feat(attachments): setup AdminSearchDto, CronJobs for archival and display route via controller

package.json

@@ -12,7 +12,7 @@
     "start:dev": "nest start --watch",
     "start:debug": "nest start --debug --watch",
     "start:prod": "node dist/main",
-    "start:variants": "node dist/attachments/workers/variants.worker.js", 
+    "start:variants": "node dist/attachments/workers/variants.worker.js",
     "lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix",
     "test": "jest",
     "test:watch": "jest --watch",

prisma/migrations/20250825173419_attachment_variants_model/migration.sql

@@ -0,0 +1,19 @@
+-- CreateTable
+CREATE TABLE "public"."attachment_variants" (
+    "id" SERIAL NOT NULL,
+    "attachment_id" INTEGER NOT NULL,
+    "variant" TEXT NOT NULL,
+    "patch" TEXT NOT NULL,
+    "bytes" INTEGER NOT NULL,
+    "width" INTEGER,
+    "height" INTEGER,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "attachment_variants_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "attachment_variants_attachment_id_variant_key" ON "public"."attachment_variants"("attachment_id", "variant");
+
+-- AddForeignKey
+ALTER TABLE "public"."attachment_variants" ADD CONSTRAINT "attachment_variants_attachment_id_fkey" FOREIGN KEY ("attachment_id") REFERENCES "public"."attachments"("id") ON DELETE CASCADE ON UPDATE CASCADE;

src/main.ts

@@ -26,7 +26,7 @@ async function bootstrap() {
   const reflector = app.get(Reflector); //setup Reflector for Roles()
 
   app.useGlobalPipes(
-    new ValidationPipe({ whitelist: true, transform: true}));
+    new ValidationPipe({ whitelist: true, transform: true, forbidNonWhitelisted: true }));
   app.useGlobalGuards(
     // new JwtAuthGuard(reflector), //Authentification JWT
     new RolesGuard(reflector), //deny-by-default and Role-based Access Control

src/modules/attachments/attachments.module.ts

@@ -0,0 +1,11 @@
+import { ScheduleModule } from "@nestjs/schedule";
+import { PrismaService } from "src/prisma/prisma.service";
+import { ArchivalAttachmentService } from "./services/archival-attachment.service";
+import { Module } from "@nestjs/common";
+
+@Module({
+    imports: [ScheduleModule.forRoot()],
+    providers: [PrismaService, ArchivalAttachmentService],
+    exports: [ArchivalAttachmentService],
+})
+export class ArchivalAttachmentModule {}

src/modules/attachments/controllers/attachments.controller.ts

@@ -19,6 +19,7 @@ import { promises as fsp } from 'node:fs';
 import { createReadStream } from "node:fs";
 import { Response } from 'express'; 
 import { VariantsQueue } from "../services/variants.queue";
+import { AdminSearchDto } from "../dtos/admin-search.dto";
 
 @Controller('attachments')
 export class AttachmentsController {
@@ -58,7 +59,6 @@ export class AttachmentsController {
             const kind = await fileTypeFromFile(abs);
             if(kind?.mime) mime = kind.mime;
         } catch {}
-
         res.set('Content-Type', mime);
         res.set('Content-Length', String(stat.size));
         res.set('ETag', `"sha256-${attachment.blob.sha256}${variant ? '.'+variant : ''}"`);
@@ -69,11 +69,10 @@ export class AttachmentsController {
         createReadStream(abs).pipe(res);
     }
 
-    @Get(':variant/:id')
+    @Get('variants/:id')
     async listVariants(@Param('id')id: string) {
         const num_id = Number(id);
-        if(!Number.isFinite(num_id)) throw new NotFoundException('Invalid id');
+        if(!Number.isFinite(num_id)) throw new NotFoundException('Invalid variant id');
-
         return this.prisma.attachmentVariants.findMany({
             where: { attachment_id: num_id },
             orderBy: { variant: 'asc'},
@@ -81,7 +80,6 @@ export class AttachmentsController {
         });
     }
 
-    // DEV version, uncomment once connected to DB and distant server
     @Delete(':id')
     async remove(@Param('id') id: string) {
         const result = await this.prisma.$transaction(async (tx) => {
@@ -181,32 +179,27 @@ export class AttachmentsController {
 
     @Get('/admin/search')
     async adminSearch(
-        @Query('owner_type') owner_type?: string,
+        @Query() query: AdminSearchDto ) {
-        @Query('owner_id') owner_id?: string,
-        @Query('date_from') date_form?: string,
-        @Query('date_to') date_to?: string,
-        @Query('page', new DefaultValuePipe(1), ParseIntPipe) page?: number,
-        @Query('page_size', new DefaultValuePipe(50), ParseIntPipe) page_size?: number,
-    ) {
         const where: any = {};
-        if (owner_type) where.owner_type = owner_type;
+        if (query.owner_type) where.owner_type = query.owner_type;
-        if (owner_id) where.owner_id = owner_id;
+        if (query.owner_id) where.owner_id = query.owner_id;
 
-        if (date_form || date_to) {
+        if (query.date_from || query.date_to) {
             where.created_at = {};
-            if (date_form) where.created_at.gte = new Date(date_form + 'T00:00:00Z');
+            if (query.date_from) where.created_at.gte = new Date(query.date_from + 'T00:00:00Z');
-            if (date_to)   where.created_at.lte = new Date(date_to   + 'T23:59:59Z');
+            if (query.date_to)   where.created_at.lte = new Date(query.date_to   + 'T23:59:59Z');
         }
 
-        const skip = (Math.max(1, page!) - 1) * Math.max(1, page_size!);
+        const page = query.page ?? 1;
-        const take = Math.min(Math.max(1, page_size!), 200);
+        const page_size = query.page_size ?? 50;
+        const skip = (page - 1)* page_size;
+        const take = page_size;
 
         const [items, total] = await this.prisma.$transaction([
             this.prisma.attachments.findMany({
             where,
             orderBy: { created_at: 'desc' },
-            skip,
+            skip, take,
-            take,
             include: {
                 blob: {
                 select: { mime: true, size: true, storage_path: true, sha256: true },
@@ -216,11 +209,6 @@ export class AttachmentsController {
             this.prisma.attachments.count({ where }),
         ]);
 
-        return {
+        return { page, page_size: take, total, items };
-            page,
-            page_size: take,
-            total,
-            items,
-        };
     }
-}
+}

src/modules/attachments/dtos/admin-search.dto.ts

@@ -0,0 +1,34 @@
+import { Type } from "class-transformer";
+import { IsInt, IsISO8601, IsOptional, IsString, Max, Min } from "class-validator";
+
+export class AdminSearchDto {
+
+    @IsOptional()
+    @IsString()
+    owner_type?: string;
+
+    @IsOptional()
+    @IsString()
+    owner_id?: string;
+
+    @IsOptional()
+    @IsISO8601()
+    date_from?: string;
+
+    @IsOptional()   
+    @IsISO8601()
+    date_to?: string;
+
+    @IsOptional()
+    @Type(()=> Number)
+    @IsInt()
+    @Min(1)
+    page?: number = 1;
+
+    @IsOptional()
+    @Type(()=> Number)
+    @IsInt()
+    @Min(1)
+    @Max(200)
+    page_size?: number = 50;
+}

src/modules/attachments/services/archival-attachment.service.ts

@@ -0,0 +1,60 @@
+import { Injectable, Logger } from "@nestjs/common";
+import { Cron } from "@nestjs/schedule";
+import { PrismaService } from "src/prisma/prisma.service";
+
+@Injectable()
+export class ArchivalAttachmentService {
+    private readonly logger = new Logger(ArchivalAttachmentService.name)
+    private readonly batch_size = Number(process.env.ARCHIVE_BATCH_SIZE || 1000);
+    private readonly cron_expression = process.env.ARCHIVE_CRON || '0 3 * * 1';
+
+    constructor( private readonly prisma: PrismaService) {}
+
+    private startOfYear(): Date {
+        const now = new Date();
+        return new Date(Date.UTC(now.getUTCFullYear(), 0, 1, 0, 0, 0, 0));
+    }
+
+    @Cron(function (this: ArchivalAttachmentService) { return this.cron_expression; } as any)
+    async runScheduled() {
+        await this.archiveCutoffToStartOfYear();
+    }
+
+    //archive everything before current year
+    async archiveCutoffToStartOfYear() {
+        const cutoff = this.startOfYear();
+        this.logger.log(`Archival: cutoff=${cutoff.toISOString()} batch=${this.batch_size}`);
+
+        let moved = 0, total = 0, i = 0;
+        do {
+            moved = await this.archiveBatch(cutoff, this.batch_size);
+            total += moved;
+            i++;
+            if(moved > 0) this.logger.log(`Batch #${i}: moved ${moved}`);
+        }while (moved === this.batch_size);
+
+        this.logger.log(`Archival done: total moved : ${total}`);
+        return { moved: total };
+    }
+
+    //only moves table content to archive and not blobs.
+    private async archiveBatch(cutoff: Date, batch_size: number): Promise<number> {
+        const moved = await this.prisma.$executeRaw`
+        WITH moved AS (
+            DELETE FROM "attachments"
+            WHERE id IN (
+                SELECT id FROM "attachments"
+                WHERE created_at < ${cutoff}
+                ORDER BY id
+                LIMIT ${batch_size}
+            )
+            RETURNING id, sha256, owner_type, owner_id, original_name, status, retention_policy, created_by, created_at
+        )
+        INSERT INTO archive.attachments_archive
+            (id, sha256, owner_type, owner_id, original_name, status, retention_policy, created_by, created_at)
+            SELECT * FROM moved;
+        `;
+    return Number(moved) || 0;
+    }
+    
+}