diff --git a/package.json b/package.json index cea3671..b1684ee 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,7 @@ "start:dev": "nest start --watch", "start:debug": "nest start --debug --watch", "start:prod": "node dist/main", - "start:variants": "node dist/attachments/workers/variants.worker.js", + "start:variants": "node dist/attachments/workers/variants.worker.js", "lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix", "test": "jest", "test:watch": "jest --watch", diff --git a/prisma/migrations/20250825173419_attachment_variants_model/migration.sql b/prisma/migrations/20250825173419_attachment_variants_model/migration.sql new file mode 100644 index 0000000..e91621a --- /dev/null +++ b/prisma/migrations/20250825173419_attachment_variants_model/migration.sql @@ -0,0 +1,19 @@ +-- CreateTable +CREATE TABLE "public"."attachment_variants" ( + "id" SERIAL NOT NULL, + "attachment_id" INTEGER NOT NULL, + "variant" TEXT NOT NULL, + "patch" TEXT NOT NULL, + "bytes" INTEGER NOT NULL, + "width" INTEGER, + "height" INTEGER, + "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "attachment_variants_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE UNIQUE INDEX "attachment_variants_attachment_id_variant_key" ON "public"."attachment_variants"("attachment_id", "variant"); + +-- AddForeignKey +ALTER TABLE "public"."attachment_variants" ADD CONSTRAINT "attachment_variants_attachment_id_fkey" FOREIGN KEY ("attachment_id") REFERENCES "public"."attachments"("id") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/src/main.ts b/src/main.ts index 504ffe1..0347cec 100644 --- a/src/main.ts +++ b/src/main.ts @@ -26,7 +26,7 @@ async function bootstrap() { const reflector = app.get(Reflector); //setup Reflector for Roles() app.useGlobalPipes( - new ValidationPipe({ whitelist: true, transform: true})); + new ValidationPipe({ whitelist: true, transform: true, forbidNonWhitelisted: true })); app.useGlobalGuards( // new JwtAuthGuard(reflector), //Authentification JWT new RolesGuard(reflector), //deny-by-default and Role-based Access Control diff --git a/src/modules/attachments/attachments.module.ts b/src/modules/attachments/attachments.module.ts new file mode 100644 index 0000000..75f5f02 --- /dev/null +++ b/src/modules/attachments/attachments.module.ts @@ -0,0 +1,11 @@ +import { ScheduleModule } from "@nestjs/schedule"; +import { PrismaService } from "src/prisma/prisma.service"; +import { ArchivalAttachmentService } from "./services/archival-attachment.service"; +import { Module } from "@nestjs/common"; + +@Module({ + imports: [ScheduleModule.forRoot()], + providers: [PrismaService, ArchivalAttachmentService], + exports: [ArchivalAttachmentService], +}) +export class ArchivalAttachmentModule {} \ No newline at end of file diff --git a/src/modules/attachments/controllers/attachments.controller.ts b/src/modules/attachments/controllers/attachments.controller.ts index 3f8e605..3a75d6a 100644 --- a/src/modules/attachments/controllers/attachments.controller.ts +++ b/src/modules/attachments/controllers/attachments.controller.ts @@ -19,6 +19,7 @@ import { promises as fsp } from 'node:fs'; import { createReadStream } from "node:fs"; import { Response } from 'express'; import { VariantsQueue } from "../services/variants.queue"; +import { AdminSearchDto } from "../dtos/admin-search.dto"; @Controller('attachments') export class AttachmentsController { @@ -58,7 +59,6 @@ export class AttachmentsController { const kind = await fileTypeFromFile(abs); if(kind?.mime) mime = kind.mime; } catch {} - res.set('Content-Type', mime); res.set('Content-Length', String(stat.size)); res.set('ETag', `"sha256-${attachment.blob.sha256}${variant ? '.'+variant : ''}"`); @@ -69,11 +69,10 @@ export class AttachmentsController { createReadStream(abs).pipe(res); } - @Get(':variant/:id') + @Get('variants/:id') async listVariants(@Param('id')id: string) { const num_id = Number(id); - if(!Number.isFinite(num_id)) throw new NotFoundException('Invalid id'); - + if(!Number.isFinite(num_id)) throw new NotFoundException('Invalid variant id'); return this.prisma.attachmentVariants.findMany({ where: { attachment_id: num_id }, orderBy: { variant: 'asc'}, @@ -81,7 +80,6 @@ export class AttachmentsController { }); } - // DEV version, uncomment once connected to DB and distant server @Delete(':id') async remove(@Param('id') id: string) { const result = await this.prisma.$transaction(async (tx) => { @@ -181,32 +179,27 @@ export class AttachmentsController { @Get('/admin/search') async adminSearch( - @Query('owner_type') owner_type?: string, - @Query('owner_id') owner_id?: string, - @Query('date_from') date_form?: string, - @Query('date_to') date_to?: string, - @Query('page', new DefaultValuePipe(1), ParseIntPipe) page?: number, - @Query('page_size', new DefaultValuePipe(50), ParseIntPipe) page_size?: number, - ) { + @Query() query: AdminSearchDto ) { const where: any = {}; - if (owner_type) where.owner_type = owner_type; - if (owner_id) where.owner_id = owner_id; + if (query.owner_type) where.owner_type = query.owner_type; + if (query.owner_id) where.owner_id = query.owner_id; - if (date_form || date_to) { + if (query.date_from || query.date_to) { where.created_at = {}; - if (date_form) where.created_at.gte = new Date(date_form + 'T00:00:00Z'); - if (date_to) where.created_at.lte = new Date(date_to + 'T23:59:59Z'); + if (query.date_from) where.created_at.gte = new Date(query.date_from + 'T00:00:00Z'); + if (query.date_to) where.created_at.lte = new Date(query.date_to + 'T23:59:59Z'); } - const skip = (Math.max(1, page!) - 1) * Math.max(1, page_size!); - const take = Math.min(Math.max(1, page_size!), 200); + const page = query.page ?? 1; + const page_size = query.page_size ?? 50; + const skip = (page - 1)* page_size; + const take = page_size; const [items, total] = await this.prisma.$transaction([ this.prisma.attachments.findMany({ where, orderBy: { created_at: 'desc' }, - skip, - take, + skip, take, include: { blob: { select: { mime: true, size: true, storage_path: true, sha256: true }, @@ -216,11 +209,6 @@ export class AttachmentsController { this.prisma.attachments.count({ where }), ]); - return { - page, - page_size: take, - total, - items, - }; + return { page, page_size: take, total, items }; } -} \ No newline at end of file +} diff --git a/src/modules/attachments/dtos/admin-search.dto.ts b/src/modules/attachments/dtos/admin-search.dto.ts new file mode 100644 index 0000000..ca1edc6 --- /dev/null +++ b/src/modules/attachments/dtos/admin-search.dto.ts @@ -0,0 +1,34 @@ +import { Type } from "class-transformer"; +import { IsInt, IsISO8601, IsOptional, IsString, Max, Min } from "class-validator"; + +export class AdminSearchDto { + + @IsOptional() + @IsString() + owner_type?: string; + + @IsOptional() + @IsString() + owner_id?: string; + + @IsOptional() + @IsISO8601() + date_from?: string; + + @IsOptional() + @IsISO8601() + date_to?: string; + + @IsOptional() + @Type(()=> Number) + @IsInt() + @Min(1) + page?: number = 1; + + @IsOptional() + @Type(()=> Number) + @IsInt() + @Min(1) + @Max(200) + page_size?: number = 50; +} \ No newline at end of file diff --git a/src/modules/attachments/services/archival-attachment.service.ts b/src/modules/attachments/services/archival-attachment.service.ts new file mode 100644 index 0000000..501923d --- /dev/null +++ b/src/modules/attachments/services/archival-attachment.service.ts @@ -0,0 +1,60 @@ +import { Injectable, Logger } from "@nestjs/common"; +import { Cron } from "@nestjs/schedule"; +import { PrismaService } from "src/prisma/prisma.service"; + +@Injectable() +export class ArchivalAttachmentService { + private readonly logger = new Logger(ArchivalAttachmentService.name) + private readonly batch_size = Number(process.env.ARCHIVE_BATCH_SIZE || 1000); + private readonly cron_expression = process.env.ARCHIVE_CRON || '0 3 * * 1'; + + constructor( private readonly prisma: PrismaService) {} + + private startOfYear(): Date { + const now = new Date(); + return new Date(Date.UTC(now.getUTCFullYear(), 0, 1, 0, 0, 0, 0)); + } + + @Cron(function (this: ArchivalAttachmentService) { return this.cron_expression; } as any) + async runScheduled() { + await this.archiveCutoffToStartOfYear(); + } + + //archive everything before current year + async archiveCutoffToStartOfYear() { + const cutoff = this.startOfYear(); + this.logger.log(`Archival: cutoff=${cutoff.toISOString()} batch=${this.batch_size}`); + + let moved = 0, total = 0, i = 0; + do { + moved = await this.archiveBatch(cutoff, this.batch_size); + total += moved; + i++; + if(moved > 0) this.logger.log(`Batch #${i}: moved ${moved}`); + }while (moved === this.batch_size); + + this.logger.log(`Archival done: total moved : ${total}`); + return { moved: total }; + } + + //only moves table content to archive and not blobs. + private async archiveBatch(cutoff: Date, batch_size: number): Promise { + const moved = await this.prisma.$executeRaw` + WITH moved AS ( + DELETE FROM "attachments" + WHERE id IN ( + SELECT id FROM "attachments" + WHERE created_at < ${cutoff} + ORDER BY id + LIMIT ${batch_size} + ) + RETURNING id, sha256, owner_type, owner_id, original_name, status, retention_policy, created_by, created_at + ) + INSERT INTO archive.attachments_archive + (id, sha256, owner_type, owner_id, original_name, status, retention_policy, created_by, created_at) + SELECT * FROM moved; + `; + return Number(moved) || 0; + } + +} \ No newline at end of file