From c45284fd84f5eb2c19117a466d5eb23bffbea58d Mon Sep 17 00:00:00 2001 From: Matthieu Haineault Date: Mon, 18 Aug 2025 10:26:01 -0400 Subject: [PATCH] feat(attachments): attachments module with minor fixes to attachments prisma model. --- package-lock.json | 30 ++-- package.json | 7 +- .../migration.sql | 41 +++++ .../migrations/20250818121746_/migration.sql | 2 + .../migration.sql | 14 ++ prisma/schema.prisma | 20 +-- .../attachments/config/upload.config.ts | 5 + .../controllers/attachments.controller.ts | 144 ++++++++++++++++++ .../dtos/upload-meta-attachments.dto.ts | 6 + .../services}/disk-storage.service.ts | 0 src/modules/attachments/utils/cas.util.ts | 4 + 11 files changed, 251 insertions(+), 22 deletions(-) create mode 100644 prisma/migrations/20250818121137_attachments_minimal/migration.sql create mode 100644 prisma/migrations/20250818121746_/migration.sql create mode 100644 prisma/migrations/20250818135721_column_naming_fixes_for_attachments/migration.sql create mode 100644 src/modules/attachments/config/upload.config.ts create mode 100644 src/modules/attachments/controllers/attachments.controller.ts create mode 100644 src/modules/attachments/dtos/upload-meta-attachments.dto.ts rename src/{attachments => modules/attachments/services}/disk-storage.service.ts (100%) create mode 100644 src/modules/attachments/utils/cas.util.ts diff --git a/package-lock.json b/package-lock.json index d064bd7..5b5f67b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,13 +14,15 @@ "@nestjs/core": "^11.0.1", "@nestjs/jwt": "^11.0.0", "@nestjs/passport": "^11.0.5", - "@nestjs/platform-express": "^11.0.1", + "@nestjs/platform-express": "^11.1.6", "@nestjs/schedule": "^6.0.0", "@nestjs/swagger": "^11.2.0", "@prisma/client": "^6.14.0", "class-transformer": "^0.5.1", "class-validator": "^0.14.2", "express-session": "^1.18.2", + "file-type": "^21.0.0", + "multer": "^2.0.2", "passport": "^0.7.0", "passport-jwt": "^4.0.1", "passport-openidconnect": "^0.1.2", @@ -38,7 +40,8 @@ "@types/express": "^5.0.0", "@types/express-session": "^1.18.2", "@types/jest": "^29.5.14", - "@types/node": "^22.10.7", + "@types/multer": "^2.0.0", + "@types/node": "^22.17.2", "@types/passport-jwt": "^4.0.1", "@types/passport-openidconnect": "^0.1.3", "@types/supertest": "^6.0.2", @@ -2858,10 +2861,9 @@ } }, "node_modules/@nestjs/platform-express": { - "version": "11.1.5", - "resolved": "https://registry.npmjs.org/@nestjs/platform-express/-/platform-express-11.1.5.tgz", - "integrity": "sha512-OsoiUBY9Shs5IG3uvDIt9/IDfY5OlvWBESuB/K4Eun8xILw1EK5d5qMfC3d2sIJ+kA3l+kBR1d/RuzH7VprLIg==", - "license": "MIT", + "version": "11.1.6", + "resolved": "https://registry.npmjs.org/@nestjs/platform-express/-/platform-express-11.1.6.tgz", + "integrity": "sha512-HErwPmKnk+loTq8qzu1up+k7FC6Kqa8x6lJ4cDw77KnTxLzsCaPt+jBvOq6UfICmfqcqCCf3dKXg+aObQp+kIQ==", "dependencies": { "cors": "2.8.5", "express": "5.1.0", @@ -3805,10 +3807,19 @@ "integrity": "sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==", "dev": true }, + "node_modules/@types/multer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@types/multer/-/multer-2.0.0.tgz", + "integrity": "sha512-C3Z9v9Evij2yST3RSBktxP9STm6OdMc5uR1xF1SGr98uv8dUlAL2hqwrZ3GVB3uyMyiegnscEK6PGtYvNrjTjw==", + "dev": true, + "dependencies": { + "@types/express": "*" + } + }, "node_modules/@types/node": { - "version": "22.16.4", - "resolved": "https://registry.npmjs.org/@types/node/-/node-22.16.4.tgz", - "integrity": "sha512-PYRhNtZdm2wH/NT2k/oAJ6/f2VD2N2Dag0lGlx2vWgMSJXGNmlce5MiTQzoWAiIJtso30mjnfQCOKVH+kAQC/g==", + "version": "22.17.2", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.17.2.tgz", + "integrity": "sha512-gL6z5N9Jm9mhY+U2KXZpteb+09zyffliRkZyZOHODGATyC5B1Jt/7TzuuiLkFsSUMLbS1OLmlj/E+/3KF4Q/4w==", "dependencies": { "undici-types": "~6.21.0" } @@ -9287,7 +9298,6 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/multer/-/multer-2.0.2.tgz", "integrity": "sha512-u7f2xaZ/UG8oLXHvtF/oWTRvT44p9ecwBBqTwgJVq0+4BW1g8OW01TyMEGWBHbyMOYVHXslaut7qEQ1meATXgw==", - "license": "MIT", "dependencies": { "append-field": "^1.0.0", "busboy": "^1.6.0", diff --git a/package.json b/package.json index 99c458e..db16934 100644 --- a/package.json +++ b/package.json @@ -45,13 +45,15 @@ "@nestjs/core": "^11.0.1", "@nestjs/jwt": "^11.0.0", "@nestjs/passport": "^11.0.5", - "@nestjs/platform-express": "^11.0.1", + "@nestjs/platform-express": "^11.1.6", "@nestjs/schedule": "^6.0.0", "@nestjs/swagger": "^11.2.0", "@prisma/client": "^6.14.0", "class-transformer": "^0.5.1", "class-validator": "^0.14.2", "express-session": "^1.18.2", + "file-type": "^21.0.0", + "multer": "^2.0.2", "passport": "^0.7.0", "passport-jwt": "^4.0.1", "passport-openidconnect": "^0.1.2", @@ -69,7 +71,8 @@ "@types/express": "^5.0.0", "@types/express-session": "^1.18.2", "@types/jest": "^29.5.14", - "@types/node": "^22.10.7", + "@types/multer": "^2.0.0", + "@types/node": "^22.17.2", "@types/passport-jwt": "^4.0.1", "@types/passport-openidconnect": "^0.1.3", "@types/supertest": "^6.0.2", diff --git a/prisma/migrations/20250818121137_attachments_minimal/migration.sql b/prisma/migrations/20250818121137_attachments_minimal/migration.sql new file mode 100644 index 0000000..4a3ac9e --- /dev/null +++ b/prisma/migrations/20250818121137_attachments_minimal/migration.sql @@ -0,0 +1,41 @@ +-- CreateEnum +CREATE TYPE "public"."AttachmentStatus" AS ENUM ('ACTIVE', 'DELETED'); + +-- CreateEnum +CREATE TYPE "public"."RetentionPolicy" AS ENUM ('EXPENSE_7Y', 'TICKET_2Y', 'PROFILE_KEEP_LAST3'); + +-- CreateTable +CREATE TABLE "public"."blobs" ( + "sha256" CHAR(64) NOT NULL, + "size" INTEGER NOT NULL, + "mime" TEXT NOT NULL, + "storage_path" TEXT NOT NULL, + "refcount" INTEGER NOT NULL DEFAULT 0, + "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "blobs_pkey" PRIMARY KEY ("sha256") +); + +-- CreateTable +CREATE TABLE "public"."attachments" ( + "id" SERIAL NOT NULL, + "sha256" CHAR(64) NOT NULL, + "owner_type" TEXT NOT NULL, + "owner_id" TEXT NOT NULL, + "orignal_name" TEXT NOT NULL, + "status" "public"."AttachmentStatus" NOT NULL DEFAULT 'ACTIVE', + "relation_policy" "public"."RetentionPolicy" NOT NULL, + "created_by" TEXT NOT NULL, + "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "attachments_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE INDEX "attachments_owner_type_owner_id_created_at_idx" ON "public"."attachments"("owner_type", "owner_id", "created_at"); + +-- CreateIndex +CREATE INDEX "attachments_sha256_idx" ON "public"."attachments"("sha256"); + +-- AddForeignKey +ALTER TABLE "public"."attachments" ADD CONSTRAINT "attachments_sha256_fkey" FOREIGN KEY ("sha256") REFERENCES "public"."blobs"("sha256") ON DELETE RESTRICT ON UPDATE CASCADE; diff --git a/prisma/migrations/20250818121746_/migration.sql b/prisma/migrations/20250818121746_/migration.sql new file mode 100644 index 0000000..e765c4a --- /dev/null +++ b/prisma/migrations/20250818121746_/migration.sql @@ -0,0 +1,2 @@ +ALTER TABLE "blobs" +ADD CONSTRAINT blobs_refcount_nonneg CHECK (refcount >= 0); \ No newline at end of file diff --git a/prisma/migrations/20250818135721_column_naming_fixes_for_attachments/migration.sql b/prisma/migrations/20250818135721_column_naming_fixes_for_attachments/migration.sql new file mode 100644 index 0000000..217475b --- /dev/null +++ b/prisma/migrations/20250818135721_column_naming_fixes_for_attachments/migration.sql @@ -0,0 +1,14 @@ +/* + Warnings: + + - You are about to drop the column `orignal_name` on the `attachments` table. All the data in the column will be lost. + - You are about to drop the column `relation_policy` on the `attachments` table. All the data in the column will be lost. + - Added the required column `original_name` to the `attachments` table without a default value. This is not possible if the table is not empty. + - Added the required column `retention_policy` to the `attachments` table without a default value. This is not possible if the table is not empty. + +*/ +-- AlterTable +ALTER TABLE "public"."attachments" DROP COLUMN "orignal_name", +DROP COLUMN "relation_policy", +ADD COLUMN "original_name" TEXT NOT NULL, +ADD COLUMN "retention_policy" "public"."RetentionPolicy" NOT NULL; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 6e4646a..71b1f02 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -280,16 +280,16 @@ model Blobs { } model Attachments { - id Int @id @default(autoincrement()) - sha256 String @db.Char(64) - blob Blobs @relation(fields: [sha256], references: [sha256], onUpdate: Cascade) - owner_type String //EXPENSES ou éventuellement autre chose comme scan ONU ou photos d'employés, etc - owner_id String //expense_id, employee_id, etc - orignal_name String - status AttachmentStatus @default(ACTIVE) - relation_policy RetentionPolicy - created_by String - created_at DateTime @default(now()) + id Int @id @default(autoincrement()) + sha256 String @db.Char(64) + blob Blobs @relation(fields: [sha256], references: [sha256], onUpdate: Cascade) + owner_type String //EXPENSES ou éventuellement autre chose comme scan ONU ou photos d'employés, etc + owner_id String //expense_id, employee_id, etc + original_name String + status AttachmentStatus @default(ACTIVE) + retention_policy RetentionPolicy + created_by String + created_at DateTime @default(now()) @@index([owner_type, owner_id, created_at]) @@index([sha256]) diff --git a/src/modules/attachments/config/upload.config.ts b/src/modules/attachments/config/upload.config.ts new file mode 100644 index 0000000..3684d01 --- /dev/null +++ b/src/modules/attachments/config/upload.config.ts @@ -0,0 +1,5 @@ +export const maxUploadBytes = () => + (Number(process.env.MAX_UPLOAD_MB || 25)) * 1024 * 1024; + +export const allowedMimes = () => + (process.env.ALLOWED_MIME || 'image/jpeg,image/png,image/webp,application/pdf').split(',').map(s =>s.trim()).filter(Boolean); \ No newline at end of file diff --git a/src/modules/attachments/controllers/attachments.controller.ts b/src/modules/attachments/controllers/attachments.controller.ts new file mode 100644 index 0000000..71ad827 --- /dev/null +++ b/src/modules/attachments/controllers/attachments.controller.ts @@ -0,0 +1,144 @@ +import { FileInterceptor } from "@nestjs/platform-express"; +import { DiskStorageService } from "../services/disk-storage.service"; +import { + Controller,NotFoundException, UseInterceptors, Post, Get, Param, Res, + UploadedFile, BadRequestException, UnsupportedMediaTypeException, Body, + Delete, NotImplementedException + } from "@nestjs/common"; +import { maxUploadBytes, allowedMimes } from "../config/upload.config"; +import { memoryStorage } from 'multer'; +import { fileTypeFromBuffer, fileTypeFromFile } from "file-type"; +import { Readable } from "node:stream"; +import { PrismaService } from "src/prisma/prisma.service"; +import { UploadMetaAttachmentsDto } from "../dtos/upload-meta-attachments.dto"; +import { resolveAttachmentsRoot } from "src/config/attachment.config"; +import { casPathFor } from "../utils/cas.util"; +import * as path from 'node:path'; +import { promises as fsp } from 'node:fs'; +import { createReadStream } from "node:fs"; +import { Response } from 'express'; + +@Controller('attachments') +export class AttachementsController { + constructor( + private readonly disk: DiskStorageService, + private readonly prisma: PrismaService, + ) {} + + @Get(':id') + async getById(@Param('id') id: string, @Res() res: Response) { + const att = await this.prisma.attachments.findUnique({ + where: { id: Number(id) }, + include: { blob: true }, + }); + if (!att) throw new NotFoundException(); + + const abs = path.join(resolveAttachmentsRoot(), att.blob.storage_path); + const stat = await fsp.stat(abs); + + res.setHeader('Content-Type', att.blob.mime); + res.setHeader('Content-Length', String(stat.size)); + res.setHeader('ETag', `"sha256-${att.blob.sha256}"`); + res.setHeader('Last-Modified', stat.mtime.toUTCString()); + res.setHeader('Cache-Control', 'private, max-age=31536000, immutable'); + res.setHeader('X-Content-Type-Options', 'nosniff'); + + createReadStream(abs).pipe(res); + } + + // DEV version, uncomment once connected to DB and distant server + @Delete(':id') + async remove(@Param('id') id: string) { + const result = await this.prisma.$transaction(async (tx) => { + const att = await tx.attachments.findUnique({ where: { id: Number(id) } }); + if (!att) throw new NotFoundException(); + + // soft-delete + await tx.attachments.update({ + where: { id: Number(id) }, + data: { status: 'DELETED' }, + }); + + // decrement refcount + const dec = await tx.$executeRawUnsafe( + `UPDATE "Blobs" SET refcount = refcount - 1 + WHERE sha256 = $1 AND refcount > 0`, att.sha256, + ); + + return { ok: true, decremented: dec > 0 }; + }); + return result; + } + + + @Post() + @UseInterceptors( + FileInterceptor('file', { storage: memoryStorage(), limits: { fileSize: maxUploadBytes() }}) + ) + async upload( + @UploadedFile() file?: Express.Multer.File, + @Body() meta?: UploadMetaAttachmentsDto, + ) { + if(!file) throw new BadRequestException('No file found'); + + //magic detection using binary signature + const kind = await fileTypeFromBuffer(file.buffer).catch(() => null); + const detectedMime = kind?.mime || file.mimetype || 'application/octet-stream'; + + //strict whitelist + if(!allowedMimes().includes(detectedMime)) { + throw new UnsupportedMediaTypeException(`This type is not supported: ${detectedMime}`); + } + + //Saving FS (hash + CAS + unDupes) + const stream = Readable.from(file.buffer); + const { sha256, storagePath, size } = await this.disk.saveStreamAndHash(stream); + + const now = new Date(); + const attachment = await this.prisma.$transaction(async (tx) => { + //upsert blob: +1 ref + await tx.blobs.upsert({ + where: { sha256 }, + create: { + sha256, + storage_path: storagePath, + size, + mime: detectedMime, + refcount: 1, + created_at: now, + }, + update: { //only increment, does not change the storage path + refcount: { increment: 1 }, + mime: detectedMime, //update mime and size to keep last image + size, + }, + }); + + const att = await tx.attachments.create({ + data: { + sha256, + owner_type: meta?.owner_type ?? 'EXPENSE', + owner_id: meta?.owner_id ?? 'unknown', + original_name: file.originalname, + status: 'ACTIVE', + retention_policy: (meta?.retention_policy ?? 'EXPENSE_7Y') as any, + created_by: meta?.created_by ?? 'system', + created_at: now, + }, + }); + return att; + }); + + return { + ok: true, + id: attachment.id, + sha256, + storagePath: storagePath, + size, + mime: detectedMime, + original_name: file.originalname, + owner_type: attachment.owner_type, + owner_id: attachment.owner_id, + }; + } +} \ No newline at end of file diff --git a/src/modules/attachments/dtos/upload-meta-attachments.dto.ts b/src/modules/attachments/dtos/upload-meta-attachments.dto.ts new file mode 100644 index 0000000..f0a3623 --- /dev/null +++ b/src/modules/attachments/dtos/upload-meta-attachments.dto.ts @@ -0,0 +1,6 @@ +export class UploadMetaAttachmentsDto { + owner_type!: string; + owner_id!: string; + retention_policy!: 'EXPENSE_7Y' | 'TICKET_2Y' | 'PROFILE_KEEP_LAST3'; + created_by!: string; +} \ No newline at end of file diff --git a/src/attachments/disk-storage.service.ts b/src/modules/attachments/services/disk-storage.service.ts similarity index 100% rename from src/attachments/disk-storage.service.ts rename to src/modules/attachments/services/disk-storage.service.ts diff --git a/src/modules/attachments/utils/cas.util.ts b/src/modules/attachments/utils/cas.util.ts new file mode 100644 index 0000000..19558db --- /dev/null +++ b/src/modules/attachments/utils/cas.util.ts @@ -0,0 +1,4 @@ +export function casPathFor(hash: string) { + const a = hash.slice(0,2), b = hash.slice(2,4); + return `sha256/${a}/${b}/${hash}`; +}