feat(attachments): cleaning and setting up module for app integration
This commit is contained in:
parent
fb0187c117
commit
4b0189ed48
|
|
@ -1,22 +1,26 @@
|
||||||
import { ArchivalAttachmentService } from "src/modules/attachments/services/archival-attachment.service";
|
import { AttachmentArchivalService } from "src/modules/attachments/services/attachment-archival.service";
|
||||||
import { GarbargeCollectorService } from "src/modules/attachments/services/garbage-collector.service";
|
import { GarbargeCollectorService } from "src/modules/attachments/services/garbage-collector.service";
|
||||||
import { AttachmentsController } from "src/modules/attachments/controllers/attachments.controller";
|
import { AttachmentsController } from "src/modules/attachments/controllers/attachments.controller";
|
||||||
import { DiskStorageService } from "src/modules/attachments/services/disk-storage.service";
|
import { DiskStorageService } from "src/modules/attachments/services/disk-storage.service";
|
||||||
// import { ScheduleModule } from "@nestjs/schedule";
|
|
||||||
import { VariantsQueue } from "src/modules/attachments/services/variants.queue";
|
import { VariantsQueue } from "src/modules/attachments/services/variants.queue";
|
||||||
import { Module } from "@nestjs/common";
|
import { Module } from "@nestjs/common";
|
||||||
|
import { AttachmentDeleteService } from "src/modules/attachments/services/attachment-delete.service";
|
||||||
|
import { AttachmentUploadService } from "src/modules/attachments/services/attachment-upload.service";
|
||||||
|
import { AttachmentGetService } from "src/modules/attachments/services/attachment-get.service";
|
||||||
|
|
||||||
@Module({
|
@Module({
|
||||||
// imports: [ScheduleModule.forRoot()],
|
|
||||||
controllers: [ AttachmentsController],
|
controllers: [ AttachmentsController],
|
||||||
providers: [
|
providers: [
|
||||||
ArchivalAttachmentService,
|
AttachmentArchivalService,
|
||||||
GarbargeCollectorService,
|
GarbargeCollectorService,
|
||||||
DiskStorageService,
|
DiskStorageService,
|
||||||
VariantsQueue,
|
VariantsQueue,
|
||||||
|
AttachmentDeleteService,
|
||||||
|
AttachmentUploadService,
|
||||||
|
AttachmentGetService,
|
||||||
],
|
],
|
||||||
exports: [
|
exports: [
|
||||||
ArchivalAttachmentService,
|
AttachmentArchivalService,
|
||||||
GarbargeCollectorService
|
GarbargeCollectorService
|
||||||
],
|
],
|
||||||
})
|
})
|
||||||
|
|
|
||||||
|
|
@ -2,4 +2,5 @@ export const maxUploadBytes = () =>
|
||||||
(Number(process.env.MAX_UPLOAD_MB || 25)) * 1024 * 1024;
|
(Number(process.env.MAX_UPLOAD_MB || 25)) * 1024 * 1024;
|
||||||
|
|
||||||
export const allowedMimes = () =>
|
export const allowedMimes = () =>
|
||||||
(process.env.ALLOWED_MIME || 'image/jpeg,image/png,image/webp,application/pdf').split(',').map(s =>s.trim()).filter(Boolean);
|
(process.env.ALLOWED_MIME || 'image/jpeg,image/png,image/webp,application/pdf')
|
||||||
|
.split(',').map(s => s.trim()).filter(Boolean);
|
||||||
|
|
@ -1,212 +1,45 @@
|
||||||
|
import { Controller, UseInterceptors, Post, Get, Param, Res, UploadedFile, Body, Delete, Query } from "@nestjs/common";
|
||||||
import { FileInterceptor } from "@nestjs/platform-express";
|
import { FileInterceptor } from "@nestjs/platform-express";
|
||||||
import { DiskStorageService } from "../services/disk-storage.service";
|
import { AttachmentDeleteService } from "src/modules/attachments/services/attachment-delete.service";
|
||||||
import {
|
import { AttachmentUploadService } from "src/modules/attachments/services/attachment-upload.service";
|
||||||
Controller,NotFoundException, UseInterceptors, Post, Get, Param, Res,
|
import { AttachmentGetService } from "src/modules/attachments/services/attachment-get.service";
|
||||||
UploadedFile, BadRequestException, UnsupportedMediaTypeException, Body, Delete,
|
|
||||||
Query,
|
|
||||||
} from "@nestjs/common";
|
|
||||||
import { maxUploadBytes, allowedMimes } from "../config/upload.config";
|
|
||||||
import { memoryStorage } from 'multer';
|
|
||||||
import { fileTypeFromBuffer, fileTypeFromFile } from "file-type";
|
|
||||||
import { Readable } from "node:stream";
|
|
||||||
import { PrismaService } from "src/prisma/prisma.service";
|
|
||||||
import { UploadMetaAttachmentsDto } from "../dtos/upload-meta-attachments.dto";
|
import { UploadMetaAttachmentsDto } from "../dtos/upload-meta-attachments.dto";
|
||||||
import { resolveAttachmentsRoot } from "src/config/attachment.config";
|
import { AdminSearchDto } from "../dtos/search-filters.dto";
|
||||||
import * as path from 'node:path';
|
import { maxUploadBytes } from "../config/upload.config";
|
||||||
import { promises as fsp } from 'node:fs';
|
import { memoryStorage } from 'multer';
|
||||||
import { createReadStream } from "node:fs";
|
|
||||||
import { Response } from 'express';
|
import { Response } from 'express';
|
||||||
import { VariantsQueue } from "../services/variants.queue";
|
|
||||||
import { AdminSearchDto } from "../dtos/admin-search.dto";
|
|
||||||
|
|
||||||
@Controller('attachments')
|
@Controller('attachments')
|
||||||
export class AttachmentsController {
|
export class AttachmentsController {
|
||||||
constructor(
|
constructor(
|
||||||
private readonly disk: DiskStorageService,
|
private readonly uploadService: AttachmentUploadService,
|
||||||
private readonly prisma: PrismaService,
|
private readonly deleteService: AttachmentDeleteService,
|
||||||
private readonly variantsQ: VariantsQueue,
|
private readonly getService: AttachmentGetService,
|
||||||
) { }
|
) { }
|
||||||
|
|
||||||
@Get(':id')
|
@Get(':id')
|
||||||
async getById(
|
async getById(@Param('id') id: string, @Query('variant') variant: string | undefined, @Res() res: Response) {
|
||||||
@Param('id') id: string,
|
return await this.getService.findAttachmentById(id, variant, res)
|
||||||
@Query('variant') variant: string | undefined,
|
|
||||||
@Res() res: Response,
|
|
||||||
) {
|
|
||||||
const num_id = Number(id);
|
|
||||||
if(!Number.isFinite(num_id)) throw new NotFoundException('Invalid id');
|
|
||||||
|
|
||||||
const attachment = await this.prisma.attachments.findUnique({
|
|
||||||
where: { id: num_id },
|
|
||||||
include: { blob: true },
|
|
||||||
});
|
|
||||||
if (!attachment) throw new NotFoundException();
|
|
||||||
|
|
||||||
const relative = variant ? `${attachment.blob.storage_path}.${variant}` : attachment.blob.storage_path;
|
|
||||||
const abs = path.join(resolveAttachmentsRoot(), relative);
|
|
||||||
|
|
||||||
let stat;
|
|
||||||
try {
|
|
||||||
stat = await fsp.stat(abs);
|
|
||||||
}catch {
|
|
||||||
throw new NotFoundException('File not found');
|
|
||||||
}
|
|
||||||
|
|
||||||
let mime = attachment.blob.mime;
|
|
||||||
try {
|
|
||||||
const kind = await fileTypeFromFile(abs);
|
|
||||||
if(kind?.mime) mime = kind.mime;
|
|
||||||
} catch {}
|
|
||||||
res.set('Content-Type', mime);
|
|
||||||
res.set('Content-Length', String(stat.size));
|
|
||||||
res.set('ETag', `"sha256-${attachment.blob.sha256}${variant ? '.'+variant : ''}"`);
|
|
||||||
res.set('Last-Modified', stat.mtime.toUTCString());
|
|
||||||
res.set('Cache-Control', 'private, max-age=31536000, immutable');
|
|
||||||
res.set('X-Content-Type-Options', 'nosniff');
|
|
||||||
|
|
||||||
createReadStream(abs).pipe(res);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Get('variants/:id')
|
@Get('variants/:id')
|
||||||
async listVariants(@Param('id')id: string) {
|
async getlistVariantsById(@Param('id') id: string) {
|
||||||
const num_id = Number(id);
|
return await this.getService.getListVariants(id);
|
||||||
if(!Number.isFinite(num_id)) throw new NotFoundException('Invalid variant id');
|
|
||||||
return this.prisma.attachmentVariants.findMany({
|
|
||||||
where: { attachment_id: num_id },
|
|
||||||
orderBy: { variant: 'asc'},
|
|
||||||
select: { variant: true, bytes: true, width: true, height: true, path: true, created_at: true },
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Delete(':id')
|
@Delete(':id')
|
||||||
async remove(@Param('id') id: string) {
|
async remove(@Param('id') id: string) {
|
||||||
const result = await this.prisma.$transaction(async (tx) => {
|
return await this.deleteService.deleteAttachment(id);
|
||||||
const att = await tx.attachments.findUnique({ where: { id: Number(id) } });
|
|
||||||
if (!att) throw new NotFoundException();
|
|
||||||
|
|
||||||
// soft-delete
|
|
||||||
await tx.attachments.update({
|
|
||||||
where: { id: Number(id) },
|
|
||||||
data: { status: 'DELETED' },
|
|
||||||
});
|
|
||||||
|
|
||||||
// decrement refcount
|
|
||||||
const dec = await tx.$executeRaw
|
|
||||||
`UPDATE "Blobs" SET refcount = refcount - 1
|
|
||||||
WHERE sha256 = ${att.sha256} AND refcount > 0`
|
|
||||||
;
|
|
||||||
|
|
||||||
return { ok: true, decremented: dec > 0 };
|
|
||||||
});
|
|
||||||
return result;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@Post()
|
@Post()
|
||||||
@UseInterceptors(
|
@UseInterceptors(FileInterceptor('file', { storage: memoryStorage(), limits: { fileSize: maxUploadBytes() } }))
|
||||||
FileInterceptor('file', { storage: memoryStorage(), limits: { fileSize: maxUploadBytes() }})
|
async upload(@UploadedFile() file?: Express.Multer.File, @Body() meta?: UploadMetaAttachmentsDto) {
|
||||||
)
|
return await this.uploadService.uploadAttachment(file, meta);
|
||||||
async upload(
|
|
||||||
@UploadedFile() file?: Express.Multer.File,
|
|
||||||
@Body() meta?: UploadMetaAttachmentsDto,
|
|
||||||
) {
|
|
||||||
if(!file) throw new BadRequestException('No file found');
|
|
||||||
|
|
||||||
//magic detection using binary signature
|
|
||||||
const kind = await fileTypeFromBuffer(file.buffer).catch(() => null);
|
|
||||||
const detected_mime = kind?.mime || file.mimetype || 'application/octet-stream';
|
|
||||||
|
|
||||||
//strict whitelist
|
|
||||||
if(!allowedMimes().includes(detected_mime)) {
|
|
||||||
throw new UnsupportedMediaTypeException(`This type is not supported: ${detected_mime}`);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
//Saving FS (hash + CAS + unDupes)
|
@Get('search/filters')
|
||||||
const stream = Readable.from(file.buffer);
|
async searchWithFilters(@Query() dto: AdminSearchDto) {
|
||||||
const { sha256, storage_path, size } = await this.disk.saveStreamAndHash(stream);
|
return await this.getService.searchAttachmentWithFilters(dto);
|
||||||
|
|
||||||
const now = new Date();
|
|
||||||
const attachment = await this.prisma.$transaction(async (tx) => {
|
|
||||||
//upsert blob: +1 ref
|
|
||||||
await tx.blobs.upsert({
|
|
||||||
where: { sha256 },
|
|
||||||
create: {
|
|
||||||
sha256,
|
|
||||||
storage_path: storage_path,
|
|
||||||
size,
|
|
||||||
mime: detected_mime,
|
|
||||||
refcount: 1,
|
|
||||||
created_at: now,
|
|
||||||
},
|
|
||||||
update: { //only increment, does not change the storage path
|
|
||||||
refcount: { increment: 1 },
|
|
||||||
mime: detected_mime, //update mime and size to keep last image
|
|
||||||
size,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
|
|
||||||
const att = await tx.attachments.create({
|
|
||||||
data: {
|
|
||||||
sha256,
|
|
||||||
owner_type: meta?.owner_type ?? 'EXPENSE',
|
|
||||||
owner_id: meta?.owner_id ?? 'unknown',
|
|
||||||
original_name: file.originalname,
|
|
||||||
status: 'ACTIVE',
|
|
||||||
retention_policy: (meta?.retention_policy ?? 'EXPENSE_7Y') as any,
|
|
||||||
created_by: meta?.created_by ?? 'system',
|
|
||||||
created_at: now,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
return att;
|
|
||||||
});
|
|
||||||
|
|
||||||
await this.variantsQ.enqueue(attachment.id, detected_mime);
|
|
||||||
|
|
||||||
return {
|
|
||||||
ok: true,
|
|
||||||
id: attachment.id,
|
|
||||||
sha256,
|
|
||||||
storage_path: storage_path,
|
|
||||||
size,
|
|
||||||
mime: detected_mime,
|
|
||||||
original_name: file.originalname,
|
|
||||||
owner_type: attachment.owner_type,
|
|
||||||
owner_id: attachment.owner_id,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
@Get('/admin/search')
|
|
||||||
async adminSearch(
|
|
||||||
@Query() query: AdminSearchDto ) {
|
|
||||||
const where: any = {};
|
|
||||||
if (query.owner_type) where.owner_type = query.owner_type;
|
|
||||||
if (query.owner_id) where.owner_id = query.owner_id;
|
|
||||||
|
|
||||||
if (query.date_from || query.date_to) {
|
|
||||||
where.created_at = {};
|
|
||||||
if (query.date_from) where.created_at.gte = new Date(query.date_from + 'T00:00:00Z');
|
|
||||||
if (query.date_to) where.created_at.lte = new Date(query.date_to + 'T23:59:59Z');
|
|
||||||
}
|
|
||||||
|
|
||||||
const page = query.page ?? 1;
|
|
||||||
const page_size = query.page_size ?? 50;
|
|
||||||
const skip = (page - 1)* page_size;
|
|
||||||
const take = page_size;
|
|
||||||
|
|
||||||
const [items, total] = await this.prisma.$transaction([
|
|
||||||
this.prisma.attachments.findMany({
|
|
||||||
where,
|
|
||||||
orderBy: { created_at: 'desc' },
|
|
||||||
skip, take,
|
|
||||||
include: {
|
|
||||||
blob: {
|
|
||||||
select: { mime: true, size: true, storage_path: true, sha256: true },
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}),
|
|
||||||
this.prisma.attachments.count({ where }),
|
|
||||||
]);
|
|
||||||
|
|
||||||
return { page, page_size: take, total, items };
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,34 +0,0 @@
|
||||||
import { Type } from "class-transformer";
|
|
||||||
import { IsInt, IsISO8601, IsOptional, IsString, Max, Min } from "class-validator";
|
|
||||||
|
|
||||||
export class AdminSearchDto {
|
|
||||||
|
|
||||||
@IsOptional()
|
|
||||||
@IsString()
|
|
||||||
owner_type?: string;
|
|
||||||
|
|
||||||
@IsOptional()
|
|
||||||
@IsString()
|
|
||||||
owner_id?: string;
|
|
||||||
|
|
||||||
@IsOptional()
|
|
||||||
@IsISO8601()
|
|
||||||
date_from?: string;
|
|
||||||
|
|
||||||
@IsOptional()
|
|
||||||
@IsISO8601()
|
|
||||||
date_to?: string;
|
|
||||||
|
|
||||||
@IsOptional()
|
|
||||||
@Type(()=> Number)
|
|
||||||
@IsInt()
|
|
||||||
@Min(1)
|
|
||||||
page?: number = 1;
|
|
||||||
|
|
||||||
@IsOptional()
|
|
||||||
@Type(()=> Number)
|
|
||||||
@IsInt()
|
|
||||||
@Min(1)
|
|
||||||
@Max(200)
|
|
||||||
page_size?: number = 50;
|
|
||||||
}
|
|
||||||
10
src/modules/attachments/dtos/search-filters.dto.ts
Normal file
10
src/modules/attachments/dtos/search-filters.dto.ts
Normal file
|
|
@ -0,0 +1,10 @@
|
||||||
|
import { IsInt, IsOptional, IsString, Max, Min } from "class-validator";
|
||||||
|
|
||||||
|
export class AdminSearchDto {
|
||||||
|
@IsOptional() @IsString() owner_type?: string;
|
||||||
|
@IsOptional() @IsString() owner_id?: string;
|
||||||
|
@IsOptional() date_from?: string;
|
||||||
|
@IsOptional() date_to?: string;
|
||||||
|
@IsOptional() @IsInt() @Min(1) page?: number = 1;
|
||||||
|
@IsOptional() @IsInt() @Min(1) @Max(200) page_size?: number = 50;
|
||||||
|
}
|
||||||
|
|
@ -1,39 +1,35 @@
|
||||||
import { Injectable, Logger } from "@nestjs/common";
|
import { Injectable } from "@nestjs/common";
|
||||||
import { Cron } from "@nestjs/schedule";
|
import { Cron } from "@nestjs/schedule";
|
||||||
|
import { startOfYear } from "src/modules/attachments/utils/cas.util";
|
||||||
import { PrismaService } from "src/prisma/prisma.service";
|
import { PrismaService } from "src/prisma/prisma.service";
|
||||||
|
|
||||||
@Injectable()
|
@Injectable()
|
||||||
export class ArchivalAttachmentService {
|
export class AttachmentArchivalService {
|
||||||
private readonly logger = new Logger(ArchivalAttachmentService.name)
|
|
||||||
private readonly batch_size = Number(process.env.ARCHIVE_BATCH_SIZE || 1000);
|
private readonly batch_size = Number(process.env.ARCHIVE_BATCH_SIZE || 1000);
|
||||||
private readonly cron_expression = process.env.ARCHIVE_CRON || '0 3 * * 1';
|
private readonly cron_expression = process.env.ARCHIVE_CRON || '0 3 * * 1';
|
||||||
|
|
||||||
constructor(private readonly prisma: PrismaService) { }
|
constructor(private readonly prisma: PrismaService) { }
|
||||||
|
|
||||||
private startOfYear(): Date {
|
|
||||||
const now = new Date();
|
|
||||||
return new Date(Date.UTC(now.getUTCFullYear(), 0, 1, 0, 0, 0, 0));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Cron(function (this: ArchivalAttachmentService) { return this.cron_expression; } as any)
|
@Cron(function (this: AttachmentArchivalService) { return this.cron_expression; } as any)
|
||||||
async runScheduled() {
|
async runScheduled() {
|
||||||
await this.archiveCutoffToStartOfYear();
|
await this.archiveCutoffToStartOfYear();
|
||||||
}
|
}
|
||||||
|
|
||||||
//archive everything before current year
|
//archive everything before current year
|
||||||
async archiveCutoffToStartOfYear() {
|
async archiveCutoffToStartOfYear() {
|
||||||
const cutoff = this.startOfYear();
|
const cutoff = startOfYear();
|
||||||
this.logger.log(`Archival: cutoff=${cutoff.toISOString()} batch=${this.batch_size}`);
|
console.log(`Archival: cutoff=${cutoff.toISOString()} batch=${this.batch_size}`);
|
||||||
|
|
||||||
let moved = 0, total = 0, i = 0;
|
let moved = 0, total = 0, i = 0;
|
||||||
do {
|
do {
|
||||||
moved = await this.archiveBatch(cutoff, this.batch_size);
|
moved = await this.archiveBatch(cutoff, this.batch_size);
|
||||||
total += moved;
|
total += moved;
|
||||||
i++;
|
i++;
|
||||||
if(moved > 0) this.logger.log(`Batch #${i}: moved ${moved}`);
|
if (moved > 0) console.log(`Batch #${i}: moved ${moved}`);
|
||||||
} while (moved === this.batch_size);
|
} while (moved === this.batch_size);
|
||||||
|
|
||||||
this.logger.log(`Archival done: total moved : ${total}`);
|
console.log(`Archival done: total moved : ${total}`);
|
||||||
return { moved: total };
|
return { moved: total };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -52,8 +48,7 @@ export class ArchivalAttachmentService {
|
||||||
)
|
)
|
||||||
INSERT INTO archive.attachments_archive
|
INSERT INTO archive.attachments_archive
|
||||||
(id, sha256, owner_type, owner_id, original_name, status, retention_policy, created_by, created_at)
|
(id, sha256, owner_type, owner_id, original_name, status, retention_policy, created_by, created_at)
|
||||||
SELECT * FROM moved;
|
SELECT * FROM moved;`;
|
||||||
`;
|
|
||||||
return Number(moved) || 0;
|
return Number(moved) || 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -0,0 +1,26 @@
|
||||||
|
import { Injectable, NotFoundException } from "@nestjs/common";
|
||||||
|
import { Result } from "src/common/errors/result-error.factory";
|
||||||
|
import { PrismaService } from "src/prisma/prisma.service";
|
||||||
|
|
||||||
|
@Injectable()
|
||||||
|
export class AttachmentDeleteService {
|
||||||
|
constructor(private readonly prisma: PrismaService) { }
|
||||||
|
|
||||||
|
async deleteAttachment(id: string): Promise<Result<boolean, string>> {
|
||||||
|
await this.prisma.$transaction(async (tx) => {
|
||||||
|
const attachment = await tx.attachments.findUnique({ where: { id: Number(id) } });
|
||||||
|
if (!attachment) return { success: false, error: 'ATTACHMENT_NOT_FOUND' };
|
||||||
|
|
||||||
|
// soft-delete
|
||||||
|
await tx.attachments.update({ where: { id: Number(id) }, data: { status: 'DELETED' } });
|
||||||
|
|
||||||
|
// decrement refcount
|
||||||
|
const dec = await tx.$executeRaw `
|
||||||
|
UPDATE "Blobs" SET refcount = refcount - 1
|
||||||
|
WHERE sha256 = ${attachment.sha256} AND refcount > 0;`;
|
||||||
|
|
||||||
|
return { ok: true, decremented: dec > 0 };
|
||||||
|
});
|
||||||
|
return { success: true, data: true };
|
||||||
|
}
|
||||||
|
}
|
||||||
100
src/modules/attachments/services/attachment-get.service.ts
Normal file
100
src/modules/attachments/services/attachment-get.service.ts
Normal file
|
|
@ -0,0 +1,100 @@
|
||||||
|
import { Injectable } from "@nestjs/common";
|
||||||
|
import { Response } from "express";
|
||||||
|
import { AdminSearchDto } from "src/modules/attachments/dtos/search-filters.dto";
|
||||||
|
import { PrismaService } from "src/prisma/prisma.service";
|
||||||
|
import { resolveAttachmentsRoot } from "src/config/attachment.config";
|
||||||
|
import * as path from 'node:path';
|
||||||
|
import { promises as fsp } from 'node:fs';
|
||||||
|
import { createReadStream } from "node:fs";
|
||||||
|
import { fileTypeFromFile } from "file-type";
|
||||||
|
import { Result } from "src/common/errors/result-error.factory";
|
||||||
|
|
||||||
|
@Injectable()
|
||||||
|
export class AttachmentGetService {
|
||||||
|
constructor(
|
||||||
|
private readonly prisma: PrismaService,
|
||||||
|
|
||||||
|
) { }
|
||||||
|
|
||||||
|
async getListVariants(id: string): Promise<Result<any, string>> {
|
||||||
|
const num_id = Number(id);
|
||||||
|
if (!Number.isFinite(num_id)) return { success: false, error: 'INVALID_ATTACHMENTS' };
|
||||||
|
const variants = await this.prisma.attachmentVariants.findMany({
|
||||||
|
where: { attachment_id: num_id },
|
||||||
|
orderBy: { variant: 'asc' },
|
||||||
|
select: { variant: true, bytes: true, width: true, height: true, path: true, created_at: true },
|
||||||
|
});
|
||||||
|
return { success: true, data: variants };
|
||||||
|
}
|
||||||
|
|
||||||
|
async searchAttachmentWithFilters(dto: AdminSearchDto): Promise<Result<any, string>> {
|
||||||
|
const where: any = {};
|
||||||
|
if (dto.owner_type) where.owner_type = dto.owner_type;
|
||||||
|
if (dto.owner_id) where.owner_id = dto.owner_id;
|
||||||
|
|
||||||
|
if (dto.date_from || dto.date_to) {
|
||||||
|
where.created_at = {};
|
||||||
|
if (dto.date_from) where.created_at.gte = new Date(dto.date_from + 'T00:00:00Z');
|
||||||
|
if (dto.date_to) where.created_at.lte = new Date(dto.date_to + 'T23:59:59Z');
|
||||||
|
}
|
||||||
|
|
||||||
|
const page = dto.page ?? 1;
|
||||||
|
const page_size = dto.page_size ?? 50;
|
||||||
|
const skip = (page - 1) * page_size;
|
||||||
|
const take = page_size;
|
||||||
|
|
||||||
|
const [items, total] = await this.prisma.$transaction([
|
||||||
|
this.prisma.attachments.findMany({
|
||||||
|
where,
|
||||||
|
orderBy: { created_at: 'desc' },
|
||||||
|
skip, take,
|
||||||
|
include: {
|
||||||
|
blob: {
|
||||||
|
select: { mime: true, size: true, storage_path: true, sha256: true },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
this.prisma.attachments.count({ where }),
|
||||||
|
]);
|
||||||
|
|
||||||
|
return { success: true, data: { page, page_size: take, total, items } };
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
async findAttachmentById(id: string, variant: string | undefined, res: Response): Promise<Result<boolean, string>> {
|
||||||
|
const num_id = Number(id);
|
||||||
|
if (!Number.isFinite(num_id)) return { success: false, error: 'INVALID_ATTACHMENTS' };
|
||||||
|
|
||||||
|
const attachment = await this.prisma.attachments.findUnique({
|
||||||
|
where: { id: num_id },
|
||||||
|
include: { blob: true },
|
||||||
|
});
|
||||||
|
if (!attachment) return { success: false, error: 'ATTACHMENT_NOT_FOUND' };
|
||||||
|
|
||||||
|
const relative = variant ? `${attachment.blob.storage_path}.${variant}` : attachment.blob.storage_path;
|
||||||
|
const abs = path.join(resolveAttachmentsRoot(), relative);
|
||||||
|
|
||||||
|
let stat;
|
||||||
|
try {
|
||||||
|
stat = await fsp.stat(abs);
|
||||||
|
} catch {
|
||||||
|
return { success: false, error: 'INVALID_FILE_PATH' };
|
||||||
|
}
|
||||||
|
|
||||||
|
let mime = attachment.blob.mime;
|
||||||
|
try {
|
||||||
|
const kind = await fileTypeFromFile(abs);
|
||||||
|
if (kind?.mime) mime = kind.mime;
|
||||||
|
} catch { }
|
||||||
|
res.set('Content-Type', mime);
|
||||||
|
res.set('Content-Length', String(stat.size));
|
||||||
|
res.set('ETag', `"sha256-${attachment.blob.sha256}${variant ? '.' + variant : ''}"`);
|
||||||
|
res.set('Last-Modified', stat.mtime.toUTCString());
|
||||||
|
res.set('Cache-Control', 'private, max-age=31536000, immutable');
|
||||||
|
res.set('X-Content-Type-Options', 'nosniff');
|
||||||
|
|
||||||
|
createReadStream(abs).pipe(res);
|
||||||
|
return { success: true, data: true };
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,87 @@
|
||||||
|
import { Injectable } from "@nestjs/common";
|
||||||
|
import { allowedMimes } from "src/modules/attachments/config/upload.config";
|
||||||
|
import { UploadMetaAttachmentsDto } from "src/modules/attachments/dtos/upload-meta-attachments.dto";
|
||||||
|
import { Readable } from "node:stream";
|
||||||
|
import { PrismaService } from "src/prisma/prisma.service";
|
||||||
|
import { fileTypeFromBuffer } from "file-type";
|
||||||
|
import { DiskStorageService } from "src/modules/attachments/services/disk-storage.service";
|
||||||
|
import { VariantsQueue } from "src/modules/attachments/services/variants.queue";
|
||||||
|
import { Result } from "src/common/errors/result-error.factory";
|
||||||
|
|
||||||
|
@Injectable()
|
||||||
|
export class AttachmentUploadService {
|
||||||
|
constructor(
|
||||||
|
private readonly prisma: PrismaService,
|
||||||
|
private readonly disk: DiskStorageService,
|
||||||
|
private readonly variantsQ: VariantsQueue,
|
||||||
|
) { }
|
||||||
|
|
||||||
|
async uploadAttachment(file?: Express.Multer.File, meta?: UploadMetaAttachmentsDto): Promise<Result<any, string>> {
|
||||||
|
if (!file) return { success: false, error: 'FILE_NOT_FOUND' };
|
||||||
|
|
||||||
|
//magic detection using binary signature
|
||||||
|
const kind = await fileTypeFromBuffer(file.buffer).catch(() => null);
|
||||||
|
const detected_mime = kind?.mime || file.mimetype || 'application/octet-stream';
|
||||||
|
|
||||||
|
//strict whitelist
|
||||||
|
if (!allowedMimes().includes(detected_mime)) {
|
||||||
|
return { success: false, error: 'INVALID_ATTACHMENT_TYPE' };
|
||||||
|
}
|
||||||
|
|
||||||
|
//Saving FS (hash + CAS + unDupes)
|
||||||
|
const stream = Readable.from(file.buffer);
|
||||||
|
const { sha256, storage_path, size } = await this.disk.saveStreamAndHash(stream);
|
||||||
|
|
||||||
|
const now = new Date();
|
||||||
|
const attachment = await this.prisma.$transaction(async (tx) => {
|
||||||
|
//upsert blob: +1 ref
|
||||||
|
await tx.blobs.upsert({
|
||||||
|
where: { sha256 },
|
||||||
|
create: {
|
||||||
|
sha256,
|
||||||
|
storage_path: storage_path,
|
||||||
|
size,
|
||||||
|
mime: detected_mime,
|
||||||
|
refcount: 1,
|
||||||
|
created_at: now,
|
||||||
|
},
|
||||||
|
update: { //only increment, does not change the storage path
|
||||||
|
refcount: { increment: 1 },
|
||||||
|
mime: detected_mime, //update mime and size to keep last image
|
||||||
|
size,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
const att = await tx.attachments.create({
|
||||||
|
data: {
|
||||||
|
sha256,
|
||||||
|
owner_type: meta?.owner_type ?? 'EXPENSE',
|
||||||
|
owner_id: meta?.owner_id ?? 'unknown',
|
||||||
|
original_name: file.originalname,
|
||||||
|
status: 'ACTIVE',
|
||||||
|
retention_policy: (meta?.retention_policy ?? 'EXPENSE_7Y') as any,
|
||||||
|
created_by: meta?.created_by ?? 'system',
|
||||||
|
created_at: now,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
return att;
|
||||||
|
});
|
||||||
|
|
||||||
|
await this.variantsQ.enqueue(attachment.id, detected_mime);
|
||||||
|
|
||||||
|
return {
|
||||||
|
success: true,
|
||||||
|
data: {
|
||||||
|
ok: true,
|
||||||
|
id: attachment.id,
|
||||||
|
sha256,
|
||||||
|
storage_path: storage_path,
|
||||||
|
size,
|
||||||
|
mime: detected_mime,
|
||||||
|
original_name: file.originalname,
|
||||||
|
owner_type: attachment.owner_type,
|
||||||
|
owner_id: attachment.owner_id,
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -4,32 +4,21 @@ import { promises as fsp } from 'node:fs';
|
||||||
import { createWriteStream, statSync, existsSync } from 'node:fs';
|
import { createWriteStream, statSync, existsSync } from 'node:fs';
|
||||||
import { join, dirname } from 'node:path';
|
import { join, dirname } from 'node:path';
|
||||||
import { pipeline } from 'node:stream/promises';
|
import { pipeline } from 'node:stream/promises';
|
||||||
import { ATT_TMP_DIR, resolveAttachmentsRoot } from 'src/config/attachment.config';
|
import { ATT_TMP_DIR } from 'src/config/attachment.config';
|
||||||
|
import { casPathFor, getAbsolutePath } from 'src/modules/attachments/utils/cas.util';
|
||||||
|
|
||||||
export type SaveResult = { sha256: string, storage_path: string, size: number };
|
export type SaveResult = { sha256: string, storage_path: string, size: number };
|
||||||
|
|
||||||
@Injectable()
|
@Injectable()
|
||||||
export class DiskStorageService {
|
export class DiskStorageService {
|
||||||
private root = resolveAttachmentsRoot();
|
// async exists(storagePathRel: string) {
|
||||||
|
// try {
|
||||||
private casPath(hash: string) {
|
// statSync(getAbsolutePath(storagePathRel));
|
||||||
const a = hash.slice(0,2), b = hash.slice(2,4);
|
// return true;
|
||||||
return `sha256/${a}/${b}/${hash}`;
|
// } catch {
|
||||||
}
|
// return false;
|
||||||
|
// }
|
||||||
//chemin absolue du storage
|
// }
|
||||||
getAbsolutePath(storagePathRel: string) {
|
|
||||||
return join(this.root, storagePathRel);
|
|
||||||
}
|
|
||||||
|
|
||||||
async exists(storagePathRel: string) {
|
|
||||||
try {
|
|
||||||
statSync(this.getAbsolutePath(storagePathRel));
|
|
||||||
return true;
|
|
||||||
}catch {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
//adds file and hash it
|
//adds file and hash it
|
||||||
async saveStreamAndHash(input: NodeJS.ReadableStream): Promise<SaveResult> {
|
async saveStreamAndHash(input: NodeJS.ReadableStream): Promise<SaveResult> {
|
||||||
|
|
@ -44,8 +33,8 @@ export class DiskStorageService {
|
||||||
await pipeline(input, tmpOut); //await end of writing stream
|
await pipeline(input, tmpOut); //await end of writing stream
|
||||||
|
|
||||||
const sha = hash.digest('hex');
|
const sha = hash.digest('hex');
|
||||||
const rel = this.casPath(sha);
|
const rel = casPathFor(sha);
|
||||||
const finalAbs = this.getAbsolutePath(rel);
|
const finalAbs = getAbsolutePath(rel);
|
||||||
|
|
||||||
// 2- is there is no destination => move (atomic renaming on the same volume)
|
// 2- is there is no destination => move (atomic renaming on the same volume)
|
||||||
if (!existsSync(finalAbs)) {
|
if (!existsSync(finalAbs)) {
|
||||||
|
|
|
||||||
|
|
@ -11,10 +11,9 @@ export class VariantsQueue {
|
||||||
}
|
}
|
||||||
|
|
||||||
enqueue(attachment_id: number, mime: string) {
|
enqueue(attachment_id: number, mime: string) {
|
||||||
if(!mime.startsWith('image/')) {
|
if (!mime.startsWith('image/')) return Promise.resolve();
|
||||||
return Promise.resolve();
|
return this.queue.add(
|
||||||
}
|
'generate',
|
||||||
return this.queue.add('generate',
|
|
||||||
{ attachment_id, mime },
|
{ attachment_id, mime },
|
||||||
{ attempts: 3, backoff: { type: 'exponential', delay: 2000 } }
|
{ attempts: 3, backoff: { type: 'exponential', delay: 2000 } }
|
||||||
);
|
);
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,17 @@
|
||||||
|
import { join } from "node:path";
|
||||||
|
|
||||||
export function casPathFor(hash: string) {
|
export function casPathFor(hash: string) {
|
||||||
const a = hash.slice(0, 2), b = hash.slice(2, 4);
|
const a = hash.slice(0, 2), b = hash.slice(2, 4);
|
||||||
return `sha256/${a}/${b}/${hash}`;
|
return `sha256/${a}/${b}/${hash}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
//chemin absolue du storage
|
||||||
|
export function getAbsolutePath(storagePathRel: string) {
|
||||||
|
return join(this.root, storagePathRel);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
export function startOfYear(): Date {
|
||||||
|
const now = new Date();
|
||||||
|
return new Date(Date.UTC(now.getUTCFullYear(), 0, 1, 0, 0, 0, 0));
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue
Block a user