#118 feat: add File Lifetime

src/domain/interfaces/IFileLocalStorage.ts

@@ -1,5 +1,6 @@
 import {IFileStorage} from './IFileStorage';
 
 export interface IFileLocalStorage extends IFileStorage {
-    getFilesPaths(): string[] | null;
+    getFilesPaths(): string[] | null,
+    getFileCreateTimeMs(fileName: string): Promise<number>,
 }

src/domain/interfaces/IFileRepository.ts

@@ -5,12 +5,13 @@ import FileStorageEnum from '../enums/FileStorageEnum';
 export const IFileRepository = 'IFileRepository';
 
 export interface IFileRepository extends ICrudRepository<FileModel> {
-    getFileWithDocument: (fileName: string) => Promise<FileModel>;
-    getFilesPathsByStorageName: (storageName: FileStorageEnum) => Promise<string[] | null>;
+    getFileWithDocument: (fileName: string) => Promise<FileModel>,
+    getFilesPathsByStorageName: (storageName: FileStorageEnum) => Promise<string[] | null>,
     getUnusedFilesIds: (config: {
-        fileNameLike: string,
-        ignoredTables: string[],
-        isEmpty: boolean,
+        fileNameLike?: string,
+        ignoredTables?: string[],
+        isEmpty?: boolean,
+        unusedFileLifetimeMs?: number,
     }) => Promise<number[]>,
     getCount: () => Promise<number>,
 }

src/domain/services/DeleteLostAndTemporaryFilesService.ts

@@ -7,14 +7,30 @@ import {
     GetFileModelsPathUsecaseToken,
     IGetFileModelsPathUsecase,
 } from '../../usecases/getFilePathModels/interfaces/IGetFileModelsPathUsecase';
+import {FileConfigService} from './FileConfigService';
+
+async function isFileJustCreated(storage: IFileLocalStorage, filePath: string, currentTimeMs: number, justUploadedFileLifetimeMs: number) {
+    let createTimeFileMs: number;
+    try {
+        createTimeFileMs = await storage.getFileCreateTimeMs(filePath);
+    } catch (error) {
+        Sentry.captureException(error);
+        return true;
+    }
+
+    return (currentTimeMs - createTimeFileMs) < justUploadedFileLifetimeMs;
+}
 
 export class DeleteLostAndTemporaryFilesService {
     constructor(
         @Inject(IFileStorageFactory)
         private fileStorageFactory: IFileStorageFactory,
+        @Inject(FileConfigService)
+        protected readonly fileConfigService: FileConfigService,
         @Optional() @Inject(GetFileModelsPathUsecaseToken)
         private getFileModelsPathUsecase: IGetFileModelsPathUsecase,
-    ) {}
+    ) {
+    }
 
     /**
      * @dev This feature is currently only available for local storage.
@@ -28,9 +44,12 @@ export class DeleteLostAndTemporaryFilesService {
         if (!storage) {
             return;
         }
+        const currentTimeMs = (new Date()).getTime();
         const lostAndTemporaryFilesPaths = await this.getLostAndTemporaryFilesPaths(storageName);
         for (const filePath of lostAndTemporaryFilesPaths) {
-            await storage.deleteFile(filePath);
+            if (!(await isFileJustCreated(storage, filePath, currentTimeMs, this.fileConfigService.justUploadedTempFileLifetimeMs))) {
+                await storage.deleteFile(filePath);
+            }
         }
     }
 

src/domain/services/FileConfigService.ts

@@ -1,6 +1,6 @@
+import {join} from 'path';
 import {toInteger as _toInteger} from 'lodash';
 import {OnModuleInit} from '@nestjs/common';
-import {join} from 'path';
 import {CronExpression} from '@nestjs/schedule';
 import {normalizeBoolean} from '@steroidsjs/nest/infrastructure/decorators/fields/BooleanField';
 import FileStorageEnum from '../enums/FileStorageEnum';
@@ -117,6 +117,33 @@ export class FileConfigService implements OnModuleInit, IFileModuleConfig {
      */
     public deleteFileFromStorage: boolean;
 
+    /**
+     * Temporary file lifetime, stored in milliseconds.
+     * The value is configured via env in seconds and will be
+     * automatically converted to milliseconds.
+     * Default: 10 seconds.
+     *
+     * Env:
+     *  - JUST_UPLOADED_TEMP_FILE_LIFETIME_S
+     */
+    public justUploadedTempFileLifetimeMs: number;
+
+    /**
+     * Unused file lifetime, stored in milliseconds.
+     * The value is configured via env in seconds and will be
+     * automatically converted to milliseconds.
+     * Default: 86400 seconds (1 day).
+     *
+     * Env:
+     *  - JUST_UPLOADED_UNUSED_FILE_LIFETIME_S
+     */
+    public justUploadedUnusedFileLifetimeMs: number;
+
+    /**
+     * Temporary file lifetime
+     */
+    public justUploadedFileLifetimeMs: number;
+
     constructor(
         private custom: IFileModuleConfig,
     ) {
@@ -181,5 +208,11 @@ export class FileConfigService implements OnModuleInit, IFileModuleConfig {
         };
 
         this.deleteFileFromStorage = custom.deleteFileFromStorage;
+
+        this.justUploadedTempFileLifetimeMs = custom.justUploadedTempFileLifetimeMs
+          || parseInt(process.env.JUST_UPLOADED_TEMP_FILE_LIFETIME_S || '10', 10) * 1000;
+
+        this.justUploadedUnusedFileLifetimeMs = custom.justUploadedUnusedFileLifetimeMs
+          || parseInt(process.env.JUST_UPLOADED_UNUSED_FILE_LIFETIME_S || String(24 * 60 * 60), 10) * 1000;
     }
 }

src/domain/services/FileService.ts

@@ -309,9 +309,10 @@ export class FileService extends ReadService<FileModel> {
     }
 
     public async getUnusedFilesIds(config: {
-        fileNameLike: string,
-        ignoredTables: string[],
-        isEmpty: boolean,
+        fileNameLike?: string,
+        ignoredTables?: string[],
+        isEmpty?: boolean,
+        unusedFileLifetimeMs?: number,
     }): Promise<number[]> {
         return this.repository.getUnusedFilesIds(config);
     }

src/domain/storages/FileLocalStorage.ts

@@ -1,6 +1,8 @@
 import {Readable} from 'stream';
 import {join} from 'path';
 import * as fs from 'fs';
+import {existsSync, Stats} from 'fs';
+import {stat} from 'fs/promises';
 import * as md5File from 'md5-file';
 import {DataMapper} from '@steroidsjs/nest/usecases/helpers/DataMapper';
 import * as Sentry from '@sentry/node';
@@ -54,6 +56,15 @@ export class FileLocalStorage implements IFileLocalStorage {
         return [this.rootUrl, file.folder, file.fileName].filter(Boolean).join('/');
     }
 
+    public async getFileCreateTimeMs(fileName: string) {
+        const filePath = join(this.rootPath, fileName);
+        if (!existsSync(filePath)) {
+            throw new Error(`File ${fileName} not exist`);
+        }
+        const stats: Stats = await stat(filePath);
+        return (stats.birthtime || stats.mtime).getTime();
+    }
+
     getFilesPaths(relativePath = ''): string[] | null {
         try {
             const folderPath = join(this.rootPath, relativePath);

src/infrastructure/commands/ClearUnusedFilesCommand.ts

@@ -3,12 +3,15 @@ import {Command, Option} from 'nestjs-command';
 import {Inject, Injectable} from '@nestjs/common';
 import {IFileService} from '@steroidsjs/nest-modules/file/services/IFileService';
 import {FileService} from '../../domain/services/FileService';
+import {FileConfigService} from '../../domain/services/FileConfigService';
 
 @Injectable()
 export class ClearUnusedFilesCommand {
     constructor(
         @Inject(IFileService)
         private fileService: FileService,
+        @Inject(FileConfigService)
+        private fileConfigService: FileConfigService,
     ) {
     }
 
@@ -67,6 +70,7 @@ export class ClearUnusedFilesCommand {
             ignoredTables,
             fileNameLike: nameLike,
             isEmpty,
+            unusedFileLifetimeMs: this.fileConfigService.justUploadedUnusedFileLifetimeMs,
         });
         console.log(`Всего файлов: ${totalFilesCount}`);
         console.log(`Файлов для удаления: ${unusedFilesIds.length}`);

src/infrastructure/repositories/FileRepository.ts

@@ -54,9 +54,10 @@ export class FileRepository extends CrudRepository<FileModel> implements IFileRe
     }
 
     public async getUnusedFilesIds(config: {
-        fileNameLike: string,
-        ignoredTables: string[],
-        isEmpty: boolean,
+        fileNameLike?: string,
+        ignoredTables?: string[],
+        isEmpty?: boolean,
+        unusedFileLifetimeMs?: number,
     }): Promise<number[]> {
         // Массив объектов, где каждый объект содержит название таблицы и колонку в этой таблице, ссылающуюся на таблицу file
         const tablesWithFileReferenceColumn: Array<{table_name: string, col_name: string}> = await this.dbRepository.query(`
@@ -83,7 +84,7 @@ export class FileRepository extends CrudRepository<FileModel> implements IFileRe
                 return [];
             }
             const tableFilesIds = await this.dbRepository.query(`
-                    SELECT DISTINCT "${table.col_name}" as id FROM ${table.table_name}
+                    SELECT DISTINCT "${table.col_name}" as id FROM "${table.table_name}"
                 `);
             return tableFilesIds.map(item => item.id);
         }));
@@ -101,6 +102,11 @@ export class FileRepository extends CrudRepository<FileModel> implements IFileRe
             allFilesQb.andWhere('(model.fileSize = 0 OR model.fileSize IS NULL)');
         }
 
+        if (config.unusedFileLifetimeMs) {
+            const thresholdDate = new Date(Date.now() - config.unusedFileLifetimeMs);
+            allFilesQb.andWhere('model."createTime" < :threshold', {threshold: thresholdDate});
+        }
+
         return (await allFilesQb.getRawMany())
             .map(file => file.model_id)
             .filter(fileId => !usedFilesIds.includes(fileId));

src/usecases/getFilePathModels/GetFileModelsPathUsecase.ts

@@ -1,4 +1,5 @@
 import {Inject, Injectable} from '@nestjs/common';
+import {IFileService} from '@steroidsjs/nest-modules/file/services/IFileService';
 import {FileService} from '../../domain/services/FileService';
 import {FileImageService} from '../../domain/services/FileImageService';
 import {FileStorageEnum} from '../../domain/enums/FileStorageEnum';
@@ -7,7 +8,7 @@ import {IGetFileModelsPathUsecase} from './interfaces/IGetFileModelsPathUsecase'
 @Injectable()
 export class GetFileModelsPathUsecase implements IGetFileModelsPathUsecase {
     constructor(
-        @Inject(FileService)
+        @Inject(IFileService)
         protected readonly fileService: FileService,
         @Inject(FileImageService)
         protected readonly fileImageService: FileImageService,