diff --git a/src/domain/interfaces/IFileLocalStorage.ts b/src/domain/interfaces/IFileLocalStorage.ts index 3e6ba52..29b8645 100644 --- a/src/domain/interfaces/IFileLocalStorage.ts +++ b/src/domain/interfaces/IFileLocalStorage.ts @@ -1,5 +1,6 @@ import {IFileStorage} from './IFileStorage'; export interface IFileLocalStorage extends IFileStorage { - getFilesPaths(): string[] | null; + getFilesPaths(): string[] | null, + getFileCreateTimeMs(fileName: string): Promise, } diff --git a/src/domain/interfaces/IFileRepository.ts b/src/domain/interfaces/IFileRepository.ts index b2ed507..84e9ca8 100644 --- a/src/domain/interfaces/IFileRepository.ts +++ b/src/domain/interfaces/IFileRepository.ts @@ -5,12 +5,13 @@ import FileStorageEnum from '../enums/FileStorageEnum'; export const IFileRepository = 'IFileRepository'; export interface IFileRepository extends ICrudRepository { - getFileWithDocument: (fileName: string) => Promise; - getFilesPathsByStorageName: (storageName: FileStorageEnum) => Promise; + getFileWithDocument: (fileName: string) => Promise, + getFilesPathsByStorageName: (storageName: FileStorageEnum) => Promise, getUnusedFilesIds: (config: { - fileNameLike: string, - ignoredTables: string[], - isEmpty: boolean, + fileNameLike?: string, + ignoredTables?: string[], + isEmpty?: boolean, + unusedFileLifetimeMs?: number, }) => Promise, getCount: () => Promise, } diff --git a/src/domain/services/DeleteLostAndTemporaryFilesService.ts b/src/domain/services/DeleteLostAndTemporaryFilesService.ts index f62d004..da6d5ed 100644 --- a/src/domain/services/DeleteLostAndTemporaryFilesService.ts +++ b/src/domain/services/DeleteLostAndTemporaryFilesService.ts @@ -7,14 +7,30 @@ import { GetFileModelsPathUsecaseToken, IGetFileModelsPathUsecase, } from '../../usecases/getFilePathModels/interfaces/IGetFileModelsPathUsecase'; +import {FileConfigService} from './FileConfigService'; + +async function isFileJustCreated(storage: IFileLocalStorage, filePath: string, currentTimeMs: number, justUploadedFileLifetimeMs: number) { + let createTimeFileMs: number; + try { + createTimeFileMs = await storage.getFileCreateTimeMs(filePath); + } catch (error) { + Sentry.captureException(error); + return true; + } + + return (currentTimeMs - createTimeFileMs) < justUploadedFileLifetimeMs; +} export class DeleteLostAndTemporaryFilesService { constructor( @Inject(IFileStorageFactory) private fileStorageFactory: IFileStorageFactory, + @Inject(FileConfigService) + protected readonly fileConfigService: FileConfigService, @Optional() @Inject(GetFileModelsPathUsecaseToken) private getFileModelsPathUsecase: IGetFileModelsPathUsecase, - ) {} + ) { + } /** * @dev This feature is currently only available for local storage. @@ -28,9 +44,12 @@ export class DeleteLostAndTemporaryFilesService { if (!storage) { return; } + const currentTimeMs = (new Date()).getTime(); const lostAndTemporaryFilesPaths = await this.getLostAndTemporaryFilesPaths(storageName); for (const filePath of lostAndTemporaryFilesPaths) { - await storage.deleteFile(filePath); + if (!(await isFileJustCreated(storage, filePath, currentTimeMs, this.fileConfigService.justUploadedTempFileLifetimeMs))) { + await storage.deleteFile(filePath); + } } } diff --git a/src/domain/services/FileConfigService.ts b/src/domain/services/FileConfigService.ts index f683080..f0e4bf8 100644 --- a/src/domain/services/FileConfigService.ts +++ b/src/domain/services/FileConfigService.ts @@ -1,6 +1,6 @@ +import {join} from 'path'; import {toInteger as _toInteger} from 'lodash'; import {OnModuleInit} from '@nestjs/common'; -import {join} from 'path'; import {CronExpression} from '@nestjs/schedule'; import {normalizeBoolean} from '@steroidsjs/nest/infrastructure/decorators/fields/BooleanField'; import FileStorageEnum from '../enums/FileStorageEnum'; @@ -117,6 +117,33 @@ export class FileConfigService implements OnModuleInit, IFileModuleConfig { */ public deleteFileFromStorage: boolean; + /** + * Temporary file lifetime, stored in milliseconds. + * The value is configured via env in seconds and will be + * automatically converted to milliseconds. + * Default: 10 seconds. + * + * Env: + * - JUST_UPLOADED_TEMP_FILE_LIFETIME_S + */ + public justUploadedTempFileLifetimeMs: number; + + /** + * Unused file lifetime, stored in milliseconds. + * The value is configured via env in seconds and will be + * automatically converted to milliseconds. + * Default: 86400 seconds (1 day). + * + * Env: + * - JUST_UPLOADED_UNUSED_FILE_LIFETIME_S + */ + public justUploadedUnusedFileLifetimeMs: number; + + /** + * Temporary file lifetime + */ + public justUploadedFileLifetimeMs: number; + constructor( private custom: IFileModuleConfig, ) { @@ -181,5 +208,11 @@ export class FileConfigService implements OnModuleInit, IFileModuleConfig { }; this.deleteFileFromStorage = custom.deleteFileFromStorage; + + this.justUploadedTempFileLifetimeMs = custom.justUploadedTempFileLifetimeMs + || parseInt(process.env.JUST_UPLOADED_TEMP_FILE_LIFETIME_S || '10', 10) * 1000; + + this.justUploadedUnusedFileLifetimeMs = custom.justUploadedUnusedFileLifetimeMs + || parseInt(process.env.JUST_UPLOADED_UNUSED_FILE_LIFETIME_S || String(24 * 60 * 60), 10) * 1000; } } diff --git a/src/domain/services/FileService.ts b/src/domain/services/FileService.ts index cd2b0ca..84f70ff 100644 --- a/src/domain/services/FileService.ts +++ b/src/domain/services/FileService.ts @@ -309,9 +309,10 @@ export class FileService extends ReadService { } public async getUnusedFilesIds(config: { - fileNameLike: string, - ignoredTables: string[], - isEmpty: boolean, + fileNameLike?: string, + ignoredTables?: string[], + isEmpty?: boolean, + unusedFileLifetimeMs?: number, }): Promise { return this.repository.getUnusedFilesIds(config); } diff --git a/src/domain/storages/FileLocalStorage.ts b/src/domain/storages/FileLocalStorage.ts index 5496b13..44cccf0 100644 --- a/src/domain/storages/FileLocalStorage.ts +++ b/src/domain/storages/FileLocalStorage.ts @@ -1,6 +1,8 @@ import {Readable} from 'stream'; import {join} from 'path'; import * as fs from 'fs'; +import {existsSync, Stats} from 'fs'; +import {stat} from 'fs/promises'; import * as md5File from 'md5-file'; import {DataMapper} from '@steroidsjs/nest/usecases/helpers/DataMapper'; import * as Sentry from '@sentry/node'; @@ -54,6 +56,15 @@ export class FileLocalStorage implements IFileLocalStorage { return [this.rootUrl, file.folder, file.fileName].filter(Boolean).join('/'); } + public async getFileCreateTimeMs(fileName: string) { + const filePath = join(this.rootPath, fileName); + if (!existsSync(filePath)) { + throw new Error(`File ${fileName} not exist`); + } + const stats: Stats = await stat(filePath); + return (stats.birthtime || stats.mtime).getTime(); + } + getFilesPaths(relativePath = ''): string[] | null { try { const folderPath = join(this.rootPath, relativePath); diff --git a/src/infrastructure/commands/ClearUnusedFilesCommand.ts b/src/infrastructure/commands/ClearUnusedFilesCommand.ts index 377e5b1..9ce242a 100644 --- a/src/infrastructure/commands/ClearUnusedFilesCommand.ts +++ b/src/infrastructure/commands/ClearUnusedFilesCommand.ts @@ -3,12 +3,15 @@ import {Command, Option} from 'nestjs-command'; import {Inject, Injectable} from '@nestjs/common'; import {IFileService} from '@steroidsjs/nest-modules/file/services/IFileService'; import {FileService} from '../../domain/services/FileService'; +import {FileConfigService} from '../../domain/services/FileConfigService'; @Injectable() export class ClearUnusedFilesCommand { constructor( @Inject(IFileService) private fileService: FileService, + @Inject(FileConfigService) + private fileConfigService: FileConfigService, ) { } @@ -67,6 +70,7 @@ export class ClearUnusedFilesCommand { ignoredTables, fileNameLike: nameLike, isEmpty, + unusedFileLifetimeMs: this.fileConfigService.justUploadedUnusedFileLifetimeMs, }); console.log(`Всего файлов: ${totalFilesCount}`); console.log(`Файлов для удаления: ${unusedFilesIds.length}`); diff --git a/src/infrastructure/repositories/FileRepository.ts b/src/infrastructure/repositories/FileRepository.ts index 27fa309..fb2193a 100644 --- a/src/infrastructure/repositories/FileRepository.ts +++ b/src/infrastructure/repositories/FileRepository.ts @@ -54,9 +54,10 @@ export class FileRepository extends CrudRepository implements IFileRe } public async getUnusedFilesIds(config: { - fileNameLike: string, - ignoredTables: string[], - isEmpty: boolean, + fileNameLike?: string, + ignoredTables?: string[], + isEmpty?: boolean, + unusedFileLifetimeMs?: number, }): Promise { // Массив объектов, где каждый объект содержит название таблицы и колонку в этой таблице, ссылающуюся на таблицу file const tablesWithFileReferenceColumn: Array<{table_name: string, col_name: string}> = await this.dbRepository.query(` @@ -83,7 +84,7 @@ export class FileRepository extends CrudRepository implements IFileRe return []; } const tableFilesIds = await this.dbRepository.query(` - SELECT DISTINCT "${table.col_name}" as id FROM ${table.table_name} + SELECT DISTINCT "${table.col_name}" as id FROM "${table.table_name}" `); return tableFilesIds.map(item => item.id); })); @@ -101,6 +102,11 @@ export class FileRepository extends CrudRepository implements IFileRe allFilesQb.andWhere('(model.fileSize = 0 OR model.fileSize IS NULL)'); } + if (config.unusedFileLifetimeMs) { + const thresholdDate = new Date(Date.now() - config.unusedFileLifetimeMs); + allFilesQb.andWhere('model."createTime" < :threshold', {threshold: thresholdDate}); + } + return (await allFilesQb.getRawMany()) .map(file => file.model_id) .filter(fileId => !usedFilesIds.includes(fileId)); diff --git a/src/usecases/getFilePathModels/GetFileModelsPathUsecase.ts b/src/usecases/getFilePathModels/GetFileModelsPathUsecase.ts index 18c0311..8a4f18d 100644 --- a/src/usecases/getFilePathModels/GetFileModelsPathUsecase.ts +++ b/src/usecases/getFilePathModels/GetFileModelsPathUsecase.ts @@ -1,4 +1,5 @@ import {Inject, Injectable} from '@nestjs/common'; +import {IFileService} from '@steroidsjs/nest-modules/file/services/IFileService'; import {FileService} from '../../domain/services/FileService'; import {FileImageService} from '../../domain/services/FileImageService'; import {FileStorageEnum} from '../../domain/enums/FileStorageEnum'; @@ -7,7 +8,7 @@ import {IGetFileModelsPathUsecase} from './interfaces/IGetFileModelsPathUsecase' @Injectable() export class GetFileModelsPathUsecase implements IGetFileModelsPathUsecase { constructor( - @Inject(FileService) + @Inject(IFileService) protected readonly fileService: FileService, @Inject(FileImageService) protected readonly fileImageService: FileImageService,