diff --git a/services/apps/alcs/src/main.ts b/services/apps/alcs/src/main.ts index e879df8a6d..b5918ecb99 100644 --- a/services/apps/alcs/src/main.ts +++ b/services/apps/alcs/src/main.ts @@ -10,23 +10,14 @@ import { import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger'; import * as config from 'config'; import { ClsMiddleware } from 'nestjs-cls'; -import { S3StreamLogger, S3StreamLoggerOptions } from 's3-streamlogger'; import { install } from 'source-map-support'; import { SPLAT } from 'triple-beam'; import * as winston from 'winston'; import { createLogger } from 'winston'; import { generateModuleGraph } from './commands/graph'; import { MainModule } from './main.module'; -import { S3ClientConfig } from '@aws-sdk/client-s3'; - -// S3StreamLogger passes its config to S3Client, but doesn't accept all the -// properties of S3ClientConfig. This allows those properties. -interface SafeS3StreamLoggerOptions extends S3StreamLoggerOptions { - config: S3StreamLoggerOptions['config'] & { - forcePathStyle: boolean; - endpoint: string; - }; -} +import { S3Client } from '@aws-sdk/client-s3'; +import { RotatingS3Writable } from './s3-logger/s3-logger'; const registerSwagger = (app: NestFastifyApplication) => { const documentBuilderConfig = new DocumentBuilder() @@ -106,23 +97,22 @@ const registerMultiPart = async (app: NestFastifyApplication) => { }; function setupLogger() { - const s3StreamOptions: SafeS3StreamLoggerOptions = { - rotate_every: 1000 * 60 * 60 * 24, //1 Day - folder: 'logs', - bucket: config.get('STORAGE.BUCKET'), - name_format: '%Y-%b-%d-console.log', //https://www.npmjs.com/package/strftime - config: { - region: 'us-east-1', - credentials: { - accessKeyId: config.get('STORAGE.ACCESS_KEY'), - secretAccessKey: config.get('STORAGE.SECRET_KEY'), - }, - forcePathStyle: true, - endpoint: config.get('STORAGE.URL'), + const s3Client = new S3Client({ + region: 'us-east-1', + credentials: { + accessKeyId: config.get('STORAGE.ACCESS_KEY'), + secretAccessKey: config.get('STORAGE.SECRET_KEY'), }, - }; + forcePathStyle: true, + endpoint: config.get('STORAGE.URL'), + }); - const s3Stream = new S3StreamLogger(s3StreamOptions); + const s3Stream = new RotatingS3Writable({ + bucket: config.get('STORAGE.BUCKET'), + folder: 'logs', + rotateEvery: 1000 * 60 * 60 * 24, // 1 day + s3Client, + }); const timeStampFormat = winston.format.timestamp({ format: 'YYYY-MMM-DD HH:mm:ss', @@ -145,20 +135,11 @@ function setupLogger() { const s3Transport = new winston.transports.Stream({ level: config.get('LOG_LEVEL'), stream: s3Stream, - format: winston.format.combine( - winston.format.errors({ stack: true }), - timeStampFormat, - messageFormat, - ), + format: winston.format.combine(winston.format.errors({ stack: true }), timeStampFormat, messageFormat), }); const consoleTransport = new winston.transports.Console({ level: config.get('LOG_LEVEL'), - format: winston.format.combine( - winston.format.errors({ stack: true }), - timeStampFormat, - messageFormat, - colorFormat, - ), + format: winston.format.combine(winston.format.errors({ stack: true }), timeStampFormat, messageFormat, colorFormat), }); return createLogger({ @@ -171,10 +152,7 @@ function setupLogger() { debug: 5, verbose: 6, }, - transports: - config.get('ENV') === 'production' - ? [consoleTransport, s3Transport] - : [consoleTransport], + transports: config.get('ENV') === 'production' ? [consoleTransport, s3Transport] : [consoleTransport], }); } diff --git a/services/apps/alcs/src/s3-logger/s3-logger.ts b/services/apps/alcs/src/s3-logger/s3-logger.ts new file mode 100644 index 0000000000..c4dad2a2ab --- /dev/null +++ b/services/apps/alcs/src/s3-logger/s3-logger.ts @@ -0,0 +1,80 @@ +import { Writable } from 'stream'; +import { S3Client, PutObjectCommand } from '@aws-sdk/client-s3'; + +const MESSAGE_SYMBOL = Symbol.for('message'); + +interface RotatingS3WritableOptions { + bucket: string; + folder?: string; + rotateEvery?: number; + s3Client: S3Client; +} + +export class RotatingS3Writable extends Writable { + private buffer: string[] = []; + private lastRotation: number; + private currentKey: string; + + constructor(private options: RotatingS3WritableOptions) { + super({ objectMode: true }); + + this.lastRotation = Date.now(); + this.currentKey = this.generateKey(this.lastRotation); + } + + _write(chunk: any, _encoding: BufferEncoding, callback: (error?: Error | null) => void) { + const message = chunk?.[MESSAGE_SYMBOL] ?? chunk?.message ?? ''; + + if (message) { + this.buffer.push(message); + } + + const now = Date.now(); + + if (this.options.rotateEvery && now - this.lastRotation >= this.options.rotateEvery) { + this.flush() + .then(() => { + this.lastRotation = now; + this.currentKey = this.generateKey(now); + callback(); + }) + .catch(callback); + } else { + callback(); + } + } + + private generateKey(dateMs: number): string { + const date = new Date(dateMs); + const year = date.getUTCFullYear(); + const month = date.toLocaleString('en-US', { month: 'short', timeZone: 'UTC' }); + const day = String(date.getUTCDate()).padStart(2, '0'); + + const filename = `${year}-${month}-${day}-console.log`; + const prefix = this.options.folder ? `${this.options.folder}/` : ''; + + return `${prefix}${filename}`; + } + + async flush(): Promise { + if (this.buffer.length === 0) return; + + const body = this.buffer.join('\n') + '\n'; + this.buffer = []; + + await this.options.s3Client.send( + new PutObjectCommand({ + Bucket: this.options.bucket, + Key: this.currentKey, + Body: body, + ContentType: 'text/plain', + }), + ); + } + + _final(callback: (error?: Error | null) => void) { + this.flush() + .then(() => callback()) + .catch(callback); + } +} diff --git a/services/package-lock.json b/services/package-lock.json index 517e3ceaf4..90e46a4b69 100644 --- a/services/package-lock.json +++ b/services/package-lock.json @@ -51,7 +51,6 @@ "reflect-metadata": "^0.1.14", "rimraf": "^5.0.5", "rxjs": "^7.8.2", - "s3-streamlogger": "^1.11.1", "source-map-support": "^0.5.21", "ts-proto": "^1.171.0", "ts-protoc-gen": "^0.15.0", @@ -12837,16 +12836,6 @@ "tslib": "^2.1.0" } }, - "node_modules/s3-streamlogger": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/s3-streamlogger/-/s3-streamlogger-1.11.1.tgz", - "integrity": "sha512-rT/feyljvWj91jzjB5lPBkNWKqMXUGH8fU2oMEBNIxNZRJO6H/9HY52yHDIEqFf3tgHffk7YRlD8eDCr94sTow==", - "license": "ISC", - "dependencies": { - "@aws-sdk/client-s3": "^3.353.0", - "strftime": "^0.10.2" - } - }, "node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -13368,15 +13357,6 @@ "node": ">=4.0.0" } }, - "node_modules/strftime": { - "version": "0.10.3", - "resolved": "https://registry.npmjs.org/strftime/-/strftime-0.10.3.tgz", - "integrity": "sha512-DZrDUeIF73eKJ4/GgGuv8UHWcUQPYDYfDeQFj3jrx+JZl6GQE656MbHIpvbo4mEG9a5DgS8GRCc5DxJXD2udDQ==", - "license": "MIT", - "engines": { - "node": ">=0.2.0" - } - }, "node_modules/string_decoder": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", diff --git a/services/package.json b/services/package.json index 679c490bd9..c5a3aadd46 100644 --- a/services/package.json +++ b/services/package.json @@ -70,7 +70,6 @@ "reflect-metadata": "^0.1.14", "rimraf": "^5.0.5", "rxjs": "^7.8.2", - "s3-streamlogger": "^1.11.1", "source-map-support": "^0.5.21", "ts-proto": "^1.171.0", "ts-protoc-gen": "^0.15.0",