Merge pull request #2430 from bcgov/develop

Deployment PR - 1645

Author: trslater

services/apps/alcs/src/main.ts

@@ -10,23 +10,14 @@ import {
 import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger';
 import * as config from 'config';
 import { ClsMiddleware } from 'nestjs-cls';
-import { S3StreamLogger, S3StreamLoggerOptions } from 's3-streamlogger';
 import { install } from 'source-map-support';
 import { SPLAT } from 'triple-beam';
 import * as winston from 'winston';
 import { createLogger } from 'winston';
 import { generateModuleGraph } from './commands/graph';
 import { MainModule } from './main.module';
-import { S3ClientConfig } from '@aws-sdk/client-s3';
-
-// S3StreamLogger passes its config to S3Client, but doesn't accept all the
-// properties of S3ClientConfig. This allows those properties.
-interface SafeS3StreamLoggerOptions extends S3StreamLoggerOptions {
-  config: S3StreamLoggerOptions['config'] & {
-    forcePathStyle: boolean;
-    endpoint: string;
-  };
-}
+import { S3Client } from '@aws-sdk/client-s3';
+import { RotatingS3Writable } from './s3-logger/s3-logger';
 
 const registerSwagger = (app: NestFastifyApplication) => {
   const documentBuilderConfig = new DocumentBuilder()
@@ -106,23 +97,22 @@ const registerMultiPart = async (app: NestFastifyApplication) => {
 };
 
 function setupLogger() {
-  const s3StreamOptions: SafeS3StreamLoggerOptions = {
-    rotate_every: 1000 * 60 * 60 * 24, //1 Day
-    folder: 'logs',
-    bucket: config.get<string>('STORAGE.BUCKET'),
-    name_format: '%Y-%b-%d-console.log', //https://www.npmjs.com/package/strftime
-    config: {
-      region: 'us-east-1',
-      credentials: {
-        accessKeyId: config.get('STORAGE.ACCESS_KEY'),
-        secretAccessKey: config.get('STORAGE.SECRET_KEY'),
-      },
-      forcePathStyle: true,
-      endpoint: config.get('STORAGE.URL'),
+  const s3Client = new S3Client({
+    region: 'us-east-1',
+    credentials: {
+      accessKeyId: config.get('STORAGE.ACCESS_KEY'),
+      secretAccessKey: config.get('STORAGE.SECRET_KEY'),
     },
-  };
+    forcePathStyle: true,
+    endpoint: config.get('STORAGE.URL'),
+  });
 
-  const s3Stream = new S3StreamLogger(s3StreamOptions);
+  const s3Stream = new RotatingS3Writable({
+    bucket: config.get<string>('STORAGE.BUCKET'),
+    folder: 'logs',
+    rotateEvery: 1000 * 60 * 60 * 24, // 1 day
+    s3Client,
+  });
 
   const timeStampFormat = winston.format.timestamp({
     format: 'YYYY-MMM-DD HH:mm:ss',
@@ -145,20 +135,11 @@ function setupLogger() {
   const s3Transport = new winston.transports.Stream({
     level: config.get('LOG_LEVEL'),
     stream: s3Stream,
-    format: winston.format.combine(
-      winston.format.errors({ stack: true }),
-      timeStampFormat,
-      messageFormat,
-    ),
+    format: winston.format.combine(winston.format.errors({ stack: true }), timeStampFormat, messageFormat),
   });
   const consoleTransport = new winston.transports.Console({
     level: config.get('LOG_LEVEL'),
-    format: winston.format.combine(
-      winston.format.errors({ stack: true }),
-      timeStampFormat,
-      messageFormat,
-      colorFormat,
-    ),
+    format: winston.format.combine(winston.format.errors({ stack: true }), timeStampFormat, messageFormat, colorFormat),
   });
 
   return createLogger({
@@ -171,10 +152,7 @@ function setupLogger() {
       debug: 5,
       verbose: 6,
     },
-    transports:
-      config.get('ENV') === 'production'
-        ? [consoleTransport, s3Transport]
-        : [consoleTransport],
+    transports: config.get('ENV') === 'production' ? [consoleTransport, s3Transport] : [consoleTransport],
   });
 }
 

services/apps/alcs/src/s3-logger/s3-logger.ts

@@ -0,0 +1,80 @@
+import { Writable } from 'stream';
+import { S3Client, PutObjectCommand } from '@aws-sdk/client-s3';
+
+const MESSAGE_SYMBOL = Symbol.for('message');
+
+interface RotatingS3WritableOptions {
+  bucket: string;
+  folder?: string;
+  rotateEvery?: number;
+  s3Client: S3Client;
+}
+
+export class RotatingS3Writable extends Writable {
+  private buffer: string[] = [];
+  private lastRotation: number;
+  private currentKey: string;
+
+  constructor(private options: RotatingS3WritableOptions) {
+    super({ objectMode: true });
+
+    this.lastRotation = Date.now();
+    this.currentKey = this.generateKey(this.lastRotation);
+  }
+
+  _write(chunk: any, _encoding: BufferEncoding, callback: (error?: Error | null) => void) {
+    const message = chunk?.[MESSAGE_SYMBOL] ?? chunk?.message ?? '';
+
+    if (message) {
+      this.buffer.push(message);
+    }
+
+    const now = Date.now();
+
+    if (this.options.rotateEvery && now - this.lastRotation >= this.options.rotateEvery) {
+      this.flush()
+        .then(() => {
+          this.lastRotation = now;
+          this.currentKey = this.generateKey(now);
+          callback();
+        })
+        .catch(callback);
+    } else {
+      callback();
+    }
+  }
+
+  private generateKey(dateMs: number): string {
+    const date = new Date(dateMs);
+    const year = date.getUTCFullYear();
+    const month = date.toLocaleString('en-US', { month: 'short', timeZone: 'UTC' });
+    const day = String(date.getUTCDate()).padStart(2, '0');
+
+    const filename = `${year}-${month}-${day}-console.log`;
+    const prefix = this.options.folder ? `${this.options.folder}/` : '';
+
+    return `${prefix}${filename}`;
+  }
+
+  async flush(): Promise<void> {
+    if (this.buffer.length === 0) return;
+
+    const body = this.buffer.join('\n') + '\n';
+    this.buffer = [];
+
+    await this.options.s3Client.send(
+      new PutObjectCommand({
+        Bucket: this.options.bucket,
+        Key: this.currentKey,
+        Body: body,
+        ContentType: 'text/plain',
+      }),
+    );
+  }
+
+  _final(callback: (error?: Error | null) => void) {
+    this.flush()
+      .then(() => callback())
+      .catch(callback);
+  }
+}

services/package-lock.json

@@ -70,7 +70,6 @@
     "reflect-metadata": "^0.1.14",
     "rimraf": "^5.0.5",
     "rxjs": "^7.8.2",
-    "s3-streamlogger": "^1.11.1",
     "source-map-support": "^0.5.21",
     "ts-proto": "^1.171.0",
     "ts-protoc-gen": "^0.15.0",