feat: s3 compatible storage (#15)

* feat: support S3 storage

* fix(test): fs mock issue

Related issue:
aws/aws-sdk-js-v3#3547 (comment)

* docs: s3 compatible storage

---------

Co-authored-by: Rémy Oudemans <31566929+remyoudemans@users.noreply.github.com>
Co-authored-by: Mo Elkhidir <mhharktat@gmail.com>

Author: 3 people

README.md

@@ -252,6 +252,7 @@ Currently, we support:
 - Supabase Storage
 - Local filesystem storage
 - Google Cloud Storage (gcs)
+- AWS S3 Compatible Storage (s3)
 
 More providers (S3, Azure, etc.) are welcome to be implemented by the community. The `StorageInterface` is quite simple and you can implement it for any blob storage service.
 </details>

__tests__/upload.test.ts

@@ -14,7 +14,6 @@ jest.mock('../apiUtils/storage/StorageFactory');
 jest.mock('../apiUtils/helpers/ZipHelper');
 jest.mock('../apiUtils/helpers/HashHelper');
 jest.mock('formidable');
-jest.mock('fs');
 jest.mock('adm-zip');
 
 describe('Upload API', () => {
@@ -48,7 +47,7 @@ describe('Upload API', () => {
 
     // Mock file system
     const mockFileContent = Buffer.from('test file content');
-    (fs.readFileSync as jest.Mock).mockReturnValue(mockFileContent);
+    jest.spyOn(fs, 'readFileSync').mockReturnValue(mockFileContent);
 
     // Mock AdmZip
     const mockZipFolder = {} as AdmZip;

apiUtils/storage/S3Storage.ts

@@ -0,0 +1,137 @@
+import {
+  S3Client,
+  GetObjectCommand,
+  HeadObjectCommand,
+  ListObjectsV2Command,
+  PutObjectCommand,
+  CopyObjectCommand,
+} from '@aws-sdk/client-s3';
+import { StorageInterface } from './StorageInterface';
+
+export class S3Storage implements StorageInterface {
+  private client: S3Client;
+  private bucketName: string;
+
+  constructor() {
+    if (!process.env.S3_ACCESS_KEY_ID || !process.env.S3_SECRET_ACCESS_KEY) {
+      throw new Error('S3 credentials not configured');
+    }
+    if (!process.env.S3_BUCKET_NAME) {
+      throw new Error('S3 bucket name not configured');
+    }
+    this.client = new S3Client({
+      region: process.env.S3_REGION ?? 'auto',
+      endpoint: process.env.S3_ENDPOINT,
+      credentials: {
+        accessKeyId: process.env.S3_ACCESS_KEY_ID,
+        secretAccessKey: process.env.S3_SECRET_ACCESS_KEY,
+      },
+    });
+    this.bucketName = process.env.S3_BUCKET_NAME;
+  }
+
+  async copyFile(sourcePath: string, destinationPath: string): Promise<void> {
+    const copyCommand = new CopyObjectCommand({
+      Bucket: this.bucketName,
+      CopySource: sourcePath,
+      Key: destinationPath,
+    });
+    await this.client.send(copyCommand);
+  }
+
+  async downloadFile(path: string): Promise<Buffer> {
+    const getCommand = new GetObjectCommand({
+      Bucket: this.bucketName,
+      Key: path,
+    });
+    const response = await this.client.send(getCommand);
+    const body = await response.Body?.transformToByteArray();
+    if (!body) {
+      throw new Error('No body found in response');
+    }
+    return Buffer.from(body);
+  }
+
+  async fileExists(path: string): Promise<boolean> {
+    try {
+      const files = await this.listDirectories(path);
+      if (files.length > 0) {
+        return true;
+      }
+    } catch {}
+    try {
+      const headCommand = new HeadObjectCommand({
+        Bucket: this.bucketName,
+        Key: path.split('/').shift(),
+      });
+      await this.client.send(headCommand);
+      return true;
+    } catch {
+      return false;
+    }
+  }
+
+  async listFiles(directory: string): Promise<
+    {
+      name: string;
+      updated_at: string;
+      created_at: string;
+      metadata: { size: number; mimetype: string };
+    }[]
+  > {
+    const prefix = `${directory}/`;
+    const listCommand = new ListObjectsV2Command({
+      Bucket: this.bucketName,
+      Prefix: prefix,
+    });
+    const response = await this.client.send(listCommand);
+    return (
+      response.Contents?.map((file) => ({
+        name: file.Key!.replace(prefix, ''),
+        updated_at: file.LastModified?.toISOString() ?? '',
+        created_at: file.LastModified?.toISOString() ?? '',
+        metadata: {
+          size: file.Size ?? 0,
+          mimetype: this.getMimeType(file.Key?.split('.').pop() ?? 'unknown'),
+        },
+      })) ?? []
+    );
+  }
+
+  async listDirectories(directory: string): Promise<string[]> {
+    const listCommand = new ListObjectsV2Command({
+      Bucket: this.bucketName,
+      Prefix: directory,
+      Delimiter: '/',
+    });
+    const response = await this.client.send(listCommand);
+    return (
+      response.CommonPrefixes?.map((prefix) =>
+        prefix.Prefix!.replace(directory, '').replace(/\/$/, '')
+      ) ?? []
+    );
+  }
+
+  async uploadFile(path: string, file: Buffer): Promise<string> {
+    const uploadCommand = new PutObjectCommand({
+      Bucket: this.bucketName,
+      Key: path,
+      Body: file,
+    });
+    await this.client.send(uploadCommand);
+    return path;
+  }
+
+  private getMimeType(ext: string): string {
+    const mimeTypes: { [key: string]: string } = {
+      js: 'application/javascript',
+      json: 'application/json',
+      png: 'image/png',
+      jpg: 'image/jpeg',
+      jpeg: 'image/jpeg',
+      gif: 'image/gif',
+      zip: 'application/zip',
+    };
+    return mimeTypes[ext.toLowerCase()] || 'application/octet-stream';
+  }
+}

apiUtils/storage/StorageFactory.ts

@@ -2,6 +2,7 @@ import { LocalStorage } from './LocalStorage';
 import { StorageInterface } from './StorageInterface';
 import { SupabaseStorage } from './SupabaseStorage';
 import { GCSStorage } from './GCSStorage';
+import { S3Storage } from './S3Storage';
 import { getLogger } from '../logger';
 
 const logger = getLogger('StorageFactory');
@@ -18,6 +19,8 @@ export class StorageFactory {
         StorageFactory.instance = new LocalStorage();
       } else if (storageType === 'gcs') {
         StorageFactory.instance = new GCSStorage();
+      } else if (storageType === 's3') {
+        StorageFactory.instance = new S3Storage();
       } else {
         logger.error('Unsupported storage type', { storageType });
         throw new Error('Unsupported storage type');

docs/supportedStorageAlternatives.md

@@ -29,6 +29,18 @@ GCP_BUCKET_NAME=your-gcs-bucket-name
 - Requires a GCP project with GCS bucket enabled
 - Bucket should be created manually before starting the server
 
+### AWS S3 Compatible Storage
+```env
+BLOB_STORAGE_TYPE=s3
+S3_REGION=auto
+S3_ENDPOINT=your-s3-endpoint
+S3_ACCESS_KEY_ID=your-access-key-id
+S3_SECRET_ACCESS_KEY=your-secret-access-key
+S3_BUCKET_NAME=your-s3-bucket-name
+```
+- Support all S3 compatible storage (AWS S3, Digital Ocean Spaces, Cloudflare R2, etc.)
+- Bucket should be created manually before starting the server
+
 ## Supported Database Providers
 Database configuration is managed via `DB_TYPE`.