chore: refactor GCSManager to use official Google Cloud Storage library

Author: rafaelromcar-parabol

packages/server/fileStorage/GCSManager.ts

@@ -1,26 +1,18 @@
-import {sign} from 'jsonwebtoken'
+import {Storage} from '@google-cloud/storage'
 import mime from 'mime-types'
 import path from 'path'
 import {Logger} from '../utils/Logger'
 import FileStoreManager, {FileAssetDir} from './FileStoreManager'
 
-interface CloudKey {
-  clientEmail: string
-  privateKeyId: string
-  privateKey: string
-}
-
 export default class GCSManager extends FileStoreManager {
-  static GOOGLE_EXPIRY = 3600
   // e.g. development, production
   private envSubDir: string
   // e.g. action-files.parabol.co
   private bucket: string
-  private accessToken: string | undefined
-
   // The CDN_BASE_URL without the env, e.g. storage.google.com/:bucket
   baseUrl: string
-  private cloudKey: CloudKey
+  private storage: Storage
+
   constructor() {
     super()
     const {
@@ -30,6 +22,7 @@ export default class GCSManager extends FileStoreManager {
       GOOGLE_CLOUD_PRIVATE_KEY,
       GOOGLE_CLOUD_PRIVATE_KEY_ID
     } = process.env
+
     if (!CDN_BASE_URL || CDN_BASE_URL === 'key_CDN_BASE_URL') {
       throw new Error('CDN_BASE_URL ENV VAR NOT SET')
     }
@@ -43,6 +36,7 @@ export default class GCSManager extends FileStoreManager {
     if (!GOOGLE_GCS_BUCKET) {
       throw new Error('GOOGLE_GCS_BUCKET ENV VAR NOT SET')
     }
+
     const baseUrl = new URL(CDN_BASE_URL.replace(/^\/+/, 'https://'))
     const {hostname, pathname} = baseUrl
     if (!hostname || !pathname) {
@@ -52,64 +46,17 @@ export default class GCSManager extends FileStoreManager {
       throw new Error('CDN_BASE_URL must end with the env, no trailing slash, e.g. /production')
 
     this.envSubDir = pathname.split('/').at(-1) as string
-
     this.baseUrl = baseUrl.href.slice(0, baseUrl.href.lastIndexOf(this.envSubDir))
-
     this.bucket = GOOGLE_GCS_BUCKET
-    this.cloudKey = {
-      clientEmail: GOOGLE_CLOUD_CLIENT_EMAIL,
-      privateKey: GOOGLE_CLOUD_PRIVATE_KEY.replace(/\\n/gm, '\n'),
-      privateKeyId: GOOGLE_CLOUD_PRIVATE_KEY_ID
-    }
-    // refresh the token every hour
-    // do this on an interval vs. on demand to reduce request latency
-    // unref it so things like pushToCDN can exit
-    setInterval(
-      async () => {
-        this.accessToken = await this.getFreshAccessToken()
-      },
-      (GCSManager.GOOGLE_EXPIRY - 100) * 1000
-    ).unref()
-  }
 
-  private async getFreshAccessToken() {
-    const authUrl = 'https://www.googleapis.com/oauth2/v4/token'
-    const {clientEmail, privateKeyId, privateKey} = this.cloudKey
-    try {
-      // GCS only accepts OAuth2 Tokens
-      // To get a token, we self-sign a JWT, then trade it in for an OAuth2 Token
-      const jwt = sign(
-        {
-          scope: 'https://www.googleapis.com/auth/devstorage.read_write'
-        },
-        privateKey,
-        {
-          algorithm: 'RS256',
-          audience: authUrl,
-          subject: clientEmail,
-          issuer: clientEmail,
-          keyid: privateKeyId,
-          expiresIn: GCSManager.GOOGLE_EXPIRY
-        }
-      )
-      const accessTokenRes = await fetch(authUrl, {
-        method: 'POST',
-        body: JSON.stringify({
-          grant_type: 'urn:ietf:params:oauth:grant-type:jwt-bearer',
-          assertion: jwt
-        })
-      })
-      const accessTokenJson = await accessTokenRes.json()
-      return accessTokenJson.access_token
-    } catch (e) {
-      return undefined
-    }
-  }
-
-  private async getAccessToken() {
-    if (this.accessToken) return this.accessToken
-    this.accessToken = await this.getFreshAccessToken()
-    return this.accessToken
+    // Initialize Google Cloud Storage client with credentials
+    this.storage = new Storage({
+      credentials: {
+        client_email: GOOGLE_CLOUD_CLIENT_EMAIL,
+        private_key: GOOGLE_CLOUD_PRIVATE_KEY.replace(/\\n/gm, '\n'),
+        private_key_id: GOOGLE_CLOUD_PRIVATE_KEY_ID
+      }
+    })
   }
 
   protected async putUserFile(file: Buffer<ArrayBufferLike>, partialPath: string) {
@@ -118,28 +65,32 @@ export default class GCSManager extends FileStoreManager {
   }
 
   protected async putFile(file: Buffer<ArrayBufferLike>, fullPath: string) {
-    const url = new URL(`https://storage.googleapis.com/upload/storage/v1/b/${this.bucket}/o`)
-    url.searchParams.append('uploadType', 'media')
-    url.searchParams.append('name', fullPath)
-    const accessToken = await this.getAccessToken()
     try {
-      await fetch(url, {
-        method: 'POST',
-        body: file,
-        headers: {
-          Authorization: `Bearer ${accessToken}`,
-          Accept: 'application/json',
-          'Content-Type': mime.lookup(fullPath) || ''
-        }
-      })
+      const bucket = this.storage.bucket(this.bucket)
+      const blob = bucket.file(fullPath)
+
+      // Set the content type based on file extension
+      const contentType = mime.lookup(fullPath) || 'application/octet-stream'
+      const options = {
+        contentType,
+        resumable: false
+      }
+
+      await blob.save(file, options)
     } catch (e) {
-      // https://github.com/nodejs/undici/issues/583#issuecomment-1577475664
-      // GCS will cause undici to error randomly with `SocketError: other side closed` `code: 'UND_ERR_SOCKET'`
-      if ((e as any).cause?.code === 'UND_ERR_SOCKET') {
+      // Handle specific socket errors that might occur with GCS
+      if (
+        (e as any).code === 'ETIMEDOUT' ||
+        (e as any).code === 'ECONNRESET' ||
+        (e as any).cause?.code === 'UND_ERR_SOCKET'
+      ) {
         Logger.log('   Retrying GCS Post:', fullPath)
         await this.putFile(file, fullPath)
+      } else {
+        throw e
       }
     }
+
     return this.getPublicFileLocation(fullPath)
   }
 
@@ -151,17 +102,47 @@ export default class GCSManager extends FileStoreManager {
   prependPath(partialPath: string, assetDir: FileAssetDir = 'store') {
     return path.join(this.envSubDir, assetDir, partialPath)
   }
+
   getPublicFileLocation(fullPath: string) {
     return encodeURI(`${this.baseUrl}${fullPath}`)
   }
+
   async checkExists(partialPath: string, assetDir?: FileAssetDir) {
-    const fullPath = encodeURIComponent(this.prependPath(partialPath, assetDir))
-    const url = `https://storage.googleapis.com/storage/v1/b/${this.bucket}/o/${fullPath}`
-    const res = await fetch(url)
-    return res.status !== 404
+    const fullPath = this.prependPath(partialPath, assetDir)
+    const bucket = this.storage.bucket(this.bucket)
+    const file = bucket.file(fullPath)
+
+    try {
+      const [exists] = await file.exists()
+      return exists
+    } catch (e) {
+      Logger.error('Error checking if file exists:', e)
+      return false
+    }
   }
-  async presignUrl(url: string): Promise<string> {
-    // not implemented yet!
-    return url
+
+  async presignUrl(url: string, expiresInMinutes: number = 60): Promise<string> {
+    // Extract the file path from the URL
+    const filePathMatch = url.match(new RegExp(`${this.baseUrl}(.*)`))
+    if (!filePathMatch || !filePathMatch[1]) {
+      return url // Return original URL if we can't extract the path
+    }
+
+    const filePath = decodeURI(filePathMatch[1])
+    const bucket = this.storage.bucket(this.bucket)
+    const file = bucket.file(filePath)
+
+    try {
+      // Generate a signed URL
+      const [signedUrl] = await file.getSignedUrl({
+        action: 'read',
+        expires: Date.now() + expiresInMinutes * 60 * 1000
+      })
+
+      return signedUrl
+    } catch (e) {
+      Logger.error('Error generating signed URL:', e)
+      return url // Return the original URL if signing fails
+    }
   }
 }

packages/server/package.json

@@ -78,6 +78,7 @@
     "@dicebear/core": "^8.0.1",
     "@dicebear/initials": "^8.0.1",
     "@emoji-mart/data": "^1.2.1",
+    "@google-cloud/storage": "^7.15.2",
     "@graphql-tools/schema": "^9.0.16",
     "@hocuspocus/extension-database": "^2.15.2",
     "@hocuspocus/extension-redis": "^2.15.2",