feat: add GCP connectors (bucket list, table creation)

packages/core/package.json

@@ -61,6 +61,8 @@
         "@aws-sdk/client-s3": "^3.826.0",
         "@aws-sdk/client-secrets-manager": "^3.826.0",
         "@faker-js/faker": "^9.8.0",
+        "@google-cloud/bigquery": "^7.0.0",
+        "@google-cloud/storage": "^7.10.0",
         "@google-cloud/vertexai": "^1.7.0",
         "@google/genai": "^1.10.0",
         "@google/generative-ai": "^0.14.1",

packages/core/src/Components/GCPBigQuery.class.ts

@@ -0,0 +1,56 @@
+import { BigQuery } from "@google-cloud/bigquery";
+import { Component } from "./Component.class";
+import { IAgent as Agent } from "@sre/types/Agent.types";
+import Joi from "joi";
+import { TemplateStringHelper } from "@sre/helpers/TemplateString.helper";
+
+export class GCPBigQuery extends Component {
+  protected configSchema = Joi.object({
+    projectId: Joi.string().max(200).required().label("GCP Project ID"),
+    keyFilename: Joi.string().max(500).allow("").label("Path to service account key JSON"),
+    datasetId: Joi.string().max(200).required(),
+    tableId: Joi.string().max(200).required(),
+    schema: Joi.string().required().label("Table schema as JSON"),
+    name: Joi.string().max(100).required(),
+    displayName: Joi.string().max(100).required(),
+    desc: Joi.string().max(5000).allow(""),
+    logoUrl: Joi.string().max(500).allow(""),
+  });
+
+  constructor() {
+    super();
+  }
+
+  init() {}
+
+  async process(input, config, agent: Agent) {
+    await super.process(input, config, agent);
+
+    const logger = this.createComponentLogger(agent, config);
+    logger.debug("=== GCP BigQuery Connector ===");
+
+    try {
+      const teamId = agent?.teamId;
+      const projectId = (await TemplateStringHelper.create(config?.data?.projectId).parseTeamKeysAsync(teamId).asyncResult) as string;
+      const keyFilename = config?.data?.keyFilename || undefined;
+      const datasetId = config?.data?.datasetId;
+      const tableId = config?.data?.tableId;
+      const schemaJson = JSON.parse(config?.data?.schema);
+
+      if (!projectId || !datasetId || !tableId || !schemaJson) {
+        return { _error: "Please provide projectId, datasetId, tableId, and schema", _debug: logger.output };
+      }
+
+      const bigquery = new BigQuery({ projectId, keyFilename });
+
+      const [table] = await bigquery.dataset(datasetId).createTable(tableId, { schema: schemaJson });
+
+      logger.debug(`Created table: ${datasetId}.${tableId}`);
+
+      return { Output: `Table ${table.id} created successfully`, _debug: logger.output };
+    } catch (error: any) {
+      logger.debug("Error: ", error?.message);
+      return { _error: error?.message || JSON.stringify(error), _debug: logger.output };
+    }
+  }
+}

packages/core/src/Components/GCPBucket.class.ts

@@ -0,0 +1,52 @@
+import { Storage } from "@google-cloud/storage";
+import { Component } from "./Component.class";
+import { IAgent as Agent } from "@sre/types/Agent.types";
+import Joi from "joi";
+import { TemplateStringHelper } from "@sre/helpers/TemplateString.helper";
+
+export class GCPBucket extends Component {
+  protected configSchema = Joi.object({
+    projectId: Joi.string().max(200).required().label("GCP Project ID"),
+    keyFilename: Joi.string().max(500).allow("").label("Path to service account key JSON"),
+    name: Joi.string().max(100).required(),
+    displayName: Joi.string().max(100).required(),
+    desc: Joi.string().max(5000).allow(""),
+    logoUrl: Joi.string().max(500).allow(""),
+  });
+
+  constructor() {
+    super();
+  }
+
+  init() {}
+
+  async process(input, config, agent: Agent) {
+    await super.process(input, config, agent);
+
+    const logger = this.createComponentLogger(agent, config);
+    logger.debug("=== GCP Bucket Connector ===");
+
+    try {
+      // Resolve project + keyfile from config (supports Smyth team key substitution)
+      const teamId = agent?.teamId;
+      const projectId = (await TemplateStringHelper.create(config?.data?.projectId).parseTeamKeysAsync(teamId).asyncResult) as string;
+      const keyFilename = config?.data?.keyFilename || undefined;
+
+      if (!projectId) {
+        return { _error: "Please provide a valid GCP Project ID", _debug: logger.output };
+      }
+
+      const storage = new Storage({ projectId, keyFilename });
+
+      const [buckets] = await storage.getBuckets();
+      const bucketNames = buckets.map(b => b.name);
+
+      logger.debug("Buckets: ", bucketNames);
+
+      return { Output: bucketNames, _debug: logger.output };
+    } catch (error: any) {
+      logger.debug("Error: ", error?.message);
+      return { _error: error?.message || JSON.stringify(error), _debug: logger.output };
+    }
+  }
+}

packages/core/src/Components/index.ts

@@ -43,6 +43,10 @@ import { MemoryReadKeyVal } from './MemoryReadKeyVal.class';
 import { MemoryDeleteKeyVal } from './MemoryDeleteKeyVal.class';
 import { MemoryWriteObject } from './MemoryWriteObject.class';
 
+import { GCPBucket } from './GCPBucket.class';
+import { GCPBigQuery } from './GCPBigQuery.class';
+
+
 const components = {
     Component: new Component(),
     Note: new Component(), //this is a fake component
@@ -92,6 +96,9 @@ const components = {
     MemoryReadKeyVal: new MemoryReadKeyVal(),
     MemoryDeleteKeyVal: new MemoryDeleteKeyVal(),
     MemoryWriteObject: new MemoryWriteObject(),
+
+    GCPBucket: new GCPBucket(),
+    GCPBigQuery: new GCPBigQuery(),
 };
 
 export const ComponentInstances = components;