feat: hourly pipeline

Author: turadg

gcp-graphql-pipeline/.gcloudignore

@@ -0,0 +1,16 @@
+# This file specifies files that are *not* uploaded to Google Cloud
+# using gcloud. It follows the same syntax as .gitignore, with the addition of
+# "#!include" directives (which insert the entries of the given .gitignore-style
+# file at that point).
+#
+# For more information, run:
+#   $ gcloud topic gcloudignore
+#
+.gcloudignore
+# If you would like to upload your .git directory, .gitignore file or files
+# from your .gitignore file, remove the corresponding line
+# below:
+.git
+.gitignore
+
+node_modules

gcp-graphql-pipeline/.gitignore

@@ -0,0 +1,2 @@
+service-account.json
+

gcp-graphql-pipeline/README.md

@@ -0,0 +1,37 @@
+# GCP GraphQL Pipeline
+
+## Prerequisites
+
+Get a `service-account.json` from https://console.cloud.google.com/iam-admin/serviceaccounts/details/111331970552820213056/keys?project=simulationlab
+
+## Development
+
+```sh
+export GOOGLE_APPLICATION_CREDENTIALS=service-account.json
+./test.js
+```
+
+## Deployment
+
+```sh
+gcloud functions deploy fetchAndStoreTransactions \
+    --runtime nodejs22 \
+    --trigger-http \
+    --allow-unauthenticated \
+    --set-env-vars GOOGLE_APPLICATION_CREDENTIALS=service-account.json
+```
+
+That will output a URL you can invoke like this,
+
+```
+curl -X POST https://us-central1-simulationlab.cloudfunctions.net/fetchAndStoreTransactions
+```
+
+That runs every hour right now because of:
+
+```sh
+gcloud scheduler jobs create http graphql-job \
+    --schedule "every 1 hours" \
+    --uri https://us-central1-simulationlab.cloudfunctions.net/fetchAndStoreTransactions \
+    --http-method POST
+```

gcp-graphql-pipeline/index.js

@@ -0,0 +1,110 @@
+const { google } = require('googleapis');
+
+const SPREADSHEET_ID = '15CZGF-GyqfimwZgrkFTIAkp8xEfHlLQhkp83flXqI84';
+const SHEET_NAME = 'live!A2'; // First row is for the existing header
+const GRAPHQL_ENDPOINT = 'https://api.subquery.network/sq/agoric-labs/internal';
+
+async function appendToGoogleSheet(data) {
+  const auth = new google.auth.GoogleAuth({
+    keyFile: 'service-account.json', // Update with your service account key file
+    scopes: ['https://www.googleapis.com/auth/spreadsheets'],
+  });
+
+  const sheets = google.sheets({ version: 'v4', auth });
+
+  console.log(`Appending ${data.length} rows to Google Sheets...`);
+
+  try {
+    await sheets.spreadsheets.values.update({
+      spreadsheetId: SPREADSHEET_ID,
+      range: SHEET_NAME,
+      valueInputOption: 'USER_ENTERED', // https://developers.google.com/sheets/api/reference/rest/v4/ValueInputOption
+      requestBody: { values: data },
+    });
+
+    console.log('Data successfully pushed to Google Sheets');
+  } catch (error) {
+    console.error('Error appending data to Google Sheets:', error.response?.data || error.message);
+    throw error;
+  }
+}
+
+exports.fetchAndStoreTransactions = async (req, res) => {
+  try {
+    console.log('Fetching transactions from GraphQL API...');
+
+    const query = `
+        query TransactionsQuery {
+            fastUsdcTransactions(orderBy: SOURCE_BLOCK_TIMESTAMP_DESC) {
+              edges {
+                    node {
+                      id
+                      sourceAddress
+                      sourceChainId
+                      sourceBlockTimestamp
+                      eud
+                      usdcAmount
+                      status
+                      statusHeight
+                      contractFee
+                      poolFee
+                      risksIdentified
+                      heightObserved
+                      heightAdvanced
+                      heightDisbursed
+                      timeObserved
+                      timeAdvanced
+                      timeDisbursed
+                    }
+                }
+            }
+        }`;
+
+    const response = await fetch(GRAPHQL_ENDPOINT, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ query }),
+    });
+
+    if (!response.ok) {
+      throw new Error(`GraphQL request failed: ${response.statusText}`);
+    }
+
+    const { data } = await response.json();
+    if (!data) throw new Error('No data received from GraphQL API');
+
+    console.log('Raw GraphQL response:', JSON.stringify(data, null, 2));
+
+    const transactions = data.fastUsdcTransactions.edges.map((edge) => edge.node);
+    console.log(`Fetched ${transactions.length} transactions`);
+
+    const rows = transactions.map((txn) => {
+      return [
+        txn.id,
+        txn.sourceAddress,
+        txn.sourceChainId,
+        txn.sourceBlockTimestamp,
+        txn.eud,
+        txn.usdcAmount,
+        txn.status,
+        txn.statusHeight,
+        txn.contractFee,
+        txn.poolFee,
+        Array.isArray(txn.risksIdentified) ? txn.risksIdentified.join(', ') : txn.risksIdentified, // Convert array to string
+        txn.heightObserved,
+        txn.heightAdvanced,
+        txn.heightDisbursed,
+        txn.timeObserved,
+        txn.timeAdvanced,
+        txn.timeDisbursed,
+      ];
+    });
+
+    await appendToGoogleSheet(rows);
+
+    res.status(200).send('Transactions stored successfully!');
+  } catch (error) {
+    console.error('Error fetching or storing transactions:', error.response?.data || error.message);
+    res.status(500).send('Error fetching or storing transactions');
+  }
+};