feat(sdk): move request fetching to sdk/src/dsr-automation/

.changeset/move-requests-to-sdk.md

@@ -0,0 +1,9 @@
+---
+'@transcend-io/sdk': minor
+'@transcend-io/cli': minor
+---
+
+Move request fetch functions from CLI to SDK
+
+- Add `dsr-automation/` module to SDK: `fetchAllRequests`, `fetchRequestsTotalCount`, `fetchRequestDataSiloActiveCount`
+- All imports updated to use `@transcend-io/sdk` directly

packages/cli/src/commands/consent/delete-preference-records/impl.ts

@@ -3,12 +3,11 @@ import { join } from 'node:path';
 
 import { createSombraGotInstance } from '@transcend-io/sdk';
 import { map } from '@transcend-io/utils';
-import cliProgress from 'cli-progress';
 import colors from 'colors';
 
 import type { LocalContext } from '../../../context.js';
 import { doneInputValidation } from '../../../lib/cli/done-input-validation.js';
-import { writeCsv } from '../../../lib/helpers/index.js';
+import { withProgressBar, writeCsv } from '../../../lib/helpers/index.js';
 import { bulkDeletePreferenceRecords } from '../../../lib/preference-management/index.js';
 import { logger } from '../../../logger.js';
 
@@ -117,33 +116,25 @@ export async function deletePreferenceRecords(
     sombraApiKey: sombraAuth,
     sombraUrl: process.env.SOMBRA_URL,
   });
-  const globalProgressBar = new cliProgress.SingleBar(
-    {
-      format: `Deletion Progress |${colors.cyan('[{bar}]')}| Duration: ${colors.red(
-        '{duration_formatted}',
-      )} | {value}/{total} Files Processed `,
-    },
-    cliProgress.Presets.shades_classic,
-  );
-  globalProgressBar.start(files.length, 0);
-
   // Process batch of files with concurrency
-  const failedResultsArrays = await map(
-    files,
-    async (filePath) => {
-      const result = await bulkDeletePreferenceRecords(sombra, {
-        partition,
-        filePath,
-        timestamp,
-        maxItemsInChunk,
-        maxConcurrency,
-      });
-      globalProgressBar.increment();
-      return result;
-    },
-    { concurrency: fileConcurrency },
-  );
-  globalProgressBar.stop();
+  const failedResultsArrays = await withProgressBar(async (bar) => {
+    bar.start(files.length);
+    return map(
+      files,
+      async (filePath) => {
+        const result = await bulkDeletePreferenceRecords(sombra, {
+          partition,
+          filePath,
+          timestamp,
+          maxItemsInChunk,
+          maxConcurrency,
+        });
+        bar.increment();
+        return result;
+      },
+      { concurrency: fileConcurrency },
+    );
+  });
   const failedResults = failedResultsArrays.flat();
 
   // Check for failed results and write receipt if any

packages/cli/src/commands/consent/delete-preference-records/tests/impl.test.ts

@@ -65,6 +65,7 @@ vi.mock('@transcend-io/sdk', () => ({
 // New CSV helpers used by impl after your refactor
 vi.mock('../../../../lib/helpers/index.js', () => ({
   writeCsv: H.writeCsv,
+  withProgressBar: (fn: any) => fn({ start: vi.fn(), update: vi.fn(), increment: vi.fn() }),
 }));
 
 // preference-management: forward and record args, then delegate to our spies

packages/cli/src/commands/consent/generate-access-tokens/impl.ts

@@ -6,13 +6,12 @@ import {
   createPreferenceAccessTokens,
   type PreferenceAccessTokenInputWithIndex,
 } from '@transcend-io/sdk';
-import cliProgress from 'cli-progress';
 import colors from 'colors';
 import * as t from 'io-ts';
 
 import type { LocalContext } from '../../../context.js';
 import { doneInputValidation } from '../../../lib/cli/done-input-validation.js';
-import { writeCsv } from '../../../lib/helpers/index.js';
+import { withProgressBar, writeCsv } from '../../../lib/helpers/index.js';
 import { readCsv } from '../../../lib/requests/index.js';
 import { logger } from '../../../logger.js';
 
@@ -121,21 +120,20 @@ export async function generateAccessTokens(
       };
     });
 
-    // Progress bar
-    const progressBar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);
-    progressBar.start(inputs.length, 0);
-
     // Kick off token creation (batched internally)
     const t0 = Date.now();
-    const results = await createPreferenceAccessTokens(client, {
-      records: inputs,
-      logger,
-      emitProgress: (progress) => {
-        progressBar.update(progress);
-      },
+    const results = await withProgressBar(async (bar) => {
+      bar.start(inputs.length);
+      const results = await createPreferenceAccessTokens(client, {
+        records: inputs,
+        logger,
+        emitProgress: (progress) => {
+          bar.update(progress);
+        },
+      });
+      bar.update(inputs.length);
+      return results;
     });
-    progressBar.update(inputs.length);
-    progressBar.stop();
 
     // Prepare output CSV rows
     const outputRows = results.map(({ accessToken, input }) => {

packages/cli/src/lib/consent-manager/uploadConsents.ts

@@ -2,12 +2,12 @@ import { ConsentPreferencesBody } from '@transcend-io/airgap.js-types';
 import { createTranscendConsentGotInstance } from '@transcend-io/sdk';
 import { decodeCodec } from '@transcend-io/type-utils';
 import { map } from '@transcend-io/utils';
-import cliProgress from 'cli-progress';
 import colors from 'colors';
 import * as t from 'io-ts';
 
 import { DEFAULT_TRANSCEND_CONSENT_API } from '../../constants.js';
 import { logger } from '../../logger.js';
+import { withProgressBar } from '../helpers/index.js';
 import { createConsentToken } from './createConsentToken.js';
 import type { ConsentPreferenceUpload } from './types.js';
 
@@ -90,62 +90,60 @@ export async function uploadConsents({
 
   // Time duration
   const t0 = new Date().getTime();
-  // create a new progress bar instance and use shades_classic theme
-  const progressBar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);
-
-  // Build a GraphQL client
-  let total = 0;
-  progressBar.start(preferences.length, 0);
-  await map(
-    preferences,
-    async ({ userId, confirmed = 'true', updated, prompted, purposes, ...consent }) => {
-      const token = createConsentToken(userId, base64EncryptionKey, base64SigningKey);
-
-      // parse usp string
-      const [, saleStatus] = consent.usp ? USP_STRING_REGEX.exec(consent.usp) || [] : [];
-
-      const input = {
-        token,
-        partition,
-        consent: {
-          confirmed: confirmed === 'true',
-          purposes: purposes
-            ? decodeCodec(PurposeMap, purposes)
-            : consent.usp
-              ? { SaleOfInfo: saleStatus === 'Y' }
-              : {},
-          ...(updated ? { updated: updated === 'true' } : {}),
-          ...(prompted ? { prompted: prompted === 'true' } : {}),
-          ...consent,
-        },
-      } as ConsentPreferencesBody;
-
-      // Make the request
-      try {
-        await transcendConsentApi
-          .post('sync', {
-            json: input,
-          })
-          .json();
-      } catch (err) {
+
+  await withProgressBar(async (bar) => {
+    let total = 0;
+    bar.start(preferences.length);
+    await map(
+      preferences,
+      async ({ userId, confirmed = 'true', updated, prompted, purposes, ...consent }) => {
+        const token = createConsentToken(userId, base64EncryptionKey, base64SigningKey);
+
+        // parse usp string
+        const [, saleStatus] = consent.usp ? USP_STRING_REGEX.exec(consent.usp) || [] : [];
+
+        const input = {
+          token,
+          partition,
+          consent: {
+            confirmed: confirmed === 'true',
+            purposes: purposes
+              ? decodeCodec(PurposeMap, purposes)
+              : consent.usp
+                ? { SaleOfInfo: saleStatus === 'Y' }
+                : {},
+            ...(updated ? { updated: updated === 'true' } : {}),
+            ...(prompted ? { prompted: prompted === 'true' } : {}),
+            ...consent,
+          },
+        } as ConsentPreferencesBody;
+
+        // Make the request
         try {
-          const parsed = JSON.parse(err?.response?.body || '{}');
-          if (parsed.error) {
-            logger.error(colors.red(`Error: ${parsed.error}`));
+          await transcendConsentApi
+            .post('sync', {
+              json: input,
+            })
+            .json();
+        } catch (err) {
+          try {
+            const parsed = JSON.parse(err?.response?.body || '{}');
+            if (parsed.error) {
+              logger.error(colors.red(`Error: ${parsed.error}`));
+            }
+          } catch {
+            // continue
           }
-        } catch {
-          // continue
+          throw new Error(`Received an error from server: ${err?.response?.body || err?.message}`);
         }
-        throw new Error(`Received an error from server: ${err?.response?.body || err?.message}`);
-      }
 
-      total += 1;
-      progressBar.update(total);
-    },
-    { concurrency },
-  );
+        total += 1;
+        bar.update(total);
+      },
+      { concurrency },
+    );
+  });
 
-  progressBar.stop();
   const t1 = new Date().getTime();
   const totalTime = t1 - t0;
 

packages/cli/src/lib/cron/markRequestDataSiloIdsCompleted.ts

@@ -1,12 +1,12 @@
 import { RequestDataSiloStatus } from '@transcend-io/privacy-types';
 import { buildTranscendGraphQLClient, makeGraphQLRequest } from '@transcend-io/sdk';
 import { map } from '@transcend-io/utils';
-import cliProgress from 'cli-progress';
 import colors from 'colors';
 
 import { DEFAULT_TRANSCEND_API } from '../../constants.js';
 import { logger } from '../../logger.js';
 import { CHANGE_REQUEST_DATA_SILO_STATUS, fetchRequestDataSilo } from '../graphql/index.js';
+import { withProgressBar } from '../helpers/index.js';
 
 /**
  * Given a CSV of Request IDs, mark associated RequestDataSilos as completed
@@ -40,8 +40,6 @@ export async function markRequestDataSiloIdsCompleted({
 
   // Time duration
   const t0 = new Date().getTime();
-  // create a new progress bar instance and use shades_classic theme
-  const progressBar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);
 
   // Notify Transcend
   logger.info(
@@ -50,43 +48,44 @@ export async function markRequestDataSiloIdsCompleted({
     ),
   );
 
-  let total = 0;
-  progressBar.start(requestIds.length, 0);
-  await map(
-    requestIds,
-    async (requestId) => {
-      const requestDataSilo = await fetchRequestDataSilo(client, {
-        requestId,
-        dataSiloId,
-      });
-
-      try {
-        await makeGraphQLRequest<{
-          /** Whether we successfully uploaded the results */
-          success: boolean;
-        }>(client, CHANGE_REQUEST_DATA_SILO_STATUS, {
-          variables: {
-            requestDataSiloId: requestDataSilo.id,
-            status,
-          },
-          logger,
+  await withProgressBar(async (bar) => {
+    let total = 0;
+    bar.start(requestIds.length);
+    await map(
+      requestIds,
+      async (requestId) => {
+        const requestDataSilo = await fetchRequestDataSilo(client, {
+          requestId,
+          dataSiloId,
         });
-      } catch (err) {
-        if (
-          !err.message.includes('Client error: Request must be active:') &&
-          !err.message.includes('Failed to find RequestDataSilo')
-        ) {
-          throw err;
+
+        try {
+          await makeGraphQLRequest<{
+            /** Whether we successfully uploaded the results */
+            success: boolean;
+          }>(client, CHANGE_REQUEST_DATA_SILO_STATUS, {
+            variables: {
+              requestDataSiloId: requestDataSilo.id,
+              status,
+            },
+            logger,
+          });
+        } catch (err) {
+          if (
+            !err.message.includes('Client error: Request must be active:') &&
+            !err.message.includes('Failed to find RequestDataSilo')
+          ) {
+            throw err;
+          }
         }
-      }
 
-      total += 1;
-      progressBar.update(total);
-    },
-    { concurrency },
-  );
+        total += 1;
+        bar.update(total);
+      },
+      { concurrency },
+    );
+  });
 
-  progressBar.stop();
   const t1 = new Date().getTime();
   const totalTime = t1 - t0;