fix(scan): insert remote CSV URLs via SQL with explicit user_id

Author: heythisischris

apps/backend/routes/internal/syncAuditUrlsFromRemoteCsv.ts

@@ -30,9 +30,9 @@ export const syncAuditUrlsFromRemoteCsv = async (auditId: string) => {
         values: [auditId],
       })
     ).rows?.[0];
-    const remoteCsvUrl = audit.remote_csv_url;
+    const remoteCsvUrl = audit?.remote_csv_url;
 
-    if (!audit.remote_csv_url.trim()) {
+    if (!audit?.remote_csv_url?.trim()) {
       // doesn't use remote csv, skip
       console.log(`Audit ${auditId} doesn't use remote CSV, skipping sync`);
       return;
@@ -57,7 +57,6 @@ export const syncAuditUrlsFromRemoteCsv = async (auditId: string) => {
         values: [auditId],
       })
     ).rows as DBUrl[];
-    await db.clean();
 
     // cache the url objects for efficiency and also why not
     const existingKeys = new Set(
@@ -98,17 +97,20 @@ export const syncAuditUrlsFromRemoteCsv = async (auditId: string) => {
     // Store updates in db
     //
 
-    // new URLs
-    for (const url of urlsToAdd) {
-      await graphqlQuery({
-        query: `mutation ($audit_id: uuid, $url: String, $type: String) {
-                insert_urls_one(object: {audit_id: $audit_id, url: $url, type: $type}) {id}
-            }`,
-        variables: {
-          audit_id: auditId,
-          url: url.url,
-          type: url.type,
-        },
+    // new URLs — inserted via SQL with an explicit user_id. The previous GraphQL
+    // mutation relied on Hasura's user-role column preset to fill user_id, which only
+    // exists when a user JWT is forwarded; from the scheduled path (no HTTP headers)
+    // it ran as admin and every insert failed the urls.user_id NOT NULL constraint.
+    if (urlsToAdd.length > 0) {
+      await db.query({
+        text: `INSERT INTO "urls" ("user_id", "audit_id", "url", "type")
+               SELECT * FROM UNNEST($1::uuid[], $2::uuid[], $3::text[], $4::text[])`,
+        values: [
+          urlsToAdd.map(() => audit.user_id),
+          urlsToAdd.map(() => auditId),
+          urlsToAdd.map((url: newUrl) => url.url),
+          urlsToAdd.map((url: newUrl) => url.type),
+        ],
       });
     }
 

apps/backend/routes/scheduled/runEveryMinute.ts

@@ -50,48 +50,53 @@ export const runEveryMinute = async () => {
     })
   ).rows.map((obj: { id: string; }) => obj.id);
   for (const scheduledAuditId of scheduledAuditIds) {
-    // hook to check for remote CSV
-    await syncAuditUrlsFromRemoteCsv(scheduledAuditId);
+    // isolate each audit so one failure can't starve the rest of the batch
+    try {
+      // hook to check for remote CSV
+      await syncAuditUrlsFromRemoteCsv(scheduledAuditId);
 
-    const urls = (
-      await db.query({
-        text: `SELECT * FROM "urls" WHERE "audit_id"=$1`,
-        values: [scheduledAuditId],
-      })
-    ).rows;
+      const urls = (
+        await db.query({
+          text: `SELECT * FROM "urls" WHERE "audit_id"=$1`,
+          values: [scheduledAuditId],
+        })
+      ).rows;
 
-    // Skip scheduled audits with no URLs to prevent hung scans
-    if (!urls || urls.length === 0) {
-      console.log("Skipping scheduled audit with no URLs:", scheduledAuditId);
-      continue;
-    }
+      // Skip scheduled audits with no URLs to prevent hung scans
+      if (!urls || urls.length === 0) {
+        console.log("Skipping scheduled audit with no URLs:", scheduledAuditId);
+        continue;
+      }
 
-    const scanId = (
-      await db.query({
-        text: `INSERT INTO "scans" ("audit_id", "status", "pages") VALUES ($1, $2, $3) RETURNING "id"`,
-        values: [
-          scheduledAuditId,
-          "processing",
-          JSON.stringify(urls.map((obj: { url: string; type: string; }) => ({ url: obj.url, type: obj.type }))),
-        ],
-      })
-    ).rows[0].id;
-    await lambda.send(
-      new InvokeCommand({
-        FunctionName: process.env.SQS_ROUTER_FUNCTION_NAME ?? "aws-lambda-scan-sqs-router",
-        InvocationType: "Event",
-        Payload: JSON.stringify({
-          urls: urls?.map((url: { id: string; url: string; type: string; }) => ({
-            auditId: scheduledAuditId,
-            scanId: scanId,
-            urlId: url.id,
-            url: url.url,
-            type: url.type,
-          })),
+      const scanId = (
+        await db.query({
+          text: `INSERT INTO "scans" ("audit_id", "status", "pages") VALUES ($1, $2, $3) RETURNING "id"`,
+          values: [
+            scheduledAuditId,
+            "processing",
+            JSON.stringify(urls.map((obj: { url: string; type: string; }) => ({ url: obj.url, type: obj.type }))),
+          ],
+        })
+      ).rows[0].id;
+      await lambda.send(
+        new InvokeCommand({
+          FunctionName: process.env.SQS_ROUTER_FUNCTION_NAME ?? "aws-lambda-scan-sqs-router",
+          InvocationType: "Event",
+          Payload: JSON.stringify({
+            urls: urls?.map((url: { id: string; url: string; type: string; }) => ({
+              auditId: scheduledAuditId,
+              scanId: scanId,
+              urlId: url.id,
+              url: url.url,
+              type: url.type,
+            })),
+          }),
         }),
-      }),
-    );
-    console.log("Scan jobs queued for audit:", scheduledAuditId);
+      );
+      console.log("Scan jobs queued for audit:", scheduledAuditId);
+    } catch (error) {
+      console.error("Scheduled scan failed for audit:", scheduledAuditId, error);
+    }
   }
 
   // See if there are any "stuck" scans that we should error out!