Skip to content

Commit 2485448

Browse files
heythisischrisheythisischris
committed
Merge branch 'staging'
# Conflicts: # apps/backend/routes/scheduled/runEveryMinute.ts # apps/frontend/src/routes/Audit.tsx
2 parents d28f5d1 + d4f2d7c commit 2485448

31 files changed

Lines changed: 1883 additions & 510 deletions

‎.gitignore‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11
node_modules
22
dist
33
.ds_store
4-
.vscode
4+
.vscode
5+
.env
6+
.env.*

‎apps/backend/routes/auth/fetchRemoteCsv.ts‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
import { event } from "#src/utils";
2+
import { fetchAndValidateRemoteCsv } from "#src/routes/internal/fetchAndValidateRemoteCsv";
3+
4+
//
5+
// Fetches a remote CSV, checks for basic validity, and returns the parsed data or an error
6+
//
7+
8+
export const fetchRemoteCsv = async () => {
9+
const csvUrl = (event.queryStringParameters as any).url;
10+
return await fetchAndValidateRemoteCsv(csvUrl);
11+
};

‎apps/backend/routes/auth/getAuditChart.ts‎

Lines changed: 120 additions & 96 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,21 @@
1-
import { db, event, graphqlQuery } from '#src/utils';
1+
import { db, event, graphqlQuery } from "#src/utils";
22

33
export const getAuditChart = async () => {
4-
const auditId = event.queryStringParameters.id;
5-
const days = parseInt((event.queryStringParameters as any).days || '7', 10);
6-
7-
await db.connect();
8-
const audit = (await db.query({
9-
text: `SELECT * FROM "audits" WHERE "id" = $1`,
10-
values: [auditId],
11-
})).rows?.[0];
12-
await db.clean();
13-
14-
// Query to get all scans for the audit with blocker counts
15-
const query = {
16-
query: `query ($audit_id: uuid!) {
4+
const auditId = (event.queryStringParameters as any).id;
5+
const days = parseInt((event.queryStringParameters as any).days || "7", 10);
6+
7+
await db.connect();
8+
const audit = (
9+
await db.query({
10+
text: `SELECT * FROM "audits" WHERE "id" = $1`,
11+
values: [auditId],
12+
})
13+
).rows?.[0];
14+
await db.clean();
15+
16+
// Query to get all scans for the audit with blocker counts
17+
const query = {
18+
query: `query ($audit_id: uuid!) {
1719
audits_by_pk(id: $audit_id) {
1820
scans(order_by: {created_at: asc}) {
1921
id
@@ -26,92 +28,114 @@ export const getAuditChart = async () => {
2628
}
2729
}
2830
}`,
29-
variables: { audit_id: auditId },
30-
};
31-
32-
console.log(JSON.stringify({ query }));
33-
const response = await graphqlQuery(query);
34-
console.log(JSON.stringify({ response }));
31+
variables: { audit_id: auditId },
32+
};
3533

36-
const scans = response.audits_by_pk?.scans || [];
37-
38-
// Process scans to get the last scan per day
39-
const scansByDate = new Map<string, { date: string; blockers: number; timestamp: string }>();
40-
41-
scans.forEach(scan => {
42-
const scanDate = new Date(scan.created_at);
43-
const dateKey = scanDate.toISOString().split('T')[0]; // YYYY-MM-DD format
44-
const blockerCount = scan.blockers_aggregate?.aggregate?.count || 0;
45-
46-
// Only keep the last scan for each day (scans are ordered by created_at asc)
47-
scansByDate.set(dateKey, {
48-
date: dateKey,
49-
blockers: blockerCount,
50-
timestamp: scan.created_at
51-
});
34+
//console.log(JSON.stringify({ query }));
35+
const response = await graphqlQuery(query);
36+
//console.log(JSON.stringify({ response }));
37+
38+
const scans = response.audits_by_pk?.scans || [];
39+
40+
// Process scans to get the last scan per day
41+
const scansByDate = new Map<
42+
string,
43+
{ date: string; blockers: number; timestamp: string }
44+
>();
45+
46+
scans.forEach((scan:any) => {
47+
const scanDate = new Date(scan.created_at);
48+
//console.log("scan.created_at", scan.created_at);
49+
//console.log("scanDate", scanDate);
50+
const dateKey = scanDate.toISOString().split("T")[0]; // YYYY-MM-DD format
51+
//console.log("dateKey", dateKey)
52+
const blockerCount = scan.blockers_aggregate?.aggregate?.count || 0;
53+
54+
// Only keep the last scan for each day (scans are ordered by created_at asc)
55+
scansByDate.set(dateKey, {
56+
date: dateKey,
57+
blockers: blockerCount,
58+
timestamp: scan.created_at,
5259
});
60+
});
61+
62+
// Generate array of the last N days
63+
const now = new Date();
64+
now.setUTCHours(0, 0, 0, 0); // Reset to start of day in UTC
65+
const chartData = [];
66+
let lastKnownValue = 0;
5367

54-
// Generate array of the last N days
55-
const now = new Date();
56-
now.setUTCHours(0, 0, 0, 0); // Reset to start of day in UTC
57-
const chartData = [];
58-
let lastKnownValue = 0;
68+
// get the blockers value of the most recent scan
69+
let mostRecentScan = null;
70+
let oldestScan = "";
71+
let mostRecentBlockersCount = 0;
72+
if (scansByDate.size > 0) { // testing we have scans to avoid error when no scans are present
73+
74+
mostRecentScan = Array.from(scansByDate.values()).sort((a, b) =>
75+
b.timestamp.localeCompare(a.timestamp),
76+
)[0];
5977

60-
// get the blockers value of the most recent scan
61-
const mostRecentScan = Array.from(scansByDate.values()).sort((a, b) => b.timestamp.localeCompare(a.timestamp))[0];
62-
const mostRecentBlockersCount = mostRecentScan ? mostRecentScan.blockers : 0;
63-
const oldestScan = Array.from(scansByDate.values()).sort((a, b) => a.timestamp.localeCompare(b.timestamp))[0].timestamp ?? "";
64-
65-
lastKnownValue = mostRecentBlockersCount;
66-
67-
for (let i = days - 1; i >= 0; i--) {
68-
const date = new Date(now);
69-
date.setUTCDate(date.getUTCDate() - i);
70-
const dateKey = date.toISOString().split('T')[0];
71-
72-
if (scansByDate.has(dateKey)) {
73-
// Use the actual scan data for this day
74-
const scanData = scansByDate.get(dateKey)!;
75-
lastKnownValue = scanData.blockers;
76-
chartData.push({
77-
date: dateKey,
78-
blockers: scanData.blockers,
79-
timestamp: scanData.timestamp
80-
});
81-
} else {
82-
// Fill with the last known value
83-
if(oldestScan === "" || date.toISOString() < oldestScan){
84-
lastKnownValue = 0; // if the date we're showing is before the first scan, set to zero
85-
}else{
86-
lastKnownValue = mostRecentBlockersCount; // otherwise use the blockers value from the latest scan
87-
}
88-
89-
if(days>30){ // If the range is >30 days, only return weekly points
90-
if(i % 7 === 0){
91-
chartData.push({
92-
date: dateKey,
93-
blockers: lastKnownValue,
94-
timestamp: null
95-
});
96-
}
97-
}else{
98-
chartData.push({
99-
date: dateKey,
100-
blockers: lastKnownValue,
101-
timestamp: null
102-
});
103-
}
78+
oldestScan =
79+
Array.from(scansByDate.values()).sort((a, b) =>
80+
a.timestamp.localeCompare(b.timestamp),
81+
)[0].timestamp ?? "";
82+
83+
mostRecentBlockersCount = mostRecentScan.blockers;
84+
}
85+
86+
87+
lastKnownValue = mostRecentBlockersCount;
88+
89+
for (let i = days - 1; i >= 0; i--) {
90+
const date = new Date(now);
91+
92+
date.setUTCDate(date.getUTCDate() - i);
93+
const dateKey = date.toISOString().split("T")[0];
94+
95+
if (scansByDate.has(dateKey)) {
96+
// Use the actual scan data for this day
97+
const scanData = scansByDate.get(dateKey)!;
98+
lastKnownValue = scanData.blockers;
99+
chartData.push({
100+
date: dateKey,
101+
blockers: scanData.blockers,
102+
timestamp: scanData.timestamp,
103+
});
104+
} else {
105+
// Fill with the last known value
106+
if (oldestScan === "" || date.toISOString() < oldestScan) {
107+
lastKnownValue = 0; // if the date we're showing is before the first scan, set to zero
108+
} else {
109+
lastKnownValue = mostRecentBlockersCount; // otherwise use the blockers value from the latest scan
110+
}
111+
112+
if (days > 30) {
113+
// If the range is >30 days, only return weekly points
114+
if (i % 7 === 0) {
115+
chartData.push({
116+
date: dateKey,
117+
blockers: lastKnownValue,
118+
timestamp: null,
119+
});
104120
}
121+
} else {
122+
chartData.push({
123+
date: dateKey,
124+
blockers: lastKnownValue,
125+
timestamp: null,
126+
});
127+
}
105128
}
129+
}
106130

107-
return {
108-
statusCode: 200,
109-
headers: { 'content-type': 'application/json' },
110-
body: {
111-
audit_id: auditId,
112-
audit_name: audit?.name,
113-
period_days: days,
114-
data: chartData,
115-
},
116-
};
117-
}
131+
return {
132+
statusCode: 200,
133+
headers: { "content-type": "application/json" },
134+
body: {
135+
audit_id: auditId,
136+
audit_name: audit?.name,
137+
period_days: days,
138+
data: chartData,
139+
},
140+
};
141+
};

‎apps/backend/routes/auth/getAuditTable.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -291,7 +291,7 @@ export const getAuditTable = async () => {
291291
id: blocker.id,
292292
short_id: blocker.short_id,
293293
created_at: blocker.created_at,
294-
url: blocker.url?.url || blocker.url_id,
294+
url: blocker.url?.url || "Unknown URL",
295295
type: blocker.url?.type || "unknown",
296296
url_id: blocker.url_id,
297297
content: blocker.content,

‎apps/backend/routes/auth/index.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,4 +16,5 @@ export * from './inviteUser'
1616
export * from './getAuditSummary'
1717
export * from './saveQuickScan'
1818
export * from './getQuickScans'
19-
export * from './fetchAndValidateRemoteCsv'
19+
export * from './fetchRemoteCsv'
20+
export * from './syncFromRemoteCsv'

‎apps/backend/routes/auth/rescanAudit.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import { db, event, isStaging } from '#src/utils';
22
import { LambdaClient, InvokeCommand } from '@aws-sdk/client-lambda';
3+
import { syncAuditUrlsFromRemoteCsv } from '../internal';
34
const lambda = new LambdaClient();
45

56
export const rescanAudit = async () => {
@@ -12,6 +13,9 @@ export const rescanAudit = async () => {
1213
})).rows;
1314
console.log('Found URLs for audit:', { auditId: audit_id, count: urls?.length, urls });
1415

16+
// hook to check for remote CSV
17+
await syncAuditUrlsFromRemoteCsv(audit_id);
18+
1519
// Handle empty URLs case - create a complete scan immediately
1620
if (!urls || urls.length === 0) {
1721
console.log('No URLs found for audit, creating completed scan with no_urls error');

‎apps/backend/routes/auth/saveAudit.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,13 @@ const lambda = new LambdaClient();
44

55
export const saveAudit = async () => {
66
try {
7-
const { auditName, scanFrequency, pages, saveAndRun, emailNotifications } = event.body;
7+
const { auditName, scanFrequency, pages, saveAndRun, emailNotifications, remoteCsvUrl } = event.body;
88
const scheduledAt = new Date();
99
await db.connect();
1010
const id = (await db.query({
11-
text: `INSERT INTO "audits" ("user_id", "name", "interval", "scheduled_at", "status", "payload", "email_notifications")
12-
VALUES ($1, $2, $3, $4, $5, $6, $7) RETURNING "id"`,
13-
values: [event.claims.sub, auditName, scanFrequency, scheduledAt, saveAndRun ? 'new' : 'draft', JSON.stringify(event.body), emailNotifications],
11+
text: `INSERT INTO "audits" ("user_id", "name", "interval", "scheduled_at", "status", "payload", "email_notifications", "remote_csv_url")
12+
VALUES ($1, $2, $3, $4, $5, $6, $7, $8) RETURNING "id"`,
13+
values: [event.claims.sub, auditName, scanFrequency, scheduledAt, saveAndRun ? 'new' : 'draft', JSON.stringify(event.body), emailNotifications, remoteCsvUrl],
1414
})).rows[0].id;
1515

1616
// Insert all URLs in a single query using UNNEST for clarity

‎apps/backend/routes/auth/syncFromRemoteCsv.ts‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
import { db, event, graphqlQuery } from '#src/utils';
2+
import { syncAuditUrlsFromRemoteCsv } from '../internal/syncAuditUrlsFromRemoteCsv';
3+
4+
5+
//
6+
// Takes an AuditId for an audit with a remote CSV,
7+
// and updates the audit's URLs from remote
8+
//
9+
10+
export const syncFromRemoteCsv = async () => {
11+
const auditId = (event.queryStringParameters as any).id;
12+
return await syncAuditUrlsFromRemoteCsv(auditId);
13+
}

apps/backend/routes/auth/fetchAndValidateRemoteCsv.ts renamed to apps/backend/routes/internal/fetchAndValidateRemoteCsv.ts

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,3 @@
1-
import { db, event, graphqlQuery, validateShortId } from "#src/utils";
2-
31
//
42
// Fetches a remote CSV, checks for basic validity, and returns the parsed data or an error
53
//
@@ -9,8 +7,7 @@ interface urlCsv {
97
type: string;
108
}
119

12-
export const fetchAndValidateRemoteCsv = async () => {
13-
const csvUrl = (event.queryStringParameters as any).url;
10+
export const fetchAndValidateRemoteCsv = async (csvUrl:string) => {
1411
try {
1512
if(!csvUrl) throw new Error(`Invalid CSV URL: ${csvUrl}`)
1613
const response = await fetch(csvUrl);

‎apps/backend/routes/internal/index.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,4 @@
11
export * from './processScheduledAuditEmails'
2-
export * from './manuallyAddTags'
2+
export * from './manuallyAddTags'
3+
export * from './fetchAndValidateRemoteCsv'
4+
export * from './syncAuditUrlsFromRemoteCsv'

0 commit comments

Comments
 (0)