Skip to content

Commit d5e2f19

Browse files
azdakazdak
committed
Backend: Remote CSV- added DB updates to sync function
1 parent 5adf8e9 commit d5e2f19

1 file changed

Lines changed: 127 additions & 72 deletions

File tree

‎apps/backend/routes/internal/syncAuditUrlsFromRemoteCsv.ts‎

Lines changed: 127 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -1,87 +1,142 @@
1-
import { db, event, graphqlQuery } from '#src/utils';
2-
import { fetchAndValidateRemoteCsv } from '#src/routes/internal/fetchAndValidateRemoteCsv';
3-
1+
import { db, event, graphqlQuery } from "#src/utils";
2+
import { fetchAndValidateRemoteCsv } from "#src/routes/internal/fetchAndValidateRemoteCsv";
43

54
//
65
// Takes an AuditId for an audit with a remote CSV,
76
// and updates the audit's URLs from remote
87
//
98

109
type newUrl = {
11-
url:string;
12-
type: string;
13-
}
10+
url: string;
11+
type: string;
12+
};
1413

1514
type DBUrl = {
16-
id: string;
17-
type: string;
18-
url: string
19-
}
20-
21-
export const syncAuditUrlsFromRemoteCsv = async (auditId:string) => {
22-
if(!auditId) throw new Error("Invalid Audit ID!");
23-
24-
// fetch audit
25-
await db.connect();
26-
const audit = (await db.query({
27-
text: `SELECT * FROM "audits" WHERE "id" = $1`,
28-
values: [auditId],
29-
})).rows?.[0];
30-
const remoteCsvUrl = audit.remote_csv_url;
31-
32-
// fetch CSV URLs
33-
const remoteCsv = await fetchAndValidateRemoteCsv(remoteCsvUrl);
34-
if(!remoteCsv.data){
35-
throw new Error("No URLs found in remote CSV.")
36-
}
37-
const remoteCsvUrls = remoteCsv.data;
38-
39-
const currentUrls = (await db.query({
40-
text: `SELECT "id", "url", "type" FROM "urls" WHERE "audit_id" = $1`,
41-
values: [auditId],
42-
})).rows as DBUrl[];
43-
await db.clean();
44-
45-
// cache the url objects for efficiency and also why not
46-
const existingKeys = new Set(
47-
currentUrls.map((item:DBUrl) => `${item.url}`)
48-
);
49-
const csvKeys = new Set(
50-
remoteCsvUrls.map((item:newUrl)=> `${item.url}|${item.type}`)
51-
);
52-
const csvKeysUrl = new Set(
53-
remoteCsvUrls.map((item:newUrl)=> `${item.url}`)
15+
id: string;
16+
type: string;
17+
url: string;
18+
};
19+
20+
export const syncAuditUrlsFromRemoteCsv = async (auditId: string) => {
21+
if (!auditId) throw new Error("Invalid Audit ID!");
22+
23+
// fetch audit
24+
await db.connect();
25+
const audit = (
26+
await db.query({
27+
text: `SELECT * FROM "audits" WHERE "id" = $1`,
28+
values: [auditId],
29+
})
30+
).rows?.[0];
31+
const remoteCsvUrl = audit.remote_csv_url;
32+
33+
// fetch CSV URLs
34+
const remoteCsv = await fetchAndValidateRemoteCsv(remoteCsvUrl);
35+
if (!remoteCsv.data) {
36+
throw new Error("No URLs found in remote CSV.");
37+
}
38+
const remoteCsvUrls = remoteCsv.data;
39+
40+
const currentUrls = (
41+
await db.query({
42+
text: `SELECT "id", "url", "type" FROM "urls" WHERE "audit_id" = $1`,
43+
values: [auditId],
44+
})
45+
).rows as DBUrl[];
46+
await db.clean();
47+
48+
// cache the url objects for efficiency and also why not
49+
const existingKeys = new Set(currentUrls.map((item: DBUrl) => `${item.url}`));
50+
const csvKeys = new Set(
51+
remoteCsvUrls.map((item: newUrl) => `${item.url}|${item.type}`),
52+
);
53+
const csvKeysUrl = new Set(
54+
remoteCsvUrls.map((item: newUrl) => `${item.url}`),
55+
);
56+
57+
// get _new_ URLs to add
58+
const urlsToAdd = remoteCsvUrls.filter((item: newUrl) => {
59+
const key = `${item.url}`;
60+
return !existingKeys.has(key);
61+
});
62+
63+
// get URLs to remove
64+
const urlsToRemove = currentUrls.filter((item: DBUrl) => {
65+
const key = `${item.url}`;
66+
return !csvKeysUrl.has(key);
67+
});
68+
69+
// get URLs to updated and generate an array with the updated values
70+
const urlsToUpdate = currentUrls.filter((item: DBUrl) => {
71+
const key = `${item.url}|${item.type}`;
72+
return csvKeysUrl.has(item.url) && !csvKeys.has(key);
73+
});
74+
urlsToUpdate.forEach((el: DBUrl, index: number, arr: DBUrl[]) => {
75+
const updatedValue = remoteCsvUrls.find(
76+
(instance) => instance.url === el.url,
5477
);
78+
if (updatedValue) arr[index].type = updatedValue.type;
79+
});
5580

56-
// get _new_ URLs to add
57-
const urlsToAdd = remoteCsvUrls.filter((item:newUrl)=>{
58-
const key = `${item.url}`;
59-
return !existingKeys.has(key);
60-
});
81+
//
82+
// Store updates in db
83+
//
6184

62-
// get URLs to remove
63-
const urlsToRemove = currentUrls.filter((item:DBUrl)=>{
64-
const key = `${item.url}`;
65-
return !csvKeysUrl.has(key);
85+
// new URLs
86+
for (const url of urlsToAdd) {
87+
await graphqlQuery({
88+
query: `mutation ($audit_id: uuid, $url: String, $type: String) {
89+
insert_urls_one(object: {audit_id: $audit_id, url: $url, type: $type}) {id}
90+
}`,
91+
variables: {
92+
audit_id: auditId,
93+
url: url.url,
94+
type: url.type,
95+
},
6696
});
97+
}
6798

68-
// get URLs to updated and generate an array with the updated values
69-
const urlsToUpdate = currentUrls.filter((item:DBUrl)=>{
70-
const key = `${item.url}|${item.type}`;
71-
return csvKeysUrl.has(item.url) && !csvKeys.has(key);
72-
});
73-
urlsToUpdate.forEach((el:DBUrl, index:number, arr:DBUrl[]) => {
74-
const updatedValue = remoteCsvUrls.find(instance => instance.url === el.url);
75-
if(updatedValue)
76-
arr[index].type = updatedValue.type;
77-
});
99+
// removed URLs
100+
for (const url of urlsToRemove) {
101+
await graphqlQuery({
102+
query: `mutation($audit_id:uuid,$url:String) {delete_urls(where: {audit_id: {_eq: $audit_id}, url: {_eq: $url}}) {affected_rows}}`,
103+
variables: {
104+
audit_id: auditId,
105+
url: url.url,
106+
},
107+
});
108+
}
109+
110+
// updated URLs
111+
for (const url of urlsToUpdate) {
112+
await graphqlQuery({
113+
query: `
114+
mutation ($audit_id: uuid, $url: String, $type: String) {
115+
update_urls(
116+
where: {
117+
audit_id: {_eq: $audit_id},
118+
_and: {url: {_eq: $url}}
119+
}, _set: {type: $type}
120+
)
121+
{
122+
returning {
123+
audit_id
124+
type
125+
updated_at
126+
url
127+
user_id
128+
}
129+
}
130+
}`,
131+
variables: {
132+
audit_id: auditId,
133+
url: url.url,
134+
type: url.type,
135+
},
136+
})
137+
}
78138

79-
// union with existing URLs
80-
return {
81-
currentUrls,
82-
remoteCsvUrls,
83-
urlsToAdd,
84-
urlsToRemove,
85-
urlsToUpdate
86-
}
87-
}
139+
return {
140+
message: `Sync complete. URLs: added ${urlsToAdd.length}, removed ${urlsToRemove.length}, updated ${urlsToUpdate.length}.`
141+
}
142+
};

0 commit comments

Comments
 (0)