From 9f353d87789d6d04c6ace105b724c5e3e3a960c6 Mon Sep 17 00:00:00 2001 From: Gibson Tang Date: Sat, 24 Aug 2024 19:01:47 +0800 Subject: [PATCH 1/5] Add URL validation function to userHandlers and utils --- .gitignore | 3 +- .../definitions/eventHandlers/userHandlers.ts | 10 ++++ functions/src/utils/utils.ts | 59 +++++++++++++++++++ 3 files changed, 71 insertions(+), 1 deletion(-) create mode 100644 functions/src/utils/utils.ts diff --git a/.gitignore b/.gitignore index aa897acc..ed3b2344 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ yarn-debug.log* yarn-error.log* firebase-debug.log* firebase-debug.*.log* +firebase-export* # Firebase cache .firebase/ @@ -77,4 +78,4 @@ testing-data/** # Others .idea .DS_Store -.vscode \ No newline at end of file +.vscode diff --git a/functions/src/definitions/eventHandlers/userHandlers.ts b/functions/src/definitions/eventHandlers/userHandlers.ts index 3b8ecbe7..f900610a 100644 --- a/functions/src/definitions/eventHandlers/userHandlers.ts +++ b/functions/src/definitions/eventHandlers/userHandlers.ts @@ -1,5 +1,6 @@ import * as admin from "firebase-admin" import * as functions from "firebase-functions" +import { validateURLS } from '../../utils/utils'; import { onMessagePublished } from "firebase-functions/v2/pubsub" import { Timestamp } from "firebase-admin/firestore" import { @@ -100,12 +101,14 @@ const userHandlerWhatsapp = async function (message: WhatsappMessageObject) { return } + console.log(`Message is of type "${type}"`) switch (type) { case "text": // info on WhatsApp text message payload: https://developers.facebook.com/docs/whatsapp/cloud-api/webhooks/payload-examples#text-messages if (!message.text || !message.text.body) { break } + console.log(`Text message is "${message.text.body}"`) const textNormalised = normalizeSpaces(message.text.body).toLowerCase() //normalise spaces needed cos of potential   when copying message on desktop whatsapp if ( checkTemplate( @@ -116,6 +119,7 @@ const userHandlerWhatsapp = async function (message: WhatsappMessageObject) { textNormalised, responses?.REFERRAL_PREPOPULATED_PREFIX_1.toLowerCase() ) + ) { step = "text_prepopulated" if (isFirstTimeUser) { @@ -123,11 +127,13 @@ const userHandlerWhatsapp = async function (message: WhatsappMessageObject) { } else { await sendMenuMessage(from, "MENU_PREFIX", "whatsapp", null, null) } + console.log(`step ${step}`) break } if (checkMenu(message.text.body)) { step = "text_menu" await sendMenuMessage(from, "MENU_PREFIX", "whatsapp", null, null) + console.log(`step ${step}`) break } step = await newTextInstanceHandler({ @@ -301,6 +307,10 @@ async function newTextInstanceHandler({ rationalisation = await rationaliseMessage(text, machineCategory) } messageRef = db.collection("messages").doc() + + const validatedURLS: any = validateURLS(text) + console.log(`Validated URLs are ${validatedURLS}`) + messageUpdateObj = { machineCategory: machineCategory, //Can be "fake news" or "scam" isMachineCategorised: isMachineAssessed, diff --git a/functions/src/utils/utils.ts b/functions/src/utils/utils.ts new file mode 100644 index 00000000..8ed49525 --- /dev/null +++ b/functions/src/utils/utils.ts @@ -0,0 +1,59 @@ +const axios = require('axios'); +const { URLSearchParams } = require('url'); + +interface URLValidationResult { + url: string; + success: boolean; + data: any; + error?: any; +} + +export async function validateURLS(text: string): Promise { + const urlRegex = /https?:\/\/[^\s/$.?#].[^\s]*/g; + const urls = text.match(urlRegex); + const results: URLValidationResult[] = []; + + if (urls) { + urls.forEach((url, index) => { + console.log(`URL ${index + 1} is: "${url}"`); + + const base64URL: string = Buffer.from(url).toString('base64'); + const virusTotalURL: string = `https://www.virustotal.com/api/v3/urls/${base64URL}` + console.log(`Calling API ${virusTotalURL} to get scan results of ${url}`) + const VIRUS_TOTAL_API_KEY = String(process.env.VIRUS_TOTAL_API_KEY) + console.log(`VIRUS_TOTAL_API_KEY: ${VIRUS_TOTAL_API_KEY}`) + const options = { + method: 'GET', + url: virusTotalURL, + headers: { + accept: 'application/json', + 'x-apikey': VIRUS_TOTAL_API_KEY + } + }; + + axios + .request(options) + .then(function (response: { data: any; }) { + console.log(`Success calling ${virusTotalURL}`) + console.log(response.data); + results.push({ + url, + success: true, + data: response.data, + error: null, + }); + }) + .catch(function (error: any) { + console.log(`Error calling ${virusTotalURL}`) + console.error(error.data); + results.push({ + url, + success: false, + data: null, + error: error.response, + }); + }); + }); + } + return results; +} \ No newline at end of file From 0a50114ac5c62826d2f48f02e888597c554cdbc7 Mon Sep 17 00:00:00 2001 From: Gibson Tang Date: Sat, 24 Aug 2024 19:24:50 +0800 Subject: [PATCH 2/5] Refactor URL validation function for better performance --- functions/src/utils/utils.ts | 71 ++++++++++++++++++++---------------- 1 file changed, 40 insertions(+), 31 deletions(-) diff --git a/functions/src/utils/utils.ts b/functions/src/utils/utils.ts index 8ed49525..4fcccdff 100644 --- a/functions/src/utils/utils.ts +++ b/functions/src/utils/utils.ts @@ -8,52 +8,61 @@ interface URLValidationResult { error?: any; } -export async function validateURLS(text: string): Promise { +export function validateURLS(text: string): Promise { const urlRegex = /https?:\/\/[^\s/$.?#].[^\s]*/g; const urls = text.match(urlRegex); const results: URLValidationResult[] = []; if (urls) { - urls.forEach((url, index) => { + // Create an array of promises for each URL request + const requests = urls.map((url, index) => { console.log(`URL ${index + 1} is: "${url}"`); const base64URL: string = Buffer.from(url).toString('base64'); - const virusTotalURL: string = `https://www.virustotal.com/api/v3/urls/${base64URL}` - console.log(`Calling API ${virusTotalURL} to get scan results of ${url}`) - const VIRUS_TOTAL_API_KEY = String(process.env.VIRUS_TOTAL_API_KEY) - console.log(`VIRUS_TOTAL_API_KEY: ${VIRUS_TOTAL_API_KEY}`) + const virusTotalURL: string = `https://www.virustotal.com/api/v3/urls/${base64URL}`; + console.log(`Calling API ${virusTotalURL} to get scan results of ${url}`); + const VIRUS_TOTAL_API_KEY = String(process.env.VIRUS_TOTAL_API_KEY); + console.log(`VIRUS_TOTAL_API_KEY: ${VIRUS_TOTAL_API_KEY}`); const options = { method: 'GET', url: virusTotalURL, headers: { - accept: 'application/json', - 'x-apikey': VIRUS_TOTAL_API_KEY + accept: 'application/json', + 'x-apikey': VIRUS_TOTAL_API_KEY } - }; + }; - axios - .request(options) - .then(function (response: { data: any; }) { - console.log(`Success calling ${virusTotalURL}`) - console.log(response.data); - results.push({ - url, - success: true, - data: response.data, - error: null, + return axios + .request(options) + .then((response: { data: any; }) => { + console.log(`Success calling ${virusTotalURL}`); + console.log(response.data); + results.push({ + url, + success: true, + data: JSON.stringify(response.data), + error: null, + }); + }) + .catch((error: { response: { data: any; }; }) => { + console.log(`Error calling ${virusTotalURL}`); + console.error(error.response.data); + results.push({ + url, + success: false, + data: null, + error: JSON.stringify(error.response.data), + }); }); - }) - .catch(function (error: any) { - console.log(`Error calling ${virusTotalURL}`) - console.error(error.data); - results.push({ - url, - success: false, - data: null, - error: error.response, - }); - }); }); + + // Wait for all requests to complete and return results + return Promise.all(requests).then(() => { + console.log('All requests completed. Results:', results); + return results; + }); + } else { + // If no URLs are found, return an empty array + return Promise.resolve([]); } - return results; } \ No newline at end of file From b63f8563ae5be6efa44fd2d8bbbaddd1c7081063 Mon Sep 17 00:00:00 2001 From: Gibson Tang Date: Sat, 24 Aug 2024 23:29:22 +0800 Subject: [PATCH 3/5] Refactor URL validation function and handling --- .../definitions/eventHandlers/userHandlers.ts | 87 ++++++++++--------- functions/src/utils/utils.ts | 31 +++++-- 2 files changed, 73 insertions(+), 45 deletions(-) diff --git a/functions/src/definitions/eventHandlers/userHandlers.ts b/functions/src/definitions/eventHandlers/userHandlers.ts index f900610a..f7c59f08 100644 --- a/functions/src/definitions/eventHandlers/userHandlers.ts +++ b/functions/src/definitions/eventHandlers/userHandlers.ts @@ -1,6 +1,6 @@ import * as admin from "firebase-admin" import * as functions from "firebase-functions" -import { validateURLS } from '../../utils/utils'; +import { validateURLs } from '../../utils/utils'; import { onMessagePublished } from "firebase-functions/v2/pubsub" import { Timestamp } from "firebase-admin/firestore" import { @@ -308,46 +308,53 @@ async function newTextInstanceHandler({ } messageRef = db.collection("messages").doc() - const validatedURLS: any = validateURLS(text) - console.log(`Validated URLs are ${validatedURLS}`) + let validatedURLS: any; + try { + const validatedURLS: any = await validateURLs(text) + console.log('Validated URLs:', validatedURLS) - messageUpdateObj = { - machineCategory: machineCategory, //Can be "fake news" or "scam" - isMachineCategorised: isMachineAssessed, - originalText: text, - text: strippedMessage, //text - caption: null, - latestInstance: null, - firstTimestamp: timestamp, //timestamp of first instance (firestore timestamp data type) - lastTimestamp: timestamp, //timestamp of latest instance (firestore timestamp data type) - lastRefreshedTimestamp: timestamp, - isPollStarted: false, //boolean, whether or not polling has started - isAssessed: isMachineAssessed, //boolean, whether or not we have concluded the voting - assessedTimestamp: null, - assessmentExpiry: null, - assessmentExpired: false, - truthScore: null, //float, the mean truth score - isIrrelevant: - isMachineAssessed && machineCategory.includes("irrelevant") - ? true - : null, //bool, if majority voted irrelevant then update this - isScam: isMachineAssessed && machineCategory === "scam" ? true : null, - isIllicit: - isMachineAssessed && machineCategory === "illicit" ? true : null, - isSpam: isMachineAssessed && machineCategory === "spam" ? true : null, - isLegitimate: null, - isUnsure: null, - isInfo: machineCategory === "info" ? true : null, - isSatire: null, - isHarmful: null, - isHarmless: null, - primaryCategory: isMachineAssessed - ? machineCategory.split("_")[0] //in case of irrelevant_length, we want to store irrelevant - : null, - customReply: null, //string - instanceCount: 0, - rationalisation: rationalisation, - } + messageUpdateObj = { + machineCategory: machineCategory, //Can be "fake news" or "scam" + isMachineCategorised: isMachineAssessed, + originalText: text, + text: strippedMessage, //text + caption: null, + latestInstance: null, + firstTimestamp: timestamp, //timestamp of first instance (firestore timestamp data type) + lastTimestamp: timestamp, //timestamp of latest instance (firestore timestamp data type) + lastRefreshedTimestamp: timestamp, + isPollStarted: false, //boolean, whether or not polling has started + isAssessed: isMachineAssessed, //boolean, whether or not we have concluded the voting + assessedTimestamp: null, + assessmentExpiry: null, + assessmentExpired: false, + truthScore: null, //float, the mean truth score + isIrrelevant: + isMachineAssessed && machineCategory.includes("irrelevant") + ? true + : null, //bool, if majority voted irrelevant then update this + isScam: isMachineAssessed && machineCategory === "scam" ? true : null, + isIllicit: + isMachineAssessed && machineCategory === "illicit" ? true : null, + isSpam: isMachineAssessed && machineCategory === "spam" ? true : null, + isLegitimate: null, + isUnsure: null, + isInfo: machineCategory === "info" ? true : null, + isSatire: null, + isHarmful: null, + isHarmless: null, + primaryCategory: isMachineAssessed + ? machineCategory.split("_")[0] //in case of irrelevant_length, we want to store irrelevant + : null, + customReply: null, //string + instanceCount: 0, + rationalisation: rationalisation, + } + } catch (error) { + console.error('Error validating URLs:', error) + // You might want to handle the error, such as setting a default value + validatedURLS = null + } } else { messageRef = matchedParentMessageRef } diff --git a/functions/src/utils/utils.ts b/functions/src/utils/utils.ts index 4fcccdff..47ad8abe 100644 --- a/functions/src/utils/utils.ts +++ b/functions/src/utils/utils.ts @@ -8,7 +8,25 @@ interface URLValidationResult { error?: any; } -export function validateURLS(text: string): Promise { +function addHttpsIfMissing(url: string) { + // Create a URL object to easily parse the URL + try { + const parsedUrl = new URL(url); + + // If URL already has a scheme, return it as is + if (parsedUrl.protocol) { + return url; + } + } catch (e) { + // If URL parsing fails, it means it's a relative or invalid URL + // We need to handle this case + } + + // Add https:// if the URL is missing a scheme + return `https://${url}`; +} + +export function validateURLs(text: string): Promise { const urlRegex = /https?:\/\/[^\s/$.?#].[^\s]*/g; const urls = text.match(urlRegex); const results: URLValidationResult[] = []; @@ -16,6 +34,7 @@ export function validateURLS(text: string): Promise { if (urls) { // Create an array of promises for each URL request const requests = urls.map((url, index) => { + url = addHttpsIfMissing(url) console.log(`URL ${index + 1} is: "${url}"`); const base64URL: string = Buffer.from(url).toString('base64'); @@ -36,22 +55,24 @@ export function validateURLS(text: string): Promise { .request(options) .then((response: { data: any; }) => { console.log(`Success calling ${virusTotalURL}`); - console.log(response.data); + let data = JSON.stringify(response.data.data.attributes.total_votes) + console.error(data); results.push({ url, success: true, - data: JSON.stringify(response.data), + data: data, error: null, }); }) .catch((error: { response: { data: any; }; }) => { console.log(`Error calling ${virusTotalURL}`); - console.error(error.response.data); + let data = JSON.stringify(error.response.data) + console.error(data); results.push({ url, success: false, data: null, - error: JSON.stringify(error.response.data), + error: data, }); }); }); From ba11153d7f157898175456dba683b6e2f9111504 Mon Sep 17 00:00:00 2001 From: Gibson Tang Date: Sun, 13 Oct 2024 20:12:18 +0800 Subject: [PATCH 4/5] Add validation for URLs in message update --- .../userGenericMessageHandlers.ts | 183 ++++++++++-------- functions/src/types.ts | 1 + functions/src/utils/utils.ts | 6 +- 3 files changed, 111 insertions(+), 79 deletions(-) diff --git a/functions/src/definitions/eventHandlers/userGenericMessageHandlers.ts b/functions/src/definitions/eventHandlers/userGenericMessageHandlers.ts index 2213644f..ae25e6a1 100644 --- a/functions/src/definitions/eventHandlers/userGenericMessageHandlers.ts +++ b/functions/src/definitions/eventHandlers/userGenericMessageHandlers.ts @@ -79,7 +79,7 @@ const userGenericMessageHandlerWhatsapp = async function ( if (!message.text) { break } - console.log(`Text message is "${message.text.body}"`) + console.log(`Text message is "${message.text}"`) const textNormalised = normalizeSpaces(message.text).toLowerCase() //normalise spaces needed cos of potential   when copying message on desktop whatsapp if ( checkTemplate( @@ -173,6 +173,8 @@ async function newTextInstanceHandler({ let hasMatch = false let messageRef: FirebaseFirestore.DocumentReference | null = null let messageUpdateObj: MessageData | null = null + let validatedURLS: any; + const machineCategory = (await classifyText(text)) ?? "error" if (from && isFirstTimeUser && machineCategory.includes("irrelevant")) { await userSnap.ref.update({ @@ -185,6 +187,7 @@ async function newTextInstanceHandler({ let embedding let textHash = hashMessage(text) // 1 - check if the exact same message exists in database + try { ;({ embedding, similarity } = await calculateSimilarity( text, @@ -240,45 +243,58 @@ async function newTextInstanceHandler({ rationalisation = await rationaliseMessage(text, machineCategory) } messageRef = db.collection("messages").doc() - messageUpdateObj = { - machineCategory: machineCategory, //Can be "fake news" or "scam" - isMachineCategorised: isMachineAssessed, - originalText: text, - text: strippedMessage, //text - caption: null, - latestInstance: null, - firstTimestamp: timestamp, //timestamp of first instance (firestore timestamp data type) - lastTimestamp: timestamp, //timestamp of latest instance (firestore timestamp data type) - lastRefreshedTimestamp: timestamp, - isPollStarted: false, //boolean, whether or not polling has started - isAssessed: isMachineAssessed, //boolean, whether or not we have concluded the voting - assessedTimestamp: null, - assessmentExpiry: null, - assessmentExpired: false, - truthScore: null, //float, the mean truth score - numberPointScale: 6, - isIrrelevant: - isMachineAssessed && machineCategory.includes("irrelevant") - ? true - : null, //bool, if majority voted irrelevant then update this - isScam: isMachineAssessed && machineCategory === "scam" ? true : null, - isIllicit: - isMachineAssessed && machineCategory === "illicit" ? true : null, - isSpam: isMachineAssessed && machineCategory === "spam" ? true : null, - isLegitimate: null, - isUnsure: null, - isInfo: machineCategory === "info" ? true : null, - isSatire: null, - isHarmful: null, - isHarmless: null, - tags: {}, - primaryCategory: isMachineAssessed - ? machineCategory.split("_")[0] //in case of irrelevant_length, we want to store irrelevant - : null, - customReply: null, //string - instanceCount: 0, - rationalisation: rationalisation, + + try { + validatedURLS = await validateURLs(text) + console.log('Validated URLs:', validatedURLS) + + messageUpdateObj = { + machineCategory: machineCategory, //Can be "fake news" or "scam" + isMachineCategorised: isMachineAssessed, + originalText: text, + text: strippedMessage, //text + caption: null, + latestInstance: null, + firstTimestamp: timestamp, //timestamp of first instance (firestore timestamp data type) + lastTimestamp: timestamp, //timestamp of latest instance (firestore timestamp data type) + lastRefreshedTimestamp: timestamp, + isPollStarted: false, //boolean, whether or not polling has started + isAssessed: isMachineAssessed, //boolean, whether or not we have concluded the voting + assessedTimestamp: null, + assessmentExpiry: null, + assessmentExpired: false, + truthScore: null, //float, the mean truth score + numberPointScale: 6, + isIrrelevant: + isMachineAssessed && machineCategory.includes("irrelevant") + ? true + : null, //bool, if majority voted irrelevant then update this + isScam: isMachineAssessed && machineCategory === "scam" ? true : null, + isIllicit: + isMachineAssessed && machineCategory === "illicit" ? true : null, + isSpam: isMachineAssessed && machineCategory === "spam" ? true : null, + isLegitimate: null, + isUnsure: null, + isInfo: machineCategory === "info" ? true : null, + isSatire: null, + isHarmful: null, + isHarmless: null, + tags: {}, + primaryCategory: isMachineAssessed + ? machineCategory.split("_")[0] //in case of irrelevant_length, we want to store irrelevant + : null, + customReply: null, //string + instanceCount: 0, + rationalisation: rationalisation, + virtualTotalResults: validatedURLS + } + console.log('messageUpdateObj:', messageUpdateObj) + } catch (error) { + console.error('Error validating URLs:', error) + // You might want to handle the error, such as setting a default value + validatedURLS = null } + } else { messageRef = matchedParentMessageRef } @@ -373,6 +389,8 @@ async function newImageInstanceHandler({ let matchedInstanceSnap let captionHash = caption ? hashMessage(caption) : null + let validatedURLS: any; + if (!mediaId) { throw new Error(`No mediaId for whatsapp message with id ${id}`) } @@ -517,44 +535,55 @@ async function newImageInstanceHandler({ ) } messageRef = db.collection("messages").doc() - messageUpdateObj = { - machineCategory: machineCategory, - isMachineCategorised: isMachineAssessed, - originalText: extractedMessage ?? null, - text: strippedMessage ?? null, //text - caption: caption ?? null, - latestInstance: null, - firstTimestamp: timestamp, //timestamp of first instance (firestore timestamp data type) - lastTimestamp: timestamp, //timestamp of latest instance (firestore timestamp data type) - lastRefreshedTimestamp: timestamp, - isPollStarted: false, //boolean, whether or not polling has started - isAssessed: isMachineAssessed, //boolean, whether or not we have concluded the voting - assessedTimestamp: null, - assessmentExpiry: null, - assessmentExpired: false, - truthScore: null, //float, the mean truth score - numberPointScale: 6, - isIrrelevant: - isMachineAssessed && machineCategory.includes("irrelevant") - ? true - : null, //bool, if majority voted irrelevant then update this - isScam: isMachineAssessed && machineCategory === "scam" ? true : null, - isIllicit: - isMachineAssessed && machineCategory === "illicit" ? true : null, - isSpam: isMachineAssessed && machineCategory === "spam" ? true : null, - isLegitimate: null, - isUnsure: null, - isInfo: !caption && machineCategory === "info" ? true : null, - isSatire: null, - isHarmful: null, - isHarmless: null, - tags: {}, - primaryCategory: isMachineAssessed - ? machineCategory.split("_")[0] //in case of irrelevant_length, we want to store irrelevant - : null, - customReply: null, //string - instanceCount: 0, - rationalisation: rationalisation, + + try { + validatedURLS = await validateURLs(extractedMessage) + console.log('Validated URLs:', validatedURLS) + + messageUpdateObj = { + machineCategory: machineCategory, + isMachineCategorised: isMachineAssessed, + originalText: extractedMessage ?? null, + text: strippedMessage ?? null, //text + caption: caption ?? null, + latestInstance: null, + firstTimestamp: timestamp, //timestamp of first instance (firestore timestamp data type) + lastTimestamp: timestamp, //timestamp of latest instance (firestore timestamp data type) + lastRefreshedTimestamp: timestamp, + isPollStarted: false, //boolean, whether or not polling has started + isAssessed: isMachineAssessed, //boolean, whether or not we have concluded the voting + assessedTimestamp: null, + assessmentExpiry: null, + assessmentExpired: false, + truthScore: null, //float, the mean truth score + numberPointScale: 6, + isIrrelevant: + isMachineAssessed && machineCategory.includes("irrelevant") + ? true + : null, //bool, if majority voted irrelevant then update this + isScam: isMachineAssessed && machineCategory === "scam" ? true : null, + isIllicit: + isMachineAssessed && machineCategory === "illicit" ? true : null, + isSpam: isMachineAssessed && machineCategory === "spam" ? true : null, + isLegitimate: null, + isUnsure: null, + isInfo: !caption && machineCategory === "info" ? true : null, + isSatire: null, + isHarmful: null, + isHarmless: null, + tags: {}, + primaryCategory: isMachineAssessed + ? machineCategory.split("_")[0] //in case of irrelevant_length, we want to store irrelevant + : null, + customReply: null, //string + instanceCount: 0, + rationalisation: rationalisation, + virtualTotalResults: validatedURLS + } + } catch (error) { + console.error('Error validating URLs:', error) + // You might want to handle the error, such as setting a default value + validatedURLS = null } } else { if (matchType === "image" && matchedInstanceSnap) { diff --git a/functions/src/types.ts b/functions/src/types.ts index 75617d28..6d998694 100644 --- a/functions/src/types.ts +++ b/functions/src/types.ts @@ -146,6 +146,7 @@ export type MessageData = { customReply: string | null instanceCount: number rationalisation: string | null // Assuming 'rationalisation' is a string; adjust as necessary if it's a different type. + virtualTotalResults: any | null } export type InstanceData = { diff --git a/functions/src/utils/utils.ts b/functions/src/utils/utils.ts index 47ad8abe..19fa33ec 100644 --- a/functions/src/utils/utils.ts +++ b/functions/src/utils/utils.ts @@ -41,7 +41,9 @@ export function validateURLs(text: string): Promise { const virusTotalURL: string = `https://www.virustotal.com/api/v3/urls/${base64URL}`; console.log(`Calling API ${virusTotalURL} to get scan results of ${url}`); const VIRUS_TOTAL_API_KEY = String(process.env.VIRUS_TOTAL_API_KEY); - console.log(`VIRUS_TOTAL_API_KEY: ${VIRUS_TOTAL_API_KEY}`); + + //Print only the last 4 characters of the API key instead of the full key for security reasons + console.log(`VIRUS_TOTAL_API_KEY: ${VIRUS_TOTAL_API_KEY.slice(-4)}`); const options = { method: 'GET', url: virusTotalURL, @@ -79,7 +81,7 @@ export function validateURLs(text: string): Promise { // Wait for all requests to complete and return results return Promise.all(requests).then(() => { - console.log('All requests completed. Results:', results); + console.log('All validate URLs requests completed. Results:', results); return results; }); } else { From 69b92de958a67ca6771fd58cf1bb0e43c2b33d0e Mon Sep 17 00:00:00 2001 From: Gibson Tang Date: Sun, 13 Oct 2024 20:29:06 +0800 Subject: [PATCH 5/5] Add initial environment variables to env.sample --- functions/env.sample | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 functions/env.sample diff --git a/functions/env.sample b/functions/env.sample new file mode 100644 index 00000000..fedd0c6b --- /dev/null +++ b/functions/env.sample @@ -0,0 +1,26 @@ +NUM_SHARDS_INSTANCE_COUNT= +NUM_SHARDS_VOTE_COUNT= +GRAPH_API_VERSION= +ENVIRONMENT= +SIMILARITY_THRESHOLD= +TYPESENSE_HOST= +EMBEDDER_HOST= +CHECKER1_ID= +CHECKER1_TELEGRAM_ID= +CHECKER1_PHONE_NUMBER= +TYPESENSE_PORT= +TYPESENSE_PROTOCOL= +TELEGRAM_REPORT_CHANNEL_ID= +TEST_IMAGE_URL= +HASHIDS_SALT= +WEBHOOK_PATH_WHATSAPP= +WEBHOOK_PATH_TELEGRAM= +CHECKER_APP_HOST= +WEBHOOK_PATH_TYPEFORM= +CHECKERS_GROUP_LINK= +USERS_WHATSAPP_NUMBER= +CHECKERS_CHAT_ID= +TYPEFORM_URL= + +#EXTERNAL APIS +VIRUS_TOTAL_API_KEY=