feat: support DAP API key in workflow and ingest

Author: mgifford

.github/workflows/daily-scan.yml

@@ -44,6 +44,7 @@ jobs:
       URL_LIMIT: ${{ inputs.url_limit }}
       TRAFFIC_WINDOW: ${{ inputs.traffic_window || 'daily' }}
       DRY_RUN: ${{ inputs.dry_run || false }}
+      DAP_API_KEY: ${{ secrets.DAP_API_KEY }}
     steps:
       - name: Checkout
         uses: actions/checkout@v4

src/cli/run-daily-scan.js

@@ -30,6 +30,7 @@ function parseArgs(argv) {
     scanMode: 'mock',
     mockFailUrl: [],
     outputRoot: null,
+    dapApiKey: undefined,
     concurrency: 4,
     timeoutMs: 20000,
     maxRetries: 1
@@ -68,6 +69,9 @@ function parseArgs(argv) {
       case '--output-root':
         args.outputRoot = argv[++index];
         break;
+      case '--dap-api-key':
+        args.dapApiKey = argv[++index];
+        break;
       case '--concurrency':
         args.concurrency = Number(argv[++index]);
         break;
@@ -234,6 +238,7 @@ export async function runDailyScan(inputArgs = parseArgs(process.argv)) {
   const args = inputArgs;
   const repoRoot = path.resolve(args.outputRoot ?? getDefaultRepoRoot());
   const configPath = args.configPath ?? getDefaultConfigPath();
+  const dapApiKey = args.dapApiKey ?? process.env.DAP_API_KEY;
 
   let runMetadata;
 
@@ -251,11 +256,17 @@ export async function runDailyScan(inputArgs = parseArgs(process.argv)) {
       source: 'dap'
     });
 
+    const dapEndpoint = runtimeConfig.sources?.dap_top_pages_endpoint;
+    if (!args.sourceFile && dapEndpoint?.includes('api.gsa.gov') && !dapApiKey) {
+      throw new Error('DAP_API_KEY is required to fetch top pages from api.gsa.gov. Set repo secret DAP_API_KEY or pass --dap-api-key.');
+    }
+
     const normalized = await getNormalizedTopPages({
-      endpoint: runtimeConfig.sources?.dap_top_pages_endpoint,
+      endpoint: dapEndpoint,
       sourceFile: args.sourceFile,
       limit: runtimeConfig.scan.url_limit,
-      sourceDate: runMetadata.run_date
+      sourceDate: runMetadata.run_date,
+      dapApiKey
     });
 
     const warningEvents = normalized.warnings.map((warning) =>

src/ingest/dap-source.js

@@ -85,6 +85,16 @@ export async function fetchDapRecords({ endpoint, fetchImpl = fetch }) {
   return extractArrayPayload(payload);
 }
 
+function buildDapEndpoint(endpoint, apiKey) {
+  const url = new URL(endpoint);
+
+  if (apiKey && !url.searchParams.has('api_key')) {
+    url.searchParams.set('api_key', apiKey);
+  }
+
+  return url.toString();
+}
+
 export async function readDapRecordsFromFile(filePath) {
   const raw = await fs.readFile(filePath, 'utf-8');
   const payload = JSON.parse(raw);
@@ -96,13 +106,15 @@ export async function getNormalizedTopPages({
   sourceFile,
   limit,
   sourceDate,
+  dapApiKey,
   fetchImpl = fetch
 }) {
   let rawRecords;
   if (sourceFile) {
     rawRecords = await readDapRecordsFromFile(sourceFile);
   } else {
-    rawRecords = await fetchDapRecords({ endpoint, fetchImpl });
+    const resolvedEndpoint = buildDapEndpoint(endpoint, dapApiKey);
+    rawRecords = await fetchDapRecords({ endpoint: resolvedEndpoint, fetchImpl });
   }
 
   return normalizeDapRecords(rawRecords, { limit, sourceDate });