fix(crawler): riusa le connessioni tra le richieste (#92)

Il client outbound creava e chiudeva un Agent per ogni richiesta, quindi le
centinaia di pagine di uno stesso monitor rifacevano handshake TCP/TLS e
risoluzione DNS: con il timeout di 10 minuti del workflow gli scan grandi
rischiavano di scadere prima di produrre output. Ora i dispatcher sono
pooled per insieme di indirizzi fissati, il DNS è risolto una volta per
host e lo scan chiude i pool in un finally.

Chiude anche due thread Codex della PR #91: le sitemap duplicate non
consumano più il budget di 32 prima di essere deduplicate al dequeue, e la
classificazione degli indirizzi copre i prefissi IPv6 non globali e di
transizione (Teredo, 6to4, NAT64, ORCHIDv2, SRv6, doc 3fff::/20) più il
relay 6to4 IPv4 192.88.99.0/24.

Co-authored-by: Claude Opus 5 <noreply@anthropic.com>

Author: max23468

src/outbound.ts

@@ -15,9 +15,14 @@ const MAX_SCAN_BYTES = 100 * 1024 * 1024;
 const EXTRA_SCAN_REQUESTS = 64;
 const blockedAddresses = createBlockedAddresses();
 
+interface PinnedAddress {
+  address: string;
+  family: 4 | 6;
+}
+
 interface OutboundDependencies {
   fetch?: typeof fetch;
-  resolve?: (hostname: string) => Promise<Array<{ address: string; family: 4 | 6 }>>;
+  resolve?: (hostname: string) => Promise<PinnedAddress[]>;
 }
 
 interface OutboundOptions {
@@ -39,6 +44,11 @@ export class OutboundClient {
   private remainingRequests: number;
   private readonly fetchImpl: typeof fetch;
   private readonly resolve: NonNullable<OutboundDependencies["resolve"]>;
+  // Un dispatcher per insieme di indirizzi fissati: le centinaia di pagine
+  // dello stesso monitor riusano connessioni TCP/TLS invece di rifarle a ogni
+  // richiesta. Il DNS resta risolto una volta sola per host per tutto lo scan.
+  private readonly agents = new Map<string, Agent>();
+  private readonly pinned = new Map<string, Promise<PinnedAddress[]>>();
 
   constructor(
     private readonly site: SiteConfig,
@@ -57,47 +67,55 @@ export class OutboundClient {
         throw new OutboundBudgetError("Budget richieste dello scan esaurito.");
       }
 
-      const addresses = await this.resolvePinnedAddresses(currentUrl);
-      const dispatcher = new Agent({
-        connect: { lookup: pinnedLookup(addresses) }
-      });
-
-      try {
-        const response = await this.fetchImpl(currentUrl, {
-          dispatcher,
-          headers: options.headers,
-          redirect: "manual",
-          signal: AbortSignal.timeout(this.site.crawl.timeoutMs)
-        } as RequestInit);
-
-        const location = response.headers.get("location");
-        if (location && isRedirect(response.status)) {
-          await response.body?.cancel();
-          if (redirects >= MAX_REDIRECTS) throw new Error(`Troppi redirect: ${rawUrl}`);
-          currentUrl = this.authorize(new URL(location, currentUrl).toString());
-          continue;
-        }
+      const response = await this.fetchImpl(currentUrl, {
+        dispatcher: this.dispatcherFor(await this.resolvePinnedAddresses(currentUrl)),
+        headers: options.headers,
+        redirect: "manual",
+        signal: AbortSignal.timeout(this.site.crawl.timeoutMs)
+      } as RequestInit);
 
-        if (response.status < 200 || response.status >= 300) {
-          await response.body?.cancel();
-          return {
-            body: new Uint8Array(),
-            headers: response.headers,
-            status: response.status,
-            url: currentUrl
-          };
-        }
+      const location = response.headers.get("location");
+      if (location && isRedirect(response.status)) {
+        await response.body?.cancel();
+        if (redirects >= MAX_REDIRECTS) throw new Error(`Troppi redirect: ${rawUrl}`);
+        currentUrl = this.authorize(new URL(location, currentUrl).toString());
+        continue;
+      }
 
-        const maxBytes =
-          typeof options.maxBytes === "function"
-            ? options.maxBytes(currentUrl, response.headers)
-            : options.maxBytes;
-        const body = await this.readBody(response, maxBytes);
-        return { body, headers: response.headers, status: response.status, url: currentUrl };
-      } finally {
-        await dispatcher.close();
+      if (response.status < 200 || response.status >= 300) {
+        await response.body?.cancel();
+        return {
+          body: new Uint8Array(),
+          headers: response.headers,
+          status: response.status,
+          url: currentUrl
+        };
       }
+
+      const maxBytes =
+        typeof options.maxBytes === "function"
+          ? options.maxBytes(currentUrl, response.headers)
+          : options.maxBytes;
+      const body = await this.readBody(response, maxBytes);
+      return { body, headers: response.headers, status: response.status, url: currentUrl };
+    }
+  }
+
+  /** Chiude i pool aperti: senza questo il processo resta appeso a fine scan. */
+  async close(): Promise<void> {
+    const agents = [...this.agents.values()];
+    this.agents.clear();
+    await Promise.all(agents.map((agent) => agent.close()));
+  }
+
+  private dispatcherFor(addresses: PinnedAddress[]): Agent {
+    const key = addresses.map(({ address }) => address).join(",");
+    let agent = this.agents.get(key);
+    if (!agent) {
+      agent = new Agent({ connect: { lookup: pinnedLookup(addresses) } });
+      this.agents.set(key, agent);
     }
+    return agent;
   }
 
   private authorize(rawUrl: string): string {
@@ -113,15 +131,22 @@ export class OutboundClient {
     return url.toString();
   }
 
-  private async resolvePinnedAddresses(url: string): Promise<Array<{ address: string; family: 4 | 6 }>> {
+  private resolvePinnedAddresses(url: string): Promise<PinnedAddress[]> {
     const rawHostname = new URL(url).hostname;
     const hostname = rawHostname.startsWith("[") ? rawHostname.slice(1, -1) : rawHostname;
-    const addresses = await this.resolve(hostname);
-    if (addresses.length === 0) throw new Error(`DNS senza indirizzi per ${hostname}`);
-    if (addresses.some(({ address, family }) => !isPublicAddress(address, family))) {
-      throw new Error(`Destinazione privata o riservata bloccata: ${hostname}`);
-    }
-    return addresses;
+    const cached = this.pinned.get(hostname);
+    if (cached) return cached;
+
+    const promise = (async () => {
+      const addresses = await this.resolve(hostname);
+      if (addresses.length === 0) throw new Error(`DNS senza indirizzi per ${hostname}`);
+      if (addresses.some(({ address, family }) => !isPublicAddress(address, family))) {
+        throw new Error(`Destinazione privata o riservata bloccata: ${hostname}`);
+      }
+      return addresses;
+    })();
+    this.pinned.set(hostname, promise);
+    return promise;
   }
 
   private async readBody(response: Response, maxBytes: number): Promise<Uint8Array> {
@@ -199,20 +224,30 @@ function createBlockedAddresses(): BlockList {
     ["192.0.2.0", 24],
     ["192.168.0.0", 16],
     ["198.18.0.0", 15],
+    ["192.88.99.0", 24],
     ["198.51.100.0", 24],
     ["203.0.113.0", 24],
     ["224.0.0.0", 4],
     ["240.0.0.0", 4]
   ] as const) {
     list.addSubnet(network, prefix, "ipv4");
   }
+  // IANA IPv6 Special-Purpose Address Registry: tutto ciò che non è "Global:
+  // True". I prefissi di transizione (NAT64, Teredo, 6to4) restano bloccati
+  // perché incapsulano destinazioni IPv4 che possono essere riservate.
   for (const [network, prefix] of [
     ["::", 128],
     ["::1", 128],
+    ["64:ff9b::", 96],
     ["64:ff9b:1::", 48],
     ["100::", 64],
+    ["2001::", 32],
     ["2001:2::", 48],
+    ["2001:20::", 28],
     ["2001:db8::", 32],
+    ["2002::", 16],
+    ["3fff::", 20],
+    ["5f00::", 16],
     ["fc00::", 7],
     ["fe80::", 10],
     ["ff00::", 8]

src/scan.ts

@@ -50,9 +50,15 @@ export async function scanSite(
   const seenUrls = new Set<string>();
   const guard = new RobotsGuard(site, client);
 
-  const queue = await buildInitialQueue(site, guard, issues, client);
   const counters: CrawlCounters = { scannedCount: 0, skippedCount: 0 };
-  await crawlQueue(site, guard, client, queue, seenUrls, issues, resources, changes, siteState, baseline, counters);
+  try {
+    const queue = await buildInitialQueue(site, guard, issues, client);
+    await crawlQueue(site, guard, client, queue, seenUrls, issues, resources, changes, siteState, baseline, counters);
+  } finally {
+    // Le connessioni riusate durante il crawling vanno chiuse qui, o il
+    // processo resta appeso anche quando lo scan è finito.
+    await client.close();
+  }
 
   if (!baseline && !hasFatalIssues(issues)) {
     changes.push(...collectRemovals(siteState, seenUrls, issues, resources));

src/sitemap.ts

@@ -20,18 +20,29 @@ export async function discoverFromSitemaps(
   client: OutboundClient
 ): Promise<QueuedUrl[]> {
   const discovered = new Map<string, QueuedUrl>();
-  const seenSitemaps = new Set<string>();
-  const queue = sitemapUrls.map((url) => ({ url, depth: 0 }));
+  const queued = new Set<string>();
+  const queue: Array<{ url: string; depth: number }> = [];
+
+  // Il budget conta le sitemap distinte: un indice che ripete lo stesso figlio
+  // non deve esaurire il limite prima delle sitemap ancora da leggere.
+  const enqueue = (url: string, depth: number): boolean => {
+    if (queued.has(url)) return true;
+    if (queued.size >= MAX_SITEMAPS) return false;
+    queued.add(url);
+    queue.push({ url, depth });
+    return true;
+  };
+
+  for (const url of sitemapUrls) {
+    if (enqueue(url, 0)) continue;
+    issues.push({ url, message: `Limite di ${MAX_SITEMAPS} sitemap raggiunto.`, fatal: false });
+    break;
+  }
 
   while (queue.length > 0 && discovered.size < site.crawl.maxUrls) {
     const item = queue.shift();
-    if (!item || seenSitemaps.has(item.url)) continue;
-    if (seenSitemaps.size >= MAX_SITEMAPS) {
-      issues.push({ url: item.url, message: `Limite di ${MAX_SITEMAPS} sitemap raggiunto.`, fatal: false });
-      break;
-    }
+    if (!item) break;
 
-    seenSitemaps.add(item.url);
     // Una sitemap illeggibile non ferma la scansione: il crawling dai roots
     // resta la fonte principale, la sitemap aggiunge le pagine non linkate.
     const body = await readSitemap(site, item.url, issues, client);
@@ -55,12 +66,10 @@ export async function discoverFromSitemaps(
         !normalized ||
         !isSameSite(normalized, site) ||
         item.depth >= MAX_SITEMAP_DEPTH ||
-        queue.length + seenSitemaps.size >= MAX_SITEMAPS
+        !enqueue(normalized, item.depth + 1)
       ) {
         rejectedChild = true;
-        continue;
       }
-      queue.push({ url: normalized, depth: item.depth + 1 });
     }
     if (rejectedChild) {
       issues.push({

test/discovery.test.ts

@@ -160,4 +160,34 @@ describe("discoverFromSitemaps", () => {
     expect(fetch).toHaveBeenCalledTimes(5);
     expect(issues[0].message).toContain("oltre budget");
   });
+
+  it("non consuma il budget con sitemap duplicate nello stesso indice", async () => {
+    const index = `<?xml version="1.0"?><sitemapindex>
+      ${'<sitemap><loc>https://example.com/doppia.xml</loc></sitemap>'.repeat(40)}
+      <sitemap><loc>https://example.com/unica.xml</loc></sitemap>
+    </sitemapindex>`;
+    const unica = `<?xml version="1.0"?><urlset>
+      <url><loc>https://example.com/pagina</loc></url>
+    </urlset>`;
+
+    vi.stubGlobal(
+      "fetch",
+      vi.fn(async (url: string) => {
+        if (url.endsWith("indice.xml")) return new Response(index, { status: 200 });
+        if (url.endsWith("unica.xml")) return new Response(unica, { status: 200 });
+        return new Response("<urlset></urlset>", { status: 200 });
+      })
+    );
+
+    const issues: ScanIssue[] = [];
+    const discovered = await discoverFromSitemaps(
+      site,
+      ["https://example.com/indice.xml"],
+      issues,
+      testOutboundClient(site, fetch)
+    );
+
+    expect(discovered.map((item) => item.url)).toEqual(["https://example.com/pagina"]);
+    expect(issues).toEqual([]);
+  });
 });

test/security.test.ts

@@ -89,4 +89,32 @@ describe("hardening input remoti", () => {
 
     expect(resolve).toHaveBeenCalledWith("2606:4700:4700::1111");
   });
+
+  it("blocca i prefissi IPv6 non globali e di transizione", () => {
+    expect(isPublicAddress("2606:4700:4700::1111", 6)).toBe(true);
+    expect(isPublicAddress("2001::1", 6)).toBe(false);
+    expect(isPublicAddress("2002:7f00:1::1", 6)).toBe(false);
+    expect(isPublicAddress("64:ff9b::a9fe:a9fe", 6)).toBe(false);
+    expect(isPublicAddress("3fff::1", 6)).toBe(false);
+    expect(isPublicAddress("192.88.99.1", 4)).toBe(false);
+  });
+
+  it("riusa lo stesso pool per gli indirizzi già fissati", async () => {
+    const resolve = vi.fn(async () => [{ address: "93.184.216.34", family: 4 as const }]);
+    const dispatchers = new Set<unknown>();
+    const client = new OutboundClient(site, {
+      fetch: async (_url, init) => {
+        dispatchers.add((init as { dispatcher?: unknown }).dispatcher);
+        return new Response("ok");
+      },
+      resolve
+    });
+
+    await client.get("https://example.com/a", { maxBytes: 10 });
+    await client.get("https://example.com/b", { maxBytes: 10 });
+    await client.close();
+
+    expect(dispatchers.size).toBe(1);
+    expect(resolve).toHaveBeenCalledOnce();
+  });
 });