Enhance downloader functionality and version bump to v0.6.1

CHANGELOG.md

@@ -7,6 +7,12 @@ and this project adheres to [Conventional Commits](https://www.conventionalcommi
 
 ## [Unreleased]
 
+## [0.6.1]
+
+### Changed
+
+- **Bulk download for `download --all`**: when downloading all 54 titles, the downloader now fetches a single `xml_uscAll@{releasePoint}.zip` instead of making 54 individual HTTP requests. Falls back to per-title downloads if the bulk zip is unavailable. No CLI changes — same `--all` flag, same output.
+
 ## [0.6.0]
 
 ### Changed

packages/cli/CHANGELOG.md

@@ -1,5 +1,14 @@
 # law2md
 
+## 0.6.1
+
+### Patch Changes
+
+- Enhance downloader for when all titles are downloaded
+- Updated dependencies
+  - @law2md/core@0.6.1
+  - @law2md/usc@0.6.1
+
 ## 0.6.0
 
 ### Minor Changes

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "law2md",
-  "version": "0.6.0",
+  "version": "0.6.1",
   "description": "Convert U.S. legislative XML (USLM) to structured Markdown for AI/RAG ingestion",
   "type": "module",
   "main": "./dist/index.js",

packages/core/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @law2md/core
 
+## 0.6.1
+
+### Patch Changes
+
+- Enhance downloader for when all titles are downloaded
+
 ## 0.6.0
 
 ### Minor Changes

packages/core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@law2md/core",
-  "version": "0.6.0",
+  "version": "0.6.1",
   "description": "Core XML parsing, AST, and Markdown rendering for law2md",
   "type": "module",
   "main": "./dist/index.js",

packages/usc/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @law2md/usc
 
+## 0.6.1
+
+### Patch Changes
+
+- Enhance downloader for when all titles are downloaded
+- Updated dependencies
+  - @law2md/core@0.6.1
+
 ## 0.6.0
 
 ### Minor Changes

packages/usc/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@law2md/usc",
-  "version": "0.6.0",
+  "version": "0.6.1",
   "description": "U.S. Code-specific element handlers and downloader for law2md",
   "type": "module",
   "main": "./dist/index.js",

packages/usc/src/downloader.test.ts

@@ -3,6 +3,7 @@ import {
   buildDownloadUrl,
   buildAllTitlesUrl,
   releasePointToPath,
+  isAllTitles,
   CURRENT_RELEASE_POINT,
   USC_TITLE_NUMBERS,
 } from "./downloader.js";
@@ -63,6 +64,36 @@ describe("buildAllTitlesUrl", () => {
   });
 });
 
+describe("isAllTitles", () => {
+  it("returns true for the full set 1-54 in order", () => {
+    expect(isAllTitles(Array.from({ length: 54 }, (_, i) => i + 1))).toBe(true);
+  });
+
+  it("returns true for the full set shuffled", () => {
+    const shuffled = Array.from({ length: 54 }, (_, i) => i + 1);
+    shuffled.reverse();
+    expect(isAllTitles(shuffled)).toBe(true);
+  });
+
+  it("returns false for a subset", () => {
+    expect(isAllTitles([1, 2, 3])).toBe(false);
+  });
+
+  it("returns false for 54 elements with duplicates (missing titles)", () => {
+    const withDupes = Array.from({ length: 54 }, (_, i) => (i < 53 ? i + 1 : 1));
+    expect(isAllTitles(withDupes)).toBe(false);
+  });
+
+  it("returns true for full set with extra duplicates", () => {
+    const withExtras = [...Array.from({ length: 54 }, (_, i) => i + 1), 1, 27, 54];
+    expect(isAllTitles(withExtras)).toBe(true);
+  });
+
+  it("returns false for an empty array", () => {
+    expect(isAllTitles([])).toBe(false);
+  });
+});
+
 describe("constants", () => {
   it("has 54 USC title numbers", () => {
     expect(USC_TITLE_NUMBERS).toHaveLength(54);

packages/usc/src/downloader.ts

@@ -34,6 +34,20 @@
 /** Valid USC title numbers (1-54) */
 export const USC_TITLE_NUMBERS = Array.from({ length: 54 }, (_, i) => i + 1);
 
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+/**
+ * Check whether a list of title numbers covers all 54 USC titles.
+ *
+ * Handles arbitrary ordering and duplicates.
+ */
+export function isAllTitles(titles: number[]): boolean {
+  const unique = new Set(titles);
+  return unique.size === 54 && USC_TITLE_NUMBERS.every((n) => unique.has(n));
+}
+
 // ---------------------------------------------------------------------------
 // Public API
 // ---------------------------------------------------------------------------
@@ -78,13 +92,27 @@
 
 /**
  * Download USC title XML files from OLRC.
+ *
+ * When all 54 titles are requested, uses the bulk `uscAll` zip for a single
+ * HTTP round-trip instead of 54 individual requests. Falls back to per-title
+ * downloads if the bulk download fails.
  */
 export async function downloadTitles(options: DownloadOptions): Promise<DownloadResult> {
   const releasePoint = options.releasePoint ?? CURRENT_RELEASE_POINT;
   const titles = options.titles ?? USC_TITLE_NUMBERS;
 
   await mkdir(options.outputDir, { recursive: true });
 
+  // Use bulk zip when all 54 titles are requested
+  if (options.titles === undefined || isAllTitles(titles)) {
+    try {
+      const files = await downloadAndExtractAllTitles(releasePoint, options.outputDir);
+      return { releasePoint, files, errors: [] };
+    } catch {
+      // Fall back to per-title downloads
+    }
+  }
+
   const files: DownloadedFile[] = [];
   const errors: DownloadError[] = [];
 
@@ -265,3 +293,121 @@
     });
   });
 }
+
+// ---------------------------------------------------------------------------
+// Bulk download (all titles in one zip)
+// ---------------------------------------------------------------------------
+
+/** Regex matching USC XML filenames like usc01.xml, usc54.xml */
+const USC_XML_RE = /^(?:.*\/)?usc(\d{2})\.xml$/;
+
+/**
+ * Extract all `usc{NN}.xml` files from a bulk zip archive.
+ *
+ * Returns an array of `{ titleNumber, filePath }` for each extracted file.
+ */
+function extractAllXmlFromZip(
+  zipPath: string,
+  outputDir: string,
+): Promise<{ titleNumber: number; filePath: string }[]> {
+  return new Promise((resolve, reject) => {
+    yauzlOpen(zipPath, { lazyEntries: true }, (err, zipFile) => {
+      if (err) {
+        reject(new Error(`Failed to open zip: ${err.message}`));
+        return;
+      }
+      if (!zipFile) {
+        reject(new Error("Failed to open zip: no zipFile returned"));
+        return;
+      }
+
+      const extracted: { titleNumber: number; filePath: string }[] = [];
+      let pending = 0;
+      let ended = false;
+
+      const maybeResolve = (): void => {
+        if (ended && pending === 0) {
+          resolve(extracted);
+        }
+      };
+
+      zipFile.on("entry", (entry: Entry) => {
+        const match = USC_XML_RE.exec(entry.fileName);
+        if (match) {
+          const titleNum = parseInt(match[1]!, 10);
+          const outPath = join(outputDir, `usc${match[1]!}.xml`);
+          pending++;
+
+          extractEntry(zipFile, entry, outPath)
+            .then(() => {
+              extracted.push({ titleNumber: titleNum, filePath: outPath });
+              pending--;
+              // Continue reading entries after extraction completes
+              zipFile.readEntry();
+              maybeResolve();
+            })
+            .catch((extractErr) => {
+              zipFile.close();
+              reject(extractErr);
+            });
+        } else {
+          zipFile.readEntry();
+        }
+      });
+
+      zipFile.on("end", () => {
+        ended = true;
+        maybeResolve();
+      });
+
+      zipFile.on("error", (zipErr: Error) => {
+        reject(new Error(`Zip error: ${zipErr.message}`));
+      });
+
+      zipFile.readEntry();
+    });
+  });
+}
+
+/**
+ * Download the bulk all-titles zip and extract every `usc{NN}.xml` file.
+ */
+async function downloadAndExtractAllTitles(
+  releasePoint: string,
+  outputDir: string,
+): Promise<DownloadedFile[]> {
+  const url = buildAllTitlesUrl(releasePoint);
+  const zipPath = join(outputDir, "uscAll.zip");
+
+  // Download the zip file
+  const response = await fetch(url);
+  if (!response.ok) {
+    throw new Error(`HTTP ${response.status}: ${response.statusText} for ${url}`);
+  }
+
+  if (!response.body) {
+    throw new Error(`No response body for ${url}`);
+  }
+
+  // Write zip to disk
+  const fileStream = createWriteStream(zipPath);
+  await pipeline(Readable.fromWeb(response.body as never), fileStream);
+
+  // Extract all XML files from zip
+  const extracted = await extractAllXmlFromZip(zipPath, outputDir);
+
+  // Clean up zip file
+  await unlink(zipPath);
+
+  // Stat each extracted file and build results
+  const files: DownloadedFile[] = [];
+  for (const { titleNumber, filePath } of extracted) {
+    const fileStat = await stat(filePath);
+    files.push({ titleNumber, filePath, size: fileStat.size });
+  }
+
+  // Sort by title number for consistent ordering
+  files.sort((a, b) => a.titleNumber - b.titleNumber);
+
+  return files;
+}

packages/usc/src/index.ts

@@ -8,6 +8,7 @@ export {
   buildDownloadUrl,
   buildAllTitlesUrl,
   releasePointToPath,
+  isAllTitles,
   CURRENT_RELEASE_POINT,
   USC_TITLE_NUMBERS,
 } from "./downloader.js";