diff --git a/DOCS.md b/DOCS.md index 4e318ac..5515c5b 100644 --- a/DOCS.md +++ b/DOCS.md @@ -859,6 +859,23 @@ runs `git init` before the best-effort commit. The `--no-harness`/ `--skip-harness` flags only apply in `full` mode (template WITHOUT the harness); in `harness` mode they are ignored with a warning. +**Extraction and the Windows symlink fallback.** Every gated download +(harness AND templates, `src/lib/template-fetch.js`) unpacks with the system +`tar` first; when it is missing or fails, a built-in pure-JS extractor +(`src/lib/tar-extract.js`) re-extracts from scratch. The case that motivates +it: the harness ships 50+ mirror symlinks (`.agents/*` and `.cursor/agents/*` +pointing into `.claude/` and `.cursor/`), and Windows' bundled tar cannot +CREATE symlinks without a privilege students don't have (admin shell or +Developer Mode) — every link died with "Invalid argument" and the install +aborted as a false "corrupted download". The built-in extractor tries a real +link first and, on the first failure, MATERIALIZES every link as a copy of +its resolved target instead — each engine still finds real content at its +mirrored path, and the machines that do allow symlinks keep them. Only when +both extractors refuse the bytes does the run fail (`extract_failed` — a +genuinely corrupted download). The same degradation exists in `imp fix` +(§14.5): restoring a `link:` manifest entry falls back to copying the +target when `symlink()` is denied. + The harness also ships `.claude/settings.json` with `CLAUDE_CODE_EXPERIMENTAL_AGENT_TEAMS=1` (required by `/team`) and the two fda-lock hooks (SessionStart warn + PreToolUse gate — the read-only guard @@ -877,7 +894,10 @@ doctor` warns about any npx MCP server missing the flag and `imp fix` adds it marker merge, template-owned paths discarded), so its keys are exactly the project-relative paths the harness shipped; each value is the sha1 of the harness content, or `link:` for a symlink (the merge copies links -verbatim, so the recorded target is what a healthy disk must show). +verbatim, so the recorded target is what a healthy disk must show). On a +Windows machine where the extractor materialized the links as copies, the +clone has real files at those paths — the manifest then records content +sha1s, which matches that disk just the same. That baseline is what makes a missing/pristine/modified classification possible for harness files: `imp doctor` reports it and `imp fix` restores @@ -950,9 +970,11 @@ without the FIA runtime. | `ui-component-researcher` | Researches/documents a single UI component into `ai-docs/components//.md`. | | `api-docs-researcher` | Researches an external API/technology and writes the project-tailored doc into `ai-docs/apis/` (also logs the four research dimensions). | -`.cursor/agents/` are symlinks to `.claude/agents/` (canonical). Cursor -additionally ships router skills (`project-workflow` + `workflow-*` wrappers -for the original 8 pipelines) because Cursor routes by skill. +`.cursor/agents/` are symlinks to `.claude/agents/` (canonical) — real +copies on a Windows machine without the symlink privilege (§8, extraction +fallback). Cursor additionally ships router skills (`project-workflow` + +`workflow-*` wrappers for the original 8 pipelines) because Cursor routes by +skill. ### 8.3 Skills shipped @@ -2024,7 +2046,7 @@ What it knows how to repair: | `skills-missing` | project | Restores agent skills that `skills-lock.json` records but `.agents/skills/` lost. | | `pi-skill-dupes` | project | Deletes skill copies duplicated into `.pi/skills/` (the "Skill conflicts" panel at every Pi launch — §6.2). | | `runtime-missing` | project | Restores FIA runtime files the stamp manifest recorded and the disk no longer has. | -| `harness-missing` | project | Re-downloads the harness from the community API and copies back ONLY the paths `imp/.harness-manifest.json` lists as missing. Dangling symlinks are re-pointed at the stamped target; a path the current harness no longer ships is reported, not invented. | +| `harness-missing` | project | Re-downloads the harness from the community API and copies back ONLY the paths `imp/.harness-manifest.json` lists as missing. Dangling symlinks are re-pointed at the stamped target (or materialized as a copy where the OS denies links — §8); a path the current harness no longer ships is reported, not invented. | | `agents-md-block` | project | Re-appends the harness block to `AGENTS.md` (or recreates the file) via the same idempotent marker merge the installer uses — your own content is kept. | ```bash diff --git a/src/lib/harness-manifest.js b/src/lib/harness-manifest.js index 100cb06..cc9b2b2 100644 --- a/src/lib/harness-manifest.js +++ b/src/lib/harness-manifest.js @@ -30,6 +30,11 @@ export function sha1(content) { * target's content. * @returns {Promise>} rel path → sha1 | `link:` */ +// readlink on Windows reports the stored target with backslashes — normalize +// to forward slashes so manifests are portable across OSes and a link stamped +// on one machine still classifies as pristine on another. +const linkTarget = (target) => String(target).replaceAll('\\', '/'); + export async function collectHarnessManifest(cloneDir) { const files = {}; async function walk(rel) { @@ -38,7 +43,7 @@ export async function collectHarnessManifest(cloneDir) { const entryRel = rel ? `${rel}/${entry.name}` : entry.name; const full = join(cloneDir, entryRel); if (entry.isSymbolicLink()) { - files[entryRel] = `link:${await readlink(full)}`; + files[entryRel] = `link:${linkTarget(await readlink(full))}`; } else if (entry.isDirectory()) { await walk(entryRel); } else { @@ -87,7 +92,7 @@ export async function classifyHarnessState(manifest, dir) { } if (String(expected).startsWith('link:')) { const target = st.isSymbolicLink() ? await readlink(dest).catch(() => null) : null; - if (target === String(expected).slice(5)) pristine++; + if (target != null && linkTarget(target) === String(expected).slice(5)) pristine++; else modified.push(rel); continue; } diff --git a/src/lib/tar-extract.js b/src/lib/tar-extract.js new file mode 100644 index 0000000..9d57d59 --- /dev/null +++ b/src/lib/tar-extract.js @@ -0,0 +1,217 @@ +// Pure-JS .tar.gz extractor — the fallback when the system `tar` cannot unpack +// a downloaded template/harness tarball. The case that motivates it: Windows' +// bundled tar (bsdtar) refuses to CREATE SYMLINKS unless the process holds the +// symlink privilege (admin shell or Developer Mode) — on a typical student +// machine every one of the harness' 50+ mirror links (.agents/* and +// .cursor/agents/* pointing into .claude/ and .cursor/) died with +// "Can't create '…': Invalid argument", tar exited 1 and the installer +// mislabeled a perfectly good download as corrupted. Here a symlink that +// cannot be created is MATERIALIZED instead: the resolved target is copied in +// its place, so every engine still finds real content at the mirrored path. +// +// Scope: the GitHub codeload tarballs the community API serves (`git archive` +// pax format). Supported entries: regular files, directories, symlinks, +// hardlinks, pax extended headers (x/g) and GNU long name/link (L/K). +// Anything else (fifo, devices) is skipped. Header checksums are not +// validated — gzip's own CRC already covers download integrity. + +import { cpSync, mkdirSync, readFileSync, rmSync, statSync, symlinkSync, writeFileSync } from 'node:fs'; +import { gunzipSync } from 'node:zlib'; +import { dirname, resolve, sep } from 'node:path'; + +const BLOCK = 512; + +/** Numeric tar field: octal text, or GNU base-256 when the high bit is set. */ +function parseNumeric(buf) { + if (buf[0] & 0x80) { + let v = buf[0] & 0x7f; + for (let i = 1; i < buf.length; i++) v = v * 256 + buf[i]; + return v; + } + const s = buf.toString('ascii').replace(/\0/g, '').trim(); + return s ? parseInt(s, 8) : 0; +} + +/** NUL-terminated string field. */ +function stringField(block, start, len) { + const raw = block.subarray(start, start + len); + const nul = raw.indexOf(0); + return raw.subarray(0, nul === -1 ? raw.length : nul).toString('utf8'); +} + +/** pax extended-header body: a sequence of " =\n" records. */ +function parsePaxRecords(body) { + const out = {}; + let i = 0; + while (i < body.length) { + const sp = body.indexOf(0x20, i); + if (sp === -1) break; + const len = parseInt(body.subarray(i, sp).toString('ascii'), 10); + if (!Number.isFinite(len) || len <= 0 || i + len > body.length) break; + const record = body.subarray(sp + 1, i + len - 1).toString('utf8'); // drops the trailing \n + const eq = record.indexOf('='); + if (eq !== -1) out[record.slice(0, eq)] = record.slice(eq + 1); + i += len; + } + return out; +} + +/** + * Iterate the entries of an (uncompressed) tar buffer. Metadata entries + * (pax x/g, GNU L/K) are folded into the entry they describe and never + * yielded themselves. + * @param {Buffer} tar + * @yields {{name: string, type: string, linkname: string, mode: number, body: Buffer}} + */ +export function* tarEntries(tar) { + let offset = 0; + let overrides = null; // accumulated pax/GNU metadata for the NEXT real entry + while (offset + BLOCK <= tar.length) { + const block = tar.subarray(offset, offset + BLOCK); + if (block.every((b) => b === 0)) break; // end-of-archive marker + const size = parseNumeric(block.subarray(124, 136)); + const body = tar.subarray(offset + BLOCK, offset + BLOCK + size); + offset += BLOCK + Math.ceil(size / BLOCK) * BLOCK; + + const type = block[156] === 0 ? '0' : String.fromCharCode(block[156]); + if (type === 'x') { + overrides = { ...overrides, ...parsePaxRecords(body) }; + continue; + } + if (type === 'g') continue; // global pax header (git archive's commit comment) + if (type === 'L' || type === 'K') { + const text = body.subarray(0, body.indexOf(0) === -1 ? body.length : body.indexOf(0)).toString('utf8'); + overrides = { ...overrides, [type === 'L' ? 'path' : 'linkpath']: text }; + continue; + } + + let name = stringField(block, 0, 100); + // The prefix field only exists in ustar-family headers. + if (stringField(block, 257, 6).startsWith('ustar')) { + const prefix = stringField(block, 345, 155); + if (prefix) name = `${prefix}/${name}`; + } + let linkname = stringField(block, 157, 100); + if (overrides?.path) name = overrides.path; + if (overrides?.linkpath) linkname = overrides.linkpath; + overrides = null; + yield { name, type, linkname, mode: parseNumeric(block.subarray(100, 108)), body }; + } +} + +/** + * Extract `tgzPath` into `destDir`, dropping the first `strip` path components + * (the GitHub tarball root `--/`), like + * `tar -xzf … --strip-components=1`. Existing files are overwritten (the + * caller may be retrying after a partial system-tar run). + * + * Symlinks: a real link is attempted first (with the right dir/file type for + * Windows). The FIRST failure flips the whole run to materialization — every + * remaining link becomes a deep copy of its resolved target — because the + * failure means the machine cannot create symlinks at all, and mixing links + * with copies would leave the tree half-mirrored. Targets are resolved inside + * the archive only; a link pointing outside `destDir` is created verbatim when + * possible and skipped (reported) otherwise — never dereferenced. + * + * @param {string} tgzPath + * @param {string} destDir + * @param {{strip?: number, makeSymlink?: typeof symlinkSync}} [opts] + * `makeSymlink` is a test seam to simulate a symlink-incapable machine. + * @returns {{files: number, links: number, materialized: boolean, skipped: string[]}} + */ +export function extractTarGz(tgzPath, destDir, opts = {}) { + const { strip = 1, makeSymlink = symlinkSync } = opts; + const tar = gunzipSync(readFileSync(tgzPath)); + const dest = resolve(destDir); + const inside = (abs) => abs === dest || abs.startsWith(dest + sep); + + // Archive path → absolute destination (or null for entries the strip eats). + const destPathOf = (rawName) => { + const parts = String(rawName) + .split('/') + .filter((p) => p && p !== '.'); + if (parts.some((p) => p === '..')) throw new Error(`unsafe path in archive: ${rawName}`); + if (parts.length <= strip) return null; + const abs = resolve(dest, parts.slice(strip).join('/')); + if (!inside(abs)) throw new Error(`unsafe path in archive: ${rawName}`); + return abs; + }; + + const links = []; + let files = 0; + for (const entry of tarEntries(tar)) { + const target = destPathOf(entry.name); + if (target === null) continue; + if (entry.type === '5' || entry.name.endsWith('/')) { + mkdirSync(target, { recursive: true }); + } else if (entry.type === '2' || entry.type === '1') { + links.push({ dest: target, linkname: entry.linkname, hard: entry.type === '1' }); + } else if (entry.type === '0') { + mkdirSync(dirname(target), { recursive: true }); + rmSync(target, { recursive: true, force: true }); + writeFileSync(target, entry.body, { mode: entry.mode || 0o644 }); + files++; + } + // Anything else (fifo/devices) never appears in git archives — skipped. + } + + // Links go LAST (their targets must exist), in passes: a link whose target + // is another still-pending link is deferred to the next round. + let cannotLink = false; + const skipped = []; + let pending = links; + let progress = true; + while (pending.length && progress) { + progress = false; + const next = []; + for (const link of pending) { + // Hardlink names are archive paths; symlink names are relative to the link. + const target = link.hard ? destPathOf(link.linkname) : resolve(dirname(link.dest), link.linkname); + const confined = target !== null && inside(target); + let stat = null; + if (confined) { + try { + stat = statSync(target); + } catch { + stat = null; // target not materialized yet (or dangling) — retry later + } + } + if (confined && !stat) { + next.push(link); + continue; + } + mkdirSync(dirname(link.dest), { recursive: true }); + rmSync(link.dest, { recursive: true, force: true }); + if (!link.hard && !cannotLink) { + try { + makeSymlink(link.linkname, link.dest, stat?.isDirectory() ? 'dir' : 'file'); + files++; + progress = true; + continue; + } catch { + cannotLink = true; // this machine cannot create symlinks — materialize all + } + } + if (!confined || !stat) { + skipped.push(link.dest); // points outside the archive and cannot be linked + progress = true; + continue; + } + cpSync(target, link.dest, { recursive: true, dereference: true }); + files++; + progress = true; + } + pending = next; + } + // Whatever is left points at a target the archive never shipped (dangling). + for (const link of pending) { + try { + makeSymlink(link.linkname, link.dest, 'file'); + files++; + } catch { + skipped.push(link.dest); + } + } + + return { files, links: links.length, materialized: cannotLink, skipped }; +} diff --git a/src/lib/template-fetch.js b/src/lib/template-fetch.js index 13e09a9..2c0c36d 100644 --- a/src/lib/template-fetch.js +++ b/src/lib/template-fetch.js @@ -9,8 +9,39 @@ import { mkdir, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { has, run } from './proc.js'; +import { extractTarGz } from './tar-extract.js'; import { downloadTemplate } from './auth-client.js'; +/** + * Unpack a downloaded tarball into `destDir`. The system `tar` goes first + * (fast, battle-tested, preserves symlinks where the OS allows them); when it + * is missing OR fails, the built-in extractor takes over — the one failure + * that matters in practice is Windows' bundled tar dying on the harness' + * symlink entries ("Invalid argument") because creating links needs a + * privilege students don't have, and the built-in extractor materializes + * those links as copies instead. `deps` is a test seam ({ run, has }). + * @returns {Promise<{ok: boolean, reason?: string}>} + */ +export async function extractDownloadedTarball(tgz, destDir, deps = {}) { + const exec = deps.run ?? run; + const hasCmd = deps.has ?? has; + await mkdir(destDir, { recursive: true }); + if (await hasCmd('tar')) { + const r = await exec('tar', ['-xzf', tgz, '-C', destDir, '--strip-components=1']); + if (r.ok) return { ok: true }; + } + try { + // Start clean: a failed system-tar run leaves a partial tree behind. + await rm(destDir, { recursive: true, force: true }); + await mkdir(destDir, { recursive: true }); + extractTarGz(tgz, destDir, { strip: 1 }); + return { ok: true }; + } catch { + // Both extractors refused the bytes — the download really is corrupted. + return { ok: false, reason: 'extract_failed' }; + } +} + /** * Downloads the template `name` (live1 | live2 | harness) and extracts it into * `destDir`. The GitHub tarball ships a root directory `--/`, @@ -23,16 +54,7 @@ export async function fetchTemplateToDir(apiBase, token, name, destDir, ref) { try { const dl = await downloadTemplate(apiBase, token, name, tgz, ref); if (!dl.ok) return { ok: false, reason: dl.reason }; - - await mkdir(destDir, { recursive: true }); - const r = await run('tar', ['-xzf', tgz, '-C', destDir, '--strip-components=1']); - if (!r.ok) { - // A missing `tar` binary is NOT transient — distinguish it from a - // corrupted download so the message tells the right recovery step. - if (!(await has('tar'))) return { ok: false, reason: 'tar_missing' }; - return { ok: false, reason: 'extract_failed' }; - } - return { ok: true }; + return await extractDownloadedTarball(tgz, destDir); } finally { await rm(tmpRoot, { recursive: true, force: true }); } @@ -50,9 +72,8 @@ export async function fetchTemplateToDir(apiBase, token, name, destDir, ref) { * GitHub token that fetches the private repos). Saying "try again in a * moment" here makes the student retry forever an error only the * maintainer can fix; - * tar_missing → the `tar` program is absent on the machine — retrying - * won't help either; the fix is installing it; - * extract_failed → the download arrived corrupted — one re-download is + * extract_failed → neither the system tar nor the built-in extractor could + * unpack the bytes — the download arrived corrupted; one re-download is * worth trying, then support; * download_timeout → the connection dropped mid-download; * network_error → the server could not be reached at all (DNS, refused, @@ -80,15 +101,6 @@ export function downloadErrorMessage(what, reason) { 'contact the community support and try again later.', ].join('\n'); } - if (reason === 'tar_missing') { - return [ - `The "tar" program was not found on this computer — it is needed to unpack the ${what}.`, - 'macOS: run xcode-select --install to restore the command line tools.', - 'Windows: tar ships with Windows 10 and newer — update Windows.', - 'Linux: install it with your package manager (e.g. sudo apt install tar ).', - 'Then run the same command again.', - ].join('\n'); - } if (reason === 'extract_failed') { return [ `The downloaded ${what} file arrived corrupted and could not be unpacked.`, diff --git a/src/steps/fix.js b/src/steps/fix.js index adcd31c..4b35eea 100644 --- a/src/steps/fix.js +++ b/src/steps/fix.js @@ -25,7 +25,7 @@ import { createInterface } from 'node:readline/promises'; import { existsSync, mkdtempSync } from 'node:fs'; import { cp, mkdir, readFile, rename, rm, symlink, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; -import { dirname, join } from 'node:path'; +import { dirname, join, resolve } from 'node:path'; import pc from 'picocolors'; import { HARNESS } from '../config.js'; import { run } from '../lib/proc.js'; @@ -227,7 +227,15 @@ export async function collectFixPlan({ cwd = process.cwd(), probes = {} } = {}) // A dangling link at dest would have classified as missing — // remove the husk before recreating, then link to the stamped target. await rm(dest, { force: true }); - await symlink(entry.slice(5), dest); + try { + await symlink(entry.slice(5), dest); + } catch { + // No symlink privilege (typical on Windows) — materialize the + // mirror instead, same degradation the installer's extractor + // applies: the fresh clone has real content at this path. + const source = existsSync(join(cloneDir, rel)) ? join(cloneDir, rel) : resolve(dirname(dest), entry.slice(5)); + await cp(source, dest, { recursive: true, dereference: true }); + } restored++; } else if (existsSync(join(cloneDir, rel))) { await rm(dest, { force: true }); diff --git a/test/tar-extract.test.js b/test/tar-extract.test.js new file mode 100644 index 0000000..954a710 --- /dev/null +++ b/test/tar-extract.test.js @@ -0,0 +1,179 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, existsSync, lstatSync, statSync } from 'node:fs'; +import { gzipSync } from 'node:zlib'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +const { extractTarGz, tarEntries } = await import('../src/lib/tar-extract.js'); + +// ── Hand-rolled tar builder ────────────────────────────────────────────────── +// Building the bytes by hand keeps these tests deterministic on every OS: no +// system tar, and no symlink privilege needed to CREATE the fixture (the whole +// point of the extractor is machines where that privilege is missing). + +function headerBlock(name, { size = 0, type = '0', linkname = '', mode = 0o644 } = {}) { + const b = Buffer.alloc(512); + b.write(name, 0, 100, 'utf8'); + b.write(mode.toString(8).padStart(7, '0') + '\0', 100); + b.write('0000000\0', 108); // uid + b.write('0000000\0', 116); // gid + b.write(size.toString(8).padStart(11, '0') + '\0', 124); + b.write('00000000000\0', 136); // mtime + b.fill(0x20, 148, 156); // checksum field counts as spaces while summing + b.write(type, 156); + if (linkname) b.write(linkname, 157, 100, 'utf8'); + b.write('ustar\0', 257); + b.write('00', 263); + let sum = 0; + for (const byte of b) sum += byte; + b.write(sum.toString(8).padStart(6, '0') + '\0 ', 148); + return b; +} + +function entry(name, content = '', opts = {}) { + const body = Buffer.isBuffer(content) ? content : Buffer.from(content, 'utf8'); + const blocks = [headerBlock(name, { ...opts, size: body.length })]; + if (body.length) { + const padded = Buffer.alloc(Math.ceil(body.length / 512) * 512); + body.copy(padded); + blocks.push(padded); + } + return blocks; +} + +/** pax body: " =\n" where len counts the whole record. */ +function paxBody(records) { + let out = ''; + for (const [k, v] of Object.entries(records)) { + const base = `${k}=${v}\n`; + let len = base.length + 2; + while (String(len).length + 1 + base.length !== len) len++; + out += `${len} ${base}`; + } + return out; +} + +function writeTgz(entries) { + const dir = mkdtempSync(join(tmpdir(), 'impactus-tarx-')); + const tgz = join(dir, 'fixture.tar.gz'); + writeFileSync(tgz, gzipSync(Buffer.concat([...entries.flat(), Buffer.alloc(1024)]))); + return { tgz, out: join(dir, 'out') }; +} + +const failingSymlink = () => { + const err = new Error('EPERM: operation not permitted (simulated Windows)'); + err.code = 'EPERM'; + throw err; +}; + +// ── Extraction basics ──────────────────────────────────────────────────────── + +test('extractTarGz: files and dirs land stripped of the tarball root', () => { + const { tgz, out } = writeTgz([ + entry('owner-repo-sha/', '', { type: '5' }), + entry('owner-repo-sha/package.json', '{"name":"x"}\n'), + entry('owner-repo-sha/src/', '', { type: '5' }), + entry('owner-repo-sha/src/deep/nested.txt', 'deep\n'), + ]); + const res = extractTarGz(tgz, out); + assert.equal(readFileSync(join(out, 'package.json'), 'utf8'), '{"name":"x"}\n'); + assert.equal(readFileSync(join(out, 'src', 'deep', 'nested.txt'), 'utf8'), 'deep\n'); + assert.ok(!existsSync(join(out, 'owner-repo-sha'))); + assert.equal(res.materialized, false); + assert.deepEqual(res.skipped, []); +}); + +test('extractTarGz: symlinks become real links when the machine allows them', () => { + const { tgz, out } = writeTgz([ + entry('r/a.txt', 'target content\n'), + entry('r/sub/inner.txt', 'inner\n'), + entry('r/link.txt', '', { type: '2', linkname: 'a.txt' }), + entry('r/dirlink', '', { type: '2', linkname: 'sub' }), + ]); + extractTarGz(tgz, out); + // Content through the link is the universal contract (link or copy). + assert.equal(readFileSync(join(out, 'link.txt'), 'utf8'), 'target content\n'); + assert.equal(readFileSync(join(out, 'dirlink', 'inner.txt'), 'utf8'), 'inner\n'); + if (process.platform !== 'win32') { + assert.ok(lstatSync(join(out, 'link.txt')).isSymbolicLink()); + assert.ok(lstatSync(join(out, 'dirlink')).isSymbolicLink()); + } +}); + +test('extractTarGz: symlink failure materializes EVERY link as a copy (the Windows path)', () => { + const { tgz, out } = writeTgz([ + entry('r/real.txt', 'X\n'), + entry('r/skills/tdd/SKILL.md', '# tdd\n'), + // Chain out of order on purpose: l1 → l2 → real.txt forces the retry pass. + entry('r/l1', '', { type: '2', linkname: 'l2' }), + entry('r/l2', '', { type: '2', linkname: 'real.txt' }), + entry('r/mirror', '', { type: '2', linkname: 'skills/tdd' }), + ]); + const res = extractTarGz(tgz, out, { makeSymlink: failingSymlink }); + assert.equal(res.materialized, true); + assert.deepEqual(res.skipped, []); + for (const p of ['l1', 'l2']) { + assert.ok(!lstatSync(join(out, p)).isSymbolicLink(), `${p} must be a real file`); + assert.equal(readFileSync(join(out, p), 'utf8'), 'X\n'); + } + assert.ok(statSync(join(out, 'mirror')).isDirectory()); + assert.equal(readFileSync(join(out, 'mirror', 'SKILL.md'), 'utf8'), '# tdd\n'); +}); + +test('extractTarGz: pax headers (git archive format) — global skipped, path override honored', () => { + const longName = 'r/' + 'very-long-directory-name/'.repeat(5) + 'file.txt'; + const { tgz, out } = writeTgz([ + entry('pax_global_header', paxBody({ comment: 'abc123' }), { type: 'g' }), + entry('pax-x-0', paxBody({ path: longName }), { type: 'x' }), + entry('r/_truncated_placeholder', 'via pax\n'), + entry('r/plain.txt', 'plain\n'), + ]); + extractTarGz(tgz, out); + assert.equal(readFileSync(join(out, longName.slice(2)), 'utf8'), 'via pax\n'); + assert.ok(!existsSync(join(out, '_truncated_placeholder'))); + assert.equal(readFileSync(join(out, 'plain.txt'), 'utf8'), 'plain\n'); +}); + +test('extractTarGz: GNU longname (L) and hardlinks resolve to real content', () => { + const gnuName = 'r/gnu/' + 'x'.repeat(120) + '.txt'; + const { tgz, out } = writeTgz([ + entry('././@LongLink', gnuName + '\0', { type: 'L' }), + entry('r/gnu/_short', 'gnu long\n'), + entry('r/orig.txt', 'HH\n'), + entry('r/hard.txt', '', { type: '1', linkname: 'r/orig.txt' }), + ]); + extractTarGz(tgz, out); + assert.equal(readFileSync(join(out, gnuName.slice(2)), 'utf8'), 'gnu long\n'); + assert.equal(readFileSync(join(out, 'hard.txt'), 'utf8'), 'HH\n'); +}); + +test('extractTarGz: path traversal in the archive throws instead of writing outside', () => { + const { tgz, out } = writeTgz([entry('r/../../evil.txt', 'nope\n')]); + assert.throws(() => extractTarGz(tgz, out), /unsafe path/); +}); + +test('extractTarGz: overwrites leftovers from a previous partial extraction', () => { + const { tgz, out } = writeTgz([entry('r/keep.txt', 'fresh\n')]); + mkdirSync(out, { recursive: true }); + writeFileSync(join(out, 'keep.txt'), 'stale partial copy\n'); + extractTarGz(tgz, out); + assert.equal(readFileSync(join(out, 'keep.txt'), 'utf8'), 'fresh\n'); +}); + +test('extractTarGz: executable bit survives (scripts must stay runnable)', { skip: process.platform === 'win32' }, () => { + const { tgz, out } = writeTgz([entry('r/hook.sh', '#!/bin/sh\n', { mode: 0o755 })]); + extractTarGz(tgz, out); + assert.ok(statSync(join(out, 'hook.sh')).mode & 0o100, 'owner-executable'); +}); + +test('tarEntries: folds pax overrides into the next entry only', () => { + const bytes = Buffer.concat([ + ...entry('pax-x', paxBody({ path: 'renamed.txt' }), { type: 'x' }), + ...entry('original.txt', 'a'), + ...entry('second.txt', 'b'), + Buffer.alloc(1024), + ]); + const names = [...tarEntries(bytes)].map((e) => e.name); + assert.deepEqual(names, ['renamed.txt', 'second.txt']); +}); diff --git a/test/template-fetch.test.js b/test/template-fetch.test.js index b7c89bb..670fec1 100644 --- a/test/template-fetch.test.js +++ b/test/template-fetch.test.js @@ -2,18 +2,19 @@ import { test, before, after } from 'node:test'; import assert from 'node:assert/strict'; import { createServer } from 'node:http'; import { execFileSync } from 'node:child_process'; -import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, existsSync } from 'node:fs'; +import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, existsSync, symlinkSync } from 'node:fs'; import { readFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -const { downloadErrorMessage, fetchTemplateToDir } = await import('../src/lib/template-fetch.js'); +const { downloadErrorMessage, extractDownloadedTarball, fetchTemplateToDir } = await import('../src/lib/template-fetch.js'); // Builds a tarball in the GitHub format (root dir --/…) and // serves its bytes — proves the extraction does strip-components=1 right. let server; let base; let tarballBytes; +let fixtureHasSymlink = false; before(async () => { const work = mkdtempSync(join(tmpdir(), 'create-iai-tar-')); @@ -21,6 +22,15 @@ before(async () => { mkdirSync(join(top, 'convex'), { recursive: true }); writeFileSync(join(top, 'package.json'), '{"name":"live1"}\n'); writeFileSync(join(top, 'convex', 'schema.ts'), 'export default {}\n'); + try { + // Mirror link like the harness ships (.agents/* → .claude/*). Guarded: + // on a Windows box without the symlink privilege the fixture just goes + // without it (the dedicated tar-extract tests cover materialization). + symlinkSync('package.json', join(top, 'link.json')); + fixtureHasSymlink = true; + } catch { + fixtureHasSymlink = false; + } const tgz = join(work, 'repo.tar.gz'); execFileSync('tar', ['-czf', tgz, '-C', work, 'elberrd-live1-abc123']); tarballBytes = readFileSync(tgz); @@ -58,6 +68,47 @@ test('fetchTemplateToDir: downloads and extracts without the tarball root direct assert.ok(existsSync(join(dest, 'package.json'))); assert.ok(existsSync(join(dest, 'convex', 'schema.ts'))); assert.equal(JSON.parse(await readFile(join(dest, 'package.json'), 'utf8')).name, 'live1'); + if (fixtureHasSymlink) { + // Whether the OS gave us a real link or the extractor materialized a + // copy, the mirrored path must resolve to the target's content. + assert.equal(JSON.parse(await readFile(join(dest, 'link.json'), 'utf8')).name, 'live1'); + } +}); + +test('extractDownloadedTarball: no system tar → built-in extractor takes over', async () => { + const work = mkdtempSync(join(tmpdir(), 'create-iai-fallback-')); + const tgz = join(work, 'repo.tar.gz'); + writeFileSync(tgz, tarballBytes); + const dest = join(work, 'out'); + const res = await extractDownloadedTarball(tgz, dest, { has: async () => false }); + assert.equal(res.ok, true); + assert.ok(existsSync(join(dest, 'convex', 'schema.ts'))); +}); + +test('extractDownloadedTarball: system tar fails (Windows symlink privilege) → clean re-extract', async () => { + const work = mkdtempSync(join(tmpdir(), 'create-iai-fallback-')); + const tgz = join(work, 'repo.tar.gz'); + writeFileSync(tgz, tarballBytes); + const dest = join(work, 'out'); + // Simulate bsdtar's partial run: some files landed, then exit 1 on a link. + mkdirSync(dest, { recursive: true }); + writeFileSync(join(dest, 'partial-leftover.txt'), 'from the failed tar run\n'); + const res = await extractDownloadedTarball(tgz, dest, { + has: async () => true, + run: async () => ({ ok: false, exitCode: 1, stdout: '', stderr: 'Invalid argument' }), + }); + assert.equal(res.ok, true); + assert.ok(existsSync(join(dest, 'package.json'))); + assert.ok(!existsSync(join(dest, 'partial-leftover.txt')), 'partial tar output must be wiped'); +}); + +test('extractDownloadedTarball: both extractors refuse the bytes → extract_failed', async () => { + const work = mkdtempSync(join(tmpdir(), 'create-iai-fallback-')); + const tgz = join(work, 'repo.tar.gz'); + writeFileSync(tgz, 'this is not a tarball'); + const res = await extractDownloadedTarball(tgz, join(work, 'out'), { has: async () => false }); + assert.equal(res.ok, false); + assert.equal(res.reason, 'extract_failed'); }); test('fetchTemplateToDir: null token sends NO Authorization header (guest harness download)', async () => { @@ -105,13 +156,7 @@ test('fetchTemplateToDir: corrupted tarball → extract_failed (tar is present)' assert.equal(res.reason, 'extract_failed'); }); -test('downloadErrorMessage: tar missing vs corrupted download vs stalled connection', () => { - // Missing tar is not transient — the message says how to install it. - const missing = downloadErrorMessage('template', 'tar_missing'); - assert.match(missing, /"tar" program was not found/); - assert.match(missing, /xcode-select --install/); - assert.doesNotMatch(missing, /Try again in a moment/); - +test('downloadErrorMessage: corrupted download vs stalled connection', () => { // Corrupted download → suggest ONE re-download, then support. const corrupt = downloadErrorMessage('template', 'extract_failed'); assert.match(corrupt, /corrupted/);