|
| 1 | +// Pure-JS .tar.gz extractor — the fallback when the system `tar` cannot unpack |
| 2 | +// a downloaded template/harness tarball. The case that motivates it: Windows' |
| 3 | +// bundled tar (bsdtar) refuses to CREATE SYMLINKS unless the process holds the |
| 4 | +// symlink privilege (admin shell or Developer Mode) — on a typical student |
| 5 | +// machine every one of the harness' 50+ mirror links (.agents/* and |
| 6 | +// .cursor/agents/* pointing into .claude/ and .cursor/) died with |
| 7 | +// "Can't create '…': Invalid argument", tar exited 1 and the installer |
| 8 | +// mislabeled a perfectly good download as corrupted. Here a symlink that |
| 9 | +// cannot be created is MATERIALIZED instead: the resolved target is copied in |
| 10 | +// its place, so every engine still finds real content at the mirrored path. |
| 11 | +// |
| 12 | +// Scope: the GitHub codeload tarballs the community API serves (`git archive` |
| 13 | +// pax format). Supported entries: regular files, directories, symlinks, |
| 14 | +// hardlinks, pax extended headers (x/g) and GNU long name/link (L/K). |
| 15 | +// Anything else (fifo, devices) is skipped. Header checksums are not |
| 16 | +// validated — gzip's own CRC already covers download integrity. |
| 17 | + |
| 18 | +import { cpSync, mkdirSync, readFileSync, rmSync, statSync, symlinkSync, writeFileSync } from 'node:fs'; |
| 19 | +import { gunzipSync } from 'node:zlib'; |
| 20 | +import { dirname, resolve, sep } from 'node:path'; |
| 21 | + |
| 22 | +const BLOCK = 512; |
| 23 | + |
| 24 | +/** Numeric tar field: octal text, or GNU base-256 when the high bit is set. */ |
| 25 | +function parseNumeric(buf) { |
| 26 | + if (buf[0] & 0x80) { |
| 27 | + let v = buf[0] & 0x7f; |
| 28 | + for (let i = 1; i < buf.length; i++) v = v * 256 + buf[i]; |
| 29 | + return v; |
| 30 | + } |
| 31 | + const s = buf.toString('ascii').replace(/\0/g, '').trim(); |
| 32 | + return s ? parseInt(s, 8) : 0; |
| 33 | +} |
| 34 | + |
| 35 | +/** NUL-terminated string field. */ |
| 36 | +function stringField(block, start, len) { |
| 37 | + const raw = block.subarray(start, start + len); |
| 38 | + const nul = raw.indexOf(0); |
| 39 | + return raw.subarray(0, nul === -1 ? raw.length : nul).toString('utf8'); |
| 40 | +} |
| 41 | + |
| 42 | +/** pax extended-header body: a sequence of "<len> <key>=<value>\n" records. */ |
| 43 | +function parsePaxRecords(body) { |
| 44 | + const out = {}; |
| 45 | + let i = 0; |
| 46 | + while (i < body.length) { |
| 47 | + const sp = body.indexOf(0x20, i); |
| 48 | + if (sp === -1) break; |
| 49 | + const len = parseInt(body.subarray(i, sp).toString('ascii'), 10); |
| 50 | + if (!Number.isFinite(len) || len <= 0 || i + len > body.length) break; |
| 51 | + const record = body.subarray(sp + 1, i + len - 1).toString('utf8'); // drops the trailing \n |
| 52 | + const eq = record.indexOf('='); |
| 53 | + if (eq !== -1) out[record.slice(0, eq)] = record.slice(eq + 1); |
| 54 | + i += len; |
| 55 | + } |
| 56 | + return out; |
| 57 | +} |
| 58 | + |
| 59 | +/** |
| 60 | + * Iterate the entries of an (uncompressed) tar buffer. Metadata entries |
| 61 | + * (pax x/g, GNU L/K) are folded into the entry they describe and never |
| 62 | + * yielded themselves. |
| 63 | + * @param {Buffer} tar |
| 64 | + * @yields {{name: string, type: string, linkname: string, mode: number, body: Buffer}} |
| 65 | + */ |
| 66 | +export function* tarEntries(tar) { |
| 67 | + let offset = 0; |
| 68 | + let overrides = null; // accumulated pax/GNU metadata for the NEXT real entry |
| 69 | + while (offset + BLOCK <= tar.length) { |
| 70 | + const block = tar.subarray(offset, offset + BLOCK); |
| 71 | + if (block.every((b) => b === 0)) break; // end-of-archive marker |
| 72 | + const size = parseNumeric(block.subarray(124, 136)); |
| 73 | + const body = tar.subarray(offset + BLOCK, offset + BLOCK + size); |
| 74 | + offset += BLOCK + Math.ceil(size / BLOCK) * BLOCK; |
| 75 | + |
| 76 | + const type = block[156] === 0 ? '0' : String.fromCharCode(block[156]); |
| 77 | + if (type === 'x') { |
| 78 | + overrides = { ...overrides, ...parsePaxRecords(body) }; |
| 79 | + continue; |
| 80 | + } |
| 81 | + if (type === 'g') continue; // global pax header (git archive's commit comment) |
| 82 | + if (type === 'L' || type === 'K') { |
| 83 | + const text = body.subarray(0, body.indexOf(0) === -1 ? body.length : body.indexOf(0)).toString('utf8'); |
| 84 | + overrides = { ...overrides, [type === 'L' ? 'path' : 'linkpath']: text }; |
| 85 | + continue; |
| 86 | + } |
| 87 | + |
| 88 | + let name = stringField(block, 0, 100); |
| 89 | + // The prefix field only exists in ustar-family headers. |
| 90 | + if (stringField(block, 257, 6).startsWith('ustar')) { |
| 91 | + const prefix = stringField(block, 345, 155); |
| 92 | + if (prefix) name = `${prefix}/${name}`; |
| 93 | + } |
| 94 | + let linkname = stringField(block, 157, 100); |
| 95 | + if (overrides?.path) name = overrides.path; |
| 96 | + if (overrides?.linkpath) linkname = overrides.linkpath; |
| 97 | + overrides = null; |
| 98 | + yield { name, type, linkname, mode: parseNumeric(block.subarray(100, 108)), body }; |
| 99 | + } |
| 100 | +} |
| 101 | + |
| 102 | +/** |
| 103 | + * Extract `tgzPath` into `destDir`, dropping the first `strip` path components |
| 104 | + * (the GitHub tarball root `<owner>-<repo>-<sha>/`), like |
| 105 | + * `tar -xzf … --strip-components=1`. Existing files are overwritten (the |
| 106 | + * caller may be retrying after a partial system-tar run). |
| 107 | + * |
| 108 | + * Symlinks: a real link is attempted first (with the right dir/file type for |
| 109 | + * Windows). The FIRST failure flips the whole run to materialization — every |
| 110 | + * remaining link becomes a deep copy of its resolved target — because the |
| 111 | + * failure means the machine cannot create symlinks at all, and mixing links |
| 112 | + * with copies would leave the tree half-mirrored. Targets are resolved inside |
| 113 | + * the archive only; a link pointing outside `destDir` is created verbatim when |
| 114 | + * possible and skipped (reported) otherwise — never dereferenced. |
| 115 | + * |
| 116 | + * @param {string} tgzPath |
| 117 | + * @param {string} destDir |
| 118 | + * @param {{strip?: number, makeSymlink?: typeof symlinkSync}} [opts] |
| 119 | + * `makeSymlink` is a test seam to simulate a symlink-incapable machine. |
| 120 | + * @returns {{files: number, links: number, materialized: boolean, skipped: string[]}} |
| 121 | + */ |
| 122 | +export function extractTarGz(tgzPath, destDir, opts = {}) { |
| 123 | + const { strip = 1, makeSymlink = symlinkSync } = opts; |
| 124 | + const tar = gunzipSync(readFileSync(tgzPath)); |
| 125 | + const dest = resolve(destDir); |
| 126 | + const inside = (abs) => abs === dest || abs.startsWith(dest + sep); |
| 127 | + |
| 128 | + // Archive path → absolute destination (or null for entries the strip eats). |
| 129 | + const destPathOf = (rawName) => { |
| 130 | + const parts = String(rawName) |
| 131 | + .split('/') |
| 132 | + .filter((p) => p && p !== '.'); |
| 133 | + if (parts.some((p) => p === '..')) throw new Error(`unsafe path in archive: ${rawName}`); |
| 134 | + if (parts.length <= strip) return null; |
| 135 | + const abs = resolve(dest, parts.slice(strip).join('/')); |
| 136 | + if (!inside(abs)) throw new Error(`unsafe path in archive: ${rawName}`); |
| 137 | + return abs; |
| 138 | + }; |
| 139 | + |
| 140 | + const links = []; |
| 141 | + let files = 0; |
| 142 | + for (const entry of tarEntries(tar)) { |
| 143 | + const target = destPathOf(entry.name); |
| 144 | + if (target === null) continue; |
| 145 | + if (entry.type === '5' || entry.name.endsWith('/')) { |
| 146 | + mkdirSync(target, { recursive: true }); |
| 147 | + } else if (entry.type === '2' || entry.type === '1') { |
| 148 | + links.push({ dest: target, linkname: entry.linkname, hard: entry.type === '1' }); |
| 149 | + } else if (entry.type === '0') { |
| 150 | + mkdirSync(dirname(target), { recursive: true }); |
| 151 | + rmSync(target, { recursive: true, force: true }); |
| 152 | + writeFileSync(target, entry.body, { mode: entry.mode || 0o644 }); |
| 153 | + files++; |
| 154 | + } |
| 155 | + // Anything else (fifo/devices) never appears in git archives — skipped. |
| 156 | + } |
| 157 | + |
| 158 | + // Links go LAST (their targets must exist), in passes: a link whose target |
| 159 | + // is another still-pending link is deferred to the next round. |
| 160 | + let cannotLink = false; |
| 161 | + const skipped = []; |
| 162 | + let pending = links; |
| 163 | + let progress = true; |
| 164 | + while (pending.length && progress) { |
| 165 | + progress = false; |
| 166 | + const next = []; |
| 167 | + for (const link of pending) { |
| 168 | + // Hardlink names are archive paths; symlink names are relative to the link. |
| 169 | + const target = link.hard ? destPathOf(link.linkname) : resolve(dirname(link.dest), link.linkname); |
| 170 | + const confined = target !== null && inside(target); |
| 171 | + let stat = null; |
| 172 | + if (confined) { |
| 173 | + try { |
| 174 | + stat = statSync(target); |
| 175 | + } catch { |
| 176 | + stat = null; // target not materialized yet (or dangling) — retry later |
| 177 | + } |
| 178 | + } |
| 179 | + if (confined && !stat) { |
| 180 | + next.push(link); |
| 181 | + continue; |
| 182 | + } |
| 183 | + mkdirSync(dirname(link.dest), { recursive: true }); |
| 184 | + rmSync(link.dest, { recursive: true, force: true }); |
| 185 | + if (!link.hard && !cannotLink) { |
| 186 | + try { |
| 187 | + makeSymlink(link.linkname, link.dest, stat?.isDirectory() ? 'dir' : 'file'); |
| 188 | + files++; |
| 189 | + progress = true; |
| 190 | + continue; |
| 191 | + } catch { |
| 192 | + cannotLink = true; // this machine cannot create symlinks — materialize all |
| 193 | + } |
| 194 | + } |
| 195 | + if (!confined || !stat) { |
| 196 | + skipped.push(link.dest); // points outside the archive and cannot be linked |
| 197 | + progress = true; |
| 198 | + continue; |
| 199 | + } |
| 200 | + cpSync(target, link.dest, { recursive: true, dereference: true }); |
| 201 | + files++; |
| 202 | + progress = true; |
| 203 | + } |
| 204 | + pending = next; |
| 205 | + } |
| 206 | + // Whatever is left points at a target the archive never shipped (dangling). |
| 207 | + for (const link of pending) { |
| 208 | + try { |
| 209 | + makeSymlink(link.linkname, link.dest, 'file'); |
| 210 | + files++; |
| 211 | + } catch { |
| 212 | + skipped.push(link.dest); |
| 213 | + } |
| 214 | + } |
| 215 | + |
| 216 | + return { files, links: links.length, materialized: cannotLink, skipped }; |
| 217 | +} |
0 commit comments