Skip to content

Commit 2c15d32

Browse files
soheiokamotosoheiokamoto
committed
Fetch Logpush dataset pages from middlecache
Fetch validated Logpush dataset pages from middlecache during docs builds while retaining checked-in pages as a safe fallback. - Verify archive downloads before replacing pages - Sync generated Markdown into existing dataset scopes and remove stale pages - Abort replacements that would remove all managed pages - Refresh the archive on every development and build run - Correct the documented product metadata directory - Require a separate manual changelog when a dataset change needs a customer-facing announcement
1 parent f3c4d7d commit 2c15d32

4 files changed

Lines changed: 327 additions & 2 deletions

File tree

‎AGENTS.md‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,8 @@ cloudflare-docs/
3737
├── public/ # Static files served as-is (images, redirects, robots.txt)
3838
├── worker/ # Cloudflare Worker for serving the site
3939
├── bin/ # Build scripts and CI helpers
40-
│ └── fetch-skills.ts # Downloads skills.tar.gz from middlecache, extracts to skills/
40+
│ ├── fetch-skills.ts # Downloads skills.tar.gz from middlecache, extracts to skills/
41+
│ └── fetch-logpush-datasets.ts # Syncs generated Logpush dataset pages
4142
├── skills/ # Agent Skills served at /.well-known/skills/ — GENERATED, do not edit
4243
│ # Fetched from https://middlecache.ced.cloudflare.com/v1/cloudflare-skills/skills.tar.gz
4344
│ # by bin/fetch-skills.ts, which runs automatically via prebuild/predev hooks.

‎bin/fetch-logpush-datasets.ts‎

Lines changed: 320 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,320 @@
1+
#!/usr/bin/env tsx
2+
3+
import fs from "fs";
4+
import { spawnSync } from "node:child_process";
5+
import { createHash } from "node:crypto";
6+
import { dirname, join } from "path";
7+
import YAML from "yaml";
8+
9+
import {
10+
downloadToDotTempIfNotPresent,
11+
extractTarGz,
12+
getDotTmpPath,
13+
} from "../src/util/custom-loaders";
14+
15+
const MIDDLECACHE_BASE_URL = `${(
16+
process.env.MIDDLECACHE_BASE_URL ?? "https://middlecache.ced.cloudflare.com"
17+
).replace(/\/+$/, "")}/`;
18+
const ARCHIVE_MIDDLECACHE_PATH = "v1/logpush-datasets/datasets.tar.gz";
19+
const ARCHIVE_DOT_TMP_PATH = `middlecache/${ARCHIVE_MIDDLECACHE_PATH}`;
20+
const MARKDOWN_PAGE_GLOB = "**/*.md";
21+
const DOT_TMP_DIR = getDotTmpPath();
22+
const REPO_ROOT = dirname(DOT_TMP_DIR);
23+
const DATASETS_GIT_PATH = "src/content/docs/logs/logpush/logpush-job/datasets";
24+
const DATASETS_DIR = join(REPO_ROOT, DATASETS_GIT_PATH);
25+
const EXTRACTED_DIR = join(DOT_TMP_DIR, "logpush-datasets-extracted");
26+
const SYNC_STATE_PATH = join(DOT_TMP_DIR, "logpush-datasets.state");
27+
const PENDING_STATE_PATH = `${SYNC_STATE_PATH}.pending`;
28+
const TRANSACTION_DIR = join(DOT_TMP_DIR, "logpush-datasets-transaction");
29+
const STAGING_DIR = join(TRANSACTION_DIR, "staging");
30+
const BACKUP_DIR = join(TRANSACTION_DIR, "backup");
31+
32+
const validatePage = (page: string) => {
33+
const content = fs.readFileSync(join(EXTRACTED_DIR, page), "utf8");
34+
const frontmatter = /^---\r?\n([\s\S]*?)\r?\n---(?:\r?\n|$)/.exec(
35+
content,
36+
)?.[1];
37+
const metadata = frontmatter
38+
? (YAML.parse(frontmatter) as unknown)
39+
: undefined;
40+
if (
41+
!metadata ||
42+
typeof metadata !== "object" ||
43+
!("title" in metadata) ||
44+
typeof metadata.title !== "string" ||
45+
metadata.title.trim() === ""
46+
) {
47+
throw new Error(`Logpush dataset page has invalid frontmatter: ${page}`);
48+
}
49+
};
50+
51+
const getMarkdownPages = (directory: string) =>
52+
fs.globSync(MARKDOWN_PAGE_GLOB, { cwd: directory });
53+
54+
const isManagedPagePath = (page: string) => page.split(/[\\/]/).length === 2;
55+
const pageScope = (page: string) => page.split(/[\\/]/)[0];
56+
57+
const assertManagedPageLayout = (pages: string[]) => {
58+
const nestedPages = pages.filter((page) => !isManagedPagePath(page));
59+
if (nestedPages.length > 0) {
60+
throw new Error(
61+
`Logpush dataset pages must use <scope>/<page>.md paths: ${nestedPages.join(", ")}`,
62+
);
63+
}
64+
return pages;
65+
};
66+
67+
const getManagedPages = (directory: string) =>
68+
assertManagedPageLayout(getMarkdownPages(directory));
69+
70+
const directoryDigest = (directory: string) => {
71+
const hash = createHash("sha256");
72+
const files = getManagedPages(directory)
73+
.filter((file) => fs.statSync(join(directory, file)).isFile())
74+
.sort();
75+
for (const file of files) {
76+
hash.update(file);
77+
hash.update("\0");
78+
hash.update(fs.readFileSync(join(directory, file)));
79+
hash.update("\0");
80+
}
81+
return hash.digest("hex");
82+
};
83+
84+
const directoryMatchesState = (directory: string, statePath: string) =>
85+
fs.existsSync(statePath) &&
86+
directoryDigest(directory) === fs.readFileSync(statePath, "utf8");
87+
88+
const writeState = (statePath: string, digest: string) => {
89+
const temporaryPath = `${statePath}.${process.pid}.tmp`;
90+
fs.writeFileSync(temporaryPath, digest);
91+
fs.renameSync(temporaryPath, statePath);
92+
};
93+
94+
const promotePendingState = () => {
95+
writeState(SYNC_STATE_PATH, fs.readFileSync(PENDING_STATE_PATH, "utf8"));
96+
fs.rmSync(PENDING_STATE_PATH, { force: true });
97+
};
98+
99+
const ensureDatasetsUnmodified = () => {
100+
const status = spawnSync(
101+
"git",
102+
[
103+
"status",
104+
"--porcelain",
105+
"--untracked-files=all",
106+
"--",
107+
`:(glob)${DATASETS_GIT_PATH}/**/*.md`,
108+
],
109+
{ cwd: REPO_ROOT, encoding: "utf8" },
110+
);
111+
if (status.error) {
112+
throw status.error;
113+
}
114+
if (status.status !== 0) {
115+
throw new Error(`git status failed: ${status.stderr.trim()}`);
116+
}
117+
if (
118+
status.stdout.trim() &&
119+
!directoryMatchesState(DATASETS_DIR, SYNC_STATE_PATH)
120+
) {
121+
throw new Error(
122+
`Logpush dataset pages have uncommitted changes and do not match sync state ${SYNC_STATE_PATH}; commit or restore the managed pages before rebuilding`,
123+
);
124+
}
125+
};
126+
127+
const validateArchive = async (filePath: string) => {
128+
const archive = spawnSync("tar", ["-tzf", filePath], { encoding: "utf8" });
129+
if (archive.error) {
130+
throw archive.error;
131+
}
132+
if (archive.status !== 0) {
133+
throw new Error(
134+
`cached Logpush dataset archive is invalid: ${archive.stderr.trim()}`,
135+
);
136+
}
137+
};
138+
139+
const archivePath = join(DOT_TMP_DIR, ...ARCHIVE_DOT_TMP_PATH.split("/"));
140+
const previousArchivePath = `${archivePath}.previous`;
141+
142+
// Package scripts invoke one sync at a time; concurrent processes are unsupported.
143+
let archiveRefreshStarted = false;
144+
try {
145+
if (!fs.existsSync(DATASETS_DIR) && fs.existsSync(BACKUP_DIR)) {
146+
fs.renameSync(BACKUP_DIR, DATASETS_DIR);
147+
}
148+
if (
149+
!fs.existsSync(DATASETS_DIR) ||
150+
!fs.statSync(DATASETS_DIR).isDirectory()
151+
) {
152+
throw new Error(
153+
`Logpush dataset directory does not exist: ${DATASETS_DIR}`,
154+
);
155+
}
156+
if (fs.existsSync(PENDING_STATE_PATH)) {
157+
if (directoryMatchesState(DATASETS_DIR, PENDING_STATE_PATH)) {
158+
promotePendingState();
159+
} else {
160+
if (fs.existsSync(BACKUP_DIR)) {
161+
ensureDatasetsUnmodified();
162+
}
163+
fs.rmSync(PENDING_STATE_PATH, { force: true });
164+
}
165+
}
166+
fs.rmSync(STAGING_DIR, { recursive: true, force: true });
167+
if (fs.existsSync(BACKUP_DIR)) {
168+
console.warn(
169+
`Warning: removing stale Logpush dataset backup: ${BACKUP_DIR}`,
170+
);
171+
fs.rmSync(BACKUP_DIR, { recursive: true, force: true });
172+
}
173+
if (fs.existsSync(archivePath)) {
174+
try {
175+
await validateArchive(archivePath);
176+
} catch {
177+
fs.rmSync(archivePath, { force: true });
178+
}
179+
}
180+
if (fs.existsSync(archivePath)) {
181+
fs.rmSync(previousArchivePath, { force: true });
182+
fs.renameSync(archivePath, previousArchivePath);
183+
}
184+
185+
archiveRefreshStarted = true;
186+
await downloadToDotTempIfNotPresent(
187+
`${MIDDLECACHE_BASE_URL}${ARCHIVE_MIDDLECACHE_PATH}`,
188+
ARCHIVE_DOT_TMP_PATH,
189+
{ validate: validateArchive },
190+
);
191+
console.log("Fetched Logpush dataset archive from middlecache");
192+
193+
fs.rmSync(EXTRACTED_DIR, { recursive: true, force: true });
194+
await extractTarGz(archivePath, EXTRACTED_DIR);
195+
196+
const destinationPages = getManagedPages(DATASETS_DIR);
197+
if (destinationPages.length === 0) {
198+
throw new Error("Logpush dataset directory contains no managed pages");
199+
}
200+
const destinationScopes = new Set(
201+
destinationPages.map((page) => dirname(page)),
202+
);
203+
const archivePages = getMarkdownPages(EXTRACTED_DIR);
204+
const skippedArchiveScopes = [
205+
...new Set(
206+
archivePages
207+
.filter(isManagedPagePath)
208+
.map(pageScope)
209+
.filter((scope) => !destinationScopes.has(scope)),
210+
),
211+
];
212+
if (skippedArchiveScopes.length > 0) {
213+
console.warn(
214+
`Warning: skipping Logpush dataset scopes not seeded in the docs: ${skippedArchiveScopes.join(", ")}`,
215+
);
216+
}
217+
// Validate layout drift within managed scopes, but ignore unrelated archive files.
218+
const pagesToCopy = assertManagedPageLayout(
219+
archivePages.filter((page) => destinationScopes.has(pageScope(page))),
220+
);
221+
for (const page of pagesToCopy) {
222+
validatePage(page);
223+
}
224+
const sourcePages = new Set(pagesToCopy);
225+
// Destination pages missing from the filtered archive are stale.
226+
const pagesToRemove = destinationPages.filter(
227+
(page) => !sourcePages.has(page),
228+
);
229+
const sourceScopes = new Set(pagesToCopy.map((page) => dirname(page)));
230+
const missingScopes = [...destinationScopes].filter(
231+
(scope) => !sourceScopes.has(scope),
232+
);
233+
234+
if (missingScopes.length > 0) {
235+
throw new Error(
236+
`Logpush dataset archive is missing scopes: ${missingScopes.join(", ")}. If intentional, remove those scopes' checked-in generated pages in the same change`,
237+
);
238+
}
239+
const unsafeScopes = [...destinationScopes].filter((scope) => {
240+
const scopePageCount = destinationPages.filter(
241+
(page) => dirname(page) === scope,
242+
).length;
243+
const scopeRemovalCount = pagesToRemove.filter(
244+
(page) => dirname(page) === scope,
245+
).length;
246+
return scopeRemovalCount > scopePageCount * 0.25;
247+
});
248+
if (unsafeScopes.length > 0) {
249+
throw new Error(
250+
`Logpush dataset sync would remove more than 25% of pages in scopes: ${unsafeScopes.join(", ")}. If intentional, delete the affected checked-in generated pages first and commit them with this change`,
251+
);
252+
}
253+
254+
fs.mkdirSync(TRANSACTION_DIR, { recursive: true });
255+
fs.cpSync(DATASETS_DIR, STAGING_DIR, { recursive: true });
256+
for (const page of pagesToCopy) {
257+
fs.copyFileSync(join(EXTRACTED_DIR, page), join(STAGING_DIR, page));
258+
}
259+
for (const page of pagesToRemove) {
260+
fs.rmSync(join(STAGING_DIR, page));
261+
}
262+
263+
ensureDatasetsUnmodified();
264+
const stagingDigest = directoryDigest(STAGING_DIR);
265+
writeState(PENDING_STATE_PATH, stagingDigest);
266+
if (directoryDigest(DATASETS_DIR) === stagingDigest) {
267+
promotePendingState();
268+
fs.rmSync(STAGING_DIR, { recursive: true });
269+
} else {
270+
fs.renameSync(DATASETS_DIR, BACKUP_DIR);
271+
try {
272+
fs.renameSync(STAGING_DIR, DATASETS_DIR);
273+
} catch (err) {
274+
try {
275+
fs.renameSync(BACKUP_DIR, DATASETS_DIR);
276+
} catch {
277+
throw new Error(
278+
`Logpush dataset swap failed; original pages remain at ${BACKUP_DIR}`,
279+
{ cause: err },
280+
);
281+
}
282+
throw err;
283+
}
284+
promotePendingState();
285+
try {
286+
fs.rmSync(BACKUP_DIR, { recursive: true, force: true });
287+
} catch (err) {
288+
console.warn(
289+
`Warning: failed to remove Logpush dataset backup: ${(err as Error).message}`,
290+
);
291+
}
292+
}
293+
console.log("Logpush dataset pages ready");
294+
} catch (err) {
295+
if (archiveRefreshStarted) {
296+
try {
297+
fs.rmSync(archivePath, { force: true });
298+
if (fs.existsSync(previousArchivePath)) {
299+
fs.renameSync(previousArchivePath, archivePath);
300+
}
301+
} catch {
302+
// Preserve the original error.
303+
}
304+
}
305+
try {
306+
fs.rmSync(STAGING_DIR, { recursive: true, force: true });
307+
} catch {
308+
// Preserve the original error.
309+
}
310+
if (!fs.existsSync(DATASETS_DIR) && fs.existsSync(BACKUP_DIR)) {
311+
console.error(
312+
`Error: Logpush dataset replacement failed; original pages remain at ${BACKUP_DIR}`,
313+
);
314+
process.exit(1);
315+
}
316+
console.error(
317+
`Error: Logpush dataset fetch failed: ${(err as Error).message}`,
318+
);
319+
process.exit(1);
320+
}

‎package.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@
4040
"lint": "eslint",
4141
"prepare": "husky",
4242
"prebuild:incremental": "pnpm run fetch:assets",
43-
"fetch:assets": "tsx bin/fetch-skills.ts && tsx bin/fetch-openapi.ts"
43+
"fetch:assets": "tsx bin/fetch-skills.ts && tsx bin/fetch-openapi.ts && tsx bin/fetch-logpush-datasets.ts"
4444
},
4545
"devDependencies": {
4646
"@actions/core": "3.0.1",
@@ -146,6 +146,7 @@
146146
"vite": "^8.2.2",
147147
"vitest": "4.1.11",
148148
"wrangler": "4.129.0",
149+
"yaml": "2.9.0",
149150
"zod": "4.5.4"
150151
},
151152
"lint-staged": {

‎pnpm-lock.yaml‎

Lines changed: 3 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)