Skip to content

Commit 830c0fb

Browse files
soheiokamotosoheiokamoto
committed
Fetch Logpush dataset pages from middlecache
Fetch validated Logpush dataset pages from middlecache during docs builds while retaining checked-in pages as a safe fallback. - Verify archive downloads before replacing pages - Sync generated Markdown into existing dataset scopes and remove stale pages - Abort replacements that would remove all managed pages - Refresh the archive on every development and build run - Correct the documented product metadata directory - Require a separate manual changelog when a dataset change needs a customer-facing announcement
1 parent f3c4d7d commit 830c0fb

3 files changed

Lines changed: 120 additions & 30 deletions

File tree

‎AGENTS.md‎

Lines changed: 29 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -16,34 +16,35 @@ This is the source for [developers.cloudflare.com](https://developers.cloudflare
1616
cloudflare-docs/
1717
├── src/
1818
│ ├── content/
19-
│ │ ├── docs/ # 5,400+ MDX pages — the user-facing documentation
20-
│ │ ├── partials/ # 1,200+ reusable MDX snippets (by product)
21-
│ │ ├── changelog/ # Product changelogs (by product subdirectory)
22-
│ │ ├── glossary/ # Glossary term definitions (YAML)
23-
│ │ ├── products/ # Product metadata (YAML, 135 files)
24-
│ │ └── ... # Other data collections (plans, fields, models, etc.)
25-
│ ├── components/ # Astro + React components (barrel: components.ts)
26-
│ ├── components.ts # MDX component barrel — all MDX imports come from here
27-
│ ├── layouts/ # Page layout components
28-
│ ├── pages/ # Dynamic route pages (changelog, llms.txt, RSS, etc.)
29-
│ ├── schemas/ # Zod schemas for all content collections
30-
│ ├── plugins/ # Satteri hast pipeline plugins
31-
│ ├── scripts/ # Client-side scripts (analytics, mermaid, webmcp, etc.)
32-
│ ├── styles/ # CSS (Tailwind 4)
33-
│ ├── icons/ # Product SVG icons (~110)
34-
│ ├── assets/ # Processed images (optimized by Astro)
35-
│ ├── util/ # App utility functions
36-
│ └── content.config.ts # Content collection definitions
37-
├── public/ # Static files served as-is (images, redirects, robots.txt)
38-
├── worker/ # Cloudflare Worker for serving the site
39-
├── bin/ # Build scripts and CI helpers
40-
│ └── fetch-skills.ts # Downloads skills.tar.gz from middlecache, extracts to skills/
41-
├── skills/ # Agent Skills served at /.well-known/skills/ — GENERATED, do not edit
42-
│ # Fetched from https://middlecache.ced.cloudflare.com/v1/cloudflare-skills/skills.tar.gz
43-
│ # by bin/fetch-skills.ts, which runs automatically via prebuild/predev hooks.
44-
│ # skills/ is in .gitignore and is NOT committed to the repository.
45-
├── .flue/ # Flue cloudflare-docs-bot — see .flue/AGENTS.md
46-
├── astro.config.ts # Astro + Nimbus configuration
19+
│ │ ├── docs/ # 5,400+ MDX pages — the user-facing documentation
20+
│ │ ├── partials/ # 1,200+ reusable MDX snippets (by product)
21+
│ │ ├── changelog/ # Product changelogs (by product subdirectory)
22+
│ │ ├── glossary/ # Glossary term definitions (YAML)
23+
│ │ ├── directory/ # Product and feature directory entries (YAML)
24+
│ │ └── ... # Other data collections (plans, fields, models, etc.)
25+
│ ├── components/ # Astro + React components (barrel: components.ts)
26+
│ ├── components.ts # MDX component barrel — all MDX imports come from here
27+
│ ├── layouts/ # Page layout components
28+
│ ├── pages/ # Dynamic route pages (changelog, llms.txt, RSS, etc.)
29+
│ ├── schemas/ # Zod schemas for all content collections
30+
│ ├── plugins/ # Satteri hast pipeline plugins
31+
│ ├── scripts/ # Client-side scripts (analytics, mermaid, webmcp, etc.)
32+
│ ├── styles/ # CSS (Tailwind 4)
33+
│ ├── icons/ # Product SVG icons (~110)
34+
│ ├── assets/ # Processed images (optimized by Astro)
35+
│ ├── util/ # App utility functions
36+
│ └── content.config.ts # Content collection definitions
37+
├── public/ # Static files served as-is (images, redirects, robots.txt)
38+
├── worker/ # Cloudflare Worker for serving the site
39+
├── bin/ # Build scripts and CI helpers
40+
│ ├── fetch-skills.ts # Downloads skills.tar.gz from middlecache, extracts to skills/
41+
│ └── fetch-logpush-datasets.ts # Syncs generated Logpush pages into existing scopes; fetch failures retain checked-in pages
42+
├── skills/ # Agent Skills served at /.well-known/skills/ — GENERATED, do not edit
43+
│ # Fetched from https://middlecache.ced.cloudflare.com/v1/cloudflare-skills/skills.tar.gz
44+
│ # by bin/fetch-skills.ts, which runs automatically via prebuild/predev hooks.
45+
│ # skills/ is in .gitignore and is NOT committed to the repository.
46+
├── .flue/ # Flue cloudflare-docs-bot — see .flue/AGENTS.md
47+
├── astro.config.ts # Astro + Nimbus configuration
4748
├── package.json
4849
└── tsconfig.json
4950
```

‎bin/fetch-logpush-datasets.ts‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
#!/usr/bin/env tsx
2+
3+
import fs from "fs";
4+
import { dirname, join } from "path";
5+
6+
import {
7+
downloadToDotTempIfNotPresent,
8+
extractTarGz,
9+
} from "../src/util/custom-loaders";
10+
11+
const MIDDLECACHE_BASE_URL =
12+
process.env.MIDDLECACHE_BASE_URL ?? "https://middlecache.ced.cloudflare.com/";
13+
const ARCHIVE_MIDDLECACHE_PATH = "v1/logpush-datasets/datasets.tar.gz";
14+
const ARCHIVE_DOT_TMP_PATH = `middlecache/${ARCHIVE_MIDDLECACHE_PATH}`;
15+
const DATASETS_DIR =
16+
process.env.LOGPUSH_DATASETS_DIR ??
17+
"src/content/docs/logs/logpush/logpush-job/datasets";
18+
const EXTRACTED_DIR = join(".tmp", "logpush-datasets");
19+
20+
// Builds soft-fail so middlecache availability cannot block unrelated docs changes.
21+
const soft = process.argv.includes("--soft");
22+
const force = process.argv.includes("--force");
23+
24+
const fail = (message: string): never => {
25+
if (soft) {
26+
console.warn(
27+
`Warning: ${message} - continuing with checked-in Logpush dataset pages`,
28+
);
29+
process.exit(0);
30+
}
31+
console.error(`Error: ${message}`);
32+
process.exit(1);
33+
};
34+
35+
const archivePath = join(".tmp", ...ARCHIVE_DOT_TMP_PATH.split("/"));
36+
if (!fs.existsSync(DATASETS_DIR) || !fs.statSync(DATASETS_DIR).isDirectory()) {
37+
fail(`Logpush dataset directory does not exist: ${DATASETS_DIR}`);
38+
}
39+
if (force) {
40+
fs.rmSync(archivePath, { force: true });
41+
}
42+
43+
console.log("Fetching Logpush dataset pages from middlecache");
44+
45+
let mutationStarted = false;
46+
try {
47+
await downloadToDotTempIfNotPresent(
48+
`${MIDDLECACHE_BASE_URL}${ARCHIVE_MIDDLECACHE_PATH}`,
49+
ARCHIVE_DOT_TMP_PATH,
50+
);
51+
52+
fs.rmSync(EXTRACTED_DIR, { recursive: true, force: true });
53+
await extractTarGz(archivePath, EXTRACTED_DIR);
54+
55+
// The archive contains <scope>/<page>.md; only sync existing docs scopes.
56+
const pagesToCopy = fs
57+
.globSync("*/*.md", { cwd: EXTRACTED_DIR })
58+
.filter((page) =>
59+
fs
60+
.statSync(join(DATASETS_DIR, dirname(page)), { throwIfNoEntry: false })
61+
?.isDirectory(),
62+
);
63+
const sourcePages = new Set(pagesToCopy);
64+
// Destination pages missing from the filtered archive are stale.
65+
const pagesToRemove = fs
66+
.globSync("*/*.md", { cwd: DATASETS_DIR })
67+
.filter((page) => !sourcePages.has(page));
68+
69+
if (pagesToCopy.length === 0 && pagesToRemove.length > 0) {
70+
throw new Error("Logpush dataset sync would remove all pages");
71+
}
72+
73+
mutationStarted = true;
74+
for (const page of pagesToCopy) {
75+
fs.copyFileSync(join(EXTRACTED_DIR, page), join(DATASETS_DIR, page));
76+
}
77+
for (const page of pagesToRemove) {
78+
fs.rmSync(join(DATASETS_DIR, page));
79+
}
80+
console.log("Logpush dataset pages ready");
81+
} catch (err) {
82+
if (mutationStarted) {
83+
console.error(
84+
`Error: Logpush dataset replacement failed: ${(err as Error).message}`,
85+
);
86+
process.exit(1);
87+
}
88+
fail(`Logpush dataset fetch failed: ${(err as Error).message}`);
89+
}

‎package.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
"check": "pnpm run check:astro && pnpm run check:worker",
1313
"check:astro": "astro check --minimumFailingSeverity=hint",
1414
"check:worker": "tsc --noEmit -p ./worker/tsconfig.json",
15-
"predev": "tsx bin/fetch-skills.ts --soft && tsx bin/fetch-openapi.ts --soft",
15+
"predev": "tsx bin/fetch-skills.ts --soft && tsx bin/fetch-openapi.ts --soft && tsx bin/fetch-logpush-datasets.ts --soft --force",
1616
"dev": "astro dev",
1717
"format": "pnpm run format:core:fix && pnpm run format:data:fix && pnpm run format:content:fix",
1818
"format:check": "pnpm run format:core:check && pnpm run format:data:check && pnpm run format:content:check",
@@ -40,7 +40,7 @@
4040
"lint": "eslint",
4141
"prepare": "husky",
4242
"prebuild:incremental": "pnpm run fetch:assets",
43-
"fetch:assets": "tsx bin/fetch-skills.ts && tsx bin/fetch-openapi.ts"
43+
"fetch:assets": "tsx bin/fetch-skills.ts && tsx bin/fetch-openapi.ts && tsx bin/fetch-logpush-datasets.ts --soft --force"
4444
},
4545
"devDependencies": {
4646
"@actions/core": "3.0.1",

0 commit comments

Comments
 (0)