diff --git a/.github/workflows/docs-flags.yml b/.github/workflows/docs-flags.yml new file mode 100644 index 0000000000..20899c24a7 --- /dev/null +++ b/.github/workflows/docs-flags.yml @@ -0,0 +1,35 @@ +permissions: {} +name: Docs Flag Parity + +on: + pull_request: + branches: + - main + paths: + - "docs/**" + schedule: + - cron: "0 8 * * 1" + workflow_dispatch: + +jobs: + flag-parity: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + # Full history: the published docs are checked against their release tag. + fetch-depth: 0 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v6.0.0 + with: + node-version-file: docs/.nvmrc + + - name: Check docs flags against the binary + run: node docs/check-flags.js + + - name: Alert Slack on drift + if: failure() && github.event_name != 'pull_request' + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + run: | + MESSAGE="🚨 *Juno docs drift:* flags on juno.nethermind.io no longer match the binary.\n\nA flag was likely renamed or added in \`cmd/juno/juno.go\` after the docs were written. Run \`node docs/check-flags.js\` for the list.\n\n- *Run:* " + curl -X POST -H 'Content-type: application/json' --data "{\"text\":\"$MESSAGE\"}" "$SLACK_WEBHOOK_URL" diff --git a/docs/check-flags.js b/docs/check-flags.js new file mode 100644 index 0000000000..7a4a80ee21 --- /dev/null +++ b/docs/check-flags.js @@ -0,0 +1,188 @@ +#!/usr/bin/env node +// Verifies the docs and the binary agree about flags, in both directions. Next docs +// are checked against the working tree; published docs against their line's newest stable tag. + +const fs = require("fs"); +const path = require("path"); +const { execFileSync } = require("child_process"); +const { + preprocessCodebase, + extractConfigs, + generateConfigTable, +} = require("./generate-config.js"); + +const repoRoot = path.resolve(__dirname, ".."); +// Registered by cobra itself, so real flags without a Flags() call in juno.go. +const COBRA_BUILTINS = new Set(["help", "version"]); + +const findings = []; +const fail = (file, msg) => + findings.push(`${path.relative(repoRoot, file)}: ${msg}`); + +function flagNames(goSource) { + const names = extractConfigs(preprocessCodebase(goSource)) + .map((c) => c.configName) + .filter((n) => typeof n === "string"); + return new Set([...names, ...COBRA_BUILTINS]); +} + +// Flags a fenced line passes to Juno. Docker's own flags sit before the image, +// and helm charts are also named nethermind/juno, so the image rule is docker-only. +function flagsFromCommand(line) { + // Env-var prefixes (`JUNO_HTTP=true juno ...`) would otherwise hide the command. + let args = line.trim().replace(/^(?:[A-Z_][A-Z0-9_]*=\S+\s+)+/, ""); + // Anchored so nethermind/juno-plugin and friends do not match. + const image = + /\bdocker\s+(run|create)\b/.test(args) && + args.match(/nethermind\/juno(?::\S+|@\S+)?(?=\s|$)/); + if (image) { + args = args.slice(image.index + image[0].length); + } else if (/^(\S*\/)?juno\s/.test(args)) { + args = args.replace(/^(\S*\/)?juno\s+/, ""); + } else { + return []; + } + return [...args.matchAll(/(?:^|\s)--([a-z][a-z0-9-]*)/g)].map((m) => m[1]); +} + +// Every --flag a page mentions: backticked in prose, or passed to Juno in a fence. +function flagMentions(markdown) { + const mentions = new Map(); // name -> first line, for the report + const lines = markdown.split("\n"); + let fenceMarker = null; // a fence only closes on its own marker + let joined = ""; + for (let i = 0; i < lines.length; i++) { + const l = lines[i]; + const fence = l.trim().match(/^(```|~~~)/); + if (fence && (!fenceMarker || fence[1] === fenceMarker)) { + fenceMarker = fenceMarker ? null : fence[1]; + joined = ""; + continue; + } + const inFence = fenceMarker !== null; + if (inFence) { + joined += l.endsWith("\\") ? l.slice(0, -1) + " " : l; + if (l.endsWith("\\")) continue; + for (const f of flagsFromCommand(joined)) { + if (!mentions.has(f)) mentions.set(f, i + 1); + } + joined = ""; + } else { + // A removed or renamed flag must stay documentable, so those lines are exempt. + if (/\b(removed|renamed|deprecated)\b/i.test(l)) continue; + for (const m of l.matchAll(/`--([a-z][a-z0-9-]*)[^`]*`/g)) { + if (!mentions.has(m[1])) mentions.set(m[1], i + 1); + } + } + } + return mentions; +} + +// First cell of each row in a generated config table. +function tableNames(markdown) { + const names = new Map(); + markdown.split("\n").forEach((l, i) => { + const m = l.match(/^\| `([a-z0-9-]+)` \|/); + if (m && !names.has(m[1])) names.set(m[1], i + 1); + }); + return names; +} + +function checkTree(treeDir, source) { + const names = flagNames(source.text); + + // Documented flags must exist in the binary: the --log-port class. + for (const file of fs.readdirSync(treeDir).filter((f) => f.endsWith(".md"))) { + const p = path.join(treeDir, file); + const md = fs.readFileSync(p, "utf8"); + const documented = file.startsWith("_") ? tableNames(md) : flagMentions(md); + for (const [flag, line] of documented) { + if (!names.has(flag)) { + fail(p, `line ${line}: \`--${flag}\` is not a flag in ${source.label}`); + } + } + } + + // Registered flags must be in the table: the undocumented-flag class. + const tableFile = path.join(treeDir, "_config-options.md"); + if (!fs.existsSync(tableFile)) { + fail(tableFile, "config table missing from this tree"); + return; + } + const inTable = tableNames(fs.readFileSync(tableFile, "utf8")); + for (const name of names) { + if (COBRA_BUILTINS.has(name)) continue; + if (!inTable.has(name)) { + fail(tableFile, `missing \`${name}\`, which ${source.label} registers`); + } + } +} + +function main() { + // next: the working tree is the truth. + const localSource = fs.readFileSync( + path.join(repoRoot, "cmd", "juno", "juno.go"), + "utf8", + ); + checkTree(path.join(__dirname, "docs"), { + label: "cmd/juno/juno.go (working tree)", + text: localSource, + }); + + // next only: the committed table must byte-match the generator's output. + const expected = generateConfigTable( + extractConfigs(preprocessCodebase(localSource)), + ); + const tablePath = path.join(__dirname, "docs", "_config-options.md"); + if (fs.readFileSync(tablePath, "utf8") !== expected) { + fail( + tablePath, + "stale against cmd/juno/juno.go (working tree); run `cd docs && node generate-config.js` and commit the result", + ); + } + + // published: checked against the newest stable tag of its line. + const published = JSON.parse( + fs.readFileSync(path.join(__dirname, "versions.json"), "utf8"), + )[0]; + const line = published.split(".").slice(0, 2).join("."); + const tags = execFileSync("git", ["tag", "-l", `v${line}.*`], { + cwd: repoRoot, + }) + .toString() + .split("\n") + .filter((t) => /^v\d+\.\d+\.\d+$/.test(t)) // stable releases only + .sort((a, b) => Number(a.split(".")[2]) - Number(b.split(".")[2])); + const tag = tags[tags.length - 1]; + if (!tag) { + // Shallow clone without tags: warn instead of failing; CI fetches full history. + console.warn(`no v${line}.* tag found; skipping the published-version check`); + } else { + const tagSource = execFileSync( + "git", + ["show", `${tag}:cmd/juno/juno.go`], + { cwd: repoRoot }, + ).toString(); + checkTree(path.join(__dirname, "versioned_docs", `version-${published}`), { + label: `${tag} (newest stable of the ${line} line)`, + text: tagSource, + }); + } + + if (findings.length) { + console.error(`${findings.length} finding(s):\n`); + for (const f of findings) console.error(` FAIL ${f}`); + process.exit(1); + } + console.log( + `docs and binary agree on flags (next vs cmd/juno/juno.go, ${published} vs ${tag || "skipped"})`, + ); +} + +try { + main(); +} catch (err) { + // Environment problems (no git, broken tree) are not docs findings. + console.error(`check-flags: ${err.message}`); + process.exit(2); +} diff --git a/docs/docs/_config-options.md b/docs/docs/_config-options.md index 3e143bea5b..1ee6fa6ace 100644 --- a/docs/docs/_config-options.md +++ b/docs/docs/_config-options.md @@ -8,12 +8,15 @@ | `http` | `false` | Enables the HTTP RPC server on the default port and interface | | `http-host` | `localhost` | The interface on which the HTTP RPC server will listen for requests | | `http-port` | `6060` | The port on which the HTTP server will listen for requests | +| `rpc-batch-concurrency` | `2 * CPU Cores` | Maximum batch calls that run at the same time, per RPC version. All batch requests to a version share this limit. Default is set based on available hardware resources | | `rpc-call-max-gas` | `100000000` | Maximum number of Sierra gas to be executed in starknet_call requests | | `rpc-call-max-steps` | `4000000` | Maximum number of steps to be executed in starknet_call requests | | `rpc-cors-enable` | `false` | Enable CORS on RPC endpoints | +| `rpc-max-batch-response-size` | `64` | Size (in MBs) at which a batch stops being processed. The calls answered so far are returned, the rest are not executed. 0 disables the limit | +| `rpc-max-batch-size` | `1000` | Maximum number of calls in a single batch request. 0 disables the limit | | `rpc-max-block-scan` | `18446744073709551615` | Maximum number of blocks scanned in single starknet_getEvents call | | `rpc-max-concurrent-requests` | `256000` | Maximum concurrent HTTP RPC requests; 0 disables the limit | -| `rpc-max-request-queue` | `256000` | Maximum number of HTTP RPC requests to queue after reaching rpc-max-concurrent-requests before rejecting incoming requests | +| `rpc-max-request-queue` | `256000` | Maximum number of HTTP RPC requests to queue after reaching rpc-max-concurrent-requests limit | | `rpc-request-timeout` | `1m` | Maximum time for an RPC request to complete | ### WebSocket RPC @@ -103,11 +106,12 @@ | Config Option | Default Value | Description | | - | - | - | | `max-compilation-cpu-time` | `10` | Maximum CPU time (in seconds) each Sierra compilation process may consume; a compilation exceeding it is aborted. Enforced on Linux only. 0 disables the limit | -| `max-compilation-memory` | `4 * 1024` | Maximum memory (in MB) each Sierra compilation process may use; a compilation exceeding it is aborted. Enforced on Linux only. 0 disables the limit | -| `max-compilation-queue` | `2 * max-concurrent-compilations` | Maximum number of compilation requests to queue after reaching max-concurrent-compilations before starting to reject incoming requests | -| `max-concurrent-compilations` | `CPU Cores` | Maximum concurrent Sierra compilations | +| `max-compilation-memory` | `4096` | Maximum virtual memory (in MB) a Sierra compilation process may use; a compilation exceeding it is aborted. Enforced on Linux only. 0 disables the limit | +| `max-compilation-queue` | `2 * max-concurrent-compilations` | Maximum number of compilation requests to queue after reaching max-concurrent-compilations. Default sets the queue to twice the concurrency limit | +| `max-concurrent-compilations` | `auto (memory-aware)` | Maximum concurrent Sierra compilations. Default is set based on available hardware resources. Derived as `min(cpu_cores, (available_memory - node_memory_reserve) / max_compilation_memory)`, at least 1 | | `max-vm-queue` | `2 * max-vms` | Maximum number for requests to queue after reaching max-vms before starting to reject incoming requests | | `max-vms` | `3 * CPU Cores` | Maximum number for VM instances to be used for RPC calls concurrently | +| `node-memory-reserve` | `4096` | Memory (in MB) excluded from the compilations memory budget when calculating the default for `max-concurrent-compilations` | | `versioned-constants-file` | | Use custom versioned constants from provided file | ### Custom Network @@ -145,10 +149,10 @@ | Config Option | Default Value | Description | | - | - | - | -| `seq-block-time` | `60` | Time to build a block, in seconds | -| `seq-disable-fees` | `false` | Skip charge fee for sequencer execution | -| `seq-enable` | `false` | Enables sequencer mode of operation | -| `seq-genesis-file` | | Path to the genesis file | +| `seq-block-time` | `60` | EXPERIMENTAL: Time to build a block, in seconds | +| `seq-disable-fees` | `false` | EXPERIMENTAL: Skip charging fees for sequencer execution | +| `seq-enable` | `false` | EXPERIMENTAL: Enables sequencer mode of operation | +| `seq-genesis-file` | | EXPERIMENTAL: Path to the genesis file | ### gRPC diff --git a/docs/docs/monitoring.md b/docs/docs/monitoring.md index 3e39a6e333..90a2fdebd9 100644 --- a/docs/docs/monitoring.md +++ b/docs/docs/monitoring.md @@ -203,14 +203,18 @@ In case you want to change the log level in runtime without the need to restart To enable this feature, use the following configuration options: -- `log-host`: The interface to listen for requests. Defaults to `localhost`. -- `log-port`: The port to listen for requests. REQUIRED +:::note +These flags were renamed from `--log-host` and `--log-port` in v0.14.4. When running in a container, set `--http-update-host=0.0.0.0`; the default binds localhost. +::: + +- `http-update-host`: The interface to listen for requests. Defaults to `localhost`. +- `http-update-port`: The port to listen for requests. REQUIRED Examples: ```console -# Start juno specifying the log port -juno --log-port=6789 --log-level=error ... +# Start juno specifying the http-update port +juno --http-update-port=6789 --log-level=error ... # Get current level curl -X GET 'localhost:6789/log/level' diff --git a/docs/generate-config.js b/docs/generate-config.js index 0e8156c4b0..9547b3eee2 100644 --- a/docs/generate-config.js +++ b/docs/generate-config.js @@ -189,6 +189,12 @@ function parseValue(value) { return `${multipliedTime[1]}${timeUnitSuffix[multipliedTime[2]]}`; } + // An integer product like `4 * 1024` is a value, not a formula, so render the result. + const product = value.match(/^(\d+)\s*\*\s*(\d+)$/); + if (product) { + return Number(product[1]) * Number(product[2]); + } + // Handle large unsigned integer value if (value === "math.MaxUint") { return "18446744073709551615"; @@ -277,10 +283,7 @@ function generateConfigTable(configs) { const fileWarning = "\n\n"; - fs.writeFileSync( - "docs/_config-options.md", - fileWarning + sections.join("\n"), - ); + return fileWarning + sections.join("\n"); } function fetchUrl(url) { @@ -319,7 +322,7 @@ async function main() { const configs = extractConfigs(preprocessedCode); console.log("Extracted Juno's configuration"); - generateConfigTable(configs); + fs.writeFileSync("docs/_config-options.md", generateConfigTable(configs)); console.log("Generated the configuration options table"); } catch (error) { console.error( @@ -329,4 +332,9 @@ async function main() { } } -main(); +// Run as a CLI when invoked directly; check-flags.js imports the parser instead. +if (require.main === module) { + main(); +} + +module.exports = { preprocessCodebase, extractConfigs, generateConfigTable }; diff --git a/docs/versioned_docs/version-0.15.0/monitoring.md b/docs/versioned_docs/version-0.15.0/monitoring.md index 80c0879ebd..4756b7d69e 100644 --- a/docs/versioned_docs/version-0.15.0/monitoring.md +++ b/docs/versioned_docs/version-0.15.0/monitoring.md @@ -185,14 +185,18 @@ In case you want to change the log level in runtime without the need to restart To enable this feature, use the following configuration options: -- `log-host`: The interface to listen for requests. Defaults to `localhost`. -- `log-port`: The port to listen for requests. REQUIRED +:::note +These flags were renamed from `--log-host` and `--log-port` in v0.14.4. When running in a container, set `--http-update-host=0.0.0.0`; the default binds localhost. +::: + +- `http-update-host`: The interface to listen for requests. Defaults to `localhost`. +- `http-update-port`: The port to listen for requests. REQUIRED Examples: ```console -# Start juno specifying the log port -juno --log-port=6789 --log-level=error ... +# Start juno specifying the http-update port +juno --http-update-port=6789 --log-level=error ... # Get current level curl -X GET 'localhost:6789/log/level' diff --git a/docs/versioned_docs/version-0.16.0/_config-options.md b/docs/versioned_docs/version-0.16.0/_config-options.md index 191488ce96..1ee6fa6ace 100644 --- a/docs/versioned_docs/version-0.16.0/_config-options.md +++ b/docs/versioned_docs/version-0.16.0/_config-options.md @@ -8,10 +8,15 @@ | `http` | `false` | Enables the HTTP RPC server on the default port and interface | | `http-host` | `localhost` | The interface on which the HTTP RPC server will listen for requests | | `http-port` | `6060` | The port on which the HTTP server will listen for requests | +| `rpc-batch-concurrency` | `2 * CPU Cores` | Maximum batch calls that run at the same time, per RPC version. All batch requests to a version share this limit. Default is set based on available hardware resources | | `rpc-call-max-gas` | `100000000` | Maximum number of Sierra gas to be executed in starknet_call requests | | `rpc-call-max-steps` | `4000000` | Maximum number of steps to be executed in starknet_call requests | | `rpc-cors-enable` | `false` | Enable CORS on RPC endpoints | +| `rpc-max-batch-response-size` | `64` | Size (in MBs) at which a batch stops being processed. The calls answered so far are returned, the rest are not executed. 0 disables the limit | +| `rpc-max-batch-size` | `1000` | Maximum number of calls in a single batch request. 0 disables the limit | | `rpc-max-block-scan` | `18446744073709551615` | Maximum number of blocks scanned in single starknet_getEvents call | +| `rpc-max-concurrent-requests` | `256000` | Maximum concurrent HTTP RPC requests; 0 disables the limit | +| `rpc-max-request-queue` | `256000` | Maximum number of HTTP RPC requests to queue after reaching rpc-max-concurrent-requests limit | | `rpc-request-timeout` | `1m` | Maximum time for an RPC request to complete | ### WebSocket RPC @@ -35,6 +40,7 @@ | Config Option | Default Value | Description | | - | - | - | +| `disable-sync` | `false` | Disables L2 synchronization | | `preconfirmed-poll-interval` | `500ms` | Sets how frequently pre_confirmed block will be updated(0s will disable fetching of pre_confirmed block) | | `readiness-block-tolerance` | `6` | Maximum blocks behind latest for /ready endpoints to return 200 OK | | `remote-db` | | gRPC URL of a remote Juno node | @@ -83,7 +89,7 @@ | `db-cache-size` | `1024` | Determines the amount of memory (in megabytes) allocated for caching data in the database | | `db-compaction-concurrency` | | DB compaction concurrency range. Format: N (lower=1, upper=N) or M,N (lower=M, upper=N). Default: 1,GOMAXPROCS/2 | | `db-compression` | `zstd` | Database compression profile. Options: zstd, snappy, minlz. Use zstd for low storage | -| `db-max-handles` | `1024` | A soft limit on the number of open files that can be used by the DB | +| `db-max-handles` | `half of process fd limit (min 1024, max 1048576)` | A soft limit on the number of open files that can be used by the DB. When not set, defaults to half of the process fd limit (min 1024, max 1048576) | | `db-memtable-count` | `2` | Determines the number of memtables the database can queue before stalling writes | | `db-memtable-size` | `256` | Determines the amount of memory (in MBs) allocated for database memtables | | `db-path` | `juno` | Location of the database files | @@ -100,11 +106,12 @@ | Config Option | Default Value | Description | | - | - | - | | `max-compilation-cpu-time` | `10` | Maximum CPU time (in seconds) each Sierra compilation process may consume; a compilation exceeding it is aborted. Enforced on Linux only. 0 disables the limit | -| `max-compilation-memory` | `4 * 1024` | Maximum memory (in MB) each Sierra compilation process may use; a compilation exceeding it is aborted. Enforced on Linux only. 0 disables the limit | -| `max-compilation-queue` | `2 * max-concurrent-compilations` | Maximum number of compilation requests to queue after reaching max-concurrent-compilations before starting to reject incoming requests | -| `max-concurrent-compilations` | `CPU Cores` | Maximum concurrent Sierra compilations | +| `max-compilation-memory` | `4096` | Maximum virtual memory (in MB) a Sierra compilation process may use; a compilation exceeding it is aborted. Enforced on Linux only. 0 disables the limit | +| `max-compilation-queue` | `2 * max-concurrent-compilations` | Maximum number of compilation requests to queue after reaching max-concurrent-compilations. Default sets the queue to twice the concurrency limit | +| `max-concurrent-compilations` | `auto (memory-aware)` | Maximum concurrent Sierra compilations. Default is set based on available hardware resources. Derived as `min(cpu_cores, (available_memory - node_memory_reserve) / max_compilation_memory)`, at least 1 | | `max-vm-queue` | `2 * max-vms` | Maximum number for requests to queue after reaching max-vms before starting to reject incoming requests | | `max-vms` | `3 * CPU Cores` | Maximum number for VM instances to be used for RPC calls concurrently | +| `node-memory-reserve` | `4096` | Memory (in MB) excluded from the compilations memory budget when calculating the default for `max-concurrent-compilations` | | `versioned-constants-file` | | Use custom versioned constants from provided file | ### Custom Network @@ -142,10 +149,10 @@ | Config Option | Default Value | Description | | - | - | - | -| `seq-block-time` | `60` | Time to build a block, in seconds | -| `seq-disable-fees` | `false` | Skip charge fee for sequencer execution | -| `seq-enable` | `false` | Enables sequencer mode of operation | -| `seq-genesis-file` | | Path to the genesis file | +| `seq-block-time` | `60` | EXPERIMENTAL: Time to build a block, in seconds | +| `seq-disable-fees` | `false` | EXPERIMENTAL: Skip charging fees for sequencer execution | +| `seq-enable` | `false` | EXPERIMENTAL: Enables sequencer mode of operation | +| `seq-genesis-file` | | EXPERIMENTAL: Path to the genesis file | ### gRPC diff --git a/docs/versioned_docs/version-0.16.0/monitoring.md b/docs/versioned_docs/version-0.16.0/monitoring.md index 03df43090c..8dc1c531cc 100644 --- a/docs/versioned_docs/version-0.16.0/monitoring.md +++ b/docs/versioned_docs/version-0.16.0/monitoring.md @@ -198,14 +198,18 @@ In case you want to change the log level in runtime without the need to restart To enable this feature, use the following configuration options: -- `log-host`: The interface to listen for requests. Defaults to `localhost`. -- `log-port`: The port to listen for requests. REQUIRED +:::note +These flags were renamed from `--log-host` and `--log-port` in v0.14.4. When running in a container, set `--http-update-host=0.0.0.0`; the default binds localhost. +::: + +- `http-update-host`: The interface to listen for requests. Defaults to `localhost`. +- `http-update-port`: The port to listen for requests. REQUIRED Examples: ```console -# Start juno specifying the log port -juno --log-port=6789 --log-level=error ... +# Start juno specifying the http-update port +juno --http-update-port=6789 --log-level=error ... # Get current level curl -X GET 'localhost:6789/log/level'