diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..4691974 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,28 @@ +# Keeps the SHA-pinned actions in .github/workflows current. +# +# Pinning by commit SHA is what stops a moved tag from changing what runs in CI, +# but a pin never updates itself: without this, the workflows would silently +# freeze on whatever was current the day they were written, security fixes +# included. Dependabot reads the `# vX.Y.Z` comment beside each SHA, so its pull +# requests bump both together and stay readable. +version: 2 +updates: + - package-ecosystem: github-actions + directory: "/" + schedule: + interval: weekly + # Repository tooling must not use fix:/feat:, or it would land in the chart's + # changelog and could trigger a release of the chart. `ci:` is a hidden + # changelog section, so these never reach a release. + commit-message: + prefix: ci + # One pull request for the routine bumps rather than five. Majors stay + # separate: those are the ones that change behaviour and deserve their own + # review. + groups: + actions-minor-patch: + patterns: ["*"] + update-types: ["minor", "patch"] + open-pull-requests-limit: 5 + labels: + - dependencies diff --git a/.github/scripts/ci_values_summary.py b/.github/scripts/ci_values_summary.py new file mode 100644 index 0000000..0248b3d --- /dev/null +++ b/.github/scripts/ci_values_summary.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +"""Report the values a ct run actually used, as Markdown for a job summary. + +`ct` discovers charts/graylog/ci/*-values.yaml on its own and the workflow passes +no sizing flags, so nothing in the log says what the chart was installed with. +This renders the chart with the same overlay and reports what came out, which is +the only view that accounts for chart defaults the overlay does not mention - +MongoDB's containers being the obvious case. + +Reads the rendered manifests as JSON on stdin: + + helm template ci charts/graylog -f | yq ea -o=json '[.]' - \ + | ci_values_summary.py --overlay [--extra-set "flag, flag"] +""" + +import argparse +import json +import sys + +CPU_SUFFIX = {"m": 0.001, "": 1.0} +MEM_SUFFIX = { + "Ki": 1 / 1024, + "Mi": 1.0, + "Gi": 1024.0, + "K": 1000 / 1024 / 1024, + "M": 1000 * 1000 / 1024 / 1024, + "G": 1000 * 1000 * 1000 / 1024 / 1024, + "": 1 / 1024 / 1024, +} + + +def cpu_cores(value: str | None) -> float: + """Parse a Kubernetes CPU quantity into cores.""" + if not value: + return 0.0 + text = str(value) + if text.endswith("m"): + return float(text[:-1]) * CPU_SUFFIX["m"] + return float(text) + + +def mem_mib(value: str | None) -> float: + """Parse a Kubernetes memory quantity into MiB.""" + if not value: + return 0.0 + text = str(value) + for suffix in ("Ki", "Mi", "Gi", "K", "M", "G"): + if text.endswith(suffix): + return float(text[: -len(suffix)]) * MEM_SUFFIX[suffix] + return float(text) * MEM_SUFFIX[""] + + +def fmt_cpu(cores: float) -> str: + return f"{cores:g}" if cores >= 1 else f"{round(cores * 1000)}m" + + +def fmt_mem(mib: float) -> str: + return f"{mib / 1024:g}Gi" if mib >= 1024 else f"{mib:g}Mi" + + +def requests_of(container: dict, key: str) -> str | None: + return (container.get("resources") or {}).get(key, {}).get("cpu"), ( + container.get("resources") or {} + ).get(key, {}).get("memory") + + +def pod_reservation(containers: list[dict], init: list[dict]) -> tuple[float, float]: + """A pod reserves max(max(initContainer), sum(containers)) on each axis.""" + run_cpu = sum(cpu_cores(requests_of(c, "requests")[0]) for c in containers) + run_mem = sum(mem_mib(requests_of(c, "requests")[1]) for c in containers) + init_cpu = max( + (cpu_cores(requests_of(c, "requests")[0]) for c in init), default=0.0 + ) + init_mem = max((mem_mib(requests_of(c, "requests")[1]) for c in init), default=0.0) + return max(run_cpu, init_cpu), max(run_mem, init_mem) + + +def workload_rows(docs: list[dict]) -> tuple[list[list[str]], float, float]: + """One row per workload, plus the cluster-wide request totals.""" + rows: list[list[str]] = [] + total_cpu = total_mem = 0.0 + + for doc in docs: + if not isinstance(doc, dict): + continue + + if doc.get("kind") == "StatefulSet": + spec = doc["spec"]["template"]["spec"] + replicas = int(doc["spec"].get("replicas", 1)) + cpu, mem = pod_reservation( + spec.get("containers") or [], spec.get("initContainers") or [] + ) + limits = [ + ( + (c.get("resources") or {}).get("limits", {}).get("cpu"), + (c.get("resources") or {}).get("limits", {}).get("memory"), + ) + for c in spec.get("containers") or [] + ] + limit_text = ", ".join( + f"{l[0] or '–'} / {l[1] or '–'}" for l in limits + ) + grace = spec.get("terminationGracePeriodSeconds", "cluster default") + rows.append( + [ + f"`{doc['metadata']['name']}`", + str(replicas), + f"{fmt_cpu(cpu)} / {fmt_mem(mem)}", + limit_text or "–", + f"{grace}s" if isinstance(grace, int) else str(grace), + ] + ) + total_cpu += cpu * replicas + total_mem += mem * replicas + + elif doc.get("kind") == "MongoDBCommunity": + spec = doc["spec"]["statefulSet"]["spec"]["template"]["spec"] + members = int(doc["spec"].get("members", 1)) + int( + doc["spec"].get("arbiters", 0) or 0 + ) + cpu, mem = pod_reservation( + spec.get("containers") or [], spec.get("initContainers") or [] + ) + limit_text = ", ".join( + f"{(c.get('resources') or {}).get('limits', {}).get('cpu') or '–'}" + f" / {(c.get('resources') or {}).get('limits', {}).get('memory') or '–'}" + for c in spec.get("containers") or [] + ) + rows.append( + [ + f"`{doc['metadata']['name']}` (MongoDB {doc['spec'].get('version', '?')})", + str(members), + f"{fmt_cpu(cpu)} / {fmt_mem(mem)}", + limit_text or "operator defaults", + "operator-owned", + ] + ) + total_cpu += cpu * members + total_mem += mem * members + + return rows, total_cpu, total_mem + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--overlay", required=True, help="values file ct discovered") + ap.add_argument("--extra-set", default="", help="flags the workflow adds") + ap.add_argument("--heading", default="Values used") + args = ap.parse_args() + + docs = json.load(sys.stdin) + rows, total_cpu, total_mem = workload_rows(docs) + + out = [f"### {args.heading}", ""] + out.append(f"Overlay: `{args.overlay}` (discovered by `ct`)") + out.append("") + if args.extra_set: + out.append(f"Extra flags: `{args.extra_set}`") + out.append("") + out.append("| workload | replicas | request / pod | limits per container | grace |") + out.append("|---|---|---|---|---|") + for row in rows: + out.append("| " + " | ".join(row) + " |") + out.append("") + out.append( + f"**Whole stack requests {fmt_cpu(total_cpu)} CPU and {fmt_mem(total_mem)}**, " + "counting every replica. A pod reserves " + "`max(max(initContainer), sum(containers))`, so init containers are " + "included where they set the floor." + ) + print("\n".join(out)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/lint-and-test.yaml b/.github/workflows/lint-and-test.yaml index bf90edf..860d3cc 100644 --- a/.github/workflows/lint-and-test.yaml +++ b/.github/workflows/lint-and-test.yaml @@ -6,9 +6,118 @@ on: push: branches: ["main"] +# One run per ref. Pushing three times to a branch used to leave three full runs +# in flight - up to 27 jobs, of which only the last mattered - and they compete +# for the same concurrency slots as everyone else's runs. +# +# Pushes to `main` are never cancelled: each merge commit's result is a record of +# whether that commit is good, and nothing else re-establishes it. +concurrency: + group: tests-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: + # Decides which test jobs are worth running for this change. + # + # Deliberately not `on..paths`: a workflow skipped by a path filter + # creates no check runs at all, so if these checks are ever made required, a + # docs-only PR waits forever on checks that will never report. Jobs skipped by + # an `if:` condition still report - as skipped, which branch protection + # accepts - and cost a few seconds rather than a runner. + changes: + runs-on: ubuntu-latest + outputs: + chart: ${{ steps.filter.outputs.chart }} + examples: ${{ steps.filter.outputs.examples }} + steps: + - name: Checkout + # The only job that needs history: the diff below is computed against the + # merge base, which a shallow clone does not contain. + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 0 + + - name: Classify changed paths + id: filter + env: + BASE_REF: ${{ github.base_ref }} + BEFORE: ${{ github.event.before }} + run: | + set -u + # On anything unexpected, fall through to running everything: a false + # negative here silently ships an untested chart. + if [[ "${{ github.event_name }}" == "pull_request" ]]; then + git fetch --quiet origin "$BASE_REF" + range="origin/${BASE_REF}...HEAD" + elif [[ -n "${BEFORE:-}" ]] && git cat-file -e "${BEFORE}^{commit}" 2>/dev/null; then + range="${BEFORE}..HEAD" + else + echo "no usable diff range; running every test job" + { + echo "chart=true" + echo "examples=true" + } >> "$GITHUB_OUTPUT" + exit 0 + fi + + files="$(git diff --name-only "$range")" + echo "::group::changed files ($range)" + printf '%s\n' "$files" + echo "::endgroup::" + + chart=false + examples=false + while IFS= read -r f; do + [[ -n "$f" ]] || continue + case "$f" in + # Markdown inside the chart is documentation - the values + # reference and the chart README. It cannot change a rendered + # manifest, so it does not earn a four-leg install matrix. + charts/graylog/*) + if [[ "$f" != *.md ]]; then chart=true; fi + ;; + # Shipped examples are values overlays, rendered by the lint job. + examples/*) + examples=true + ;; + # The CI definition itself: run everything, or a change to how the + # chart is tested would never be tested. + .github/workflows/lint-and-test.yaml|.github/ct.yaml) + chart=true + examples=true + ;; + esac + done <<< "$files" + + echo "chart=$chart" | tee -a "$GITHUB_OUTPUT" + echo "examples=$examples" | tee -a "$GITHUB_OUTPUT" + + mark() { [[ "$1" == "true" ]] && echo "run" || echo "skipped"; } + { + echo "## What this run tests" + echo + echo "| job | verdict | why |" + echo "|-----|---------|-----|" + echo "| \`helm-ct-lint\` | $( [[ "$chart" == "true" || "$examples" == "true" ]] && echo run || echo skipped ) | chart or examples changed |" + echo "| \`helm-unittest\` | $(mark "$chart") | chart templates/values/tests changed |" + echo "| \`helm-ct-install\` | $(mark "$chart") | chart changed (4 legs, full stack each) |" + echo + echo "Markdown under \`charts/graylog/\` and everything in \`docs/\` cannot" + echo "change a rendered manifest, so they test nothing." + echo + echo "
Changed files ($(printf '%s\n' "$files" | grep -c . || true))" + echo + echo '```' + printf '%s\n' "$files" + echo '```' + echo + echo "
" + } >> "$GITHUB_STEP_SUMMARY" + helm-ct-lint: runs-on: ubuntu-latest + needs: changes + if: needs.changes.outputs.chart == 'true' || needs.changes.outputs.examples == 'true' strategy: fail-fast: false matrix: @@ -17,20 +126,131 @@ jobs: - v4.2.0 steps: - name: Checkout - uses: actions/checkout@v6 - with: - fetch-depth: 0 + # Shallow on purpose: `ct` is invoked with --all and never diffs against a + # target branch, so no job past `changes` needs history. + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Set up Helm - uses: azure/setup-helm@v5 + uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 with: version: ${{ matrix.helm-version }} - name: Set up chart-testing - uses: helm/chart-testing-action@v2.8.0 + uses: helm/chart-testing-action@6ec842c01de15ebb84c8627d2744a0c2f2755c9f # v2.8.0 - name: Run ct lint - run: ct lint --config .github/ct.yaml --all + id: ct-lint + run: | + set -o pipefail + ct lint --config .github/ct.yaml --all 2>&1 | tee /tmp/ct-lint.log + + # `ct` discovers ci/*-values.yaml on its own and the workflow passes no + # sizing flags, so without this nothing in the run says what the chart was + # linted against. + - name: Report values used + if: always() + # Advisory only: this step describes the run, it must never fail it. + continue-on-error: true + run: | + helm template ci charts/graylog -f charts/graylog/ci/ci-values.yaml \ + | yq ea -o=json '[.]' - \ + | python3 .github/scripts/ci_values_summary.py \ + --overlay charts/graylog/ci/ci-values.yaml \ + --heading "Values linted — Helm ${{ matrix.helm-version }}" \ + >> "$GITHUB_STEP_SUMMARY" + + # The rendered manifests are the chart's actual output, and reading them is + # how you answer "what does this PR change" without installing anything. + # Kept as an artifact rather than printed: a full render is thousands of + # lines, too much for a log and far too much for a job summary. + - name: Render full manifests + if: always() + continue-on-error: true + run: | + set -u + mkdir -p rendered + + render() { + local name="$1"; shift + if helm template graylog charts/graylog "$@" > "rendered/${name}.yaml" 2>"rendered/${name}.err"; then + rm -f "rendered/${name}.err" + printf '%-46s %6s lines, %4s objects\n' "$name" \ + "$(wc -l < "rendered/${name}.yaml" | tr -d ' ')" \ + "$(grep -c '^kind:' "rendered/${name}.yaml" || true)" + else + mv "rendered/${name}.yaml" "rendered/${name}.failed.yaml" 2>/dev/null || true + printf '%-46s FAILED (see %s.err)\n' "$name" "$name" + fi + } + + # Chart defaults are the production shape, and the one people actually + # deploy; the CI overlay is what this workflow tests. + render "00-chart-defaults" + render "01-ci-values" -f charts/graylog/ci/ci-values.yaml + + while IFS= read -r f; do + grep -q '^apiVersion:' "$f" && continue # raw manifest, not an overlay + render "example-$(basename "${f%.yaml}")" -f "$f" + done < <(find examples -name '*.yaml' | sort) + + # `helm template` generates credentials for every key left empty - the + # root password, the password_secret pepper, both MongoDB passwords. + # They are throwaway values that never reach a cluster, but artifacts on + # a public repository are world-downloadable and these would read as + # leaked credentials to anyone (and to a secret scanner). Redact the + # values, keep the keys, so a diff still shows a key appearing or + # disappearing. The placeholder is valid base64, so the manifests stay + # loadable. + for f in rendered/*.yaml; do + yq -i '(select(.kind == "Secret" and has("data")) | .data) |= with_entries(.value = "UkVEQUNURUQ=")' "$f" + yq -i '(select(.kind == "Secret" and has("stringData")) | .stringData) |= with_entries(.value = "REDACTED")' "$f" + done + + # A failed redaction must not become a published credential, so prove + # it worked and throw the bundle away if it did not. + leaked="$(yq ea 'select(.kind == "Secret") | .data // {} | to_entries | .[] | select(.value != "UkVEQUNURUQ=") | .key' rendered/*.yaml | grep -c . || true)" + if [[ "$leaked" != "0" ]]; then + echo "::error::redaction left ${leaked} secret value(s); discarding renders instead of uploading" + rm -rf rendered + exit 1 + fi + echo "all Secret values redacted" + + ls -l rendered/ + + - name: Upload rendered manifests + id: upload-rendered + if: always() + continue-on-error: true + # v7 rather than v4: v4 runs on Node 20, which the runners now force onto + # Node 24 with a deprecation warning on every run. Inputs and the + # artifact-url output the summary step reads are unchanged. + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: rendered-manifests-helm-${{ matrix.helm-version }} + path: rendered/ + # Long enough to review a PR and bisect a regression, short enough not + # to accumulate a render of every push forever. + retention-days: 14 + if-no-files-found: warn + + - name: Note the artifact in the summary + if: always() + continue-on-error: true + run: | + { + echo "### Rendered manifests" + echo + echo "Full \`helm template\` output for chart defaults, the CI overlay and every" + echo "shipped example: [\`rendered-manifests-helm-${{ matrix.helm-version }}\`](${{ steps.upload-rendered.outputs.artifact-url }})" + echo "(kept 14 days). Download and diff against the same artifact on \`main\` to" + echo "see exactly what a change does to the output." + echo + echo "\`Secret\` values are redacted — keys are kept, so a diff still shows one" + echo "appearing or disappearing. Everything Helm generates for an empty value" + echo "(root password, \`password_secret\` pepper, MongoDB passwords) would" + echo "otherwise be world-downloadable from a public repository." + } >> "$GITHUB_STEP_SUMMARY" # `ct lint` never looks at examples/, so a shipped example could reference a # removed key, miss a required one, or trip a template-time guard and no one @@ -40,26 +260,59 @@ jobs: # Raw manifests in examples/ (Secrets, Pods) are not values overlays and are # skipped by looking for a top-level `apiVersion:`. - name: Render shipped examples + id: examples + # always() so a ct lint failure still yields the examples table and a + # summary for this leg, rather than an empty run page. + if: always() run: | set -u rc=0 + rows="" + ok=0; skipped=0; failed=0 while IFS= read -r f; do if grep -q '^apiVersion:' "$f"; then echo "skip $f (raw manifest, not a values overlay)" + rows="${rows}| \`${f}\` | – | raw manifest, not a values overlay |"$'\n' + skipped=$((skipped + 1)) continue fi if out=$(helm template ci-examples charts/graylog -f "$f" 2>&1); then echo "ok $f" + objects=$(printf '%s\n' "$out" | grep -c '^kind:' || true) + rows="${rows}| \`${f}\` | ok | ${objects} objects |"$'\n' + ok=$((ok + 1)) else echo "FAILED $f" printf '%s\n' "$out" | sed 's/^/ /' + # The first `Error:` line is generic ("values don't meet the + # specifications of the schema"); the offending key is on the + # lines after it, so keep a few and flatten them into the cell. + reason=$(printf '%s\n' "$out" | grep -i -m1 -A3 'error' | tr '\n' ' ' | tr -s ' ' | cut -c1-220) + rows="${rows}| \`${f}\` | **FAILED** | ${reason//|/\\|} |"$'\n' + failed=$((failed + 1)) rc=1 fi done < <(find examples -name '*.yaml' | sort) + + { + echo "## Lint — Helm ${{ matrix.helm-version }}" + echo + echo "\`ct lint\`: **$( [[ "${{ steps.ct-lint.outcome }}" == "success" ]] && echo passed || echo failed )**" + echo + echo "Shipped examples: **${ok} rendered**, ${skipped} skipped, ${failed} failed" + echo + echo "| example | result | detail |" + echo "|---------|--------|--------|" + printf '%s' "$rows" + } >> "$GITHUB_STEP_SUMMARY" exit $rc helm-unittest: runs-on: ubuntu-latest + needs: changes + # Templates, values and the suites themselves all live under + # charts/graylog/, so a chart change is the only thing this can regress. + if: needs.changes.outputs.chart == 'true' strategy: fail-fast: false matrix: @@ -68,12 +321,12 @@ jobs: - v4.2.0 steps: - name: Checkout - uses: actions/checkout@v6 - with: - fetch-depth: 0 + # Shallow on purpose: `ct` is invoked with --all and never diffs against a + # target branch, so no job past `changes` needs history. + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Set up Helm - uses: azure/setup-helm@v5 + uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 with: version: ${{ matrix.helm-version }} @@ -91,23 +344,75 @@ jobs: helm plugin install /tmp/helm-unittest - name: Run helm unittest - run: helm unittest charts/graylog + id: unittest + run: | + set -o pipefail + helm unittest charts/graylog 2>&1 | tee /tmp/unittest.log # helm-unittest can only assert on rendered text. These two run the shipped # shell scripts for real: the preStop drain against a stubbed metrics # endpoint, and the init script against fixture volumes (including the # upgrade path, where an existing volume must not be clobbered). - name: preStop drain behaviour tests + id: drain + if: always() run: sh charts/graylog/tests/scripts/prestop-drain-behavior-test.sh - name: init-script behaviour tests + id: init-script # always() so a drain failure above does not hide an init failure here. if: always() run: sh charts/graylog/tests/scripts/init-graylog-behavior-test.sh + - name: Summary + if: always() + run: | + set -u + verdict() { [[ "$1" == "success" ]] && echo "passed" || echo "**FAILED**"; } + + { + echo "## Unit tests — Helm ${{ matrix.helm-version }}" + echo + if [[ -f /tmp/unittest.log ]]; then + # helm-unittest prints "Test Suites: 28 passed, 28 total" and the + # same shape for Tests; lift those two lines verbatim. + suites=$(grep -m1 '^Test Suites:' /tmp/unittest.log | sed 's/Test Suites:[[:space:]]*//') + tests=$(grep -m1 '^Tests:' /tmp/unittest.log | sed 's/Tests:[[:space:]]*//') + echo "| suite group | result |" + echo "|-------------|--------|" + echo "| \`helm unittest\` suites | ${suites:-unknown} |" + echo "| \`helm unittest\` tests | ${tests:-unknown} |" + echo "| preStop drain behaviour | $(verdict "${{ steps.drain.outcome }}") |" + echo "| init-script behaviour | $(verdict "${{ steps.init-script.outcome }}") |" + else + echo "\`helm unittest\` did not run." + fi + + if [[ "${{ steps.unittest.outcome }}" != "success" && -f /tmp/unittest.log ]]; then + echo + echo "
Failing suites" + echo + echo '```' + grep -A3 '^ FAIL' /tmp/unittest.log | head -60 || true + echo '```' + echo + echo "
" + fi + } >> "$GITHUB_STEP_SUMMARY" + helm-ct-install: runs-on: ubuntu-latest - needs: [helm-ct-lint, helm-unittest] + needs: [changes, helm-ct-lint, helm-unittest] + # The expensive job: four legs, each standing up Graylog, a datanode and a + # MongoDB replica set. Only a change that alters what the chart renders can + # justify it. + if: needs.changes.outputs.chart == 'true' + # Backstop only. A healthy leg spends ~2 minutes on setup and a few more + # bringing the stack up; the default job ceiling is 360 minutes, so without + # this a wedged install burns hours per leg before anyone notices. Namespace + # termination can genuinely hang on finalizers, and that is not covered by + # `helm-extra-args: --timeout=900s`, which bounds only the install itself. + timeout-minutes: 30 strategy: fail-fast: false # Asymmetric K8s × Helm matrix: full K8s coverage on Helm 3 (the chart's @@ -125,34 +430,59 @@ jobs: k8s-version: v1.34.3 steps: - name: Checkout - uses: actions/checkout@v6 - with: - fetch-depth: 0 + # Shallow on purpose: `ct` is invoked with --all and never diffs against a + # target branch, so no job past `changes` needs history. + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Set up Helm - uses: azure/setup-helm@v5 + uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 with: version: ${{ matrix.helm-version }} - name: Set up chart-testing - uses: helm/chart-testing-action@v2.8.0 + uses: helm/chart-testing-action@6ec842c01de15ebb84c8627d2744a0c2f2755c9f # v2.8.0 - name: Create kind cluster - uses: helm/kind-action@v1.14.0 + uses: helm/kind-action@ef37e7f390d99f746eb8b610417061a60e82a6cc # v1.14.0 with: node_image: kindest/node:${{ matrix.k8s-version }} - name: Install MongoDB Kubernetes Operator + # The operator's own chart requests 500m CPU by default — an eighth of + # the runner for a controller reconciling a single one-member replica + # set. Shrinking it leaves that CPU for the workloads under test. run: | helm upgrade --install mongodb-kubernetes-operator mongodb-kubernetes \ --repo https://mongodb.github.io/helm-charts \ --version "1.6.1" \ --set operator.watchNamespace="*" \ + --set operator.resources.requests.cpu=100m \ + --set operator.resources.requests.memory=200Mi \ + --set operator.resources.limits.cpu=500m \ + --set operator.resources.limits.memory=500Mi \ --namespace operators \ --create-namespace \ --wait \ --timeout 5m + # The CPU budget on this node decides whether the stack can be scheduled at + # all. Printing it once makes "Insufficient cpu" a one-line diagnosis + # instead of an inference from pod descriptions. + - name: Show node capacity and reservations + run: | + kubectl get nodes -o custom-columns=\ + 'NAME:.metadata.name,CPU:.status.allocatable.cpu,MEM:.status.allocatable.memory' + kubectl describe node | sed -n '/Allocated resources/,/Events/p' + + # Carried into the job summary: whether the stack fits is the single + # most common reason this job fails, and CPU is the binding constraint. + cpu=$(kubectl get nodes -o jsonpath='{.items[0].status.allocatable.cpu}') + mem=$(kubectl get nodes -o jsonpath='{.items[0].status.allocatable.memory}') + { + echo "NODE_CPU=$cpu" + echo "NODE_MEM=$mem" + } >> "$GITHUB_ENV" + - name: Generate ephemeral CI rootPassword and mask in workflow logs id: ci-root-password run: | @@ -160,20 +490,114 @@ jobs: echo "::add-mask::${password}" echo "value=${password}" >> "$GITHUB_OUTPUT" + # Written before the install rather than after, so the summary says what + # was attempted even when the install times out or the job is cancelled. + - name: Report values used + # Advisory only: this step describes the run, it must never fail it. + continue-on-error: true + run: | + helm template ci charts/graylog -f charts/graylog/ci/ci-values.yaml \ + | yq ea -o=json '[.]' - \ + | python3 .github/scripts/ci_values_summary.py \ + --overlay charts/graylog/ci/ci-values.yaml \ + --extra-set "--set graylog.config.rootPassword=" \ + --heading "Values installed — Helm ${{ matrix.helm-version }} / Kubernetes ${{ matrix.k8s-version }}" \ + >> "$GITHUB_STEP_SUMMARY" + + # All sizing lives in charts/graylog/ci/ci-values.yaml, which ct discovers + # automatically. It used to be duplicated here as --set flags, which take + # precedence over the values file — so ci-values.yaml's graylog memory + # request was silently overridden and the two disagreed. Only the generated + # password is passed here, because it cannot be committed. + # + # `helm --wait` prints nothing until it succeeds or times out, so this step + # used to sit silent for the full 15 minutes and a stuck install looked + # exactly like a slow one. The watcher below streams anything not yet + # Running plus recent warning events, so a pod that cannot be scheduled is + # visible within seconds instead of in a post-mortem dump. - name: Run ct install + id: ct-install run: | + started=$(date +%s) + echo "INSTALL_STARTED=$started" >> "$GITHUB_ENV" + + watch_cluster() { + while true; do + echo "::group::cluster state $(date -u +%H:%M:%S)" + kubectl get pods -A \ + --field-selector=status.phase!=Running,status.phase!=Succeeded \ + -o wide 2>/dev/null || true + kubectl get events -A --field-selector type=Warning \ + --sort-by=.lastTimestamp 2>/dev/null | tail -8 || true + echo "::endgroup::" + + # Latest full pod state, kept for the job summary: ct deletes its + # namespace when it finishes, so by summary time the workloads + # under test are gone and cannot be queried any more. + kubectl get pods -A -o custom-columns=\ + 'NS:.metadata.namespace,POD:.metadata.name,CPU:.spec.containers[*].resources.requests.cpu,MEM:.spec.containers[*].resources.requests.memory,STATUS:.status.phase' \ + > /tmp/pods-latest.txt 2>/dev/null || true + + sleep 20 + done + } + watch_cluster & + watcher=$! + trap 'kill "$watcher" 2>/dev/null || true' EXIT + + set -o pipefail ct install --config .github/ct.yaml --all \ --helm-extra-set-args "\ - --set graylog.config.rootPassword=${{ steps.ci-root-password.outputs.value }} \ - --set graylog.replicas=1 \ - --set datanode.replicas=1 \ - --set mongodb.replicas=1 \ - --set mongodb.arbiters=0 \ - --set graylog.resources.requests.cpu=250m \ - --set graylog.resources.requests.memory=512Mi \ - --set-string graylog.resources.limits.cpu=1 \ - --set graylog.resources.limits.memory=1Gi \ - --set datanode.resources.requests.cpu=250m \ - --set datanode.resources.requests.memory=1Gi \ - --set-string datanode.resources.limits.cpu=1 \ - --set datanode.resources.limits.memory=2Gi" + --set graylog.config.rootPassword=${{ steps.ci-root-password.outputs.value }}" \ + 2>&1 | tee /tmp/ct-install.log + + - name: Summary + if: always() + run: | + set -u + elapsed="unknown" + if [[ -n "${INSTALL_STARTED:-}" ]]; then + elapsed="$(( ( $(date +%s) - INSTALL_STARTED ) / 60 ))m $(( ( $(date +%s) - INSTALL_STARTED ) % 60 ))s" + fi + passed="${{ steps.ct-install.outcome }}" + + { + echo "## Install — Helm ${{ matrix.helm-version }} on Kubernetes ${{ matrix.k8s-version }}" + echo + echo "| | |" + echo "|--|--|" + echo "| result | $( [[ "$passed" == "success" ]] && echo "passed" || echo "**FAILED**" ) |" + echo "| install duration | ${elapsed} |" + echo "| node allocatable | ${NODE_CPU:-?} CPU / ${NODE_MEM:-?} |" + echo "| sizing | \`charts/graylog/ci/ci-values.yaml\` |" + echo + + # What was actually admitted to the node, which is what "does it + # fit" means in practice, snapshotted by the watcher before ct tore + # the namespace down. + echo "
Pod requests, as last observed during the install" + echo + echo '```' + cat /tmp/pods-latest.txt 2>/dev/null || echo "no snapshot captured" + echo '```' + echo + echo "
" + + if [[ "$passed" != "success" ]]; then + echo + echo "### Why it failed" + echo + echo '```' + grep -iE 'error|failed|timed out|unhealthy' /tmp/ct-install.log 2>/dev/null | tail -20 || echo "no log" + echo '```' + echo + echo "
Pods not Running, and recent warnings" + echo + echo '```' + kubectl get pods -A --field-selector=status.phase!=Running,status.phase!=Succeeded -o wide 2>/dev/null || true + kubectl get events -A --field-selector type=Warning --sort-by=.lastTimestamp 2>/dev/null | tail -25 || true + echo '```' + echo + echo "
" + fi + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/release-please.yaml b/.github/workflows/release-please.yaml index 0c5d263..96a447e 100644 --- a/.github/workflows/release-please.yaml +++ b/.github/workflows/release-please.yaml @@ -18,7 +18,7 @@ jobs: release-please: runs-on: ubuntu-latest steps: - - uses: googleapis/release-please-action@v5 + - uses: googleapis/release-please-action@45996ed1f6d02564a971a2fa1b5860e934307cf7 # v5 id: release with: token: ${{ secrets.GITHUB_TOKEN }} diff --git a/charts/graylog/ci/ci-values.yaml b/charts/graylog/ci/ci-values.yaml index 81eeaed..13a68e2 100644 --- a/charts/graylog/ci/ci-values.yaml +++ b/charts/graylog/ci/ci-values.yaml @@ -4,6 +4,12 @@ graylog: replicas: 1 + # The chart default is 300s, to give the preStop drain room to flush the + # journal before SIGKILL. ct deletes the namespace after every install and + # waits for termination, so that budget is paid on every leg of the matrix. + # Nothing here holds data worth draining, and the drain hook is off by + # default anyway. Do not change the chart default. + terminationGracePeriodSeconds: 30 config: serverJavaOpts: "-Xms512m -Xmx768m" resources: