Migrate CI workflow to scripts, allow ci/validate (#1377)

### What does this PR do?

This commit builds on the insights gained from the automated bollard
    dependency update and moves our CI correctness testing into locally
    executable scripts with a single 'validate' to call the whole suite.
    CLAUDE.md is updated to prefer the use of `ci/validate`.

Author: blt

.github/workflows/ci.yml

@@ -7,6 +7,13 @@ on:
       - main
 
 jobs:
+  shellcheck:
+    name: Shellcheck
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - run: ci/shellcheck
+
   rust_actions:
     name: Rust Actions (Check/Fmt/Clippy)
     runs-on: ${{ matrix.os }}
@@ -20,17 +27,17 @@ jobs:
             protobuf: true
             fuse: true
             components: ""
-            command: cargo check --locked --all-features
+            command: ci/check
           - tool: fmt
             protobuf: true
             fuse: true
             components: "rustfmt"
-            command: cargo fmt --all -- --check
+            command: ci/fmt
           - tool: clippy
             protobuf: true
             fuse: true
             components: "clippy"
-            command: cargo clippy --all-features
+            command: ci/clippy
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - uses: actions-rust-lang/setup-rust-toolchain@9d7e65c320fdb52dcd45ffaa68deb6c02c8754d9 # v1.10
@@ -49,7 +56,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-      - uses: EmbarkStudios/cargo-deny-action@34899fc7ba81ca6268d5947a7a16b4649013fea1 # v2.0.11
+      - uses: actions-rust-lang/setup-rust-toolchain@9d7e65c320fdb52dcd45ffaa68deb6c02c8754d9 # v1.10
+      - name: Install Protobuf
+        uses: ./.github/actions/install-protobuf
+      - name: Install FUSE
+        uses: ./.github/actions/install-fuse
+      - run: ci/deny
 
   test:
     name: Test Suite
@@ -63,7 +75,7 @@ jobs:
         uses: ./.github/actions/install-fuse
       - name: Install nextest
         uses: taiki-e/install-action@49be99c627fae102cb8c86414e9605869641782a # nextest
-      - run: cargo nextest run --all-features
+      - run: ci/test
 
   integration-test:
     name: Integration Tests
@@ -75,7 +87,7 @@ jobs:
         uses: ./.github/actions/install-protobuf
       - name: Install FUSE
         uses: ./.github/actions/install-fuse
-      - run: cargo test --locked -p sheepdog
+      - run: ci/integration-test
         timeout-minutes: 30
 
   kani:
@@ -93,43 +105,23 @@ jobs:
         uses: ./.github/actions/install-fuse
       - name: Install kani
         run: cargo install kani-verifier
-      - run: cargo kani --solver cadical
-        working-directory: ${{ matrix.crate }}
+      - run: ci/kani ${{ matrix.crate }}
         timeout-minutes: 30
 
-  loom:
-    name: Loom Proofs
+  buf:
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        crate: [lading_signal]
     steps:
+      # Check our protobufs for lint cleanliness and for lack of breaking
+      # changes
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - uses: actions-rust-lang/setup-rust-toolchain@9d7e65c320fdb52dcd45ffaa68deb6c02c8754d9 # v1.10
       - name: Install Protobuf
         uses: ./.github/actions/install-protobuf
       - name: Install FUSE
         uses: ./.github/actions/install-fuse
-      - name: Install nextest
-        uses: taiki-e/install-action@49be99c627fae102cb8c86414e9605869641782a # nextest
-      - run: RUSTFLAGS="--cfg loom" cargo nextest run --release
-        working-directory: ${{ matrix.crate }}
-        timeout-minutes: 30
-
-  buf:
-    runs-on: ubuntu-latest
-    steps:
-      # Check our protobufs for lint cleanliness and for lack of breaking
-      # changes
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - name: buf-setup
         uses: bufbuild/buf-setup-action@a47c93e0b1648d5651a065437926377d060baa99 # v1.50.0
-      - name: buf-lint
-        uses: bufbuild/buf-lint-action@06f9dd823d873146471cfaaf108a993fe00e5325 # v1.1.1
-      - name: buf-breaking
-        uses: bufbuild/buf-breaking-action@c57b3d842a5c3f3b454756ef65305a50a587c5ba # v1.1.4
-        with:
-          against: 'https://github.com/datadog/lading.git#branch=main'
+      - run: ci/buf
 
   actionlint:
     runs-on: ubuntu-latest

CLAUDE.md

@@ -1,3 +1,17 @@
+# Architecture Overview
+
+Lading measures performance of long-running programs (daemons) using synthetic,
+repeatable load generation. It operates with three components:
+
+- **Generators**: Create deterministic load and push to targets
+- **Targets**: Programs being tested (run as subprocesses by lading)
+- **Blackholes**: Optionally receive output from targets
+
+Data flows: Generator → Target → Blackhole
+
+Key principle: Pre-compute everything possible. This minimizes runtime overhead
+and interference with the target being measured.
+
 # Design Goals
 
 Lading is a performance tool. It's behavior has consequences on the development
@@ -15,22 +29,45 @@ this means that lading will pre-allocate where possible, will explicitly thread
 randomness and other sources of non-determinism into code paths and will
 preference algorithms that have better worst-case behavior.
 
-# Code Style
+## Performance Guidance
+
+Always consider performance implications of changes. Focus on "obviously fast" patterns:
+- Pre-allocate collections when size is known
+- Avoid allocations in hot paths
+- Choose algorithms with good worst-case behavior over average-case
+- Prefer bounded operations over unbounded ones
+- Avoid unnecessary cloning or copying
+
+Do NOT dive into performance optimization rabbit holes without profiling data.
+Benchmarks alone are insufficient - real performance work requires profiling.
+Stick to obviously correct performance patterns.
 
-This project is subject attempts to automate code style to a great degree, using
-these tools:
+## Error Handling
 
-* `cargo clippy`
-* `cargo fmt`
+Lading MUST NOT panic. Use controlled shutdowns only. All errors should be
+propagated using Result types. Panics are only acceptable for truly exceptional
+circumstances that cannot be handled (e.g., fundamental invariant violations that
+indicate memory corruption).
+
+When handling errors:
+- Always use Result<T, E> for fallible operations
+- Propagate errors up to where they can be meaningfully handled
+- Provide context when wrapping errors
+- Design for graceful degradation where possible
+
+# Code Style
+
+This project enforces code style through automated tooling. Use `ci/validate` to
+check style compliance - it will run formatting and linting checks for you.
 
 We do not allow for warnings: all warnings are errors. Deprecation warnings MUST
 be treated as errors. Lading is written in a "naive" style where abstraction is
 not preferred if a duplicated pattern will satisfy. Our reasoning for this is it
 makes ramp-up for new engineers easier: all you must do is follow the pattern,
 not internalize a complex type hierarchy. There are obvious places in the code
 base where replicated patterns have been made into abstractions -- we follow the
-"shape" rule, if you have three or more repeats, make a jig -- but we do not
-start there.
+"shape" rule: if you have three or more repeats, make a jig. This is a hard rule:
+3+ repetitions = create an abstraction. Less than 3 = duplicate the pattern.
 
 Lading is written to be as easy to contribute to as possible. We ask that any
 dependency used in the project in more than one crate be present in the
@@ -46,28 +83,98 @@ or two of code. Do add comments that explain the "why" of a block of code, how
 it functions or is unusual in a way that an experienced engineer might not
 understand.
 
+Always add comments for:
+- Non-obvious performance optimizations
+- Complex state machines or control flow
+- Unusual algorithm choices
+- Workarounds for external limitations
+- Any code that would make an experienced engineer pause and wonder "why?"
+
 Crate versions are always given as XX.YY and not XX.YY.ZZ.
 
 # Testing
 
-Where possible lading prefers property tests over unit tests and in especially
-critical components we require proofs. We use
-[proptest](https://github.com/proptest-rs/proptest) in this project for property
-tests and the [kani](https://github.com/model-checking/kani) proof tool.
+ALWAYS prefer property tests over unit tests. Unit tests are insufficient for
+lading's requirements. We use [proptest](https://github.com/proptest-rs/proptest)
+for property testing.
+
+Critical components are those which must function correctly or lading itself
+cannot function. These require proofs using [kani](https://github.com/model-checking/kani):
+- Throttling (MUST be correct - lading is used to make throughput claims)
+- Core data generation
+- Fundamental algorithms
 
-## Workflow
+The throttle is especially critical: incorrect throttling invalidates all
+performance claims made using lading.
 
-Changes to lading are subject this flow:
+When writing tests:
+- Default to property tests
+- Unit tests are only acceptable for simple pure functions
+- Think about the properties your code should maintain
+- Test invariants, not specific examples
 
-* `cargo check`
-* `cargo clippy`
-* `cargo nextest run`
+# CRITICAL: Validation Workflow
 
-Proofs must be run with the `cargo kani` tool in the crate where proofs reside.
+You MUST run `ci/validate` after making any code changes. This is not optional.
+The script runs all checks in optimal order and exits on first failure.
 
+Do not run individual cargo commands - use the ci scripts instead:
+- Use `ci/validate` for full validation
+- Use `ci/fmt` instead of `cargo fmt`
+- Use `ci/check` instead of `cargo check`
+- Use `ci/clippy` instead of `cargo clippy`
+- Use `ci/test` instead of `cargo nextest run`
+- Use `ci/kani <crate>` for kani proofs (valid crates: lading_throttle, lading_payload)
+- Use `ci/outdated` instead of `cargo outdated`
 
 # Tools
 
-To identify outdated dependencies: `cargo outdated --root-deps-only`.
+To identify outdated dependencies: Use `ci/outdated`
 
 To run micro-benchmarks: `cargo criterion`
+
+## Dependencies
+
+Due to lading's unusual constraints (performance, determinism, correctness),
+we often must implement functionality rather than use external crates.
+
+Before suggesting a new dependency, consider:
+- Does it meet our performance requirements?
+- Is it deterministic?
+- Does it give us sufficient control?
+- Is the functionality core to lading's purpose?
+
+When in doubt, implement rather than import.
+
+## Common Patterns and Anti-Patterns
+
+### DO:
+- Pre-allocate buffers and collections when size is known
+- Use bounded channels/queues over unbounded
+- Design for worst-case scenarios
+- Use fixed-size data structures where possible
+- Propagate errors up to where they can be handled meaningfully
+
+### DON'T:
+- Use `.unwrap()` or `.expect()` (these panic)
+- Allocate in hot paths
+- Use unbounded growth patterns
+- Add external dependencies without careful consideration
+- Implement complex abstractions for fewer than 3 use cases
+
+# Key Reminders for Claude
+
+1. ALWAYS use `ci/validate` after code changes - never skip this
+2. Do NOT run cargo commands directly - use the ci/ scripts
+3. When users ask about build/test failures, suggest running `ci/validate`
+4. If users are confused about project correctness criteria, point them to `ci/validate`
+5. The ci/ scripts are the single source of truth for validation
+6. Always consider performance - suggest "obviously fast" patterns
+7. Do NOT suggest performance optimizations without profiling data
+8. When in doubt, prefer pre-allocation and good worst-case behavior
+9. NEVER suggest code that could panic - use Result types
+10. Be skeptical of external dependencies - we often need custom implementations
+11. Remember the throttle is critical - any changes there need extra scrutiny
+12. Generators must be deterministic - no randomness without explicit seeding
+13. Pre-compute in initialization, not in hot paths
+14. Think about how your code affects the measurement of the target

ci/buf

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+set -o errexit
+set -o nounset
+set -o pipefail
+
+# Check if buf is installed
+if ! command -v buf &> /dev/null; then
+    echo "buf is not installed. Please install it first."
+    echo "See: https://docs.buf.build/installation"
+    exit 1
+fi
+
+echo "Running buf lint..."
+buf lint
+
+echo "Running buf breaking check..."
+buf breaking --against 'https://github.com/datadog/lading.git#branch=main'
+
+echo "Buf checks passed!"

ci/check

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+set -o errexit
+set -o nounset
+set -o pipefail
+
+echo "Running cargo check..."
+cargo check --locked --all-features
+
+echo "Cargo check passed!"

ci/clippy

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+set -o errexit
+set -o nounset
+set -o pipefail
+
+echo "Running cargo clippy..."
+cargo clippy --all-features
+
+echo "Cargo clippy passed!"

ci/deny

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+set -o errexit
+set -o nounset
+set -o pipefail
+
+# Check if cargo-deny is installed
+if ! command -v cargo-deny &> /dev/null; then
+    echo "cargo-deny is not installed. Installing it now..."
+    cargo install cargo-deny
+fi
+
+echo "Running cargo deny..."
+cargo deny check
+
+echo "Cargo deny passed!"

ci/fmt

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+set -o errexit
+set -o nounset
+set -o pipefail
+
+echo "Running cargo fmt..."
+cargo fmt --all -- --check
+
+echo "Cargo fmt passed!"

ci/integration-test

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+set -o errexit
+set -o nounset
+set -o pipefail
+
+echo "Running integration tests..."
+cargo test --locked -p sheepdog
+
+echo "Integration tests passed!"