sh: support building multiple kernel variants

.buildkite/pipeline_pr.py

@@ -4,13 +4,56 @@
 
 """Generate Buildkite pipelines dynamically"""
 
+from pathlib import Path
+
 from common import (
     BKPipeline,
     ci_artifacts_change_mode,
     get_changed_files,
     run_all_tests,
 )
 
+# Secret hiding kernel variants live in per-variant subfolders here. The
+# generator runs from the repo root, hence the repo-relative path.
+HIDING_KERNELS_DIR = Path("resources/hiding_ci/kernels")
+
+
+def hiding_kernel_variants():
+    """Discover the secret hiding kernel variants to build."""
+    if not HIDING_KERNELS_DIR.is_dir():
+        return []
+    return sorted(p.name for p in HIDING_KERNELS_DIR.iterdir() if p.is_dir())
+
+
+def affected_hiding_kernel_variants(changed_files):
+    """Return hiding kernel variants whose inputs changed."""
+    variants = hiding_kernel_variants()
+
+    if not changed_files:
+        return variants
+
+    affected = set()
+    shared_hiding_ci_changed = False
+
+    for path in changed_files:
+        parts = path.parts
+
+        if len(parts) < 2 or parts[:2] != ("resources", "hiding_ci"):
+            continue
+
+        if len(parts) >= 4 and parts[:3] == ("resources", "hiding_ci", "kernels"):
+            variant = parts[3]
+            if variant in variants:
+                affected.add(variant)
+        else:
+            shared_hiding_ci_changed = True
+
+    if shared_hiding_ci_changed:
+        return variants
+
+    return sorted(affected)
+
+
 # Buildkite default job priority is 0. Setting this to 1 prioritizes PRs over
 # scheduled jobs and other batch jobs.
 DEFAULT_PRIORITY = 1
@@ -85,13 +128,17 @@
         depends_on_build=False,
     )
 
-if not changed_files or (
-    any(parent.name == "hiding_ci" for x in changed_files for parent in x.parents)
-):
+for variant in affected_hiding_kernel_variants(changed_files):
+    # One build job per variant x arch, so each variant gets parallel
+    # wall-clock and its own pass/fail signal in the Buildkite UI.
     pipeline.build_group_per_arch(
-        "🕵️ Build Secret Hiding Kernel",
+        f"🕵️ Build Secret Hiding Kernel [{variant}]",
         pipeline.devtool_test(
-            pytest_opts="-m secret_hiding integration_tests/build/test_hiding_kernel.py",
+            pytest_opts=(
+                "-m secret_hiding "
+                f'-k "test_build_hiding_kernel[{variant}]" '
+                "integration_tests/build/test_hiding_kernel.py"
+            ),
         ),
         depends_on_build=False,
     )

resources/hiding_ci/README.md

@@ -0,0 +1,74 @@
+# Secret Hiding kernel CI
+
+This directory builds the "Secret Freedom" / direct-map-removal kernels used by
+Firecracker's secret-hiding CI. Its layout lets you build **multiple kernel
+variants of different versions** while sharing as much as possible between them.
+
+## Layout
+
+```
+resources/hiding_ci/
+├── build_and_install_kernel.sh   # builds (and optionally installs) a variant
+├── apply_kernel_patches.sh       # applies a variant's patch series to a kernel tree
+├── dkms.conf                     # shared ENA driver DKMS config (AL2023)
+├── install_ena.sh                # shared ENA driver installer (AL2023)
+├── kernel_url                    # shared default git repo to clone from
+├── base_config                   # shared base kernel config overrides
+└── kernels/
+    └── <version>-<feature>/      # one subfolder per variant, e.g. 6.18-secret-hiding
+        ├── kernel_commit_hash    # REQUIRED: base commit to check out
+        ├── kernel_url            # OPTIONAL: repo override (defaults to ../../kernel_url)
+        ├── config_overrides      # OPTIONAL: config overrides merged on top of base_config
+        └── linux_patches/        # REQUIRED: patch series applied on top of the base commit
+            ├── GPL-2.0           # license for the patches (travels with them)
+            ├── README.md
+            └── NN-feature/*.patch
+```
+
+### Shared vs per-variant
+
+- **Shared (root):** the build scripts, the ENA helpers, the default
+  `kernel_url`, and `base_config` (config knobs common to every hiding kernel).
+- **Per-variant (`kernels/<variant>/`):** the base commit, the patch series, and
+  any version-specific config or repo overrides.
+
+The build layers config overrides **base first, then the variant's
+`config_overrides` on top** (later values win), matching
+`scripts/kconfig/merge_config.sh -m` semantics. `resources/rebuild.sh` uses the
+same approach for guest kernels.
+
+## Naming convention
+
+Name variant folders `<base-version>-<feature>`, e.g. `6.18-secret-hiding`. The
+version prefix is the upstream kernel version of the pinned base commit
+(`make -s kernelversion` at that commit).
+
+## Adding a new variant
+
+1. Create `kernels/<version>-<feature>/`.
+1. Add `kernel_commit_hash` with the base commit to check out.
+1. Add a `linux_patches/` directory with the patch series (include a `GPL-2.0`
+   copy alongside the patches, since the license must travel with them).
+1. Optionally add `kernel_url` if the variant pulls from a different repo than
+   the shared root default.
+1. Optionally add `config_overrides` for config knobs specific to this variant;
+   anything common to all variants belongs in the root `base_config`.
+
+The pytest test (`tests/integration_tests/build/test_hiding_kernel.py`) and the
+Buildkite PR pipeline (`.buildkite/pipeline_pr.py`) discover variants from
+`kernels/`, so a new variant gets its own build job with no further wiring.
+
+## Building locally
+
+```sh
+cd resources/hiding_ci
+# Build a specific variant without installing it, cleaning up the temp tree after:
+./build_and_install_kernel.sh 6.18-secret-hiding --no-install --tidy
+
+# If exactly one variant exists, the selector may be omitted:
+./build_and_install_kernel.sh --no-install --tidy
+```
+
+Pass `--install` (or answer the prompt) to install the built kernel. Use
+`apply_kernel_patches.sh <variant>` from inside an already checked-out kernel
+tree to apply the patch series on its own.

resources/hiding_ci/apply_kernel_patches.sh

@@ -34,9 +34,38 @@ apply_all_patches() {
   done
 }
 
-SCRIPT_DIR="$(dirname "$0")"
-KERNEL_COMMIT_HASH=$(cat "$SCRIPT_DIR"/kernel_commit_hash)
-KERNEL_PATCHES_DIR="$SCRIPT_DIR"/linux_patches
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+
+list_variants() {
+  for d in "$SCRIPT_DIR"/kernels/*/; do
+    [ -d "$d" ] && basename "$d"
+  done
+}
+
+# The variant is the first argument. Default to the sole variant if exactly one
+# exists, otherwise require an explicit choice.
+VARIANT="${1:-}"
+if [ -z "$VARIANT" ]; then
+  mapfile -t _variants < <(list_variants)
+  if [ "${#_variants[@]}" -eq 1 ]; then
+    VARIANT="${_variants[0]}"
+  else
+    echo "Usage: $0 <variant>" >&2
+    echo "Available variants:" >&2
+    list_variants | sed 's/^/  - /' >&2
+    exit 1
+  fi
+fi
+
+VARIANT_DIR="$SCRIPT_DIR/kernels/$VARIANT"
+if [ ! -d "$VARIANT_DIR" ]; then
+  echo "Unknown variant '$VARIANT'. Available variants:" >&2
+  list_variants | sed 's/^/  - /' >&2
+  exit 1
+fi
+
+KERNEL_COMMIT_HASH=$(cat "$VARIANT_DIR"/kernel_commit_hash)
+KERNEL_PATCHES_DIR="$VARIANT_DIR"/linux_patches
 
 HEAD_HASH="$(git rev-parse HEAD)"
 if [ $? != 0 ]; then

resources/hiding_ci/build_and_install_kernel.sh

@@ -116,12 +116,26 @@ check_new_config() {
 }
 
 check_override_presence() {
-  while IFS= read -r line; do
-    if ! grep -Fq "$line" .config; then
+  # Build the effective set of overrides across all override files. Later files
+  # win, so resolving the last value per CONFIG_* key avoids false failures when
+  # a variant deliberately overrides a base value (e.g. base X=y, variant X=n).
+  declare -A effective
+  local f line key
+  for f in "${CONFIG_OVERRIDE_FILES[@]}"; do
+    while IFS= read -r line; do
+      [ -z "$line" ] && continue
+      case "$line" in \#*) continue ;; esac
+      key="${line%%=*}"
+      effective["$key"]="$line"
+    done <"$f"
+  done
+
+  for line in "${effective[@]}"; do
+    if ! grep -Fxq "$line" .config; then
       echo "Missing config: $line"
       exit 1
     fi
-  done <"$KERNEL_CONFIG_OVERRIDES"
+  done
 
   echo "All overrides correctly applied.."
 }
@@ -166,17 +180,80 @@ update_boot_config() {
   esac
 }
 
+list_variants() {
+  for d in "$START_DIR"/kernels/*/; do
+    [ -d "$d" ] && basename "$d"
+  done
+}
+
+die_with_variants() {
+  echo "$1" >&2
+  echo "Available variants:" >&2
+  list_variants | sed 's/^/  - /' >&2
+  exit 1
+}
+
 check_userspace
 install_build_deps
 
-KERNEL_URL=$(cat kernel_url)
-KERNEL_COMMIT_HASH=$(cat kernel_commit_hash)
-KERNEL_PATCHES_DIR=$(pwd)/linux_patches
-KERNEL_CONFIG_OVERRIDES=$(pwd)/kernel_config_overrides
+START_DIR=$(pwd)
 
-TMP_BUILD_DIR=$(mktemp -d -t kernel-build-XXXX)
+# Separate the variant selector (first positional arg) from the --flags. The
+# flags are forwarded verbatim to `confirm`, so the variant name can never be
+# mistaken for an --install/--no-install/--tidy flag.
+VARIANT=""
+FLAGS=()
+for arg in "$@"; do
+  case "$arg" in
+  --*) FLAGS+=("$arg") ;;
+  *)
+    if [ -z "$VARIANT" ]; then
+      VARIANT="$arg"
+    else
+      echo "Unexpected extra argument: $arg" >&2
+      exit 1
+    fi
+    ;;
+  esac
+done
+
+# Default to the sole variant if exactly one exists, otherwise require a choice.
+if [ -z "$VARIANT" ]; then
+  mapfile -t _variants < <(list_variants)
+  if [ "${#_variants[@]}" -eq 1 ]; then
+    VARIANT="${_variants[0]}"
+    echo "No variant specified; defaulting to the only one: $VARIANT"
+  else
+    die_with_variants "No variant specified and multiple are available."
+  fi
+fi
 
-START_DIR=$(pwd)
+VARIANT_DIR="$START_DIR/kernels/$VARIANT"
+[ -d "$VARIANT_DIR" ] || die_with_variants "Unknown variant '$VARIANT'."
+
+echo "Building kernel variant: $VARIANT"
+
+# Repository URL: per-variant override falls back to the shared root default.
+if [ -f "$VARIANT_DIR/kernel_url" ]; then
+  KERNEL_URL=$(cat "$VARIANT_DIR/kernel_url")
+else
+  KERNEL_URL=$(cat "$START_DIR/kernel_url")
+fi
+
+# Commit hash and patches are per-variant.
+[ -f "$VARIANT_DIR/kernel_commit_hash" ] ||
+  die_with_variants "Variant '$VARIANT' is missing kernel_commit_hash."
+KERNEL_COMMIT_HASH=$(cat "$VARIANT_DIR/kernel_commit_hash")
+KERNEL_PATCHES_DIR="$VARIANT_DIR/linux_patches"
+
+# Config overrides: shared base first, then optional per-variant file on top.
+# Order matters: merge_config.sh -m applies later files over earlier ones.
+CONFIG_OVERRIDE_FILES=("$START_DIR/base_config")
+if [ -f "$VARIANT_DIR/config_overrides" ]; then
+  CONFIG_OVERRIDE_FILES+=("$VARIANT_DIR/config_overrides")
+fi
+
+TMP_BUILD_DIR=$(mktemp -d -t kernel-build-XXXX)
 
 cd $TMP_BUILD_DIR
 
@@ -202,8 +279,8 @@ make olddefconfig
 scripts/config --disable SYSTEM_TRUSTED_KEYS
 scripts/config --disable SYSTEM_REVOCATION_KEYS
 
-# Apply our config overrides on top of the config
-scripts/kconfig/merge_config.sh -m .config $KERNEL_CONFIG_OVERRIDES
+# Apply our config overrides on top of the config (base first, variant on top)
+scripts/kconfig/merge_config.sh -m .config "${CONFIG_OVERRIDE_FILES[@]}"
 
 check_override_presence
 
@@ -222,7 +299,8 @@ KERNEL_VERSION=$(KERNELVERSION=$(make -s kernelversion) ./scripts/setlocalversio
 echo "New kernel version:" $KERNEL_VERSION
 
 # Make sure a user really wants to install this kernel
-confirm "$@"
+# Forward only the --flags, never the variant selector.
+confirm "${FLAGS[@]}"
 
 check_root
 

resources/hiding_ci/kernels/6.18-amazon-linux/kernel_commit_hash

@@ -0,0 +1 @@
+275d294b2b24abcd65452198551cd8a5b8d4f775

resources/hiding_ci/kernels/6.18-amazon-linux/kernel_url

@@ -0,0 +1 @@
+https://github.com/amazonlinux/linux.git

tests/integration_tests/build/test_hiding_kernel.py

@@ -1,17 +1,33 @@
 # Copyright 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
-"""A test which checks that the secret hiding enable kernel builds successfully."""
+"""A test which checks that the secret hiding kernel variants build successfully."""
+
+from pathlib import Path
 
 import pytest
 
 from framework import utils
 
+# Tests run with `tests/` as the working directory, so the hiding CI assets live
+# one level up under `resources/`.
+HIDING_CI_DIR = Path("../resources/hiding_ci")
+KERNELS_DIR = HIDING_CI_DIR / "kernels"
+
+
+def _discover_variants():
+    """Discover the kernel variants available under hiding_ci/kernels."""
+    if not KERNELS_DIR.is_dir():
+        return []
+    return sorted(p.name for p in KERNELS_DIR.iterdir() if p.is_dir())
+
 
 @pytest.mark.timeout(600)
 @pytest.mark.secret_hiding
-def test_build_hiding_kernel():
+@pytest.mark.parametrize("variant", _discover_variants())
+def test_build_hiding_kernel(variant):
     """
-    In the test we will run our kernel build script to check it succeeds and builds the hidden kernel
+    In the test we will run our kernel build script for each secret hiding
+    variant to check it succeeds and builds the hidden kernel
     """
 
     # We have some extra deps for building the kernel that are not in the dev container
@@ -26,5 +42,5 @@ def test_build_hiding_kernel():
     utils.check_output('git config --global user.email "ci@email.com"')
 
     utils.check_output(
-        "cd ../resources/hiding_ci; ./build_and_install_kernel.sh --no-install --tidy"
+        f"cd {HIDING_CI_DIR}; ./build_and_install_kernel.sh {variant} --no-install --tidy"
     )