bare_metal.yaml

# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: LicenseRef-NvidiaProprietary

# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual
# property and proprietary rights in and to this material, related
# documentation and any modifications thereto. Any use, reproduction,
# disclosure or distribution of this material and related documentation
# without an express license agreement from NVIDIA CORPORATION or
# its affiliates is strictly prohibited.

# Bare Metal (BMaaS) Validation - Canonical Configuration
#
# THE canonical bare-metal test suite. Provider configs import this file
# and override commands with platform-specific stubs.
#
# Architecture:
#   - Scripts: Platform-specific (YOU implement these) - do all BM work
#   - Validations: Platform-agnostic (provided) - just check JSON output
#
# Contract:
#   Each script MUST print a JSON object to stdout. The validations
#   only inspect that JSON. As long as your scripts output the right
#   fields, the validations pass regardless of which cloud you use.
#
# Steps:
#   1. launch_instance      (setup)    - Provision bare-metal GPU instance
#   2. list_instances       (test)     - List instances, verify target exists
#   3. verify_tags          (test)     - Verify user-defined tags on instance
#   4. topology_placement   (test)     - Validate topology-based placement support
#   5. serial_console       (test)     - Retrieve serial console output (read-only)
#   6. stop_instance        (test)     - Power off node, verify stopped state
#   7. start_instance       (test)     - Power on node, verify recovery
#   8. reboot_instance      (test)     - Reboot instance, validate recovery
#   9. power_cycle_instance (test)     - Hard power off + on, validate cold-start recovery
#  10. describe_instance    (test)     - Describe current state + SSH info
#                                        (runs after power_cycle so IP is current)
#  11. reinstall_instance   (test)     - Reinstall OS from stock image [skip: true by default]
#  12. deploy_nim           (test)     - Deploy NIM container via SSH
#  13. teardown_nim         (teardown) - Stop NIM container
#  14. teardown             (teardown) - Terminate instance
#  15. verify_teardown      (teardown) - Confirm resources deleted
#
# Dev workflow (reuse existing instance):
#   # Run 1: launch + test, keep instance alive
#   BM_SKIP_TEARDOWN=true uv run isvctl test run -f isvctl/configs/my-isv/bare_metal.yaml
#
#   # Run 2+: reuse instance
#   BM_INSTANCE_ID=<id> BM_KEY_FILE=/path/to/key.pem \
#     BM_SKIP_TEARDOWN=true uv run isvctl test run -f isvctl/configs/my-isv/bare_metal.yaml
#
# Quick start (new provider):
#   1. Create your provider folder: mkdir -p isvctl/configs/providers/my-isv/
#   2. Create providers/my-isv/bare_metal.yaml with: import: ../../tests/bare_metal.yaml
#   3. Override commands with your platform stubs
#   4. Run: uv run isvctl test run -f isvctl/configs/providers/my-isv/bare_metal.yaml
#
# See also:
#   - AWS reference implementation: ../aws/bare_metal.yaml + ../stubs/aws/bare_metal/
#   - JSON schemas: isvctl/src/isvctl/config/output_schemas.py

version: "1.0"

commands:
  bare_metal:
    phases: ["setup", "test", "teardown"]
    steps:
      # ─── Step 1: Launch Bare-Metal Instance ────────────────────────
      # YOUR SCRIPT must output JSON with at minimum:
      #   {
      #     "success": true,
      #     "platform": "bm",
      #     "instance_id": "<unique-id>",
      #     "public_ip": "<ip-address>",
      #     "key_file": "<path-to-ssh-key>",
      #     "vpc_id": "<network-id>",
      #     "state": "running",
      #     "security_group_id": "<sg-id>",
      #     "key_name": "<key-pair-name>"
      #   }
      - name: launch_instance
        phase: setup
        command: "python3 ../stubs/bare_metal/launch_instance.py"
        args:
          - "--name"
          - "isv-bm-test-gpu"
          - "--instance-type"
          - "{{instance_type}}"
          - "--region"
          - "{{region}}"
        timeout: 1200

      # ─── Step 2: List Instances ────────────────────────────────────
      # Reuses the VM list_instances stub (generic instance listing).
      - name: list_instances
        phase: test
        command: "python3 ../stubs/vm/list_instances.py"
        args:
          - "--vpc-id"
          - "{{steps.launch_instance.vpc_id}}"
          - "--instance-id"
          - "{{steps.launch_instance.instance_id}}"
          - "--region"
          - "{{region}}"
        timeout: 120

      # ─── Step 3: Verify Tags ────────────────────────────────────
      # Uses the bare-metal describe_tags stub (provider implements retrieval).
      # YOUR SCRIPT must output JSON with at minimum:
      #   {
      #     "success": true,
      #     "platform": "bm",
      #     "instance_id": "...",
      #     "tags": {"Name": "...", "CreatedBy": "isvtest"},
      #     "tag_count": 2
      #   }
      - name: verify_tags
        phase: test
        command: "python3 ../stubs/bare_metal/describe_tags.py"
        args:
          - "--instance-id"
          - "{{steps.launch_instance.instance_id}}"
          - "--region"
          - "{{region}}"
        timeout: 60

      # ─── Step 4: Topology Placement ──────────────────────────────
      # YOUR SCRIPT must validate topology-based placement support.
      #   {
      #     "success": true,
      #     "platform": "bm",
      #     "instance_id": "...",
      #     "placement_supported": true,
      #     "availability_zone": "...",
      #     "placement_group": "...",
      #     "placement_strategy": "cluster",
      #     "operations": {
      #       "create_group":    {"passed": true},
      #       "verify_instance": {"passed": true},
      #       "describe_group":  {"passed": true},
      #       "delete_group":    {"passed": true}
      #     }
      #   }
      - name: topology_placement
        phase: test
        command: "python3 ../stubs/bare_metal/topology_placement.py"
        args:
          - "--instance-id"
          - "{{steps.launch_instance.instance_id}}"
          - "--region"
          - "{{region}}"
        timeout: 120

      # ─── Step 5: Serial Console Access ──────────────────────────────
      # YOUR SCRIPT must output JSON with at minimum:
      #   {
      #     "success": true,
      #     "platform": "bm",
      #     "instance_id": "...",
      #     "console_available": true,
      #     "serial_access_enabled": true,
      #     "output_length": 4096
      #   }
      - name: serial_console
        phase: test
        command: "python3 ../stubs/bare_metal/serial_console.py"
        args:
          - "--instance-id"
          - "{{steps.launch_instance.instance_id}}"
          - "--region"
          - "{{region}}"
        timeout: 60

      # ─── Step 6: Stop Instance ────────────────────────────────────
      # YOUR SCRIPT must output JSON with at minimum:
      #   {
      #     "success": true,
      #     "platform": "bm",
      #     "instance_id": "...",
      #     "state": "stopped",
      #     "stop_initiated": true
      #   }
      - name: stop_instance
        phase: test
        command: "python3 ../stubs/bare_metal/stop_instance.py"
        args:
          - "--instance-id"
          - "{{steps.launch_instance.instance_id}}"
          - "--region"
          - "{{region}}"
        timeout: 1800

      # ─── Step 7: Start Instance ───────────────────────────────────
      # YOUR SCRIPT must output JSON with at minimum:
      #   {
      #     "success": true,
      #     "platform": "bm",
      #     "instance_id": "...",
      #     "state": "running",
      #     "public_ip": "...",
      #     "key_file": "...",
      #     "start_initiated": true,
      #     "ssh_ready": true
      #   }
      - name: start_instance
        phase: test
        command: "python3 ../stubs/bare_metal/start_instance.py"
        args:
          - "--instance-id"
          - "{{steps.launch_instance.instance_id}}"
          - "--region"
          - "{{region}}"
          - "--key-file"
          - "{{steps.launch_instance.key_file}}"
          - "--public-ip"
          - "{{steps.launch_instance.public_ip}}"
        timeout: 2700

      # ─── Step 8: Reboot Instance ──────────────────────────────────
      # BM-specific: longer waits for hardware POST/BIOS/OS boot.
      # YOUR SCRIPT must output JSON with at minimum:
      #   {
      #     "success": true,
      #     "platform": "bm",
      #     "instance_id": "...",
      #     "state": "running",
      #     "public_ip": "...",
      #     "key_file": "...",
      #     "reboot_initiated": true,
      #     "uptime_seconds": <int>,
      #     "ssh_ready": true
      #   }
      - name: reboot_instance
        phase: test
        command: "python3 ../stubs/bare_metal/reboot_instance.py"
        args:
          - "--instance-id"
          - "{{steps.launch_instance.instance_id}}"
          - "--region"
          - "{{region}}"
          - "--key-file"
          - "{{steps.launch_instance.key_file}}"
          - "--public-ip"
          - "{{steps.start_instance.public_ip}}"
        timeout: 1800

      # ─── Step 9: Power-Cycle Instance ────────────────────────────────
      # Hard power off + power on (cold start). Unlike reboot (OS-level),
      # this exercises firmware init, BIOS POST, and a cold OS boot.
      # YOUR SCRIPT must output JSON with at minimum:
      #   {
      #     "success": true,
      #     "platform": "bm",
      #     "instance_id": "...",
      #     "state": "running",
      #     "public_ip": "54.x.x.x",
      #     "key_file": "/tmp/key.pem",
      #     "power_cycle_initiated": true,
      #     "power_was_off": true,
      #     "ssh_ready": true,
      #     "recovery_seconds": 180
      #   }
      - name: power_cycle_instance
        phase: test
        command: "python3 ../stubs/bare_metal/power_cycle_instance.py"
        args:
          - "--instance-id"
          - "{{steps.launch_instance.instance_id}}"
          - "--region"
          - "{{region}}"
          - "--key-file"
          - "{{steps.launch_instance.key_file}}"
          - "--public-ip"
          - "{{steps.reboot_instance.public_ip}}"
        timeout: 2700

      # ─── Describe Instance ─────────────────────────────────────────
      # Runs AFTER power_cycle so the IP is current (stop/start may reassign).
      # This step is the SSH/GPU validation anchor: most host-level checks
      # bind to describe_instance for connection details.
      # YOUR SCRIPT must output JSON with at minimum:
      #   {
      #     "success": true,
      #     "platform": "bm",
      #     "instance_id": "...",
      #     "state": "running",
      #     "public_ip": "...",
      #     "key_file": "..."
      #   }
      - name: describe_instance
        phase: test
        command: "python3 ../stubs/bare_metal/describe_instance.py"
        args:
          - "--instance-id"
          - "{{steps.launch_instance.instance_id}}"
          - "--region"
          - "{{region}}"
          - "--key-file"
          - "{{steps.launch_instance.key_file}}"
        timeout: 120

      # ─── Step 11: Reinstall Instance ────────────────────────────────
      # Reinstall the node from its configured stock OS.
      # Skipped by default: can be very slow depending on platform.
      # Set skip: false and implement stubs/bare_metal/reinstall_instance.py to enable.
      # See ../aws/bare_metal.yaml + stubs/aws/bare_metal/reinstall_instance.py for reference.
      - name: reinstall_instance
        phase: test
        skip: true
        command: "python3 ../stubs/bare_metal/reinstall_instance.py"
        args:
          - "--instance-id"
          - "{{steps.launch_instance.instance_id}}"
          - "--region"
          - "{{region}}"
          - "--key-file"
          - "{{steps.launch_instance.key_file}}"
          - "--public-ip"
          - "{{steps.power_cycle_instance.public_ip}}"
        timeout: 3600

      # ─── Step 12: Deploy NIM Container ───────────────────────────
      # Uses post-power-cycle IP/key since power-cycle may reassign the public IP
      - name: deploy_nim
        phase: test
        command: "python3 ../stubs/common/deploy_nim.py"
        args:
          - "--host"
          - "{{steps.power_cycle_instance.public_ip}}"
          - "--key-file"
          - "{{steps.power_cycle_instance.key_file}}"
        timeout: 1800

      # ─── Step 13: Teardown NIM ───────────────────────────────────
      # Uses post-power-cycle IP/key to match what deploy_nim used
      - name: teardown_nim
        phase: teardown
        command: "python3 ../stubs/common/teardown_nim.py"
        args:
          - "--host"
          - "{{steps.power_cycle_instance.public_ip}}"
          - "--key-file"
          - "{{steps.power_cycle_instance.key_file}}"
        timeout: 120

      # ─── Step 14: Teardown Instance ──────────────────────────────
      - name: teardown
        phase: teardown
        command: "python3 ../stubs/bare_metal/teardown.py"
        args:
          - "--instance-id"
          - "{{steps.launch_instance.instance_id}}"
          - "--region"
          - "{{region}}"
          - "--delete-key-pair"
          - "--delete-security-group"
          - "{{teardown_flag}}"
        timeout: 1800

      # ─── Step 15: Verify Teardown ────────────────────────────────
      # Post-teardown sanitization: verify instance terminated and
      # resources deleted.
      - name: verify_teardown
        phase: teardown
        command: "python3 ../stubs/bare_metal/verify_terminated.py"
        args:
          - "--instance-id"
          - "{{steps.launch_instance.instance_id}}"
          - "--region"
          - "{{region}}"
          - "--security-group-id"
          - "{{steps.launch_instance.security_group_id}}"
          - "--key-name"
          - "{{steps.launch_instance.key_name}}"
        timeout: 120

tests:
  platform: bare_metal
  cluster_name: "bm-validation"
  description: "Bare metal (BMaaS) GPU validation tests"

  settings:
    # Provider-specific -- override in your provider config
    region: ""
    instance_type: "{{instance_type}}" # Supply your platform-equivalent bare-metal/GPU instance type
    teardown_flag: "{{(env.BM_SKIP_TEARDOWN == 'true') | ternary('--skip-destroy', '')}}"

  # ─── Validations ─────────────────────────────────────────────────
  # These are PROVIDER-AGNOSTIC. They only check JSON field names/values.
  validations:
    setup_checks:
      step: launch_instance
      checks:
        InstanceStateCheck:
          expected_state: "running"

    list_instances:
      step: list_instances
      checks:
        InstanceListCheck:
          min_count: 1

    tag_checks:
      step: verify_tags
      checks:
        InstanceTagCheck:
          required_keys: ["Name", "CreatedBy"]

    topology_placement:
      step: topology_placement
      checks:
        StepSuccessCheck: {}
        TopologyPlacementCheck: {}

    serial_console:
      step: serial_console
      checks:
        SerialConsoleCheck: {}

    cloud_init:
      step: launch_instance
      checks:
        CloudInitCheck: {}

    instance_info:
      step: describe_instance
      checks:
        InstanceStateCheck:
          expected_state: "running"

    ssh:
      step: describe_instance
      checks:
        ConnectivityCheck: {}
        OsCheck:
          expected_os: "ubuntu"

    gpu:
      step: describe_instance
      checks:
        GpuCheck:
          expected_gpus: 8

    host_os:
      step: describe_instance
      checks:
        HostSoftwareCheck: {}

    # GPU stress test - PyTorch matrix multiplications on all GPUs
    gpu_stress:
      step: describe_instance
      checks:
        GpuStressCheck:
          runtime: 30
          memory_gb: 16

    # NCCL AllReduce - validates GPU-to-GPU communication (NVLink/NVSwitch)
    nccl:
      step: describe_instance
      checks:
        NcclCheck: {}

    # DDP training workload - validates full training stack
    training:
      step: describe_instance
      checks:
        TrainingCheck:
          steps: 50

    # Network / interconnect checks
    nvlink:
      step: describe_instance
      checks:
        NvlinkCheck: {}

    infiniband:
      step: describe_instance
      checks:
        InfiniBandCheck: {}

    ethernet:
      step: describe_instance
      checks:
        EthernetCheck:
          ping_target: "8.8.8.8"

    driver_info:
      step: describe_instance
      checks:
        DriverCheck: {}

    cpu_info:
      step: describe_instance
      checks:
        CpuInfoCheck: {}

    container_runtime:
      step: describe_instance
      checks:
        ContainerRuntimeCheck: {}

    stop_checks:
      step: stop_instance
      checks:
        InstanceStopCheck: {}

    start_checks:
      step: start_instance
      checks:
        InstanceStartCheck: {}
        StableIdentifierCheck:
          reference_id: "{{steps.launch_instance.instance_id}}"

    start_ssh:
      step: start_instance
      checks:
        ConnectivityCheck: {}
        OsCheck:
          expected_os: "ubuntu"

    start_gpu:
      step: start_instance
      checks:
        GpuCheck: {}

    reboot_checks:
      step: reboot_instance
      checks:
        InstanceRebootCheck:
          max_uptime: 600
        StableIdentifierCheck:
          reference_id: "{{steps.launch_instance.instance_id}}"

    reboot_state:
      step: reboot_instance
      checks:
        InstanceStateCheck:
          expected_state: "running"

    reboot_ssh:
      step: reboot_instance
      checks:
        ConnectivityCheck: {}
        OsCheck:
          expected_os: "ubuntu"

    reboot_gpu:
      step: reboot_instance
      checks:
        GpuCheck:
          expected_gpus: 8

    power_cycle_checks:
      step: power_cycle_instance
      checks:
        InstancePowerCycleCheck:
          max_recovery_time: 900
        StableIdentifierCheck:
          reference_id: "{{steps.launch_instance.instance_id}}"

    power_cycle_state:
      step: power_cycle_instance
      checks:
        InstanceStateCheck:
          expected_state: "running"

    power_cycle_ssh:
      step: power_cycle_instance
      checks:
        ConnectivityCheck: {}
        OsCheck:
          expected_os: "ubuntu"

    power_cycle_gpu:
      step: power_cycle_instance
      checks:
        GpuCheck:
          expected_gpus: 8

    # Reinstall validations (skipped when reinstall_instance step is skipped)
    reinstall_state:
      step: reinstall_instance
      checks:
        InstanceStateCheck:
          expected_state: "running"

    reinstall_ssh:
      step: reinstall_instance
      checks:
        ConnectivityCheck: {}
        OsCheck:
          expected_os: "ubuntu"

    reinstall_gpu:
      step: reinstall_instance
      checks:
        GpuCheck: {}

    nim_health:
      step: deploy_nim
      checks:
        NimHealthCheck: {}

    nim_models:
      step: deploy_nim
      checks:
        NimModelCheck: {}

    nim_inference:
      step: deploy_nim
      checks:
        NimInferenceCheck:
          prompt: "What is CUDA?"
          max_tokens: 50

    nim_teardown:
      step: teardown_nim
      checks:
        StepSuccessCheck: {}

    teardown_checks:
      step: teardown
      checks:
        StepSuccessCheck: {}

    sanitization:
      step: verify_teardown
      checks:
        StepSuccessCheck: {}

  exclude:
    markers: []