.gitlab-ci.yml

# =============================================================================
# GitLab CI/CD Pipeline for GCO (Global Capacity Orchestrator on AWS)
# =============================================================================
#
# FROZEN REFERENCE — NOT MAINTAINED
# ---------------------------------
# This file is preserved as a frozen reference for anyone forking on GitLab.
# It is NOT updated to match `.github/workflows/*.yml` and may drift over
# time. GitHub Actions is authoritative for this project.
#
# The last-verified-working state of this pipeline is the commit that
# introduced the `.github/workflows/*.yml` workflows. Check `git log` on
# this file for that commit; anything after will be drift.
#
# Why keep it at all? Fork users running on GitLab may use this as a
# starting point. They are welcome to open PRs against the GitHub repo to
# keep it functional for their use case; the GCO maintainers will not
# actively maintain it.
#
# For the active pipeline definitions, see `.github/workflows/`:
#   - unit-tests.yml
#   - integration-tests.yml
#   - security.yml
#   - lint.yml
#   - release.yml           (manual dispatch)
#   - deps-scan.yml         (monthly)
#   - cve-scan.yml          (weekly)
#
# PIPELINE STRUCTURE (maximized parallelism):
# -------------------------------------------
# Stage 1 (parallel): lint, test, security, build - all run simultaneously
# Stage 2: integration - runs after build completes
# Stage 3: deploy-preview - coverage report publishing
# Stage 4: release - manual version bumps
# Stage 5: maintenance - manual dependency checks
#
# SETUP INSTRUCTIONS (GitLab only):
# ----------------------------------
# 1. Create a Project Access Token:
#    - Go to Settings → Access Tokens
#    - Name: "CI Release Token" (or similar)
#    - Role: Maintainer
#    - Scopes: api, read_repository, write_repository
#    - Create token and copy the value
#
# 2. Add the token as a CI/CD variable:
#    - Go to Settings → CI/CD → Variables
#    - Add variable:
#      - Key: RELEASE_TOKEN
#      - Value: <paste token>
#      - Type: Variable
#      - Flags: Masked (checked), Protected (optional)
#
# MANUAL JOBS:
# ------------
# - release:patch/minor/major - Bump version and create git tag
# - dependency-check - Check for outdated packages, creates issue if found
#
# POLICY: All scan/lint/security jobs are STRICT (no allow_failure, no || true).
# If a scan finds issues, the pipeline fails. Fix the code, don't silence the tool.
#
# =============================================================================

stages:
  - validate    # lint + test + security + build (all parallel)
  - integration # needs build artifacts
  - deploy-preview
  - release
  - maintenance # manual + monthly scheduled dependency checks

variables:
  PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
  PYTHONUNBUFFERED: "1"
  AWS_DEFAULT_REGION: "us-east-1"
  # Pin Helm/kubectl to match lambda/helm-installer/Dockerfile so the dependency
  # scan uses the same binaries as the runtime Lambda. Bump these in lockstep
  # with the Dockerfile (grep for helm-v and dl.k8s.io/release/v).
  HELM_VERSION: "v4.1.3"
  KUBECTL_VERSION: "v1.35.4"
  # Pin Trivy so the scheduled weekly CVE scan uses the same version as the
  # pipeline's security:trivy job. Bump in lockstep with the security:trivy
  # image tag below.
  TRIVY_VERSION: "v0.70.0"

# Shared script to build the Lambda package (used by test, build:cdk, test:cdk-config-matrix)
.build-lambda-package: &build-lambda-package
  - rm -rf lambda/kubectl-applier-simple-build
  - mkdir -p lambda/kubectl-applier-simple-build
  - cp lambda/kubectl-applier-simple/handler.py lambda/kubectl-applier-simple-build/
  - cp -r lambda/kubectl-applier-simple/manifests lambda/kubectl-applier-simple-build/
  - pip install kubernetes pyyaml urllib3 -t lambda/kubectl-applier-simple-build/
  - rm -rf lambda/helm-installer-build
  - cp -r lambda/helm-installer lambda/helm-installer-build
  - rm -rf lambda/helm-installer-build/__pycache__

# =============================================================================
# VALIDATE STAGE - All checks run in parallel
# =============================================================================

# --- LINTING ---
lint:python:
  stage: validate
  image: python:3.14.4-slim
  before_script:
    - apt-get update && apt-get install -y git
    - pip install --upgrade pip
    - pip install -e ".[lint]"
  script:
    - black --check --diff --exclude '/(\.venv|cdk\.out|lambda/kubectl-applier-simple-build|lambda/helm-installer-build)/' gco/ cli/ mcp/ tests/ lambda/ scripts/
    - isort --check-only --diff --skip-glob 'cdk.out*' --skip-glob 'lambda/kubectl-applier-simple-build*' --skip-glob 'lambda/helm-installer-build*' gco/ cli/ mcp/ tests/ lambda/ scripts/
    - ruff check gco/ cli/ mcp/ tests/ --exclude lambda/kubectl-applier-simple-build --exclude lambda/helm-installer-build
    - flake8 gco/ cli/ mcp/ tests/ lambda/ scripts/ --exclude lambda/kubectl-applier-simple-build,lambda/helm-installer-build
    - yamllint .
  cache:
    key: lint-${CI_COMMIT_REF_SLUG}
    paths:
      - .cache/pip

lint:typecheck:
  stage: validate
  image: python:3.14.4-slim
  before_script:
    - pip install --upgrade pip
    # Install full project + mcp extras so mypy can resolve real types
    # for boto3, kubernetes, fastapi, click, fastmcp, etc. Strict mode in
    # pyproject.toml would otherwise flag untyped decorators and Any-subclass
    # relationships when the runtime deps are missing.
    #
    # CDK is intentionally NOT installed here — gco/stacks/ and app.py are
    # checked in a separate job (lint:typecheck-stacks) with CDK installed.
    - pip install -e ".[typecheck,mcp]"
  script:
    # Main application code. gco/stacks/ and app.py are excluded because
    # mypy's --exclude only skips checking, not following imports; without
    # aws_cdk installed, mypy resolves `Stack` as Any and flags every
    # subclass. lint:typecheck-stacks handles that path with CDK installed.
    - mypy gco/ cli/ mcp/ scripts/ --exclude 'gco/stacks/'
  cache:
    key: typecheck-${CI_COMMIT_REF_SLUG}
    paths:
      - .cache/pip
      - .mypy_cache/

lint:typecheck-stacks:
  stage: validate
  image: python:3.14.4-slim
  before_script:
    - pip install --upgrade pip
    - pip install -e ".[cdk,typecheck]"
  script:
    # CDK stacks need aws_cdk installed to resolve imports. Keep this in a
    # separate job so the main typecheck job stays fast (no CDK install).
    # app.py is the CDK app entry point and imports every stack, so it
    # belongs here too.
    - mypy gco/stacks/ app.py
  cache:
    key: typecheck-stacks-${CI_COMMIT_REF_SLUG}
    paths:
      - .cache/pip
      - .mypy_cache/

lint:typecheck-lambda:
  stage: validate
  image: python:3.14.4-slim
  before_script:
    - pip install --upgrade pip
    # Lambda handlers import boto3, kubernetes, urllib3, yaml at module scope
    # so we need the runtime deps installed to resolve their types.
    - pip install -e ".[typecheck]"
  script:
    # Each Lambda directory ships its own `handler.py`, so we can't check
    # them together (mypy's module resolver treats them as duplicates).
    # Loop through each one and fail the job on the first error.
    - |
      FAILED=0
      for d in lambda/alb-header-validator lambda/api-gateway-proxy \
               lambda/cross-region-aggregator lambda/drift-detection \
               lambda/ga-registration lambda/helm-installer \
               lambda/kubectl-applier-simple lambda/proxy-shared \
               lambda/regional-api-proxy lambda/secret-rotation; do
        echo "=== typecheck: $d ==="
        if ! mypy "$d"; then
          FAILED=1
        fi
      done
      exit $FAILED
  cache:
    key: typecheck-lambda-${CI_COMMIT_REF_SLUG}
    paths:
      - .cache/pip
      - .mypy_cache/

lint:dockerfile:
  stage: validate
  image: hadolint/hadolint:v2.14.0-debian
  script:
    # Auto-discover all Dockerfiles, skip build artifacts
    - |
      find . -type f \( -name 'Dockerfile*' -o -name '*dockerfile' -o -name '*Dockerfile' \) \
        -not -path './.git/*' \
        -not -path './cdk.out/*' \
        -not -path './.venv/*' \
        -not -path './lambda/kubectl-applier-simple-build/*' \
        -not -path './lambda/helm-installer-build/*' \
        | while read -r df; do
          echo "=== Linting $df ==="
          hadolint "$df" --ignore DL3008 --ignore DL3013
        done

lint:shellcheck:
  stage: validate
  image: koalaman/shellcheck-alpine:v0.11.0
  script:
    # Auto-discover all shell scripts, skip build artifacts
    # -x follows source directives so shellcheck can resolve sourced files
    - |
      find . -name '*.sh' \
        -not -path './.git/*' \
        -not -path './cdk.out/*' \
        -not -path './.venv/*' \
        -not -path './lambda/kubectl-applier-simple-build/*' \
        -not -path './lambda/helm-installer-build/*' \
        | while read -r sh; do
          echo "=== Checking $sh ==="
          shellcheck -x "$sh"
        done

test:bats:
  stage: validate
  image:
    name: bats/bats:1.13.0
    entrypoint: [""]
  before_script:
    - apk add --no-cache jq bash python3 py3-yaml
  script:
    - /opt/bats/bin/bats tests/BATS/

# --- TESTING (consolidated into one job) ---
test:test-dir-pytest:
  stage: validate
  image: python:3.14.4-slim
  before_script:
    - apt-get update && apt-get install -y curl
    - curl -fsSL https://deb.nodesource.com/setup_20.x | bash -
    - apt-get install -y nodejs
    - pip install --upgrade pip
    - pip install -e ".[dev,mcp]"
    # Build Lambda package for CDK tests
    - *build-lambda-package
  script:
    - pytest tests/ -v --ignore=tests/test_integration.py --cov=gco --cov=cli --cov-report=xml --cov-report=html --cov-report=json --cov-report=term-missing --cov-fail-under=90 --junitxml=report.xml -x
  coverage: '/^TOTAL\s+.*\s+(\d+)%/'
  cache:
    key: test-${CI_COMMIT_REF_SLUG}
    paths:
      - .cache/pip
  artifacts:
    when: always
    paths:
      - htmlcov/
      - coverage.xml
      - coverage.json
      - report.xml
    reports:
      junit: report.xml
      coverage_report:
        coverage_format: cobertura
        path: coverage.xml
    expire_in: 7 days

test:cli:
  stage: validate
  image: python:3.14.4-slim
  before_script:
    - pip install --upgrade pip
    - pip install -e .
  script:
    - python -c "from cli import GCOConfig, JobManager, CapacityChecker; print('CLI imports OK')"
    - gco --help
    - gco jobs --help
    - gco capacity --help
    - gco stacks --help
    - gco costs --help
    - gco dag --help
    - gco inference --help
  cache:
    key: cli-${CI_COMMIT_REF_SLUG}
    paths:
      - .cache/pip

# --- SECURITY (all run in parallel, all strict) ---
security:bandit:
  stage: validate
  image: python:3.14.4-slim
  before_script:
    - pip install --upgrade pip
    - pip install -e ".[security]"
  script:
    - bandit -r gco/ cli/ -c pyproject.toml --severity-level medium -f json -o bandit-report.json
  artifacts:
    when: always
    paths:
      - bandit-report.json
    expire_in: 30 days

security:safety:
  stage: validate
  image: python:3.14.4-slim
  before_script:
    - pip install --upgrade pip
    - pip install -e ".[security]"
  script:
    - pip-compile --no-emit-index-url --strip-extras -o /tmp/requirements-lock.txt pyproject.toml -q
    - safety check -r /tmp/requirements-lock.txt --output json > safety-report.json
  artifacts:
    when: always
    paths:
      - safety-report.json
    expire_in: 30 days

security:trivy:
  stage: validate
  image:
    name: aquasec/trivy:0.70.0
    entrypoint: [""]
  script:
    - trivy fs --exit-code 1 --severity HIGH,CRITICAL --format json -o trivy-report.json .
  artifacts:
    when: always
    paths:
      - trivy-report.json
    expire_in: 30 days

security:secrets:
  stage: validate
  image:
    name: trufflesecurity/trufflehog:3.94.3
    entrypoint: [""]
  script:
    - trufflehog filesystem . --only-verified --fail --json > trufflehog-report.json 2>&1
  artifacts:
    when: always
    paths:
      - trufflehog-report.json
    expire_in: 30 days

security:gitleaks:
  stage: validate
  image:
    name: zricethezav/gitleaks:v8.30.1
    entrypoint: [""]
  script:
    # Scan only current files, not git history (use --no-git)
    # Uses .gitleaks.toml for path exclusions and allowlists
    - gitleaks detect --source . --no-git --config .gitleaks.toml --report-format json --report-path gitleaks-report.json
  artifacts:
    when: always
    paths:
      - gitleaks-report.json
    expire_in: 30 days

security:semgrep:
  stage: validate
  image: semgrep/semgrep:1.160.0
  script:
    - semgrep scan --config auto --error --json -o semgrep-report.json .
  artifacts:
    when: always
    paths:
      - semgrep-report.json
    expire_in: 30 days

security:checkov:
  stage: validate
  image:
    name: bridgecrew/checkov:3.2.500
    entrypoint: [""]
  script:
    - checkov -d . --config-file .checkov.yaml -o json > checkov-report.json
  artifacts:
    when: always
    paths:
      - checkov-report.json
    expire_in: 30 days

security:kics:
  stage: validate
  image:
    name: checkmarx/kics:v2.1.20
    entrypoint: [""]
  needs: ["build:cdk"]
  script:
    - /app/bin/kics scan -p cdk.out -o . --report-formats json,html --output-name kics-report --config .kics.yaml --fail-on high
  artifacts:
    when: always
    paths:
      - kics-report.json
      - kics-report.html
    expire_in: 30 days

security:pip-audit:
  stage: validate
  image: python:3.14.4-slim
  before_script:
    - pip install --upgrade pip
    - pip install pip-audit
  script:
    # Audit only the project's production lockfile — dev/test/build tooling
    # (pytest, pip, wheel, etc.) is intentionally excluded since it is not
    # deployed. pip-audit defaults to exiting non-zero when vulnerabilities
    # are found.
    #
    # First invocation writes the JSON artifact (always uploaded even on failure).
    - pip-audit -r requirements-lock.txt --format json --output pip-audit-report.json --desc || true
    # Second invocation fails the job when vulnerabilities are present.
    - pip-audit -r requirements-lock.txt --desc
  artifacts:
    when: always
    paths:
      - pip-audit-report.json
    expire_in: 30 days

# --- BUILD ---
build:cdk:
  stage: validate
  image: python:3.14.4-slim
  before_script:
    - apt-get update && apt-get install -y nodejs npm
    - pip install --upgrade pip
    - pip install -e ".[cdk]"
    - npm install -g aws-cdk
    - *build-lambda-package
  script:
    - cdk synth --quiet
  cache:
    key: cdk-${CI_COMMIT_REF_SLUG}
    paths:
      - .cache/pip
  artifacts:
    when: always
    paths:
      - cdk.out/
    expire_in: 7 days

build:package:
  stage: validate
  image: python:3.14.4-slim
  before_script:
    - pip install --upgrade pip build wheel
    - pip install -e .
  script:
    - python -m build
  artifacts:
    when: always
    paths:
      - dist/
    expire_in: 7 days

# --- EKS WORKLOAD SMOKE TESTS ---
test:workload-imports:
  stage: validate
  image: python:3.14.4-slim
  before_script:
    - apt-get update && apt-get install -y --no-install-recommends curl > /dev/null 2>&1
    - pip install --upgrade pip -q
  script:
    # Verify health-monitor and manifest-processor can import (install from pyproject.toml)
    - echo "=== Testing health-monitor ==="
    - pip install -e . -q
    - python -c "from gco.services.health_api import app; print('health-monitor OK')"
    - echo "=== Testing manifest-processor ==="
    - python -c "from gco.services.manifest_api import app; print('manifest-processor OK')"
    # Verify inference-monitor can import (uses its own minimal requirements)
    - echo "=== Testing inference-monitor ==="
    - pip install -e ".[inference-monitor]" -q
    - python -c "from gco.services.inference_monitor import InferenceMonitor; print('inference-monitor OK')"
    # Verify services can be fully imported (catches circular imports that simple imports miss)
    # importlib.import_module executes all module-level code including deferred router imports
    - echo "=== Testing full module imports (circular import detection) ==="
    - python -c "import importlib; importlib.import_module('gco.services.manifest_api'); print('manifest-api full import OK')"
    - python -c "import importlib; importlib.import_module('gco.services.health_api'); print('health-api full import OK')"
    - python -c "import importlib; importlib.import_module('gco.services.inference_monitor'); print('inference-monitor full import OK')"
    # Verify all api_routes can be imported alongside manifest_api (the circular import vector)
    - python -c "from gco.services.api_routes import jobs, manifests, queue, templates, webhooks; print('all api_routes OK')"

# --- CDK CONFIG MATRIX ---
test:cdk-config-matrix:
  stage: validate
  image: python:3.14.4-slim
  before_script:
    - apt-get update && apt-get install -y --no-install-recommends nodejs npm > /dev/null 2>&1
    - npm install --location=global aws-cdk@2 > /dev/null 2>&1
    - pip install --upgrade pip > /dev/null 2>&1
    - pip install -e ".[cdk]" > /dev/null 2>&1
    # Build Lambda package (required for CDK synth)
    - *build-lambda-package
  script:
    - python3 scripts/test_cdk_synthesis.py --verbose
  variables:
    AWS_ACCESS_KEY_ID: "fake"
    AWS_SECRET_ACCESS_KEY: "fake"
    AWS_DEFAULT_REGION: "us-east-1"
    CDK_DEFAULT_REGION: "us-east-1"
  cache:
    key: cdk-config-matrix
    paths:
      - .cache/pip/
      - node_modules/

# --- LOCKFILE FRESHNESS CHECK ---
test:lockfile:
  stage: validate
  image: python:3.14.4-slim
  before_script:
    - pip install --upgrade pip
    - pip install pip-tools
  script:
    # Pre-seed the candidate lockfile with the committed one so pip-compile
    # treats the existing pins as the current resolution state. pip-compile
    # only rewrites pins when pyproject.toml requires it (new/changed/removed
    # direct dep). It will NOT bump a transitive just because PyPI published
    # a newer version — the existing pin still satisfies the constraint, so
    # it stays. This removes a race where an unrelated transitive release
    # (for example botocore shipping a new patch) between the local
    # pip-compile run and the CI run would falsely flag the lockfile as
    # stale. Genuine drift (a direct dep added or bumped without regenerating
    # requirements-lock.txt) still produces a diff and fails the job.
    - cp requirements-lock.txt /tmp/requirements-lock-fresh.txt
    - pip-compile --no-emit-index-url --strip-extras -o /tmp/requirements-lock-fresh.txt pyproject.toml -q
    - grep -v '^#' requirements-lock.txt > /tmp/lock-no-comments.txt
    - grep -v '^#' /tmp/requirements-lock-fresh.txt > /tmp/fresh-no-comments.txt
    - |
      if ! diff /tmp/lock-no-comments.txt /tmp/fresh-no-comments.txt; then
        echo ""
        echo "ERROR: requirements-lock.txt is stale."
        echo "Run: pip-compile --no-emit-index-url --strip-extras -o requirements-lock.txt pyproject.toml"
        echo "Then commit the updated lockfile."
        exit 1
      fi
    - echo "Lockfile is up to date"
  cache:
    key: lockfile-${CI_COMMIT_REF_SLUG}
    paths:
      - .cache/pip

# --- FRESH INSTALL VERIFICATION ---
test:fresh-install:
  stage: validate
  image: python:3.14.4-slim
  before_script:
    - apt-get update && apt-get install -y --no-install-recommends nodejs npm > /dev/null 2>&1
  script:
    - pip install --upgrade pip
    - pip install -e ".[cdk]"
    # Verify all required packages are importable
    - python3 -c "import aws_cdk; import cdk_nag; import aws_cdk.aws_eks_v2; print('All CDK imports OK')"
    - python3 -c "import click; import boto3; import requests; import yaml; print('All runtime imports OK')"
    - python3 -c "from cli.main import cli; print('CLI entry point OK')"
    - python3 -c "from gco.stacks.regional_stack import GCORegionalStack; print('Regional stack import OK')"

# =============================================================================
# INTEGRATION STAGE - Needs build artifacts
# =============================================================================

integration:k8s:
  stage: integration
  image: python:3.14.4-slim
  needs: ["build:cdk"]
  before_script:
    - pip install --upgrade pip
    - pip install -e .
  script:
    # Validate YAML syntax for all manifests (skip files with template variables)
    - echo "=== Validating Kubernetes manifest YAML syntax ==="
    - |
      FAILED=0
      for f in lambda/kubectl-applier-simple/manifests/*.yaml; do
        echo "Validating $f..."
        # Skip files with template variables (they need substitution before parsing)
        if grep -q '{{' "$f"; then
          echo "  Skipping (contains template variables)"
          continue
        fi
        if ! python -c "import yaml; list(yaml.safe_load_all(open('$f')))" 2>&1; then
          echo "  FAILED: $f"
          FAILED=1
        else
          echo "  OK"
        fi
      done
    # Validate example manifests
    - echo "=== Validating example manifest YAML syntax ==="
    - |
      for f in examples/*.yaml; do
        echo "Validating $f..."
        if ! python -c "import yaml; list(yaml.safe_load_all(open('$f')))" 2>&1; then
          echo "  FAILED: $f"
          FAILED=1
        else
          echo "  OK"
        fi
      done
    # Verify required resources exist in manifests
    - echo "=== Checking for required Kubernetes resources ==="
    - |
      REQUIRED_KINDS="Namespace ServiceAccount ClusterRole ClusterRoleBinding Deployment"
      for kind in $REQUIRED_KINDS; do
        if grep -r "kind: $kind" lambda/kubectl-applier-simple/manifests/ > /dev/null; then
          echo "  Found $kind"
        else
          echo "  MISSING: $kind"
          FAILED=1
        fi
      done
    # Validate DAG pipeline definitions
    - echo "=== Validating DAG pipeline definitions ==="
    - |
      for f in examples/*dag*.yaml examples/*pipeline*.yaml; do
        [ -f "$f" ] || continue
        echo "Validating DAG $f..."
        python -c "
      import yaml, sys
      with open('$f') as fh:
          data = yaml.safe_load(fh)
      if 'steps' not in data:
          print('  Not a DAG file, skipping')
          sys.exit(0)
      names = set()
      for step in data['steps']:
          if 'name' not in step or 'manifest' not in step:
              print('  FAILED: step missing name or manifest')
              sys.exit(1)
          if step['name'] in names:
              print(f'  FAILED: duplicate step name {step[\"name\"]}')
              sys.exit(1)
          names.add(step['name'])
          for dep in step.get('depends_on', []):
              if dep not in names:
                  # Forward reference — check it exists somewhere
                  all_names = {s['name'] for s in data['steps']}
                  if dep not in all_names:
                      print(f'  FAILED: unknown dependency {dep}')
                      sys.exit(1)
      print(f'  OK ({len(data[\"steps\"])} steps)')
      "
      done
    # Verify manifests have required fields (skip files with template variables)
    - echo "=== Checking manifest structure ==="
    - |
      python << 'EOF'
      import yaml
      import sys
      import os
      import re

      failed = False
      for root, dirs, files in os.walk('lambda/kubectl-applier-simple/manifests'):
          for f in files:
              if not f.endswith('.yaml'):
                  continue
              path = os.path.join(root, f)
              # Read file content to check for template variables
              with open(path) as fh:
                  content = fh.read()
              # Skip files with template variables
              if '{{' in content:
                  print(f"  {path}: skipping (contains template variables)")
                  continue
              try:
                  docs = list(yaml.safe_load_all(content))
                  for doc in docs:
                      if doc is None:
                          continue
                      if 'kind' not in doc:
                          print(f"  {path}: missing 'kind' field")
                          failed = True
                      if 'apiVersion' not in doc:
                          print(f"  {path}: missing 'apiVersion' field")
                          failed = True
              except Exception as e:
                  print(f"  {path}: {e}")
                  failed = True

      if failed:
          sys.exit(1)
      print("  All manifests have required fields")
      EOF
    # Fail if any validation failed
    - |
      if [ "$FAILED" -eq 1 ]; then
        echo "=== VALIDATION FAILED ==="
        exit 1
      fi
      echo "=== All validations passed ==="

integration:lambda:
  stage: integration
  image: python:3.14.4-slim
  needs: ["build:cdk"]
  before_script:
    - pip install --upgrade pip
    - pip install -e .
  script:
    - python -c "import sys; sys.path.insert(0, 'lambda/kubectl-applier-simple'); from handler import lambda_handler; print('kubectl-applier OK')"
    - python -c "import sys; sys.path.insert(0, 'lambda/api-gateway-proxy'); from handler import lambda_handler; print('api-gateway-proxy OK')"
    - python -c "import sys; sys.path.insert(0, 'lambda/ga-registration'); from handler import lambda_handler; print('ga-registration OK')"
    - python -c "import sys; sys.path.insert(0, 'lambda/cross-region-aggregator'); from handler import lambda_handler; print('cross-region-agg OK')"
    - python -c "import sys; sys.path.insert(0, 'lambda/regional-api-proxy'); from handler import lambda_handler; print('regional-api-proxy OK')"
    - python -c "import sys; sys.path.insert(0, 'lambda/alb-header-validator'); from handler import lambda_handler; print('alb-header-validator OK')"
    - python -c "import sys; sys.path.insert(0, 'lambda/secret-rotation'); from handler import lambda_handler; print('secret-rotation OK')"
    - python -c "import sys; sys.path.insert(0, 'lambda/helm-installer'); from handler import lambda_handler; print('helm-installer OK')"

integration:comprehensive:
  stage: integration
  image: python:3.14.4-slim
  needs: ["build:cdk"]
  before_script:
    - pip install --upgrade pip
    - pip install -e ".[test]"
  script:
    - pytest tests/test_integration.py -v --tb=short --junitxml=test-integration-report.xml
  artifacts:
    when: always
    paths:
      - test-integration-report.xml
    reports:
      junit: test-integration-report.xml
    expire_in: 7 days

integration:mcp:
  stage: integration
  image: python:3.14.4-slim
  needs: ["build:cdk"]
  before_script:
    - pip install --upgrade pip
    - pip install -e ".[dev,mcp]"
  script:
    - pytest tests/test_mcp_integration.py -v --tb=short --junitxml=test-mcp-integration-report.xml
  artifacts:
    when: always
    paths:
      - test-mcp-integration-report.xml
    reports:
      junit: test-mcp-integration-report.xml
    expire_in: 7 days

# =============================================================================
# DEPLOY PREVIEW
# =============================================================================

pages:
  stage: deploy-preview
  needs: ["test:test-dir-pytest"]
  script:
    - mkdir -p public
    - cp -r htmlcov/* public/ || echo "No coverage report found"
  artifacts:
    paths:
      - public
    expire_in: 30 days
  rules:
    - if: '$CI_COMMIT_BRANCH == "main" || $CI_COMMIT_BRANCH == "master"'
  allow_failure: true

# =============================================================================
# RELEASE - Manual triggers only
# =============================================================================

.release-base:
  stage: release
  image: python:3.14.4-slim
  needs: ["lint:python", "test:test-dir-pytest"]
  allow_failure: true
  before_script:
    - apt-get update && apt-get install -y git
    - git config --global user.email "ci@gitlab.com"
    - git config --global user.name "GitLab CI"
  script:
    - python scripts/bump_version.py $BUMP_TYPE
    - export NEW_VERSION=$(python -c "from gco._version import __version__; print(__version__)")
    - git add gco/_version.py cli/__init__.py
    - git commit -m "Release v${NEW_VERSION}"
    - git tag -a "v${NEW_VERSION}" -m "Release v${NEW_VERSION}"
    - git push "https://oauth2:${RELEASE_TOKEN}@${CI_SERVER_HOST}/${CI_PROJECT_PATH}.git" HEAD:${CI_COMMIT_REF_NAME} --tags
  rules:
    - if: '$CI_COMMIT_BRANCH == "main" || $CI_COMMIT_BRANCH == "master"'
      when: manual

release:patch:
  extends: .release-base
  variables:
    BUMP_TYPE: patch

release:minor:
  extends: .release-base
  variables:
    BUMP_TYPE: minor

release:major:
  extends: .release-base
  variables:
    BUMP_TYPE: major

# =============================================================================
# MAINTENANCE - Manual + Scheduled jobs
# =============================================================================

# Shared dependency scan script (used by both manual and scheduled jobs)
.dependency-scan-script: &dependency-scan-script
  - |
    set +e  # Don't exit on errors - we want to continue checking

    # =======================================================================
    # Check Python dependencies
    # =======================================================================
    echo "=== Checking for outdated Python dependencies ==="

    pip install -e . --quiet --root-user-action=ignore
    OUTDATED=$(pip list --outdated --format=json)
    PYTHON_COUNT=$(echo "$OUTDATED" | jq 'length')

    if [ "$PYTHON_COUNT" -eq 0 ]; then
      echo "All Python dependencies are up to date!"
      PYTHON_OUTDATED=""
    else
      echo "Found $PYTHON_COUNT outdated Python package(s)"
      echo "$OUTDATED" | jq -r '.[] | "  - \(.name): \(.version) -> \(.latest_version)"'
      PYTHON_OUTDATED="$OUTDATED"
    fi

    # =======================================================================
    # Check Docker image versions
    # =======================================================================
    echo ""
    echo "=== Checking for outdated Docker images ==="

    # Create temp file for Docker results (avoids subshell variable issues)
    DOCKER_RESULTS=$(mktemp)

    # Function to check a single image
    check_image() {
      local image="$1"
      local current_tag="$2"

      # Skip non-semver tags
      if ! echo "$current_tag" | grep -qE "^v?[0-9]+\.[0-9]+(\.[0-9]+)?"; then
        return
      fi

      # Skip gco project images (we control those)
      if echo "$image" | grep -q "^gco/"; then
        return
      fi

      # Determine registry and repo
      local registry=""
      local repo=""
      case "$image" in
        nvcr.io/*|gcr.io/*|quay.io/*|ghcr.io/*|registry.k8s.io/*)
          registry=$(echo "$image" | cut -d'/' -f1)
          repo=$(echo "$image" | cut -d'/' -f2-)
          ;;
        */*)
          # Docker Hub org/repo
          registry="docker.io"
          repo="$image"
          ;;
        *)
          # Docker Hub library image
          registry="docker.io"
          repo="library/$image"
          ;;
      esac

      # Query registry for tags
      local tags=""
      tags=$(skopeo list-tags "docker://${registry}/${repo}" 2>/dev/null | \
        jq -r '.Tags[]' 2>/dev/null | \
        grep -E "^v?[0-9]+\.[0-9]+\.[0-9]+$" | \
        sort -V | tail -10) || return

      [ -z "$tags" ] && return

      local latest_tag=$(echo "$tags" | tail -1)
      local current_ver=$(echo "$current_tag" | sed 's/^v//')
      local latest_ver=$(echo "$latest_tag" | sed 's/^v//')

      # Check if latest is newer
      if [ "$current_ver" != "$latest_ver" ]; then
        local newer=$(printf '%s\n%s' "$current_ver" "$latest_ver" | sort -V | tail -1)
        if [ "$newer" = "$latest_ver" ]; then
          echo "  - ${image}:${current_tag} -> ${latest_tag}"
          echo "${image}|${current_tag}|${latest_tag}" >> "$DOCKER_RESULTS"
        fi
      fi
    }

    # Collect all images to check
    ALL_IMAGES=$(mktemp)

    # From .gitlab-ci.yml
    echo "Checking CI images..."
    grep -oE "image: [a-zA-Z0-9_./-]+:[a-zA-Z0-9._-]+" .gitlab-ci.yml 2>/dev/null | \
      sed 's/image: //' >> "$ALL_IMAGES"
    # Also check nested image definitions (e.g. image: { name: ... }) used by Trivy, KICS, etc.
    grep -oE "name: [a-zA-Z0-9_./-]+:[a-zA-Z0-9._-]+" .gitlab-ci.yml 2>/dev/null | \
      grep -v 'name: keda\|name: nvidia\|name: kedacore\|name: gpu-nodes\|name: workers' | \
      sed 's/name: //' >> "$ALL_IMAGES"

    # From K8s manifests (skip template variables)
    echo "Checking K8s manifest images..."
    grep -rhoE "image: [a-zA-Z0-9_./-]+:[a-zA-Z0-9._-]+" lambda/kubectl-applier-simple/manifests/ 2>/dev/null | \
      grep -v '{{' | sed 's/image: //' >> "$ALL_IMAGES"

    # From example manifests
    echo "Checking example manifest images..."
    grep -rhoE "image: [a-zA-Z0-9_./-]+:[a-zA-Z0-9._-]+" examples/ 2>/dev/null | \
      sed 's/image: //' >> "$ALL_IMAGES"

    # From Helm chart values in charts.yaml (repository + tag pairs)
    echo "Checking Helm chart value images..."
    python3 -c "
    import yaml
    with open('lambda/helm-installer/charts.yaml') as f:
        data = yaml.safe_load(f)
    def find_images(d):
        if isinstance(d, dict):
            repo = d.get('repository', '')
            tag = d.get('tag', '')
            if repo and tag and '/' in repo:
                print(f'{repo}:{tag}')
            for v in d.values():
                find_images(v)
        elif isinstance(d, list):
            for item in d:
                find_images(item)
    for name, cfg in data.get('charts', {}).items():
        find_images(cfg.get('values', {}))
    " >> "$ALL_IMAGES" 2>/dev/null || true

    # Check each unique image
    sort -u "$ALL_IMAGES" | while read -r img; do
      [ -z "$img" ] && continue
      image=$(echo "$img" | cut -d':' -f1)
      tag=$(echo "$img" | cut -d':' -f2)
      check_image "$image" "$tag"
    done

    rm -f "$ALL_IMAGES"

    DOCKER_COUNT=$(wc -l < "$DOCKER_RESULTS" | tr -d ' ')
    [ "$DOCKER_COUNT" = "" ] && DOCKER_COUNT=0

    # =======================================================================
    # Check Helm chart versions (from charts.yaml)
    # =======================================================================
    echo ""
    echo "=== Checking Helm chart versions ==="

    HELM_RESULTS=$(mktemp)
    CHARTS_FILE="lambda/helm-installer/charts.yaml"

    if [ -f "$CHARTS_FILE" ]; then
      # Extract chart entries: name, repo_url, chart, version, use_oci
      python3 -c "
    import yaml, json
    with open('$CHARTS_FILE') as f:
        data = yaml.safe_load(f)
    for name, cfg in data.get('charts', {}).items():
        print(json.dumps({'name': name, 'repo_url': cfg.get('repo_url',''), 'chart': cfg.get('chart',''), 'version': cfg.get('version',''), 'use_oci': cfg.get('use_oci', False)}))
    " | while IFS= read -r entry; do
        chart_name=$(echo "$entry" | jq -r '.name')
        repo_url=$(echo "$entry" | jq -r '.repo_url')
        chart=$(echo "$entry" | jq -r '.chart')
        current=$(echo "$entry" | jq -r '.version')
        use_oci=$(echo "$entry" | jq -r '.use_oci')

        [ -z "$current" ] && continue

        latest=""
        if [ "$use_oci" = "true" ]; then
          # OCI charts: use helm show chart
          latest=$(helm show chart "${repo_url}/${chart}" 2>/dev/null | grep '^version:' | awk '{print $2}') || true
        else
          # Traditional repos
          helm repo add "$chart_name" "$repo_url" --force-update > /dev/null 2>&1 || true
          latest=$(helm search repo "${chart_name}/${chart}" --output json 2>/dev/null | jq -r '.[0].version // empty') || true
        fi

        if [ -n "$latest" ] && [ "$current" != "$latest" ]; then
          # Strip v prefix for comparison (e.g., 0.5.25 vs v0.5.25)
          current_stripped=$(echo "$current" | sed 's/^v//')
          latest_stripped=$(echo "$latest" | sed 's/^v//')
          if [ "$current_stripped" != "$latest_stripped" ]; then
            echo "  - ${chart_name} (${chart}): ${current} -> ${latest}"
            echo "${chart_name}|${chart}|${current}|${latest}" >> "$HELM_RESULTS"
          fi
        fi
      done
    fi

    HELM_COUNT=$(wc -l < "$HELM_RESULTS" 2>/dev/null | tr -d ' ')
    [ "$HELM_COUNT" = "" ] && HELM_COUNT=0

    # =======================================================================
    # Check EKS add-on versions
    # =======================================================================
    echo ""
    echo "=== Checking EKS add-on versions ==="

    ADDON_RESULTS=$(mktemp)
    K8S_VERSION=$(python3 -c "import json; print(json.load(open('cdk.json'))['context']['kubernetes_version'])" 2>/dev/null || echo "1.35")

    # Extract addon versions from regional_stack.py
    python3 -c "
    import re
    with open('gco/stacks/regional_stack.py') as f:
        text = f.read()
    for m in re.finditer(r'addon_name=\"([^\"]+)\".*?addon_version=\"([^\"]+)\"', text, re.DOTALL):
        print(f'{m.group(1)}|{m.group(2)}')
    " | while IFS='|' read -r addon_name current_version; do
        [ -z "$addon_name" ] && continue
        latest=$(aws eks describe-addon-versions --addon-name "$addon_name" --kubernetes-version "$K8S_VERSION" \
          --query 'addons[0].addonVersions[0].addonVersion' --output text 2>/dev/null) || true

        if [ -n "$latest" ] && [ "$latest" != "None" ] && [ "$current_version" != "$latest" ]; then
          echo "  - ${addon_name}: ${current_version} -> ${latest}"
          echo "${addon_name}|${current_version}|${latest}" >> "$ADDON_RESULTS"
        fi
      done

    ADDON_COUNT=$(wc -l < "$ADDON_RESULTS" 2>/dev/null | tr -d ' ')
    [ "$ADDON_COUNT" = "" ] && ADDON_COUNT=0

    # =======================================================================
    # Check kubectl version matches EKS version
    # =======================================================================
    echo ""
    echo "=== Checking kubectl version alignment ==="

    KUBECTL_ISSUE=""
    # Get kubectl version from helm-installer Dockerfile
    KUBECTL_VER=$(grep -oP 'dl\.k8s\.io/release/v\K[\d.]+' lambda/helm-installer/Dockerfile 2>/dev/null || echo "")
    if [ -n "$KUBECTL_VER" ]; then
      # Check if kubectl minor version matches EKS version
      KUBECTL_MINOR=$(echo "$KUBECTL_VER" | cut -d. -f1-2)
      if [ "$KUBECTL_MINOR" != "$K8S_VERSION" ]; then
        echo "  ⚠ kubectl $KUBECTL_VER doesn't match EKS $K8S_VERSION"
        KUBECTL_ISSUE="kubectl ${KUBECTL_VER} vs EKS ${K8S_VERSION}"
      else
        # Check for latest patch version from dl.k8s.io
        LATEST_KUBECTL=$(curl -sL "https://dl.k8s.io/release/stable-${K8S_VERSION}.txt" 2>/dev/null | sed 's/^v//')
        if [ -n "$LATEST_KUBECTL" ] && [ "$KUBECTL_VER" != "$LATEST_KUBECTL" ]; then
          echo "  - kubectl: $KUBECTL_VER -> $LATEST_KUBECTL"
          KUBECTL_ISSUE="kubectl ${KUBECTL_VER} -> ${LATEST_KUBECTL}"
        else
          echo "  kubectl $KUBECTL_VER is up to date for EKS $K8S_VERSION"
        fi
      fi
    fi

    # =======================================================================
    # Report results
    # =======================================================================
    echo ""
    echo "=== Summary ==="
    echo "Python packages outdated: $PYTHON_COUNT"
    echo "Docker images outdated: $DOCKER_COUNT"
    echo "Helm charts outdated: $HELM_COUNT"
    echo "EKS add-ons outdated: $ADDON_COUNT"

    if [ "$PYTHON_COUNT" -eq 0 ] && [ "$DOCKER_COUNT" -eq 0 ] && [ "$HELM_COUNT" -eq 0 ] && [ "$ADDON_COUNT" -eq 0 ]; then
      echo ""
      echo "All dependencies are up to date!"
      rm -f "$DOCKER_RESULTS" "$HELM_RESULTS" "$ADDON_RESULTS"
      exit 0
    fi

    # Check if RELEASE_TOKEN is available for creating issues
    if [ -z "$RELEASE_TOKEN" ]; then
      echo ""
      echo "WARNING: RELEASE_TOKEN not set - cannot create GitLab issue"
      echo "Please update dependencies manually based on the output above"
      rm -f "$DOCKER_RESULTS" "$HELM_RESULTS" "$ADDON_RESULTS"
      exit 0
    fi

    # Build issue description
    ISSUE_TITLE="[Automated] Dependency Updates Available ($(date +%Y-%m-%d))"
    ISSUE_BODY="# Dependency Update Report\n\n"

    # Add Python section
    if [ "$PYTHON_COUNT" -gt 0 ]; then
      ISSUE_BODY+="## Python Packages\n\n"
      ISSUE_BODY+="| Package | Current | Latest |\n"
      ISSUE_BODY+="|---------|---------|--------|\n"
      while IFS= read -r line; do
        NAME=$(echo "$line" | jq -r '.name')
        CURRENT=$(echo "$line" | jq -r '.version')
        LATEST=$(echo "$line" | jq -r '.latest_version')
        ISSUE_BODY+="| $NAME | $CURRENT | $LATEST |\n"
      done < <(echo "$PYTHON_OUTDATED" | jq -c '.[]')
      ISSUE_BODY+="\n"
    fi

    # Add Docker section
    if [ "$DOCKER_COUNT" -gt 0 ]; then
      ISSUE_BODY+="## Docker Images\n\n"
      ISSUE_BODY+="| Image | Current | Latest |\n"
      ISSUE_BODY+="|-------|---------|--------|\n"
      while IFS='|' read -r img cur lat; do
        ISSUE_BODY+="| $img | $cur | $lat |\n"
      done < "$DOCKER_RESULTS"
      ISSUE_BODY+="\n"
    fi

    rm -f "$DOCKER_RESULTS"

    # Add Helm chart section
    if [ "$HELM_COUNT" -gt 0 ]; then
      ISSUE_BODY+="## Helm Charts\n\n"
      ISSUE_BODY+="| Chart | Name | Current | Latest |\n"
      ISSUE_BODY+="|-------|------|---------|--------|\n"
      while IFS='|' read -r cname chart cur lat; do
        ISSUE_BODY+="| $cname | $chart | $cur | $lat |\n"
      done < "$HELM_RESULTS"
      ISSUE_BODY+="\n"
    fi
    rm -f "$HELM_RESULTS"

    # Add EKS add-on section
    if [ "$ADDON_COUNT" -gt 0 ]; then
      ISSUE_BODY+="## EKS Add-ons\n\n"
      ISSUE_BODY+="| Add-on | Current | Latest |\n"
      ISSUE_BODY+="|--------|---------|--------|\n"
      while IFS='|' read -r addon cur lat; do
        ISSUE_BODY+="| $addon | $cur | $lat |\n"
      done < "$ADDON_RESULTS"
      ISSUE_BODY+="\n"
    fi
    rm -f "$ADDON_RESULTS"

    ISSUE_BODY+="## Action Required\n\n"
    ISSUE_BODY+="1. Review changelogs for breaking changes\n"
    ISSUE_BODY+="2. Update versions in \`pyproject.toml\` or manifests\n"
    ISSUE_BODY+="3. Run tests locally to verify compatibility\n"
    ISSUE_BODY+="4. Create a merge request with the updates\n\n"
    ISSUE_BODY+="---\n*This issue was automatically created by the CI pipeline.*"

    # Create GitLab issue via API
    echo ""
    echo "Creating GitLab issue..."

    ESCAPED_BODY=$(echo -e "$ISSUE_BODY" | jq -Rs .)

    HTTP_CODE=$(curl --silent --output /tmp/response.json --write-out "%{http_code}" \
      --request POST \
      --header "PRIVATE-TOKEN: ${RELEASE_TOKEN}" \
      --header "Content-Type: application/json" \
      --data "{\"title\": \"$ISSUE_TITLE\", \"description\": $ESCAPED_BODY, \"labels\": \"dependencies,automated\"}" \
      "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/issues")

    if [ "$HTTP_CODE" -eq 201 ]; then
      ISSUE_URL=$(jq -r '.web_url' /tmp/response.json)
      echo "Issue created successfully: $ISSUE_URL"
    else
      echo "WARNING: Failed to create issue (HTTP $HTTP_CODE)"
      echo "Response: $(cat /tmp/response.json)"
      echo ""
      echo "Please update dependencies manually based on the output above"
    fi

.dependency-scan-base:
  stage: maintenance
  image: python:3.14.4-slim
  needs: []
  allow_failure: true
  before_script:
    - apt-get update && apt-get install -y curl jq skopeo
    - pip install --upgrade pip --root-user-action=ignore
    - pip install -e ".[security]" --root-user-action=ignore
    # Install Helm pinned to the same version as lambda/helm-installer/Dockerfile
    # so the scan compares chart versions against the Helm client we actually
    # deploy with. See HELM_VERSION in the top-level variables block.
    - curl -fsSL "https://get.helm.sh/helm-${HELM_VERSION}-linux-amd64.tar.gz" | tar xz -C /tmp
    - mv /tmp/linux-amd64/helm /usr/local/bin/helm
    - rm -rf /tmp/linux-amd64
    - helm version --short
    # Install kubectl pinned to match the helm-installer Lambda and EKS version.
    - curl -fsSL -o /usr/local/bin/kubectl "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl"
    - chmod +x /usr/local/bin/kubectl
    - kubectl version --client --output=yaml | head -5
    # Install AWS CLI (for EKS addon checks — fails gracefully without credentials)
    - pip install awscli --root-user-action=ignore -q
  script:
    - *dependency-scan-script

# Manual trigger — click "Play" in the pipeline UI
dependency-check:
  extends: .dependency-scan-base
  when: manual

# Scheduled monthly scan — runs on the 1st of each month at 09:00 UTC
# Setup: CI/CD → Schedules → New schedule
#   Description: "Monthly dependency scan"
#   Interval pattern: 0 9 1 * *
#   Cron timezone: UTC
#   Target branch: main
#   Variable: SCHEDULED_SCAN = "true"
dependency-scan:monthly:
  extends: .dependency-scan-base
  rules:
    - if: '$CI_PIPELINE_SOURCE == "schedule" && $SCHEDULED_SCAN == "true"'

# Weekly CVE scan — re-checks dependencies against latest vulnerability databases
# Catches newly published CVEs for packages you already depend on.
# Setup: CI/CD → Schedules → New schedule
#   Description: "Weekly security scan"
#   Interval pattern: 0 9 * * 1
#   Cron timezone: UTC
#   Target branch: main
#   Variable: SECURITY_SCAN = "true"
security-scan:weekly:
  stage: maintenance
  image: python:3.14.4-slim
  needs: []
  rules:
    - if: '$CI_PIPELINE_SOURCE == "schedule" && $SECURITY_SCAN == "true"'
  before_script:
    - pip install --upgrade pip --root-user-action=ignore
    - pip install -e ".[security]"
    - apt-get update && apt-get install -y --no-install-recommends curl
    # Install Trivy pinned to TRIVY_VERSION to match the security:trivy job image.
    - curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin "${TRIVY_VERSION}"
  script:
    - echo "=== Trivy CVE scan ==="
    - trivy fs --exit-code 1 --severity HIGH,CRITICAL --format table .
    - echo "=== Safety dependency scan ==="
    - pip-compile --no-emit-index-url --strip-extras -o /tmp/requirements-lock.txt pyproject.toml -q
    - safety check -r /tmp/requirements-lock.txt