release.yaml

---
apiVersion: source.toolkit.fluxcd.io/v1
kind: HelmRepository
metadata:
  name: runner-x86-v2-pub
  namespace: gitlab
spec:
  interval: 10m
  url: https://charts.gitlab.io

---
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
  name: runner-x86-v2-pub
  namespace: gitlab
spec:
  interval: 10m
  chart:
    spec:
      chart: gitlab-runner
      version: 0.86.0 # gitlab-runner@18.9.0
      sourceRef:
        kind: HelmRepository
        name: runner-x86-v2-pub
  dependsOn:
    - name: gitlab
      namespace: gitlab

  valuesFrom:
    # See terraform/modules/sentry/sentry.tf
    - kind: ConfigMap
      name: gitlab-runner-sentry-config
      valuesKey: values.yaml

  values:
    imagePullPolicy: IfNotPresent
    replicas: 6
    probeTimeoutSeconds: 70

    gitlabUrl: "https://gitlab.spack.io/"
    unregisterRunners: true
    terminationGracePeriodSeconds: 21600 # six hours
    concurrent: 20
    checkInterval: 30

    metrics:
      enabled: true

    rbac:
      serviceAccountName: runner

    runners:
      config: |
        [[runners]]
          pre_build_script = """
          if [ ${CI_OIDC_REQUIRED:-1} == 1 ]; then
            echo 'Executing Spack pre-build setup script'

            for cmd in "${PY3:-}" python3 python; do
              if command -v > /dev/null "$cmd"; then
                export PY3="$(command -v "$cmd")"
                break
              fi
            done

            if [ -z "${PY3:-}" ]; then
              echo "Unable to find python3 executable"
              exit 1
            fi

            $PY3 -c "import urllib.request;urllib.request.urlretrieve('https://raw.githubusercontent.com/spack/spack-infrastructure/main/scripts/gitlab_runner_pre_build/pre_build.py', 'pre_build.py')"
            $PY3 pre_build.py > envvars

            . ./envvars
            rm -f envvars
            unset GITLAB_OIDC_TOKEN
          fi
          """

          output_limit = 20480
          environment = ["FF_GITLAB_REGISTRY_HELPER_IMAGE=1"]
          [runners.kubernetes]
            privileged = false
            helper_memory_request = "512M"

            cpu_request = "750m"
            cpu_request_overwrite_max_allowed = "12"
            cpu_limit_overwrite_max_allowed = "24"

            memory_request = "2G"
            memory_request_overwrite_max_allowed = "64G"
            memory_limit = "96G"
            memory_limit_overwrite_max_allowed = "96G"

            namespace = "pipeline"
            poll_timeout = 600  # ten minutes
            service_account = "runner"

            [runners.kubernetes.affinity]
              [runners.kubernetes.affinity.node_affinity]

              # Schedule this pod on any node with x86_64 >= v2
              [runners.kubernetes.affinity.node_affinity.required_during_scheduling_ignored_during_execution]
                [[runners.kubernetes.affinity.node_affinity.required_during_scheduling_ignored_during_execution.node_selector_terms]]
                  [[runners.kubernetes.affinity.node_affinity.required_during_scheduling_ignored_during_execution.node_selector_terms.match_expressions]]
                      key = "spack.io/x86_64"
                      operator = "In"
                      values = ["v2", "v3", "v4"]
                  [[runners.kubernetes.affinity.node_affinity.required_during_scheduling_ignored_during_execution.node_selector_terms.match_expressions]]
                      key = "spack.io/pipeline"
                      operator = "Exists"

              # Weight this pod towards x86-64-v2 nodes
              [[runners.kubernetes.affinity.node_affinity.preferred_during_scheduling_ignored_during_execution]]
                  weight = 3
                  [[runners.kubernetes.affinity.node_affinity.preferred_during_scheduling_ignored_during_execution.preference.match_expressions]]
                    key = "spack.io/x86_64"
                    operator = "In"
                    values = ["v2"]
              [[runners.kubernetes.affinity.node_affinity.preferred_during_scheduling_ignored_during_execution]]
                  weight = 2
                  [[runners.kubernetes.affinity.node_affinity.preferred_during_scheduling_ignored_during_execution.preference.match_expressions]]
                    key = "spack.io/x86_64"
                    operator = "In"
                    values = ["v3"]
              [[runners.kubernetes.affinity.node_affinity.preferred_during_scheduling_ignored_during_execution]]
                  weight = 1
                  [[runners.kubernetes.affinity.node_affinity.preferred_during_scheduling_ignored_during_execution.preference.match_expressions]]
                    key = "spack.io/x86_64"
                    operator = "In"
                    values = ["v4"]

              # Place pod close to other pipeline pods if possible ("pack" the pods tightly)
              # This takes precedence over the above weights, prioritizing pod packing
              # Docs: https://docs.gitlab.com/runner/executors/kubernetes.html#define-nodes-where-pods-are-scheduled
              [runners.kubernetes.affinity.pod_affinity]
                [[runners.kubernetes.affinity.pod_affinity.preferred_during_scheduling_ignored_during_execution]]
                weight = 4
                [runners.kubernetes.affinity.pod_affinity.preferred_during_scheduling_ignored_during_execution.pod_affinity_term]
                  topology_key = "topology.kubernetes.io/zone"
                  [runners.kubernetes.affinity.pod_affinity.preferred_during_scheduling_ignored_during_execution.pod_affinity_term.label_selector]
                    [[runners.kubernetes.affinity.pod_affinity.preferred_during_scheduling_ignored_during_execution.pod_affinity_term.label_selector.match_expressions]]
                      key = "spack.io/runner"
                      operator = "In"
                      values = ["true"]

            [runners.kubernetes.node_tolerations]
              "spack.io/runner-taint=true" = "NoSchedule"

            [runners.kubernetes.pod_annotations]
              "pod-cleanup.gitlab.com/ttl" = "12h"
              "fluentbit.io/exclude" = "true"
              "karpenter.sh/do-not-disrupt" = "true"
              "gitlab/ci_pipeline_url" = "$CI_PIPELINE_URL"
              "gitlab/ci_job_url" = "$CI_JOB_URL"
              "gitlab/ci_project_url" = "$CI_PROJECT_URL"
              "gitlab/ci_runner_description" = "$CI_RUNNER_DESCRIPTION"
              "gitlab/ci_job_id" = "$CI_JOB_ID"
              "metrics/spack_job_spec_pkg_name" = "$SPACK_JOB_SPEC_PKG_NAME"
              "metrics/spack_job_spec_hash" = "$SPACK_JOB_SPEC_DAG_HASH"
              "metrics/spack_job_spec_pkg_version" = "$SPACK_JOB_SPEC_PKG_VERSION"
              "metrics/spack_job_spec_compiler_name" = "$SPACK_JOB_SPEC_COMPILER_NAME"
              "metrics/spack_job_spec_compiler_version" = "$SPACK_JOB_SPEC_COMPILER_VERSION"
              "metrics/spack_job_spec_arch" = "$SPACK_JOB_SPEC_ARCH"
              "metrics/spack_job_spec_variants" = "$SPACK_JOB_SPEC_VARIANTS"
              "metrics/spack_job_build_jobs" = "$SPACK_BUILD_JOBS"
              "metrics/spack_ci_stack_name" = "$SPACK_CI_STACK_NAME"
            [runners.kubernetes.pod_labels]
              "spack.io/runner" = "true"
              "gitlab/ci_job_id" = "$CI_JOB_ID"
              "gitlab/ci_job_size" = "$CI_JOB_SIZE"
              "metrics/gitlab_ci_pipeline_id" = "$CI_PIPELINE_ID"
              "metrics/gitlab_ci_project_namespace" = "$CI_PROJECT_NAMESPACE"
              "metrics/gitlab_ci_project_name" = "$CI_PROJECT_NAME"
              "metrics/gitlab_ci_job_stage" = "$CI_JOB_STAGE"
              "metrics/gitlab_ci_commit_ref_name" = "$CI_COMMIT_REF_NAME"
              "metrics/spack_ci_stack_name" = "$SPACK_CI_STACK_NAME"
              "metrics/spack_job_spec_pkg_name" = "$SPACK_JOB_SPEC_PKG_NAME"
              "metrics/spack_spec_needs_rebuild" = "$SPACK_SPEC_NEEDS_REBUILD"

      # default image
      image: "busybox:1.32.0"
      imagePullPolicy: "if-not-present"
      locked: false

      tags: "x86_64,x86_64_v2,small,medium,large,huge,public,aws,spack,service,service_noop"
      runUntagged: false
      secret: spack-group-runner-secret

      cache: {}

      services: {}

      helpers: {}

    nodeSelector:
      spack.io/node-pool: base # pool for the runner

    resources:
      requests:
        # Based on this prometheus query:
        # sum by (pod) (container_memory_max_usage_bytes{namespace="gitlab", pod=~"runner-x86.*"})
        memory: 1G

    podAnnotations:
      karpenter.sh/do-not-disrupt: true