.gitlab-ci.yml

---
default:
  image: quay.io/testing-farm/python-ci-image:latest

variables:
  # GitLab CI runs in world writable directory, we need to explicitely set ANSIBLE_CONFIG
  # https://docs.ansible.com/ansible/devel/reference_appendices/config.html#cfg-in-world-writable-dir
  ANSIBLE_CONFIG: ansible.cfg
  # Used only for staging/ci environment, no-op for others
  STAGING_CI_SUFFIX: $CI_MERGE_REQUEST_ID-$CI_JOB_ID
  # Set USER used to derive certain dev environment resource names
  USER: gitlab-ci-$CI_JOB_ID

workflow:
  rules:
    # for merge requests
    - if: $CI_MERGE_REQUEST_ID
    # for the tip of the default branch
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
    # for schedules
    - if: $CI_PIPELINE_SOURCE == "schedule"

.rules:
  not_default_branch:
    # do not run for default branch
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
      when: never

  not_schedule:
    # do not run for schedules
    - if: $CI_PIPELINE_SOURCE == "schedule"
      when: never

  only_default_branch:
    # run only for default branch
    - if: $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH
      when: never

  only_schedule:
    # run only for schedule
    - if: $CI_PIPELINE_SOURCE != "schedule"
      when: never

  gitlab_ci:
    - changes:
        - .gitlab-ci.yml

  secrets:
    - changes:
        - ansible/secrets/credentials.yaml

  infrastructure:
    - !reference [.rules, secrets]
    - !reference [.rules, gitlab_ci]
    - changes:
        - .envrc
        - poetry.lock
        - Makefile*
        - setup/*

  ansible:
    - changes:
        - ansible/files/**/*
        - ansible/inventory/**/*
        - ansible/requirements.yaml
        - ansible/roles/**/*
        - ansible/tasks/**/*
        - ansible/vars/**/*

  terragrunt_modules_infra:
    - changes:
        - terragrunt/modules/gitlab/**/*

  terragrunt_modules_testing_farm:
    - changes:
        - terragrunt/modules/artemis/**/*
        - terragrunt/modules/eks/**/*
        - terragrunt/modules/localhost/**/*
        - terragrunt/modules/vault/**/*
        - terragrunt/modules/worker/**/*

  terragrunt_infra:
    - !reference [.rules, infrastructure]
    - !reference [.rules, terragrunt_modules_infra]
    - changes:
        - terragrunt/environments/infra/**/*

  terragrunt_testing_farm:
    - !reference [.rules, infrastructure]
    - !reference [.rules, terragrunt_modules_testing_farm]

  terragrunt_dev:
    - !reference [.rules, terragrunt_testing_farm]
    - changes:
        # NOTE: need to be explicit here, no nice excludes supported yet
        # We do not run this when variables-composes changed for example
        # https://gitlab.com/gitlab-org/gitlab/-/issues/198688
        #
        # NOTE: we do not want this run on config changes, validation is done against staging
        - terragrunt/environments/dev/*/terragrunt.hcl
        - terragrunt/environments/dev/artemis/**/*
        - terragrunt/environments/dev/server/**/*
        - terragrunt/environments/dev/worker-public/jobs/**/*

  terragrunt_staging:
    - !reference [.rules, terragrunt_testing_farm]
    - changes:
        - terragrunt/environments/staging/**/*

  terragrunt_staging_ci:
    - !reference [.rules, terragrunt_testing_farm]
    - changes:
        - terragrunt/environments/staging/artemis/**/*
        - terragrunt/environments/staging/worker-local/**/*

  terragrunt_production:
    - !reference [.rules, infrastructure]
    - changes:
        - terragrunt/environments/production/**/*

  public_compose_tests:
    - changes:
        - tests/worker/public/compose*

  public_pipeline_tests:
    - changes:
        - tests/worker/base-scenarios/*
        - tests/worker/public/tmt*
        - tests/worker/public/sti*

direnv:
  stage: test
  before_script:
    - dnf -y install direnv libffi libffi-devel
    - echo -n "$VAULT_PASSWORD" > .vault_pass
  script:
    - direnv allow
    - eval "$(direnv export bash)"
    - make generate/staging/citool-config
  rules:
    - !reference [.rules, not_default_branch]
    - !reference [.rules, not_schedule]

.direnv:
  before_script:
    - dnf -y install direnv libffi libffi-devel podman rsync awk
    - echo -n "$VAULT_PASSWORD" > .vault_pass
    - poetry env use python3.12
    - direnv allow
    - eval "$(direnv export bash)"
  script:
    - eval "$(direnv export bash)"
  artifacts:
    paths:
      - .direnv/envrc.log
    when: always

# Workaround for after_script timing-out
.script_with_cleanup:
  script:
    - /bin/bash -c "${SCRIPT}" && EXIT_CODE=$? || EXIT_CODE=$?
    - if [ ${EXIT_CODE} -ne 0 ] && [ -n "$ERROR_SCRIPT" ]; then /bin/bash -c "${ERROR_SCRIPT}" || true; fi
    - /bin/bash -c "${AFTER_SCRIPT}"
    - "[ ${EXIT_CODE} = 0 ]"

# Archive pytest-gluetool artifacts
.archive_tests:
  variables:
    CI_ARTIFACT_URL_PREFIX_FILE:
      "https://$CI_PAGES_HOSTNAME/-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts"
    CI_ARTIFACT_URL_PREFIX_DIR:
      "$CI_PROJECT_URL/-/jobs/$CI_JOB_ID/artifacts/browse"
  artifacts:
    expose_as: report
    paths:
      - report.html
      - assets/style.css
      - .pytest/
      - .direnv/envrc.log
    when: always

ansible-vault:
  extends: .direnv
  stage: test
  script:
    - !reference [.direnv, script]
    - dnf -y install ansible-lint
    - >
      ansible-vault view $SECRETS_FILE | yamllint -d "rules: {line-length: {level: warning}}" -
  rules:
    - !reference [.rules, not_default_branch]
    - !reference [.rules, not_schedule]
    - !reference [.rules, secrets]

pre-commit:
  extends: .direnv
  stage: test
  variables:
    # Disable parallelism on all levels to mitigate test flakes
    GOMAXPROCS: 1
    TF_PARALLELISM: 1
    TERRAGRUNT_PARALLELISM: 1
  script:
    - !reference [.direnv, script]
    # Need to init the modules, or pre-commit terragrunt_validate will fail :(
    - make -j4 dev/init staging/init staging/init/ci production/init
    # Check if no changes after init (replacement for unstable terragrunt_validate pre-commit hook)
    - "if ! git diff; then echo 'Error: terragrunt init shows some changes.'; exit 1; fi"
    - pre-commit run --config .pre-commit-config-maintainer.yaml --all-files
  rules:
    - !reference [.rules, not_default_branch]
    - !reference [.rules, not_schedule]
    - when: always

test/dev:
  extends: .direnv
  stage: test
  script:
    - !reference [.direnv, script]
    - !reference [.script_with_cleanup, script]
  variables:
    TF_VAR_cluster_name: testing-farm-gitlab-ci-$CI_JOB_ID
    SCRIPT: >-
      make dev/init &&
      make dev/apply &&
      make wait/artemis/dev &&
      make dev/create/public/worker &&
      testing-farm-dev-public-developer request --git-url https://gitlab.com/testing-farm/tests --plan /testing-farm/sanity
    ERROR_SCRIPT: >-
      make dev/kubeconfig &&
      echo &&
      echo "==============================================" &&
      echo "==   Kubernetes Container Logs" &&
      echo "==============================================" &&
      echo &&
      kubectl stern --no-follow --tail 100 -A . &&
      echo &&
      echo "==============================================" &&
      echo "==   Kubernetes Event Logs" &&
      echo "==============================================" &&
      echo &&
      kubectl get events --all-namespaces -o wide
    AFTER_SCRIPT: make dev/destroy > /dev/null
  rules:
    - !reference [.rules, not_default_branch]
    - !reference [.rules, not_schedule]
    - !reference [.rules, ansible]
    - !reference [.rules, terragrunt_dev]

test/staging/pipeline:
  extends: [.direnv, .archive_tests]
  stage: test
  script:
    - !reference [.direnv, script]
    - !reference [.script_with_cleanup, script]
  variables:
    SCRIPT: >-
      git clone https://gitlab.com/testing-farm/profiles &&
      ANSIBLE_COLLECTIONS_PATH=/etc/testing-farm/profiles make -C profiles install/local &&
      make staging/apply/ci &&
      PYTEST_OPTIONS="--profiles /etc/testing-farm/profiles" make test/staging/pipeline/ci
    ERROR_SCRIPT: >-
      make staging/kubeconfig &&
      NAMESPACE="$(env -C terragrunt/environments/staging/ci/artemis terragrunt output --raw namespace)" &&
      echo &&
      echo "==============================================" &&
      echo "==   Kubernetes Container Logs" &&
      echo "==============================================" &&
      echo &&
      kubectl stern --no-follow --tail 100 -n "$NAMESPACE" .
      echo &&
      echo "==============================================" &&
      echo "==   Kubernetes Event Logs" &&
      echo "==============================================" &&
      echo &&
      kubectl get events -n "$NAMESPACE" -o wide
    AFTER_SCRIPT: >-
      make staging/destroy/ci > /dev/null &&
      make test/pytest/oculus
  rules:
    - !reference [.rules, not_default_branch]
    - !reference [.rules, not_schedule]
    - !reference [.rules, terragrunt_staging_ci]
    - !reference [.rules, public_pipeline_tests]

test/staging/pipeline/container:
  extends: [.direnv, .archive_tests]
  stage: test
  script:
    - !reference [.direnv, script]
    - !reference [.script_with_cleanup, script]
  variables:
    PYTEST_PARALLEL_RUNS: 1
    SCRIPT: >-
      git clone https://gitlab.com/testing-farm/profiles &&
      ANSIBLE_COLLECTIONS_PATH=/etc/testing-farm/profiles make -C profiles install/local &&
      PYTEST_OPTIONS="--profiles /etc/testing-farm/profiles" make test/staging/pipeline/container/ci
    AFTER_SCRIPT: >-
      make test/pytest/oculus
  rules:
    - !reference [.rules, not_default_branch]
    - !reference [.rules, not_schedule]
    - !reference [.rules, terragrunt_staging_ci]
    - !reference [.rules, public_pipeline_tests]

test/staging/compose:
  extends: [.direnv, .archive_tests]
  stage: test
  script:
    - !reference [.direnv, script]
    - !reference [.script_with_cleanup, script]
  variables:
    SCRIPT: >-
      git clone https://gitlab.com/testing-farm/profiles &&
      ANSIBLE_COLLECTIONS_PATH=/etc/testing-farm/profiles make -C profiles install/local &&
      make staging/apply/ci &&
      PYTEST_OPTIONS="--profiles /etc/testing-farm/profiles" make test/staging/compose/ci
    ERROR_SCRIPT: >-
      make staging/kubeconfig &&
      NAMESPACE="$(env -C terragrunt/environments/staging/ci/artemis terragrunt output --raw namespace)" &&
      echo &&
      echo "==============================================" &&
      echo "==   Kubernetes Container Logs" &&
      echo "==============================================" &&
      echo &&
      kubectl stern --no-follow --tail 100 -n "$NAMESPACE" .
      echo &&
      echo "==============================================" &&
      echo "==   Kubernetes Event Logs" &&
      echo "==============================================" &&
      echo &&
      kubectl get events -n "$NAMESPACE" -o wide
    AFTER_SCRIPT: >-
      make staging/destroy/ci > /dev/null &&
      make test/pytest/oculus
  rules:
    - !reference [.rules, not_default_branch]
    - !reference [.rules, not_schedule]
    - !reference [.rules, terragrunt_staging_ci]
    - !reference [.rules, public_compose_tests]

test/terraform/modules:
  extends: [.direnv]
  stage: test
  script:
    - dnf -y install golang
    - make test/terraform/modules
  rules:
    - !reference [.rules, not_default_branch]
    - !reference [.rules, not_schedule]
    - !reference [.rules, terragrunt_modules_infra]
    - !reference [.rules, terragrunt_modules_testing_farm]

test/infra:
  extends: .direnv
  stage: test
  script:
    - !reference [.direnv, script]
    - !reference [.script_with_cleanup, script]
  variables:
    TERRAGRUNT_PARALLELISM: 3
    SCRIPT: >-
      make infra/init &&
      make infra/plan
  rules:
    - !reference [.rules, not_default_branch]
    - !reference [.rules, not_schedule]
    - !reference [.rules, terragrunt_infra]

test/staging:
  extends: .direnv
  stage: test
  script:
    - !reference [.direnv, script]
    - !reference [.script_with_cleanup, script]
  variables:
    SCRIPT: >-
      make staging/plan
  rules:
    - !reference [.rules, not_default_branch]
    - !reference [.rules, not_schedule]
    - !reference [.rules, terragrunt_staging]

test/production:
  extends: .direnv
  stage: test
  script:
    - !reference [.direnv, script]
    - !reference [.script_with_cleanup, script]
  variables:
    SCRIPT: >-
      make production/plan

  rules:
    - !reference [.rules, not_default_branch]
    - !reference [.rules, not_schedule]
    - !reference [.rules, terragrunt_production]

redeploy/staging:
  extends: .direnv
  stage: deploy
  script:
    - !reference [.direnv, script]
    - !reference [.script_with_cleanup, script]
  variables:
    SCRIPT: >-
      make staging/init &&
      make staging/destroy &&
      make staging/apply &&
      make wait/artemis/staging &&
      make staging/create/public/worker
  rules:
    - !reference [.rules, only_default_branch]
    - !reference [.rules, only_schedule]
    - if: '$SCHEDULED_JOB == "redeploy/staging"'

cleanup/ci:
  extends: .direnv
  stage: deploy
  script:
    - !reference [.direnv, script]
    - !reference [.script_with_cleanup, script]
  variables:
    SCRIPT: make cleanup/ci
  rules:
    - !reference [.rules, only_default_branch]
    - !reference [.rules, only_schedule]
    - if: '$SCHEDULED_JOB == "cleanup/ci"'

cleanup/dev:
  extends: .direnv
  stage: deploy
  script:
    - !reference [.direnv, script]
    - !reference [.script_with_cleanup, script]
  variables:
    SCRIPT: make dev/destroy/force/all
  rules:
    - !reference [.rules, only_default_branch]
    - !reference [.rules, only_schedule]
    - if: '$SCHEDULED_JOB == "cleanup/dev"'

deploy/infra:
  extends: .direnv
  stage: deploy
  script:
    - !reference [.direnv, script]
    - !reference [.script_with_cleanup, script]
  variables:
    SCRIPT: >-
      make infra/init &&
      make infra/apply
  rules:
    - !reference [.rules, only_default_branch]
    - !reference [.rules, not_schedule]
    - !reference [.rules, terragrunt_infra]

deploy/staging:
  extends: .direnv
  stage: deploy
  script:
    - !reference [.direnv, script]
    - !reference [.script_with_cleanup, script]
  variables:
    SCRIPT: >-
      make staging/apply
  rules:
    - !reference [.rules, only_default_branch]
    - !reference [.rules, not_schedule]
    - !reference [.rules, terragrunt_staging]

deploy/production:
  extends: .direnv
  stage: deploy
  needs:
    # NOTE: prevent artifacts to be copied from dependant jobs, we want a fresh direnv setup here
    - job: deploy/staging
      optional: true
      artifacts: false
  script:
    - !reference [.direnv, script]
    - !reference [.script_with_cleanup, script]
  variables:
    SCRIPT: >-
      make production/apply
  rules:
    - !reference [.rules, only_default_branch]
    - !reference [.rules, not_schedule]
    - !reference [.rules, terragrunt_production]

update/composes/public:
  extends: .direnv
  stage: deploy
  script:
    - !reference [.direnv, script]
    - make compose/update/public
    - git remote set-url origin https://oauth2:${GITLAB_PRIVATE_TOKEN}@gitlab.com/testing-farm/infrastructure.git
    - gitlab create-merge-request --title="Update public composes $(date -u --rfc-3339=date)"
  rules:
    - !reference [.rules, only_schedule]
    - if: '$SCHEDULED_JOB == "compose/update/public"'