.github: add v1.18.x to the nightly tests (#10429)

Author: sam-heilbron

.github/workflows/docs-gen.yaml

@@ -2,15 +2,15 @@ name: Docs
 
 env:
     SLACK_DEBUG_TESTING: false      # when set to "true", send notifications to #slack-integration-testing.  Otherwise, post to #edge-team-bots
-    MIN_SCANNED_VERSION: 'v1.14.0'  # ⚠️ you should also change trivy-analysis-scheduled.yaml ⚠️
+    MIN_SCANNED_VERSION: 'v1.15.0'  # ⚠️ you should also change trivy-analysis-scheduled.yaml ⚠️
 on:
   push:
     branches:
     - 'main'
+    - 'v1.18.x'
     - 'v1.17.x'
     - 'v1.16.x'
     - 'v1.15.x'
-    - 'v1.14.x'
   pull_request:
     types: [opened, synchronize, reopened, ready_for_review]
     branches:

.github/workflows/nightly-tests.yaml

@@ -12,19 +12,19 @@ on:
   # Day of the week ([0,6] with 0=Sunday)
   schedule:
     - cron: "0 5 * * *" # every day @ 05:00 UTC, run tests against latest main
-    - cron: "0 6 * * 1" # monday    @ 06:00 UTC, run expanded tests against v1.17.x
-    - cron: "0 7 * * 1" # monday    @ 07:00 UTC, run expanded tests against v1.16.x
-    - cron: "0 8 * * 1" # monday    @ 08:00 UTC, run expanded tests against v1.15.x
+    - cron: "0 6 * * 1" # monday    @ 06:00 UTC, run expanded tests against v1.18.x
+    - cron: "0 7 * * 1" # monday    @ 07:00 UTC, run expanded tests against v1.17.x
+    - cron: "0 8 * * 1" # monday    @ 08:00 UTC, run expanded tests against v1.16.x
   workflow_dispatch:
     inputs:
       branch:
         description: "The branch to run tests against"
         type: choice
         options:
           - main
+          - v1.18.x
           - v1.17.x
           - v1.16.x
-          - v1.15.x
           - workflow_initiating_branch
       run-regression:
         description: "Run regression tests"
@@ -177,9 +177,71 @@ jobs:
           matrix-label: ${{ matrix.version-files.label }}
 
   # Reminder: when setting up the job next release branch, copy from "end_to_end_tests_main" not the previous release job as configuration may have changed
+  end_to_end_tests_18:
+    name: End-to-End (branch=v1.18.x, cluster=${{ matrix.test.cluster-name }}, version=${{ matrix.version-files.label }} )
+    if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run-kubernetes-end-to-end && inputs.branch == 'v1.18.x') || github.event.schedule == '0 6 * * 1' }}
+    runs-on: ubuntu-22.04
+    timeout-minutes: 180
+    strategy:
+      # Since we are running these on a schedule, there is no value in failing fast
+      # In fact, we want to ensure that all tests run, so that we have a clearer picture of which tests are prone to flaking
+      fail-fast: false
+      matrix:
+        test:
+          # When running the tests at night, there is no value in splitting the tests across multiple clusters and running them in parallel.
+          # As a result, we increase the threshold for the tests, since they all run serially on a single cluster
+          - cluster-name: 'cluster-one'
+            go-test-args: '-v -timeout=150m'
+            # Specifying an empty regex means all tests will be run.
+            go-test-run-regex: ""
+        # In our nightly tests, we run the suite of tests using the lower and upper ends of versions that we claim to support
+        # The versions should mirror: https://docs.solo.io/gloo-edge/latest/reference/support/
+        version-files:
+          - label: 'min'
+            file: './.github/workflows/.env/nightly-tests/min_versions.env'
+          - label: 'max'
+            file: './.github/workflows/.env/nightly-tests/max_versions.env'
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: main
+      # The dotenv action is used to load key-value pairs from files.
+      # In this case, the file is specified in the matrix and will contain the versions of the tools to use
+      - name: Dotenv Action
+        uses: falti/[email protected]
+        id: dotenv
+        with:
+          path: ${{ matrix.version-files.file }}
+          log-variables: true
+      - name: Prep Go Runner
+        uses: ./.github/workflows/composite-actions/prep-go-runner
+      # Set up the KinD cluster that the tests will use
+      - id: setup-kind-cluster
+        name: Setup KinD Cluster
+        uses: ./.github/workflows/composite-actions/setup-kind-cluster
+        with:
+          cluster-name: ${{ matrix.test.cluster-name }}
+          kind-node-version: ${{ steps.dotenv.outputs.node_version }}
+          kind-version: ${{ steps.dotenv.outputs.kind_version }}
+          kubectl-version: ${{ steps.dotenv.outputs.kubectl_version }}
+          helm-version: ${{ steps.dotenv.outputs.helm_version }}
+          istio-version: ${{ steps.dotenv.outputs.istio_version }}
+          k8sgateway-api-version: ${{ steps.dotenv.outputs.k8sgateway_api_version }}
+      # Run the tests
+      - id: run-tests
+        name: Run Kubernetes e2e Tests
+        uses: ./.github/workflows/composite-actions/kubernetes-e2e-tests
+        with:
+          cluster-name: ${{ matrix.test.cluster-name }}
+          test-args: ${{ matrix.test.go-test-args }}
+          run-regex: ${{ matrix.test.go-test-run-regex }}
+          istio-version: ${{ steps.dotenv.outputs.istio_version }}
+          matrix-label: ${{ matrix.version-files.label }}
+
   end_to_end_tests_17:
     name: End-to-End (branch=v1.17.x, cluster=${{ matrix.test.cluster-name }}, version=${{ matrix.version-files.label }} )
-    if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run-kubernetes-end-to-end && inputs.branch == 'v1.17.x') || github.event.schedule == '0 6 * * 1' }}
+    if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run-kubernetes-end-to-end && inputs.branch == 'v1.17.x') || github.event.schedule == '0 7 * * 1' }}
     runs-on: ubuntu-22.04
     timeout-minutes: 150
     strategy:
@@ -286,9 +348,39 @@ jobs:
         ref: main
     - uses: ./.github/workflows/composite-actions/regression-tests
 
+  regression_tests_18:
+    name: v1.18.x regression tests
+    if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run-regression && inputs.branch == 'v1.18.x') || github.event.schedule == '0 6 * * 1' }}
+    runs-on: ubuntu-22.04
+    timeout-minutes: 60
+    # Previously, there was an environment variable, RELEASED_VERSION="LATEST" set.  This made use of some internal code:
+    #       https://github.com/solo-io/gloo/blob/main/test/kube2e/util.go#L229-L241
+    # which modified our testing process to pull the latest beta release.
+    #
+    # NOW, however, running this job is the same as normal CI.  (building a local chart, then using it)
+    strategy:
+      fail-fast: false
+      matrix:
+        # TODO:
+        #   As part of the end_to_end_tests_main job, we added support for importing versions from a .env file
+        #   We should extend the support/usage of those .env files to these other jobs.
+        #   The tests are currently in flux, and some of these regression tests are being migrated, so we decided
+        #   to limit the scope (and potentially unnecessary work) for now
+        kube-e2e-test-type: [ 'gateway', 'gloo', 'upgrade' ]
+        kube-version: [ { node: 'v1.27.3@sha256:3966ac761ae0136263ffdb6cfd4db23ef8a83cba8a463690e98317add2c9ba72', kubectl: 'v1.27.3', kind: 'v0.20.0', helm: 'v3.13.2' },
+                        { node: 'v1.31.0@sha256:53df588e04085fd41ae12de0c3fe4c72f7013bba32a20e7325357a1ac94ba865', kubectl: 'v1.31.0', kind: 'v0.24.0', helm: 'v3.14.4' } ]
+        image-variant:
+          - standard
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: v1.18.x
+      - uses: ./.github/workflows/composite-actions/regression-tests
+
+
   regression_tests_17:
     name: v1.17.x regression tests
-    if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run-regression && inputs.branch == 'v1.17.x') || github.event.schedule == '0 6 * * 1' }}
+    if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run-regression && inputs.branch == 'v1.17.x') || github.event.schedule == '0 7 * * 1' }}
     runs-on: ubuntu-22.04
     timeout-minutes: 60
     strategy:
@@ -306,7 +398,7 @@ jobs:
 
   regression_tests_16:
     name: v1.16.x regression tests
-    if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run-regression && inputs.branch == 'v1.16.x') || github.event.schedule == '0 7 * * 1' }}
+    if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run-regression && inputs.branch == 'v1.16.x') || github.event.schedule == '0 8 * * 1' }}
     runs-on: ubuntu-22.04
     timeout-minutes: 60
     strategy:
@@ -321,23 +413,6 @@ jobs:
           ref: v1.16.x
       - uses: ./.github/workflows/composite-actions/regression-tests
 
-  regression_tests_15:
-    name: v1.15.x regression tests
-    if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run-regression && inputs.branch == 'v1.15.x') || github.event.schedule == '0 8 * * 1' }}
-    runs-on: ubuntu-22.04
-    timeout-minutes: 60
-    strategy:
-      fail-fast: false
-      matrix:
-        kube-e2e-test-type: ['gateway', 'gloo', 'ingress', 'helm', 'gloomtls', 'glooctl', 'upgrade']
-        kube-version: [ { node: 'v1.23.13@sha256:ef453bb7c79f0e3caba88d2067d4196f427794086a7d0df8df4f019d5e336b61', kubectl: 'v1.23.17', kind: 'v0.17.0', helm: 'v3.11.2' },
-                        { node: 'v1.27.3@sha256:3966ac761ae0136263ffdb6cfd4db23ef8a83cba8a463690e98317add2c9ba72', kubectl: 'v1.27.3', kind: 'v0.20.0', helm: 'v3.13.2' } ]
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          ref: v1.15.x
-      - uses: ./.github/workflows/composite-actions/regression-tests
-
   performance_tests_on_demand:
     name: on demand performance tests
     if: ${{ github.event_name == 'workflow_dispatch' && inputs.run-performance && inputs.branch == 'workflow_initiating_branch' }}
@@ -362,39 +437,39 @@ jobs:
       - uses: ./.github/workflows/composite-actions/prep-go-runner
       - uses: ./.github/workflows/composite-actions/performance-tests
 
-  performance_tests_17:
-    name: v1.17.x performance tests
-    if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run-performance && inputs.branch == 'v1.17.x') || github.event.schedule == '0 6 * * 1' }}
+  performance_tests_18:
+    name: v1.18.x performance tests
+    if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run-performance && inputs.branch == 'v1.18.x') || github.event.schedule == '0 6 * * 1' }}
     runs-on: ubuntu-22.04
     timeout-minutes: 60
     steps:
       - uses: actions/checkout@v4
         with:
-          ref: v1.17.x
+          ref: v1.18.x
       - uses: ./.github/workflows/composite-actions/prep-go-runner
       - uses: ./.github/workflows/composite-actions/performance-tests
 
-  performance_tests_16:
-    name: v1.16.x performance tests
-    if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run-performance && inputs.branch == 'v1.16.x') || github.event.schedule == '0 7 * * 1' }}
+  performance_tests_17:
+    name: v1.17.x performance tests
+    if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run-performance && inputs.branch == 'v1.17.x') || github.event.schedule == '0 7 * * 1' }}
     runs-on: ubuntu-22.04
     timeout-minutes: 60
     steps:
       - uses: actions/checkout@v4
         with:
-          ref: v1.16.x
+          ref: v1.17.x
       - uses: ./.github/workflows/composite-actions/prep-go-runner
       - uses: ./.github/workflows/composite-actions/performance-tests
 
-  performance_tests_15:
-    name: v1.15.x performance tests
-    if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run-performance && inputs.branch == 'v1.15.x') || github.event.schedule == '0 8 * * 1' }}
+  performance_tests_16:
+    name: v1.16.x performance tests
+    if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run-performance && inputs.branch == 'v1.16.x') || github.event.schedule == '0 8 * * 1' }}
     runs-on: ubuntu-22.04
     timeout-minutes: 60
     steps:
       - uses: actions/checkout@v4
         with:
-          ref: v1.15.x
+          ref: v1.16.x
       - uses: ./.github/workflows/composite-actions/prep-go-runner
       - uses: ./.github/workflows/composite-actions/performance-tests
 
@@ -416,9 +491,28 @@ jobs:
         ref: main
     - uses: ./.github/workflows/composite-actions/kube-gateway-api-conformance-tests
 
+  kube_gateway_api_conformance_tests_18:
+    name: Conformance (branch=v1.18.x, type=Kubernetes Gateway API, version=${{matrix.kube-version.node}} )
+    if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run-conformance && inputs.branch == 'v1.18.x') || github.event.schedule == '0 6 * * 1' }}
+    runs-on: ubuntu-22.04
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        kube-version: [ { node: 'v1.27.3@sha256:3966ac761ae0136263ffdb6cfd4db23ef8a83cba8a463690e98317add2c9ba72', kubectl: 'v1.27.3', kind: 'v0.20.0', helm: 'v3.13.2' },
+                        { node: 'v1.31.0@sha256:53df588e04085fd41ae12de0c3fe4c72f7013bba32a20e7325357a1ac94ba865', kubectl: 'v1.31.0', kind: 'v0.24.0', helm: 'v3.14.4' }]
+        image-variant:
+          - standard
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: v1.18.x
+      - uses: ./.github/workflows/composite-actions/kube-gateway-api-conformance-tests
+
+
   kube_gateway_api_conformance_tests_17:
     name: Conformance (branch=v1.17.x, type=Kubernetes Gateway API, version=${{matrix.kube-version.node}} )
-    if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run-conformance && inputs.branch == 'v1.17.x') || github.event.schedule == '0 6 * * 1' }}
+    if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run-conformance && inputs.branch == 'v1.17.x') || github.event.schedule == '0 7 * * 1' }}
     runs-on: ubuntu-22.04
     timeout-minutes: 60
     strategy:
@@ -440,16 +534,18 @@ jobs:
     if: ${{ always() }}
     needs:
       - end_to_end_tests_main
+      - end_to_end_tests_18
       - end_to_end_tests_17
       - regression_tests_main
+      - regression_tests_18
       - regression_tests_17
       - regression_tests_16
-      - regression_tests_15
       - performance_tests_main
+      - performance_tests_18
       - performance_tests_17
       - performance_tests_16
-      - performance_tests_15
       - kube_gateway_api_conformance_tests_main
+      - kube_gateway_api_conformance_tests_18
       - kube_gateway_api_conformance_tests_17
       - end_to_end_tests_on_demand
       - regression_tests_on_demand
@@ -470,13 +566,13 @@ jobs:
             branch="main"
           elif [[ ${{github.event.schedule == '0 6 * * 1'}} = true ]]; then
             trigger="Gloo OSS weeklies"
-            branch="v1.17.x"
+            branch="v1.18.x"
           elif [[ ${{github.event.schedule == '0 7 * * 1'}} = true ]]; then
             trigger="Gloo OSS weeklies"
-            branch="v1.16.x"
+            branch="v1.17.x"
           elif [[ ${{github.event.schedule == '0 8 * * 1'}} = true ]]; then
             trigger="Gloo OSS nightlies"
-            branch="v1.15.x"
+            branch="v1.16.x"
           fi
           preamble="$trigger ($branch)"
           echo "Setting PREAMBLE as $preamble"

.github/workflows/trivy-analysis-scheduled.yaml

@@ -38,7 +38,7 @@ jobs:
           SCAN_DIR: _output/scans
           IMAGE_REGISTRY: quay.io/solo-io
           # ON_LTS_UPDATE - bump version
-          MIN_SCANNED_VERSION: 'v1.14.0' # ⚠️ you should also change docs-gen.yaml ⚠️
+          MIN_SCANNED_VERSION: 'v1.15.0' # ⚠️ you should also change docs-gen.yaml ⚠️
         run: |
           mkdir -p $SCAN_DIR
           make run-security-scan

Makefile

@@ -1250,7 +1250,7 @@ SCAN_DIR ?= $(OUTPUT_DIR)/scans
 SCAN_BUCKET ?= solo-gloo-security-scans
 # The minimum version to scan with trivy
 # ON_LTS_UPDATE - bump version
-MIN_SCANNED_VERSION ?= v1.14.0
+MIN_SCANNED_VERSION ?= v1.15.0
 
 .PHONY: run-security-scans
 run-security-scan:

changelog/v1.19.0-beta1/1-19-main-branch-cleanup.yaml

@@ -0,0 +1,7 @@
+changelog:
+  - type: NON_USER_FACING
+    description: >-
+      Update the main branch nightly tests after cutting the 1.18.x branch
+      
+      skipCI-kube-tests:true
+      skipCI-docs-build:true

docs/Makefile

@@ -11,7 +11,7 @@ HUGO_VERSION := 0.81.0
 SOLO_HUGO_THEME_REVISION := v0.0.29
 
 # The minimum version to maintain in our public docs
-MIN_SCANNED_VERSION ?= v1.13.0
+MIN_SCANNED_VERSION ?= v1.15.0
 
 #----------------------------------------------------------------------------------
 # Docs