.circleci/config.yml

@@ -1,65 +1,89 @@
-version: 2
+version: 2.1
+
+# Add additional CircleCI Orbs dependencies
+orbs:
+  # https://circleci.com/orbs/registry/orb/circleci/kubernetes
+  kubernetes: circleci/kubernetes@1.3.1
+  # https://circleci.com/orbs/registry/orb/circleci/helm
+  helm: circleci/helm@3.0.0
+  # https://circleci.com/orbs/registry/orb/ccpgames/minikube
+  minikube: ccpgames/minikube@0.0.1
+
 jobs:
-  # Run Helm Lint checks
-  helm-lint:
-    working_directory: ~/stackstorm-ha
-    docker:
-      - image: lachlanevenson/k8s-helm
+  # Spin up minikube K8s cluster and run Helm chart & e2e tests on it
+  e2e-k8s:
+    parameters:
+      kubernetes-version:
+        type: string
+    # 'large' 4 vCPUs & 15GB RAM CircleCI machine executor
+    # required to deploy heavy 'stackstorm-ha' Helm release with RabbitMQ, MongoDB, Redis clusters and 25+ st2 Pods.
+    # https://circleci.com/docs/2.0/configuration-reference/#machine-executor-linux
+    resource_class: large
+    machine:
+      # Available images https://circleci.com/docs/2.0/configuration-reference/#available-machine-images
+      image: ubuntu-2204:current
     steps:
       - checkout
+      - kubernetes/install
+      - minikube/minikube-install:
+          # https://github.com/kubernetes/minikube/releases
+          version: v1.31.2
       - run:
-          name: Prepare Helm
-          command: |
-            set -x
-            helm init --client-only
-            helm repo add incubator https://kubernetes-charts-incubator.storage.googleapis.com/
-            helm dependency update
+          name: Install Helm v3
+          command: curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash
       - run:
-          name: Helm Lint Check (Community)
-          command: helm lint
+          name: Create new K8s cluster
+          command: minikube start --vm-driver=docker --memory 8192 --cpus 4 --kubernetes-version=<< parameters.kubernetes-version >>
       - run:
-          name: Helm Lint Check (Enterprise)
-          command: helm lint --set enterprise.enabled=true --set enterprise.license=123asd456fake
+          name: Update stackstorm-ha chart dependencies
+          command: helm dependency update
       - run:
-          name: Helm template
-          command: |
-            mkdir -p enterprise community
-            helm template --output-dir community .
-            helm template --output-dir enterprise --set enterprise.enabled=true --set enterprise.license=123asd456fake .
-      - persist_to_workspace:
-          root: ~/stackstorm-ha/
-          paths:
-            - community
-            - enterprise
-            # TODO: Fill an issue in https://github.com/garethr/kubeval
-            # 'charts' contains 3rd party templates which doesn't validate against schema due to minor 'object != null' API validation issues
-            # See: https://circleci.com/gh/StackStorm/stackstorm-enterprise-ha/18
-            #- charts
-
-  # Run Kubernetes lint checks
-  k8s-lint:
-    docker:
-      - image: garethr/kubeval
-    steps:
-      - attach_workspace:
-          at: .
+          name: Helm install stackstorm-ha chart
+          command: helm install --timeout 15m0s --debug --wait --name-template stackstorm-ha .
       - run:
-          name: K8s Kubeval Lint Check (Community)
-          command: kubeval $(find . -type f)
-          working_directory: community/stackstorm-ha/templates/
+          name: Helm test
+          command: helm test stackstorm-ha
       - run:
-          name: K8s Kubeval Lint Check (Enterprise)
-          command: kubeval $(find . -type f)
-          working_directory: enterprise/stackstorm-ha/templates/
+          name: Helm upgrade with RBAC enabled
+          command: helm upgrade --set st2.rbac.enabled=true --timeout 5m0s --debug --wait stackstorm-ha .
+      - run:
+          name: Helm test with RBAC enabled
+          command: helm test stackstorm-ha
+      - run:
+          when: always
+          name: Show created K8s resources
+          command: kubectl get all
 
 workflows:
   version: 2
-  helm:
+  e2e:
+    jobs:
+      - e2e-k8s:
+          matrix:
+            parameters:
+              # https://kubernetes.io/releases
+              kubernetes-version:
+                - "v1.28.3"
+                - "v1.27.7"
+                - "v1.26.10"
+  # Run periodic nightly Helm tests to ensure there are no regressions
+  e2e-nightly:
     jobs:
-      - helm-lint
-      - k8s-lint:
-          requires:
-            - helm-lint
+      - e2e-k8s:
+          matrix:
+            parameters:
+              # https://kubernetes.io/releases
+              kubernetes-version:
+                - "v1.28.3"
+                - "v1.27.7"
+                - "v1.26.10"
+    triggers:
+      - schedule:
+          cron: "0 1 * * *"
+          filters:
+            branches:
+              only:
+                - master
 
 experimental:
   notify:

.github/FUNDING.yml

@@ -0,0 +1,4 @@
+# https://stackstorm.com/2020/06/12/sponsoring-stackstorm/
+# FAQ: https://stackstorm.com/donate/
+# Expenses: https://github.com/StackStorm/discussions/issues/36
+community_bridge: stackstorm

.github/workflows/e2e.yaml

@@ -0,0 +1,68 @@
+name: E2E Tests
+
+on:
+  pull_request:
+    types:
+      - opened
+      - synchronize
+  push:
+    branches:
+      - master
+  schedule:
+    - cron: "0 1 * * *"
+  workflow_dispatch:
+
+jobs:
+  k3s:
+    name: "k3s (experimental)"
+    runs-on: ubuntu-22.04
+    # NOTE: Just a thought in case the timeouts fail; might not be
+    # necessary, but might not hurt either, would vary based on the
+    # size of the testing matrix, too.
+    timeout-minutes: 30
+    strategy:
+      fail-fast: false
+      max-parallel: 1
+      matrix:
+        # TODO: Document which versions we support and cover them.
+        # https://github.com/StackStorm/stackstorm-k8s/issues/342
+        # https://github.com/k3s-io/k3s/releases
+        k3s-channel:
+          - "v1.28.3+k3s1"
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v3
+
+      - name: Set up K3s
+        id: k3s
+        uses: jupyterhub/action-k3s-helm@v3
+        with:
+          k3s-channel: ${{ matrix.k3s-channel }}
+
+      - name: Update stackstorm-ha chart dependencies
+        run: |
+          set -x
+          helm dependency update
+
+      - name: Helm install
+        run: |
+          helm install --timeout 15m0s --debug --wait \
+            --name-template stackstorm-ha .
+
+      - name: Helm test
+        run: |
+          helm test stackstorm-ha
+
+      - name: Helm upgrade with RBAC enabled
+        run: |
+          helm upgrade --set st2.rbac.enabled=true \
+            --timeout 10m0s --debug --wait stackstorm-ha .
+
+      - name: Helm test
+        run: |
+          helm test stackstorm-ha
+
+      - name: Show all Kubernetes resources
+        if: ${{ always() }}
+        run: |
+          kubectl get all

.github/workflows/lint.yaml

@@ -0,0 +1,71 @@
+name: Lint
+
+on:
+  pull_request:
+    types:
+      - opened
+      - synchronize
+  push:
+    branches:
+      - master
+  schedule:
+    - cron: "0 1 * * *"
+  workflow_dispatch:
+
+jobs:
+  helm-lint:
+    runs-on: ubuntu-22.04
+    timeout-minutes: 10
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v3
+
+      - name: Update stackstorm-ha chart dependencies
+        run: |
+          set -x
+          helm dependency update
+
+      - name: Helm lint
+        run: |
+          helm lint
+
+      - name: Cache community
+        id: cache-community
+        uses: actions/cache@v3
+        with:
+          path: community
+          key: ${{ runner.os }}-community-${{ hashFiles('conf/**', 'templates/**', 'Chart.yaml', 'values.yaml') }}
+
+      - name: Helm template
+        if: steps.cache-community.outputs.cache-hit != 'true'
+        shell: bash
+        run: |
+          helm template --output-dir community .
+
+  k8s-lint:
+    runs-on: ubuntu-22.04
+    timeout-minutes: 10
+    needs: [helm-lint]
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v3
+
+      - name: Install kubeconform-helm
+        run: |
+          helm plugin install https://github.com/jtyr/kubeconform-helm  --version v0.1.17
+
+      - name: Update stackstorm-ha chart dependencies
+        run: |
+          set -x
+          helm dependency update
+
+      - name: Cache community
+        id: cache-community
+        uses: actions/cache@v3
+        with:
+          path: community
+          key: ${{ runner.os }}-community-${{ hashFiles('conf/**', 'templates/**', 'Chart.yaml', 'values.yaml') }}
+
+      - name: Kubernetes kubeconform-helm Lint
+        run: |
+          helm kubeconform .

.github/workflows/unit.yaml

@@ -0,0 +1,47 @@
+name: Unit Tests
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    types:
+      - opened
+      - synchronize
+  schedule:
+    - cron: "0 1 * * *"
+  workflow_dispatch:
+
+jobs:
+  helm-unittest:
+    runs-on: ubuntu-22.04
+    # strategy:
+    #   matrix:
+    # Relevant tools installed by default on ubuntu 20.04:
+    #   - helm 3.8.0
+    #   - jq 1.6
+    #   - kind 0.11.1
+    #   - kubectl 1.23.3
+    #   - minikube 1.25.1
+    #   - python 3.8.10
+    #   - yamllint 1.26.3
+    #   - yq 4.19.1
+    # see: https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Install helm-unittest
+        # We should periodically check to see if another fork has taken over maintenance,
+        # as the de-facto "best" fork has changed several times over the years.
+        run: |
+          helm plugin install https://github.com/helm-unittest/helm-unittest.git  --version v0.5.1
+
+      - name: Install chart dependencies
+        run: |
+          helm dependency update
+
+      - name: Run helm-unittest
+        # by default looks for tests/*_test.yaml
+        run: |
+          helm unittest --color -f 'tests/unit/*_test.yaml' .

.kubeconform

@@ -0,0 +1,8 @@
+# Command line options that can be set multiple times can be defined as an array
+schema-location:
+  - default
+  - https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json
+# Command line options that can be specified without a value must have boolean
+# value in the config file
+summary: true
+verbose: true