Horizontal pod autoscaler (#286)

* Load test DEVOPS-428

* horizontal pod autoscaler DEVOPS-392

* Load test workflow DEVOPS-428

* Prevent parallel deployment DEVOPS-429

Co-authored-by: Tero Virtanen <tero.virtanen@mavericks.fi>

Author: terovirtanen

.github/workflows/load_tests.yml

@@ -0,0 +1,34 @@
+name: k6 Load Tests
+
+# Only trigger when the staging workflow succeeded
+on:
+  workflow_run:
+    workflows: ["Build & Staging"]
+    types:
+      - completed
+
+# allow only one run at a time
+concurrency: ${{ github.workflow }}
+
+jobs:
+  k6_load_test:
+    name: k6 Load Tests
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Run k6 tests
+        uses: k6io/action@v0.1
+        with:
+          filename: k6LoadTests.js
+      - name: k6 Load Tests failed notifications
+        uses: rtCamp/action-slack-notify@v2
+        env:
+          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
+          SLACK_USERNAME: K6LoadTest
+          SLACK_ICON: https://a.slack-edge.com/80588/img/services/outgoing-webhook_48.png
+          SLACK_TITLE: K6 Load Tests has failed.
+          SLACK_MESSAGE: Some of the k6 load tests has failed.
+        if: failure()

.github/workflows/production.yml

@@ -4,6 +4,9 @@ on:
     tags:
       - 'release-*'
 
+# allow only one run at a time
+concurrency: ${{ github.workflow }}
+
 env:
   CONTAINER_REGISTRY: ghcr.io
   CONTAINER_REGISTRY_USER: ${{ secrets.GHCR_CONTAINER_REGISTRY_USER }}
@@ -19,7 +22,12 @@ env:
   K8S_REQUEST_RAM: 400Mi
   K8S_LIMIT_CPU: 800m
   K8S_LIMIT_RAM: 800Mi
-  K8S_REPLICACOUNT: 2
+  # horizontal pod autoscaler
+  K8S_HPA_ENABLED: true
+  K8S_HPA_MAX_REPLICAS: 8
+  K8S_HPA_MIN_REPLICAS: 2
+  K8S_HPA_MAX_CPU_AVG: 80 # 80% from request
+#  K8S_HPA_MAX_RAM_AVG: 80 # 80% from request
 
 jobs:
   build:

.github/workflows/review.yml

@@ -19,6 +19,12 @@ env:
   K8S_REQUEST_RAM: 250Mi
   K8S_LIMIT_CPU: 100m
   K8S_LIMIT_RAM: 350Mi
+  # horizontal pod autoscaler
+  K8S_HPA_ENABLED: true
+  K8S_HPA_MAX_REPLICAS: 2
+  K8S_HPA_MIN_REPLICAS: 1
+  K8S_HPA_MAX_CPU_AVG: 80 # 80% from request
+#  K8S_HPA_MAX_RAM_AVG: 80 # 80% from request
 
 jobs:
   build:
@@ -33,6 +39,9 @@ jobs:
     runs-on: ubuntu-latest
     needs: build
     name: Review
+    # allow only one deployment at a time
+    concurrency: ${{ github.workflow }}-${{ github.head_ref }}
+
     steps:
       - uses: actions/checkout@v2
       - uses: andersinno/kolga-setup-action@v2

.github/workflows/staging.yml

@@ -20,7 +20,12 @@ env:
   K8S_REQUEST_RAM: 250Mi
   K8S_LIMIT_CPU: 100m
   K8S_LIMIT_RAM: 350Mi
-  K8S_REPLICACOUNT: 2
+  # horizontal pod autoscaler
+  K8S_HPA_ENABLED: true
+  K8S_HPA_MAX_REPLICAS: 3
+  K8S_HPA_MIN_REPLICAS: 1
+  K8S_HPA_MAX_CPU_AVG: 80 # 80% from request
+#  K8S_HPA_MAX_RAM_AVG: 80 # 80% from request
 
 jobs:
   build:
@@ -35,6 +40,10 @@ jobs:
     runs-on: ubuntu-latest
     needs: build
     name: Staging
+
+    # allow only one deployment at a time
+    concurrency: ${{ github.workflow }}-${{ github.ref }}
+
     steps:
       - uses: actions/checkout@v2
       - uses: andersinno/kolga-setup-action@v2

k6LoadTests.js

@@ -0,0 +1,24 @@
+/* eslint-disable */
+import { sleep } from 'k6';
+import http from 'k6/http';
+
+export const options = {
+  duration: '10m',
+  vus: 100,
+  //  vus: 1,
+  thresholds: {
+    //avg is around ?100ms? on https://kukkuu.test.kuva.hel.ninja
+    http_req_duration: ['p(95)<5000'],
+  },
+};
+
+export default () => {
+  const url = 'https://kukkuu.test.kuva.hel.ninja/graphql';
+  const data = 'query=query Organisations {organisations {edges {node {id } } } }';
+  const res = http.post(url, data, {
+    headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
+  });
+
+  //10 loads per minute
+  sleep(6);
+};