From 71666765e24233c8fb28a9b9a8566f2a69875ed0 Mon Sep 17 00:00:00 2001 From: Maja Massarini Date: Wed, 15 Jan 2025 15:31:35 +0100 Subject: [PATCH 1/6] Revert "Remove cpu limits" This reverts commit 453ef729106982453a852d1d9a0cd42b8497163d. If we don't set cpu limits then default limits appear (500m). We need more than this in different pods, thus I keep the limits and increase it even for postgres. --- openshift/dashboard.yml.j2 | 1 + openshift/flower.yml.j2 | 1 + openshift/nginx.yml.j2 | 1 + openshift/packit-service-beat.yml.j2 | 1 + openshift/packit-service-fedmsg.yml.j2 | 1 + openshift/packit-service.yml.j2 | 3 ++- openshift/packit-worker.yml.j2 | 3 ++- openshift/postgres.yml.j2 | 1 + openshift/pushgateway.yml.j2 | 1 + openshift/redict.yml.j2 | 1 + openshift/redis-commander.yml.j2 | 1 + openshift/redis.yml.j2 | 1 + openshift/valkey.yml.j2 | 1 + playbooks/deploy.yml | 1 + roles/deploy/defaults/main.yml | 1 + roles/deploy/tasks/main.yml | 3 +++ vars/packit/prod_template.yml | 1 + 17 files changed, 21 insertions(+), 2 deletions(-) diff --git a/openshift/dashboard.yml.j2 b/openshift/dashboard.yml.j2 index 76f95ab6..b9846f03 100644 --- a/openshift/dashboard.yml.j2 +++ b/openshift/dashboard.yml.j2 @@ -48,6 +48,7 @@ spec: cpu: "5m" limits: memory: "256Mi" + cpu: "50m" volumes: - name: packit-secrets secret: diff --git a/openshift/flower.yml.j2 b/openshift/flower.yml.j2 index afa72188..a1175a47 100644 --- a/openshift/flower.yml.j2 +++ b/openshift/flower.yml.j2 @@ -40,6 +40,7 @@ spec: cpu: "5m" limits: memory: "128Mi" + cpu: "50m" replicas: 1 strategy: type: Recreate diff --git a/openshift/nginx.yml.j2 b/openshift/nginx.yml.j2 index 62077486..faef227a 100644 --- a/openshift/nginx.yml.j2 +++ b/openshift/nginx.yml.j2 @@ -59,6 +59,7 @@ spec: cpu: "5m" limits: memory: "32Mi" + cpu: "10m" replicas: 1 strategy: type: Recreate diff --git a/openshift/packit-service-beat.yml.j2 b/openshift/packit-service-beat.yml.j2 index 27d04fdf..2057f03f 100644 --- a/openshift/packit-service-beat.yml.j2 +++ b/openshift/packit-service-beat.yml.j2 @@ -79,6 +79,7 @@ spec: ephemeral-storage: "80Ki" limits: memory: "256Mi" + cpu: "50m" replicas: 1 strategy: type: Recreate diff --git a/openshift/packit-service-fedmsg.yml.j2 b/openshift/packit-service-fedmsg.yml.j2 index 256aae89..7cdbdb5f 100644 --- a/openshift/packit-service-fedmsg.yml.j2 +++ b/openshift/packit-service-fedmsg.yml.j2 @@ -55,6 +55,7 @@ spec: cpu: "5m" limits: memory: "128Mi" + cpu: "50m" livenessProbe: exec: command: diff --git a/openshift/packit-service.yml.j2 b/openshift/packit-service.yml.j2 index 6d2c7e9d..ca6fc67e 100644 --- a/openshift/packit-service.yml.j2 +++ b/openshift/packit-service.yml.j2 @@ -104,6 +104,7 @@ spec: # you have to temporarily increase (in webUI/console) the limit # and once the alembic upgrade passes, revert. memory: "{{ '4Gi' if project == 'packit--prod' else '512Mi' }}" + cpu: "200m" # In TLS world, hostname needs to match whatever is set in the cert # in our cause, k8s is doing here something like curl https://172.15.2.4:8443/api/healthz/ # which will fail TLS validation. @@ -142,7 +143,7 @@ spec: readOnly: true resources: requests: {memory: "128Mi", cpu: "10m"} - limits: {memory: "128Mi"} + limits: {memory: "128Mi", cpu: "100m"} {% endif %} --- apiVersion: v1 diff --git a/openshift/packit-worker.yml.j2 b/openshift/packit-worker.yml.j2 index 1c86cd0a..9269e0da 100644 --- a/openshift/packit-worker.yml.j2 +++ b/openshift/packit-worker.yml.j2 @@ -166,6 +166,7 @@ spec: cpu: {{ worker_requests_cpu }} limits: memory: {{ worker_limits_memory }} + cpu: {{ worker_limits_cpu }} livenessProbe: exec: command: @@ -196,7 +197,7 @@ spec: readOnly: true resources: requests: {memory: "128Mi", cpu: "10m"} - limits: {memory: "128Mi"} + limits: {memory: "128Mi", cpu: "100m"} {% endif %} --- kind: ImageStream diff --git a/openshift/postgres.yml.j2 b/openshift/postgres.yml.j2 index 52fede4f..541ef120 100644 --- a/openshift/postgres.yml.j2 +++ b/openshift/postgres.yml.j2 @@ -71,6 +71,7 @@ spec: # based on this limit, so keep the limit reasonably high. memory: "{{ '4Gi' if project == 'packit--prod' else '512Mi' }}" # Not utilized most of the time, but useful during migrations and for some queries + cpu: "1" volumeMounts: - name: postgres-data mountPath: /var/lib/pgsql/data diff --git a/openshift/pushgateway.yml.j2 b/openshift/pushgateway.yml.j2 index 508eca0c..6bdab578 100644 --- a/openshift/pushgateway.yml.j2 +++ b/openshift/pushgateway.yml.j2 @@ -42,6 +42,7 @@ spec: cpu: "5m" limits: memory: "32Mi" + cpu: "10m" replicas: 1 strategy: type: Recreate diff --git a/openshift/redict.yml.j2 b/openshift/redict.yml.j2 index f21c2493..4be86ecb 100644 --- a/openshift/redict.yml.j2 +++ b/openshift/redict.yml.j2 @@ -43,6 +43,7 @@ spec: cpu: "10m" limits: memory: "256Mi" + cpu: "10m" volumes: - name: redict-pv persistentVolumeClaim: diff --git a/openshift/redis-commander.yml.j2 b/openshift/redis-commander.yml.j2 index 8414353a..0274e233 100644 --- a/openshift/redis-commander.yml.j2 +++ b/openshift/redis-commander.yml.j2 @@ -51,6 +51,7 @@ spec: cpu: "10m" limits: memory: "32Mi" + cpu: "10m" replicas: 1 strategy: type: Recreate diff --git a/openshift/redis.yml.j2 b/openshift/redis.yml.j2 index 6bb7b16d..dfd9c627 100644 --- a/openshift/redis.yml.j2 +++ b/openshift/redis.yml.j2 @@ -39,6 +39,7 @@ spec: cpu: "10m" limits: memory: "64Mi" + cpu: "10m" volumes: - name: redis-pv persistentVolumeClaim: diff --git a/openshift/valkey.yml.j2 b/openshift/valkey.yml.j2 index 3cf982b5..fb54e3d0 100644 --- a/openshift/valkey.yml.j2 +++ b/openshift/valkey.yml.j2 @@ -43,6 +43,7 @@ spec: cpu: "10m" limits: memory: "256Mi" + cpu: "10m" volumes: - name: valkey-pv persistentVolumeClaim: diff --git a/playbooks/deploy.yml b/playbooks/deploy.yml index f903862c..ad845ebd 100644 --- a/playbooks/deploy.yml +++ b/playbooks/deploy.yml @@ -63,6 +63,7 @@ cpu: "5m" limits: memory: "128Mi" + cpu: "50m" appcode: PCKT-002 servicephase: lab costcenter: "700" diff --git a/roles/deploy/defaults/main.yml b/roles/deploy/defaults/main.yml index d3e89b23..89480d5a 100644 --- a/roles/deploy/defaults/main.yml +++ b/roles/deploy/defaults/main.yml @@ -56,6 +56,7 @@ tokman: cpu: "5m" limits: memory: "128Mi" + cpu: "50m" appcode: PCKT-002 servicephase: lab costcenter: "700" diff --git a/roles/deploy/tasks/main.yml b/roles/deploy/tasks/main.yml index d950de9d..4fee46d3 100644 --- a/roles/deploy/tasks/main.yml +++ b/roles/deploy/tasks/main.yml @@ -188,6 +188,7 @@ worker_requests_memory: "384Mi" worker_requests_cpu: "100m" worker_limits_memory: "1024Mi" + worker_limits_cpu: "400m" ansible.builtin.include_tasks: k8s.yml loop: - "{{ lookup('template', '{{ project_dir }}/openshift/packit-worker.yml.j2') }}" @@ -205,6 +206,7 @@ worker_requests_memory: "320Mi" worker_requests_cpu: "80m" worker_limits_memory: "640Mi" + worker_limits_cpu: "2" ansible.builtin.include_tasks: k8s.yml loop: - "{{ lookup('template', '{{ project_dir }}/openshift/packit-worker.yml.j2') }}" @@ -236,6 +238,7 @@ worker_requests_memory: "768Mi" worker_requests_cpu: "100m" worker_limits_memory: "2048Mi" + worker_limits_cpu: "600m" ansible.builtin.include_tasks: k8s.yml loop: - "{{ lookup('template', '{{ project_dir }}/openshift/packit-worker.yml.j2') }}" diff --git a/vars/packit/prod_template.yml b/vars/packit/prod_template.yml index b0bd0688..1048e5a3 100644 --- a/vars/packit/prod_template.yml +++ b/vars/packit/prod_template.yml @@ -99,3 +99,4 @@ tokman: cpu: "20m" limits: memory: "160Mi" + cpu: "50m" From 175fb85272d3dd9b8c4beb46c478a52121bb095f Mon Sep 17 00:00:00 2001 From: Maja Massarini Date: Wed, 15 Jan 2025 15:45:33 +0100 Subject: [PATCH 2/6] Increase postgreas cpu limit Today we were above 1 cpu usage. Let put here the max we can. I also slightly increased valkey and pushgateway since sometimes we are close to the limit (looking at metrics). --- openshift/postgres.yml.j2 | 2 +- openshift/pushgateway.yml.j2 | 2 +- openshift/valkey.yml.j2 | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/openshift/postgres.yml.j2 b/openshift/postgres.yml.j2 index 541ef120..99749ee6 100644 --- a/openshift/postgres.yml.j2 +++ b/openshift/postgres.yml.j2 @@ -71,7 +71,7 @@ spec: # based on this limit, so keep the limit reasonably high. memory: "{{ '4Gi' if project == 'packit--prod' else '512Mi' }}" # Not utilized most of the time, but useful during migrations and for some queries - cpu: "1" + cpu: "2" volumeMounts: - name: postgres-data mountPath: /var/lib/pgsql/data diff --git a/openshift/pushgateway.yml.j2 b/openshift/pushgateway.yml.j2 index 6bdab578..4d1b7beb 100644 --- a/openshift/pushgateway.yml.j2 +++ b/openshift/pushgateway.yml.j2 @@ -42,7 +42,7 @@ spec: cpu: "5m" limits: memory: "32Mi" - cpu: "10m" + cpu: "50m" replicas: 1 strategy: type: Recreate diff --git a/openshift/valkey.yml.j2 b/openshift/valkey.yml.j2 index fb54e3d0..81b5156f 100644 --- a/openshift/valkey.yml.j2 +++ b/openshift/valkey.yml.j2 @@ -43,7 +43,7 @@ spec: cpu: "10m" limits: memory: "256Mi" - cpu: "10m" + cpu: "50m" volumes: - name: valkey-pv persistentVolumeClaim: From 6deb2c319ba67486df2ab575eaf4c4ae8433149d Mon Sep 17 00:00:00 2001 From: Maja Massarini Date: Wed, 15 Jan 2025 15:47:35 +0100 Subject: [PATCH 3/6] Skip host check The host key is not saved in the template anymore (for security reasons). Skip checking differences in this key between template and local file. --- tasks/check-up-to-date.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tasks/check-up-to-date.yml b/tasks/check-up-to-date.yml index c3182396..84fd9903 100644 --- a/tasks/check-up-to-date.yml +++ b/tasks/check-up-to-date.yml @@ -31,7 +31,8 @@ # Remove the keys expected to be different and lines specific to the diff-format. cmd: >- diff vars/{{ service }}/{{ deployment }}.yml vars/{{ service }}/{{ deployment }}_template.yml | - sed '/api_key/d;/[0-9]\+c[0-9]\+/d;/---/d' + sed '/api_key/d;/[0-9]\+c[0-9]\+/d;/---/d' | + sed '/host/d;/[0-9]\+c[0-9]\+/d;/---/d' register: vars_diff changed_when: vars_diff.rc != 0 failed_when: vars_diff.rc != 0 From dd2d7bb5ca22d3f368add2ced7028c00a3bebbbb Mon Sep 17 00:00:00 2001 From: Matej Focko Date: Wed, 15 Jan 2025 23:42:54 +0100 Subject: [PATCH 4/6] fix(k8s): don't define empty volumeClaimTemplates When conditionalizing the templates for volume claim, do not create even an empty object when it's not defined. Better to avoid it altogether than nesting something by mistake. Signed-off-by: Matej Focko --- openshift/packit-worker.yml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openshift/packit-worker.yml.j2 b/openshift/packit-worker.yml.j2 index 9269e0da..621711de 100644 --- a/openshift/packit-worker.yml.j2 +++ b/openshift/packit-worker.yml.j2 @@ -24,8 +24,8 @@ spec: component: {{ component }} serviceName: "{{ component }}" replicas: {{ worker_replicas }} - volumeClaimTemplates: {% if with_repository_cache and 'long-running' in queues %} + volumeClaimTemplates: - metadata: name: packit-worker-repository-cache {% if managed_platform %} From 8025910d5800678df76592c598080d7402576e19 Mon Sep 17 00:00:00 2001 From: Matej Focko Date: Wed, 15 Jan 2025 23:44:28 +0100 Subject: [PATCH 5/6] fix(k8s): quote dist-git URL URL contains colon for the protocol, quote it to prevent YAML parsing issues. Signed-off-by: Matej Focko --- openshift/packit-worker.yml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openshift/packit-worker.yml.j2 b/openshift/packit-worker.yml.j2 index 621711de..30c92be9 100644 --- a/openshift/packit-worker.yml.j2 +++ b/openshift/packit-worker.yml.j2 @@ -94,7 +94,7 @@ spec: - name: DEPLOYMENT value: {{ deployment }} - name: DISTGIT_URL - value: {{ distgit_url }} + value: "{{ distgit_url }}" - name: DISTGIT_NAMESPACE value: {{ distgit_namespace }} {% if sourcegit_namespace %} From 972d20a01c3aee8a066577eb5a18707c116b093b Mon Sep 17 00:00:00 2001 From: Maja Massarini Date: Thu, 16 Jan 2025 09:25:48 +0100 Subject: [PATCH 6/6] Do not deploy tokman --- vars/packit/dev_template.yml | 2 +- vars/packit/prod_template.yml | 2 +- vars/packit/stg_template.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/vars/packit/dev_template.yml b/vars/packit/dev_template.yml index b8e0b615..0af8fc18 100644 --- a/vars/packit/dev_template.yml +++ b/vars/packit/dev_template.yml @@ -24,7 +24,7 @@ check_up_to_date: false # Check that the current vars file us up to date with the template # check_vars_template_diff: true -# with_tokman: true +with_tokman: true # if you want to deploy fedmsg, please make sure to # edit the queue name in secrets/*/fedora.toml diff --git a/vars/packit/prod_template.yml b/vars/packit/prod_template.yml index 1048e5a3..08e40148 100644 --- a/vars/packit/prod_template.yml +++ b/vars/packit/prod_template.yml @@ -26,7 +26,7 @@ api_key: "" # Check that the current vars file us up to date with the template # check_vars_template_diff: true -# with_tokman: false +with_tokman: false # if you want to deploy fedmsg, please make sure to # edit the queue name in secrets/*/fedora.toml diff --git a/vars/packit/stg_template.yml b/vars/packit/stg_template.yml index 45bdc427..13829769 100644 --- a/vars/packit/stg_template.yml +++ b/vars/packit/stg_template.yml @@ -24,7 +24,7 @@ api_key: "" # Check that the current vars file us up to date with the template # check_vars_template_diff: true -# with_tokman: false +with_tokman: false # if you want to deploy fedmsg, please make sure to # edit the queue name in secrets/*/fedora.toml