From bad1d82eff20c1e8640763cd80a378dd0957cb2f Mon Sep 17 00:00:00 2001 From: obetsun Date: Mon, 26 May 2025 15:19:35 +0300 Subject: [PATCH 1/8] feat(KONFLUX-8225): add log forwarder for KubeArchive Signed-off-by: obetsun --- .../infra-deployments/kustomization.yaml | 1 + .../kustomization.yaml | 6 + .../vector-kubearchive-logs-collector.yaml | 41 +++++ .../base/kustomization.yaml | 21 +++ .../base/vector-helm-generator.yaml | 10 ++ .../base/vector-helm-values.yaml | 150 ++++++++++++++++++ .../base/vector-pre.yaml | 54 +++++++ .../development/kustomization.yaml | 6 + .../development/loki-helm-generator.yaml | 10 ++ .../development/loki-helm-values.yaml | 77 +++++++++ .../development/values.minio.yaml | 20 +++ .../production/kustomization.yaml | 6 + .../production/loki-helm-generator.yaml | 10 ++ .../production/loki-helm-values.yaml | 77 +++++++++ .../production/values.minio.yaml | 20 +++ .../staging/kustomization.yaml | 7 + .../staging/loki-helm-generator.yaml | 10 ++ .../staging/loki-helm-values.yaml | 77 +++++++++ .../staging/values.minio.yaml | 20 +++ 19 files changed, 623 insertions(+) create mode 100644 argo-cd-apps/base/member/infra-deployments/vector-kubearchive-log-collector/kustomization.yaml create mode 100644 argo-cd-apps/base/member/infra-deployments/vector-kubearchive-log-collector/vector-kubearchive-logs-collector.yaml create mode 100644 components/vector-kubearchive-log-collector/base/kustomization.yaml create mode 100644 components/vector-kubearchive-log-collector/base/vector-helm-generator.yaml create mode 100644 components/vector-kubearchive-log-collector/base/vector-helm-values.yaml create mode 100644 components/vector-kubearchive-log-collector/base/vector-pre.yaml create mode 100644 components/vector-kubearchive-log-collector/development/kustomization.yaml create mode 100644 components/vector-kubearchive-log-collector/development/loki-helm-generator.yaml create mode 100644 components/vector-kubearchive-log-collector/development/loki-helm-values.yaml create mode 100644 components/vector-kubearchive-log-collector/development/values.minio.yaml create mode 100644 components/vector-kubearchive-log-collector/production/kustomization.yaml create mode 100644 components/vector-kubearchive-log-collector/production/loki-helm-generator.yaml create mode 100644 components/vector-kubearchive-log-collector/production/loki-helm-values.yaml create mode 100644 components/vector-kubearchive-log-collector/production/values.minio.yaml create mode 100644 components/vector-kubearchive-log-collector/staging/kustomization.yaml create mode 100644 components/vector-kubearchive-log-collector/staging/loki-helm-generator.yaml create mode 100644 components/vector-kubearchive-log-collector/staging/loki-helm-values.yaml create mode 100644 components/vector-kubearchive-log-collector/staging/values.minio.yaml diff --git a/argo-cd-apps/base/member/infra-deployments/kustomization.yaml b/argo-cd-apps/base/member/infra-deployments/kustomization.yaml index 01b9b81e338..404807fd695 100644 --- a/argo-cd-apps/base/member/infra-deployments/kustomization.yaml +++ b/argo-cd-apps/base/member/infra-deployments/kustomization.yaml @@ -30,6 +30,7 @@ resources: - konflux-ui - konflux-rbac - konflux-info + - vector-kubearchive-logs-collector - vector-tekton-logs-collector - kyverno - namespace-lister diff --git a/argo-cd-apps/base/member/infra-deployments/vector-kubearchive-log-collector/kustomization.yaml b/argo-cd-apps/base/member/infra-deployments/vector-kubearchive-log-collector/kustomization.yaml new file mode 100644 index 00000000000..b4d56f5d507 --- /dev/null +++ b/argo-cd-apps/base/member/infra-deployments/vector-kubearchive-log-collector/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- vector-kubearchive-logs-collector.yaml +components: + - ../../../../k-components/deploy-to-member-cluster-merge-generator diff --git a/argo-cd-apps/base/member/infra-deployments/vector-kubearchive-log-collector/vector-kubearchive-logs-collector.yaml b/argo-cd-apps/base/member/infra-deployments/vector-kubearchive-log-collector/vector-kubearchive-logs-collector.yaml new file mode 100644 index 00000000000..f92f37e67ad --- /dev/null +++ b/argo-cd-apps/base/member/infra-deployments/vector-kubearchive-log-collector/vector-kubearchive-logs-collector.yaml @@ -0,0 +1,41 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: vector-kubearchive-logs-collector +spec: + generators: + - merge: + mergeKeys: + - nameNormalized + generators: + - clusters: + values: + sourceRoot: components/vector-kubearchive-logs-collector + environment: staging + clusterDir: "" + - list: + elements: [] + template: + metadata: + name: vector-kubearchive-logs-collector-{{nameNormalized}} + spec: + project: default + source: + path: '{{values.sourceRoot}}/{{values.environment}}/{{values.clusterDir}}' + repoURL: https://github.com/redhat-appstudio/infra-deployments.git + targetRevision: main + destination: + namespace: kubearchive-logging + server: '{{server}}' + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + retry: + limit: -1 + backoff: + duration: 10s + factor: 2 + maxDuration: 3m diff --git a/components/vector-kubearchive-log-collector/base/kustomization.yaml b/components/vector-kubearchive-log-collector/base/kustomization.yaml new file mode 100644 index 00000000000..3c88f50d1e4 --- /dev/null +++ b/components/vector-kubearchive-log-collector/base/kustomization.yaml @@ -0,0 +1,21 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: kubearchive-logging + +commonAnnotations: + argocd.argoproj.io/sync-wave: "-1" + ignore-check.kube-linter.io/run-as-non-root: > + "Vector Runs as Root and attach host Path." + ignore-check.kube-linter.io/sensitive-host-mounts: > + "Vector Runs requires certain host mounts to watch + files being created by pods." + ignore-check.kube-linter.io/drop-net-raw-capability: > + "Vector Runs requires access to socket." + +generators: + - vector-helm-generator.yaml + +resources: + - vector-pre.yaml + diff --git a/components/vector-kubearchive-log-collector/base/vector-helm-generator.yaml b/components/vector-kubearchive-log-collector/base/vector-helm-generator.yaml new file mode 100644 index 00000000000..638556e44a2 --- /dev/null +++ b/components/vector-kubearchive-log-collector/base/vector-helm-generator.yaml @@ -0,0 +1,10 @@ +apiVersion: builtin +kind: HelmChartInflationGenerator +metadata: + name: vector +name: vector +repo: https://helm.vector.dev +version: 0.46.1 +releaseName: vector-kubearchive-logs-collector +namespace: kubearchive-logging +valuesFile: vector-helm-values.yaml diff --git a/components/vector-kubearchive-log-collector/base/vector-helm-values.yaml b/components/vector-kubearchive-log-collector/base/vector-helm-values.yaml new file mode 100644 index 00000000000..5d11aaa4e43 --- /dev/null +++ b/components/vector-kubearchive-log-collector/base/vector-helm-values.yaml @@ -0,0 +1,150 @@ +--- +role: Agent +resources: + requests: + cpu: 512m + memory: 4096Mi + limits: + cpu: 2000m + memory: 4096Mi +customConfig: + data_dir: /vector-data-dir + api: + enabled: true + address: 127.0.0.1:8686 + playground: false + sources: + k8s_logs: + type: kubernetes_logs + rotate_wait_secs: 5 + glob_minimum_cooldown_ms: 500 + max_line_bytes: 3145728 + auto_partial_merge: true + transforms: + reduce_events: + type: reduce + inputs: + - k8s_logs + group_by: + - file + flush_period_ms: 2000 + end_every_period_ms: 2000 + merge_strategies: + message: concat_newline + remap_app_logs: + type: remap + inputs: + - reduce_events + source: |- + .tmp = del(.) + # Handling Tekton-specific labels + if exists(.tmp.kubernetes.pod_labels."tekton.dev/taskRunUID") { + .taskRunUID = del(.tmp.kubernetes.pod_labels."tekton.dev/taskRunUID") + } else { + .taskRunUID = "none" + } + if exists(.tmp.kubernetes.pod_labels."tekton.dev/pipelineRunUID") { + .pipelineRunUID = del(.tmp.kubernetes.pod_labels."tekton.dev/pipelineRunUID") + .result = .pipelineRunUID + } else { + .result = .taskRunUID + } + # --- Start: Cronjob Specific Handling --- + # Check for cronjob related labels. Common labels include `job-name` and `controller-uid` + # from the job created by the cronjob, and potentially `cronjob-name` if explicitly set. + if exists(.tmp.kubernetes.pod_labels."job-name") { + .job_name = del(.tmp.kubernetes.pod_labels."job-name") + .log_type = "cronjob" # Tag logs from cronjobs + if exists(.tmp.kubernetes.pod_labels."cronjob-name") { + .cronjob_name = del(.tmp.kubernetes.pod_labels."cronjob-name") + } else { + # Attempt to infer cronjob name from job-name (common pattern: -) + if .job_name =~ r"^(.*)-\d{8,10}$" { # Basic pattern for job names from cronjobs (e.g., mycronjob-12345678) + .cronjob_name = capture(.job_name, r"^(.*)-\d{8,10}$")[0] + } else { + .cronjob_name = "unknown_cronjob" + } + } + if exists(.tmp.kubernetes.pod_labels."controller-uid") { + .job_uid = del(.tmp.kubernetes.pod_labels."controller-uid") + } + } else { + .log_type = "application" # Default for other application logs + } + # --- End: Cronjob Specific Handling --- + # Handling general Kubernetes labels + if exists(.tmp.kubernetes.pod_labels) { + .pod_labels = .tmp.kubernetes.pod_labels + } else { + .pod_labels = "no_labels" + } + if exists(.tmp.kubernetes.pod_namespace) { + .namespace = del(.tmp.kubernetes.pod_namespace) + } else { + .namespace = "unlabeled" + } + # General Kubernetes container name + if exists(.tmp.kubernetes.container_name) { + .container = del(.tmp.kubernetes.container_name) + } else { + .container = "unknown_container" + } + # General message field handling + if exists(.tmp.message) { + .message = del(.tmp.message) + } else { + .message = "no_message" + } + # Clean up temporary fields + del(.tmp) + sinks: + loki: + type: loki + inputs: ["remap_app_logs"] + endpoint: "http://loki.kubearchive-logging.svc.cluster.local:3100" + labels: + namespace: .kubernetes.namespace_name + pod: .kubernetes.pod_name + container: .kubernetes.container_name + batch: + max_bytes: 10485760 + timeout_secs: 300 + compression: "none" + encoding: + codec: "text" + key_prefix: "/logs/{{ `{{ .namespace }}` }}/{{`{{ .result }}`}}/{{`{{ .taskRunUID }}`}}/{{`{{ .container }}`}}" + filename_time_format: "-%s" + filename_append_uuid: false + buffer: + type: "memory" + max_events: 10000 + when_full: "block" + +nodeSelector: + konflux-ci.dev/workload: konflux-tenants +tolerations: + - effect: NoSchedule + key: konflux-ci.dev/workload + operator: Equal + value: konflux-tenants +image: + repository: quay.io/kubearchive/vector + tag: 0.46.1-distroless-libc +securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - CHOWN + - DAC_OVERRIDE + - FOWNER + - FSETID + - KILL + - NET_BIND_SERVICE + - SETGID + - SETPCAP + - SETUID + readOnlyRootFilesystem: true + seLinuxOptions: + type: spc_t + seccompProfile: + type: RuntimeDefault diff --git a/components/vector-kubearchive-log-collector/base/vector-pre.yaml b/components/vector-kubearchive-log-collector/base/vector-pre.yaml new file mode 100644 index 00000000000..050efb7c0c6 --- /dev/null +++ b/components/vector-kubearchive-log-collector/base/vector-pre.yaml @@ -0,0 +1,54 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: kubearchive-logging +--- +allowHostDirVolumePlugin: true +allowHostIPC: false +allowHostNetwork: false +allowHostPID: false +allowHostPorts: false +allowPrivilegeEscalation: false +allowPrivilegedContainer: false +allowedCapabilities: null +apiVersion: security.openshift.io/v1 +defaultAddCapabilities: null +defaultAllowPrivilegeEscalation: false +forbiddenSysctls: +- '*' +fsGroup: + type: RunAsAny +groups: [] +kind: SecurityContextConstraints +metadata: + name: logging-scc + namespace: kubearchive-logging +priority: null +readOnlyRootFilesystem: true +requiredDropCapabilities: +- CHOWN +- DAC_OVERRIDE +- FSETID +- FOWNER +- SETGID +- SETUID +- SETPCAP +- NET_BIND_SERVICE +- KILL +runAsUser: + type: RunAsAny +seLinuxContext: + type: RunAsAny +seccompProfiles: +- runtime/default +supplementalGroups: + type: RunAsAny +users: +- system:serviceaccount:kubearchive-logging:vector-kubearchive-logs-collector +volumes: +- configMap +- emptyDir +- hostPath +- projected +- secret diff --git a/components/vector-kubearchive-log-collector/development/kustomization.yaml b/components/vector-kubearchive-log-collector/development/kustomization.yaml new file mode 100644 index 00000000000..49c39a1824d --- /dev/null +++ b/components/vector-kubearchive-log-collector/development/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../base + - loki-helm-generator.yaml diff --git a/components/vector-kubearchive-log-collector/development/loki-helm-generator.yaml b/components/vector-kubearchive-log-collector/development/loki-helm-generator.yaml new file mode 100644 index 00000000000..f155e3dbbf0 --- /dev/null +++ b/components/vector-kubearchive-log-collector/development/loki-helm-generator.yaml @@ -0,0 +1,10 @@ +apiVersion: builtin +kind: HelmChartInflationGenerator +metadata: + name: loki +name: loki +repo: https://grafana.github.io/helm-charts +version: 0.0.1 +releaseName: vector-kubearchive-logs-collector +namespace: kubearchive-logging +valuesFile: loki-helm-values.yaml \ No newline at end of file diff --git a/components/vector-kubearchive-log-collector/development/loki-helm-values.yaml b/components/vector-kubearchive-log-collector/development/loki-helm-values.yaml new file mode 100644 index 00000000000..6d947309372 --- /dev/null +++ b/components/vector-kubearchive-log-collector/development/loki-helm-values.yaml @@ -0,0 +1,77 @@ +--- +loki: + auth_enabled: false + storage: + type: s3 + bucketNames: + chunks: loki + ruler: loki-ruler + admin: loki-admin + s3: + endpoint: minio.kubearchive-logging.svc.cluster.local:9000 + region: us-east-1 + accessKeyId: admin + s3ForcePathStyle: true + insecure: true + + schemaConfig: + configs: + - from: 2024-01-01 + store: tsdb + object_store: s3 + schema: v13 + index: + prefix: index_ + period: 24h + + commonConfig: + replication_factor: 1 + + compactor: + retention_enabled: true + delete_request_store: s3 + ruler: + storage: + type: local + local: + directory: /tmp/rules + +deploymentMode: SingleBinary + +singleBinary: + replicas: 1 + persistence: + enabled: false + extraVolumeMounts: + - name: loki-storage + mountPath: /var/loki + extraVolumes: + - name: loki-storage + emptyDir: {} + +# Zero out replica counts of other deployment modes +backend: + replicas: 0 +read: + replicas: 0 +write: + replicas: 0 + +ingester: + replicas: 0 +querier: + replicas: 0 +queryFrontend: + replicas: 0 +queryScheduler: + replicas: 0 +distributor: + replicas: 0 +compactor: + replicas: 0 +indexGateway: + replicas: 0 +bloomCompactor: + replicas: 0 +bloomGateway: + replicas: 0 diff --git a/components/vector-kubearchive-log-collector/development/values.minio.yaml b/components/vector-kubearchive-log-collector/development/values.minio.yaml new file mode 100644 index 00000000000..b72cd53fbc3 --- /dev/null +++ b/components/vector-kubearchive-log-collector/development/values.minio.yaml @@ -0,0 +1,20 @@ +# Copyright KubeArchive Authors +# SPDX-License-Identifier: Apache-2.0 +mode: standalone + +persistence: + enabled: false + +rootUser: minioadmin +rootPassword: minioadmin + +provisioning: + enabled: true + buckets: + - name: loki + - name: loki-ruler + - name: loki-admin + +resources: + requests: + memory: 512Mi diff --git a/components/vector-kubearchive-log-collector/production/kustomization.yaml b/components/vector-kubearchive-log-collector/production/kustomization.yaml new file mode 100644 index 00000000000..49c39a1824d --- /dev/null +++ b/components/vector-kubearchive-log-collector/production/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../base + - loki-helm-generator.yaml diff --git a/components/vector-kubearchive-log-collector/production/loki-helm-generator.yaml b/components/vector-kubearchive-log-collector/production/loki-helm-generator.yaml new file mode 100644 index 00000000000..f155e3dbbf0 --- /dev/null +++ b/components/vector-kubearchive-log-collector/production/loki-helm-generator.yaml @@ -0,0 +1,10 @@ +apiVersion: builtin +kind: HelmChartInflationGenerator +metadata: + name: loki +name: loki +repo: https://grafana.github.io/helm-charts +version: 0.0.1 +releaseName: vector-kubearchive-logs-collector +namespace: kubearchive-logging +valuesFile: loki-helm-values.yaml \ No newline at end of file diff --git a/components/vector-kubearchive-log-collector/production/loki-helm-values.yaml b/components/vector-kubearchive-log-collector/production/loki-helm-values.yaml new file mode 100644 index 00000000000..6d947309372 --- /dev/null +++ b/components/vector-kubearchive-log-collector/production/loki-helm-values.yaml @@ -0,0 +1,77 @@ +--- +loki: + auth_enabled: false + storage: + type: s3 + bucketNames: + chunks: loki + ruler: loki-ruler + admin: loki-admin + s3: + endpoint: minio.kubearchive-logging.svc.cluster.local:9000 + region: us-east-1 + accessKeyId: admin + s3ForcePathStyle: true + insecure: true + + schemaConfig: + configs: + - from: 2024-01-01 + store: tsdb + object_store: s3 + schema: v13 + index: + prefix: index_ + period: 24h + + commonConfig: + replication_factor: 1 + + compactor: + retention_enabled: true + delete_request_store: s3 + ruler: + storage: + type: local + local: + directory: /tmp/rules + +deploymentMode: SingleBinary + +singleBinary: + replicas: 1 + persistence: + enabled: false + extraVolumeMounts: + - name: loki-storage + mountPath: /var/loki + extraVolumes: + - name: loki-storage + emptyDir: {} + +# Zero out replica counts of other deployment modes +backend: + replicas: 0 +read: + replicas: 0 +write: + replicas: 0 + +ingester: + replicas: 0 +querier: + replicas: 0 +queryFrontend: + replicas: 0 +queryScheduler: + replicas: 0 +distributor: + replicas: 0 +compactor: + replicas: 0 +indexGateway: + replicas: 0 +bloomCompactor: + replicas: 0 +bloomGateway: + replicas: 0 diff --git a/components/vector-kubearchive-log-collector/production/values.minio.yaml b/components/vector-kubearchive-log-collector/production/values.minio.yaml new file mode 100644 index 00000000000..b72cd53fbc3 --- /dev/null +++ b/components/vector-kubearchive-log-collector/production/values.minio.yaml @@ -0,0 +1,20 @@ +# Copyright KubeArchive Authors +# SPDX-License-Identifier: Apache-2.0 +mode: standalone + +persistence: + enabled: false + +rootUser: minioadmin +rootPassword: minioadmin + +provisioning: + enabled: true + buckets: + - name: loki + - name: loki-ruler + - name: loki-admin + +resources: + requests: + memory: 512Mi diff --git a/components/vector-kubearchive-log-collector/staging/kustomization.yaml b/components/vector-kubearchive-log-collector/staging/kustomization.yaml new file mode 100644 index 00000000000..a4717cf3ed4 --- /dev/null +++ b/components/vector-kubearchive-log-collector/staging/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../base + - loki-helm-generator.yaml + diff --git a/components/vector-kubearchive-log-collector/staging/loki-helm-generator.yaml b/components/vector-kubearchive-log-collector/staging/loki-helm-generator.yaml new file mode 100644 index 00000000000..f155e3dbbf0 --- /dev/null +++ b/components/vector-kubearchive-log-collector/staging/loki-helm-generator.yaml @@ -0,0 +1,10 @@ +apiVersion: builtin +kind: HelmChartInflationGenerator +metadata: + name: loki +name: loki +repo: https://grafana.github.io/helm-charts +version: 0.0.1 +releaseName: vector-kubearchive-logs-collector +namespace: kubearchive-logging +valuesFile: loki-helm-values.yaml \ No newline at end of file diff --git a/components/vector-kubearchive-log-collector/staging/loki-helm-values.yaml b/components/vector-kubearchive-log-collector/staging/loki-helm-values.yaml new file mode 100644 index 00000000000..6d947309372 --- /dev/null +++ b/components/vector-kubearchive-log-collector/staging/loki-helm-values.yaml @@ -0,0 +1,77 @@ +--- +loki: + auth_enabled: false + storage: + type: s3 + bucketNames: + chunks: loki + ruler: loki-ruler + admin: loki-admin + s3: + endpoint: minio.kubearchive-logging.svc.cluster.local:9000 + region: us-east-1 + accessKeyId: admin + s3ForcePathStyle: true + insecure: true + + schemaConfig: + configs: + - from: 2024-01-01 + store: tsdb + object_store: s3 + schema: v13 + index: + prefix: index_ + period: 24h + + commonConfig: + replication_factor: 1 + + compactor: + retention_enabled: true + delete_request_store: s3 + ruler: + storage: + type: local + local: + directory: /tmp/rules + +deploymentMode: SingleBinary + +singleBinary: + replicas: 1 + persistence: + enabled: false + extraVolumeMounts: + - name: loki-storage + mountPath: /var/loki + extraVolumes: + - name: loki-storage + emptyDir: {} + +# Zero out replica counts of other deployment modes +backend: + replicas: 0 +read: + replicas: 0 +write: + replicas: 0 + +ingester: + replicas: 0 +querier: + replicas: 0 +queryFrontend: + replicas: 0 +queryScheduler: + replicas: 0 +distributor: + replicas: 0 +compactor: + replicas: 0 +indexGateway: + replicas: 0 +bloomCompactor: + replicas: 0 +bloomGateway: + replicas: 0 diff --git a/components/vector-kubearchive-log-collector/staging/values.minio.yaml b/components/vector-kubearchive-log-collector/staging/values.minio.yaml new file mode 100644 index 00000000000..b72cd53fbc3 --- /dev/null +++ b/components/vector-kubearchive-log-collector/staging/values.minio.yaml @@ -0,0 +1,20 @@ +# Copyright KubeArchive Authors +# SPDX-License-Identifier: Apache-2.0 +mode: standalone + +persistence: + enabled: false + +rootUser: minioadmin +rootPassword: minioadmin + +provisioning: + enabled: true + buckets: + - name: loki + - name: loki-ruler + - name: loki-admin + +resources: + requests: + memory: 512Mi From 62872f77791cae6e10834675346d5a3beb734b51 Mon Sep 17 00:00:00 2001 From: "rh-tap-build-team[bot]" <127938674+rh-tap-build-team[bot]@users.noreply.github.com> Date: Wed, 21 May 2025 12:45:54 +0000 Subject: [PATCH 2/8] update components/mintmaker/production/base/kustomization.yaml (#6419) Co-authored-by: rh-tap-build-team[bot] <127938674+rh-tap-build-team[bot]@users.noreply.github.com> From 0631715ec0d53ff3eaf503f9350607098a60d828 Mon Sep 17 00:00:00 2001 From: obetsun Date: Mon, 2 Jun 2025 16:27:16 +0300 Subject: [PATCH 3/8] feat(KONFLUX-8225): fix log forwarder namespace Signed-off-by: obetsun --- .../vector-kubearchive-logs-collector.yaml | 2 +- .../base/kustomization.yaml | 2 +- .../base/vector-helm-generator.yaml | 2 +- .../base/vector-helm-values.yaml | 2 +- .../vector-kubearchive-log-collector/base/vector-pre.yaml | 6 +++--- .../development/loki-helm-values.yaml | 2 +- .../production/loki-helm-generator.yaml | 2 +- .../staging/loki-helm-generator.yaml | 2 +- 8 files changed, 10 insertions(+), 10 deletions(-) diff --git a/argo-cd-apps/base/member/infra-deployments/vector-kubearchive-log-collector/vector-kubearchive-logs-collector.yaml b/argo-cd-apps/base/member/infra-deployments/vector-kubearchive-log-collector/vector-kubearchive-logs-collector.yaml index f92f37e67ad..26883222d8b 100644 --- a/argo-cd-apps/base/member/infra-deployments/vector-kubearchive-log-collector/vector-kubearchive-logs-collector.yaml +++ b/argo-cd-apps/base/member/infra-deployments/vector-kubearchive-log-collector/vector-kubearchive-logs-collector.yaml @@ -25,7 +25,7 @@ spec: repoURL: https://github.com/redhat-appstudio/infra-deployments.git targetRevision: main destination: - namespace: kubearchive-logging + namespace: product-kubearchive-logging server: '{{server}}' syncPolicy: automated: diff --git a/components/vector-kubearchive-log-collector/base/kustomization.yaml b/components/vector-kubearchive-log-collector/base/kustomization.yaml index 3c88f50d1e4..2ff0a985f61 100644 --- a/components/vector-kubearchive-log-collector/base/kustomization.yaml +++ b/components/vector-kubearchive-log-collector/base/kustomization.yaml @@ -1,7 +1,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization -namespace: kubearchive-logging +namespace: product-kubearchive-logging commonAnnotations: argocd.argoproj.io/sync-wave: "-1" diff --git a/components/vector-kubearchive-log-collector/base/vector-helm-generator.yaml b/components/vector-kubearchive-log-collector/base/vector-helm-generator.yaml index 638556e44a2..388ec3c0623 100644 --- a/components/vector-kubearchive-log-collector/base/vector-helm-generator.yaml +++ b/components/vector-kubearchive-log-collector/base/vector-helm-generator.yaml @@ -6,5 +6,5 @@ name: vector repo: https://helm.vector.dev version: 0.46.1 releaseName: vector-kubearchive-logs-collector -namespace: kubearchive-logging +namespace: product-kubearchive-logging valuesFile: vector-helm-values.yaml diff --git a/components/vector-kubearchive-log-collector/base/vector-helm-values.yaml b/components/vector-kubearchive-log-collector/base/vector-helm-values.yaml index 5d11aaa4e43..1203e0c9519 100644 --- a/components/vector-kubearchive-log-collector/base/vector-helm-values.yaml +++ b/components/vector-kubearchive-log-collector/base/vector-helm-values.yaml @@ -101,7 +101,7 @@ customConfig: loki: type: loki inputs: ["remap_app_logs"] - endpoint: "http://loki.kubearchive-logging.svc.cluster.local:3100" + endpoint: "http://loki.product-kubearchive-logging.svc.cluster.local:3100" labels: namespace: .kubernetes.namespace_name pod: .kubernetes.pod_name diff --git a/components/vector-kubearchive-log-collector/base/vector-pre.yaml b/components/vector-kubearchive-log-collector/base/vector-pre.yaml index 050efb7c0c6..5efa2bfc66f 100644 --- a/components/vector-kubearchive-log-collector/base/vector-pre.yaml +++ b/components/vector-kubearchive-log-collector/base/vector-pre.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: Namespace metadata: - name: kubearchive-logging + name: product-kubearchive-logging --- allowHostDirVolumePlugin: true allowHostIPC: false @@ -23,7 +23,7 @@ groups: [] kind: SecurityContextConstraints metadata: name: logging-scc - namespace: kubearchive-logging + namespace: product-kubearchive-logging priority: null readOnlyRootFilesystem: true requiredDropCapabilities: @@ -45,7 +45,7 @@ seccompProfiles: supplementalGroups: type: RunAsAny users: -- system:serviceaccount:kubearchive-logging:vector-kubearchive-logs-collector +- system:serviceaccount:product-kubearchive-logging:vector-kubearchive-logs-collector volumes: - configMap - emptyDir diff --git a/components/vector-kubearchive-log-collector/development/loki-helm-values.yaml b/components/vector-kubearchive-log-collector/development/loki-helm-values.yaml index 6d947309372..5a3cf66990d 100644 --- a/components/vector-kubearchive-log-collector/development/loki-helm-values.yaml +++ b/components/vector-kubearchive-log-collector/development/loki-helm-values.yaml @@ -8,7 +8,7 @@ loki: ruler: loki-ruler admin: loki-admin s3: - endpoint: minio.kubearchive-logging.svc.cluster.local:9000 + endpoint: minio.product-kubearchive-logging.svc.cluster.local:9000 region: us-east-1 accessKeyId: admin s3ForcePathStyle: true diff --git a/components/vector-kubearchive-log-collector/production/loki-helm-generator.yaml b/components/vector-kubearchive-log-collector/production/loki-helm-generator.yaml index f155e3dbbf0..11647a68311 100644 --- a/components/vector-kubearchive-log-collector/production/loki-helm-generator.yaml +++ b/components/vector-kubearchive-log-collector/production/loki-helm-generator.yaml @@ -6,5 +6,5 @@ name: loki repo: https://grafana.github.io/helm-charts version: 0.0.1 releaseName: vector-kubearchive-logs-collector -namespace: kubearchive-logging +namespace: product-kubearchive-logging valuesFile: loki-helm-values.yaml \ No newline at end of file diff --git a/components/vector-kubearchive-log-collector/staging/loki-helm-generator.yaml b/components/vector-kubearchive-log-collector/staging/loki-helm-generator.yaml index f155e3dbbf0..11647a68311 100644 --- a/components/vector-kubearchive-log-collector/staging/loki-helm-generator.yaml +++ b/components/vector-kubearchive-log-collector/staging/loki-helm-generator.yaml @@ -6,5 +6,5 @@ name: loki repo: https://grafana.github.io/helm-charts version: 0.0.1 releaseName: vector-kubearchive-logs-collector -namespace: kubearchive-logging +namespace: product-kubearchive-logging valuesFile: loki-helm-values.yaml \ No newline at end of file From ff476b95d20cb1d1b87fda06e572687e1b5a04f9 Mon Sep 17 00:00:00 2001 From: obetsun Date: Mon, 26 May 2025 15:19:35 +0300 Subject: [PATCH 4/8] feat(KONFLUX-8225): add log forwarder for KubeArchive Signed-off-by: obetsun --- .../vector-kubearchive-logs-collector.yaml | 2 +- .../vector-kubearchive-log-collector/base/kustomization.yaml | 2 -- .../development/loki-helm-generator.yaml | 2 +- .../staging/loki-helm-values.yaml | 2 +- 4 files changed, 3 insertions(+), 5 deletions(-) diff --git a/argo-cd-apps/base/member/infra-deployments/vector-kubearchive-log-collector/vector-kubearchive-logs-collector.yaml b/argo-cd-apps/base/member/infra-deployments/vector-kubearchive-log-collector/vector-kubearchive-logs-collector.yaml index 26883222d8b..0e0393f3e76 100644 --- a/argo-cd-apps/base/member/infra-deployments/vector-kubearchive-log-collector/vector-kubearchive-logs-collector.yaml +++ b/argo-cd-apps/base/member/infra-deployments/vector-kubearchive-log-collector/vector-kubearchive-logs-collector.yaml @@ -22,7 +22,7 @@ spec: project: default source: path: '{{values.sourceRoot}}/{{values.environment}}/{{values.clusterDir}}' - repoURL: https://github.com/redhat-appstudio/infra-deployments.git + repoURL: https://github.com/olegbet/infra-deployments.git targetRevision: main destination: namespace: product-kubearchive-logging diff --git a/components/vector-kubearchive-log-collector/base/kustomization.yaml b/components/vector-kubearchive-log-collector/base/kustomization.yaml index 2ff0a985f61..886544c0559 100644 --- a/components/vector-kubearchive-log-collector/base/kustomization.yaml +++ b/components/vector-kubearchive-log-collector/base/kustomization.yaml @@ -1,8 +1,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization - namespace: product-kubearchive-logging - commonAnnotations: argocd.argoproj.io/sync-wave: "-1" ignore-check.kube-linter.io/run-as-non-root: > diff --git a/components/vector-kubearchive-log-collector/development/loki-helm-generator.yaml b/components/vector-kubearchive-log-collector/development/loki-helm-generator.yaml index f155e3dbbf0..11647a68311 100644 --- a/components/vector-kubearchive-log-collector/development/loki-helm-generator.yaml +++ b/components/vector-kubearchive-log-collector/development/loki-helm-generator.yaml @@ -6,5 +6,5 @@ name: loki repo: https://grafana.github.io/helm-charts version: 0.0.1 releaseName: vector-kubearchive-logs-collector -namespace: kubearchive-logging +namespace: product-kubearchive-logging valuesFile: loki-helm-values.yaml \ No newline at end of file diff --git a/components/vector-kubearchive-log-collector/staging/loki-helm-values.yaml b/components/vector-kubearchive-log-collector/staging/loki-helm-values.yaml index 6d947309372..5a3cf66990d 100644 --- a/components/vector-kubearchive-log-collector/staging/loki-helm-values.yaml +++ b/components/vector-kubearchive-log-collector/staging/loki-helm-values.yaml @@ -8,7 +8,7 @@ loki: ruler: loki-ruler admin: loki-admin s3: - endpoint: minio.kubearchive-logging.svc.cluster.local:9000 + endpoint: minio.product-kubearchive-logging.svc.cluster.local:9000 region: us-east-1 accessKeyId: admin s3ForcePathStyle: true From da0f5391d6378ac9a92b205958381191fb8f9f90 Mon Sep 17 00:00:00 2001 From: obetsun Date: Mon, 26 May 2025 15:19:35 +0300 Subject: [PATCH 5/8] feat(KONFLUX-8225): add log forwarder for KubeArchive Signed-off-by: obetsun --- .../vector-kubearchive-logs-collector.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/argo-cd-apps/base/member/infra-deployments/vector-kubearchive-log-collector/vector-kubearchive-logs-collector.yaml b/argo-cd-apps/base/member/infra-deployments/vector-kubearchive-log-collector/vector-kubearchive-logs-collector.yaml index 0e0393f3e76..89555f1e357 100644 --- a/argo-cd-apps/base/member/infra-deployments/vector-kubearchive-log-collector/vector-kubearchive-logs-collector.yaml +++ b/argo-cd-apps/base/member/infra-deployments/vector-kubearchive-log-collector/vector-kubearchive-logs-collector.yaml @@ -23,7 +23,7 @@ spec: source: path: '{{values.sourceRoot}}/{{values.environment}}/{{values.clusterDir}}' repoURL: https://github.com/olegbet/infra-deployments.git - targetRevision: main + targetRevision: KONFLUX-8225_add_log_forwarder_for_loki destination: namespace: product-kubearchive-logging server: '{{server}}' From 6b1d1bb850004264ac18db9c207683a2838ca3aa Mon Sep 17 00:00:00 2001 From: obetsun Date: Tue, 3 Jun 2025 18:37:07 +0300 Subject: [PATCH 6/8] add test git branch reference Signed-off-by: obetsun --- .../vector-kubearchive-logs-collector.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/argo-cd-apps/base/member/infra-deployments/vector-kubearchive-log-collector/vector-kubearchive-logs-collector.yaml b/argo-cd-apps/base/member/infra-deployments/vector-kubearchive-log-collector/vector-kubearchive-logs-collector.yaml index 89555f1e357..93d02658f5d 100644 --- a/argo-cd-apps/base/member/infra-deployments/vector-kubearchive-log-collector/vector-kubearchive-logs-collector.yaml +++ b/argo-cd-apps/base/member/infra-deployments/vector-kubearchive-log-collector/vector-kubearchive-logs-collector.yaml @@ -23,7 +23,7 @@ spec: source: path: '{{values.sourceRoot}}/{{values.environment}}/{{values.clusterDir}}' repoURL: https://github.com/olegbet/infra-deployments.git - targetRevision: KONFLUX-8225_add_log_forwarder_for_loki + targetRevision: origin/KONFLUX-8225_add_log_forwarder_for_loki destination: namespace: product-kubearchive-logging server: '{{server}}' From 30cf047176ceea3b8dfada3e6648df9d79e00cdf Mon Sep 17 00:00:00 2001 From: obetsun Date: Tue, 3 Jun 2025 19:04:42 +0300 Subject: [PATCH 7/8] Branch name for test corrected Signed-off-by: obetsun --- .../vector-kubearchive-logs-collector.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/argo-cd-apps/base/member/infra-deployments/vector-kubearchive-log-collector/vector-kubearchive-logs-collector.yaml b/argo-cd-apps/base/member/infra-deployments/vector-kubearchive-log-collector/vector-kubearchive-logs-collector.yaml index 93d02658f5d..89555f1e357 100644 --- a/argo-cd-apps/base/member/infra-deployments/vector-kubearchive-log-collector/vector-kubearchive-logs-collector.yaml +++ b/argo-cd-apps/base/member/infra-deployments/vector-kubearchive-log-collector/vector-kubearchive-logs-collector.yaml @@ -23,7 +23,7 @@ spec: source: path: '{{values.sourceRoot}}/{{values.environment}}/{{values.clusterDir}}' repoURL: https://github.com/olegbet/infra-deployments.git - targetRevision: origin/KONFLUX-8225_add_log_forwarder_for_loki + targetRevision: KONFLUX-8225_add_log_forwarder_for_loki destination: namespace: product-kubearchive-logging server: '{{server}}' From 0213502644d46f39387c8aa4f97609c12f23b199 Mon Sep 17 00:00:00 2001 From: obetsun Date: Wed, 4 Jun 2025 13:08:19 +0300 Subject: [PATCH 8/8] add rbac for kubearchive logging Signed-off-by: obetsun --- .../kustomization.yaml | 5 +++ .../pipeline-service-sre.yaml | 45 +++++++++++++++++++ hack/secret-creator/create-plnsvc-secrets.sh | 1 + 3 files changed, 51 insertions(+) create mode 100644 components/pipeline-service/base/rbac/product-kubearchive-logging/kustomization.yaml create mode 100644 components/pipeline-service/base/rbac/product-kubearchive-logging/pipeline-service-sre.yaml diff --git a/components/pipeline-service/base/rbac/product-kubearchive-logging/kustomization.yaml b/components/pipeline-service/base/rbac/product-kubearchive-logging/kustomization.yaml new file mode 100644 index 00000000000..64e6254b834 --- /dev/null +++ b/components/pipeline-service/base/rbac/product-kubearchive-logging/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: product-kubearchive-logging +resources: + - pipeline-service-sre.yaml diff --git a/components/pipeline-service/base/rbac/product-kubearchive-logging/pipeline-service-sre.yaml b/components/pipeline-service/base/rbac/product-kubearchive-logging/pipeline-service-sre.yaml new file mode 100644 index 00000000000..193b2af5c16 --- /dev/null +++ b/components/pipeline-service/base/rbac/product-kubearchive-logging/pipeline-service-sre.yaml @@ -0,0 +1,45 @@ +--- +# Grant access to the tekton-logging namespace +# This binding is needed to allow the pipelines team to manage the pods +# which happen to stuck during an upgrade. +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: pipeline-service-sre-manage-vector-pods + namespace: product-kubearchive-logging +rules: + - apiGroups: + - "" + verbs: + - get + - list + - watch + - delete + resources: + - pods + - apiGroups: + - "apps" + verbs: + - get + - list + - watch + - delete + resources: + - daemonsets +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: pipeline-service-sre-manage-vector-pods + namespace: product-kubearchive-logging +subjects: + - kind: Group + apiGroup: rbac.authorization.k8s.io + name: konflux-pipeline-service + - apiGroup: rbac.authorization.k8s.io + kind: Group + name: konflux-sre +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: pipeline-service-sre-manage-vector-pods diff --git a/hack/secret-creator/create-plnsvc-secrets.sh b/hack/secret-creator/create-plnsvc-secrets.sh index de241aed24a..a9a36aa87bb 100755 --- a/hack/secret-creator/create-plnsvc-secrets.sh +++ b/hack/secret-creator/create-plnsvc-secrets.sh @@ -4,6 +4,7 @@ main() { echo "Setting secrets for pipeline-service" create_namespace tekton-results create_namespace tekton-logging + create_namespace product-kubearchive-logging create_db_secret create_s3_secret tekton-results tekton-results-s3 create_s3_secret tekton-logging tekton-results-s3