feature(kpi): WIP Application audit ingestion performance improvement (PROD)

Author: marco-vit-pagopa

commons/prod-analytics/configmaps/kafka-configmap.yaml

@@ -7,7 +7,7 @@ data:
   KAFKA_BROKERS: "b-1.interopplatformevents.gfkljw.c4.kafka.eu-south-1.amazonaws.com:9098,b-2.interopplatformevents.gfkljw.c4.kafka.eu-south-1.amazonaws.com:9098,b-3.interopplatformevents.gfkljw.c4.kafka.eu-south-1.amazonaws.com:9098"
   KAFKA_AVERAGE_MESSAGE_SIZE_IN_BYTES: "1000"
   KAFKA_MAX_WAIT_BATCH_MILLIS: "60000"
-  KAFKA_MESSAGES_TO_READ_PER_BATCH: "1000"
+  KAFKA_MESSAGES_TO_READ_PER_BATCH: "10000"
   KAFKA_TOPIC_STARTING_OFFSET: "earliest"
   APPLICATION_AUDIT_TOPIC: "prod_application.audit"
   CATALOG_TOPIC: "event-store.prod_catalog.events"

microservices/application-audit-analytics-writer/prod-analytics/values.yaml

@@ -9,6 +9,13 @@ service:
 configmap:
   SERVICE_NAME: pagopa-interop-application-audit-analytics-writer-prod
   KAFKA_GROUP_ID: "{{.Values.namespace}}-application-audit-analytics-writer"
+  ACCUMULATOR_MAX_MESSAGES: 1000000
+  ACCUMULATOR_FLUSH_TIMEOUT_MS: 60000
+  S3_COPY_BUCKET: interop-analytics-etl-application-audit-{{.Values.local.env}}-es1
+  S3_DELETE_AFTER_COPY: true
+  REDSHIFT_COPY_IAM_ROLE_ARN: "arn:aws:iam::697818730278:role/interop-analytics-redshift-s3-copy-{{.Values.local.env}}"
+  GZ_COMPRESSION_LEVEL: 3
+  DB_INGEST_MODE: "COPY"
 
 deployment:
   replicas: 0