chore(kpi): Tuning of performance parameter regarding audit log ingestion (#163)

Author: marco-vit-pagopa

microservices/application-audit-analytics-writer/dev-analytics/values.yaml

@@ -9,8 +9,9 @@ service:
 configmap:
   SERVICE_NAME: pagopa-interop-application-audit-analytics-writer-dev
   KAFKA_GROUP_ID: "{{.Values.namespace}}-application-audit-analytics-writer"
-  ACCUMULATOR_MAX_MESSAGES: 1000000
-  ACCUMULATOR_FLUSH_TIMEOUT_MS: 60000
+  KAFKA_MAX_WAIT_BATCH_MILLIS: 300000
+  ACCUMULATOR_MAX_MESSAGES: 500000
+  ACCUMULATOR_FLUSH_TIMEOUT_MS: 30000
   S3_COPY_BUCKET: interop-analytics-etl-application-audit-{{.Values.local.env}}-es1
   S3_DELETE_AFTER_COPY: true
   REDSHIFT_COPY_IAM_ROLE_ARN: "arn:aws:iam::505630707203:role/interop-analytics-redshift-s3-copy-dev"
@@ -42,7 +43,7 @@ deployment:
   envFromConfigmaps:
     KAFKA_BROKERS: "common-kafka.KAFKA_BROKERS"
     KAFKA_AVERAGE_MESSAGE_SIZE_IN_BYTES: "common-kafka.KAFKA_AVERAGE_MESSAGE_SIZE_IN_BYTES"
-    KAFKA_MAX_WAIT_BATCH_MILLIS: "common-kafka.KAFKA_MAX_WAIT_BATCH_MILLIS"
+    # KAFKA_MAX_WAIT_BATCH_MILLIS: "common-kafka.KAFKA_MAX_WAIT_BATCH_MILLIS" (commented because a custom value is set for this microservice)
     KAFKA_MESSAGES_TO_READ_PER_BATCH: "common-kafka.KAFKA_MESSAGES_TO_READ_PER_BATCH"
     KAFKA_TOPIC_STARTING_OFFSET: "common-kafka.KAFKA_TOPIC_STARTING_OFFSET"
     KAFKA_ENABLE_AUTOCOMMIT: "common-kafka.KAFKA_ENABLE_AUTOCOMMIT"

microservices/application-audit-analytics-writer/prod-analytics/values.yaml

@@ -9,8 +9,9 @@ service:
 configmap:
   SERVICE_NAME: pagopa-interop-application-audit-analytics-writer-prod
   KAFKA_GROUP_ID: "{{.Values.namespace}}-application-audit-analytics-writer"
-  ACCUMULATOR_MAX_MESSAGES: 1000000
-  ACCUMULATOR_FLUSH_TIMEOUT_MS: 60000
+  KAFKA_MAX_WAIT_BATCH_MILLIS: 300000
+  ACCUMULATOR_MAX_MESSAGES: 500000
+  ACCUMULATOR_FLUSH_TIMEOUT_MS: 30000
   S3_COPY_BUCKET: interop-analytics-etl-application-audit-{{.Values.local.env}}-es1
   S3_DELETE_AFTER_COPY: true
   REDSHIFT_COPY_IAM_ROLE_ARN: "arn:aws:iam::697818730278:role/interop-analytics-redshift-s3-copy-{{.Values.local.env}}"
@@ -48,7 +49,7 @@ deployment:
   envFromConfigmaps:
     KAFKA_BROKERS: "common-kafka.KAFKA_BROKERS"
     KAFKA_AVERAGE_MESSAGE_SIZE_IN_BYTES: "common-kafka.KAFKA_AVERAGE_MESSAGE_SIZE_IN_BYTES"
-    KAFKA_MAX_WAIT_BATCH_MILLIS: "common-kafka.KAFKA_MAX_WAIT_BATCH_MILLIS"
+    # KAFKA_MAX_WAIT_BATCH_MILLIS: "common-kafka.KAFKA_MAX_WAIT_BATCH_MILLIS" (commented because a custom value is set for this microservice)
     KAFKA_MESSAGES_TO_READ_PER_BATCH: "common-kafka.KAFKA_MESSAGES_TO_READ_PER_BATCH"
     KAFKA_TOPIC_STARTING_OFFSET: "common-kafka.KAFKA_TOPIC_STARTING_OFFSET"
     KAFKA_ENABLE_AUTOCOMMIT: "common-kafka.KAFKA_ENABLE_AUTOCOMMIT"