diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 07223ca..b54ce55 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1 +1,3 @@ -* @GiovanniMancini @TommasoLencioni @and-mora @petretiandrea @lucaconsalvi \ No newline at end of file +# see https://help.github.com/en/articles/about-code-owners#example-of-a-codeowners-file +# Add the repository's code owners here +* @pagopa/idpay-app-maintainer-team @pagopa/idpay-approver-team @dariopelliccioli \ No newline at end of file diff --git a/.github/workflows/anchore.yml b/.github/workflows/anchore.yml index 5d3942d..6c57785 100644 --- a/.github/workflows/anchore.yml +++ b/.github/workflows/anchore.yml @@ -13,10 +13,11 @@ name: Anchore Container Scan on: push: - branches: [ "develop", "uat", "main" ] + branches: [ "release-dev", "release-uat", "main" ] pull_request: # The branches below must be a subset of the branches above - branches: [ "develop", "uat", "main" ] + branches: [ "release-dev", "release-uat", "main" ] + workflow_dispatch: schedule: - cron: '00 07 * * *' @@ -31,30 +32,65 @@ jobs: permissions: contents: read # for actions/checkout to fetch code security-events: write # for github/codeql-action/upload-sarif to upload SARIF results - actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status + actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status runs-on: ubuntu-latest + environment: dev steps: - name: Checkout the code - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 #v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #4.2.2 - name: Build the Docker image - run: cd kafka-connect-image && docker build . --file ${{ env.DOCKERFILE }} --tag localbuild/testimage:latest - - name: Run the Anchore scan action itself with GitHub Advanced Security code scanning integration enabled + run: docker build . --file ${{ env.DOCKERFILE }} --tag localbuild/testimage:latest + - name: Run the Anchore Grype scan action uses: anchore/scan-action@24fd7c9060f3c96848dd1929fac8d796fb5ae4b4 #v3.3.6 id: scan with: image: "localbuild/testimage:latest" fail-build: true severity-cutoff: "high" + output-format: "sarif" + output-file: "results.sarif" + - name: Upload Anchore Scan Report + uses: github/codeql-action/upload-sarif@e4262713b504983e61c7728f5452be240d9385a7 #v2.14.3 + if: always() + with: + sarif_file: results.sarif - name: Inspect action SARIF report if: always() run: cat ${{ steps.scan.outputs.sarif }} - name: CVE Description escaped extraction and print - if: ${{ always() }} + if: always() run: | - SCAN_RESULTS=$(jq -r 'try .runs[0].tool.driver.rules | map(.shortDescription.text) | join("\\n")' ${{ steps.scan.outputs.sarif }}) + SCAN_RESULTS=$(jq -r 'try .runs[0].tool.driver.rules | map(.shortDescription.text) | join("\\n")' results.sarif) echo "SCAN_RESULTS=$SCAN_RESULTS" >> $GITHUB_ENV echo "CVE_CRITICAL=$(echo $SCAN_RESULTS | grep -o critical | wc -l)" >> $GITHUB_ENV echo "CVE_HIGH=$(echo $SCAN_RESULTS | grep -o high | wc -l)" >> $GITHUB_ENV echo "CVE_MEDIUM=$(echo $SCAN_RESULTS | grep -o medium | wc -l)" >> $GITHUB_ENV echo -e $SCAN_RESULTS + - name: Send notification to Slack + id: slack + if: always() && github.event_name == 'schedule' && steps.scan.outcome == 'failure' + uses: slackapi/slack-github-action@e28cf165c92ffef168d23c5c9000cffc8a25e117 #v1.24.0 + with: + payload: | + { + "blocks": [ + { + "type": "header", + "text": { + "type": "plain_text", + "text": "${{ github.event.repository.name }} - Notification Scan Result Failed on ${{ github.ref_type }} ${{ github.ref_name }}!" + } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "GitHub Action scan results:\n`CRITICAL`: ${{ env.CVE_CRITICAL }}\n`HIGH`: ${{ env.CVE_HIGH }}\n`MEDIUM`: ${{ env.CVE_MEDIUM }}\n\n" + } + } + ] + } + env: + SLACK_WEBHOOK_URL: ${{ secrets.CVE_SCAN_SLACK_WEBHOOK }} + SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK \ No newline at end of file diff --git a/.github/workflows/flow-docker-snapshot.yml b/.github/workflows/flow-docker-snapshot.yml new file mode 100644 index 0000000..ab4e240 --- /dev/null +++ b/.github/workflows/flow-docker-snapshot.yml @@ -0,0 +1,88 @@ +name: 📦 Flow Snapshot Docker + +on: + push: + branches-ignore: + - 'main' + paths-ignore: + - 'CODEOWNERS' + - '**.md' + - '.**' + workflow_dispatch: + inputs: + docker_build_enabled: + description: 'Enable Docker build' + required: false + default: 'true' + azdo_trigger_enabled: + description: 'Enable Azure DevOps trigger' + required: false + default: 'true' + argocd_target_branch: + description: 'argocd target branch name' + required: false + default: 'main' + postman_branch: + description: postman branch name' + required: false + default: 'release-dev' + +env: + # branch choosed by workflow_dispatch or by push event + CURRENT_BRANCH: ${{ github.event.inputs.branch || github.ref_name }} + +jobs: + checkout: + name: 🔖 Checkout Repository + runs-on: ubuntu-22.04 + steps: + - name: Checkout code + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #4.2.2 + with: + ref: ${{ env.CURRENT_BRANCH }} + + docker-build: + name: 📦 Docker Build and Push + needs: checkout + runs-on: ubuntu-22.04 + if: ${{ github.event_name == 'push' || github.event.inputs.docker_build_enabled == 'true' }} + steps: + - name: Checkout code + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #4.2.2 + with: + ref: ${{ env.CURRENT_BRANCH }} + + - name: Run Snapshot Docker Build/Push + # https://github.com/pagopa/github-actions-template/releases/tag/v2.0.0 + uses: pagopa/github-actions-template/payments-flow-docker-snapshot@b825ee193430395706a4a7e580b8435bdded0227 + with: + current_branch: ${{ github.ref_name }} + + azure-devops-trigger: + name: 🅰️ Azure DevOps Pipeline Trigger + needs: docker-build + runs-on: ubuntu-22.04 + if: | + always() && + needs.docker-build.result != 'failure' && + github.event.inputs.azdo_trigger_enabled == 'true' + steps: + - name: Trigger Azure DevOps Pipeline + # https://github.com/pagopa/github-actions-template/releases/tag/v2.1.0 + uses: pagopa/github-actions-template/azure-devops-trigger-pipeline@cad30356d9046af6e7b0cee43db4cf919cc408f9 + with: + enable_azure_devops_step: 'true' + azure_devops_project_url: 'https://dev.azure.com/pagopaspa/idpay-itn-projects' + azure_devops_pipeline_name: 'idpay-deploy-aks.deploy' + azure_devops_pat: ${{ secrets.AZURE_DEVOPS_TOKEN }} + azure_template_parameters: | + { + "APPS_TOP": "[idpay-kafka-connect]", + "APPS_MID": "[]", + "APPS_EXT": "[]", + "ARGOCD_TARGET_BRANCH": "${{ github.event.inputs.argocd_target_branch }}", + "POSTMAN_BRANCH": "${{ github.event.inputs.postman_branch }}", + "TRIGGER_MESSAGE": "idpay-kafka-connect" + } + + diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1af0159..3083915 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,35 +1,37 @@ name: Release on: - # Trigger the workflow on push or pull request, - # but only for the main branch + # Trigger the workflow on push on the main branch push: branches: - main paths-ignore: - - "CODEOWNERS" - - "**.md" - - ".**" + - 'CODEOWNERS' + - '**.md' + - '.**' + +permissions: + packages: write + contents: write + jobs: - release: - name: Release - runs-on: ubuntu-latest + checkout: + name: 🔖 Checkout Repository + runs-on: ubuntu-22.04 steps: - - name: Checkout - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 #v3.6.0 - with: - persist-credentials: false - fetch-depth: 0 - - name: Setup Node 20.x - uses: actions/setup-node@5e21ff4d9bc1a8cf6de233a3057d20ec6b3fb69d #v3.8.1 + - name: Checkout code + uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 with: - node-version: 20 - - name: Release - uses: cycjimmy/semantic-release-action@61680d0e9b02ff86f5648ade99e01be17f0260a4 #v4.0.0 + ref: ${{ github.ref_name }} + + flow-release: + name: 🚀 Release + runs-on: ubuntu-22.04 + needs: checkout + steps: + - name: 🚀 release + docker + # https://github.com/pagopa/github-actions-template/releases/tag/v2.1.1 + uses: pagopa/github-actions-template/payments-flow-release@e67e0ec4bb01d6cb5065e311ba0e1c92c49966a7 with: - extra_plugins: | - semantic-release/release-notes-generator#c466b7165be2a0e5988676174e4bc244c991d669 #v11.0.7 - semantic-release/git#3e934d45f97fd07a63617c0fc098c9ed3e67d97a #v10.0.1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file + current_branch: ${{ github.ref_name }} diff --git a/Dockerfile b/Dockerfile index 1f2de0d..4c881f2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,15 +5,15 @@ WORKDIR /deps COPY build.gradle build.gradle RUN gradle getMongoKafkaConnectDeps -RUN curl -L "https://github.com/microsoft/ApplicationInsights-Java/releases/download/3.4.19/applicationinsights-agent-3.4.19.jar" \ +RUN curl -L "https://github.com/microsoft/ApplicationInsights-Java/releases/download/3.5.1/applicationinsights-agent-3.5.1.jar" \ --output "applicationinsights-agent.jar" -FROM debezium/connect-base:2.4.1.Final@sha256:52297b4d15b2eff0a19b44337e34a9cd3144696a70393ab9f4eee14d697abb0e +FROM debezium/connect-base:2.6.0.Final@sha256:ea2d17592e93e06e93459f940704d9b57f2b30d4f4bb5e83699bfae28aeea568 COPY --from=deps /deps/mongo-kafka-connect/ /kafka/connect/mongo-kafka-connect/ COPY --from=deps /deps/applicationinsights-agent.jar . - USER root RUN chmod 777 -R /kafka/connect/ && chown kafka:kafka -R applicationinsights-agent.jar +RUN chmod 777 -R /tmp USER kafka \ No newline at end of file diff --git a/build.gradle b/build.gradle index 395e67c..54736c0 100644 --- a/build.gradle +++ b/build.gradle @@ -7,10 +7,23 @@ repositories { configurations { mongoKafkaConnect } - dependencies { // https://mvnrepository.com/artifact/org.mongodb.kafka/mongo-kafka-connect - mongoKafkaConnect("org.mongodb.kafka:mongo-kafka-connect:1.5.1") + mongoKafkaConnect("org.mongodb.kafka:mongo-kafka-connect:1.5.1") { + exclude group: 'org.apache.kafka', module: 'connect-api' + exclude group: 'org.apache.avro', module: 'avro' + } + + // Dipendenza di Avro con esclusione di commons-compress + mongoKafkaConnect("org.apache.avro:avro:1.11.3") { + exclude group: 'org.apache.commons', module: 'commons-compress' + } + + // Nuova dipendenza di commons-compress alla versione 1.26.1 + mongoKafkaConnect("org.apache.commons:commons-compress:1.26.1") + + // Aggiornamento di connect-api alla versione 3.7.0 + mongoKafkaConnect("org.apache.kafka:connect-api:3.7.0") } task getMongoKafkaConnectDeps(type: Copy) { diff --git a/helm/Chart.lock b/helm/Chart.lock index 7e55c9c..25348fc 100644 --- a/helm/Chart.lock +++ b/helm/Chart.lock @@ -1,6 +1,6 @@ dependencies: - name: microservice-chart repository: https://pagopa.github.io/aks-microservice-chart-blueprint - version: 5.0.0 -digest: sha256:acce690c924529c84ccd541dd046bf2fa8e6d379d0bfe6f2e200d95466cdb253 -generated: "2024-01-26T10:52:19.8539417+01:00" \ No newline at end of file + version: 5.9.0 +digest: sha256:b614dd4be4c439e182fe5e7102e959fcda019413ddb2430be77c7a080bb13de1 +generated: "2024-04-05T15:23:36.0002433+02:00" diff --git a/helm/Chart.yaml b/helm/Chart.yaml index 467b691..9f3fda4 100644 --- a/helm/Chart.yaml +++ b/helm/Chart.yaml @@ -6,5 +6,5 @@ version: 1.0.0 appVersion: 1.0.0 dependencies: - name: microservice-chart - version: 5.0.0 + version: 5.9.0 repository: "https://pagopa.github.io/aks-microservice-chart-blueprint" diff --git a/helm/values-dev.yaml b/helm/values-dev.yaml index 20bb09a..3fa761d 100644 --- a/helm/values-dev.yaml +++ b/helm/values-dev.yaml @@ -10,15 +10,15 @@ microservice-chart: resources: requests: memory: "256Mi" - cpu: "40m" + cpu: "100m" limits: - memory: "512Mi" - cpu: "300m" + memory: "800Mi" + cpu: "500m" autoscaling: enable: true minReplica: 1 - maxReplica: 2 + maxReplica: 1 pollingInterval: 30 # seconds cooldownPeriod: 300 # seconds triggers: diff --git a/helm/values-prod.yaml b/helm/values-prod.yaml index e48c23b..720107c 100644 --- a/helm/values-prod.yaml +++ b/helm/values-prod.yaml @@ -12,13 +12,13 @@ microservice-chart: memory: "512Mi" cpu: "100m" limits: - memory: "756Mi" - cpu: "300m" + memory: "800Mi" + cpu: "500m" autoscaling: enable: true minReplica: 1 - maxReplica: 2 + maxReplica: 1 pollingInterval: 30 # seconds cooldownPeriod: 300 # seconds triggers: diff --git a/helm/values-uat.yaml b/helm/values-uat.yaml index d1888ab..a6681ca 100644 --- a/helm/values-uat.yaml +++ b/helm/values-uat.yaml @@ -12,13 +12,13 @@ microservice-chart: memory: "256Mi" cpu: "100m" limits: - memory: "512Mi" - cpu: "300m" + memory: "800Mi" + cpu: "500m" autoscaling: enable: true minReplica: 1 - maxReplica: 2 + maxReplica: 1 pollingInterval: 30 # seconds cooldownPeriod: 300 # seconds triggers: diff --git a/helm/values.yaml b/helm/values.yaml index 50a0d3c..0dc05c2 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -4,18 +4,23 @@ microservice-chart: fullnameOverride: "" livenessProbe: - httpGet: - path: /connectors - port: 8083 - initialDelaySeconds: 120 - failureThreshold: 15 + create: true + handlerType: 'exec' + exec: + command: + - sh + - -c + - for connector in $(curl -s -X GET http://localhost:8083/connectors | grep -Eo '[^]",\[]+'); do curl -s -X GET http://localhost:8083/connectors/$connector/status | grep FAILED | wc -l; done | grep -Ez '^(0[[:space:]])+$' + initialDelaySeconds: 60 periodSeconds: 30 + failureThreshold: 15 + timeoutSeconds: 5 readinessProbe: httpGet: path: /connectors port: 8083 - initialDelaySeconds: 125 + initialDelaySeconds: 65 failureThreshold: 15 periodSeconds: 30 @@ -55,9 +60,23 @@ microservice-chart: APPLICATIONINSIGHTS_INSTRUMENTATION_LOGGING_LEVEL: "OFF" APPLICATIONINSIGHTS_INSTRUMENTATION_MICROMETER_ENABLED: "false" APPLICATIONINSIGHTS_PREVIEW_PROFILER_ENABLED: "false" + CONNECT_PRODUCER_METADATA_MAX_AGE_MS: "180000" + CONNECT_PRODUCER_CONNECTIONS_MAX_IDLE_MS: "180000" + CONNECT_PRODUCER_MAX_REQUEST_SIZE: "1000000" + CONNECT_PRODUCER_RETRIES: "3" + CONNECT_PRODUCER_REQUEST_TIMEOUT_MS: "30000" + CONNECT_PRODUCER_METADATA_MAX_IDLE_MS: "180000" + CONNECT_PRODUCER_LINGER_MS: "5" + CONNECT_PRODUCER_DELIVERY_TIMEOUT_MS: "180000" + CONNECT_PRODUCER_COMPRESSION_TYPE: "none" CONNECT_METADATA_MAX_AGE_MS: "180000" CONNECT_CONNECTIONS_MAX_IDLE_MS: "180000" - CONNECT_MAX_REQUEST_SIZE: "1000000" + CONNECT_HEARTBEAT_INTERVAL_MS: "3000" + CONNECT_SESSION_TIMEOUT_MS: "30000" + CONNECT_MAX_POLL_INTERVAL_MS: "300000" + + + GROUP_ID: kafka-connect-consumer-group CONFIG_STORAGE_TOPIC: idpay-cdc-configs OFFSET_STORAGE_TOPIC: idpay-cdc-offsets @@ -99,3 +118,10 @@ microservice-chart: configMaps: - name: appinsights-config key: applicationinsights.json + + tmpVolumeMount: + create: true + mounts: + - name: tmp + mountPath: /tmp +