fix(coprocessor): scheduler performance and stability fixes (#1834) #970
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: coprocessor-docker-build | |
| on: | |
| release: | |
| types: | |
| - published | |
| workflow_call: | |
| secrets: | |
| AWS_ACCESS_KEY_S3_USER: | |
| required: true | |
| AWS_SECRET_KEY_S3_USER: | |
| required: true | |
| BLOCKCHAIN_ACTIONS_TOKEN: | |
| required: true | |
| GHCR_READ_TOKEN: | |
| required: true | |
| CGR_USERNAME: | |
| required: true | |
| CGR_PASSWORD: | |
| required: true | |
| outputs: | |
| db_migration_build_result: | |
| description: "Result of the build-db-migration job" | |
| value: ${{ jobs.build-db-migration.result }} | |
| gw_listener_build_result: | |
| description: "Result of the build-gw-listener job" | |
| value: ${{ jobs.build-gw-listener.result }} | |
| host_listener_build_result: | |
| description: "Result of the build-host-listener job" | |
| value: ${{ jobs.build-host-listener.result }} | |
| sns_worker_build_result: | |
| description: "Result of the build-sns-worker job" | |
| value: ${{ jobs.build-sns-worker.result }} | |
| tfhe_worker_build_result: | |
| description: "Result of the build-tfhe-worker job" | |
| value: ${{ jobs.build-tfhe-worker.result }} | |
| tx_sender_build_result: | |
| description: "Result of the build-tx-sender job" | |
| value: ${{ jobs.build-tx-sender.result }} | |
| zkproof_worker_build_result: | |
| description: "Result of the build-zkproof-worker job" | |
| value: ${{ jobs.build-zkproof-worker.result }} | |
| workflow_dispatch: | |
| inputs: | |
| build_db_migration: | |
| description: "Enable/disable build for Coprocessor's DB Migration" | |
| type: boolean | |
| default: true | |
| build_gw_listener: | |
| description: "Enable/disable build for Coprocessor's Gateway Listener" | |
| type: boolean | |
| default: true | |
| build_host_listener: | |
| description: "Enable/disable build for Coprocessor's Host Listener" | |
| type: boolean | |
| default: true | |
| build_sns_worker: | |
| description: "Enable/disable build for Coprocessor's SNS Worker" | |
| type: boolean | |
| default: true | |
| build_tfhe_worker: | |
| description: "Enable/disable build for Coprocessor's TFHE Worker" | |
| type: boolean | |
| default: true | |
| build_tx_sender: | |
| description: "Enable/disable build for Coprocessor's Transaction Sender" | |
| type: boolean | |
| default: true | |
| build_zkproof_worker: | |
| description: "Enable/disable build for Coprocessor's ZKProof Worker" | |
| type: boolean | |
| default: true | |
| push: | |
| branches: | |
| - main | |
| permissions: {} | |
| concurrency: | |
| group: coprocessor-docker-build-${{ github.ref_name }} | |
| cancel-in-progress: true | |
| jobs: | |
| ######################################################################## | |
| # DB MIGRATION # | |
| ######################################################################## | |
| check-changes-db-migration: | |
| uses: ./.github/workflows/check-changes-for-docker-build.yml | |
| secrets: &check_changes_secrets | |
| GHCR_READ_TOKEN: ${{ secrets.GHCR_READ_TOKEN }} | |
| permissions: &check_changes_permissions | |
| actions: 'read' # Required to read workflow run information | |
| contents: 'read' # Required to checkout repository code | |
| pull-requests: 'read' # Required to read pull request information | |
| with: | |
| caller-workflow-event-name: ${{ github.event_name }} | |
| caller-workflow-event-before: ${{ github.event.before }} | |
| docker-image: fhevm/coprocessor/db-migration | |
| filters: | | |
| db-migration: | |
| - .github/workflows/coprocessor-docker-build.yml | |
| - coprocessor/fhevm-engine/db-migration/** | |
| build-db-migration: | |
| needs: check-changes-db-migration | |
| if: | | |
| github.event_name == 'release' | |
| || (github.event_name != 'workflow_dispatch' && needs.check-changes-db-migration.outputs.changes == 'true') | |
| || (github.event_name == 'workflow_dispatch' && inputs.build_db_migration) | |
| uses: zama-ai/ci-templates/.github/workflows/common-docker.yml@3cf4c2b133947d29e7a313555638621f9ca0345c # v1.0.3 | |
| secrets: &docker_secrets | |
| AWS_ACCESS_KEY_S3_USER: ${{ secrets.AWS_ACCESS_KEY_S3_USER }} | |
| AWS_SECRET_KEY_S3_USER: ${{ secrets.AWS_SECRET_KEY_S3_USER }} | |
| BLOCKCHAIN_ACTIONS_TOKEN: ${{ secrets.BLOCKCHAIN_ACTIONS_TOKEN }} | |
| CGR_USERNAME: ${{ secrets.CGR_USERNAME }} | |
| CGR_PASSWORD: ${{ secrets.CGR_PASSWORD }} | |
| permissions: &docker_permissions | |
| actions: 'read' # Required to read workflow run information | |
| contents: 'read' # Required to checkout repository code | |
| pull-requests: 'read' # Required to read pull request information | |
| attestations: 'write' # Required to create build attestations | |
| packages: 'write' # Required to publish Docker images | |
| id-token: 'write' # Required for OIDC authentication | |
| with: | |
| use-cgr-secrets: true | |
| working-directory: "." | |
| image-name: "fhevm/coprocessor/db-migration" | |
| docker-file: "coprocessor/fhevm-engine/db-migration/Dockerfile" | |
| app-cache-dir: "fhevm-coprocessor-db-migration" | |
| rust-toolchain-file-path: coprocessor/fhevm-engine/rust-toolchain.toml | |
| re-tag-db-migration-image: | |
| needs: check-changes-db-migration | |
| if: | | |
| needs.check-changes-db-migration.outputs.changes != 'true' && github.event_name == 'push' | |
| permissions: &re-tag-image-permissions | |
| actions: 'read' # Required to read workflow run information | |
| contents: 'read' # Required to checkout repository code | |
| packages: 'write' # Required to publish Docker images | |
| id-token: 'write' # Required for OIDC authentication | |
| uses: ./.github/workflows/re-tag-docker-image.yml | |
| with: | |
| image-name: "fhevm/coprocessor/db-migration" | |
| previous-tag-or-commit: ${{ needs.check-changes-db-migration.outputs.base-commit }} | |
| new-tag-or-commit: ${{ github.event.after }} | |
| ######################################################################## | |
| # GATEWAY LISTENER # | |
| ######################################################################## | |
| check-changes-gw-listener: | |
| uses: ./.github/workflows/check-changes-for-docker-build.yml | |
| secrets: *check_changes_secrets | |
| permissions: *check_changes_permissions | |
| with: | |
| caller-workflow-event-name: ${{ github.event_name }} | |
| caller-workflow-event-before: ${{ github.event.before }} | |
| docker-image: fhevm/coprocessor/gw-listener | |
| filters: | | |
| gw-listener: | |
| - .github/workflows/coprocessor-docker-build.yml | |
| - coprocessor/fhevm-engine/gw-listener/** | |
| - coprocessor/fhevm-engine/Cargo.* | |
| build-gw-listener: | |
| needs: check-changes-gw-listener | |
| if: | | |
| github.event_name == 'release' | |
| || (github.event_name != 'workflow_dispatch' && needs.check-changes-gw-listener.outputs.changes == 'true') | |
| || (github.event_name == 'workflow_dispatch' && inputs.build_gw_listener) | |
| uses: zama-ai/ci-templates/.github/workflows/common-docker.yml@3cf4c2b133947d29e7a313555638621f9ca0345c # v1.0.3 | |
| permissions: *docker_permissions | |
| secrets: *docker_secrets | |
| with: | |
| use-cgr-secrets: true | |
| working-directory: "." | |
| image-name: "fhevm/coprocessor/gw-listener" | |
| docker-file: "./coprocessor/fhevm-engine/gw-listener/Dockerfile" | |
| app-cache-dir: "fhevm-coprocessor-gw-listener" | |
| rust-toolchain-file-path: coprocessor/fhevm-engine/rust-toolchain.toml | |
| re-tag-gw-listener-image: | |
| needs: check-changes-gw-listener | |
| if: | | |
| needs.check-changes-gw-listener.outputs.changes != 'true' && github.event_name == 'push' | |
| permissions: *re-tag-image-permissions | |
| uses: ./.github/workflows/re-tag-docker-image.yml | |
| with: | |
| image-name: "fhevm/coprocessor/gw-listener" | |
| previous-tag-or-commit: ${{ needs.check-changes-gw-listener.outputs.base-commit }} | |
| new-tag-or-commit: ${{ github.event.after }} | |
| ######################################################################## | |
| # HOST LISTENER # | |
| ######################################################################## | |
| check-changes-host-listener: | |
| uses: ./.github/workflows/check-changes-for-docker-build.yml | |
| secrets: *check_changes_secrets | |
| permissions: *check_changes_permissions | |
| with: | |
| caller-workflow-event-name: ${{ github.event_name }} | |
| caller-workflow-event-before: ${{ github.event.before }} | |
| docker-image: fhevm/coprocessor/host-listener | |
| filters: | | |
| host-listener: | |
| - .github/workflows/coprocessor-docker-build.yml | |
| - coprocessor/fhevm-engine/host-listener/** | |
| - coprocessor/fhevm-engine/Cargo.* | |
| - host-contracts/contracts/*Events.sol | |
| - host-contracts/contracts/shared/** | |
| build-host-listener: | |
| needs: check-changes-host-listener | |
| if: | | |
| github.event_name == 'release' | |
| || (github.event_name != 'workflow_dispatch' && needs.check-changes-host-listener.outputs.changes == 'true') | |
| || (github.event_name == 'workflow_dispatch' && inputs.build_host_listener) | |
| uses: zama-ai/ci-templates/.github/workflows/common-docker.yml@3cf4c2b133947d29e7a313555638621f9ca0345c # v1.0.3 | |
| permissions: *docker_permissions | |
| secrets: *docker_secrets | |
| with: | |
| use-cgr-secrets: true | |
| working-directory: "." | |
| image-name: "fhevm/coprocessor/host-listener" | |
| docker-file: "coprocessor/fhevm-engine/host-listener/Dockerfile" | |
| app-cache-dir: "fhevm-coprocessor-host-listener" | |
| rust-toolchain-file-path: coprocessor/fhevm-engine/rust-toolchain.toml | |
| re-tag-host-listener-image: | |
| needs: check-changes-host-listener | |
| if: | | |
| needs.check-changes-host-listener.outputs.changes != 'true' && github.event_name == 'push' | |
| permissions: *re-tag-image-permissions | |
| uses: ./.github/workflows/re-tag-docker-image.yml | |
| with: | |
| image-name: "fhevm/coprocessor/host-listener" | |
| previous-tag-or-commit: ${{ needs.check-changes-host-listener.outputs.base-commit }} | |
| new-tag-or-commit: ${{ github.event.after }} | |
| ######################################################################## | |
| # SNS WORKER # | |
| ######################################################################## | |
| check-changes-sns-worker: | |
| uses: ./.github/workflows/check-changes-for-docker-build.yml | |
| secrets: *check_changes_secrets | |
| permissions: *check_changes_permissions | |
| with: | |
| caller-workflow-event-name: ${{ github.event_name }} | |
| caller-workflow-event-before: ${{ github.event.before }} | |
| docker-image: fhevm/coprocessor/sns-worker | |
| filters: | | |
| sns-worker: | |
| - .github/workflows/coprocessor-docker-build.yml | |
| - coprocessor/fhevm-engine/sns-worker/** | |
| - coprocessor/fhevm-engine/Cargo.* | |
| build-sns-worker: | |
| needs: check-changes-sns-worker | |
| if: | | |
| github.event_name == 'release' | |
| || (github.event_name != 'workflow_dispatch' && needs.check-changes-sns-worker.outputs.changes == 'true') | |
| || (github.event_name == 'workflow_dispatch' && inputs.build_sns_worker) | |
| uses: zama-ai/ci-templates/.github/workflows/common-docker.yml@3cf4c2b133947d29e7a313555638621f9ca0345c # v1.0.3 | |
| permissions: *docker_permissions | |
| secrets: *docker_secrets | |
| with: | |
| use-cgr-secrets: true | |
| working-directory: "." | |
| image-name: "fhevm/coprocessor/sns-worker" | |
| docker-file: "coprocessor/fhevm-engine/sns-worker/Dockerfile" | |
| app-cache-dir: "fhevm-coprocessor-sns-worker" | |
| rust-toolchain-file-path: coprocessor/fhevm-engine/rust-toolchain.toml | |
| re-tag-sns-worker-image: | |
| needs: check-changes-sns-worker | |
| if: | | |
| needs.check-changes-sns-worker.outputs.changes != 'true' && github.event_name == 'push' | |
| permissions: *re-tag-image-permissions | |
| uses: ./.github/workflows/re-tag-docker-image.yml | |
| with: | |
| image-name: "fhevm/coprocessor/sns-worker" | |
| previous-tag-or-commit: ${{ needs.check-changes-sns-worker.outputs.base-commit }} | |
| new-tag-or-commit: ${{ github.event.after }} | |
| ######################################################################## | |
| # TFHE WORKER # | |
| ######################################################################## | |
| check-changes-tfhe-worker: | |
| uses: ./.github/workflows/check-changes-for-docker-build.yml | |
| secrets: *check_changes_secrets | |
| permissions: *check_changes_permissions | |
| with: | |
| caller-workflow-event-name: ${{ github.event_name }} | |
| caller-workflow-event-before: ${{ github.event.before }} | |
| docker-image: fhevm/coprocessor/tfhe-worker | |
| filters: | | |
| tfhe-worker: | |
| - .github/workflows/coprocessor-docker-build.yml | |
| - coprocessor/fhevm-engine/tfhe-worker/** | |
| - coprocessor/fhevm-engine/Cargo.* | |
| build-tfhe-worker: | |
| needs: check-changes-tfhe-worker | |
| if: | | |
| github.event_name == 'release' | |
| || (github.event_name != 'workflow_dispatch' && needs.check-changes-tfhe-worker.outputs.changes == 'true') | |
| || (github.event_name == 'workflow_dispatch' && inputs.build_tfhe_worker) | |
| uses: zama-ai/ci-templates/.github/workflows/common-docker.yml@3cf4c2b133947d29e7a313555638621f9ca0345c # v1.0.3 | |
| permissions: *docker_permissions | |
| secrets: *docker_secrets | |
| with: | |
| use-cgr-secrets: true | |
| working-directory: "." | |
| image-name: "fhevm/coprocessor/tfhe-worker" | |
| docker-file: "coprocessor/fhevm-engine/tfhe-worker/Dockerfile" | |
| app-cache-dir: "fhevm-coprocessor-tfhe-worker" | |
| rust-toolchain-file-path: coprocessor/fhevm-engine/rust-toolchain.toml | |
| re-tag-tfhe-worker-image: | |
| needs: check-changes-tfhe-worker | |
| if: | | |
| needs.check-changes-tfhe-worker.outputs.changes != 'true' && github.event_name == 'push' | |
| permissions: *re-tag-image-permissions | |
| uses: ./.github/workflows/re-tag-docker-image.yml | |
| with: | |
| image-name: "fhevm/coprocessor/tfhe-worker" | |
| previous-tag-or-commit: ${{ needs.check-changes-tfhe-worker.outputs.base-commit }} | |
| new-tag-or-commit: ${{ github.event.after }} | |
| ######################################################################## | |
| # TRANSACTION SENDER # | |
| ######################################################################## | |
| check-changes-tx-sender: | |
| uses: ./.github/workflows/check-changes-for-docker-build.yml | |
| secrets: *check_changes_secrets | |
| permissions: *check_changes_permissions | |
| with: | |
| caller-workflow-event-name: ${{ github.event_name }} | |
| caller-workflow-event-before: ${{ github.event.before }} | |
| docker-image: fhevm/coprocessor/tx-sender | |
| filters: | | |
| tx-sender: | |
| - .github/workflows/coprocessor-docker-build.yml | |
| - coprocessor/fhevm-engine/transaction-sender/** | |
| - coprocessor/fhevm-engine/Cargo.* | |
| build-tx-sender: | |
| needs: check-changes-tx-sender | |
| if: | | |
| github.event_name == 'release' | |
| || (github.event_name != 'workflow_dispatch' && needs.check-changes-tx-sender.outputs.changes == 'true') | |
| || (github.event_name == 'workflow_dispatch' && inputs.build_tx_sender) | |
| uses: zama-ai/ci-templates/.github/workflows/common-docker.yml@3cf4c2b133947d29e7a313555638621f9ca0345c # v1.0.3 | |
| permissions: *docker_permissions | |
| secrets: *docker_secrets | |
| with: | |
| use-cgr-secrets: true | |
| working-directory: "." | |
| image-name: "fhevm/coprocessor/tx-sender" | |
| docker-file: "./coprocessor/fhevm-engine/transaction-sender/Dockerfile" | |
| app-cache-dir: "fhevm-coprocessor-tx-sender" | |
| rust-toolchain-file-path: coprocessor/fhevm-engine/rust-toolchain.toml | |
| re-tag-tx-sender-image: | |
| needs: check-changes-tx-sender | |
| if: | | |
| needs.check-changes-tx-sender.outputs.changes != 'true' && github.event_name == 'push' | |
| permissions: *re-tag-image-permissions | |
| uses: ./.github/workflows/re-tag-docker-image.yml | |
| with: | |
| image-name: "fhevm/coprocessor/tx-sender" | |
| previous-tag-or-commit: ${{ needs.check-changes-tx-sender.outputs.base-commit }} | |
| new-tag-or-commit: ${{ github.event.after }} | |
| ######################################################################## | |
| # ZKPROOF WORKER # | |
| ######################################################################## | |
| check-changes-zkproof-worker: | |
| uses: ./.github/workflows/check-changes-for-docker-build.yml | |
| secrets: *check_changes_secrets | |
| permissions: *check_changes_permissions | |
| with: | |
| caller-workflow-event-name: ${{ github.event_name }} | |
| caller-workflow-event-before: ${{ github.event.before }} | |
| docker-image: fhevm/coprocessor/zkproof-worker | |
| filters: | | |
| zkproof-worker: | |
| - .github/workflows/coprocessor-docker-build.yml | |
| - coprocessor/fhevm-engine/zkproof-worker/** | |
| - coprocessor/fhevm-engine/Cargo.* | |
| build-zkproof-worker: | |
| needs: check-changes-zkproof-worker | |
| if: | | |
| github.event_name == 'release' | |
| || (github.event_name != 'workflow_dispatch' && needs.check-changes-zkproof-worker.outputs.changes == 'true') | |
| || (github.event_name == 'workflow_dispatch' && inputs.build_zkproof_worker) | |
| uses: zama-ai/ci-templates/.github/workflows/common-docker.yml@3cf4c2b133947d29e7a313555638621f9ca0345c # v1.0.3 | |
| permissions: *docker_permissions | |
| secrets: *docker_secrets | |
| with: | |
| use-cgr-secrets: true | |
| working-directory: "." | |
| image-name: "fhevm/coprocessor/zkproof-worker" | |
| docker-file: "coprocessor/fhevm-engine/zkproof-worker/Dockerfile" | |
| app-cache-dir: "fhevm-coprocessor-zkproof-worker" | |
| rust-toolchain-file-path: coprocessor/fhevm-engine/rust-toolchain.toml | |
| re-tag-zkproof-worker-image: | |
| needs: check-changes-zkproof-worker | |
| if: | | |
| needs.check-changes-zkproof-worker.outputs.changes != 'true' && github.event_name == 'push' | |
| permissions: *re-tag-image-permissions | |
| uses: ./.github/workflows/re-tag-docker-image.yml | |
| with: | |
| image-name: "fhevm/coprocessor/zkproof-worker" | |
| previous-tag-or-commit: ${{ needs.check-changes-zkproof-worker.outputs.base-commit }} | |
| new-tag-or-commit: ${{ github.event.after }} |