release zksync os (#4212)

## What ❔

Release zksync os prover and server. For using it later in demo stands. 

## Why ❔

<!-- Why are these changes done? What goal do they contribute to? What
are the principles behind them? -->
<!-- The `Why` has to be clear to non-Matter Labs entities running their
own ZK Chain -->
<!-- Example: PR templates ensure PR reviewers, observers, and future
iterators are in context about the evolution of repos. -->

## Is this a breaking change?
- [ ] Yes
- [ ] No

## Operational changes
<!-- Any config changes? Any new flags? Any changes to any scripts? -->
<!-- Please add anything that non-Matter Labs entities running their own
ZK Chain may need to know -->

## Checklist

<!-- Check your PR fulfills the following items. -->
<!-- For draft PRs check the boxes as you complete them. -->

- [ ] PR title corresponds to the body of PR (we generate changelog
entries from PRs).
- [ ] Tests for the changes have been added / updated.
- [ ] Documentation comments have been added / updated.
- [ ] Code has been formatted via `zkstack dev fmt` and `zkstack dev
lint`.

---------

Signed-off-by: Danil <[email protected]>

Author: Deniallugo

.dockerignore

@@ -17,6 +17,7 @@ keys/setup
 !db/
 !backups/
 !core/
+!zkos_prover/
 !prover/
 !yarn.lock
 !package.json
@@ -48,3 +49,6 @@ contracts/.git
 !etc/env/consensus_secrets.yaml
 !etc/env/consensus_config.yaml
 !rust-toolchain
+!execution_environment/*
+!app.bin
+!app_logging_enabled.bin

.github/workflows/build-core-template.yml

@@ -27,6 +27,11 @@ on:
         type: string
         required: false
         default: "do nothing"
+      zksync_os:
+        description: 'Flag that determines if zksync-os should be built'
+        type: boolean
+        required: false
+        default: false
 
 jobs:
   prepare-contracts:
@@ -46,6 +51,7 @@ jobs:
           echo $HOME/.local/bin >> $GITHUB_PATH
           echo CI=1 >> .env
           echo IN_DOCKER=1 >> .env
+          echo RUSTFLAGS="" >> .env
 
       - name: Download contracts
         shell: bash
@@ -133,7 +139,7 @@ jobs:
         if: env.BUILD_CONTRACTS == 'true'
         run: |
           ./zkstack_cli/zkstackup/install --path ./zkstack_cli/zkstackup/zkstackup
-          zkstackup --local || true
+          RUSTFLAGS="" zkstackup --local || true
 
       - name: build contracts
         if: env.BUILD_CONTRACTS == 'true'
@@ -153,7 +159,7 @@ jobs:
     name: Build and Push Docker Images
     needs: prepare-contracts
     env:
-      IMAGE_TAG_SUFFIX: ${{ inputs.image_tag_suffix }}${{ (inputs.en_alpha_release && matrix.components == 'external-node') && '-alpha' || '' }}
+      IMAGE_TAG_SUFFIX: ${{ inputs.zksync_os && 'zksync_os-' || '' }}${{ inputs.image_tag_suffix }}${{ (inputs.en_alpha_release && matrix.components == 'external-node') && '-alpha' || '' }}
     runs-on: ${{ fromJSON('["matterlabs-ci-runner-high-performance", "matterlabs-ci-runner-arm"]')[contains(matrix.platforms, 'arm')] }}
     permissions:
       packages: write
@@ -164,6 +170,7 @@ jobs:
           - server-v2
           - external-node
           - snapshots-creator
+          - zksync-os-server
         platforms:
           - linux/amd64
         include:
@@ -240,9 +247,10 @@ jobs:
             SCCACHE_GCS_SERVICE_ACCOUNT=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com
             SCCACHE_GCS_RW_MODE=READ_WRITE
             RUSTC_WRAPPER=sccache
-          tags: |            
+          tags: |
             us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.components }}:${{ env.IMAGE_TAG_SHA_TS }}-${{ env.PLATFORM }}
             ghcr.io/${{ github.repository_owner }}/${{ matrix.components }}:${{ env.IMAGE_TAG_SHA_TS }}-${{ env.PLATFORM }}
+            ghcr.io/${{ github.repository_owner }}/${{ matrix.components }}:${{ inputs.zksync_os && 'zksync_os-' || '' }}latest
             matterlabs/${{ matrix.components }}:${{ env.IMAGE_TAG_SHA_TS }}-${{ env.PLATFORM }}
 
       - name: Push docker image
@@ -251,6 +259,7 @@ jobs:
           docker push us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.components }}:${{ env.IMAGE_TAG_SHA_TS }}-${{ env.PLATFORM }}
           docker push matterlabs/${{ matrix.components }}:${{ env.IMAGE_TAG_SHA_TS }}-${{ env.PLATFORM }}
           docker push ghcr.io/${{ github.repository_owner }}/${{ matrix.components }}:${{ env.IMAGE_TAG_SHA_TS }}-${{ env.PLATFORM }}
+          docker push ghcr.io/${{ github.repository_owner }}/${{ matrix.components }}:${{ inputs.zksync_os && 'zksync_os-' || '' }}latest
 
   create_manifest:
     name: Create release manifest
@@ -262,13 +271,15 @@ jobs:
         component:
           - name: server-v2
             platform: linux/amd64
+          - name: zksync-os-server
+            platform: linux/amd64
           - name: external-node
             platform: linux/amd64,linux/arm64
           - name: snapshots-creator
             platform: linux/amd64
 
     env:
-      IMAGE_TAG_SUFFIX: ${{ inputs.image_tag_suffix }}${{ (inputs.en_alpha_release && matrix.component.name == 'external-node') && '-alpha' || '' }}
+      IMAGE_TAG_SUFFIX: ${{ inputs.zksync_os && 'zksync_os-' || '' }}${{ inputs.image_tag_suffix }}${{ (inputs.en_alpha_release && matrix.component.name == 'external-node') && '-alpha' || '' }}
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
 

.github/workflows/build-zksync-os-prover-template.yml

@@ -0,0 +1,95 @@
+name: Build Prover images
+on:
+  workflow_call:
+    secrets:
+      DOCKERHUB_USER:
+        description: "DOCKERHUB_USER"
+        required: true
+      DOCKERHUB_TOKEN:
+        description: "DOCKERHUB_TOKEN"
+        required: true
+    inputs:
+      image_tag_suffix:
+        description: "Optional suffix to override tag name generation"
+        type: string
+        required: false
+      action:
+        description: "Action with docker image"
+        type: string
+        default: "push"
+        required: false
+
+jobs:
+  build-images:
+    name: Build and Push Docker Images
+    runs-on: [ matterlabs-ci-runner-high-performance ]
+    permissions:
+      packages: write
+      contents: read
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: "recursive"
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0
+
+      - name: setup-env
+        run: |
+          echo ZKSYNC_HOME=$(pwd) >> $GITHUB_ENV
+          echo CI=1 >> $GITHUB_ENV
+          echo $(pwd)/bin >> $GITHUB_PATH
+          echo CI=1 >> .env
+          echo IN_DOCKER=1 >> .env
+
+      - name: Set env vars
+        shell: bash
+        run: |
+          # Support for custom tag suffix
+          if [ -n "${{ inputs.image_tag_suffix }}" ]; then
+            echo IMAGE_TAG_SHA_TS="${{ inputs.image_tag_suffix }}" >> $GITHUB_ENV
+          else
+            echo IMAGE_TAG_SHA_TS=$(git rev-parse --short HEAD)-$(date +%s) >> $GITHUB_ENV
+          fi
+
+      - name: login to Docker registries
+        if: ${{ inputs.action == 'push' }}
+        shell: bash
+        run: |
+          docker login -u ${{ secrets.DOCKERHUB_USER }} -p ${{ secrets.DOCKERHUB_TOKEN }}
+          gcloud auth configure-docker us-docker.pkg.dev -q
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
+        if: ${{ inputs.action == 'push' }}
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push
+        uses: docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4 # v6.15.0
+        with:
+          context: .
+          load: true
+          build-args: |
+            SCCACHE_GCS_BUCKET=matterlabs-infra-sccache-storage
+            SCCACHE_GCS_SERVICE_ACCOUNT=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com
+            SCCACHE_GCS_RW_MODE=READ_WRITE
+            RUSTC_WRAPPER=sccache
+          file: docker/zksync-os-prover/Dockerfile
+          tags: |
+            us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/zksync-os-prover:${{ env.IMAGE_TAG_SHA_TS }}
+            ghcr.io/${{ github.repository_owner }}/zksync-os-prover:${{ env.IMAGE_TAG_SHA_TS }}
+            ghcr.io/${{ github.repository_owner }}/zksync-os-prover:latest
+            matterlabs/zksync-os-prover:${{ env.IMAGE_TAG_SHA_TS }}
+            matterlabs/zksync-os-prover:latest
+
+      - name: Push docker image
+        if: ${{ inputs.action == 'push' }}
+        run: |
+          docker push us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/zksync-os-prover:${{ env.IMAGE_TAG_SHA_TS }}
+          docker push ghcr.io/${{ github.repository_owner }}/zksync-os-prover:${{ env.IMAGE_TAG_SHA_TS }}
+          docker push ghcr.io/${{ github.repository_owner }}/zksync-os-prover:latest
+          docker push matterlabs/zksync-os-prover:${{ env.IMAGE_TAG_SHA_TS }}
+          docker push matterlabs/zksync-os-prover:latest

.github/workflows/release-zkos-stage.yml

@@ -0,0 +1,87 @@
+name: Build and release zksync-os images
+on:
+  push:
+    branches:
+      - zksync-os-integration
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name != 'push' || github.ref != 'refs/heads/main' }}
+
+jobs:
+  changed_files:
+    runs-on: ubuntu-latest
+    name: Test changed-files
+    outputs:
+      core: ${{ steps.changed-files-yaml.outputs.core_any_changed }}
+      prover: ${{ steps.changed-files-yaml.outputs.prover_any_changed }}
+      all: ${{ steps.changed-files-yaml.outputs.all_any_changed }}
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
+        with:
+          fetch-depth: 2
+
+      - name: Get all test, doc and src files that have changed
+        id: changed-files-yaml
+        uses: tj-actions/changed-files@ed68ef82c095e0d48ec87eccea555d944a631a4c # v46
+        with:
+          files_yaml: |
+            # TODO: make it more granular, as already implemented in CI workflow
+            # We don't want to be rebuilding and redeploying all the Docker images when eg. only document have changed
+            prover:
+              - prover/**
+              - core/lib/**
+              - zkos_prover/**
+              - '!core/lib/zksync_core_leftovers/**'
+            core:
+              - core/**
+            all:
+              - '!core/**'
+              - '!prover/**'
+  setup:
+    name: Setup
+    runs-on: [ matterlabs-deployer-stage ]
+    outputs:
+      image_tag_suffix: ${{ steps.generate-tag-suffix.outputs.image_tag_suffix }}
+      prover_fri_gpu_key_id: ${{ steps.extract-prover-fri-setup-key-ids.outputs.gpu_short_commit_sha }}
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
+
+      - name: Generate image tag suffix
+        id: generate-tag-suffix
+        run: |
+          sha=$(git rev-parse --short HEAD)
+          ts=$(date +%s | cut -b1-13)
+          echo "image_tag_suffix=${sha}-${ts}" >> $GITHUB_OUTPUT
+
+      - name: Generate outputs with Prover FRI setup data keys IDs
+        id: extract-prover-fri-setup-key-ids
+        run: |
+          ./prover/extract-setup-data-keys.sh >> $GITHUB_OUTPUT
+
+  build-push-core-images:
+    name: Build and push images
+    needs: [ setup, changed_files ]
+    uses: ./.github/workflows/build-core-template.yml
+    if: needs.changed_files.outputs.core == 'true' || needs.changed_files.outputs.all == 'true'
+    with:
+      image_tag_suffix: ${{ needs.setup.outputs.image_tag_suffix }}
+      action: "push"
+      zksync_os: true
+    secrets:
+      DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
+      DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+
+
+  build-push-prover-images:
+    name: Build and push images
+    needs: [ setup, changed_files ]
+    uses: ./.github/workflows/build-zksync-os-prover-template.yml
+    if: needs.changed_files.outputs.prover == 'true' || needs.changed_files.outputs.all == 'true'
+    with:
+      image_tag_suffix: ${{ needs.setup.outputs.image_tag_suffix }}
+      action: "push"
+    secrets:
+      DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
+      DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+

core/node/zkos_prover_input_generator/src/zkos_prover_input_generator.rs

@@ -253,7 +253,7 @@ impl ZkosProverInputGenerator {
 
         let list_source = TxListSource { transactions };
         let prover_input = zk_os_forward_system::run::generate_proof_input(
-            PathBuf::from("app_logging_enabled.bin"),
+            PathBuf::from("execution_environment/app_logging_enabled.bin"),
             context,
             storage_commitment,
             tree,

docker-compose-zksync-os.yml

@@ -0,0 +1,73 @@
+services:
+  reth:
+    restart: always
+    image: "ghcr.io/paradigmxyz/reth:v1.3.7"
+    ulimits:
+      nofile:
+        soft: 1048576
+        hard: 1048576
+    ports:
+      - 127.0.0.1:8545:8545
+    volumes:
+      - type: volume
+        source: reth-data
+        target: /rethdata
+      - type: bind
+        source: ./etc/reth/chaindata
+        target: /chaindata
+
+    command: node --dev --datadir /rethdata --http --http.addr 0.0.0.0 --http.port 8545 --http.corsdomain "*"  --chain /chaindata/reth_config --dev.block-time 300ms
+
+  postgres:
+    image: "postgres:14"
+    command: postgres -c 'max_connections=1000'
+    ulimits:
+      nofile:
+        soft: 1048576
+        hard: 1048576
+    ports:
+      - 127.0.0.1:5432:5432
+    volumes:
+      - type: volume
+        source: postgres-data
+        target: /var/lib/postgresql/data
+    environment:
+      # We bind only to 127.0.0.1, so setting insecure password is acceptable here
+      - POSTGRES_PASSWORD=notsecurepassword
+
+  zksync-os-state-keeper:
+    image: "matterlabs/zksync-os-server:latest"
+    platform: linux/x86_64
+
+    ports:
+      - 3124:3124
+    volumes:
+      - ./chains/era/configs:/configs
+    command: --config-path=/configs/general.yaml --secrets-path=/configs/secrets.yaml --genesis-path=/configs/genesis.yaml --contracts-config-path=/configs/contracts.yaml --wallets-path=/configs/wallets.yaml --components=eth,state_keeper,zkos_prover_input_generator
+    network_mode: host
+
+  zksync-os-api:
+    image: "matterlabs/zksync-os-server:latest"
+    platform: linux/x86_64
+
+    ports:
+      - 3050:3050
+      - 3051:3051
+    volumes:
+      - ./chains/era/configs:/configs
+    command: --config-path=/configs/general.yaml --secrets-path=/configs/secrets.yaml --genesis-path=/configs/genesis.yaml --contracts-config-path=/configs/contracts.yaml --wallets-path=/configs/wallets.yaml --components=api
+    network_mode: host
+
+  zksync-os-prover:
+    image: "matterlabs/zksync-os-prover:latest"
+    platform: linux/x86_64
+
+    ports:
+      - 3052:3052
+    command: --base-url=http://localhost:3124
+    network_mode: host
+
+
+volumes:
+  postgres-data:
+  reth-data: