switch to feature flag

Author: bernard-wagner

.github/workflows/arbitrator-ci.yml

@@ -128,6 +128,23 @@ jobs:
         with:
           cache: false
 
+      - name: Cache cbrotli
+        uses: actions/cache@v4
+        id: cache-cbrotli
+        with:
+          path: |
+            target/include/brotli/
+            target/lib-wasm/
+            target/lib/libbrotlicommon-static.a
+            target/lib/libbrotlienc-static.a
+            target/lib/libbrotlidec-static.a
+          key: ${{ runner.os }}-brotli-${{ hashFiles('scripts/build-brotli.sh') }}-${{ hashFiles('.github/workflows/arbitrator-ci.yaml') }}-arbitrator
+          restore-keys: ${{ runner.os }}-brotli-${{ hashFiles('scripts/build-brotli.sh') }}-${{ hashFiles('.github/workflows/arbitrator-ci.yaml') }}
+
+      - name: Build cbrotli-local
+        if: steps.cache-cbrotli.outputs.cache-hit != 'true'
+        run: ./scripts/build-brotli.sh -l
+
       - name: Setup emsdk
         if: steps.cache-cbrotli.outputs.cache-hit != 'true'
         uses: mymindstorm/setup-emsdk@v12
@@ -139,6 +156,10 @@ jobs:
           #  so make sure it doesn't conflict with anything!
           no-cache: true
 
+      - name: Build cbrotli-wasm
+        if: steps.cache-cbrotli.outputs.cache-hit != 'true'
+        run: ./scripts/build-brotli.sh -w
+
       - name: Add wabt to path
         run: echo "$HOME/wabt-prefix/bin" >> "$GITHUB_PATH"
 

.github/workflows/ci.yml

@@ -105,6 +105,26 @@ jobs:
           key: ${{ runner.os }}-cargo-${{ matrix.test-mode }}-${{ steps.rust-version.outputs.version }}-min-${{ hashFiles('arbitrator/Cargo.lock') }}
           restore-keys: ${{ runner.os }}-cargo-${{ matrix.test-mode }}-${{ steps.rust-version.outputs.version }}-
 
+      - name: Cache cbrotli
+        uses: actions/cache@v4
+        id: cache-cbrotli
+        with:
+          path: |
+            target/include/brotli/
+            target/lib-wasm/
+            target/lib/libbrotlicommon-static.a
+            target/lib/libbrotlienc-static.a
+            target/lib/libbrotlidec-static.a
+          key: ${{ runner.os }}-brotli-${{ matrix.test-mode }}-${{ hashFiles('scripts/build-brotli.sh') }}-${{ hashFiles('.github/workflows/arbitrator-ci.yaml') }}
+
+      - name: Build cbrotli-local
+        if: steps.cache-cbrotli.outputs.cache-hit != 'true'
+        run: ./scripts/build-brotli.sh -l
+
+      - name: Build cbrotli-wasm in docker
+        if: steps.cache-cbrotli.outputs.cache-hit != 'true'
+        run: ./scripts/build-brotli.sh -w -d
+
       - name: Build
         run: make build test-go-deps -j8
 

.github/workflows/codeql-analysis.yml

@@ -113,6 +113,19 @@ jobs:
         path: ~/wabt-prefix
         key: ${{ runner.os }}-wabt-codeql-${{ matrix.language }}-${{ env.WABT_VERSION }}
 
+    - name: Cache cbrotli
+      uses: actions/cache@v4
+      id: cache-cbrotli
+      with:
+        path: |
+          target/include/brotli/
+          target/lib-wasm/
+          target/lib/libbrotlicommon-static.a
+          target/lib/libbrotlienc-static.a
+          target/lib/libbrotlidec-static.a
+        key: ${{ runner.os }}-brotli-${{ matrix.language }}-${{ hashFiles('scripts/build-brotli.sh') }}-${{ hashFiles('.github/workflows/arbitrator-ci.yaml') }}-codeql
+        restore-keys: ${{ runner.os }}-brotli-${{ matrix.language }}-${{ hashFiles('scripts/build-brotli.sh') }}-${{ hashFiles('.github/workflows/arbitrator-ci.yaml') }}
+
     - name: Cache Rust Build Products
       uses: actions/cache@v4
       with:
@@ -139,6 +152,14 @@ jobs:
         # queries: security-extended,security-and-quality
         config-file: ./.github/codeql/codeql-config.yml
 
+    - name: Build cbrotli-local
+      if: steps.cache-cbrotli.outputs.cache-hit != 'true'
+      run: ./scripts/build-brotli.sh -l
+
+    - name: Build cbrotli-wasm in docker
+      if: steps.cache-cbrotli.outputs.cache-hit != 'true'
+      run: ./scripts/build-brotli.sh -w -d
+
     - name: Build Nitro for CodeQL
       run: make build
 

Dockerfile

@@ -43,7 +43,6 @@ RUN . ~/.bashrc && NITRO_BUILD_IGNORE_TIMESTAMPS=1 make build-solidity
 
 FROM debian:bookworm-20231218 AS wasm-base
 WORKDIR /workspace
-COPY brotli brotli
 RUN apt-get update && apt-get install -y curl build-essential=12.9
 
 FROM wasm-base AS wasm-libs-builder
@@ -52,9 +51,6 @@ RUN apt-get update && \
     apt-get install -y clang=1:14.0-55.7~deb12u1 lld=1:14.0-55.7~deb12u1 wabt
     # pinned rust 1.84.1
 RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain 1.84.1 --target x86_64-unknown-linux-gnu,wasm32-unknown-unknown,wasm32-wasip1
-RUN mkdir -p /opt/wasi-sdk/wasi-sysroot && \
-    curl --proto '=https' --tlsv1.2 -sSf -L https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-27/wasi-sysroot-27.0.tar.gz | \
-    tar -C /opt/wasi-sdk/wasi-sysroot -xzf - --strip-components=1
 COPY ./Makefile ./
 COPY arbitrator/Cargo.* arbitrator/
 COPY arbitrator/arbutil arbitrator/arbutil
@@ -63,11 +59,12 @@ COPY arbitrator/caller-env arbitrator/caller-env
 COPY arbitrator/prover arbitrator/prover
 COPY arbitrator/wasm-libraries arbitrator/wasm-libraries
 COPY arbitrator/tools/wasmer arbitrator/tools/wasmer
+COPY brotli brotli
 COPY scripts/build-brotli.sh scripts/
 COPY scripts/remove_reference_types.sh scripts/
 COPY --from=brotli-wasm-export / target/
 RUN apt-get update && apt-get install -y cmake
-RUN . ~/.cargo/env && NITRO_BUILD_IGNORE_TIMESTAMPS=1 WASI_SYSROOT=/opt/wasi-sdk/wasi-sysroot RUSTFLAGS='-C symbol-mangling-version=v0' make build-wasm-libs
+RUN . ~/.cargo/env && NITRO_BUILD_IGNORE_TIMESTAMPS=1 RUSTFLAGS='-C symbol-mangling-version=v0' make build-wasm-libs
 
 FROM scratch AS wasm-libs-export
 COPY --from=wasm-libs-builder /workspace/ /
@@ -148,7 +145,6 @@ RUN wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
     apt-get update && \
     apt-get install -y llvm-15-dev libclang-common-15-dev
 COPY --from=brotli-library-export / target/
-COPY brotli brotli
 COPY arbitrator/Cargo.* arbitrator/
 COPY arbitrator/arbutil arbitrator/arbutil
 COPY arbitrator/bench arbitrator/bench

Makefile

@@ -57,6 +57,8 @@ done = "%bdone!%b\n" $(color_pink) $(color_reset)
 
 replay_wasm=$(output_latest)/replay.wasm
 
+arb_brotli_files = $(wildcard arbitrator/brotli/src/*.* arbitrator/brotli/src/*/*.* arbitrator/brotli/*.toml arbitrator/brotli/*.rs) .make/cbrotli-lib .make/cbrotli-wasm
+
 arbitrator_generated_header=$(output_root)/include/arbitrator.h
 arbitrator_wasm_libs=$(patsubst %, $(output_root)/machines/latest/%.wasm, forward wasi_stub host_io soft-float arbcompress user_host program_exec)
 arbitrator_stylus_lib=$(output_root)/lib/libstylus.a
@@ -80,15 +82,15 @@ WASI_SYSROOT?=/opt/wasi-sdk/wasi-sysroot
 
 arbitrator_wasm_lib_flags=$(patsubst %, -l %, $(arbitrator_wasm_libs))
 
-rust_arbutil_files = $(wildcard arbitrator/arbutil/src/*.* arbitrator/arbutil/src/*/*.* arbitrator/arbutil/*.toml arbitrator/caller-env/src/*.* arbitrator/caller-env/src/*/*.* arbitrator/caller-env/*.toml)
+rust_arbutil_files = $(wildcard arbitrator/arbutil/src/*.* arbitrator/arbutil/src/*/*.* arbitrator/arbutil/*.toml arbitrator/caller-env/src/*.* arbitrator/caller-env/src/*/*.* arbitrator/caller-env/*.toml) .make/cbrotli-lib
 
 prover_direct_includes = $(patsubst %,$(output_latest)/%.wasm, forward forward_stub)
 prover_dir = arbitrator/prover/
-rust_prover_files = $(wildcard $(prover_dir)/src/*.* $(prover_dir)/src/*/*.* $(prover_dir)/*.toml $(prover_dir)/*.rs) $(rust_arbutil_files) $(prover_direct_includes)
+rust_prover_files = $(wildcard $(prover_dir)/src/*.* $(prover_dir)/src/*/*.* $(prover_dir)/*.toml $(prover_dir)/*.rs) $(rust_arbutil_files) $(prover_direct_includes) $(arb_brotli_files)
 
 wasm_lib = arbitrator/wasm-libraries
 wasm_lib_cargo = $(wasm_lib)/.cargo/config.toml
-wasm_lib_deps = $(wildcard $(wasm_lib)/$(1)/*.toml $(wasm_lib)/$(1)/src/*.rs $(wasm_lib)/$(1)/*.rs) $(wasm_lib_cargo) $(rust_arbutil_files)  .make/machines
+wasm_lib_deps = $(wildcard $(wasm_lib)/$(1)/*.toml $(wasm_lib)/$(1)/src/*.rs $(wasm_lib)/$(1)/*.rs) $(wasm_lib_cargo) $(rust_arbutil_files) $(arb_brotli_files) .make/machines
 wasm_lib_go_abi = $(call wasm_lib_deps,go-abi)
 wasm_lib_forward = $(call wasm_lib_deps,forward)
 wasm_lib_user_host_trait = $(call wasm_lib_deps,user-host-trait)
@@ -153,6 +155,8 @@ stylus_test_hostio-test_src       = $(call get_stylus_test_rust,hostio-test)
 stylus_test_wasms = $(stylus_test_keccak_wasm) $(stylus_test_keccak-100_wasm) $(stylus_test_fallible_wasm) $(stylus_test_storage_wasm) $(stylus_test_multicall_wasm) $(stylus_test_log_wasm) $(stylus_test_create_wasm) $(stylus_test_math_wasm) $(stylus_test_sdk-storage_wasm) $(stylus_test_erc20_wasm) $(stylus_test_read-return-data_wasm) $(stylus_test_evm-data_wasm) $(stylus_test_hostio-test_wasm) $(stylus_test_bfs:.b=.wasm)
 stylus_benchmarks = $(wildcard $(stylus_dir)/*.toml $(stylus_dir)/src/*.rs) $(stylus_test_wasms)
 
+CBROTLI_WASM_BUILD_ARGS ?=-d
+
 # user targets
 
 .PHONY: push
@@ -169,7 +173,7 @@ build: $(patsubst %,$(output_root)/bin/%, nitro deploy relay daprovider daserver
 	@printf $(done)
 
 .PHONY: build-node-deps
-build-node-deps: $(go_source) build-prover-header build-prover-lib build-jit .make/solgen
+build-node-deps: $(go_source) build-prover-header build-prover-lib build-jit .make/solgen .make/cbrotli-lib
 
 .PHONY: test-go-deps
 test-go-deps: \
@@ -289,6 +293,7 @@ clean:
 	rm -f arbitrator/wasm-libraries/soft-float/SoftFloat/build/Wasm-Clang/*.a
 	rm -f arbitrator/wasm-libraries/forward/*.wat
 	rm -rf arbitrator/stylus/tests/*/target/ arbitrator/stylus/tests/*/*.wasm
+	rm -rf brotli/buildfiles
 	@rm -rf contracts/build contracts/cache solgen/go/
 	@rm -rf contracts-legacy/build contracts-legacy/cache
 	@rm -rf contracts-local/out contracts-local/forge-cache
@@ -444,7 +449,7 @@ $(output_latest)/user_test.wasm: $(DEP_PREDICATE) $(call wasm_lib_deps,user-test
 	install arbitrator/wasm-libraries/$(wasm32_wasi)/user_test.wasm $@
 	./scripts/remove_reference_types.sh $@
 
-$(output_latest)/arbcompress.wasm: $(DEP_PREDICATE) $(call wasm_lib_deps) $(wasm_lib_go_abi)
+$(output_latest)/arbcompress.wasm: $(DEP_PREDICATE) $(call wasm_lib_deps,brotli) $(wasm_lib_go_abi)
 	cargo build --manifest-path arbitrator/wasm-libraries/Cargo.toml --release --target wasm32-wasip1 --config $(wasm_lib_cargo) --package arbcompress
 	install arbitrator/wasm-libraries/$(wasm32_wasi)/arbcompress.wasm $@
 	./scripts/remove_reference_types.sh $@
@@ -618,9 +623,23 @@ contracts/test/prover/proofs/%.json: $(arbitrator_cases)/%.wasm $(prover_bin)
 	make -C contracts-local install
 	@touch $@
 
+.make/cbrotli-lib: $(DEP_PREDICATE) $(ORDER_ONLY_PREDICATE) .make
+	test -f target/include/brotli/encode.h || ./scripts/build-brotli.sh -l
+	test -f target/include/brotli/decode.h || ./scripts/build-brotli.sh -l
+	test -f target/lib/libbrotlicommon-static.a || ./scripts/build-brotli.sh -l
+	test -f target/lib/libbrotlienc-static.a || ./scripts/build-brotli.sh -l
+	test -f target/lib/libbrotlidec-static.a || ./scripts/build-brotli.sh -l
+	@touch $@
+
+.make/cbrotli-wasm: $(DEP_PREDICATE) $(ORDER_ONLY_PREDICATE) .make
+	test -f target/lib-wasm/libbrotlicommon-static.a || ./scripts/build-brotli.sh -w $(CBROTLI_WASM_BUILD_ARGS)
+	test -f target/lib-wasm/libbrotlienc-static.a || ./scripts/build-brotli.sh -w $(CBROTLI_WASM_BUILD_ARGS)
+	test -f target/lib-wasm/libbrotlidec-static.a || ./scripts/build-brotli.sh -w $(CBROTLI_WASM_BUILD_ARGS)
+	@touch $@
+
 .make/wasm-lib: $(DEP_PREDICATE) arbitrator/wasm-libraries/soft-float/SoftFloat/build/Wasm-Clang/softfloat.a  $(ORDER_ONLY_PREDICATE) .make
-	test -f arbitrator/wasm-libraries/soft-float/bindings32.o
-	test -f arbitrator/wasm-libraries/soft-float/bindings64.o
+	test -f arbitrator/wasm-libraries/soft-float/bindings32.o || ./scripts/build-brotli.sh -f -d -t ..
+	test -f arbitrator/wasm-libraries/soft-float/bindings64.o || ./scripts/build-brotli.sh -f -d -t ..
 	@touch $@
 
 .make/machines: $(DEP_PREDICATE) $(ORDER_ONLY_PREDICATE) .make

arbitrator/brotli/Cargo.toml

@@ -15,10 +15,11 @@ wasmer = { path = "../tools/wasmer/lib/api", optional = true }
 wee_alloc.workspace = true
 
 [build-dependencies]
-cc = "1.0"
+cc = { version = "1.0", optional = true }
 
 [lib]
 crate-type = ["lib"]
 
 [features]
 wasmer_traits = ["dep:wasmer"]
+cc_brotli = ["dep:cc"]

arbitrator/brotli/build.rs

@@ -1,10 +1,26 @@
 // Copyright 2021-2024, Offchain Labs, Inc.
 // For license information, see https://github.com/OffchainLabs/nitro/blob/master/LICENSE.md
+#[cfg(not(feature = "cc_brotli"))]
+fn main() {
+    use std::env;
+
+    let target_arch = env::var("TARGET").unwrap();
 
-use std::env;
-use std::path::PathBuf;
+    if target_arch.contains("wasm32") {
+        println!("cargo:rustc-link-search=../../target/lib-wasm/");
+    } else {
+        println!("cargo:rustc-link-search=../target/lib/");
+        println!("cargo:rustc-link-search=../../target/lib/");
+    }
+    println!("cargo:rustc-link-lib=static=brotlienc-static");
+    println!("cargo:rustc-link-lib=static=brotlidec-static");
+    println!("cargo:rustc-link-lib=static=brotlicommon-static");
+}
 
+#[cfg(feature = "cc_brotli")]
 fn main() {
+    use std::env;
+    use std::path::PathBuf;
     let manifest_dir = PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").unwrap());
     let include_dir = manifest_dir.join("../../brotli/c/include");
     cc::Build::new()

arbitrator/brotli/config.toml

@@ -15,3 +15,4 @@ default = ["brotli"]
 brotli = ["dep:brotli"]
 static_caller = []
 wasmer_traits = ["dep:wasmer", "brotli?/wasmer_traits"]
+cc_brotli = ["brotli?/cc_brotli"]

arbitrator/caller-env/Cargo.toml

@@ -25,3 +25,4 @@ sha2 = "0.9.9"
 
 [features]
 llvm = ["dep:wasmer-compiler-llvm"]
+cc_brotli = ["brotli/cc_brotli"]