Skip to content

Commit edc6df3

Browse files
antoniupopantoniupop
committed
feat(coprocessor): add new scheduling infrastructure for multi-GPU
1 parent 254f5d3 commit edc6df3

15 files changed

+4178
-404
lines changed

‎.github/workflows/coprocessor-benchmark-cpu.yml‎

Lines changed: 22 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -10,13 +10,23 @@ on:
1010
type: choice
1111
options:
1212
- "erc20"
13+
- "dex"
1314
- "synthetics"
1415
- "all"
1516
batch_size:
1617
description: "Batch sizes (# FHE operations executed per batch)"
1718
required: true
1819
type: string
1920
default: "5000"
21+
scheduling_policy:
22+
description: "Scheduling policy"
23+
required: true
24+
type: choice
25+
options:
26+
- "MAX_PARALLELISM"
27+
- "MAX_LOCALITY"
28+
- "LOOP"
29+
- "FINE_GRAIN"
2030

2131
env:
2232
CARGO_TERM_COLOR: always
@@ -64,11 +74,12 @@ jobs:
6474
gcc: 11
6575

6676
steps:
67-
- name: Install git LFS
77+
- name: Wait for unattended upgrades to complete
6878
run: |
69-
sudo systemctl stop unattended-upgrades
70-
sleep 10
7179
sudo apt-get update
80+
81+
- name: Install git LFS
82+
run: |
7283
sudo apt-get install -y git-lfs
7384
git lfs install
7485
@@ -91,13 +102,16 @@ jobs:
91102
} >> "${GITHUB_ENV}"
92103
93104
- name: Install rust
94-
uses: dtolnay/rust-toolchain@888c2e1ea69ab0d4330cbf0af1ecc7b68f368cc1
105+
uses: dtolnay/rust-toolchain@a54c7afa936fefeb4456b2dd8068152669aa8203
95106
with:
96107
toolchain: nightly
97108

98109
- name: Install cargo dependencies
99110
run: |
100111
sudo systemctl stop docker
112+
DEBIAN_FRONTEND=noninteractive sudo apt-get remove -y docker docker-engine docker.io containerd runc
113+
DEBIAN_FRONTEND=noninteractive sudo apt-get purge -y docker-ce docker-ce-cli containerd.io docker-compose-plugin docker-compose
114+
sudo rm -rf /etc/bash_completion.d/docker /usr/local/bin/docker-compose /etc/bash_completion.d/docker-compose
101115
DEBIAN_FRONTEND=noninteractive sudo apt-get install -y protobuf-compiler cmake \
102116
pkg-config libssl-dev \
103117
libclang-dev docker-compose-v2 \
@@ -127,17 +141,13 @@ jobs:
127141
with:
128142
node-version: 20.x
129143

130-
- name: Build contracts
131-
env:
132-
HARDHAT_NETWORK: hardhat
144+
- name: Start localstack
133145
run: |
134-
cp ./host-contracts/.env.example ./host-contracts/.env
135-
npm --prefix ./host-contracts ci --include=optional
136-
cd host-contracts && npm install && npm run deploy:emptyProxies && npx hardhat compile
146+
docker run --rm -d -p 4566:4566 --name localstack localstack/localstack:latest
137147
138148
- name: Run benchmarks on CPU
139149
run: |
140-
BENCHMARK_BATCH_SIZE="${{ inputs.batch_size }}" make "benchmark_${{ inputs.benchmarks }}_cpu"
150+
DATABASE_URL=postgresql://postgres:postgres@localhost:5432/coprocessor TXN_SENDER_TEST_GLOBAL_LOCALSTACK=1 BENCHMARK_BATCH_SIZE="${{ inputs.batch_size }}" FHEVM_DF_SCHEDULE="${{ inputs.scheduling_policy }}" make -e "benchmark_${{ inputs.benchmarks }}_cpu"
141151
working-directory: coprocessor/fhevm-engine/coprocessor
142152

143153
- name: Parse results
@@ -151,7 +161,7 @@ jobs:
151161
--commit-date "${{ env.COMMIT_DATE }}" \
152162
--bench-date "${{ env.BENCH_DATE }}" \
153163
--walk-subdirs \
154-
--name-suffix "operation_batch_size_${{ inputs.batch_size }}"
164+
--name-suffix "operation_batch_size_${{ inputs.batch_size }}-schedule_${{ inputs.scheduling_policy }}"
155165
156166
- name: Upload parsed results artifact
157167
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02

‎.github/workflows/coprocessor-benchmark-gpu.yml‎

Lines changed: 31 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,13 +23,40 @@ on:
2323
type: choice
2424
options:
2525
- "erc20"
26+
- "dex"
2627
- "synthetics"
2728
- "all"
2829
batch_size:
2930
description: "Batch sizes (# FHE operations executed per batch)"
3031
required: true
3132
type: string
3233
default: "5000"
34+
scheduling_policy:
35+
description: "Scheduling policy"
36+
required: true
37+
type: choice
38+
options:
39+
- "MAX_PARALLELISM"
40+
- "MAX_LOCALITY"
41+
- "LOOP"
42+
- "FINE_GRAIN"
43+
optimization_target:
44+
description: "Optimization target"
45+
required: false
46+
type: choice
47+
options:
48+
- "throughput"
49+
- "latency"
50+
default: "throughput"
51+
benchmark_type:
52+
description: "Benchmark type"
53+
required: false
54+
type: choice
55+
options:
56+
- "ALL"
57+
- "THROUGHPUT"
58+
- "LATENCY"
59+
default: "all"
3360

3461
env:
3562
CARGO_TERM_COLOR: always
@@ -124,7 +151,7 @@ jobs:
124151
{
125152
echo "BENCH_DATE=$(date --iso-8601=seconds)";
126153
echo "COMMIT_DATE=$(git --no-pager show -s --format=%cd --date=iso8601-strict ${{ github.sha }})";
127-
echo "COMMIT_HASH=$(git describe --tags --dirty)";
154+
echo "COMMIT_HASH=$(git rev-parse HEAD)";
128155
} >> "${GITHUB_ENV}"
129156
130157
- name: Install rust
@@ -173,7 +200,7 @@ jobs:
173200
174201
- name: Run benchmarks on GPU
175202
run: |
176-
BENCHMARK_BATCH_SIZE="${{ inputs.batch_size }}" make "benchmark_${{ inputs.benchmarks }}_gpu"
203+
BENCHMARK_BATCH_SIZE="${{ inputs.batch_size }}" FHEVM_DF_SCHEDULE="${{ inputs.scheduling_policy }}" BENCHMARK_TYPE="${{ inputs.benchmark_type }}" OPTIMIZATION_TARGET="${{ inputs.optimization_target }}" make -e "benchmark_${{ inputs.benchmarks }}_gpu"
177204
working-directory: coprocessor/fhevm-engine/coprocessor
178205

179206
- name: Parse results
@@ -187,7 +214,8 @@ jobs:
187214
--commit-date "${{ env.COMMIT_DATE }}" \
188215
--bench-date "${{ env.BENCH_DATE }}" \
189216
--walk-subdirs \
190-
--name-suffix "operation_batch_size_${{ inputs.batch_size }}"
217+
--crate "coprocessor/fhevm-engine/coprocessor" \
218+
--name-suffix "operation_batch_size_${{ inputs.batch_size }}-schedule_${{ inputs.scheduling_policy }}-optimization_target_${{ inputs.optimization_target }}"
191219
192220
- name: Upload parsed results artifact
193221
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02

‎ci/benchmark_parser.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,7 @@
9999
parser.add_argument(
100100
"--crate",
101101
dest="crate",
102-
default="fhevm-engine/coprocessor",
102+
default="coprocessor/fhevm-engine/coprocessor",
103103
help="Crate for which benchmarks have run",
104104
)
105105

‎coprocessor/fhevm-engine/coprocessor/.sqlx/query-514d2ee9ff416b42cc48b33df6a2ef5bb25f3ed9abdd2a03957d42252812278a.json‎

Lines changed: 46 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎coprocessor/fhevm-engine/coprocessor/.sqlx/query-9d7dd33f60a053cf0f7e9a480d32ae6aaf684011580f0534afb25e0dd39d50a9.json‎

Lines changed: 0 additions & 46 deletions
This file was deleted.

‎coprocessor/fhevm-engine/coprocessor/Cargo.toml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,9 @@ scheduler = { path = "../scheduler" }
5050
[features]
5151
nightly-avx512 = ["tfhe/nightly-avx512"]
5252
gpu = ["tfhe/gpu", "scheduler/gpu", "fhevm-engine-common/gpu"]
53+
bench = []
54+
latency = []
55+
throughput = []
5356

5457
[dev-dependencies]
5558
testcontainers = { workspace = true }
@@ -81,3 +84,8 @@ harness = false
8184
name = "synthetics"
8285
path = "benches/synthetics.rs"
8386
harness = false
87+
88+
[[bench]]
89+
name = "dex"
90+
path = "benches/dex.rs"
91+
harness = false

‎coprocessor/fhevm-engine/coprocessor/Makefile‎

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11

22
DB_URL ?= DATABASE_URL=postgres://postgres:postgres@127.0.0.1:5432/coprocessor
3+
OPTIMIZATION_TARGET ?= throughput
34

45
.PHONY: build
56
build:
@@ -41,25 +42,33 @@ clean_run:
4142

4243
.PHONY: benchmark_erc20_gpu # Run ECR20 transfer benchmarks on GPU
4344
benchmark_erc20_gpu:
44-
RUSTFLAGS="-C target-cpu=native" $(DB_URL) cargo +nightly bench --bench erc20 --features=gpu --
45+
RUSTFLAGS="-C target-cpu=native" $(DB_URL) cargo +nightly bench --bench erc20 --features=gpu,bench,$(OPTIMIZATION_TARGET) --
46+
47+
.PHONY: benchmark_dex_gpu # Run DEX benchmarks on GPU
48+
benchmark_dex_gpu:
49+
RUSTFLAGS="-C target-cpu=native" $(DB_URL) cargo +nightly bench --bench dex --features=gpu,bench,$(OPTIMIZATION_TARGET) --
4550

4651
.PHONY: benchmark_synthetics_gpu # Run all benchmarks on GPU
4752
benchmark_synthetics_gpu:
48-
RUSTFLAGS="-C target-cpu=native" $(DB_URL) cargo +nightly bench --bench synthetics --features=gpu --
53+
RUSTFLAGS="-C target-cpu=native" $(DB_URL) cargo +nightly bench --bench synthetics --features=gpu,bench,$(OPTIMIZATION_TARGET) --
4954

5055
.PHONY: benchmark_all_gpu # Run all benchmarks on GPU
5156
benchmark_all_gpu:
52-
RUSTFLAGS="-C target-cpu=native" $(DB_URL) cargo +nightly bench --bench erc20 --bench synthetics --features=gpu --
57+
RUSTFLAGS="-C target-cpu=native" $(DB_URL) cargo +nightly bench --bench erc20 --bench dex --bench synthetics --features=gpu,bench,$(OPTIMIZATION_TARGET) --
5358

5459

5560
.PHONY: benchmark_erc20_cpu # Run ECR20 transfer benchmarks on CPU
5661
benchmark_erc20_cpu:
57-
RUSTFLAGS="-C target-cpu=native" $(DB_URL) cargo +nightly bench --bench erc20 --
62+
RUSTFLAGS="-C target-cpu=native" $(DB_URL) cargo +nightly bench --bench erc20 --features=bench --
63+
64+
.PHONY: benchmark_dex_cpu # Run DEX benchmarks on CPU
65+
benchmark_dex_cpu:
66+
RUSTFLAGS="-C target-cpu=native" $(DB_URL) cargo +nightly bench --bench dex --features=bench --
5867

5968
.PHONY: benchmark_synthetics_cpu # Run all benchmarks on CPU
6069
benchmark_synthetics_cpu:
61-
RUSTFLAGS="-C target-cpu=native" $(DB_URL) cargo +nightly bench --bench synthetics --
70+
RUSTFLAGS="-C target-cpu=native" $(DB_URL) cargo +nightly bench --bench synthetics --features=bench --
6271

6372
.PHONY: benchmark_all_cpu # Run all benchmarks on CPU
6473
benchmark_all_cpu:
65-
RUSTFLAGS="-C target-cpu=native" $(DB_URL) cargo +nightly bench --bench erc20 --bench synthetics --
74+
RUSTFLAGS="-C target-cpu=native" $(DB_URL) cargo +nightly bench --bench erc20 --bench dex --bench synthetics --features=bench --

0 commit comments

Comments
 (0)