Skip to content

Commit bcb5c6e

Browse files
tamirhemotamirhemo
authored
feat: merkle tree, transpose, CI (#4)
* init * init * params * merkle tree traits * d * poseidon2_lernels * compiles * fix indices * fix index * fix kernels * yay * refactor traits * checkpoint * config * default clone implementation * update paths * fix openings * ten * refs * checkpoint * checkpoint * init fixed * dimensions * indexing * indexing * index * view mut * tensor indexing * save * works * test init * mtree * fix dir * with warmup in merkle test * transpose * transpose works * gr * transpose * alloc mut * derive cuda send * runner * remote * try * try * action * setup * better * comment out toolchain * try it * try * ff * repo * gchange settung * with * rm runs-on * try * merkle tree * try test * try * runs-on * verify cuda * different runner * back to default * runs-on * check if needed * check if needed * echo * try with apt * from installer * install from repo * setup cuda via paths * try echo cuda * try * fix cuda version * add clippy action
1 parent 9cffa8d commit bcb5c6e

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

44 files changed

+4014
-438
lines changed

‎.github/actions/setup/action.yml‎

Lines changed: 82 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,18 @@
1+
# Note: this is only compatible with Linux runners.
2+
13
name: Test setup
24
inputs:
35
pull_token:
46
description: "Token to use for private repo access"
57
required: true
8+
setup_gcc:
9+
description: "Whether to setup GCC or not"
10+
required: false
11+
default: 'true'
12+
setup_aws_cli:
13+
description: "Whether to install AWS CLI or not"
14+
required: false
15+
default: 'true'
616
runs:
717
using: "composite"
818
steps:
@@ -12,6 +22,51 @@ runs:
1222
git config --global url."https://${{ inputs.pull_token }}@github.com/".insteadOf ssh://[email protected]
1323
git config --global url."https://${{ inputs.pull_token }}@github.com".insteadOf https://github.com
1424
25+
- name: Install Go 1.22
26+
uses: actions/setup-go@v5
27+
with:
28+
go-version: "1.22"
29+
cache-dependency-path: "**/go.sum"
30+
31+
- name: Print go version
32+
shell: bash
33+
run: go version
34+
35+
- name: Check GCC version
36+
id: check-gcc
37+
shell: bash
38+
run: |
39+
if command -v gcc &> /dev/null; then
40+
echo "gcc_exists=true" >> $GITHUB_OUTPUT
41+
echo "gcc_version=$(gcc --version | head -n1 | awk '{print $NF}')" >> $GITHUB_OUTPUT
42+
else
43+
echo "gcc_exists=false" >> $GITHUB_OUTPUT
44+
fi
45+
46+
- name: Setup GCC
47+
uses: Dup4/actions-setup-gcc@v1
48+
if: inputs.setup_gcc == 'true' && steps.check-gcc.outputs.gcc_exists != 'true'
49+
with:
50+
version: latest
51+
52+
- uses: actions/setup-python@v5
53+
with:
54+
python-version: '3.10'
55+
56+
- name: Install AWS CLI v2
57+
if: inputs.setup_aws_cli == 'true'
58+
shell: bash
59+
run: |
60+
if ! command -v aws &> /dev/null; then
61+
echo "AWS CLI not found. Installing..."
62+
python3 -m pip install --user awscli
63+
echo "$HOME/.local/bin" >> $GITHUB_PATH
64+
else
65+
echo "AWS CLI is already installed."
66+
fi
67+
export PATH="$HOME/.local/bin:$PATH"
68+
aws --version
69+
1570
- name: rust-cache
1671
uses: actions/cache@v3
1772
with:
@@ -22,7 +77,7 @@ runs:
2277
~/.cargo/git/db/
2378
target/
2479
~/.rustup/
25-
key: rust-1.81.0-${{ hashFiles('**/Cargo.toml') }}
80+
key: rust-1.81.0-${{ hashFiles('**/Cargo.toml') }}-x
2681
restore-keys: rust-1.81.0-
2782

2883
- name: Setup toolchain
@@ -31,4 +86,29 @@ runs:
3186
run: |
3287
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --default-toolchain 1.81.0 -y
3388
. "$HOME/.cargo/env"
34-
echo "$HOME/.cargo/bin" >> $GITHUB_PATH
89+
echo "$HOME/.cargo/bin" >> $GITHUB_PATH
90+
91+
# install pkg-config and openssl
92+
- name: Install pkg-config and openssl
93+
shell: bash
94+
run: |
95+
if ! dpkg -s pkg-config libssl-dev &> /dev/null; then
96+
echo "pkg-config and/or libssl-dev not found. Installing..."
97+
sudo apt-get update
98+
sudo apt-get install -y pkg-config libssl-dev
99+
else
100+
echo "pkg-config and libssl-dev are already installed."
101+
fi
102+
103+
- name: Echo docker buildx version
104+
shell: bash
105+
run: docker buildx version
106+
107+
- name: Set up Docker
108+
uses: crazy-max/ghaction-setup-docker@v3
109+
110+
- name: Set up Docker Buildx
111+
uses: docker/setup-buildx-action@v3
112+
with:
113+
driver-opts: |
114+
image=public.ecr.aws/vend/moby/buildkit:buildx-stable-1

‎.github/runs-on.yml‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
images:
2+
nvidia-linux:
3+
platform: "linux"
4+
arch: "x64"
5+
ami: "ami-0a63dc9cb9e934ba3"
6+
owner: "421253708207"
7+
8+
dlami-x64:
9+
platform: "linux"
10+
arch: "x64"
11+
owner: "898082745236" # AWS
12+
name: "Deep Learning Base OSS Nvidia Driver GPU AMI (Ubuntu 22.04)*"
13+
14+
runners:
15+
gpu-nvidia:
16+
family: ["g6.4xlarge"]
17+
image: dlami-x64

‎.github/workflows/pr.yml‎

Lines changed: 41 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -18,85 +18,79 @@ concurrency:
1818
cancel-in-progress: true
1919

2020
jobs:
21-
test-x86:
22-
name: Test (x86-64)
21+
test:
22+
name: Test
2323
runs-on:
2424
[
25-
runs-on,
26-
runner=64cpu-linux-x64,
27-
spot=false,
25+
"runs-on",
26+
"family=g6.4xlarge",
27+
"hdd=200",
28+
"ami=ami-0a63dc9cb9e934ba3",
29+
"spot=false",
2830
"run-id=${{ github.run_id }}",
2931
]
3032
env:
3133
CARGO_NET_GIT_FETCH_WITH_CLI: "true"
3234
steps:
3335
- name: Checkout sources
34-
uses: actions/checkout@v4
36+
uses: actions/checkout@v1
3537

3638
- name: Setup CI
3739
uses: ./.github/actions/setup
38-
39-
- name: Run cargo check
40-
uses: actions-rs/cargo@v1
4140
with:
42-
command: check
43-
toolchain: 1.81.0
44-
args: --all-targets --all-features
41+
pull_token: ${{ secrets.PRIVATE_PULL_TOKEN }}
42+
43+
- name: Set up CUDA environment
44+
run: |
45+
echo "CUDA_HOME=/usr/local/cuda-12.6" >> $GITHUB_ENV
46+
echo "LD_LIBRARY_PATH=/usr/local/cuda-12.6/lib64:${LD_LIBRARY_PATH}" >> $GITHUB_ENV
47+
echo "/usr/local/cuda-12.6/bin" >> $GITHUB_PATH
48+
49+
- name: Confirm CUDA installation
50+
run: |
51+
echo "CUDA_HOME=$CUDA_HOME"
52+
which nvcc
4553
4654
- name: Run cargo test
4755
uses: actions-rs/cargo@v1
4856
with:
4957
command: test
50-
toolchain: 1.81.0
51-
args: --release --workspace
58+
# toolchain: 1.81.0
59+
args: --release --workspace -- --test-threads=1
5260
env:
5361
RUSTFLAGS: -Copt-level=3 -Cdebug-assertions -Coverflow-checks=y -Cdebuginfo=0 -C target-cpu=native
5462
RUST_BACKTRACE: 1
5563

56-
test-arm:
57-
name: Test (ARM)
64+
lint:
65+
name: Formatting & Clippy
5866
runs-on:
5967
[
60-
runs-on,
61-
runner=64cpu-linux-arm64,
62-
spot=false,
68+
"runs-on",
69+
"family=g6.4xlarge",
70+
"hdd=200",
71+
"ami=ami-0a63dc9cb9e934ba3",
72+
"spot=false",
6373
"run-id=${{ github.run_id }}",
6474
]
65-
env:
66-
CARGO_NET_GIT_FETCH_WITH_CLI: "true"
6775
steps:
6876
- name: Checkout sources
69-
uses: actions/checkout@v4
77+
uses: actions/checkout@v1
7078

7179
- name: Setup CI
7280
uses: ./.github/actions/setup
73-
74-
- name: Run cargo check
75-
uses: actions-rs/cargo@v1
76-
with:
77-
command: check
78-
toolchain: 1.81.0
79-
args: --all-targets --all-features
80-
81-
- name: Run cargo test
82-
uses: actions-rs/cargo@v1
8381
with:
84-
command: test
85-
toolchain: 1.81.0
86-
args: --release --workspace
87-
env:
88-
RUSTFLAGS: -Copt-level=3 -Cdebug-assertions -Coverflow-checks=y -Cdebuginfo=0 -C target-cpu=native
89-
RUST_BACKTRACE: 1
90-
91-
lint:
92-
name: Formatting & Clippy
93-
runs-on: [runs-on, runner=8cpu-linux-x64, "run-id=${{ github.run_id }}"]
94-
steps:
95-
- name: Checkout sources
96-
uses: actions/checkout@v4
82+
pull_token: ${{ secrets.PRIVATE_PULL_TOKEN }}
9783

98-
- name: Setup CI
99-
uses: ./.github/actions/setup
84+
- name: Set up CUDA environment
85+
run: |
86+
echo "CUDA_HOME=/usr/local/cuda-12.6" >> $GITHUB_ENV
87+
echo "LD_LIBRARY_PATH=/usr/local/cuda-12.6/lib64:${LD_LIBRARY_PATH}" >> $GITHUB_ENV
88+
echo "/usr/local/cuda-12.6/bin" >> $GITHUB_PATH
89+
90+
- name: Confirm CUDA installation
91+
run: |
92+
echo "CUDA_HOME=$CUDA_HOME"
93+
which nvcc
10094
10195
- name: Run cargo fmt
10296
uses: actions-rs/cargo@v1

0 commit comments

Comments
 (0)