Skip to content

Commit 2668fc7

Browse files
jtguibasjtguibas
authored
chore: readme updates (#86)
* update banner * latest changes * README * latest oss changes * getting ready for oss * feat: test docker builds [docker] * testing more [docker] * fix docker issues [docker] * cargo fmt [docker] * new docker build system [docker] * changes
1 parent 643ccf8 commit 2668fc7

File tree

40 files changed

+538
-559
lines changed

40 files changed

+538
-559
lines changed

‎.github/assets/banner.png‎

-33.2 KB
Binary file not shown.

‎.github/assets/image.png‎

135 KB
Loading

‎.github/workflows/docker.yml‎

Lines changed: 99 additions & 212 deletions
Original file line numberDiff line numberDiff line change
@@ -1,212 +1,99 @@
1-
# name: Build and Push Images
2-
3-
# on:
4-
# push:
5-
# branches:
6-
# - '**'
7-
# paths:
8-
# - 'workflows/**'
9-
# - 'crates/**'
10-
# - 'bin/**'
11-
# - 'Dockerfile.gpu'
12-
# - 'Dockerfile.cpu'
13-
# - 'Cargo.toml'
14-
# merge_group:
15-
16-
# env:
17-
# ECR_REPOSITORY: public.ecr.aws/succinct-labs/prover
18-
19-
# jobs:
20-
# build-amd64:
21-
# runs-on:
22-
# [
23-
# 'runs-on',
24-
# 'runner=32cpu-linux-x64',
25-
# 'run-id=${{ github.run_id }}',
26-
# 'hdd=41',
27-
# 'spot=false',
28-
# 'tag=gpu',
29-
# 'disk=large',
30-
# ]
31-
# steps:
32-
# - name: Add SHORT_SHA env property with commit short sha
33-
# run: echo "SHORT_SHA=`echo ${GITHUB_SHA} | cut -c1-7`" >> $GITHUB_ENV
34-
35-
# # https://github.com/orgs/community/discussions/25678
36-
# - name: Delete huge unnecessary tools folder
37-
# run: |
38-
# df -h
39-
# sudo rm -rf /opt/hostedtoolcache
40-
# sudo rm -rf /usr/share/dotnet
41-
# sudo rm -rf /usr/local/share/boost
42-
# sudo rm -rf "$AGENT_TOOLSDIRECTORY"
43-
# df -h
44-
45-
# - name: Checkout repo
46-
# uses: actions/checkout@v4
47-
# with:
48-
# submodules: true
49-
50-
# - name: Setup CI
51-
# uses: ./.github/actions/setup
52-
53-
# - name: Configure AWS credentials
54-
# uses: 'aws-actions/configure-aws-credentials@v1'
55-
# with:
56-
# aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
57-
# aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
58-
# aws-region: ${{ secrets.AWS_REGION }}
59-
60-
# - name: Set up Docker
61-
# uses: docker/setup-buildx-action@v3
62-
# with:
63-
# platforms: linux/amd64,linux/arm64
64-
65-
# - name: Set up NVIDIA Container Toolkit
66-
# run: |
67-
# distribution=$(. /etc/os-release;echo $ID$VERSION_ID)
68-
# curl -s -L https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add -
69-
# curl -s -L https://nvidia.github.io/nvidia-docker/$distribution/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list
70-
# sudo apt-get update && sudo apt-get install -y nvidia-container-toolkit
71-
# sudo systemctl restart docker
72-
73-
# - name: Login to Amazon ECR Public
74-
# uses: aws-actions/amazon-ecr-login@v2
75-
# with:
76-
# registry-type: public
77-
78-
# - name: Build and Push AMD64 GPU Image
79-
# uses: docker/build-push-action@v5
80-
# with:
81-
# context: .
82-
# file: ./Dockerfile.gpu
83-
# platforms: linux/amd64
84-
# push: true
85-
# tags: ${{ env.ECR_REPOSITORY }}-gpu:${{ env.SHORT_SHA }}-amd64
86-
# cache-from: type=gha
87-
# cache-to: type=gha,mode=max
88-
89-
# - name: Build and Push AMD64 CPU Image
90-
# uses: docker/build-push-action@v5
91-
# with:
92-
# context: .
93-
# file: ./Dockerfile.cpu
94-
# platforms: linux/amd64
95-
# push: true
96-
# tags: ${{ env.ECR_REPOSITORY }}-cpu:${{ env.SHORT_SHA }}-amd64
97-
# cache-from: type=gha
98-
# cache-to: type=gha,mode=max
99-
100-
# build-arm64:
101-
# runs-on:
102-
# [
103-
# 'runs-on',
104-
# 'runner=32cpu-linux-arm64',
105-
# 'run-id=${{ github.run_id }}',
106-
# 'hdd=41',
107-
# 'spot=false',
108-
# 'tag=gpu',
109-
# 'disk=large',
110-
# ]
111-
# steps:
112-
# - name: Add SHORT_SHA env property with commit short sha
113-
# run: echo "SHORT_SHA=`echo ${GITHUB_SHA} | cut -c1-7`" >> $GITHUB_ENV
114-
115-
# - name: Delete huge unnecessary tools folder
116-
# run: |
117-
# df -h
118-
# sudo rm -rf /opt/hostedtoolcache
119-
# sudo rm -rf /usr/share/dotnet
120-
# sudo rm -rf /usr/local/share/boost
121-
# sudo rm -rf "$AGENT_TOOLSDIRECTORY"
122-
# df -h
123-
124-
# - name: Checkout repo
125-
# uses: actions/checkout@v4
126-
# with:
127-
# submodules: true
128-
129-
# - name: Setup CI
130-
# uses: ./.github/actions/setup
131-
132-
# - name: Configure AWS credentials
133-
# uses: 'aws-actions/configure-aws-credentials@v1'
134-
# with:
135-
# aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
136-
# aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
137-
# aws-region: ${{ secrets.AWS_REGION }}
138-
139-
# - name: Set up Docker
140-
# uses: docker/setup-buildx-action@v3
141-
142-
# - name: Set up NVIDIA Container Toolkit
143-
# run: |
144-
# distribution=$(. /etc/os-release;echo $ID$VERSION_ID)
145-
# curl -s -L https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add -
146-
# curl -s -L https://nvidia.github.io/nvidia-docker/$distribution/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list
147-
# sudo apt-get update && sudo apt-get install -y nvidia-container-toolkit
148-
# sudo systemctl restart docker
149-
150-
# - name: Login to Amazon ECR Public
151-
# uses: aws-actions/amazon-ecr-login@v2
152-
# with:
153-
# registry-type: public
154-
155-
# - name: Build and Push ARM64 GPU Image
156-
# uses: docker/build-push-action@v5
157-
# with:
158-
# context: .
159-
# file: ./Dockerfile.gpu
160-
# platforms: linux/arm64
161-
# push: true
162-
# tags: ${{ env.ECR_REPOSITORY }}-gpu:${{ env.SHORT_SHA }}-arm64
163-
# cache-from: type=gha
164-
# cache-to: type=gha,mode=max
165-
166-
# - name: Build and Push ARM64 CPU Image
167-
# uses: docker/build-push-action@v5
168-
# with:
169-
# context: .
170-
# file: ./Dockerfile.cpu
171-
# platforms: linux/arm64
172-
# push: true
173-
# tags: ${{ env.ECR_REPOSITORY }}-cpu:${{ env.SHORT_SHA }}-arm64
174-
# cache-from: type=gha
175-
# cache-to: type=gha,mode=max
176-
177-
# create-manifest:
178-
# needs: [build-amd64, build-arm64]
179-
# runs-on: ubuntu-latest
180-
# steps:
181-
# - name: Add SHORT_SHA env property with commit short sha
182-
# run: echo "SHORT_SHA=`echo ${GITHUB_SHA} | cut -c1-7`" >> $GITHUB_ENV
183-
184-
# - name: Configure AWS credentials
185-
# uses: 'aws-actions/configure-aws-credentials@v1'
186-
# with:
187-
# aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
188-
# aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
189-
# aws-region: ${{ secrets.AWS_REGION }}
190-
191-
# - name: Login to Amazon ECR Public
192-
# uses: aws-actions/amazon-ecr-login@v2
193-
# with:
194-
# registry-type: public
195-
196-
# - name: Create and push GPU manifest
197-
# run: |
198-
# docker buildx imagetools create -t ${{ env.ECR_REPOSITORY }}-gpu:${{ env.SHORT_SHA }} \
199-
# ${{ env.ECR_REPOSITORY }}-gpu:${{ env.SHORT_SHA }}-amd64 \
200-
# ${{ env.ECR_REPOSITORY }}-gpu:${{ env.SHORT_SHA }}-arm64
201-
# docker buildx imagetools create -t ${{ env.ECR_REPOSITORY }}-gpu:latest \
202-
# ${{ env.ECR_REPOSITORY }}-gpu:${{ env.SHORT_SHA }}-amd64 \
203-
# ${{ env.ECR_REPOSITORY }}-gpu:${{ env.SHORT_SHA }}-arm64
204-
205-
# - name: Create and push CPU manifest
206-
# run: |
207-
# docker buildx imagetools create -t ${{ env.ECR_REPOSITORY }}-cpu:${{ env.SHORT_SHA }} \
208-
# ${{ env.ECR_REPOSITORY }}-cpu:${{ env.SHORT_SHA }}-amd64 \
209-
# ${{ env.ECR_REPOSITORY }}-cpu:${{ env.SHORT_SHA }}-arm64
210-
# docker buildx imagetools create -t ${{ env.ECR_REPOSITORY }}-cpu:latest \
211-
# ${{ env.ECR_REPOSITORY }}-cpu:${{ env.SHORT_SHA }}-amd64 \
212-
# ${{ env.ECR_REPOSITORY }}-cpu:${{ env.SHORT_SHA }}-arm64
1+
name: Build and Push Images
2+
3+
on:
4+
release:
5+
types: [published]
6+
push:
7+
branches:
8+
- '**'
9+
10+
env:
11+
ECR_REPOSITORY: public.ecr.aws/succinct-labs/spn-node
12+
13+
jobs:
14+
build:
15+
if: github.event_name == 'release' || (github.event_name == 'push' && contains(github.event.head_commit.message, '[docker]'))
16+
runs-on:
17+
[
18+
'runs-on',
19+
'runner=32cpu-linux-x64',
20+
'run-id=${{ github.run_id }}',
21+
'hdd=41',
22+
'spot=false',
23+
'tag=gpu',
24+
'disk=large',
25+
]
26+
steps:
27+
- name: Add SHORT_SHA env property with commit short sha
28+
run: echo "SHORT_SHA=`echo ${GITHUB_SHA} | cut -c1-7`" >> $GITHUB_ENV
29+
30+
# https://github.com/orgs/community/discussions/25678
31+
- name: Delete huge unnecessary tools folder
32+
run: |
33+
df -h
34+
sudo rm -rf /opt/hostedtoolcache
35+
sudo rm -rf /usr/share/dotnet
36+
sudo rm -rf /usr/local/share/boost
37+
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
38+
df -h
39+
40+
- name: Checkout repo
41+
uses: actions/checkout@v4
42+
with:
43+
submodules: true
44+
45+
- name: Setup CI
46+
uses: ./.github/actions/setup
47+
48+
- name: Configure AWS credentials
49+
uses: 'aws-actions/configure-aws-credentials@v1'
50+
with:
51+
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
52+
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
53+
aws-region: ${{ secrets.AWS_REGION }}
54+
55+
- name: Set up Docker
56+
uses: docker/setup-buildx-action@v3
57+
58+
- name: Set up NVIDIA Container Toolkit
59+
run: |
60+
distribution=$(. /etc/os-release;echo $ID$VERSION_ID)
61+
curl -s -L https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add -
62+
curl -s -L https://nvidia.github.io/nvidia-docker/$distribution/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list
63+
sudo apt-get update && sudo apt-get install -y nvidia-container-toolkit
64+
sudo systemctl restart docker
65+
66+
- name: Login to Amazon ECR Public
67+
uses: aws-actions/amazon-ecr-login@v2
68+
with:
69+
registry-type: public
70+
71+
- name: Build and Push GPU Image
72+
uses: docker/build-push-action@v5
73+
with:
74+
context: .
75+
file: ./Dockerfile
76+
target: gpu
77+
platforms: linux/amd64
78+
push: true
79+
tags: |
80+
${{ env.ECR_REPOSITORY }}:${{ env.SHORT_SHA }}-gpu
81+
${{ env.ECR_REPOSITORY }}:latest-gpu
82+
${{ github.event_name == 'release' && format('{0}:{1}-gpu', env.ECR_REPOSITORY, github.ref_name) || '' }}
83+
cache-from: type=gha
84+
cache-to: type=gha,mode=max
85+
86+
- name: Build and Push CPU Image
87+
uses: docker/build-push-action@v5
88+
with:
89+
context: .
90+
file: ./Dockerfile
91+
target: cpu
92+
platforms: linux/amd64
93+
push: true
94+
tags: |
95+
${{ env.ECR_REPOSITORY }}:${{ env.SHORT_SHA }}-cpu
96+
${{ env.ECR_REPOSITORY }}:latest-cpu
97+
${{ github.event_name == 'release' && format('{0}:{1}-cpu', env.ECR_REPOSITORY, github.ref_name) || '' }}
98+
cache-from: type=gha
99+
cache-to: type=gha,mode=max

‎Cargo.toml‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -9,15 +9,15 @@ categories = ["cryptography"]
99

1010
[workspace]
1111
members = [
12-
"bin/cli",
12+
"bin/node",
13+
"crates/network/artifacts",
14+
"crates/network/rpc",
15+
"crates/network/utils",
16+
"crates/node/calibrator",
17+
"crates/node/core",
18+
"crates/node/metrics",
1319
"crates/types/artifact",
1420
"crates/types/network",
15-
"crates/utils",
16-
"crates/node",
17-
"crates/artifacts",
18-
"crates/calibrator",
19-
"crates/rpc",
20-
"crates/metrics",
2121
"crates/vapp",
2222
"programs/examples/fibonacci",
2323
"programs/examples/is-prime",
@@ -28,13 +28,13 @@ resolver = "2"
2828

2929
[workspace.dependencies]
3030
# spn
31-
spn-artifacts = { path = "crates/artifacts" }
31+
spn-artifacts = { path = "crates/network/artifacts" }
32+
spn-rpc = { path = "crates/network/rpc" }
33+
spn-utils = { path = "crates/network/utils" }
34+
spn-calibrator = { path = "crates/node/calibrator" }
35+
spn-node-core = { path = "crates/node/core" }
3236
spn-artifact-types = { path = "crates/types/artifact" }
33-
spn-calibrator = { path = "crates/calibrator" }
3437
spn-network-types = { path = "crates/types/network" }
35-
spn-node = { path = "crates/node" }
36-
spn-utils = { path = "crates/utils" }
37-
spn-rpc = { path = "crates/rpc" }
3838
spn-vapp-core = { path = "crates/vapp" }
3939

4040
# sp1

0 commit comments

Comments
 (0)