Skip to content

Commit 13bcefe

Browse files
isegall-dastephencompall-DA
isegall-da
and
stephencompall-DA
authored
Support running static tests on gh-hosted runners (hyperledger-labs#1668)
Signed-off-by: Itai Segall <itai.segall@digitalasset.com> Co-authored-by: Stephen Compall <stephen.compall@digitalasset.com>
1 parent 6d2ec21 commit 13bcefe

File tree

9 files changed

+310
-255
lines changed

9 files changed

+310
-255
lines changed

‎.github/actions/nix/setup_nix/action.yml‎

Lines changed: 235 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,235 @@
1+
name: "Setup Nix"
2+
description: "Setup Nix"
3+
inputs:
4+
artifactory_user:
5+
description: "The Artifactory user"
6+
required: true
7+
artifactory_password:
8+
description: "The Artifactory password"
9+
required: true
10+
oss_only:
11+
description: "Restrict upstream dependencies (e.g. Canton) to OSS versions (the equivalent of OSS_ONLY=1 in local checkouts)"
12+
default: "false"
13+
required: false
14+
cache_version:
15+
description: "The cache version"
16+
required: true
17+
should_save:
18+
description: "If the nix cache should be saved"
19+
# this should be run just from one job to ensure we avoid multi write conflicts, which makes everything worse
20+
default: "false"
21+
should_save_gcp:
22+
description: "If the nix cache should be saved to the public GCP bucket"
23+
default: "false"
24+
upload_workload_identity_provider:
25+
description: "The workload identity provider to use for uploading the cache"
26+
required: false
27+
default: ""
28+
upload_service_account:
29+
description: "The service account to use for uploading the cache"
30+
required: false
31+
default: ""
32+
33+
runs:
34+
using: "composite"
35+
steps:
36+
- name: Compute cache Key
37+
id: cache_key
38+
shell: bash
39+
run: |
40+
set -euxo pipefail
41+
git ls-files nix/ | grep -v '[.]md$' | LC_ALL=C sort | xargs sha256sum -b > /tmp/nix-cache-key
42+
uname -m >> /tmp/nix-cache-key # Add architecture to the cache key
43+
echo "gh_cache_version: ${{ inputs.cache_version }}" >> /tmp/nix-cache-key # Add cache version to the cache key
44+
if [ "${{ inputs.oss_only }}" == true ]; then
45+
echo "Using OSS only dependencies"
46+
echo "oss_only: ${{ inputs.oss_only }}" >> /tmp/nix-cache-key
47+
touch /tmp/oss-only # Create a file to indicate that we are using OSS only dependencies (so we don't need to re-specifify oss_only to run_bash_command_in_nix)
48+
fi
49+
cat /tmp/nix-cache-key
50+
cache_key=($(md5sum "/tmp/nix-cache-key"))
51+
echo "cache_key=$cache_key" >> $GITHUB_ENV
52+
53+
- name: Download cache (for non-self-hosted)
54+
if: ${{ !startsWith(runner.name, 'self-hosted') }}
55+
shell: bash
56+
run: |
57+
set -euxo pipefail
58+
59+
if [ ${{ inputs.oss_only }} != 'true' ]; then
60+
echo "Must use OSS only dependencies in GitHub-hosted runners"
61+
exit 1
62+
fi
63+
64+
echo "Latest nix cache:"
65+
66+
wget -q "https://storage.googleapis.com/splice-nix-cache-public/${cache_key}.tar.gz" -O cache.tar.gz || true
67+
if [ ! -s cache.tar.gz ]; then
68+
echo "Cache not found, fetching latest instead"
69+
latest=$(curl https://storage.googleapis.com/storage/v1/b/splice-nix-cache-public/o | jq -r '.items | sort_by(.updated) | .[-1].name')
70+
wget -q "https://storage.googleapis.com/splice-nix-cache-public/${latest}" -O cache.tar.gz
71+
72+
fi
73+
74+
sudo mkdir -p /cache/nix/${cache_key}
75+
sudo tar -xzf cache.tar.gz -C /cache/nix/${cache_key}
76+
77+
- name: Restore nix
78+
id: restore_nix
79+
shell: bash
80+
run: |
81+
set -euxo pipefail
82+
sudo mkdir -p /nix/store
83+
sudo chown -R $(whoami):$(whoami) /nix
84+
if [ -f "/cache/nix/$cache_key/cached" ]; then
85+
echo "Restoring nix cache (key $cache_key)"
86+
# we use rsync here because it's simply faster to install
87+
rsync -avi /cache/nix/$cache_key/.nix-* $HOME/
88+
rsync -avi "/cache/nix/$cache_key/nix" $HOME/.config/
89+
rsync -avi "/cache/nix/$cache_key/nix_store/var/" /nix/var
90+
sudo mount --bind /cache/nix/$cache_key/nix_store/store /nix/store
91+
else
92+
sudo mkdir -p "/cache/nix/$cache_key"
93+
sudo chown $(whoami):$(whoami) "/cache/nix/$cache_key"
94+
sudo chown $(whoami):$(whoami) "/cache/nix"
95+
fi
96+
- name: Setup Nix
97+
shell: bash
98+
run: |
99+
set -exuo pipefail
100+
echo 'source ~/.nix-profile/etc/profile.d/nix.sh' > nix.rc
101+
if [[ -f ~/.config/nix/nix.conf && -f ~/.nix-profile/etc/profile.d/nix.sh ]]; then
102+
echo "nix.conf or nix.sh already exists, skipping Nix setup"
103+
exit 0
104+
else
105+
# Disabling sandbox because:
106+
# 1. It doesn't work on CircleCI (sethostname is not allowed)
107+
# 2. We don't plan to build anything, so the risk is fairly low
108+
mkdir -p ~/.config/nix
109+
if [ true ]; then
110+
cat <<EOF > ~/.config/nix/nix.conf
111+
sandbox = false
112+
netrc-file = /etc/nix/netrc
113+
extra-experimental-features = nix-command flakes
114+
substituters = file:///cache/nix/binary_cache?trusted=1 https://cache.nixos.org/
115+
trusted-substituters = file:///cache/nix/binary_cache?trusted=1
116+
trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
117+
cores = 4
118+
max-jobs = 16
119+
EOF
120+
else
121+
cat <<EOF > ~/.config/nix/nix.conf
122+
sandbox = false
123+
netrc-file = /etc/nix/netrc
124+
extra-experimental-features = nix-command flakes
125+
cores = 4
126+
max-jobs = 16
127+
EOF
128+
fi
129+
sh <(curl -fsSL --retry 8 https://releases.nixos.org/nix/nix-2.13.3/install) --no-daemon
130+
sudo mkdir -p /etc/nix
131+
sudo chmod a+rw /etc/nix
132+
if [[ "${{ inputs.oss_only }}" == true ]]; then
133+
echo "Using OSS only dependencies, not setting up Artifactory credentials"
134+
else
135+
cat <<EOF > /etc/nix/netrc
136+
machine digitalasset.jfrog.io
137+
login ${{ inputs.artifactory_user }}
138+
password ${{ inputs.artifactory_password }}
139+
EOF
140+
fi
141+
export USER=$(whoami)
142+
echo "Running nix.sh"
143+
. ~/.nix-profile/etc/profile.d/nix.sh
144+
if [[ "${{ inputs.oss_only }}" == true ]]; then
145+
target="oss"
146+
else
147+
target="default"
148+
fi
149+
nix develop path:nix#${target} -v --profile "$HOME/.nix-shell" --command echo "Done loading packages"
150+
echo "Garbage collecting to reduce cache size"
151+
nix-store --gc
152+
fi
153+
154+
- name: Invoke nix before saving cache
155+
uses: ./.github/actions/nix/run_bash_command_in_nix
156+
with:
157+
cmd: |
158+
echo "Validated nix"
159+
ls -al
160+
161+
# The nix cache does not change in the workflow, so we can save it immediately, rather than splitting it into pre-&post- steps
162+
- name: Save nix cache
163+
shell: bash
164+
if: ${{ inputs.should_save == 'true' }}
165+
run: |
166+
set -euxo pipefail
167+
echo ~
168+
chown -R $(whoami):$(whoami) ~
169+
cat /tmp/nix-cache-key
170+
if [ ! -f "/cache/nix/$cache_key/cached" ]; then
171+
echo "Saving nix"
172+
173+
sudo -v ; curl https://rclone.org/install.sh | sudo bash
174+
175+
echo "sourcing nix profile"
176+
export USER=$(whoami)
177+
. ~/.nix-profile/etc/profile.d/nix.sh
178+
179+
nix copy --all --to 'file:///cache/nix/binary_cache?trusted=1' -v
180+
181+
CLONE_COMMAND="rclone --no-update-dir-modtime --no-update-modtime --size-only --multi-thread-streams=32 --transfers=32 --ignore-existing --links --create-empty-src-dirs --fast-list --metadata --order-by name,mixed --retries 10 copy"
182+
${CLONE_COMMAND} "$HOME/" "/cache/nix/$cache_key/" --include ".nix-*/**" --include ".nix-*"
183+
${CLONE_COMMAND} $HOME/.config/nix "/cache/nix/$cache_key/nix"
184+
185+
mkdir -p "/cache/nix/$cache_key/nix_store/store"
186+
mkdir -p "/cache/nix/$cache_key/nix_store/var"
187+
188+
#requires to preserve read only during clone
189+
sudo ${CLONE_COMMAND} /nix/store/ /cache/nix/$cache_key/nix_store/store
190+
sudo ${CLONE_COMMAND} /nix/var/ "/cache/nix/$cache_key/nix_store/var"
191+
192+
echo "done" > "/cache/nix/$cache_key/cached"
193+
fi
194+
195+
- name: Check if cache already exists in GCP
196+
id: already_exists
197+
if: ${{ inputs.should_save_gcp == 'true' }}
198+
shell: bash
199+
run: |
200+
if curl -Isf https://storage.googleapis.com/splice-nix-cache-public/${cache_key}.tar.gz &> /dev/null; then
201+
echo "Cache with key ${cache_key} already exists in GCP, not uploading again"
202+
echo "already_exists=true" >> $GITHUB_OUTPUT;
203+
fi
204+
- name: Authenticate to GCP
205+
id: auth
206+
if: ${{ inputs.should_save_gcp == 'true' && steps.already_exists.outputs.already_exists != 'true' }}
207+
uses: "google-github-actions/auth@ba79af03959ebeac9769e648f473a284504d9193" #v2.1.10
208+
with:
209+
workload_identity_provider: "${{ inputs.upload_workload_identity_provider }}"
210+
service_account: "${{ inputs.upload_service_account }}"
211+
212+
- name: tar-gz the cache
213+
shell: bash
214+
if: ${{ inputs.should_save_gcp == 'true' && steps.already_exists.outputs.already_exists != 'true' }}
215+
id: prep_cache_upload
216+
run: |
217+
set -euxo pipefail
218+
echo "Compressing nix cache to /cache/nix/${cache_key}.tar.gz"
219+
mkdir -p /tmp/nix-upload
220+
221+
tar -czf "/tmp/nix-upload/${cache_key}.tar.gz" -C "/cache/nix/$cache_key" .
222+
223+
echo "Cache compressed to /tmp/nix-upload/${cache_key}.tar.gz"
224+
ls /tmp/nix-upload
225+
echo "cache_file=/tmp/nix-upload/${cache_key}.tar.gz" >> $GITHUB_OUTPUT
226+
227+
- name: Upload nix cache
228+
if: ${{ inputs.should_save_gcp == 'true' && steps.already_exists.outputs.already_exists != 'true' }}
229+
uses: google-github-actions/upload-cloud-storage@v2
230+
with:
231+
destination: splice-nix-cache-public
232+
path: "${{ steps.prep_cache_upload.outputs.cache_file }}"
233+
parent: false # upload to root of the bucket
234+
process_gcloudignore: false # no gcloud ignore file in this repo, must set this to false
235+
gzip: false # it's already gzipped

0 commit comments

Comments
 (0)