Skip to content

Commit a54c848

Browse files
milahumilahu
committed
fail: use dynamic bins
dynamic bins fail because the interpreter ld-linux-x86-64.so.2 is not available from /nix/store and we cannot do patchelf --set-interpreter '$ORIGIN/../lib/ld-linux-x86-64.so.2' so we must use static bins from pkgsStatic
1 parent 9ded0d3 commit a54c848

File tree

3 files changed

+326
-49
lines changed

3 files changed

+326
-49
lines changed

‎builder.sh

+237-21
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,216 @@ busybox=@busybox@
1111
caBundleZstd=@caBundleZstd@
1212
storeTar=@storeTar@
1313
bundledExe=@bundledExe@
14+
patchelf=@patchelf@
15+
16+
set -x
17+
18+
# https://github.com/NixOS/nixpkgs/blob/e101e9465d47dd7a7eb95b0477ae67091c02773c/lib/strings.nix#L1716
19+
function removePrefix() {
20+
local prefix="$1"
21+
local str="$2"
22+
local preLen=${#prefix}
23+
if [[ "${str:0:$preLen}" == "$prefix" ]]; then
24+
echo "${str:$preLen}"
25+
else
26+
echo "$str"
27+
fi
28+
}
29+
30+
add_file_list=()
31+
32+
# in stage1, we only have bash and coreutils
33+
# so we cannot unzip busybox, so we get the file offsets
34+
# se we can unpack files with "tail" and "head" commands
35+
# tail -c+$((offset + 1)) $zip | head -c$size
36+
37+
stage1_file_path_list=()
38+
stage1_file_offset_list=()
39+
stage1_file_size_list=()
40+
41+
# add a stage1 executable file and its dependencies (libraries)
42+
function add_stage1_bin() {
43+
local bin="$1"
44+
if add_file -1 "$bin"; then
45+
echo
46+
echo "adding binary: $bin"
47+
add_stage1_libs "$bin"
48+
echo
49+
# exit 1 # debug
50+
fi
51+
}
52+
53+
# add stage1 library files
54+
function add_stage1_libs() {
55+
local bin="$1"
56+
# ldd "$bin" # debug
57+
for lib in $(ldd "$bin" 2>/dev/null | grep -oE '/nix/store/[^ ]+'); do
58+
# echo " adding library: $lib"
59+
if add_file -1 "$lib"; then
60+
# recurse: add dependencies of lib
61+
add_stage1_libs "$lib"
62+
fi
63+
done
64+
}
65+
66+
function add_stage1_file_offset_size() {
67+
local file="$1"
68+
#local size=$(stat -c %s "$file")
69+
local size=$(stat -c %s -L "$file") # dereference symlinks
70+
local hash=$(sha1sum "$file" | head -c40)
71+
# locate the first 1000 bytes of file in the zip archive
72+
local grep_size=1000
73+
if ((grep_size > size)); then grep_size=$size; fi
74+
local skip=0
75+
if [ ${#stage1_file_offset_list[@]} != 0 ]; then
76+
# start search from previous file
77+
# skip bytes until previous offset + size
78+
skip=$((${stage1_file_offset_list[-1]} + ${stage1_file_size_list[-1]}))
79+
fi
80+
# exploit that files are appended
81+
while read offset; do
82+
offset=$((offset / 2)) # hex to bin
83+
offset=$((skip + offset))
84+
# debug
85+
# echo "zipped file header:"
86+
# tail -c+$((offset + 1)) "$out"/bin/nix-portable.zip | head -c1000 | basenc --base16 -w0 || true
87+
# verify offset
88+
# FIXME tail: error writing 'standard output': Broken pipe
89+
set +o pipefail
90+
hash2=$(tail -c+$((offset + 1)) "$out"/bin/nix-portable.zip | head -c"$size" |
91+
sha1sum - | head -c40 || true)
92+
set -o pipefail
93+
if [ "$hash" = "$hash2" ]; then
94+
stage1_file_offset_list+=("$offset")
95+
stage1_file_size_list+=("$size")
96+
return
97+
fi
98+
done < <(
99+
# bin to hex
100+
# rg is 25x faster than grep
101+
# basenc is 10x faster than xxd
102+
tail -c+$((skip + 1)) "$out"/bin/nix-portable.zip | basenc --base16 -w0 |
103+
rg -boF $(head -c"$grep_size" "$file" | basenc --base16 -w0) |
104+
cut -d: -f1
105+
)
106+
echo "error: file was not found in zip archive: $file"
107+
exit 1
108+
}
109+
110+
defer_zip=false # dt: 10.5362 # TODO remove
111+
defer_zip=true # dt: 3.20238
112+
113+
function add_file() {
114+
local is_stage1=false
115+
if [ "$1" = "-1" ]; then is_stage1=true; shift; fi
116+
local file="$1"
117+
if $is_stage1; then
118+
# change file path to build a FHS filesystem layout for stage1
119+
local file2="stage1/${file#/*/*/*/*}"
120+
if ! [ -e "$file2" ]; then
121+
mkdir -p "${file2%/*}"
122+
cp -Lp "$file" "$file2"
123+
chmod +w "$file2"
124+
fi
125+
file="$file2"
126+
# set relative rpath to create relocatable bins and libs
127+
$patchelf/bin/patchelf --set-rpath '$ORIGIN/../lib' "$file"
128+
# FIXME patch interpreter paths like /nix/store/rmy663w9p7xb202rcln4jjzmvivznmz8-glibc-2.40-66/lib/ld-linux-x86-64.so.2
129+
# no. this is not working
130+
# https://stackoverflow.com/questions/48452793/using-origin-to-specify-the-interpreter-in-elf-binaries-isnt-working
131+
#$patchelf/bin/patchelf --set-interpreter '$ORIGIN/../lib' "$file"
132+
fi
133+
if ! $defer_zip; then
134+
# dont defer the zip command = add one file now
135+
# check if file exists in zip archive
136+
if unzip -p "$out"/bin/nix-portable.zip "$(removePrefix "/" "$file")" 2>/dev/null | head -c0; then
137+
return 1
138+
fi
139+
$zip "$out"/bin/nix-portable.zip "$file"
140+
if $is_stage1; then
141+
stage1_file_path_list+=("$file")
142+
add_stage1_file_offset_size "$file"
143+
fi
144+
else
145+
# defer the zip command = add all files later
146+
# check if file exists in zip archive
147+
local f
148+
for f in "${add_file_list[@]}"; do
149+
if [ "$f" = "$file" ]; then
150+
return 1
151+
fi
152+
done
153+
echo " adding file: $file"
154+
add_file_list+=("$file")
155+
if $is_stage1; then stage1_file_path_list+=("$file"); fi
156+
fi
157+
}
158+
159+
function dump_array() {
160+
local name=$1
161+
local -n arr=$1
162+
echo "$name=("
163+
local val
164+
for val in "${arr[@]}"; do
165+
printf "%q\n" "$val"
166+
done
167+
echo ")"
168+
}
169+
170+
function assert_equal_array_size() {
171+
local -n name1=$1
172+
local -n arr1=$1
173+
shift
174+
local size1=${#arr1[@]}
175+
while (($# > 0)); do
176+
local -n name2=$1
177+
local -n arr2=$1
178+
shift
179+
local size2=${#arr2[@]}
180+
if [ "$size1" != "$size2" ]; then
181+
echo "error: $name2 should have $size1 values, has $size2"
182+
return 1
183+
fi
184+
done
185+
}
186+
187+
function add_file_done() {
188+
if $defer_zip; then
189+
$zip "$out"/bin/nix-portable.zip "${add_file_list[@]}"
190+
add_file_list=()
191+
local file
192+
for file in "${stage1_file_path_list[@]}"; do
193+
add_stage1_file_offset_size "$file"
194+
done
195+
fi
196+
rm -rf stage1
197+
# check internal consistency
198+
assert_equal_array_size \
199+
stage1_file_path_list \
200+
stage1_file_offset_list \
201+
stage1_file_size_list
202+
# store file offsets in the zip archive
203+
{
204+
dump_array stage1_file_path_list
205+
dump_array stage1_file_offset_list
206+
dump_array stage1_file_size_list
207+
} >stage1_files.sh
208+
touch -d1970-01-01 stage1_files.sh
209+
stage1_file_path_list=()
210+
stage1_file_offset_list=()
211+
stage1_file_size_list=()
212+
# add_file -1 stage1_files.sh
213+
$zip "$out"/bin/nix-portable.zip stage1_files.sh
214+
add_stage1_file_offset_size stage1_files.sh
215+
rm stage1_files.sh
216+
stage1_files_sh_offset=${stage1_file_offset_list[0]}
217+
stage1_files_sh_size=${stage1_file_size_list[0]}
218+
stage1_file_path_list=()
219+
stage1_file_offset_list=()
220+
stage1_file_size_list=()
221+
sed -i "0,/@stage1_files_sh_offset@/s//$(printf "%-24s" "$stage1_files_sh_offset")/; \
222+
0,/@stage1_files_sh_size@/s//$(printf "%-22s" "$stage1_files_sh_size")/" "$out"/bin/nix-portable.zip
223+
}
14224

15225
mkdir -p "$out"/bin
16226
cp $runtimeScript "$out"/bin/nix-portable.zip
@@ -86,27 +296,33 @@ unzip -vl "$out"/bin/nix-portable.zip
86296

87297
zip="$zip/bin/zip -0"
88298

89-
$zip "$out"/bin/nix-portable.zip $busybox/bin/busybox
90-
91-
# we cannot unzip busybox, so we need offset and size
92-
# locate the first 1000 bytes of busybox in the zip archive
93-
busyboxOffset=$(
94-
cat "$out"/bin/nix-portable.zip | xxd -p -c0 |
95-
grep -bo -m1 $(head -c1000 $busybox/bin/busybox | xxd -p -c0) |
96-
cut -d: -f1
97-
)
98-
# hex to bin
99-
busyboxOffset=$((busyboxOffset / 2))
100-
busyboxSize=$(stat -c %s busybox/bin/busybox)
101-
sed -i "0,/@busyboxOffset@/s//$(printf "%-15s" $busyboxOffset)/; \
102-
0,/@busyboxSize@/s//$(printf "%-13s" "$busyboxSize")/" "$out"/bin/nix-portable.zip
103-
104-
$zip "$out"/bin/nix-portable.zip $bubblewrap/bin/bwrap
105-
$zip "$out"/bin/nix-portable.zip $nix/bin/nix
106-
$zip "$out"/bin/nix-portable.zip $proot/bin/proot
107-
$zip "$out"/bin/nix-portable.zip $zstd/bin/zstd
108-
$zip "$out"/bin/nix-portable.zip $storeTar/tar
109-
$zip "$out"/bin/nix-portable.zip $caBundleZstd
299+
# we cannot unzip busybox, so we need offset and size of all needed files (bins and libs)
300+
# add_file does not work here
301+
#$zip $out/bin/nix-portable.zip $busybox/bin/busybox
302+
add_stage1_bin $busybox/bin/busybox
303+
304+
t1=$(date +%s.%N)
305+
306+
#add_stage1_bin $bubblewrap/bin/bwrap
307+
#add_stage1_bin $proot/bin/proot
308+
add_stage1_bin $zstd/bin/zstd
309+
#add_stage1_bin $nix/bin/nix # 150M result/bin/nix-portable
310+
# # nix needs too many libs, so we dont use add_stage1_bin
311+
# add_file $nix/bin/nix # 99M result/bin/nix-portable
312+
313+
# TODO move stage1_files.sh up in the zip archive for faster access
314+
#stage1_file_done
315+
316+
add_file $storeTar/closureInfo/store-paths
317+
add_file $storeTar/tar
318+
add_file $caBundleZstd
319+
320+
add_file_done
321+
322+
t2=$(date +%s.%N)
323+
dt=$(echo "$t1" "$t2" | awk '{ print ($2 - $1) }')
324+
echo "dt: $dt"
325+
# exit 1
110326

111327
# create fingerprint
112328
fp=$(sha256sum "$out"/bin/nix-portable.zip | head -c64)

‎default.nix

+6
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,9 @@ with builtins;
77
unixtools,
88
substituteAll,
99
lib,
10+
glibc,
11+
ripgrep,
12+
patchelf,
1013
cacert,
1114
pkgs,
1215
# no. pkgsStatic.nix and pkgsStatic.proot are not cached
@@ -103,13 +106,16 @@ let
103106
busybox
104107
caBundleZstd
105108
storeTar
109+
patchelf
106110
;
107111
};
108112

109113
nixPortable = pkgs.runCommand pname {
110114
nativeBuildInputs = [
111115
unixtools.xxd
112116
unzip
117+
glibc # ldd
118+
ripgrep # rg
113119
];
114120
}
115121
''

0 commit comments

Comments
 (0)