diff --git a/README.md b/README.md index b77e735..10b9571 100644 --- a/README.md +++ b/README.md @@ -153,7 +153,7 @@ nix-portable is tested continuously on the following platforms: ### Under the hood - The nix-portable executable is a self extracting archive, caching its contents in $HOME/.nix-portable -- Either nix, bubblewrap or proot is used to virtualize the /nix/store directory which actually resides in $HOME/.nix-portable/store +- Either nix, bubblewrap or proot is used to virtualize the /nix/store directory which actually resides in $HOME/.nix-portable/nix/store - A default nixpkgs channel is included and the NIX_PATH variable is set accordingly. - Features `flakes` and `nix-command` are enabled out of the box. diff --git a/build-trace.sh b/build-trace.sh new file mode 100755 index 0000000..c4729c9 --- /dev/null +++ b/build-trace.sh @@ -0,0 +1,7 @@ +#!/bin/sh +set -eux +nix-build -E 'with import { }; callPackage ./. { }' +mkdir -p ~/.nix-portable/bin/ +unzip -jo result/bin/nix-portable -d ~/.nix-portable/bin/ +ldd ~/.nix-portable/bin/proot +strace -ff ~/.nix-portable/bin/proot 2>&1 | grep /nix/store | grep -v -i "no such file" | cut -d'"' -f2 | grep -v '^none$' | sort -u diff --git a/build.sh b/build.sh new file mode 100755 index 0000000..f227835 --- /dev/null +++ b/build.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -x +export NIX_BUILD_CORES=1 +exec nix-build -E 'with import { }; callPackage ./. { }' diff --git a/builder.sh b/builder.sh new file mode 100644 index 0000000..a2d0864 --- /dev/null +++ b/builder.sh @@ -0,0 +1,341 @@ +#!/usr/bin/env bash + +# substituteAll interface +runtimeScript=@runtimeScript@ +zip=@zip@ +zstd=@zstd@ +proot=@proot@ +bubblewrap=@bubblewrap@ +nix=@nix@ +busybox=@busybox@ +caBundleZstd=@caBundleZstd@ +storeTar=@storeTar@ +bundledExe=@bundledExe@ +patchelf=@patchelf@ + +set -x + +# https://github.com/NixOS/nixpkgs/blob/e101e9465d47dd7a7eb95b0477ae67091c02773c/lib/strings.nix#L1716 +function removePrefix() { + local prefix="$1" + local str="$2" + local preLen=${#prefix} + if [[ "${str:0:$preLen}" == "$prefix" ]]; then + echo "${str:$preLen}" + else + echo "$str" + fi +} + +add_file_list=() + +# in stage1, we only have bash and coreutils +# so we cannot unzip busybox, so we get the file offsets +# se we can unpack files with "tail" and "head" commands +# tail -c+$((offset + 1)) $zip | head -c$size + +stage1_file_path_list=() +stage1_file_offset_list=() +stage1_file_size_list=() + +# add a stage1 executable file and its dependencies (libraries) +function add_stage1_bin() { + local bin="$1" + if add_file -1 "$bin"; then + echo + echo "adding binary: $bin" + add_stage1_libs "$bin" + echo + # exit 1 # debug + fi +} + +# add stage1 library files +function add_stage1_libs() { + local bin="$1" + # ldd "$bin" # debug + for lib in $(ldd "$bin" 2>/dev/null | grep -oE '/nix/store/[^ ]+'); do + # echo " adding library: $lib" + if add_file -1 "$lib"; then + # recurse: add dependencies of lib + add_stage1_libs "$lib" + fi + done +} + +function add_stage1_file_offset_size() { + local file="$1" + #local size=$(stat -c %s "$file") + local size=$(stat -c %s -L "$file") # dereference symlinks + local hash=$(sha1sum "$file" | head -c40) + # locate the first 1000 bytes of file in the zip archive + local grep_size=1000 + if ((grep_size > size)); then grep_size=$size; fi + local skip=0 + if [ ${#stage1_file_offset_list[@]} != 0 ]; then + # start search from previous file + # skip bytes until previous offset + size + skip=$((${stage1_file_offset_list[-1]} + ${stage1_file_size_list[-1]})) + fi + # exploit that files are appended + while read offset; do + offset=$((offset / 2)) # hex to bin + offset=$((skip + offset)) + # debug + # echo "zipped file header:" + # tail -c+$((offset + 1)) "$out"/bin/nix-portable.zip | head -c1000 | basenc --base16 -w0 || true + # verify offset + # FIXME tail: error writing 'standard output': Broken pipe + set +o pipefail + hash2=$(tail -c+$((offset + 1)) "$out"/bin/nix-portable.zip | head -c"$size" | + sha1sum - | head -c40 || true) + set -o pipefail + if [ "$hash" = "$hash2" ]; then + stage1_file_offset_list+=("$offset") + stage1_file_size_list+=("$size") + return + fi + done < <( + # bin to hex + # rg is 25x faster than grep + # basenc is 10x faster than xxd + tail -c+$((skip + 1)) "$out"/bin/nix-portable.zip | basenc --base16 -w0 | + rg -boF $(head -c"$grep_size" "$file" | basenc --base16 -w0) | + cut -d: -f1 + ) + echo "error: file was not found in zip archive: $file" + exit 1 +} + +defer_zip=false # dt: 10.5362 # TODO remove +defer_zip=true # dt: 3.20238 + +function add_file() { + local is_stage1=false + if [ "$1" = "-1" ]; then is_stage1=true; shift; fi + local file="$1" + if $is_stage1; then + if false; then + # change file path to build a FHS filesystem layout for stage1 + #local file2="stage1/${file#/*/*/*/*}" + local file2="${file#/*/*/*/*}" + if ! [ -e "$file2" ]; then + mkdir -p "${file2%/*}" + cp -Lp "$file" "$file2" + chmod +w "$file2" + fi + file="$file2" + fi + # set relative rpath to create relocatable bins and libs + #$patchelf/bin/patchelf --set-rpath '$ORIGIN/../lib' "$file" + # FIXME patch interpreter paths like /nix/store/rmy663w9p7xb202rcln4jjzmvivznmz8-glibc-2.40-66/lib/ld-linux-x86-64.so.2 + # no. this is not working + # https://stackoverflow.com/questions/48452793/using-origin-to-specify-the-interpreter-in-elf-binaries-isnt-working + #$patchelf/bin/patchelf --set-interpreter '$ORIGIN/../lib' "$file" + fi + if ! $defer_zip; then + # dont defer the zip command = add one file now + # check if file exists in zip archive + if unzip -p "$out"/bin/nix-portable.zip "$(removePrefix "/" "$file")" 2>/dev/null | head -c0; then + return 1 + fi + $zip "$out"/bin/nix-portable.zip "$file" + if $is_stage1; then + stage1_file_path_list+=("$file") + add_stage1_file_offset_size "$file" + fi + else + # defer the zip command = add all files later + # check if file exists in zip archive + local f + for f in "${add_file_list[@]}"; do + if [ "$f" = "$file" ]; then + return 1 + fi + done + echo " adding file: $file" + add_file_list+=("$file") + if $is_stage1; then stage1_file_path_list+=("$file"); fi + fi +} + +function dump_array() { + local name=$1 + local -n arr=$1 + echo "$name=(" + local val + for val in "${arr[@]}"; do + printf "%q\n" "$val" + done + echo ")" +} + +function assert_equal_array_size() { + local -n name1=$1 + local -n arr1=$1 + shift + local size1=${#arr1[@]} + while (($# > 0)); do + local -n name2=$1 + local -n arr2=$1 + shift + local size2=${#arr2[@]} + if [ "$size1" != "$size2" ]; then + echo "error: $name2 should have $size1 values, has $size2" + return 1 + fi + done +} + +function add_file_done() { + if $defer_zip; then + $zip "$out"/bin/nix-portable.zip "${add_file_list[@]}" + add_file_list=() + local file + for file in "${stage1_file_path_list[@]}"; do + add_stage1_file_offset_size "$file" + done + fi + rm -rf stage1 + # check internal consistency + assert_equal_array_size \ + stage1_file_path_list \ + stage1_file_offset_list \ + stage1_file_size_list + # store file offsets in the zip archive + { + dump_array stage1_file_path_list + dump_array stage1_file_offset_list + dump_array stage1_file_size_list + } >stage1_files.sh + touch -d1970-01-01 stage1_files.sh + stage1_file_path_list=() + stage1_file_offset_list=() + stage1_file_size_list=() + # add_file -1 stage1_files.sh + $zip "$out"/bin/nix-portable.zip stage1_files.sh + add_stage1_file_offset_size stage1_files.sh + rm stage1_files.sh + stage1_files_sh_offset=${stage1_file_offset_list[0]} + stage1_files_sh_size=${stage1_file_size_list[0]} + stage1_file_path_list=() + stage1_file_offset_list=() + stage1_file_size_list=() + sed -i "0,/@stage1_files_sh_offset@/s//$(printf "%-24s" "$stage1_files_sh_offset")/; \ + 0,/@stage1_files_sh_size@/s//$(printf "%-22s" "$stage1_files_sh_size")/" "$out"/bin/nix-portable.zip +} + +mkdir -p "$out"/bin +cp $runtimeScript "$out"/bin/nix-portable.zip +chmod +w "$out"/bin/nix-portable.zip + +file_name=runtimeScript.sh +file_name_size_hex=$(printf "%04x" ${#file_name} | tac -rs ..) +file_name_hex=$(echo -n "$file_name" | xxd -p) + +# note: zip fails to extract the file because the local file header comes after the file contents + +# Local file header +local_file_header_offset_hex=$(printf "%08x" $(stat -c "%s" "$out"/bin/nix-portable.zip) | tac -rs ..) +{ + echo 50 4b 03 04 # Local file header signature + echo 00 00 # Version needed to extract (minimum) + echo 00 00 # General purpose bit flag + echo 00 00 # Compression method + echo 00 00 # File last modification time + echo 00 00 # File last modification date + echo 00 00 00 00 # CRC-32 of uncompressed data + echo 00 00 00 00 # Compressed size + echo 00 00 00 00 # Uncompressed size + echo $file_name_size_hex # File name length (n) + echo 00 00 # Extra field length (m) + echo $file_name_hex # File name + # Extra field +} | xxd -r -p >> "$out"/bin/nix-portable.zip + +# Central directory file header +central_directory_file_header_offset_hex=$(printf "%08x" $(stat -c "%s" "$out"/bin/nix-portable.zip) | tac -rs ..) +{ + echo 50 4b 01 02 # Central directory file header signature + echo 00 00 # Version made by + echo 00 00 # Version needed to extract (minimum) + echo 00 00 # General purpose bit flag + echo 00 00 # Compression method + echo 00 00 # File last modification time + echo 00 00 # File last modification date + echo 00 00 00 00 # CRC-32 of uncompressed data + echo 00 00 00 00 # Compressed size + echo 00 00 00 00 # Uncompressed size + echo $file_name_size_hex # File name length (n) + echo 00 00 # Extra field length (m) + echo 00 00 # File comment length (k) + echo 00 00 # Disk number where file starts (or 0xffff for ZIP64) + echo 00 00 # Internal file attributes + echo 00 00 00 00 # External file attributes + echo "$local_file_header_offset_hex" # Relative offset of local file header + echo $file_name_hex # File name + # Extra field + # File comment +} | xxd -r -p >> "$out"/bin/nix-portable.zip + +# 62 -> 3e 00 00 00 +central_directory_size_hex=$(printf "%08x" $((46 + ${#file_name})) | tac -rs ..) + +# End of central directory record +{ + echo 50 4b 05 06 # End of central directory signature + echo 00 00 # Number of this disk + echo 00 00 # Disk where central directory starts + echo 00 00 # Number of central directory records on this disk + echo 01 00 # Total number of central directory records + echo $central_directory_size_hex # Size of central directory (bytes) + echo "$central_directory_file_header_offset_hex" # Offset of start of central directory, relative to start of archive + echo 00 00 # Comment length + # Comment + echo 00 00 00 00 00 00 00 00 00 00 # 10 null bytes (?) +} | xxd -r -p >> "$out"/bin/nix-portable.zip + +unzip -vl "$out"/bin/nix-portable.zip + +zip="$zip/bin/zip -0" + +# we cannot unzip busybox, so we need offset and size of all needed files (bins and libs) +# add_file does not work here +#$zip $out/bin/nix-portable.zip $busybox/bin/busybox +add_stage1_bin $busybox/bin/busybox + +t1=$(date +%s.%N) + +# decompressor for $storeTar/tar +add_stage1_bin $zstd/bin/zstd + +# runtime bins +add_stage1_bin $bubblewrap/bin/bwrap +add_stage1_bin $proot/bin/proot +add_stage1_bin $nix/bin/nix + +# TODO move stage1_files.sh up in the zip archive for faster access +#stage1_file_done + +add_file $storeTar/closureInfo/store-paths +add_file $storeTar/tar +add_file $caBundleZstd + +add_file_done + +t2=$(date +%s.%N) +dt=$(echo "$t1" "$t2" | awk '{ print ($2 - $1) }') +echo "dt: $dt" +# exit 1 + +# create fingerprint +fp=$(sha256sum "$out"/bin/nix-portable.zip | head -c64) +sed -i "0,/_FINGERPRINT_PLACEHOLDER_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx/s//$fp/" "$out"/bin/nix-portable.zip + +if [ "$bundledExe" == "" ]; then + target="$out/bin/nix-portable" +else + target="$out/bin/$(basename "$bundledExe")" +fi +mv "$out"/bin/nix-portable.zip "$target" +chmod +x "$target" diff --git a/default.nix b/default.nix index 330967c..3bb407b 100644 --- a/default.nix +++ b/default.nix @@ -1,31 +1,72 @@ with builtins; { - bwrap, + bubblewrapStatic ? pkgsStatic.bubblewrap, + # fix: builder failed to produce output path for output 'man' + # https://github.com/milahu/nixpkgs/issues/83 + #nixStatic ? pkgsStatic.nix, + # use nix 2.21.0 + # https://discourse.nixos.org/t/where-can-i-get-a-statically-built-nix/34253/15 + # https://hydra.nixos.org/job/nix/master/buildStatic.x86_64-linux/all + stdenv, nix, - proot, + nixGitStatic ? (stdenv.mkDerivation { + name = "nix-static-x86_64-unknown-linux-musl-2.21.0pre20240311_25bf671"; + src = fetchurl { + url = "https://hydra.nixos.org/build/252984554/download/1/nix"; + sha256 = "sha256:0rcxm2p38lhxz4cbxwbw432mpi8i5lmkmw6gzrw4i48ra90hn89q"; + }; + # ls -l $(dirname $(readlink -f $(which nix))) | grep -- '->' | cut -d' ' -f17 | xargs echo + nixBins = lib.escapeShellArgs (attrNames (lib.filterAttrs (d: type: type == "symlink") (readDir "${nix}/bin"))); + buildCommand = '' + mkdir -p $out/bin + cp $src $out/bin/nix + chmod +x $out/bin/nix + for bin in $nixBins; do + ln -s nix $out/bin/$bin + done + ''; + }), unzip, zip, unixtools, - - busybox ? pkgs.pkgsStatic.busybox, - cacert ? pkgs.cacert, - compression ? "zstd -19 -T0", - gnutar ? pkgs.pkgsStatic.gnutar, - lib ? pkgs.lib, - perl ? pkgs.perl, - pkgs ? import {}, - xz ? pkgs.pkgsStatic.xz, - zstd ? pkgs.pkgsStatic.zstd, - nixStatic ? pkgs.pkgsStatic.nix, - + substituteAll, + lib, + glibc, + ripgrep, + patchelf, + cacert, + pkgs, + pkgsStatic, + busyboxStatic ? pkgsStatic.busybox, + gnutar, + xz, + zstdStatic ? pkgsStatic.zstd, + # fix: ld: attempted static link of dynamic object + # https://gitlab.com/ita1024/waf/-/issues/2467 + #prootStatic ? pkgsStatic.proot, + callPackage, + prootStatic ? (callPackage ./proot/alpine.nix { }), + compression ? "zstd -3 -T1", buildSystem ? builtins.currentSystem, + # # tar crashed on emulated aarch64 system + # buildSystem ? "x86_64-linux", # hardcode executable to run. Useful when creating a bundle. bundledPackage ? null, ... -}@inp: +}: + with lib; let + nixStatic = nixGitStatic; + + # stage1 bins + busybox = busyboxStatic; + zstd = zstdStatic; + nix = nixStatic; + bubblewrap = bubblewrapStatic; + proot = prootStatic; + pname = if bundledPackage == null then "nix-portable" @@ -45,587 +86,81 @@ let maketar = targets: pkgsBuild.stdenv.mkDerivation { name = "nix-portable-store-tarball"; - nativeBuildInputs = [ pkgsBuild.perl pkgsBuild.zstd ]; - exportReferencesGraph = map (x: [("closure-" + baseNameOf x) x]) targets; + nativeBuildInputs = [ pkgsBuild.zstd ]; buildCommand = '' - storePaths=$(perl ${pkgsBuild.pathsFromGraph} ./closure-*) mkdir $out - echo $storePaths > $out/index cp -r ${pkgsBuild.closureInfo { rootPaths = targets; }} $out/closureInfo - tar -cf - \ --owner=0 --group=0 --mode=u+rw,uga+r \ --hard-dereference \ - $storePaths | ${compression} > $out/tar + $(cat $out/closureInfo/store-paths) | ${compression} > $out/tar ''; }; - packStaticBin = binPath: let - binName = (last (splitString "/" binPath)); in - pkgs.runCommand - binName - { nativeBuildInputs = [ pkgs.upx ]; } - '' - mkdir -p $out/bin - upx -9 -o $out/bin/${binName} ${binPath} - ''; - - installBin = pkg: bin: '' - unzip -qqoj "\$self" ${ lib.removePrefix "/" "${pkg}/bin/${bin}"} -d \$dir/bin - chmod +wx \$dir/bin/${bin}; - ''; - - caBundleZstd = pkgs.runCommand "cacerts" {} "cat ${cacert}/etc/ssl/certs/ca-bundle.crt | ${inp.zstd}/bin/zstd -19 > $out"; + caBundleZstd = pkgs.runCommand "cacerts" {} "cat ${cacert}/etc/ssl/certs/ca-bundle.crt | ${zstd}/bin/zstd -19 > $out"; - bwrap = packStaticBin "${inp.bwrap}/bin/bwrap"; - nixStatic = packStaticBin "${inp.nixStatic}/bin/nix"; - proot = packStaticBin "${inp.proot}/bin/proot"; - zstd = packStaticBin "${inp.zstd}/bin/zstd"; # the default nix store contents to extract when first used - storeTar = maketar ([ cacert nix nixpkgsSrc ] ++ lib.optional (bundledPackage != null) bundledPackage); + storeTar = maketar ([ + cacert + nix + # nix.man # not with nix 2.21.0 + nixpkgsSrc + ] ++ lib.optional (bundledPackage != null) bundledPackage); # The runtime script which unpacks the necessary files to $HOME/.nix-portable # and then executes nix via proot or bwrap # Some shell expressions will be evaluated at build time and some at run time. # Variables/expressions escaped via `\$` will be evaluated at run time - runtimeScript = '' - #!/usr/bin/env bash - - set -eo pipefail - - start=\$(date +%s%N) # start time in nanoseconds - - # dump environment on exit if debug is enabled - if [ -n "\$NP_DEBUG" ] && [ "\$NP_DEBUG" -ge 1 ]; then - trap "declare -p > /tmp/np_env" EXIT - fi - - # there seem to be less issues with proot when disabling seccomp - export PROOT_NO_SECCOMP=\''${PROOT_NO_SECCOMP:-1} - - set -e - if [ -n "\$NP_DEBUG" ] && [ "\$NP_DEBUG" -ge 2 ]; then - set -x - fi - - # &3 is our error out which we either forward to &2 or to /dev/null - # depending on the setting - if [ -n "\$NP_DEBUG" ] && [ "\$NP_DEBUG" -ge 1 ]; then - debug(){ - echo \$@ || true - } - exec 3>&2 - else - debug(){ - true - } - exec 3>/dev/null - fi - - # to reference this script's file - self="\$(realpath \''${BASH_SOURCE[0]})" - - # fingerprint will be inserted by builder - fingerprint="_FINGERPRINT_PLACEHOLDER_" - - # user specified location for program files and nix store - [ -z "\$NP_LOCATION" ] && NP_LOCATION="\$HOME" - NP_LOCATION="\$(readlink -f "\$NP_LOCATION")" - dir="\$NP_LOCATION/.nix-portable" - store="\$dir/nix/store" - # create /nix/var/nix to prevent nix from falling back to chroot store. - mkdir -p \$dir/{bin,nix/var/nix,nix/store} - # sanitize the tmpbin directory - rm -rf "\$dir/tmpbin" - # create a directory to hold executable symlinks for overriding - mkdir -p "\$dir/tmpbin" - - # create minimal drv file for nix to spawn a nix shell - echo 'builtins.derivation {name="foo"; builder="/bin/sh"; args = ["-c" "echo hello \> \\\$out"]; system=builtins.currentSystem;}' > "\$dir/mini-drv.nix" - - # the fingerprint being present inside a file indicates that - # this version of nix-portable has already been initialized - if test -e \$dir/conf/fingerprint && [ "\$(cat \$dir/conf/fingerprint)" == "\$fingerprint" ]; then - newNPVersion=false - else - newNPVersion=true - fi - - # Nix portable ships its own nix.conf - export NIX_CONF_DIR=\$dir/conf/ - - NP_CONF_SANDBOX=\''${NP_CONF_SANDBOX:-false} - NP_CONF_STORE=\''${NP_CONF_STORE:-auto} - - - recreate_nix_conf(){ - mkdir -p "\$NIX_CONF_DIR" - rm -f "\$NIX_CONF_DIR/nix.conf" - - # static config - echo "build-users-group = " >> \$dir/conf/nix.conf - echo "experimental-features = nix-command flakes" >> \$dir/conf/nix.conf - echo "ignored-acls = security.selinux system.nfs4_acl" >> \$dir/conf/nix.conf - echo "use-sqlite-wal = false" >> \$dir/conf/nix.conf - echo "sandbox-paths = /bin/sh=\$dir/busybox/bin/busybox" >> \$dir/conf/nix.conf - - # configurable config - echo "sandbox = \$NP_CONF_SANDBOX" >> \$dir/conf/nix.conf - echo "store = \$NP_CONF_STORE" >> \$dir/conf/nix.conf - } - - - ### install files - - PATH_OLD="\$PATH" - - # as soon as busybox is unpacked, restrict PATH to busybox to ensure reproducibility of this script - # only unpack binaries if necessary - if [ "\$newNPVersion" == "false" ]; then - - debug "binaries already installed" - export PATH="\$dir/busybox/bin" - - else - - debug "installing files" - - mkdir -p \$dir/emptyroot - - # install busybox - mkdir -p \$dir/busybox/bin - (base64 -d> "\$dir/busybox/bin/busybox" && chmod +x "\$dir/busybox/bin/busybox") << END - $(cat ${busybox}/bin/busybox | base64) - END - busyBins="${toString (attrNames (filterAttrs (d: type: type == "symlink") (readDir "${inp.busybox}/bin")))}" - for bin in \$busyBins; do - [ ! -e "\$dir/busybox/bin/\$bin" ] && ln -s busybox "\$dir/busybox/bin/\$bin" - done - - export PATH="\$dir/busybox/bin" - - # install other binaries - ${installBin zstd "zstd"} - ${installBin proot "proot"} - ${installBin bwrap "bwrap"} - ${installBin nixStatic "nix"} - - # install ssl cert bundle - unzip -poj "\$self" ${ lib.removePrefix "/" "${caBundleZstd}"} | \$dir/bin/zstd -d > \$dir/ca-bundle.crt - - recreate_nix_conf - fi - - - - ### setup SSL - # find ssl certs or use from nixpkgs - debug "figuring out ssl certs" - if [ -z "\$SSL_CERT_FILE" ]; then - debug "SSL_CERT_FILE not defined. trying to find certs automatically" - if [ -e /etc/ssl/certs/ca-bundle.crt ]; then - export SSL_CERT_FILE=\$(realpath /etc/ssl/certs/ca-bundle.crt) - debug "found /etc/ssl/certs/ca-bundle.crt with real path \$SSL_CERT_FILE" - elif [ -e /etc/ssl/certs/ca-certificates.crt ]; then - export SSL_CERT_FILE=\$(realpath /etc/ssl/certs/ca-certificates.crt) - debug "found /etc/ssl/certs/ca-certificates.crt with real path \$SSL_CERT_FILE" - elif [ ! -e /etc/ssl/certs ]; then - debug "/etc/ssl/certs does not exist. Will use certs from nixpkgs." - export SSL_CERT_FILE=\$dir/ca-bundle.crt - else - debug "certs seem to reside in /etc/ssl/certs. No need to set up anything" - fi - fi - if [ -n "\$SSL_CERT_FILE" ]; then - sslBind="\$(realpath \$SSL_CERT_FILE) \$dir/ca-bundle.crt" - export SSL_CERT_FILE="\$dir/ca-bundle.crt" - else - sslBind="/etc/ssl /etc/ssl" - fi - - - - ### detecting existing git installation - # we need to install git inside the wrapped environment - # unless custom git executable path is specified in NP_GIT, - # since the existing git might be incompatible to Nix (e.g. v1.x) - if [ -n "\$NP_GIT" ]; then - doInstallGit=false - ln -s "\$NP_GIT" "\$dir/tmpbin/git" - else - doInstallGit=true - fi - - - - storePathOfFile(){ - file=\$(realpath \$1) - sPath="\$(echo \$file | awk -F "/" 'BEGIN{OFS="/";}{print \$2,\$3,\$4}')" - echo "/\$sPath" - } - - - collectBinds(){ - ### gather paths to bind for proot - # we cannot bind / to / without running into a lot of trouble, therefore - # we need to collect all top level directories and bind them inside an empty root - pathsTopLevel="\$(find / -mindepth 1 -maxdepth 1 -not -name nix -not -name dev)" - - - toBind="" - for p in \$pathsTopLevel; do - if [ -e "\$p" ]; then - real=\$(realpath \$p) - if [ -e "\$real" ]; then - if [[ "\$real" == /nix/store/* ]]; then - storePath=\$(storePathOfFile \$real) - toBind="\$toBind \$storePath \$storePath" - else - toBind="\$toBind \$real \$p" - fi - fi - fi - done - - - # TODO: add /var/run/dbus/system_bus_socket - paths="/etc/host.conf /etc/hosts /etc/hosts.equiv /etc/mtab /etc/netgroup /etc/networks /etc/passwd /etc/group /etc/nsswitch.conf /etc/resolv.conf /etc/localtime \$HOME" - - for p in \$paths; do - if [ -e "\$p" ]; then - real=\$(realpath \$p) - if [ -e "\$real" ]; then - if [[ "\$real" == /nix/store/* ]]; then - storePath=\$(storePathOfFile \$real) - toBind="\$toBind \$storePath \$storePath" - else - toBind="\$toBind \$real \$real" - fi - fi - fi - done - - # if we're on a nixos, the /bin/sh symlink will point - # to a /nix/store path which doesn't exit inside the wrapped env - # we fix this by binding busybox/bin to /bin - if test -s /bin/sh && [[ "\$(realpath /bin/sh)" == /nix/store/* ]]; then - toBind="\$toBind \$dir/busybox/bin /bin" - fi - } - - - makeBindArgs(){ - arg=\$1; shift - sep=\$1; shift - binds="" - while :; do - if [ -n "\$1" ]; then - from="\$1"; shift - to="\$1"; shift || { echo "no bind destination provided for \$from!"; exit 3; } - binds="\$binds \$arg \$from\$sep\$to"; - else - break - fi - done - } - - - - ### select container runtime - debug "figuring out which runtime to use" - [ -z "\$NP_BWRAP" ] && NP_BWRAP=\$(PATH="\$PATH_OLD:\$PATH" which bwrap 2>/dev/null) || true - [ -z "\$NP_BWRAP" ] && NP_BWRAP=\$dir/bin/bwrap - debug "bwrap executable: \$NP_BWRAP" - # [ -z "\$NP_NIX ] && NP_NIX=\$(PATH="\$PATH_OLD:\$PATH" which nix 2>/dev/null) || true - [ -z "\$NP_NIX" ] && NP_NIX=\$dir/bin/nix - debug "nix executable: \$NP_NIX" - [ -z "\$NP_PROOT" ] && NP_PROOT=\$(PATH="\$PATH_OLD:\$PATH" which proot 2>/dev/null) || true - [ -z "\$NP_PROOT" ] && NP_PROOT=\$dir/bin/proot - debug "proot executable: \$NP_PROOT" - debug "testing all available runtimes..." - if [ -z "\$NP_RUNTIME" ]; then - # read last automatic selected runtime from disk - if [ "\$newNPVersion" == "true" ]; then - debug "removing cached auto selected runtime" - rm -f "\$dir/conf/last_auto_runtime" - fi - if [ -f "\$dir/conf/last_auto_runtime" ]; then - last_auto_runtime="\$(cat "\$dir/conf/last_auto_runtime")" - else - last_auto_runtime= - fi - debug "last auto selected runtime: \$last_auto_runtime" - if [ "\$last_auto_runtime" != "" ]; then - NP_RUNTIME="\$last_auto_runtime" - # check if nix --store works - elif \\ - debug "testing nix --store" \\ - && mkdir -p \$dir/tmp/ \\ - && touch \$dir/tmp/testfile \\ - && "\$NP_NIX" --store "\$dir/tmp/__store" shell -f "\$dir/mini-drv.nix" -c "\$dir/bin/nix" store add-file --store "\$dir/tmp/__store" "\$dir/tmp/testfile" >/dev/null 2>&3; then - chmod -R +w \$dir/tmp/__store - rm -r \$dir/tmp/__store - debug "nix --store works on this system -> will use nix as runtime" - NP_RUNTIME=nix - # check if bwrap works properly - elif \\ - debug "nix --store failed -> testing bwrap" \\ - && \$NP_BWRAP --bind \$dir/emptyroot / --bind \$dir/ /nix --bind \$dir/busybox/bin/busybox "\$dir/true" "\$dir/true" 2>&3 ; then - debug "bwrap seems to work on this system -> will use bwrap" - NP_RUNTIME=bwrap - else - debug "bwrap doesn't work on this system -> will use proot" - NP_RUNTIME=proot - fi - echo -n "\$NP_RUNTIME" > "\$dir/conf/last_auto_runtime" - else - debug "runtime selected via NP_RUNTIME: \$NP_RUNTIME" - fi - debug "NP_RUNTIME: \$NP_RUNTIME" - if [ "\$NP_RUNTIME" == "nix" ]; then - run="\$NP_NIX shell -f \$dir/mini-drv.nix -c" - export PATH="\$PATH:\$store${lib.removePrefix "/nix/store" nix}/bin" - NP_CONF_STORE="\$dir" - recreate_nix_conf - elif [ "\$NP_RUNTIME" == "bwrap" ]; then - collectBinds - makeBindArgs --bind " " \$toBind \$sslBind - run="\$NP_BWRAP \$BWRAP_ARGS \\ - --bind \$dir/emptyroot /\\ - --dev-bind /dev /dev\\ - --bind \$dir/nix /nix\\ - \$binds" - # --bind \$dir/busybox/bin/busybox /bin/sh\\ - else - # proot - collectBinds - makeBindArgs -b ":" \$toBind \$sslBind - run="\$NP_PROOT \$PROOT_ARGS\\ - -r \$dir/emptyroot\\ - -b /dev:/dev\\ - -b \$dir/nix:/nix\\ - \$binds" - # -b \$dir/busybox/bin/busybox:/bin/sh\\ - fi - debug "base command will be: \$run" - - - - ### setup environment - export NIX_PATH="\$dir/channels:nixpkgs=\$dir/channels/nixpkgs" - mkdir -p \$dir/channels - [ -h \$dir/channels/nixpkgs ] || ln -s ${nixpkgsSrc} \$dir/channels/nixpkgs - - - ### install nix store - # Install all the nix store paths necessary for the current nix-portable version - # We only unpack missing store paths from the tar archive. - index="$(cat ${storeTar}/index)" - - # if [ ! "\$NP_RUNTIME" == "nix" ]; then - export missing=\$( - for path in \$index; do - if [ ! -e \$store/\$(basename \$path) ]; then - echo "nix/store/\$(basename \$path)" - fi - done - ) - - if [ -n "\$missing" ]; then - debug "extracting missing store paths" - ( - mkdir -p \$dir/tmp \$store/ - rm -rf \$dir/tmp/* - cd \$dir/tmp - unzip -qqp "\$self" ${ lib.removePrefix "/" "${storeTar}/tar"} \ - | \$dir/bin/zstd -d \ - | tar -x \$missing --strip-components 2 - mv \$dir/tmp/* \$store/ - ) - rm -rf \$dir/tmp - fi - - if [ -n "\$missing" ]; then - debug "registering new store paths to DB" - reg="$(cat ${storeTar}/closureInfo/registration)" - cmd="\$run \$store${lib.removePrefix "/nix/store" nix}/bin/nix-store --load-db" - debug "running command: \$cmd" - # echo "\$reg" | \$cmd - fi - # fi - - - ### select executable - # the executable can either be selected by - # - executing './nix-portable BIN_NAME', - # - symlinking to nix-portable, in which case the name of the symlink selects the nix executable - # Alternatively the executable can be hardcoded by specifying the argument 'executable' of nix-portable's default.nix file. - executable="${if bundledPackage == null then "" else bundledExe}" - if [ "\$executable" != "" ]; then - bin="\$executable" - debug "executable is hardcoded to: \$bin" - elif [[ "\$(basename \$0)" == nix-portable* ]]; then\ - if [ -z "\$1" ]; then - echo "Error: please specify the nix binary to execute" - echo "Alternatively symlink against \$0" - exit 1 - elif [ "\$1" == "debug" ]; then - bin="\$(which \$2)" - shift; shift - else - bin="\$store${lib.removePrefix "/nix/store" nix}/bin/\$1" - shift - fi - else - bin="\$store${lib.removePrefix "/nix/store" nix}/bin/\$(basename \$0)" - fi - - - - ### check which runtime has been used previously - if [ -f "\$dir/conf/last_runtime" ]; then - lastRuntime=\$(cat "\$dir/conf/last_runtime") - else - lastRuntime= - fi - - - - ### check if nix is functional with or without sandbox - # sandbox-fallback is not reliable: https://github.com/NixOS/nix/issues/4719 - if [ "\$newNPVersion" == "true" ] || [ "\$lastRuntime" != "\$NP_RUNTIME" ]; then - nixBin="\$store${lib.removePrefix "/nix/store" nix}/bin/nix" - # if [ "\$NP_RUNTIME" == "nix" ]; then - # nixBin="nix" - # else - # fi - debug "Testing if nix can build stuff without sandbox" - if ! \$run "\$nixBin" build --no-link -f "\$dir/mini-drv.nix" --option sandbox false >&3 2>&3; then - echo "Fatal error: nix is unable to build packages" - exit 1 - fi - - debug "Testing if nix sandbox is functional" - if ! \$run "\$nixBin" build --no-link -f "\$dir/mini-drv.nix" --option sandbox true >&3 2>&3; then - debug "Sandbox doesn't work -> disabling sandbox" - NP_CONF_SANDBOX=false - recreate_nix_conf - else - debug "Sandboxed builds work -> enabling sandbox" - NP_CONF_SANDBOX=true - recreate_nix_conf - fi - - fi - - - ### save fingerprint and lastRuntime - if [ "\$newNPVersion" == "true" ]; then - echo -n "\$fingerprint" > "\$dir/conf/fingerprint" - fi - if [ "\$lastRuntime" != \$NP_RUNTIME ]; then - echo -n \$NP_RUNTIME > "\$dir/conf/last_runtime" - fi - - - - ### set PATH - # restore original PATH and append busybox - export PATH="\$PATH_OLD:\$dir/busybox/bin" - # apply overriding executable paths in \$dir/tmpbin/ - export PATH="\$dir/tmpbin:\$PATH" - - - - ### install git via nix, if git installation is not in /nix path - if \$doInstallGit && [ ! -e \$store${lib.removePrefix "/nix/store" git.out} ] ; then - echo "Installing git. Disable this by specifying the git executable path with 'NP_GIT'" - \$run \$store${lib.removePrefix "/nix/store" nix}/bin/nix build --impure --no-link --expr " - (import ${nixpkgsSrc} {}).${gitAttribute}.out - " - else - debug "git already installed or manually specified" - fi - - ### override the possibly existing git in the environment with the installed one - # excluding the case NP_GIT is set. - if \$doInstallGit; then - export PATH="${git.out}/bin:\$PATH" - fi - - - ### print elapsed time - end=\$(date +%s%N) # end time in nanoseconds - # time elapsed in millis with two decimal places - # elapsed=\$(echo "scale=2; (\$end - \$start)/1000000000" | bc) - elapsed=\$(echo "scale=2; (\$end - \$start)/1000000" | bc) - debug "Time to initialize nix-portable: \$elapsed millis" + runtimeScript = substituteAll { + src = ./runtimeScript.sh; + busyboxBins = lib.escapeShellArgs (attrNames (filterAttrs (d: type: type == "symlink") (readDir "${busybox}/bin"))); + bundledExe = if bundledPackage == null then "" else bundledExe; + git = git.out; # TODO why not just "git" + inherit + bubblewrap + nix + proot + zstd + busybox + caBundleZstd + storeTar + nixpkgsSrc + gitAttribute + ; + }; + builderScript = substituteAll { + src = ./builder.sh; + executable = true; + bundledExe = if bundledPackage == null then "" else bundledExe; + inherit + runtimeScript + zip + bubblewrap + nix + proot + zstd + busybox + caBundleZstd + storeTar + patchelf + ; + }; - ### run commands - [ -z "\$NP_RUN" ] && NP_RUN="\$run" - if [ "\$NP_RUNTIME" == "proot" ]; then - debug "running command: \$NP_RUN \$bin \$@" - exec \$NP_RUN \$bin "\$@" - else - cmd="\$NP_RUN \$bin \$@" - debug "running command: \$cmd" - exec \$NP_RUN \$bin "\$@" - fi - exit + nixPortable = pkgs.runCommand pname { + nativeBuildInputs = [ + unixtools.xxd + unzip + glibc # ldd + ripgrep # rg + ]; + } + '' + bash ${builderScript} ''; - runtimeScriptEscaped = replaceStrings ["\""] ["\\\""] runtimeScript; - - nixPortable = pkgs.runCommand pname {nativeBuildInputs = [unixtools.xxd unzip];} '' - mkdir -p $out/bin - echo "${runtimeScriptEscaped}" > $out/bin/nix-portable.zip - xxd $out/bin/nix-portable.zip | tail - - sizeA=$(printf "%08x" `stat -c "%s" $out/bin/nix-portable.zip` | tac -rs ..) - echo 504b 0304 0000 0000 0000 0000 0000 0000 | xxd -r -p >> $out/bin/nix-portable.zip - echo 0000 0000 0000 0000 0000 0200 0000 4242 | xxd -r -p >> $out/bin/nix-portable.zip - - sizeB=$(printf "%08x" `stat -c "%s" $out/bin/nix-portable.zip` | tac -rs ..) - echo 504b 0102 0000 0000 0000 0000 0000 0000 | xxd -r -p >> $out/bin/nix-portable.zip - echo 0000 0000 0000 0000 0000 0000 0200 0000 | xxd -r -p >> $out/bin/nix-portable.zip - echo 0000 0000 0000 0000 0000 $sizeA 4242 | xxd -r -p >> $out/bin/nix-portable.zip - - echo 504b 0506 0000 0000 0000 0100 3000 0000 | xxd -r -p >> $out/bin/nix-portable.zip - echo $sizeB 0000 0000 0000 0000 0000 0000 | xxd -r -p >> $out/bin/nix-portable.zip - - unzip -vl $out/bin/nix-portable.zip - - zip="${zip}/bin/zip -0" - $zip $out/bin/nix-portable.zip ${bwrap}/bin/bwrap - $zip $out/bin/nix-portable.zip ${nixStatic}/bin/nix - $zip $out/bin/nix-portable.zip ${proot}/bin/proot - $zip $out/bin/nix-portable.zip ${zstd}/bin/zstd - $zip $out/bin/nix-portable.zip ${storeTar}/tar - $zip $out/bin/nix-portable.zip ${caBundleZstd} - - # create fingerprint - fp=$(sha256sum $out/bin/nix-portable.zip | cut -d " " -f 1) - sed -i "s/_FINGERPRINT_PLACEHOLDER_/$fp/g" $out/bin/nix-portable.zip - # fix broken zip header due to manual modification - ${zip}/bin/zip -F $out/bin/nix-portable.zip --out $out/bin/nix-portable-fixed.zip - - rm $out/bin/nix-portable.zip - executable=${if bundledPackage == null then "" else bundledExe} - if [ "$executable" == "" ]; then - target="$out/bin/nix-portable" - else - target="$out/bin/$(basename "$executable")" - fi - mv $out/bin/nix-portable-fixed.zip "$target" - chmod +x "$target" - ''; in -nixPortable.overrideAttrs (prev: { - passthru = (prev.passthru or {}) // { - inherit bwrap proot; - }; -}) +nixPortable diff --git a/flake.nix b/flake.nix index 0f40e1b..c235874 100644 --- a/flake.nix +++ b/flake.nix @@ -131,27 +131,21 @@ # the static proot built with nix somehow didn't work on other systems, # therefore using the proot static build from proot gitlab - proot = if crossSystem != null then throw "fix proot for crossSytem" else import ./proot/alpine.nix { inherit pkgs; }; + prootStatic = if crossSystem != null then throw "fix proot for crossSytem" else import ./proot/alpine.nix { inherit pkgs; }; in # crashes if nixpkgs updated: error: executing 'git': No such file or directory pkgs.callPackage ./default.nix { - inherit proot; + #proot = prootStatic; pkgs = pkgsDefaultChannel; lib = inp.nixpkgs.lib; - compression = "zstd -3 -T1"; nix = inp.nix.packages.${system}.nix; nixStatic = inp.nix.packages.${system}.nix-static; - busybox = pkgs.pkgsStatic.busybox; - bwrap = pkgs.pkgsStatic.bubblewrap; - gnutar = pkgs.pkgsStatic.gnutar; - perl = pkgs.pkgsBuildBuild.perl; - xz = pkgs.pkgsStatic.xz; - zstd = pkgs.pkgsStatic.zstd; + pkgsStatic = pkgs.pkgsStatic; # tar crashed on emulated aarch64 system buildSystem = "x86_64-linux"; diff --git a/runtimeScript.sh b/runtimeScript.sh new file mode 100644 index 0000000..b8a3344 --- /dev/null +++ b/runtimeScript.sh @@ -0,0 +1,599 @@ +#!/usr/bin/env bash + +# substituteAll interface +zstd=@zstd@ +proot=@proot@ +bubblewrap=@bubblewrap@ +nix=@nix@ +busybox=@busybox@ +busyboxBins=(@busyboxBins@) +caBundleZstd=@caBundleZstd@ +storeTar=@storeTar@ +git=@git@ +gitAttribute=@gitAttribute@ +nixpkgsSrc=@nixpkgsSrc@ +bundledExe=@bundledExe@ + +# sed interface +# busyboxOffset=@busyboxOffset@ +# busyboxSize=@busyboxSize@ +stage1_files_sh_offset=@stage1_files_sh_offset@ +stage1_files_sh_size=@stage1_files_sh_size@ + +set -eo pipefail + +start="$(date +%s%N)" # start time in nanoseconds + +unzip_quiet="-qq" + +# dump environment on exit if debug is enabled +if [ -n "$NP_DEBUG" ] && [ "$NP_DEBUG" -ge 1 ]; then + trap "declare -p > /tmp/np_env" EXIT + unzip_quiet= +fi + +# there seem to be less issues with proot when disabling seccomp +export PROOT_NO_SECCOMP="${PROOT_NO_SECCOMP:-1}" + +set -e +if [ -n "$NP_DEBUG" ] && [ "$NP_DEBUG" -ge 2 ]; then + set -x +fi + +# &3 is our error out which we either forward to &2 or to /dev/null +# depending on the setting +if [ -n "$NP_DEBUG" ] && [ "$NP_DEBUG" -ge 1 ]; then + debug(){ + echo "$@" || true + } + exec 3>&2 +else + debug(){ + true + } + exec 3>/dev/null +fi + +# to reference this script's file +self="$(realpath "${BASH_SOURCE[0]}")" + +# fingerprint will be inserted by builder +fingerprint="_FINGERPRINT_PLACEHOLDER_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + +# user specified location for program files and nix store +[ -z "$NP_LOCATION" ] && NP_LOCATION="$HOME" +NP_LOCATION="$(readlink -f "$NP_LOCATION")" +dir="$NP_LOCATION/.nix-portable" +store="$dir/nix/store" +# create /nix/var/nix to prevent nix from falling back to chroot store. +mkdir -p "$dir"/{bin,nix/var/nix,nix/store} +# sanitize the tmpbin directory +rm -rf "$dir/tmpbin" +# create a directory to hold executable symlinks for overriding +mkdir -p "$dir/tmpbin" + +# create minimal drv file for nix to spawn a nix shell +miniDrv="$(cat <<'EOF' +builtins.derivation { + name = "foo"; + builder = "/bin/sh"; + args = [ + "-c" + "echo hello >$out" + ]; + system = builtins.currentSystem; +} +EOF +)" +if [ "$(cat "$dir/mini-drv.nix" || true)" != "$miniDrv" ]; then + echo "$miniDrv" >"$dir/mini-drv.nix" +fi + +# the fingerprint being present inside a file indicates that +# this version of nix-portable has already been initialized +if test -e "$dir"/conf/fingerprint && [ "$(cat "$dir"/conf/fingerprint)" == "$fingerprint" ]; then + newNPVersion=false +else + newNPVersion=true +fi + +# Nix portable ships its own nix.conf +export NIX_CONF_DIR="$dir"/conf/ + +NP_CONF_SANDBOX="${NP_CONF_SANDBOX:-false}" +NP_CONF_STORE="${NP_CONF_STORE:-auto}" + + +recreate_nix_conf(){ + mkdir -p "$NIX_CONF_DIR" + rm -f "$NIX_CONF_DIR/nix.conf" + + { + # static config + echo "build-users-group = " + echo "experimental-features = nix-command flakes" + echo "ignored-acls = security.selinux system.nfs4_acl" + echo "use-sqlite-wal = false" + echo "sandbox-paths = /bin/sh=$dir/bin/busybox" + + # configurable config + echo "sandbox = $NP_CONF_SANDBOX" + echo "store = $NP_CONF_STORE" + } > "$NIX_CONF_DIR/nix.conf" +} + + +### install files + +# https://github.com/NixOS/nixpkgs/blob/e101e9465d47dd7a7eb95b0477ae67091c02773c/lib/strings.nix#L1716 +function removePrefix() { + local prefix="$1" + local str="$2" + local preLen=${#prefix} + if [[ "${str:0:$preLen}" == "$prefix" ]]; then + echo "${str:$preLen}" + else + echo "$str" + fi +} + +function installBin() { + local pkg="$1" + local bin="$2" + unzip $unzip_quiet -oj "$self" "$(removePrefix "/" "$pkg/bin/$bin")" -d "$dir"/bin + chmod +wx "$dir"/bin/"$bin"; +} + +PATH_OLD="$PATH" + +# as soon as busybox is unpacked, restrict PATH to busybox to ensure reproducibility of this script +# only unpack binaries if necessary +if [ "$newNPVersion" == "false" ]; then + + debug "binaries already installed" + export PATH="$dir/bin:$dir/bin" + +else + + debug "installing files" + + mkdir -p "$dir"/emptyroot + + # define arrays + stage1_file_path_list=() + stage1_file_offset_list=() + stage1_file_size_list=() + source <(tail -c+$((stage1_files_sh_offset + 1)) "$self" | head -c$stage1_files_sh_size || true) + + # install stage1 files + for ((i=0; i<${#stage1_file_path_list[@]}; i++)); do + path=${stage1_file_path_list[$i]} + offset=${stage1_file_offset_list[$i]} + size=${stage1_file_size_list[$i]} + path="${path#/*/*/*/*}" # remove "/nix/store/*/" prefix + if ! [ -e "$dir/$path" ]; then + mkdir -p "$dir/${path%/*}" + tail -c+$((offset + 1)) "$self" | head -c$size >"$dir/$path" || true + chmod +x "$dir/$path" # TODO better. add stage1_file_mode_list + fi + # if [ ${path#*/*/*/*/} = bin/busybox ]; then + if [ "$path" = bin/busybox ]; then + # install busybox symlinks + for bin in "${busyboxBins[@]}"; do + [ ! -e "$dir/${path%/*}/$bin" ] && ln -s busybox "$dir/${path%/*}/$bin" + # ~/.nix-portable/bin/ + done + fi + done + + export PATH="$dir/bin" + + # TODO use files from nix store + # install ssl cert bundle + # TODO? move to "$dir/etc/ssl/certs/ca-bundle.crt" + unzip $unzip_quiet -poj "$self" "$(removePrefix "/" "$caBundleZstd")" | zstd -d > "$dir"/ca-bundle.crt + + recreate_nix_conf +fi + + + +### setup SSL +# find ssl certs or use from nixpkgs +debug "figuring out ssl certs" +if [ -z "$SSL_CERT_FILE" ]; then + debug "SSL_CERT_FILE not defined. trying to find certs automatically" + if [ -e /etc/ssl/certs/ca-bundle.crt ]; then + SSL_CERT_FILE="$(realpath /etc/ssl/certs/ca-bundle.crt)" + export SSL_CERT_FILE + debug "found /etc/ssl/certs/ca-bundle.crt with real path $SSL_CERT_FILE" + elif [ -e /etc/ssl/certs/ca-certificates.crt ]; then + SSL_CERT_FILE="$(realpath /etc/ssl/certs/ca-certificates.crt)" + export SSL_CERT_FILE + debug "found /etc/ssl/certs/ca-certificates.crt with real path $SSL_CERT_FILE" + elif [ ! -e /etc/ssl/certs ]; then + debug "/etc/ssl/certs does not exist. Will use certs from nixpkgs." + export SSL_CERT_FILE="$dir"/ca-bundle.crt + else + debug "certs seem to reside in /etc/ssl/certs. No need to set up anything" + fi +fi +if [ -n "$SSL_CERT_FILE" ]; then + sslBind="$(realpath "$SSL_CERT_FILE") $dir/ca-bundle.crt" + export SSL_CERT_FILE="$dir/ca-bundle.crt" +else + sslBind="/etc/ssl /etc/ssl" +fi + + + +### detecting existing git installation +# we need to install git inside the wrapped environment +# unless custom git executable path is specified in NP_GIT, +# since the existing git might be incompatible to Nix (e.g. v1.x) +if [ -n "$NP_GIT" ]; then + doInstallGit=false + ln -s "$NP_GIT" "$dir/tmpbin/git" +else + doInstallGit=true +fi + + + +# stage2: now we can use unzip and zstd + +### install nix store +# Install all the nix store paths necessary for the current nix-portable version +missing="$( + while read -r path; do + if [ -e "$store/${path##*/}" ]; then continue; fi + echo "${path#*/}" # remove leading "/" + done < <( + unzip $unzip_quiet -p "$self" "$(removePrefix "/" "$storeTar/closureInfo/store-paths")" + ) +)" +if [ -n "$missing" ]; then + debug "extracting store paths" + mkdir -p "$store" + # "tar -k" has return code 2 when output files exist + unzip $unzip_quiet -p "$self" "$(removePrefix "/" "$storeTar/tar")" \ + | zstd -d \ + | tar x --strip-components 2 -C "$store" -- $missing +fi + + + +storePathOfFile(){ + file="$(realpath "$1")" + sPath="$(echo "$file" | awk -F "/" 'BEGIN{OFS="/";}{print $2,$3,$4}')" + echo "/$sPath" +} + + +collectBinds(){ + ### gather paths to bind for proot + # we cannot bind / to / without running into a lot of trouble, therefore + # we need to collect all top level directories and bind them inside an empty root + pathsTopLevel="$(find / -mindepth 1 -maxdepth 1 -not -name nix -not -name dev)" + + + toBind="" + for p in $pathsTopLevel; do + if [ -e "$p" ]; then + real="$(realpath "$p")" + if [ -e "$real" ]; then + if [[ "$real" == /nix/store/* ]]; then + storePath="$(storePathOfFile "$real")" + toBind="$toBind $storePath $storePath" + else + toBind="$toBind $real $p" + fi + fi + fi + done + + + # TODO: add /var/run/dbus/system_bus_socket + paths="/etc/host.conf /etc/hosts /etc/hosts.equiv /etc/mtab /etc/netgroup /etc/networks /etc/passwd /etc/group /etc/nsswitch.conf /etc/resolv.conf /etc/localtime $HOME" + + for p in $paths; do + if [ -e "$p" ]; then + real="$(realpath "$p")" + if [ -e "$real" ]; then + if [[ "$real" == /nix/store/* ]]; then + storePath="$(storePathOfFile "$real")" + toBind="$toBind $storePath $storePath" + else + toBind="$toBind $real $real" + fi + fi + fi + done + + # if we're on a nixos, the /bin/sh symlink will point + # to a /nix/store path which doesn't exit inside the wrapped env + # we fix this by binding busybox/bin to /bin + if test -s /bin/sh && [[ "$(realpath /bin/sh)" == /nix/store/* ]]; then + toBind="$toBind $dir/bin /bin" + fi + + # TODO remove + if false; then + # bind all libs required by our bins + find "$dir/lib/nix/store" -not -type d | while read -r path; do + toBind="$toBind $path $(removePrefix "$dir" "$path")" + done + fi +} + + +makeBindArgs(){ + arg="$1"; shift + sep="$1"; shift + binds="" + while :; do + if [ -n "$1" ]; then + from="$1"; shift + to="$1"; shift || { echo "no bind destination provided for $from!"; exit 3; } + binds="$binds $arg $from$sep$to"; + else + break + fi + done +} + + + +### get runtime paths +if [ -z "$NP_BWRAP" ]; then NP_BWRAP="$(PATH="$PATH_OLD:$PATH" which bwrap 2>/dev/null || true)"; fi +if [ -z "$NP_BWRAP" ]; then NP_BWRAP="$dir"/bin/bwrap; fi +debug "bwrap executable: $NP_BWRAP" +# if [ -z "$NP_NIX ]; then NP_NIX="$(PATH="$PATH_OLD:$PATH" which nix 2>/dev/null || true)"; fi +if [ -z "$NP_NIX" ]; then NP_NIX="$dir"/bin/nix; fi +debug "nix executable: $NP_NIX" +if [ -z "$NP_PROOT" ]; then NP_PROOT="$(PATH="$PATH_OLD:$PATH" which proot 2>/dev/null || true)"; fi +if [ -z "$NP_PROOT" ]; then NP_PROOT="$dir"/bin/proot; fi +debug "proot executable: $NP_PROOT" + + + +### select container runtime +debug "figuring out which runtime to use" +if [ -z "$NP_RUNTIME" ]; then + rm -rf "$dir"/tmp/__store + # check if nix --store works + if \ + debug "testing nix --store" \ + && mkdir -p "$dir"/tmp/ \ + && touch "$dir"/tmp/testfile \ + && "$NP_NIX" --store "$dir/tmp/__store" shell -f "$dir/mini-drv.nix" -c "$dir/bin/nix" store add-file --store "$dir/tmp/__store" "$dir/tmp/testfile" >/dev/null 2>&3; then + chmod -R +w "$dir"/tmp/__store + rm -rf "$dir"/tmp/__store + debug "nix --store works on this system -> will use nix as runtime" + NP_RUNTIME=nix + # check if bwrap works properly + # TODO? --bind "$PWD" "$PWD" + elif \ + debug "nix --store failed -> testing bwrap" \ + && $NP_BWRAP --bind "$dir"/emptyroot / --bind "$dir"/nix /nix --bind "$dir"/bin/busybox "$dir/true" "$dir/true" 2>&3 ; then + debug "bwrap seems to work on this system -> will use bwrap" + NP_RUNTIME=bwrap + # check if proot works properly + # TODO? -b "$PWD:$PWD" + elif \ + debug "bwrap failed -> testing proot" \ + && $NP_PROOT -b "$dir"/emptyroot:/ -b "$dir"/nix:/nix -b "$dir/bin/busybox:$dir/true" "$dir/true" 2>&3 ; then + debug "proot seems to work on this system -> will use proot" + NP_RUNTIME=proot + else + echo "error: no runtime is working on this system" + exit 1 + fi +else + debug "runtime selected via NP_RUNTIME: $NP_RUNTIME" +fi +rm -rf "$dir"/tmp/__store +debug "NP_RUNTIME: $NP_RUNTIME" + + + +### setup runtime args +if [ "$NP_RUNTIME" == "nix" ]; then + run="$NP_NIX shell -f $dir/mini-drv.nix -c" + PATH="$PATH:$store$(removePrefix "/nix/store" $nix)/bin" + export PATH + NP_CONF_STORE="$dir" + recreate_nix_conf +elif [ "$NP_RUNTIME" == "bwrap" ]; then + collectBinds + # shellcheck disable=SC2086 + makeBindArgs --bind " " $toBind $sslBind + run="$NP_BWRAP $BWRAP_ARGS \ + --bind $dir/emptyroot / \ + --dev-bind /dev /dev \ + --bind $dir/nix /nix \ + $binds" + # --bind $dir/bin/busybox /bin/sh \ +else + # proot + collectBinds + # shellcheck disable=SC2086 + makeBindArgs -b ":" $toBind $sslBind + run="$NP_PROOT $PROOT_ARGS \ + -r $dir/emptyroot \ + -b /dev:/dev \ + -b $dir/nix:/nix \ + $binds" + # -b $dir/bin/busybox:/bin/sh \ +fi +debug "base command will be: $run" + + + +### setup environment +export NIX_PATH="$dir/channels:nixpkgs=$dir/channels/nixpkgs" +mkdir -p "$dir"/channels +[ -h "$dir"/channels/nixpkgs ] || ln -s $nixpkgsSrc "$dir"/channels/nixpkgs + + +if false; then +### install nix store +# Install all the nix store paths necessary for the current nix-portable version +# We only unpack missing store paths from the tar archive. +index="$(cat $storeTar/closureInfo/store-paths)" + +# if [ ! "$NP_RUNTIME" == "nix" ]; then + # TODO reduce to boolean + missing="$( + for path in $index; do + if [ ! -e "$store/$(basename "$path")" ]; then + echo "nix/store/$(basename "$path")" + fi + done + )" + # TODO why? + export missing + + if [ -n "$missing" ]; then + debug "extracting missing store paths" + ( + mkdir -p "$dir"/tmp "$store"/ + rm -rf "$dir"/tmp/* + cd "$dir"/tmp + # shellcheck disable=SC2086 + unzip $unzip_quiet -p "$self" "$(removePrefix "/" "$storeTar/tar")" \ + | zstd -d \ + | tar x -k --strip-components 2 + mv "$dir"/tmp/* "$store"/ + ) + rm -rf "$dir"/tmp + fi + + # TODO remove? + if false; then + if [ -n "$missing" ]; then + debug "registering new store paths to DB" + # reg="$(cat $storeTar/closureInfo/registration)" + cmd="$run $store$(removePrefix "/nix/store" $nix)/bin/nix-store --load-db" + debug "running command: $cmd" + # echo "$reg" | $cmd + fi + fi +# fi +fi + + + +### select executable +# the executable can either be selected by +# - executing './nix-portable BIN_NAME', +# - symlinking to nix-portable, in which case the name of the symlink selects the nix executable +# Alternatively the executable can be hardcoded by specifying the argument 'executable' of nix-portable's default.nix file. +executable="$bundledExe" +if [ "$executable" != "" ]; then + bin="$executable" + debug "executable is hardcoded to: $bin" +elif [[ "$(basename "$0")" == nix-portable* ]]; then\ + if [ -z "$1" ]; then + echo "Error: please specify the nix binary to execute" + echo "Alternatively symlink against $0" + exit 1 + elif [ "$1" == "debug" ]; then + bin="$(which "$2")" + shift; shift + else + bin="$store$(removePrefix "/nix/store" $nix)/bin/$1" + shift + fi +else + bin="$store$(removePrefix "/nix/store" $nix)/bin/$(basename "$0")" +fi + + + +### check which runtime has been used previously +if [ -f "$dir/conf/last_runtime" ]; then + lastRuntime="$(cat "$dir/conf/last_runtime")" +else + lastRuntime= +fi + + + +### check if nix is functional with or without sandbox +# sandbox-fallback is not reliable: https://github.com/NixOS/nix/issues/4719 +if [ "$newNPVersion" == "true" ] || [ "$lastRuntime" != "$NP_RUNTIME" ]; then + nixBin="$store$(removePrefix "/nix/store" $nix)/bin/nix" + # if [ "$NP_RUNTIME" == "nix" ]; then + # nixBin="nix" + # else + # fi + debug "Testing if nix can build stuff without sandbox" + if ! $run "$nixBin" build --no-link -f "$dir/mini-drv.nix" --option sandbox false >&3 2>&3; then + echo "Fatal error: nix is unable to build packages" + exit 1 + fi + + debug "Testing if nix sandbox is functional" + if ! $run "$nixBin" build --no-link -f "$dir/mini-drv.nix" --option sandbox true >&3 2>&3; then + debug "Sandbox doesn't work -> disabling sandbox" + NP_CONF_SANDBOX=false + recreate_nix_conf + else + debug "Sandboxed builds work -> enabling sandbox" + NP_CONF_SANDBOX=true + recreate_nix_conf + fi + +fi + + +### save fingerprint and lastRuntime +if [ "$newNPVersion" == "true" ]; then + echo -n "$fingerprint" > "$dir/conf/fingerprint" +fi +if [ "$lastRuntime" != "$NP_RUNTIME" ]; then + echo -n "$NP_RUNTIME" > "$dir/conf/last_runtime" +fi + + + +### set PATH +# restore original PATH and append busybox +export PATH="$PATH_OLD:$dir/bin" +# apply overriding executable paths in $dir/tmpbin/ +export PATH="$dir/tmpbin:$PATH" + + + +### install git via nix, if git installation is not in /nix path +if $doInstallGit && [ ! -e "$store$(removePrefix "/nix/store" $git)" ] ; then + echo "Installing git. Disable this by specifying the git executable path with 'NP_GIT'" + $run "$store$(removePrefix "/nix/store" $nix)/bin/nix" build --impure --no-link --expr " + (import $nixpkgsSrc {}).$gitAttribute.out + " +else + debug "git already installed or manually specified" +fi + +### override the possibly existing git in the environment with the installed one +# excluding the case NP_GIT is set. +if $doInstallGit; then + export PATH="$git/bin:$PATH" +fi + + +### print elapsed time +end="$(date +%s%N)" # end time in nanoseconds +# time elapsed in millis with two decimal places +# elapsed="$(echo "scale=2; ($end - $start)/1000000000" | bc)" +elapsed="$(echo "scale=2; ($end - $start)/1000000" | bc)" +debug "Time to initialize nix-portable: $elapsed millis" + + + +### run commands +[ -z "$NP_RUN" ] && NP_RUN="$run" +debug "running command: $NP_RUN $bin $*" +exec $NP_RUN "$bin" "$@" +exit