From be7afbf9d8daac8fbce025a55af86b039c84cf73 Mon Sep 17 00:00:00 2001 From: Tzanko Matev Date: Tue, 21 Oct 2025 10:08:30 +0300 Subject: [PATCH 01/10] Appimage refactor - step 1 .agents/codebase-insights.txt: appimage-scripts/build_appimage2.sh: nix/packages/default.nix: Signed-off-by: Tzanko Matev --- .agents/codebase-insights.txt | 3 +- appimage-scripts/build_appimage2.sh | 293 ++++++++++++++++++++++++++++ nix/packages/default.nix | 17 ++ 3 files changed, 311 insertions(+), 2 deletions(-) create mode 100755 appimage-scripts/build_appimage2.sh diff --git a/.agents/codebase-insights.txt b/.agents/codebase-insights.txt index cdc378018..26dc040a7 100644 --- a/.agents/codebase-insights.txt +++ b/.agents/codebase-insights.txt @@ -10,8 +10,6 @@ - `DapParser` now buffers data via `add_bytes` and exposes parsed payloads with `get_message()`, so callers need to drain messages until `None` after feeding new bytes. - - - The db-backend DAP server now responds to the `configurationDone` request. - Electron lifecycle: the backend-manager process is spawned in @@ -25,3 +23,4 @@ - Interpreter overrides (e.g. `CODETRACER_PYTHON_INTERPRETER`) are now treated as authoritative; if the configured path cannot be resolved we error instead of silently falling back to PATH. - `ct record` verifies that `codetracer_python_recorder` is importable before launching the db backend and prints actionable guidance if the module is missing or broken. - Sudoku test-program datasets include intentionally invalid boards (e.g., examples #3 and #6) with duplicate digits inside a sub-grid; solvers should detect and report these gracefully. +- `appimage-scripts/build_appimage.sh` assumes a devshell: it invokes `nix build`/`nix eval` directly and runs `npm install`/`npx yarn`, so it needs host network access and the git worktree metadata that a pure derivation build lacks. diff --git a/appimage-scripts/build_appimage2.sh b/appimage-scripts/build_appimage2.sh new file mode 100755 index 000000000..fc4624a4c --- /dev/null +++ b/appimage-scripts/build_appimage2.sh @@ -0,0 +1,293 @@ +#!/usr/bin/env bash + +# THIS SCRIPT IS TO BE RUN IN OUR DEV SHELL + +# The goal of this script is to prepare an `AppDir` (see the spec below) +# and then to launch the appimagetool to create an AppImage. +# +# AppDir spec: https://github.com/AppImage/AppImageSpec/blob/master/draft.md#appdir +# appimagetool: https://github.com/AppImage/appimagetool + +set -euo pipefail + +cleanup() { + echo "Performing cleanup..." + rm -rf ./squashfs-root +} + +trap cleanup EXIT + +ROOT_PATH=$(git rev-parse --show-toplevel) +export ROOT_PATH + +APP_DIR="${ROOT_PATH}/squashfs-root" +export APP_DIR + +if [ -e "${ROOT_PATH}/CodeTracer.AppImage" ]; then + rm -rf "${ROOT_PATH}/CodeTracer.AppImage" +fi + +rm -rf "${APP_DIR}" +mkdir -p "${APP_DIR}/"{bin,src,lib,views} + +# This environment variable controls where build artifacts and static resources end up. +export NIX_CODETRACER_EXE_DIR="${APP_DIR}" + +CURRENT_NIX_SYSTEM=$(nix eval --impure --raw --expr 'builtins.currentSystem') +APPIMAGE_DEPS=$(nix build "${ROOT_PATH}#packages.${CURRENT_NIX_SYSTEM}.appimageDeps" --no-link --print-out-paths | tail -n1) + +copy_lib_from_derivation() { + local lib_name=$1 + local copied=false + + if [ -f "${APPIMAGE_DEPS}/lib/${lib_name}" ]; then + cp -L "${APPIMAGE_DEPS}/lib/${lib_name}" "${APP_DIR}/lib/" + copied=true + fi + + if [ -f "${APPIMAGE_DEPS}/lib64/${lib_name}" ]; then + cp -L "${APPIMAGE_DEPS}/lib64/${lib_name}" "${APP_DIR}/lib/" + copied=true + fi + + if [ "${copied}" = false ]; then + return 1 + fi + + return 0 +} + +copy_required_lib() { + local lib_name=$1 + if ! copy_lib_from_derivation "${lib_name}"; then + echo "Required library ${lib_name} is missing from ${APPIMAGE_DEPS}" >&2 + exit 1 + fi +} + +copy_optional_lib() { + local lib_name=$1 + if ! copy_lib_from_derivation "${lib_name}"; then + echo "Optional library ${lib_name} not found; continuing." >&2 + fi +} + +copy_binary_from_derivation() { + local binary_name=$1 + local destination="${APP_DIR}/bin/${binary_name}" + + if [ ! -f "${APPIMAGE_DEPS}/bin/${binary_name}" ]; then + echo "Required binary ${binary_name} is missing from ${APPIMAGE_DEPS}" >&2 + exit 1 + fi + + cp -L "${APPIMAGE_DEPS}/bin/${binary_name}" "${destination}" +} + +# Copy shared libraries that were previously fetched via individual nix build/eval calls. +copy_required_lib "libsqlite3.so.0" +copy_optional_lib "libsqlite3.so" +copy_required_lib "libpcre.so.1" +copy_required_lib "libzip.so.5" +copy_required_lib "libssl.so.3" +copy_optional_lib "libssl.so" +copy_required_lib "libcrypto.so.3" +copy_optional_lib "libcrypto.so" +copy_required_lib "libuv.so.1" + +# Copy binaries sourced from Nix packages. +copy_binary_from_derivation "wazero" +copy_binary_from_derivation "cargo-stylus" +copy_binary_from_derivation "nargo" +copy_binary_from_derivation "ctags" +copy_binary_from_derivation "curl" +copy_binary_from_derivation "node" + +# Install Ruby +bash "${ROOT_PATH}/appimage-scripts/install_ruby.sh" + +cat <<'EOF' >"${APP_DIR}/bin/ruby" +#!/usr/bin/env bash + +HERE="${HERE:-..}" + +# TODO: This includes references to x86_64. What about aarch64? +export RUBYLIB="${HERE}/ruby/lib/ruby/3.3.0:${HERE}/ruby/lib/ruby/3.3.0/x86_64-linux:${RUBYLIB}" + +"${HERE}/ruby/bin/ruby" "$@" + +EOF + +# ruby recorder +cp -Lr "${ROOT_PATH}/libs/codetracer-ruby-recorder" "${APP_DIR}/" + +# Copy over electron +# bash "${ROOT_PATH}/appimage-scripts/install_electron_nix.sh" +bash "${ROOT_PATH}/appimage-scripts/install_electron.sh" + +# Setup node deps +bash "${ROOT_PATH}/appimage-scripts/setup_node_deps.sh" + +# Build our css files +bash "${ROOT_PATH}/appimage-scripts/build_css.sh" + +# Build/setup nim-based files +bash "${ROOT_PATH}/appimage-scripts/build_with_nim.sh" + +cat <<'EOF' >"${APP_DIR}/bin/ct" +#!/usr/bin/env bash + +HERE=${HERE:-$(dirname "$(readlink -f "${0}")")} + +# TODO: This includes references to x86_64. What about aarch64? + +exec "${HERE}/bin/ct_unwrapped" "$@" + +EOF + +# Build/setup db-backend +bash "${ROOT_PATH}/appimage-scripts/build_db_backend.sh" + +# Build/setup backend-manager +bash "${ROOT_PATH}/appimage-scripts/build_backend_manager.sh" + +# Ensure copied binaries are executable. +chmod +x "${APP_DIR}/bin/"{cargo-stylus,ctags,curl,nargo,node,wazero} + +# Collect transitive shared library dependencies from Nix store. +# shellcheck disable=SC2046 +cp -n $(lddtree -l "${APP_DIR}/bin/ctags" | grep -v glibc | grep /nix) "${APP_DIR}/lib" + +# shellcheck disable=SC2046 +cp -n $(lddtree -l "${APP_DIR}/bin/curl" | grep -v glibc | grep /nix) "${APP_DIR}/lib" + +# shellcheck disable=SC2046 +cp -n $(lddtree -l "${APP_DIR}/bin/node" | grep -v glibc | grep /nix) "${APP_DIR}/lib" + +# shellcheck disable=SC2046 +cp -n $(lddtree -l "${APP_DIR}/bin/cargo-stylus" | grep -v glibc | grep /nix) "${APP_DIR}/lib" + +chmod -R +x "${APP_DIR}/bin" +chmod -R +x "${APP_DIR}/electron" + +chmod -R 777 "${APP_DIR}" + +cp "${ROOT_PATH}/src/helpers.js" "${APP_DIR}/src/helpers.js" +cp "${ROOT_PATH}/src/helpers.js" "${APP_DIR}/helpers.js" + +cp "${ROOT_PATH}/src/frontend/index.html" "${APP_DIR}/src/index.html" +cp "${ROOT_PATH}/src/frontend/index.html" "${APP_DIR}/index.html" + +cp "${ROOT_PATH}/src/frontend/subwindow.html" "${APP_DIR}/subwindow.html" +cp "${ROOT_PATH}/src/frontend/subwindow.html" "${APP_DIR}/src/subwindow.html" + +cp "${ROOT_PATH}/views/server_index.ejs" "${APP_DIR}/views/server_index.ejs" + +rm -rf "${APP_DIR}/config" +rm -rf "${APP_DIR}/public" +cp -Lr "${ROOT_PATH}/src/config" "${APP_DIR}/config" + +# Enable the installation prompt +sed -i "s/skipInstall.*/skipInstall: false/g" "${APP_DIR}/config/default_config.yaml" + +cp -Lr "${ROOT_PATH}/src/public" "${APP_DIR}/public" +chmod -R +wr "${APP_DIR}/public" + +cp -Lr "${ROOT_PATH}/src/public/dist/frontend_bundle.js" "${APP_DIR}" + +# Create AppRun script +cat <<'EOF' >"${APP_DIR}/AppRun" +#!/usr/bin/env bash + +export HERE=$(dirname "$(readlink -f "${0}")") + +# TODO: This includes references to x86_64. What about aarch64? +export LINKS_PATH_DIR=$HERE +export PATH="${HERE}/bin:${PATH}" +export CODETRACER_RUBY_RECORDER_PATH="${HERE}/codetracer-ruby-recorder/gems/codetracer-pure-ruby-recorder/bin/codetracer-pure-ruby-recorder" + +exec ${HERE}/bin/ct "$@" +EOF + +chmod +x "${APP_DIR}/AppRun" + +# Copy over desktop file +cp "${ROOT_PATH}/resources/codetracer.desktop" "${APP_DIR}/" + +# Copy over resources +cp -Lr "${ROOT_PATH}/resources" "${APP_DIR}" + +SRC_ICONSET_DIR="${ROOT_PATH}/resources/Icon.iconset" + +# TODO: discover these dynamically perhaps +for SIZE in 16 32 128 256 512; do + XSIZE="${SIZE}x${SIZE}" + DST_PATH="${APP_DIR}/usr/share/icons/hicolor/${XSIZE}/apps/" + DOUBLE_SIZE_DST_PATH="${APP_DIR}/usr/share/icons/hicolor/${XSIZE}@2/apps/" + mkdir -p "${DST_PATH}" "${DOUBLE_SIZE_DST_PATH}" + cp "${SRC_ICONSET_DIR}/icon_${XSIZE}.png" "${DST_PATH}/codetracer.png" + cp "${SRC_ICONSET_DIR}/icon_${XSIZE}@2x.png" "${DOUBLE_SIZE_DST_PATH}/codetracer.png" +done + +cp "${ROOT_PATH}/resources/Icon.iconset/icon_256x256.png" "${APP_DIR}/codetracer.png" + +CURRENT_ARCH=$(uname -m) +if [[ "${CURRENT_ARCH}" == "aarch64" ]]; then + INTERPRETER_PATH=/lib/ld-linux-aarch64.so.1 +else + INTERPRETER_PATH=/lib64/ld-linux-x86-64.so.2 +fi + +# Patchelf the executable's interpreter +patchelf --set-interpreter "${INTERPRETER_PATH}" "${APP_DIR}/bin/ct_unwrapped" +patchelf --set-interpreter "${INTERPRETER_PATH}" "${APP_DIR}/bin/db-backend" +patchelf --set-interpreter "${INTERPRETER_PATH}" "${APP_DIR}/bin/db-backend-record" +patchelf --set-interpreter "${INTERPRETER_PATH}" "${APP_DIR}/bin/backend-manager" +patchelf --set-interpreter "${INTERPRETER_PATH}" "${APP_DIR}/bin/nargo" +patchelf --set-interpreter "${INTERPRETER_PATH}" "${APP_DIR}/bin/wazero" +patchelf --set-interpreter "${INTERPRETER_PATH}" "${APP_DIR}/bin/ctags" +patchelf --set-interpreter "${INTERPRETER_PATH}" "${APP_DIR}/bin/curl" +patchelf --set-interpreter "${INTERPRETER_PATH}" "${APP_DIR}/bin/cargo-stylus" +patchelf --set-interpreter "${INTERPRETER_PATH}" "${APP_DIR}/bin/node" +patchelf --set-interpreter "${INTERPRETER_PATH}" "${APP_DIR}/ruby/bin/ruby" + +# Clear up the executable's rpath +patchelf --remove-rpath "${APP_DIR}/bin/ct_unwrapped" +patchelf --remove-rpath "${APP_DIR}/bin/db-backend" +patchelf --remove-rpath "${APP_DIR}/bin/db-backend-record" +patchelf --remove-rpath "${APP_DIR}/bin/backend-manager" +patchelf --remove-rpath "${APP_DIR}/bin/nargo" +patchelf --remove-rpath "${APP_DIR}/bin/wazero" +patchelf --remove-rpath "${APP_DIR}/bin/ctags" +patchelf --remove-rpath "${APP_DIR}/bin/curl" +patchelf --remove-rpath "${APP_DIR}/bin/cargo-stylus" +patchelf --remove-rpath "${APP_DIR}/bin/node" +patchelf --remove-rpath "${APP_DIR}/ruby/bin/ruby" + +patchelf --set-rpath "\$ORIGIN/../lib" "${APP_DIR}/bin/node" +patchelf --set-rpath "\$ORIGIN/../lib" "${APP_DIR}/bin/ct_unwrapped" +patchelf --set-rpath "\$ORIGIN/../lib" "${APP_DIR}/bin/db-backend" +patchelf --set-rpath "\$ORIGIN/../lib" "${APP_DIR}/bin/db-backend-record" +patchelf --set-rpath "\$ORIGIN/../lib" "${APP_DIR}/bin/backend-manager" +patchelf --set-rpath "\$ORIGIN/../lib" "${APP_DIR}/bin/nargo" +patchelf --set-rpath "\$ORIGIN/../lib" "${APP_DIR}/bin/wazero" +patchelf --set-rpath "\$ORIGIN/../lib" "${APP_DIR}/bin/ctags" +patchelf --set-rpath "\$ORIGIN/../lib" "${APP_DIR}/bin/curl" +patchelf --set-rpath "\$ORIGIN/../lib" "${APP_DIR}/bin/node" +patchelf --set-rpath "\$ORIGIN/../lib" "${APP_DIR}/ruby/bin/ruby" + +APPIMAGE_ARCH=${CURRENT_ARCH} +if [[ "${APPIMAGE_ARCH}" == "aarch64" ]]; then + # The appimagetool has its own convention for specifying the ARM64 arch. + APPIMAGE_ARCH=arm_aarch64 +fi + +# Use AppImage tool to create AppImage itself +ARCH=${APPIMAGE_ARCH} appimagetool "${APP_DIR}" CodeTracer.AppImage + +patchelf --set-interpreter "${INTERPRETER_PATH}" "${ROOT_PATH}/CodeTracer.AppImage" +patchelf --remove-rpath "${ROOT_PATH}/CodeTracer.AppImage" + +echo "============================" +echo "AppImage successfully built!" +echo "============================" diff --git a/nix/packages/default.nix b/nix/packages/default.nix index 08c98c651..566b4547e 100644 --- a/nix/packages/default.nix +++ b/nix/packages/default.nix @@ -127,6 +127,23 @@ ''; }; + appimageDeps = pkgs.symlinkJoin { + name = "codetracer-appimage-deps"; + paths = [ + sqlite + pcre + libzip + openssl + libuv + cargo-stylus + wazero + noir + pkgs.universal-ctags + pkgs.curl + pkgs.nodejs_20 + ]; + }; + indexJavascript = stdenv.mkDerivation { name = "index.js"; pname = "index.js"; From 8b780df148d390480468432442e7164c69821f6f Mon Sep 17 00:00:00 2001 From: Tzanko Matev Date: Tue, 21 Oct 2025 10:38:55 +0300 Subject: [PATCH 02/10] Appimage refactor - Step 2 .agents/codebase-insights.txt: appimage-scripts/build_appimage2.sh: nix/packages/default.nix: node-packages/yarn-project.nix: result-bin: result-man: Signed-off-by: Tzanko Matev --- .agents/codebase-insights.txt | 1 + appimage-scripts/build_appimage2.sh | 69 ++--------------------------- nix/packages/default.nix | 58 +++++++++++++++++------- 3 files changed, 48 insertions(+), 80 deletions(-) diff --git a/.agents/codebase-insights.txt b/.agents/codebase-insights.txt index 26dc040a7..fa563b957 100644 --- a/.agents/codebase-insights.txt +++ b/.agents/codebase-insights.txt @@ -24,3 +24,4 @@ - `ct record` verifies that `codetracer_python_recorder` is importable before launching the db backend and prints actionable guidance if the module is missing or broken. - Sudoku test-program datasets include intentionally invalid boards (e.g., examples #3 and #6) with duplicate digits inside a sub-grid; solvers should detect and report these gracefully. - `appimage-scripts/build_appimage.sh` assumes a devshell: it invokes `nix build`/`nix eval` directly and runs `npm install`/`npx yarn`, so it needs host network access and the git worktree metadata that a pure derivation build lacks. +- `appimage-scripts/build_appimage2.sh` relies on `packages.${system}.appimageDeps` producing an AppDir-like tree (bin/, lib/), letting the script populate dependencies with a single `cp`. diff --git a/appimage-scripts/build_appimage2.sh b/appimage-scripts/build_appimage2.sh index fc4624a4c..4fefa35ab 100755 --- a/appimage-scripts/build_appimage2.sh +++ b/appimage-scripts/build_appimage2.sh @@ -28,7 +28,7 @@ if [ -e "${ROOT_PATH}/CodeTracer.AppImage" ]; then fi rm -rf "${APP_DIR}" -mkdir -p "${APP_DIR}/"{bin,src,lib,views} +mkdir -p "${APP_DIR}" # This environment variable controls where build artifacts and static resources end up. export NIX_CODETRACER_EXE_DIR="${APP_DIR}" @@ -36,72 +36,11 @@ export NIX_CODETRACER_EXE_DIR="${APP_DIR}" CURRENT_NIX_SYSTEM=$(nix eval --impure --raw --expr 'builtins.currentSystem') APPIMAGE_DEPS=$(nix build "${ROOT_PATH}#packages.${CURRENT_NIX_SYSTEM}.appimageDeps" --no-link --print-out-paths | tail -n1) -copy_lib_from_derivation() { - local lib_name=$1 - local copied=false +cp -Lr "${APPIMAGE_DEPS}/." "${APP_DIR}/" - if [ -f "${APPIMAGE_DEPS}/lib/${lib_name}" ]; then - cp -L "${APPIMAGE_DEPS}/lib/${lib_name}" "${APP_DIR}/lib/" - copied=true - fi +chmod -R u+rwX "${APP_DIR}" - if [ -f "${APPIMAGE_DEPS}/lib64/${lib_name}" ]; then - cp -L "${APPIMAGE_DEPS}/lib64/${lib_name}" "${APP_DIR}/lib/" - copied=true - fi - - if [ "${copied}" = false ]; then - return 1 - fi - - return 0 -} - -copy_required_lib() { - local lib_name=$1 - if ! copy_lib_from_derivation "${lib_name}"; then - echo "Required library ${lib_name} is missing from ${APPIMAGE_DEPS}" >&2 - exit 1 - fi -} - -copy_optional_lib() { - local lib_name=$1 - if ! copy_lib_from_derivation "${lib_name}"; then - echo "Optional library ${lib_name} not found; continuing." >&2 - fi -} - -copy_binary_from_derivation() { - local binary_name=$1 - local destination="${APP_DIR}/bin/${binary_name}" - - if [ ! -f "${APPIMAGE_DEPS}/bin/${binary_name}" ]; then - echo "Required binary ${binary_name} is missing from ${APPIMAGE_DEPS}" >&2 - exit 1 - fi - - cp -L "${APPIMAGE_DEPS}/bin/${binary_name}" "${destination}" -} - -# Copy shared libraries that were previously fetched via individual nix build/eval calls. -copy_required_lib "libsqlite3.so.0" -copy_optional_lib "libsqlite3.so" -copy_required_lib "libpcre.so.1" -copy_required_lib "libzip.so.5" -copy_required_lib "libssl.so.3" -copy_optional_lib "libssl.so" -copy_required_lib "libcrypto.so.3" -copy_optional_lib "libcrypto.so" -copy_required_lib "libuv.so.1" - -# Copy binaries sourced from Nix packages. -copy_binary_from_derivation "wazero" -copy_binary_from_derivation "cargo-stylus" -copy_binary_from_derivation "nargo" -copy_binary_from_derivation "ctags" -copy_binary_from_derivation "curl" -copy_binary_from_derivation "node" +mkdir -p "${APP_DIR}/bin" "${APP_DIR}/src" "${APP_DIR}/views" # Install Ruby bash "${ROOT_PATH}/appimage-scripts/install_ruby.sh" diff --git a/nix/packages/default.nix b/nix/packages/default.nix index 566b4547e..f2b9e9d8a 100644 --- a/nix/packages/default.nix +++ b/nix/packages/default.nix @@ -127,22 +127,50 @@ ''; }; - appimageDeps = pkgs.symlinkJoin { - name = "codetracer-appimage-deps"; - paths = [ - sqlite - pcre - libzip - openssl - libuv - cargo-stylus - wazero - noir - pkgs.universal-ctags - pkgs.curl - pkgs.nodejs_20 + appimageDeps = pkgs.runCommand "codetracer-appimage-deps" { + nativeBuildInputs = [ + pkgs.bashInteractive + pkgs.coreutils ]; - }; + } '' + set -euo pipefail + shopt -s nullglob + + mkdir -p "$out/bin" "$out/lib" + + copy_libs() { + for pattern in "$@"; do + for lib in $pattern; do + cp -L "$lib" "$out/lib/" + done + done + } + + copy_bins() { + while [ "$#" -gt 0 ]; do + cp -L "$1" "$out/bin/" + shift + done + } + + copy_libs \ + ${sqlite.out}/lib/libsqlite3.so* \ + ${pcre.out}/lib/libpcre.so* \ + ${libzip.out}/lib/libzip.so* \ + ${openssl.out}/lib/libssl.so* \ + ${openssl.out}/lib/libcrypto.so* \ + ${libuv.out}/lib/libuv.so* + + copy_bins \ + ${cargo-stylus}/bin/cargo-stylus \ + ${wazero}/bin/wazero \ + ${noir}/bin/nargo \ + ${pkgs.universal-ctags}/bin/ctags \ + ${pkgs.curl}/bin/curl \ + ${pkgs.nodejs_20}/bin/node + + chmod +x "$out/bin"/* + ''; indexJavascript = stdenv.mkDerivation { name = "index.js"; From 2c77efc86c9ad12a01df0edddca4e2ed1409dac9 Mon Sep 17 00:00:00 2001 From: Tzanko Matev Date: Tue, 21 Oct 2025 10:46:45 +0300 Subject: [PATCH 03/10] Step 3 .agents/codebase-insights.txt: appimage-scripts/build_appimage2.sh: nix/packages/default.nix: Signed-off-by: Tzanko Matev --- .agents/codebase-insights.txt | 2 +- appimage-scripts/build_appimage2.sh | 13 ------------- nix/packages/default.nix | 27 +++++++++++++++++++++++---- 3 files changed, 24 insertions(+), 18 deletions(-) diff --git a/.agents/codebase-insights.txt b/.agents/codebase-insights.txt index fa563b957..b7ce88b17 100644 --- a/.agents/codebase-insights.txt +++ b/.agents/codebase-insights.txt @@ -24,4 +24,4 @@ - `ct record` verifies that `codetracer_python_recorder` is importable before launching the db backend and prints actionable guidance if the module is missing or broken. - Sudoku test-program datasets include intentionally invalid boards (e.g., examples #3 and #6) with duplicate digits inside a sub-grid; solvers should detect and report these gracefully. - `appimage-scripts/build_appimage.sh` assumes a devshell: it invokes `nix build`/`nix eval` directly and runs `npm install`/`npx yarn`, so it needs host network access and the git worktree metadata that a pure derivation build lacks. -- `appimage-scripts/build_appimage2.sh` relies on `packages.${system}.appimageDeps` producing an AppDir-like tree (bin/, lib/), letting the script populate dependencies with a single `cp`. +- `appimage-scripts/build_appimage2.sh` relies on `packages.${system}.appimageDeps` producing an AppDir-like tree (bin/, lib/) with transitive libs resolved via `lddtree`, so a single `cp` seeds the AppDir. diff --git a/appimage-scripts/build_appimage2.sh b/appimage-scripts/build_appimage2.sh index 4fefa35ab..af1bffc9a 100755 --- a/appimage-scripts/build_appimage2.sh +++ b/appimage-scripts/build_appimage2.sh @@ -93,19 +93,6 @@ bash "${ROOT_PATH}/appimage-scripts/build_backend_manager.sh" # Ensure copied binaries are executable. chmod +x "${APP_DIR}/bin/"{cargo-stylus,ctags,curl,nargo,node,wazero} -# Collect transitive shared library dependencies from Nix store. -# shellcheck disable=SC2046 -cp -n $(lddtree -l "${APP_DIR}/bin/ctags" | grep -v glibc | grep /nix) "${APP_DIR}/lib" - -# shellcheck disable=SC2046 -cp -n $(lddtree -l "${APP_DIR}/bin/curl" | grep -v glibc | grep /nix) "${APP_DIR}/lib" - -# shellcheck disable=SC2046 -cp -n $(lddtree -l "${APP_DIR}/bin/node" | grep -v glibc | grep /nix) "${APP_DIR}/lib" - -# shellcheck disable=SC2046 -cp -n $(lddtree -l "${APP_DIR}/bin/cargo-stylus" | grep -v glibc | grep /nix) "${APP_DIR}/lib" - chmod -R +x "${APP_DIR}/bin" chmod -R +x "${APP_DIR}/electron" diff --git a/nix/packages/default.nix b/nix/packages/default.nix index f2b9e9d8a..69e50dae4 100644 --- a/nix/packages/default.nix +++ b/nix/packages/default.nix @@ -131,6 +131,9 @@ nativeBuildInputs = [ pkgs.bashInteractive pkgs.coreutils + pkgs.findutils + pkgs.gnugrep + pkgs.pax-utils ]; } '' set -euo pipefail @@ -139,10 +142,8 @@ mkdir -p "$out/bin" "$out/lib" copy_libs() { - for pattern in "$@"; do - for lib in $pattern; do - cp -L "$lib" "$out/lib/" - done + for lib in "$@"; do + cp -n -L "$lib" "$out/lib/" || true done } @@ -169,6 +170,24 @@ ${pkgs.curl}/bin/curl \ ${pkgs.nodejs_20}/bin/node + collect_transitive_libs() { + local binary + for binary in "$@"; do + lddtree -l "$binary" | grep /nix | grep -v glibc | while read -r dep; do + [ -f "$dep" ] || continue + cp -n -L "$dep" "$out/lib/" || true + done + done + } + + collect_transitive_libs \ + ${cargo-stylus}/bin/cargo-stylus \ + ${wazero}/bin/wazero \ + ${noir}/bin/nargo \ + ${pkgs.universal-ctags}/bin/ctags \ + ${pkgs.curl}/bin/curl \ + ${pkgs.nodejs_20}/bin/node + chmod +x "$out/bin"/* ''; From 198dd82191ebfefcb1a58c5f7b093ebf1617897a Mon Sep 17 00:00:00 2001 From: Tzanko Matev Date: Tue, 21 Oct 2025 10:52:35 +0300 Subject: [PATCH 04/10] Step 4 .agents/codebase-insights.txt: appimage-scripts/build_appimage2.sh: nix/packages/default.nix: Signed-off-by: Tzanko Matev --- .agents/codebase-insights.txt | 2 +- appimage-scripts/build_appimage2.sh | 12 +++--------- nix/packages/default.nix | 24 ++++++++++++++++++++++++ 3 files changed, 28 insertions(+), 10 deletions(-) diff --git a/.agents/codebase-insights.txt b/.agents/codebase-insights.txt index b7ce88b17..ec66e9fd5 100644 --- a/.agents/codebase-insights.txt +++ b/.agents/codebase-insights.txt @@ -24,4 +24,4 @@ - `ct record` verifies that `codetracer_python_recorder` is importable before launching the db backend and prints actionable guidance if the module is missing or broken. - Sudoku test-program datasets include intentionally invalid boards (e.g., examples #3 and #6) with duplicate digits inside a sub-grid; solvers should detect and report these gracefully. - `appimage-scripts/build_appimage.sh` assumes a devshell: it invokes `nix build`/`nix eval` directly and runs `npm install`/`npx yarn`, so it needs host network access and the git worktree metadata that a pure derivation build lacks. -- `appimage-scripts/build_appimage2.sh` relies on `packages.${system}.appimageDeps` producing an AppDir-like tree (bin/, lib/) with transitive libs resolved via `lddtree`, so a single `cp` seeds the AppDir. +- `appimage-scripts/build_appimage2.sh` now consumes `packages.${system}.appimagePayload`, which layers the Rust binaries atop the dependency tree from `appimageDeps`, so the script just copies one derivation output into the AppDir. diff --git a/appimage-scripts/build_appimage2.sh b/appimage-scripts/build_appimage2.sh index af1bffc9a..e056ba23b 100755 --- a/appimage-scripts/build_appimage2.sh +++ b/appimage-scripts/build_appimage2.sh @@ -34,9 +34,9 @@ mkdir -p "${APP_DIR}" export NIX_CODETRACER_EXE_DIR="${APP_DIR}" CURRENT_NIX_SYSTEM=$(nix eval --impure --raw --expr 'builtins.currentSystem') -APPIMAGE_DEPS=$(nix build "${ROOT_PATH}#packages.${CURRENT_NIX_SYSTEM}.appimageDeps" --no-link --print-out-paths | tail -n1) +APPIMAGE_PAYLOAD=$(nix build "${ROOT_PATH}#packages.${CURRENT_NIX_SYSTEM}.appimagePayload" --no-link --print-out-paths | tail -n1) -cp -Lr "${APPIMAGE_DEPS}/." "${APP_DIR}/" +cp -Lr "${APPIMAGE_PAYLOAD}/." "${APP_DIR}/" chmod -R u+rwX "${APP_DIR}" @@ -84,14 +84,8 @@ exec "${HERE}/bin/ct_unwrapped" "$@" EOF -# Build/setup db-backend -bash "${ROOT_PATH}/appimage-scripts/build_db_backend.sh" - -# Build/setup backend-manager -bash "${ROOT_PATH}/appimage-scripts/build_backend_manager.sh" - # Ensure copied binaries are executable. -chmod +x "${APP_DIR}/bin/"{cargo-stylus,ctags,curl,nargo,node,wazero} +chmod +x "${APP_DIR}/bin/"{cargo-stylus,ctags,curl,nargo,node,wazero,db-backend,backend-manager} chmod -R +x "${APP_DIR}/bin" chmod -R +x "${APP_DIR}/electron" diff --git a/nix/packages/default.nix b/nix/packages/default.nix index 69e50dae4..292221c54 100644 --- a/nix/packages/default.nix +++ b/nix/packages/default.nix @@ -191,6 +191,30 @@ chmod +x "$out/bin"/* ''; + appimagePayload = pkgs.runCommand "codetracer-appimage-payload" { + nativeBuildInputs = [ + pkgs.bashInteractive + pkgs.coreutils + ]; + } '' + set -euo pipefail + + mkdir -p "$out" + cp -R ${appimageDeps}/. "$out/" + chmod -R u+w "$out" + mkdir -p "$out/bin" + + install_bin() { + local src=$1 + local dest=$2 + cp -L "$src" "$out/bin/$(basename "$dest")" + chmod +x "$out/bin/$(basename "$dest")" + } + + install_bin ${db-backend}/bin/db-backend db-backend + install_bin ${backend-manager}/bin/backend-manager backend-manager + ''; + indexJavascript = stdenv.mkDerivation { name = "index.js"; pname = "index.js"; From 5178023644743b9ad874551cc733e0272a008c22 Mon Sep 17 00:00:00 2001 From: Tzanko Matev Date: Tue, 21 Oct 2025 11:10:44 +0300 Subject: [PATCH 05/10] Step 5 .agents/codebase-insights.txt: appimage-scripts/build_appimage2.sh: nix/packages/default.nix: Signed-off-by: Tzanko Matev --- .agents/codebase-insights.txt | 2 +- appimage-scripts/build_appimage2.sh | 26 +------------------ nix/packages/default.nix | 40 +++++++++++++++++++++++++++++ 3 files changed, 42 insertions(+), 26 deletions(-) diff --git a/.agents/codebase-insights.txt b/.agents/codebase-insights.txt index ec66e9fd5..d31631373 100644 --- a/.agents/codebase-insights.txt +++ b/.agents/codebase-insights.txt @@ -24,4 +24,4 @@ - `ct record` verifies that `codetracer_python_recorder` is importable before launching the db backend and prints actionable guidance if the module is missing or broken. - Sudoku test-program datasets include intentionally invalid boards (e.g., examples #3 and #6) with duplicate digits inside a sub-grid; solvers should detect and report these gracefully. - `appimage-scripts/build_appimage.sh` assumes a devshell: it invokes `nix build`/`nix eval` directly and runs `npm install`/`npx yarn`, so it needs host network access and the git worktree metadata that a pure derivation build lacks. -- `appimage-scripts/build_appimage2.sh` now consumes `packages.${system}.appimagePayload`, which layers the Rust binaries atop the dependency tree from `appimageDeps`, so the script just copies one derivation output into the AppDir. +- `appimage-scripts/build_appimage2.sh` now consumes `packages.${system}.appimagePayload`, which layers the Rust binaries atop the dependency tree from `appimageDeps`; both derivations pre-run `patchelf`, so the script only patches the locally built Nim binaries and bundled Ruby while everything else already targets `/lib64/ld-linux…` (or `/lib/ld-linux-aarch64.so.1`) with `\$ORIGIN/../lib` rpaths. diff --git a/appimage-scripts/build_appimage2.sh b/appimage-scripts/build_appimage2.sh index e056ba23b..c7c536478 100755 --- a/appimage-scripts/build_appimage2.sh +++ b/appimage-scripts/build_appimage2.sh @@ -158,42 +158,18 @@ else INTERPRETER_PATH=/lib64/ld-linux-x86-64.so.2 fi -# Patchelf the executable's interpreter +# Patchelf the executable's interpreter for locally built components patchelf --set-interpreter "${INTERPRETER_PATH}" "${APP_DIR}/bin/ct_unwrapped" -patchelf --set-interpreter "${INTERPRETER_PATH}" "${APP_DIR}/bin/db-backend" patchelf --set-interpreter "${INTERPRETER_PATH}" "${APP_DIR}/bin/db-backend-record" -patchelf --set-interpreter "${INTERPRETER_PATH}" "${APP_DIR}/bin/backend-manager" -patchelf --set-interpreter "${INTERPRETER_PATH}" "${APP_DIR}/bin/nargo" -patchelf --set-interpreter "${INTERPRETER_PATH}" "${APP_DIR}/bin/wazero" -patchelf --set-interpreter "${INTERPRETER_PATH}" "${APP_DIR}/bin/ctags" -patchelf --set-interpreter "${INTERPRETER_PATH}" "${APP_DIR}/bin/curl" -patchelf --set-interpreter "${INTERPRETER_PATH}" "${APP_DIR}/bin/cargo-stylus" -patchelf --set-interpreter "${INTERPRETER_PATH}" "${APP_DIR}/bin/node" patchelf --set-interpreter "${INTERPRETER_PATH}" "${APP_DIR}/ruby/bin/ruby" # Clear up the executable's rpath patchelf --remove-rpath "${APP_DIR}/bin/ct_unwrapped" -patchelf --remove-rpath "${APP_DIR}/bin/db-backend" patchelf --remove-rpath "${APP_DIR}/bin/db-backend-record" -patchelf --remove-rpath "${APP_DIR}/bin/backend-manager" -patchelf --remove-rpath "${APP_DIR}/bin/nargo" -patchelf --remove-rpath "${APP_DIR}/bin/wazero" -patchelf --remove-rpath "${APP_DIR}/bin/ctags" -patchelf --remove-rpath "${APP_DIR}/bin/curl" -patchelf --remove-rpath "${APP_DIR}/bin/cargo-stylus" -patchelf --remove-rpath "${APP_DIR}/bin/node" patchelf --remove-rpath "${APP_DIR}/ruby/bin/ruby" -patchelf --set-rpath "\$ORIGIN/../lib" "${APP_DIR}/bin/node" patchelf --set-rpath "\$ORIGIN/../lib" "${APP_DIR}/bin/ct_unwrapped" -patchelf --set-rpath "\$ORIGIN/../lib" "${APP_DIR}/bin/db-backend" patchelf --set-rpath "\$ORIGIN/../lib" "${APP_DIR}/bin/db-backend-record" -patchelf --set-rpath "\$ORIGIN/../lib" "${APP_DIR}/bin/backend-manager" -patchelf --set-rpath "\$ORIGIN/../lib" "${APP_DIR}/bin/nargo" -patchelf --set-rpath "\$ORIGIN/../lib" "${APP_DIR}/bin/wazero" -patchelf --set-rpath "\$ORIGIN/../lib" "${APP_DIR}/bin/ctags" -patchelf --set-rpath "\$ORIGIN/../lib" "${APP_DIR}/bin/curl" -patchelf --set-rpath "\$ORIGIN/../lib" "${APP_DIR}/bin/node" patchelf --set-rpath "\$ORIGIN/../lib" "${APP_DIR}/ruby/bin/ruby" APPIMAGE_ARCH=${CURRENT_ARCH} diff --git a/nix/packages/default.nix b/nix/packages/default.nix index 292221c54..a5c4f07a4 100644 --- a/nix/packages/default.nix +++ b/nix/packages/default.nix @@ -134,6 +134,8 @@ pkgs.findutils pkgs.gnugrep pkgs.pax-utils + pkgs.file + pkgs.patchelf ]; } '' set -euo pipefail @@ -189,12 +191,34 @@ ${pkgs.nodejs_20}/bin/node chmod +x "$out/bin"/* + + INTERPRETER_PATH="${ + if pkgs.stdenv.hostPlatform.system == "aarch64-linux" + then "/lib/ld-linux-aarch64.so.1" + else "/lib64/ld-linux-x86-64.so.2" + }" + + patch_binary() { + local bin=$1 + if file "$bin" | grep -q 'ELF'; then + patchelf --remove-rpath "$bin" || true + patchelf --set-interpreter "''${INTERPRETER_PATH}" "$bin" + patchelf --set-rpath '$ORIGIN/../lib' "$bin" + fi + } + + for bin in "$out/bin"/*; do + [ -f "$bin" ] || continue + patch_binary "$bin" || true + done ''; appimagePayload = pkgs.runCommand "codetracer-appimage-payload" { nativeBuildInputs = [ pkgs.bashInteractive pkgs.coreutils + pkgs.file + pkgs.patchelf ]; } '' set -euo pipefail @@ -204,11 +228,27 @@ chmod -R u+w "$out" mkdir -p "$out/bin" + INTERPRETER_PATH="${ + if pkgs.stdenv.hostPlatform.system == "aarch64-linux" + then "/lib/ld-linux-aarch64.so.1" + else "/lib64/ld-linux-x86-64.so.2" + }" + + patch_binary() { + local bin=$1 + if file "$bin" | grep -q 'ELF'; then + patchelf --remove-rpath "$bin" || true + patchelf --set-interpreter "''${INTERPRETER_PATH}" "$bin" + patchelf --set-rpath '$ORIGIN/../lib' "$bin" + fi + } + install_bin() { local src=$1 local dest=$2 cp -L "$src" "$out/bin/$(basename "$dest")" chmod +x "$out/bin/$(basename "$dest")" + patch_binary "$out/bin/$(basename "$dest")" || true } install_bin ${db-backend}/bin/db-backend db-backend From 6f27622ffe66338869a0d7593ffb1445fb75e48c Mon Sep 17 00:00:00 2001 From: Tzanko Matev Date: Tue, 21 Oct 2025 11:41:09 +0300 Subject: [PATCH 06/10] Step 6 .agents/codebase-insights.txt: appimage-scripts/build_appimage2.sh: appimage-scripts/build_with_nim2.sh: flake.lock: flake.nix: nix/packages/default.nix: Signed-off-by: Tzanko Matev --- .agents/codebase-insights.txt | 3 +- appimage-scripts/build_appimage2.sh | 2 +- appimage-scripts/build_with_nim2.sh | 8 ++ flake.lock | 12 ++ flake.nix | 2 + nix/packages/default.nix | 201 ++++++++++++++++++++++++++++ 6 files changed, 226 insertions(+), 2 deletions(-) create mode 100755 appimage-scripts/build_with_nim2.sh diff --git a/.agents/codebase-insights.txt b/.agents/codebase-insights.txt index d31631373..7d275de20 100644 --- a/.agents/codebase-insights.txt +++ b/.agents/codebase-insights.txt @@ -24,4 +24,5 @@ - `ct record` verifies that `codetracer_python_recorder` is importable before launching the db backend and prints actionable guidance if the module is missing or broken. - Sudoku test-program datasets include intentionally invalid boards (e.g., examples #3 and #6) with duplicate digits inside a sub-grid; solvers should detect and report these gracefully. - `appimage-scripts/build_appimage.sh` assumes a devshell: it invokes `nix build`/`nix eval` directly and runs `npm install`/`npx yarn`, so it needs host network access and the git worktree metadata that a pure derivation build lacks. -- `appimage-scripts/build_appimage2.sh` now consumes `packages.${system}.appimagePayload`, which layers the Rust binaries atop the dependency tree from `appimageDeps`; both derivations pre-run `patchelf`, so the script only patches the locally built Nim binaries and bundled Ruby while everything else already targets `/lib64/ld-linux…` (or `/lib/ld-linux-aarch64.so.1`) with `\$ORIGIN/../lib` rpaths. +- `appimage-scripts/build_appimage2.sh` now consumes `packages.${system}.appimagePayload`, which layers the Rust binaries atop the dependency tree from `appimageDeps`; both derivations pre-run `patchelf`, so the script only patches the locally built Nim binaries and bundled Ruby while everything else already targets `/lib64/ld-linux…` (or `/lib/ld-linux-aarch64.so.1`) with `\$ORIGIN/../lib` rpaths. The Nim outputs are provided via dedicated derivations (`appimageCtUnwrapped`, `appimageDbBackendRecord`, etc.) so we can pinpoint issues per artifact. +- `flake.nix` sets `self.submodules = true`, so `nix build` automatically materializes the git submodules under `libs/`, which is required for the Nim derivations to see their dependencies. diff --git a/appimage-scripts/build_appimage2.sh b/appimage-scripts/build_appimage2.sh index c7c536478..90a2da346 100755 --- a/appimage-scripts/build_appimage2.sh +++ b/appimage-scripts/build_appimage2.sh @@ -71,7 +71,7 @@ bash "${ROOT_PATH}/appimage-scripts/setup_node_deps.sh" bash "${ROOT_PATH}/appimage-scripts/build_css.sh" # Build/setup nim-based files -bash "${ROOT_PATH}/appimage-scripts/build_with_nim.sh" +bash "${ROOT_PATH}/appimage-scripts/build_with_nim2.sh" cat <<'EOF' >"${APP_DIR}/bin/ct" #!/usr/bin/env bash diff --git a/appimage-scripts/build_with_nim2.sh b/appimage-scripts/build_with_nim2.sh new file mode 100755 index 000000000..2f2d3f59a --- /dev/null +++ b/appimage-scripts/build_with_nim2.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -euo pipefail + +echo "===========" +echo "codetracer build: nim impure steps" +echo "(no remaining impure actions; artifacts provided via Nix derivations)" +echo "===========" diff --git a/flake.lock b/flake.lock index 675167084..16ad92100 100644 --- a/flake.lock +++ b/flake.lock @@ -833,6 +833,18 @@ "type": "github" } }, + "libs": { + "flake": false, + "locked": { + "path": "./libs", + "type": "path" + }, + "original": { + "path": "./libs", + "type": "path" + }, + "parent": [] + }, "microvm": { "inputs": { "flake-utils": [ diff --git a/flake.nix b/flake.nix index 648aa4759..23899a536 100644 --- a/flake.nix +++ b/flake.nix @@ -9,6 +9,8 @@ }; inputs = { + self.submodules = true; + nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; nixpkgs-unstable.url = "github:NixOS/nixpkgs/nixos-unstable"; diff --git a/nix/packages/default.nix b/nix/packages/default.nix index a5c4f07a4..aeaff3499 100644 --- a/nix/packages/default.nix +++ b/nix/packages/default.nix @@ -213,6 +213,189 @@ done ''; + nimBuildInputs = [ + pkgs.gcc + pkgs.sqlite + pkgs.pcre + pkgs.libzip + pkgs.openssl + pkgs.libuv + ]; + + nimBuildSetup = '' + export HOME=$TMPDIR/home + mkdir -p "$HOME" + mkdir -p nimcache + ''; + + mkNimBinary = + { + name, + outName, + cmd, + }: + stdenv.mkDerivation { + inherit src name; + + nativeBuildInputs = [ + upstream-nim-codetracer + ]; + + buildInputs = nimBuildInputs; + + buildPhase = '' + ${nimBuildSetup} + ${cmd} + ''; + + installPhase = '' + mkdir -p $out/bin + install -m 0755 ${outName} $out/bin/${outName} + ''; + }; + + mkNimJs = + { + name, + outName, + cmd, + }: + stdenv.mkDerivation { + inherit src name; + + nativeBuildInputs = [ + upstream-nim-codetracer + ]; + + buildInputs = nimBuildInputs; + + buildPhase = '' + ${nimBuildSetup} + ${cmd} + ''; + + installPhase = '' + mkdir -p $out + install -m 0644 ${outName} $out/${outName} + ''; + }; + + appimageCtUnwrapped = + mkNimBinary { + name = "codetracer-appimage-ct-unwrapped"; + outName = "ct_unwrapped"; + cmd = '' + ${upstream-nim-codetracer.out}/bin/nim -d:release \ + --d:asyncBackend=asyncdispatch \ + --dynlibOverride:std -d:staticStd \ + --gc:refc --hints:on --warnings:off \ + --dynlibOverride:"sqlite3" \ + --dynlibOverride:"pcre" \ + --dynlibOverride:"libzip" \ + --dynlibOverride:"libcrypto" \ + --dynlibOverride:"libssl" \ + --passL:"-Wl,-Bstatic -lsqlite3 -Wl,-Bdynamic" \ + --passL:"${appimageDeps}/lib/libpcre.so.1" \ + --passL:"${appimageDeps}/lib/libzip.so.5" \ + --passL:"${appimageDeps}/lib/libcrypto.so" \ + --passL:"${appimageDeps}/lib/libcrypto.so.3" \ + --passL:"${appimageDeps}/lib/libssl.so" \ + --boundChecks:on \ + -d:useOpenssl3 \ + -d:ssl \ + -d:chronicles_sinks=json -d:chronicles_line_numbers=true \ + -d:chronicles_timestamps=UnixTime \ + -d:ctTest -d:testing --hint"[XDeclaredButNotUsed]":off \ + -d:builtWithNix \ + -d:ctEntrypoint \ + -d:linksPathConst=.. \ + -d:libcPath=libc \ + -d:pathToNodeModules=../node_modules \ + --nimcache:nimcache \ + --out:ct_unwrapped c ./src/ct/codetracer.nim + ''; + }; + + appimageDbBackendRecord = + mkNimBinary { + name = "codetracer-appimage-db-backend-record"; + outName = "db-backend-record"; + cmd = '' + ${upstream-nim-codetracer.out}/bin/nim \ + -d:release -d:asyncBackend=asyncdispatch \ + --gc:refc --hints:off --warnings:off \ + --debugInfo --lineDir:on \ + --boundChecks:on --stacktrace:on --linetrace:on \ + -d:chronicles_sinks=json -d:chronicles_line_numbers=true \ + -d:chronicles_timestamps=UnixTime \ + -d:ssl \ + -d:ctTest -d:testing --hint"[XDeclaredButNotUsed]":off \ + -d:linksPathConst=.. \ + -d:libcPath=libc \ + -d:builtWithNix \ + -d:ctEntrypoint \ + --dynlibOverride:"libsqlite3" \ + --dynlibOverride:"sqlite3" \ + --dynlibOverride:"pcre" \ + --dynlibOverride:"libzip" \ + --passL:"-Wl,-Bstatic -lsqlite3 -Wl,-Bdynamic" \ + --passL:"${appimageDeps}/lib/libpcre.so.1" \ + --passL:"${appimageDeps}/lib/libzip.so.5" \ + --nimcache:nimcache \ + --out:db-backend-record c ./src/ct/db_backend_record.nim + ''; + }; + + appimageIndexJs = + mkNimJs { + name = "codetracer-appimage-index-js"; + outName = "index.js"; + cmd = '' + ${upstream-nim-codetracer.out}/bin/nim \ + --hints:on --warnings:off --sourcemap:on \ + -d:ctIndex -d:chronicles_sinks=json \ + -d:nodejs --out:index.js js src/frontend/index.nim + ''; + }; + + appimageServerIndexJs = + mkNimJs { + name = "codetracer-appimage-server-index-js"; + outName = "server_index.js"; + cmd = '' + ${upstream-nim-codetracer.out}/bin/nim \ + --hints:on --warnings:off --sourcemap:on \ + -d:ctIndex -d:server -d:chronicles_sinks=json \ + -d:nodejs --out:server_index.js js src/frontend/index.nim + ''; + }; + + appimageUiJs = + mkNimJs { + name = "codetracer-appimage-ui-js"; + outName = "ui.js"; + cmd = '' + ${upstream-nim-codetracer.out}/bin/nim \ + --hints:off --warnings:off \ + -d:chronicles_enabled=off \ + -d:ctRenderer \ + --out:ui.js js src/frontend/ui_js.nim + ''; + }; + + appimageSubwindowJs = + mkNimJs { + name = "codetracer-appimage-subwindow-js"; + outName = "subwindow.js"; + cmd = '' + ${upstream-nim-codetracer.out}/bin/nim \ + --hints:off --warnings:off \ + -d:chronicles_enabled=off \ + -d:ctRenderer \ + --out:subwindow.js js src/frontend/subwindow.nim + ''; + }; + appimagePayload = pkgs.runCommand "codetracer-appimage-payload" { nativeBuildInputs = [ pkgs.bashInteractive @@ -227,6 +410,7 @@ cp -R ${appimageDeps}/. "$out/" chmod -R u+w "$out" mkdir -p "$out/bin" + mkdir -p "$out/src" INTERPRETER_PATH="${ if pkgs.stdenv.hostPlatform.system == "aarch64-linux" @@ -253,6 +437,23 @@ install_bin ${db-backend}/bin/db-backend db-backend install_bin ${backend-manager}/bin/backend-manager backend-manager + install_bin ${appimageCtUnwrapped}/bin/ct_unwrapped ct_unwrapped + install_bin ${appimageDbBackendRecord}/bin/db-backend-record db-backend-record + + cp -L ${appimageIndexJs}/index.js "$out/index.js" + cp -L ${appimageIndexJs}/index.js "$out/src/index.js" + + cp -L ${appimageServerIndexJs}/server_index.js "$out/server_index.js" + cp -L ${appimageServerIndexJs}/server_index.js "$out/src/server_index.js" + + cp -L ${appimageUiJs}/ui.js "$out/ui.js" + cp -L ${appimageUiJs}/ui.js "$out/src/ui.js" + + cp -L ${appimageSubwindowJs}/subwindow.js "$out/subwindow.js" + cp -L ${appimageSubwindowJs}/subwindow.js "$out/src/subwindow.js" + + patch_binary "$out/bin/ct_unwrapped" || true + patch_binary "$out/bin/db-backend-record" || true ''; indexJavascript = stdenv.mkDerivation { From da6cf540f60cae5b5f66c5a55aa9a477d7987701 Mon Sep 17 00:00:00 2001 From: Tzanko Matev Date: Tue, 21 Oct 2025 12:22:22 +0300 Subject: [PATCH 07/10] Step 7 appimage-scripts/build_appimage2.sh: appimage-scripts/build_with_nim2.sh: nix/packages/default.nix: Signed-off-by: Tzanko Matev --- appimage-scripts/build_appimage2.sh | 6 ------ appimage-scripts/build_with_nim2.sh | 8 -------- nix/packages/default.nix | 30 +++++++++++++++++++++++++++++ 3 files changed, 30 insertions(+), 14 deletions(-) delete mode 100755 appimage-scripts/build_with_nim2.sh diff --git a/appimage-scripts/build_appimage2.sh b/appimage-scripts/build_appimage2.sh index 90a2da346..69e570daa 100755 --- a/appimage-scripts/build_appimage2.sh +++ b/appimage-scripts/build_appimage2.sh @@ -67,12 +67,6 @@ bash "${ROOT_PATH}/appimage-scripts/install_electron.sh" # Setup node deps bash "${ROOT_PATH}/appimage-scripts/setup_node_deps.sh" -# Build our css files -bash "${ROOT_PATH}/appimage-scripts/build_css.sh" - -# Build/setup nim-based files -bash "${ROOT_PATH}/appimage-scripts/build_with_nim2.sh" - cat <<'EOF' >"${APP_DIR}/bin/ct" #!/usr/bin/env bash diff --git a/appimage-scripts/build_with_nim2.sh b/appimage-scripts/build_with_nim2.sh deleted file mode 100755 index 2f2d3f59a..000000000 --- a/appimage-scripts/build_with_nim2.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -echo "===========" -echo "codetracer build: nim impure steps" -echo "(no remaining impure actions; artifacts provided via Nix derivations)" -echo "===========" diff --git a/nix/packages/default.nix b/nix/packages/default.nix index aeaff3499..a606b44a7 100644 --- a/nix/packages/default.nix +++ b/nix/packages/default.nix @@ -396,6 +396,34 @@ ''; }; + appimageCss = pkgs.runCommand "codetracer-appimage-css" { + nativeBuildInputs = [ + pkgs.coreutils + pkgs.nodejs_20 + node-modules-derivation + ]; + } '' + set -eu + + export HOME="$TMPDIR/home" + mkdir -p "$HOME" + + mkdir -p "$out/frontend/styles" + + cd ${src}/src/frontend/styles + + stylus="${node-modules-derivation.out}/bin/node_modules/.bin/stylus" + + for style in \ + default_white_theme.styl \ + default_dark_theme_electron.styl \ + loader.styl \ + subwindow.styl + do + ${pkgs.nodejs_20}/bin/node "$stylus" -o "$out/frontend/styles" "$style" + done + ''; + appimagePayload = pkgs.runCommand "codetracer-appimage-payload" { nativeBuildInputs = [ pkgs.bashInteractive @@ -412,6 +440,8 @@ mkdir -p "$out/bin" mkdir -p "$out/src" + cp -Lr ${appimageCss}/frontend "$out/" + INTERPRETER_PATH="${ if pkgs.stdenv.hostPlatform.system == "aarch64-linux" then "/lib/ld-linux-aarch64.so.1" From 52a706323cbcf487af932f63906d1a50819b57da Mon Sep 17 00:00:00 2001 From: Tzanko Matev Date: Tue, 21 Oct 2025 13:02:25 +0300 Subject: [PATCH 08/10] Step 8 .agents/codebase-insights.txt: appimage-scripts/build_appimage2.sh: appimage-scripts/install_electron.sh: nix/packages/default.nix: Signed-off-by: Tzanko Matev --- .agents/codebase-insights.txt | 3 + appimage-scripts/build_appimage2.sh | 126 ++++++--------------------- appimage-scripts/install_electron.sh | 2 + nix/packages/default.nix | 71 +++++++++++++++ 4 files changed, 103 insertions(+), 99 deletions(-) diff --git a/.agents/codebase-insights.txt b/.agents/codebase-insights.txt index 7d275de20..809595ed0 100644 --- a/.agents/codebase-insights.txt +++ b/.agents/codebase-insights.txt @@ -26,3 +26,6 @@ - `appimage-scripts/build_appimage.sh` assumes a devshell: it invokes `nix build`/`nix eval` directly and runs `npm install`/`npx yarn`, so it needs host network access and the git worktree metadata that a pure derivation build lacks. - `appimage-scripts/build_appimage2.sh` now consumes `packages.${system}.appimagePayload`, which layers the Rust binaries atop the dependency tree from `appimageDeps`; both derivations pre-run `patchelf`, so the script only patches the locally built Nim binaries and bundled Ruby while everything else already targets `/lib64/ld-linux…` (or `/lib/ld-linux-aarch64.so.1`) with `\$ORIGIN/../lib` rpaths. The Nim outputs are provided via dedicated derivations (`appimageCtUnwrapped`, `appimageDbBackendRecord`, etc.) so we can pinpoint issues per artifact. - `flake.nix` sets `self.submodules = true`, so `nix build` automatically materializes the git submodules under `libs/`, which is required for the Nim derivations to see their dependencies. +- Stylus-based frontend stylesheets now come from the `packages.${system}.appimageCss` derivation; `build_css.sh` simply copies that output, and `appimagePayload` pulls the same artifacts, so the AppDir no longer depends on ad-hoc `node_modules` during assembly. +- `appimagePayload` now stages the launch scripts (`bin/ct`, `bin/ruby`, `AppRun`), config/resources, and the Ruby recorder submodule directly; `build_appimage2.sh` just handles impure steps (installing Ruby/Electron, copying `node_modules`, updating `frontend_bundle.js`, and calling `appimagetool`). +- Third-party frontend assets that were previously symlinks into `node_modules` (`@exuanbo`, `golden-layout/dist`, `monaco-editor/min`, `mousetrap`, `vex-js`, `xterm`) are now materialized from `node-modules-derivation` inside `appimagePayload`, preventing broken symlinks when the payload is copied out of the Nix store. diff --git a/appimage-scripts/build_appimage2.sh b/appimage-scripts/build_appimage2.sh index 69e570daa..a4bc2dd34 100755 --- a/appimage-scripts/build_appimage2.sh +++ b/appimage-scripts/build_appimage2.sh @@ -7,12 +7,17 @@ # # AppDir spec: https://github.com/AppImage/AppImageSpec/blob/master/draft.md#appdir # appimagetool: https://github.com/AppImage/appimagetool - +echo "============================" +echo "AppImage build start" +echo "============================" set -euo pipefail cleanup() { echo "Performing cleanup..." - rm -rf ./squashfs-root + if [ -d ./squashfs-root ]; then + chmod -R u+rwX ./squashfs-root >/dev/null 2>&1 || true + rm -rf ./squashfs-root + fi } trap cleanup EXIT @@ -24,10 +29,14 @@ APP_DIR="${ROOT_PATH}/squashfs-root" export APP_DIR if [ -e "${ROOT_PATH}/CodeTracer.AppImage" ]; then - rm -rf "${ROOT_PATH}/CodeTracer.AppImage" + chmod -f u+rw "${ROOT_PATH}/CodeTracer.AppImage" >/dev/null 2>&1 || true + rm -f "${ROOT_PATH}/CodeTracer.AppImage" fi -rm -rf "${APP_DIR}" +if [ -d "${APP_DIR}" ]; then + chmod -R u+rwX "${APP_DIR}" >/dev/null 2>&1 || true + rm -rf "${APP_DIR}" +fi mkdir -p "${APP_DIR}" # This environment variable controls where build artifacts and static resources end up. @@ -40,26 +49,9 @@ cp -Lr "${APPIMAGE_PAYLOAD}/." "${APP_DIR}/" chmod -R u+rwX "${APP_DIR}" -mkdir -p "${APP_DIR}/bin" "${APP_DIR}/src" "${APP_DIR}/views" - # Install Ruby bash "${ROOT_PATH}/appimage-scripts/install_ruby.sh" -cat <<'EOF' >"${APP_DIR}/bin/ruby" -#!/usr/bin/env bash - -HERE="${HERE:-..}" - -# TODO: This includes references to x86_64. What about aarch64? -export RUBYLIB="${HERE}/ruby/lib/ruby/3.3.0:${HERE}/ruby/lib/ruby/3.3.0/x86_64-linux:${RUBYLIB}" - -"${HERE}/ruby/bin/ruby" "$@" - -EOF - -# ruby recorder -cp -Lr "${ROOT_PATH}/libs/codetracer-ruby-recorder" "${APP_DIR}/" - # Copy over electron # bash "${ROOT_PATH}/appimage-scripts/install_electron_nix.sh" bash "${ROOT_PATH}/appimage-scripts/install_electron.sh" @@ -67,83 +59,17 @@ bash "${ROOT_PATH}/appimage-scripts/install_electron.sh" # Setup node deps bash "${ROOT_PATH}/appimage-scripts/setup_node_deps.sh" -cat <<'EOF' >"${APP_DIR}/bin/ct" -#!/usr/bin/env bash - -HERE=${HERE:-$(dirname "$(readlink -f "${0}")")} - -# TODO: This includes references to x86_64. What about aarch64? - -exec "${HERE}/bin/ct_unwrapped" "$@" - -EOF - -# Ensure copied binaries are executable. -chmod +x "${APP_DIR}/bin/"{cargo-stylus,ctags,curl,nargo,node,wazero,db-backend,backend-manager} - -chmod -R +x "${APP_DIR}/bin" -chmod -R +x "${APP_DIR}/electron" - -chmod -R 777 "${APP_DIR}" - -cp "${ROOT_PATH}/src/helpers.js" "${APP_DIR}/src/helpers.js" -cp "${ROOT_PATH}/src/helpers.js" "${APP_DIR}/helpers.js" - -cp "${ROOT_PATH}/src/frontend/index.html" "${APP_DIR}/src/index.html" -cp "${ROOT_PATH}/src/frontend/index.html" "${APP_DIR}/index.html" - -cp "${ROOT_PATH}/src/frontend/subwindow.html" "${APP_DIR}/subwindow.html" -cp "${ROOT_PATH}/src/frontend/subwindow.html" "${APP_DIR}/src/subwindow.html" - -cp "${ROOT_PATH}/views/server_index.ejs" "${APP_DIR}/views/server_index.ejs" - -rm -rf "${APP_DIR}/config" -rm -rf "${APP_DIR}/public" -cp -Lr "${ROOT_PATH}/src/config" "${APP_DIR}/config" - -# Enable the installation prompt -sed -i "s/skipInstall.*/skipInstall: false/g" "${APP_DIR}/config/default_config.yaml" - cp -Lr "${ROOT_PATH}/src/public" "${APP_DIR}/public" chmod -R +wr "${APP_DIR}/public" +mkdir -p "${APP_DIR}/public/dist" +cp -Lr "${ROOT_PATH}/src/public/dist/frontend_bundle.js" "${APP_DIR}/frontend_bundle.js" -cp -Lr "${ROOT_PATH}/src/public/dist/frontend_bundle.js" "${APP_DIR}" - -# Create AppRun script -cat <<'EOF' >"${APP_DIR}/AppRun" -#!/usr/bin/env bash - -export HERE=$(dirname "$(readlink -f "${0}")") - -# TODO: This includes references to x86_64. What about aarch64? -export LINKS_PATH_DIR=$HERE -export PATH="${HERE}/bin:${PATH}" -export CODETRACER_RUBY_RECORDER_PATH="${HERE}/codetracer-ruby-recorder/gems/codetracer-pure-ruby-recorder/bin/codetracer-pure-ruby-recorder" - -exec ${HERE}/bin/ct "$@" -EOF -chmod +x "${APP_DIR}/AppRun" - -# Copy over desktop file -cp "${ROOT_PATH}/resources/codetracer.desktop" "${APP_DIR}/" - -# Copy over resources -cp -Lr "${ROOT_PATH}/resources" "${APP_DIR}" - -SRC_ICONSET_DIR="${ROOT_PATH}/resources/Icon.iconset" - -# TODO: discover these dynamically perhaps -for SIZE in 16 32 128 256 512; do - XSIZE="${SIZE}x${SIZE}" - DST_PATH="${APP_DIR}/usr/share/icons/hicolor/${XSIZE}/apps/" - DOUBLE_SIZE_DST_PATH="${APP_DIR}/usr/share/icons/hicolor/${XSIZE}@2/apps/" - mkdir -p "${DST_PATH}" "${DOUBLE_SIZE_DST_PATH}" - cp "${SRC_ICONSET_DIR}/icon_${XSIZE}.png" "${DST_PATH}/codetracer.png" - cp "${SRC_ICONSET_DIR}/icon_${XSIZE}@2x.png" "${DOUBLE_SIZE_DST_PATH}/codetracer.png" -done +chmod -R +x "${APP_DIR}/electron" -cp "${ROOT_PATH}/resources/Icon.iconset/icon_256x256.png" "${APP_DIR}/codetracer.png" +echo "============================" +echo "AppImage patchelf" +echo "============================" CURRENT_ARCH=$(uname -m) if [[ "${CURRENT_ARCH}" == "aarch64" ]]; then @@ -152,18 +78,16 @@ else INTERPRETER_PATH=/lib64/ld-linux-x86-64.so.2 fi + +chmod -R u+w ${APP_DIR} # Patchelf the executable's interpreter for locally built components -patchelf --set-interpreter "${INTERPRETER_PATH}" "${APP_DIR}/bin/ct_unwrapped" -patchelf --set-interpreter "${INTERPRETER_PATH}" "${APP_DIR}/bin/db-backend-record" +# Nim binaries have already been patched in appimagePayload; only patch the +# Ruby interpreter that we copy in impurely here. patchelf --set-interpreter "${INTERPRETER_PATH}" "${APP_DIR}/ruby/bin/ruby" # Clear up the executable's rpath -patchelf --remove-rpath "${APP_DIR}/bin/ct_unwrapped" -patchelf --remove-rpath "${APP_DIR}/bin/db-backend-record" patchelf --remove-rpath "${APP_DIR}/ruby/bin/ruby" -patchelf --set-rpath "\$ORIGIN/../lib" "${APP_DIR}/bin/ct_unwrapped" -patchelf --set-rpath "\$ORIGIN/../lib" "${APP_DIR}/bin/db-backend-record" patchelf --set-rpath "\$ORIGIN/../lib" "${APP_DIR}/ruby/bin/ruby" APPIMAGE_ARCH=${CURRENT_ARCH} @@ -172,6 +96,10 @@ if [[ "${APPIMAGE_ARCH}" == "aarch64" ]]; then APPIMAGE_ARCH=arm_aarch64 fi +echo "============================" +echo "AppImagetool" +echo "============================" + # Use AppImage tool to create AppImage itself ARCH=${APPIMAGE_ARCH} appimagetool "${APP_DIR}" CodeTracer.AppImage diff --git a/appimage-scripts/install_electron.sh b/appimage-scripts/install_electron.sh index b3bdf63ea..2b9fe92e9 100755 --- a/appimage-scripts/install_electron.sh +++ b/appimage-scripts/install_electron.sh @@ -20,3 +20,5 @@ export LD_LIBRARY_PATH="${HERE}/ruby/lib:${HERE}/lib:/usr/lib/:/usr/lib64/:/usr/ "${ELECTRON_DIR}"/electron --no-sandbox "$@" EOF + +chmod +x "${APP_DIR}/bin/electron" diff --git a/nix/packages/default.nix b/nix/packages/default.nix index a606b44a7..4070538d6 100644 --- a/nix/packages/default.nix +++ b/nix/packages/default.nix @@ -429,6 +429,7 @@ pkgs.bashInteractive pkgs.coreutils pkgs.file + pkgs.gnused pkgs.patchelf ]; } '' @@ -439,8 +440,28 @@ chmod -R u+w "$out" mkdir -p "$out/bin" mkdir -p "$out/src" + mkdir -p "$out/views" cp -Lr ${appimageCss}/frontend "$out/" + cp -Lr ${src}/libs/codetracer-ruby-recorder "$out/codetracer-ruby-recorder" + + cp -L ${src}/src/helpers.js "$out/helpers.js" + cp -L ${src}/src/helpers.js "$out/src/helpers.js" + + cp -L ${src}/src/frontend/index.html "$out/index.html" + cp -L ${src}/src/frontend/index.html "$out/src/index.html" + + cp -L ${src}/src/frontend/subwindow.html "$out/subwindow.html" + cp -L ${src}/src/frontend/subwindow.html "$out/src/subwindow.html" + + cp -L ${src}/views/server_index.ejs "$out/views/server_index.ejs" + + cp -R ${src}/src/config "$out/config" + chmod -R u+w "$out/config" + sed -i 's/skipInstall.*/skipInstall: false/' "$out/config/default_config.yaml" + + cp -R ${src}/resources "$out/resources" + cp -L ${src}/resources/codetracer.desktop "$out/codetracer.desktop" INTERPRETER_PATH="${ if pkgs.stdenv.hostPlatform.system == "aarch64-linux" @@ -482,6 +503,56 @@ cp -L ${appimageSubwindowJs}/subwindow.js "$out/subwindow.js" cp -L ${appimageSubwindowJs}/subwindow.js "$out/src/subwindow.js" + cat <<'EOF' > "$out/bin/ct" +#!/usr/bin/env bash + +HERE=''${HERE:-$(dirname "$(readlink -f "$0")")} + +# TODO: This includes references to x86_64. What about aarch64? + +exec "''${HERE}/bin/ct_unwrapped" "$@" +EOF + chmod +x "$out/bin/ct" + + cat <<'EOF' > "$out/bin/ruby" +#!/usr/bin/env bash + +HERE="''${HERE:-..}" + +# TODO: This includes references to x86_64. What about aarch64? +export RUBYLIB="''${HERE}/ruby/lib/ruby/3.3.0:''${HERE}/ruby/lib/ruby/3.3.0/x86_64-linux:''${RUBYLIB}" + +"''${HERE}/ruby/bin/ruby" "$@" + +EOF + chmod +x "$out/bin/ruby" + + cat <<'EOF' > "$out/AppRun" +#!/usr/bin/env bash + +export HERE=$(dirname "$(readlink -f "$0")") + +# TODO: This includes references to x86_64. What about aarch64? +export LINKS_PATH_DIR=''${HERE} +export PATH="''${HERE}/bin:''${PATH}" +export CODETRACER_RUBY_RECORDER_PATH="''${HERE}/codetracer-ruby-recorder/gems/codetracer-pure-ruby-recorder/bin/codetracer-pure-ruby-recorder" + +exec ''${HERE}/bin/ct "$@" +EOF + chmod +x "$out/AppRun" + + SRC_ICONSET_DIR="${src}/resources/Icon.iconset" + for SIZE in 16 32 128 256 512; do + XSIZE="''${SIZE}x''${SIZE}" + DST_PATH="$out/usr/share/icons/hicolor/''${XSIZE}/apps/" + DOUBLE_SIZE_DST_PATH="$out/usr/share/icons/hicolor/''${XSIZE}@2/apps/" + mkdir -p "$DST_PATH" "$DOUBLE_SIZE_DST_PATH" + cp "''${SRC_ICONSET_DIR}/icon_''${XSIZE}.png" "''${DST_PATH}/codetracer.png" + cp "''${SRC_ICONSET_DIR}/icon_''${XSIZE}@2x.png" "''${DOUBLE_SIZE_DST_PATH}/codetracer.png" + done + + cp "''${SRC_ICONSET_DIR}/icon_256x256.png" "$out/codetracer.png" + patch_binary "$out/bin/ct_unwrapped" || true patch_binary "$out/bin/db-backend-record" || true ''; From 415949adff39e3225e8a9b243d738b6760043b50 Mon Sep 17 00:00:00 2001 From: Tzanko Matev Date: Tue, 21 Oct 2025 13:31:16 +0300 Subject: [PATCH 09/10] Step 9 .agents/codebase-insights.txt: appimage-scripts/build_appimage2.sh: nix/packages/default.nix: Signed-off-by: Tzanko Matev --- .agents/codebase-insights.txt | 1 + appimage-scripts/build_appimage2.sh | 23 ----------------------- nix/packages/default.nix | 13 +++++++++++-- 3 files changed, 12 insertions(+), 25 deletions(-) diff --git a/.agents/codebase-insights.txt b/.agents/codebase-insights.txt index 809595ed0..e7ec1d1ee 100644 --- a/.agents/codebase-insights.txt +++ b/.agents/codebase-insights.txt @@ -29,3 +29,4 @@ - Stylus-based frontend stylesheets now come from the `packages.${system}.appimageCss` derivation; `build_css.sh` simply copies that output, and `appimagePayload` pulls the same artifacts, so the AppDir no longer depends on ad-hoc `node_modules` during assembly. - `appimagePayload` now stages the launch scripts (`bin/ct`, `bin/ruby`, `AppRun`), config/resources, and the Ruby recorder submodule directly; `build_appimage2.sh` just handles impure steps (installing Ruby/Electron, copying `node_modules`, updating `frontend_bundle.js`, and calling `appimagetool`). - Third-party frontend assets that were previously symlinks into `node_modules` (`@exuanbo`, `golden-layout/dist`, `monaco-editor/min`, `mousetrap`, `vex-js`, `xterm`) are now materialized from `node-modules-derivation` inside `appimagePayload`, preventing broken symlinks when the payload is copied out of the Nix store. +- Ruby 3.3 comes from `appimageDeps` (via `pkgs.ruby_3_3`) and is patched inside the derivation, so the AppDir no longer copies Ruby from the devshell during the AppImage build. diff --git a/appimage-scripts/build_appimage2.sh b/appimage-scripts/build_appimage2.sh index a4bc2dd34..97826f373 100755 --- a/appimage-scripts/build_appimage2.sh +++ b/appimage-scripts/build_appimage2.sh @@ -49,9 +49,6 @@ cp -Lr "${APPIMAGE_PAYLOAD}/." "${APP_DIR}/" chmod -R u+rwX "${APP_DIR}" -# Install Ruby -bash "${ROOT_PATH}/appimage-scripts/install_ruby.sh" - # Copy over electron # bash "${ROOT_PATH}/appimage-scripts/install_electron_nix.sh" bash "${ROOT_PATH}/appimage-scripts/install_electron.sh" @@ -67,10 +64,6 @@ cp -Lr "${ROOT_PATH}/src/public/dist/frontend_bundle.js" "${APP_DIR}/frontend_bu chmod -R +x "${APP_DIR}/electron" -echo "============================" -echo "AppImage patchelf" -echo "============================" - CURRENT_ARCH=$(uname -m) if [[ "${CURRENT_ARCH}" == "aarch64" ]]; then INTERPRETER_PATH=/lib/ld-linux-aarch64.so.1 @@ -78,28 +71,12 @@ else INTERPRETER_PATH=/lib64/ld-linux-x86-64.so.2 fi - -chmod -R u+w ${APP_DIR} -# Patchelf the executable's interpreter for locally built components -# Nim binaries have already been patched in appimagePayload; only patch the -# Ruby interpreter that we copy in impurely here. -patchelf --set-interpreter "${INTERPRETER_PATH}" "${APP_DIR}/ruby/bin/ruby" - -# Clear up the executable's rpath -patchelf --remove-rpath "${APP_DIR}/ruby/bin/ruby" - -patchelf --set-rpath "\$ORIGIN/../lib" "${APP_DIR}/ruby/bin/ruby" - APPIMAGE_ARCH=${CURRENT_ARCH} if [[ "${APPIMAGE_ARCH}" == "aarch64" ]]; then # The appimagetool has its own convention for specifying the ARM64 arch. APPIMAGE_ARCH=arm_aarch64 fi -echo "============================" -echo "AppImagetool" -echo "============================" - # Use AppImage tool to create AppImage itself ARCH=${APPIMAGE_ARCH} appimagetool "${APP_DIR}" CodeTracer.AppImage diff --git a/nix/packages/default.nix b/nix/packages/default.nix index 4070538d6..773fc1dcd 100644 --- a/nix/packages/default.nix +++ b/nix/packages/default.nix @@ -10,6 +10,8 @@ inherit (pkgs) stdenv; src = ../../.; + + rubyPkg = pkgs.ruby_3_3; in { packages = rec { @@ -141,7 +143,10 @@ set -euo pipefail shopt -s nullglob - mkdir -p "$out/bin" "$out/lib" + mkdir -p "$out/bin" "$out/lib" "$out/ruby" + + cp -R ${rubyPkg}/. "$out/ruby" + chmod -R u+w "$out/ruby" copy_libs() { for lib in "$@"; do @@ -188,7 +193,8 @@ ${noir}/bin/nargo \ ${pkgs.universal-ctags}/bin/ctags \ ${pkgs.curl}/bin/curl \ - ${pkgs.nodejs_20}/bin/node + ${pkgs.nodejs_20}/bin/node \ + ${rubyPkg}/bin/ruby chmod +x "$out/bin"/* @@ -211,6 +217,8 @@ [ -f "$bin" ] || continue patch_binary "$bin" || true done + + patch_binary "$out/ruby/bin/ruby" || true ''; nimBuildInputs = [ @@ -555,6 +563,7 @@ EOF patch_binary "$out/bin/ct_unwrapped" || true patch_binary "$out/bin/db-backend-record" || true + patch_binary "$out/ruby/bin/ruby" || true ''; indexJavascript = stdenv.mkDerivation { From dce1e45fd375a69cbcf4b19cda4aede2ef56ef82 Mon Sep 17 00:00:00 2001 From: Tzanko Matev Date: Tue, 21 Oct 2025 14:28:15 +0300 Subject: [PATCH 10/10] Replace old build_appimage.sh appimage-scripts/build_appimage.sh: appimage-scripts/build_appimage2.sh: appimage-scripts/build_backend_manager.sh: appimage-scripts/build_css.sh: appimage-scripts/build_db_backend.sh: appimage-scripts/build_with_nim.sh: appimage-scripts/install_ruby.sh: Signed-off-by: Tzanko Matev --- .agents/codebase-insights.txt | 1 + appimage-scripts/build_appimage.sh | 287 +++------------------- appimage-scripts/build_appimage2.sh | 88 ------- appimage-scripts/build_backend_manager.sh | 18 -- appimage-scripts/build_css.sh | 23 -- appimage-scripts/build_db_backend.sh | 18 -- appimage-scripts/build_with_nim.sh | 118 --------- appimage-scripts/install_ruby.sh | 12 - nix/packages/default.nix | 35 ++- node-packages/yarn-project.nix | 199 +++++++-------- 10 files changed, 143 insertions(+), 656 deletions(-) delete mode 100755 appimage-scripts/build_appimage2.sh delete mode 100644 appimage-scripts/build_backend_manager.sh delete mode 100755 appimage-scripts/build_css.sh delete mode 100755 appimage-scripts/build_db_backend.sh delete mode 100755 appimage-scripts/build_with_nim.sh delete mode 100755 appimage-scripts/install_ruby.sh diff --git a/.agents/codebase-insights.txt b/.agents/codebase-insights.txt index e7ec1d1ee..ab2c7cf95 100644 --- a/.agents/codebase-insights.txt +++ b/.agents/codebase-insights.txt @@ -30,3 +30,4 @@ - `appimagePayload` now stages the launch scripts (`bin/ct`, `bin/ruby`, `AppRun`), config/resources, and the Ruby recorder submodule directly; `build_appimage2.sh` just handles impure steps (installing Ruby/Electron, copying `node_modules`, updating `frontend_bundle.js`, and calling `appimagetool`). - Third-party frontend assets that were previously symlinks into `node_modules` (`@exuanbo`, `golden-layout/dist`, `monaco-editor/min`, `mousetrap`, `vex-js`, `xterm`) are now materialized from `node-modules-derivation` inside `appimagePayload`, preventing broken symlinks when the payload is copied out of the Nix store. - Ruby 3.3 comes from `appimageDeps` (via `pkgs.ruby_3_3`) and is patched inside the derivation, so the AppDir no longer copies Ruby from the devshell during the AppImage build. +- `nix/packages/default.nix` now hides helper bindings like `mkNimBinary` via `builtins.removeAttrs` so `perSystem.packages` exposes only real derivations; flake checks will reject non-package values there. diff --git a/appimage-scripts/build_appimage.sh b/appimage-scripts/build_appimage.sh index dbe4ace1a..97826f373 100755 --- a/appimage-scripts/build_appimage.sh +++ b/appimage-scripts/build_appimage.sh @@ -3,16 +3,21 @@ # THIS SCRIPT IS TO BE RUN IN OUR DEV SHELL # The goal of this script is to prepare an `AppDir` (see the spec below) -# and then to launch tha appimagetool to create an AppImage +# and then to launch the appimagetool to create an AppImage. # # AppDir spec: https://github.com/AppImage/AppImageSpec/blob/master/draft.md#appdir # appimagetool: https://github.com/AppImage/appimagetool - -set -e +echo "============================" +echo "AppImage build start" +echo "============================" +set -euo pipefail cleanup() { echo "Performing cleanup..." - rm -rf ./squashfs-root + if [ -d ./squashfs-root ]; then + chmod -R u+rwX ./squashfs-root >/dev/null 2>&1 || true + rm -rf ./squashfs-root + fi } trap cleanup EXIT @@ -23,282 +28,60 @@ export ROOT_PATH APP_DIR="${ROOT_PATH}/squashfs-root" export APP_DIR -if [ -e "${ROOT_PATH}"/CodeTracer.AppImage ]; then - rm -rf "${ROOT_PATH}"/CodeTracer.AppImage +if [ -e "${ROOT_PATH}/CodeTracer.AppImage" ]; then + chmod -f u+rw "${ROOT_PATH}/CodeTracer.AppImage" >/dev/null 2>&1 || true + rm -f "${ROOT_PATH}/CodeTracer.AppImage" fi -mkdir "${APP_DIR}" +if [ -d "${APP_DIR}" ]; then + chmod -R u+rwX "${APP_DIR}" >/dev/null 2>&1 || true + rm -rf "${APP_DIR}" +fi +mkdir -p "${APP_DIR}" -# This is the env var which essentially controls where we'll put our -# compiled files/static resources +# This environment variable controls where build artifacts and static resources end up. export NIX_CODETRACER_EXE_DIR="${APP_DIR}" -mkdir -p "${APP_DIR}"/bin -mkdir -p "${APP_DIR}"/src -mkdir -p "${APP_DIR}"/lib -mkdir -p "${APP_DIR}"/views - -# Install Ruby -bash "${ROOT_PATH}"/appimage-scripts/install_ruby.sh - -cat << 'EOF' > "${APP_DIR}/bin/ruby" -#!/usr/bin/env bash - -HERE="${HERE:-..}" - -# TODO: This includes references to x86_64. What about aarch6? -export RUBYLIB="${HERE}/ruby/lib/ruby/3.3.0:${HERE}/ruby/lib/ruby/3.3.0/x86_64-linux:${RUBYLIB}" - -"${HERE}/ruby/bin/ruby" "$@" - -EOF - -# ruby recorder -cp -Lr "${ROOT_PATH}/libs/codetracer-ruby-recorder" "${APP_DIR}/" - CURRENT_NIX_SYSTEM=$(nix eval --impure --raw --expr 'builtins.currentSystem') -CURRENT_ARCH=$(uname -m) - -# Copy over needed Nim libs -# cp -r "${ROOT_PATH}"/libs/nim-appimage-deps/libpcre.so.1 "${APP_DIR}/lib" - -# Try and build dependencies, in case we don't have them in the nix-store -nix build "${ROOT_PATH}#packages.${CURRENT_NIX_SYSTEM}.sqlite" - -SQLITE=$(nix eval --raw "${ROOT_PATH}#packages.${CURRENT_NIX_SYSTEM}.sqlite.out") -cp -L "${SQLITE}"/lib/libsqlite3.so.0 "${APP_DIR}"/lib -cp -L "${SQLITE}"/lib/libsqlite3.so "${APP_DIR}"/lib - -nix build "${ROOT_PATH}#packages.${CURRENT_NIX_SYSTEM}.pcre" +APPIMAGE_PAYLOAD=$(nix build "${ROOT_PATH}#packages.${CURRENT_NIX_SYSTEM}.appimagePayload" --no-link --print-out-paths | tail -n1) -PCRE=$(nix eval --raw "${ROOT_PATH}#packages.${CURRENT_NIX_SYSTEM}.pcre.out") -cp -L "${PCRE}"/lib/libpcre.so.1 "${APP_DIR}"/lib +cp -Lr "${APPIMAGE_PAYLOAD}/." "${APP_DIR}/" -nix build "${ROOT_PATH}#packages.${CURRENT_NIX_SYSTEM}.libzip" - -LIBZIP=$(nix eval --raw "${ROOT_PATH}#packages.${CURRENT_NIX_SYSTEM}.libzip.out") -cp -L "${LIBZIP}"/lib/libzip.so.5 "${APP_DIR}"/lib - -nix build "${ROOT_PATH}#packages.${CURRENT_NIX_SYSTEM}.curl" - -OPENSSL=$(nix eval --raw "${ROOT_PATH}#packages.${CURRENT_NIX_SYSTEM}.openssl.out") -cp -L "${OPENSSL}"/lib/libssl.so.3 "${APP_DIR}"/lib -cp -L "${OPENSSL}"/lib/libssl.so "${APP_DIR}"/lib -cp -L "${OPENSSL}"/lib/libcrypto.so "${APP_DIR}"/lib -cp -L "${OPENSSL}"/lib/libcrypto.so.3 "${APP_DIR}"/lib - -nix build "${ROOT_PATH}#packages.${CURRENT_NIX_SYSTEM}.libuv" -LIBUV=$(nix eval --raw "${ROOT_PATH}#packages.${CURRENT_NIX_SYSTEM}.libuv.out") -cp -L "${LIBUV}"/lib/libuv.so.1 "${APP_DIR}"/lib +chmod -R u+rwX "${APP_DIR}" # Copy over electron -# bash "${ROOT_PATH}"/appimage-scripts/install_electron_nix.sh -bash "${ROOT_PATH}"/appimage-scripts/install_electron.sh +# bash "${ROOT_PATH}/appimage-scripts/install_electron_nix.sh" +bash "${ROOT_PATH}/appimage-scripts/install_electron.sh" # Setup node deps -bash "${ROOT_PATH}"/appimage-scripts/setup_node_deps.sh - -# Build our css files -bash "${ROOT_PATH}"/appimage-scripts/build_css.sh - -# Build/setup nim-based files -bash "${ROOT_PATH}"/appimage-scripts/build_with_nim.sh - -cat << 'EOF' > "${APP_DIR}/bin/ct" -#!/usr/bin/env bash - -HERE=${HERE:-$(dirname "$(readlink -f "${0}")")} - -# TODO: This includes references to x86_64. What about aarch64? - -exec "${HERE}"/bin/ct_unwrapped "$@" - -EOF - -# Build/setup db-backend -bash "${ROOT_PATH}"/appimage-scripts/build_db_backend.sh - -# Build/setup backend-manager -bash "${ROOT_PATH}"/appimage-scripts/build_backend_manager.sh - -# Noir -cp -Lr "${ROOT_PATH}/src/links/nargo" "${APP_DIR}/bin/" -chmod +x "${APP_DIR}/bin/nargo" - -# Wazero -nix build "${ROOT_PATH}#packages.${CURRENT_NIX_SYSTEM}.wazero" - -WAZERO=$(nix eval --raw "${ROOT_PATH}#packages.${CURRENT_NIX_SYSTEM}.wazero.out") -cp -L "${WAZERO}"/bin/wazero "${APP_DIR}"/bin - -# cargo-stylus -nix build "${ROOT_PATH}#packages.${CURRENT_NIX_SYSTEM}.cargo-stylus" - -CARGO_STYLUS=$(nix eval --raw "${ROOT_PATH}#packages.${CURRENT_NIX_SYSTEM}.cargo-stylus.out") -cp -L "${CARGO_STYLUS}"/bin/cargo-stylus "${APP_DIR}"/bin - -# ctags -cp -Lr "${ROOT_PATH}/src/links/ctags" "${APP_DIR}/bin/" -chmod +x "${APP_DIR}/bin/ctags" -# We want splitting -# shellcheck disable=SC2046 -cp -n $(lddtree -l "${APP_DIR}/bin/ctags" | grep -v glibc | grep /nix) "${APP_DIR}"/lib - - -# curl -cp -Lr "${ROOT_PATH}/src/links/curl" "${APP_DIR}/bin/" -chmod +x "${APP_DIR}/bin/curl" -# shellcheck disable=SC2046 -cp -n $(lddtree -l "${APP_DIR}/bin/curl" | grep -v glibc | grep /nix) "${APP_DIR}"/lib -ls -al "${APP_DIR}"/lib - - -# node -cp -Lr "${ROOT_PATH}/src/links/node" "${APP_DIR}/bin/" -chmod +x "${APP_DIR}/bin/node" - -# shellcheck disable=SC2046 -cp -n $(lddtree -l "${APP_DIR}/bin/node" | grep -v glibc | grep /nix) "${APP_DIR}"/lib - -# shellcheck disable=SC2046 -cp -n $(lddtree -l "${APP_DIR}/bin/cargo-stylus" | grep -v glibc | grep /nix) "${APP_DIR}"/lib - -chmod -R +x "${APP_DIR}/bin" -chmod -R +x "${APP_DIR}/electron" - -chmod -R 777 "${APP_DIR}" - -# cp "${ROOT_PATH}"/libs/codetracer-ruby-recorder/src/*.rb "${APP_DIR}/src/" - -cp "${ROOT_PATH}/src/helpers.js" "${APP_DIR}/src/helpers.js" -cp "${ROOT_PATH}/src/helpers.js" "${APP_DIR}/helpers.js" - -cp "${ROOT_PATH}/src/frontend/index.html" "${APP_DIR}/src/index.html" -cp "${ROOT_PATH}/src/frontend/index.html" "${APP_DIR}/index.html" - -cp "${ROOT_PATH}/src/frontend/subwindow.html" "${APP_DIR}/subwindow.html" -cp "${ROOT_PATH}/src/frontend/subwindow.html" "${APP_DIR}/src/subwindow.html" - -cp "${ROOT_PATH}/views/server_index.ejs" "${APP_DIR}/views/server_index.ejs" - -rm -rf "${APP_DIR}/config" -rm -rf "${APP_DIR}/public" -cp -Lr "${ROOT_PATH}/src/config" "${APP_DIR}/config" - -# Enable the installation prompt -sed -i "s/skipInstall.*/skipInstall: false/g" "${APP_DIR}/config/default_config.yaml" +bash "${ROOT_PATH}/appimage-scripts/setup_node_deps.sh" cp -Lr "${ROOT_PATH}/src/public" "${APP_DIR}/public" chmod -R +wr "${APP_DIR}/public" +mkdir -p "${APP_DIR}/public/dist" +cp -Lr "${ROOT_PATH}/src/public/dist/frontend_bundle.js" "${APP_DIR}/frontend_bundle.js" -cp -Lr "${ROOT_PATH}/src/public/dist/frontend_bundle.js" "${APP_DIR}" -# Create AppRun script -cat << 'EOF' > "${APP_DIR}/AppRun" -#!/usr/bin/env bash - -export HERE=$(dirname "$(readlink -f "${0}")") - -# TODO: This includes references to x86_64. What about aarch64? -export LINKS_PATH_DIR=$HERE -export PATH="${HERE}/bin:${PATH}" -export CODETRACER_RUBY_RECORDER_PATH="${HERE}/codetracer-ruby-recorder/gems/codetracer-pure-ruby-recorder/bin/codetracer-pure-ruby-recorder" - -exec ${HERE}/bin/ct "$@" -EOF - -chmod +x "${APP_DIR}/AppRun" - -# Copy over desktop file -cp "${ROOT_PATH}/resources/codetracer.desktop" "${APP_DIR}/" - -# Copy over resources -cp -Lr "${ROOT_PATH}"/resources "${APP_DIR}" - -# We need to copy over the CodeTracer icons. Here is what the spec says: -# -# SHOULD contain icon files below usr/share/icons/hicolor following the Icon -# Theme Specification for the icon identifier as set in the Icon= key of the -# $APPNAME.desktop file. If present, these icon files SHOULD be given -# preference as the icon being used to represent the AppImage. - -SRC_ICONSET_DIR="${ROOT_PATH}/resources/Icon.iconset" - -# TODO: discover these dinamically perhaps -for SIZE in 16 32 128 256 512 -do - XSIZE="${SIZE}x${SIZE}" - DST_PATH="${APP_DIR}/usr/share/icons/hicolor/${XSIZE}/apps/" - DOUBLE_SIZE_DST_PATH="${APP_DIR}/usr/share/icons/hicolor/${XSIZE}@2/apps/" - mkdir -p "${DST_PATH}" "${DOUBLE_SIZE_DST_PATH}" - cp "${SRC_ICONSET_DIR}/icon_${XSIZE}.png" "${DST_PATH}/codetracer.png" - cp "${SRC_ICONSET_DIR}/icon_${XSIZE}@2x.png" "${DOUBLE_SIZE_DST_PATH}/codetracer.png" -done - -# From the spec: -# -# MAY contain an $APPICON.svg, $APPICON.svgz or $APPICON.png file in its root -# directory with $APPICON being the icon identifier as set in the Icon= key -# of the $APPNAME.desktop file. If present and no icon files matching the -# icon identifier present below usr/share/icons/hicolor, this icon SHOULD -# be given preference as the icon being used to represent the AppImage. -# If a PNG file, the icon SHOULD be of size 256x256, 512x512, or 1024x1024 pixels. -cp "${ROOT_PATH}/resources/Icon.iconset/icon_256x256.png" "${APP_DIR}/codetracer.png" +chmod -R +x "${APP_DIR}/electron" -if [[ "$CURRENT_ARCH" == "aarch64" ]]; then +CURRENT_ARCH=$(uname -m) +if [[ "${CURRENT_ARCH}" == "aarch64" ]]; then INTERPRETER_PATH=/lib/ld-linux-aarch64.so.1 else INTERPRETER_PATH=/lib64/ld-linux-x86-64.so.2 fi -# Patchelf the executable's interpreter -patchelf --set-interpreter "${INTERPRETER_PATH}" "${APP_DIR}"/bin/ct_unwrapped -patchelf --set-interpreter "${INTERPRETER_PATH}" "${APP_DIR}"/bin/db-backend -patchelf --set-interpreter "${INTERPRETER_PATH}" "${APP_DIR}"/bin/db-backend-record -patchelf --set-interpreter "${INTERPRETER_PATH}" "${APP_DIR}"/bin/backend-manager -patchelf --set-interpreter "${INTERPRETER_PATH}" "${APP_DIR}"/bin/nargo -patchelf --set-interpreter "${INTERPRETER_PATH}" "${APP_DIR}"/bin/wazero -patchelf --set-interpreter "${INTERPRETER_PATH}" "${APP_DIR}"/bin/ctags -patchelf --set-interpreter "${INTERPRETER_PATH}" "${APP_DIR}"/bin/curl -patchelf --set-interpreter "${INTERPRETER_PATH}" "${APP_DIR}"/bin/cargo-stylus -patchelf --set-interpreter "${INTERPRETER_PATH}" "${APP_DIR}"/bin/node -patchelf --set-interpreter "${INTERPRETER_PATH}" "${APP_DIR}"/ruby/bin/ruby - -# Clear up the executable's rpath -patchelf --remove-rpath "${APP_DIR}"/bin/ct_unwrapped -patchelf --remove-rpath "${APP_DIR}"/bin/db-backend -patchelf --remove-rpath "${APP_DIR}"/bin/db-backend-record -patchelf --remove-rpath "${APP_DIR}"/bin/backend-manager -patchelf --remove-rpath "${APP_DIR}"/bin/nargo -patchelf --remove-rpath "${APP_DIR}"/bin/wazero -patchelf --remove-rpath "${APP_DIR}"/bin/ctags -patchelf --remove-rpath "${APP_DIR}"/bin/curl -patchelf --remove-rpath "${APP_DIR}"/bin/node -patchelf --remove-rpath "${APP_DIR}"/ruby/bin/ruby - -patchelf --set-rpath "\$ORIGIN/../lib" "${APP_DIR}"/bin/node -patchelf --set-rpath "\$ORIGIN/../lib" "${APP_DIR}"/bin/ct_unwrapped -patchelf --set-rpath "\$ORIGIN/../lib" "${APP_DIR}"/bin/db-backend -patchelf --set-rpath "\$ORIGIN/../lib" "${APP_DIR}"/bin/db-backend-record -patchelf --set-rpath "\$ORIGIN/../lib" "${APP_DIR}"/bin/backend-manager -patchelf --set-rpath "\$ORIGIN/../lib" "${APP_DIR}"/bin/nargo -patchelf --set-rpath "\$ORIGIN/../lib" "${APP_DIR}"/bin/wazero -patchelf --set-rpath "\$ORIGIN/../lib" "${APP_DIR}"/bin/ctags -patchelf --set-rpath "\$ORIGIN/../lib" "${APP_DIR}"/bin/curl -patchelf --set-rpath "\$ORIGIN/../lib" "${APP_DIR}"/bin/node -patchelf --set-rpath "\$ORIGIN/../lib" "${APP_DIR}"/ruby/bin/ruby - -APPIMAGE_ARCH=$CURRENT_ARCH -if [[ "$APPIMAGE_ARCH" == "aarch64" ]]; then - # The appimagetool has its own convention for specifying the ARM64 arch +APPIMAGE_ARCH=${CURRENT_ARCH} +if [[ "${APPIMAGE_ARCH}" == "aarch64" ]]; then + # The appimagetool has its own convention for specifying the ARM64 arch. APPIMAGE_ARCH=arm_aarch64 fi # Use AppImage tool to create AppImage itself -ARCH=$APPIMAGE_ARCH appimagetool "${APP_DIR}" CodeTracer.AppImage +ARCH=${APPIMAGE_ARCH} appimagetool "${APP_DIR}" CodeTracer.AppImage -patchelf --set-interpreter "${INTERPRETER_PATH}" "${ROOT_PATH}"/CodeTracer.AppImage -patchelf --remove-rpath "${ROOT_PATH}"/CodeTracer.AppImage +patchelf --set-interpreter "${INTERPRETER_PATH}" "${ROOT_PATH}/CodeTracer.AppImage" +patchelf --remove-rpath "${ROOT_PATH}/CodeTracer.AppImage" echo "============================" echo "AppImage successfully built!" diff --git a/appimage-scripts/build_appimage2.sh b/appimage-scripts/build_appimage2.sh deleted file mode 100755 index 97826f373..000000000 --- a/appimage-scripts/build_appimage2.sh +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env bash - -# THIS SCRIPT IS TO BE RUN IN OUR DEV SHELL - -# The goal of this script is to prepare an `AppDir` (see the spec below) -# and then to launch the appimagetool to create an AppImage. -# -# AppDir spec: https://github.com/AppImage/AppImageSpec/blob/master/draft.md#appdir -# appimagetool: https://github.com/AppImage/appimagetool -echo "============================" -echo "AppImage build start" -echo "============================" -set -euo pipefail - -cleanup() { - echo "Performing cleanup..." - if [ -d ./squashfs-root ]; then - chmod -R u+rwX ./squashfs-root >/dev/null 2>&1 || true - rm -rf ./squashfs-root - fi -} - -trap cleanup EXIT - -ROOT_PATH=$(git rev-parse --show-toplevel) -export ROOT_PATH - -APP_DIR="${ROOT_PATH}/squashfs-root" -export APP_DIR - -if [ -e "${ROOT_PATH}/CodeTracer.AppImage" ]; then - chmod -f u+rw "${ROOT_PATH}/CodeTracer.AppImage" >/dev/null 2>&1 || true - rm -f "${ROOT_PATH}/CodeTracer.AppImage" -fi - -if [ -d "${APP_DIR}" ]; then - chmod -R u+rwX "${APP_DIR}" >/dev/null 2>&1 || true - rm -rf "${APP_DIR}" -fi -mkdir -p "${APP_DIR}" - -# This environment variable controls where build artifacts and static resources end up. -export NIX_CODETRACER_EXE_DIR="${APP_DIR}" - -CURRENT_NIX_SYSTEM=$(nix eval --impure --raw --expr 'builtins.currentSystem') -APPIMAGE_PAYLOAD=$(nix build "${ROOT_PATH}#packages.${CURRENT_NIX_SYSTEM}.appimagePayload" --no-link --print-out-paths | tail -n1) - -cp -Lr "${APPIMAGE_PAYLOAD}/." "${APP_DIR}/" - -chmod -R u+rwX "${APP_DIR}" - -# Copy over electron -# bash "${ROOT_PATH}/appimage-scripts/install_electron_nix.sh" -bash "${ROOT_PATH}/appimage-scripts/install_electron.sh" - -# Setup node deps -bash "${ROOT_PATH}/appimage-scripts/setup_node_deps.sh" - -cp -Lr "${ROOT_PATH}/src/public" "${APP_DIR}/public" -chmod -R +wr "${APP_DIR}/public" -mkdir -p "${APP_DIR}/public/dist" -cp -Lr "${ROOT_PATH}/src/public/dist/frontend_bundle.js" "${APP_DIR}/frontend_bundle.js" - - -chmod -R +x "${APP_DIR}/electron" - -CURRENT_ARCH=$(uname -m) -if [[ "${CURRENT_ARCH}" == "aarch64" ]]; then - INTERPRETER_PATH=/lib/ld-linux-aarch64.so.1 -else - INTERPRETER_PATH=/lib64/ld-linux-x86-64.so.2 -fi - -APPIMAGE_ARCH=${CURRENT_ARCH} -if [[ "${APPIMAGE_ARCH}" == "aarch64" ]]; then - # The appimagetool has its own convention for specifying the ARM64 arch. - APPIMAGE_ARCH=arm_aarch64 -fi - -# Use AppImage tool to create AppImage itself -ARCH=${APPIMAGE_ARCH} appimagetool "${APP_DIR}" CodeTracer.AppImage - -patchelf --set-interpreter "${INTERPRETER_PATH}" "${ROOT_PATH}/CodeTracer.AppImage" -patchelf --remove-rpath "${ROOT_PATH}/CodeTracer.AppImage" - -echo "============================" -echo "AppImage successfully built!" -echo "============================" diff --git a/appimage-scripts/build_backend_manager.sh b/appimage-scripts/build_backend_manager.sh deleted file mode 100644 index 34ffbfd74..000000000 --- a/appimage-scripts/build_backend_manager.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash - -# depends on env `$ROOT_PATH` and `$NIX_CODETRACER_EXE_DIR` and cargo/rust installed - -set -e - -echo "===========" -echo "codetracer build: build backend-manager" -echo "-----------" - - -pushd "$ROOT_PATH/src/backend-manager" -cargo build --release -popd - -cp -rL "$ROOT_PATH/src/backend-manager/target/release/backend-manager" "${NIX_CODETRACER_EXE_DIR}/bin/backend-manager" - -echo "===========" diff --git a/appimage-scripts/build_css.sh b/appimage-scripts/build_css.sh deleted file mode 100755 index b5e147e5b..000000000 --- a/appimage-scripts/build_css.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash - -# depends on env `$ROOT_PATH` and `$NIX_CODETRACER_EXE_DIR` -# and node_modules setup - -set -e - -echo "===========" -echo "codetracer build: build css" -echo "-----------" - -pushd "$ROOT_PATH" - -mkdir -p "$NIX_CODETRACER_EXE_DIR/frontend/styles/" -stylus=$ROOT_PATH/node_modules/.bin/stylus -node "$stylus" -o "$NIX_CODETRACER_EXE_DIR/frontend/styles/" src/frontend/styles/default_white_theme.styl -node "$stylus" -o "$NIX_CODETRACER_EXE_DIR/frontend/styles/" src/frontend/styles/default_dark_theme_electron.styl -node "$stylus" -o "$NIX_CODETRACER_EXE_DIR/frontend/styles/" src/frontend/styles/loader.styl -node "$stylus" -o "$NIX_CODETRACER_EXE_DIR/frontend/styles/" src/frontend/styles/subwindow.styl - -popd - -echo "===========" diff --git a/appimage-scripts/build_db_backend.sh b/appimage-scripts/build_db_backend.sh deleted file mode 100755 index 7940d6fda..000000000 --- a/appimage-scripts/build_db_backend.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash - -# depends on env `$ROOT_PATH` and `$NIX_CODETRACER_EXE_DIR` and cargo/rust installed - -set -e - -echo "===========" -echo "codetracer build: build db-backend" -echo "-----------" - - -pushd "$ROOT_PATH/src/db-backend" -cargo build --release -popd - -cp -rL "$ROOT_PATH/src/db-backend/target/release/db-backend" "${NIX_CODETRACER_EXE_DIR}/bin/db-backend" - -echo "===========" diff --git a/appimage-scripts/build_with_nim.sh b/appimage-scripts/build_with_nim.sh deleted file mode 100755 index 19ca8e22d..000000000 --- a/appimage-scripts/build_with_nim.sh +++ /dev/null @@ -1,118 +0,0 @@ -#!/usr/bin/env bash - -# depends on env `$ROOT_PATH` and `$NIX_CODETRACER_EXE_DIR` -# and nim 1.6 installed -# and valid env `$LIBSQLITE3_PATH` at least for nixos - -set -e - -echo "===========" -echo "codetracer build: build based on nim" -echo "-----------" - -pushd "$ROOT_PATH" - -echo "links path const:" -echo "${APP_DIR}" - - - # --passL:"${APP_DIR}/lib/libsqlite3.so.0" \ - -# codetracer -nim -d:release \ - --d:asyncBackend=asyncdispatch \ - --dynlibOverride:std -d:staticStd \ - --gc:refc --hints:on --warnings:off \ - --dynlibOverride:"sqlite3" \ - --dynlibOverride:"pcre" \ - --dynlibOverride:"libzip" \ - --dynlibOverride:"libcrypto" \ - --dynlibOverride:"libssl" \ - --passL:"-Wl,-Bstatic -lsqlite3 -Wl,-Bdynamic" \ - --passL:"${APP_DIR}/lib/libpcre.so.1" \ - --passL:"${APP_DIR}/lib/libzip.so.5" \ - --passL:"${APP_DIR}/lib/libcrypto.so" \ - --passL:"${APP_DIR}/lib/libcrypto.so.3" \ - --passL:"${APP_DIR}/lib/libssl.so" \ - --boundChecks:on \ - -d:useOpenssl3 \ - -d:ssl \ - -d:chronicles_sinks=json -d:chronicles_line_numbers=true \ - -d:chronicles_timestamps=UnixTime \ - -d:ctTest -d:testing --hint"[XDeclaredButNotUsed]":off \ - -d:builtWithNix \ - -d:ctEntrypoint \ - -d:linksPathConst=.. \ - -d:libcPath=libc \ - -d:pathToNodeModules=../node_modules \ - --nimcache:nimcache \ - --out:"${APP_DIR}/bin/ct_unwrapped" c ./src/ct/codetracer.nim - - -nim \ - -d:release -d:asyncBackend=asyncdispatch \ - --gc:refc --hints:off --warnings:off \ - --debugInfo --lineDir:on \ - --boundChecks:on --stacktrace:on --linetrace:on \ - -d:chronicles_sinks=json -d:chronicles_line_numbers=true \ - -d:chronicles_timestamps=UnixTime \ - -d:ssl \ - -d:ctTest -d:testing --hint"[XDeclaredButNotUsed]":off \ - -d:linksPathConst=.. \ - -d:libcPath=libc \ - -d:builtWithNix \ - -d:ctEntrypoint \ - --dynlibOverride:"libsqlite3" \ - --dynlibOverride:"sqlite3" \ - --dynlibOverride:"pcre" \ - --dynlibOverride:"libzip" \ - --passL:"-Wl,-Bstatic -lsqlite3 -Wl,-Bdynamic" \ - --passL:"${APP_DIR}/lib/libpcre.so.1" \ - --passL:"${APP_DIR}/lib/libzip.so.5" \ - --nimcache:nimcache \ - --out:"${APP_DIR}/bin/db-backend-record" c ./src/ct/db_backend_record.nim - - # --passL:"-lsqlite3" \ - #--passL:"${APPDIR}/lib/libcrypto.so.3" \ - #--passL:"${APPDIR}/lib/libssl.so.3" \ - # --passL:"${APP_DIR}/lib/libz.so.1" \ - -# this works --passL:/nix/store/f6afb4jw9g5f94ixw0jn6cl0ah4liy35-sqlite-3.45.3/lib/libsqlite3.so.0 \ - - # TODO conditional for nixos?--passL:$LIBSQLITE3_PATH - # -# patchelf --set-rpath ${APP_DIR}/lib ${APP_DIR}/bin/ct - - -# index.js -nim \ - --hints:on --warnings:off --sourcemap:on \ - -d:ctIndex -d:chronicles_sinks=json \ - -d:nodejs --out:"${APP_DIR}/index.js" js src/frontend/index.nim -cp "${APP_DIR}/index.js" "${APP_DIR}/src/index.js" - -nim \ - --hints:on --warnings:off --sourcemap:on \ - -d:ctIndex -d:server -d:chronicles_sinks=json \ - -d:nodejs --out:"${APP_DIR}/server_index.js" js src/frontend/index.nim -cp "${APP_DIR}/server_index.js" "${APP_DIR}/src/server_index.js" - -# ui.js -nim \ - --hints:off --warnings:off \ - -d:chronicles_enabled=off \ - -d:ctRenderer \ - --out:"${APP_DIR}/ui.js" js src/frontend/ui_js.nim -cp "${APP_DIR}/ui.js" "${APP_DIR}/src/ui.js" - -# subwindow.js -nim \ - --hints:off --warnings:off \ - -d:chronicles_enabled=off \ - -d:ctRenderer \ - --out:"${APP_DIR}/subwindow.js" js src/frontend/subwindow.nim -cp "${APP_DIR}/subwindow.js" "${APP_DIR}/src/subwindow.js" - -popd - -echo "===========" diff --git a/appimage-scripts/install_ruby.sh b/appimage-scripts/install_ruby.sh deleted file mode 100755 index 69e855bd5..000000000 --- a/appimage-scripts/install_ruby.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash - -APP_DIR="${APP_DIR:-.}" - -# Install Ruby -RUBY="$(dirname "$(dirname "$(which ruby)")")" -echo "${RUBY}" - -mkdir -p "${APP_DIR}/ruby" - -cp -Lr --no-preserve=ownership "${RUBY}"/* "${APP_DIR}"/ruby -chmod -R +x "${APP_DIR}/ruby" diff --git a/nix/packages/default.nix b/nix/packages/default.nix index 773fc1dcd..ab8b51c6b 100644 --- a/nix/packages/default.nix +++ b/nix/packages/default.nix @@ -14,7 +14,9 @@ rubyPkg = pkgs.ruby_3_3; in { - packages = rec { + packages = + let + rawPackages = rec { upstream-nim-codetracer = pkgs.buildPackages.nim1.overrideAttrs (_: { postInstallPhase = '' mv $out/nim $out/upstream-nim @@ -150,7 +152,7 @@ copy_libs() { for lib in "$@"; do - cp -n -L "$lib" "$out/lib/" || true + cp -n -L "$lib" "$out/lib/" done } @@ -182,7 +184,7 @@ for binary in "$@"; do lddtree -l "$binary" | grep /nix | grep -v glibc | while read -r dep; do [ -f "$dep" ] || continue - cp -n -L "$dep" "$out/lib/" || true + cp -n -L "$dep" "$out/lib/" done done } @@ -207,7 +209,8 @@ patch_binary() { local bin=$1 if file "$bin" | grep -q 'ELF'; then - patchelf --remove-rpath "$bin" || true + chmod u+w "$bin" + patchelf --remove-rpath "$bin" patchelf --set-interpreter "''${INTERPRETER_PATH}" "$bin" patchelf --set-rpath '$ORIGIN/../lib' "$bin" fi @@ -215,10 +218,10 @@ for bin in "$out/bin"/*; do [ -f "$bin" ] || continue - patch_binary "$bin" || true + patch_binary "$bin" done - patch_binary "$out/ruby/bin/ruby" || true + patch_binary "$out/ruby/bin/ruby" ''; nimBuildInputs = [ @@ -480,7 +483,8 @@ patch_binary() { local bin=$1 if file "$bin" | grep -q 'ELF'; then - patchelf --remove-rpath "$bin" || true + chmod u+w "$bin" + patchelf --remove-rpath "$bin" patchelf --set-interpreter "''${INTERPRETER_PATH}" "$bin" patchelf --set-rpath '$ORIGIN/../lib' "$bin" fi @@ -491,7 +495,7 @@ local dest=$2 cp -L "$src" "$out/bin/$(basename "$dest")" chmod +x "$out/bin/$(basename "$dest")" - patch_binary "$out/bin/$(basename "$dest")" || true + patch_binary "$out/bin/$(basename "$dest")" } install_bin ${db-backend}/bin/db-backend db-backend @@ -514,7 +518,7 @@ cat <<'EOF' > "$out/bin/ct" #!/usr/bin/env bash -HERE=''${HERE:-$(dirname "$(readlink -f "$0")")} +HERE=''${HERE:-$(dirname "$(readlink -f "$0")")/..} # TODO: This includes references to x86_64. What about aarch64? @@ -561,9 +565,9 @@ EOF cp "''${SRC_ICONSET_DIR}/icon_256x256.png" "$out/codetracer.png" - patch_binary "$out/bin/ct_unwrapped" || true - patch_binary "$out/bin/db-backend-record" || true - patch_binary "$out/ruby/bin/ruby" || true + patch_binary "$out/bin/ct_unwrapped" + patch_binary "$out/bin/db-backend-record" + patch_binary "$out/ruby/bin/ruby" ''; indexJavascript = stdenv.mkDerivation { @@ -1123,5 +1127,12 @@ EOF default = codetracer; }; + in + builtins.removeAttrs rawPackages [ + "nimBuildInputs" + "nimBuildSetup" + "mkNimBinary" + "mkNimJs" + ]; }; } diff --git a/node-packages/yarn-project.nix b/node-packages/yarn-project.nix index d155f8bcb..299f9b832 100644 --- a/node-packages/yarn-project.nix +++ b/node-packages/yarn-project.nix @@ -1,21 +1,8 @@ # This file is generated by running "yarn install" inside your project. # Manual changes might be lost - proceed with caution! -{ - lib, - stdenv, - nodejs, - git, - cacert, - fetchurl, - writeShellScript, - writeShellScriptBin, -}: -{ - src, - overrideAttrs ? null, - ... -}@args: +{ lib, stdenv, nodejs, git, cacert, fetchurl, writeShellScript, writeShellScriptBin }: +{ src, overrideAttrs ? null, ... } @ args: let @@ -25,7 +12,8 @@ let lockfile = ./yarn.lock; # Call overrideAttrs on a derivation if a function is provided. - optionalOverride = fn: drv: if fn == null then drv else drv.overrideAttrs fn; + optionalOverride = fn: drv: + if fn == null then drv else drv.overrideAttrs fn; # Simple stub that provides the global yarn command. yarn = writeShellScriptBin "yarn" '' @@ -35,10 +23,7 @@ let # Common attributes between Yarn derivations. drvCommon = { # Make sure the build uses the right Node.js version everywhere. - buildInputs = [ - nodejs - yarn - ]; + buildInputs = [ nodejs yarn ]; # All dependencies should already be cached. yarn_enable_network = "0"; # Tell node-gyp to use the provided Node.js headers for native code builds. @@ -55,16 +40,9 @@ let export yarn_enable_nixify=false ''; - cacheDrv = stdenv.mkDerivation rec { + cacheDrv = stdenv.mkDerivation { name = "yarn-cache"; - buildInputs = [ - yarn - git - cacert - ]; - - nativeBuildInputs = buildInputs ++ [ nodejs ]; - + buildInputs = [ yarn git cacert ]; buildCommand = '' cp --reflink=auto --recursive '${src}' ./src cd ./src/ @@ -74,105 +52,96 @@ let rm $out/.gitignore ''; outputHashMode = "recursive"; - outputHash = ( - if stdenv.isAarch64 then - "sha512-+f3z4zwF2YlzskL7uDwVvQ3Fg0EYJ1HGPEqIGNb2VsiQpXoY2tU+EufqX79YRCOW7lXkuDfMQHYI3XcPioAEvg==" - else - "sha512-8spVLTYEciHn5q0N1R4fRB9EBM4gzZ1Zd4EeZdLwQM6ob+wvgP3oMH6JHQXUxKraP8TlpuVg7l2gNsTaAH5NoQ==" - ); + outputHash = "sha512-8spVLTYEciHn5q0N1R4fRB9EBM4gzZ1Zd4EeZdLwQM6ob+wvgP3oMH6JHQXUxKraP8TlpuVg7l2gNsTaAH5NoQ=="; }; # Main project derivation. - project = stdenv.mkDerivation ( - drvCommon - // { - inherit src; - name = "codetracer"; - - configurePhase = '' - ${buildVars} - - # Copy over the Yarn cache. - rm -fr '${cacheFolder}' - mkdir -p '${cacheFolder}' - cp --reflink=auto --recursive ${cacheDrv}/* '${cacheFolder}/' - - # Yarn may need a writable home directory. - export yarn_global_folder="$TMP" - - # Ensure global cache is disabled. Cache must be part of our output. - touch .yarnrc.yml - sed -i -e '/^enableGlobalCache/d' .yarnrc.yml - echo 'enableGlobalCache: false' >> .yarnrc.yml - - # Some node-gyp calls may call out to npm, which could fail due to an - # read-only home dir. - export HOME="$TMP" - - # running preConfigure after the cache is populated allows for - # preConfigure to contain substituteInPlace for dependencies as well as the - # main project. This is necessary for native bindings that maybe have - # hardcoded values. - runHook preConfigure - - # Run normal Yarn install to complete dependency installation. - yarn install --immutable --immutable-cache - - runHook postConfigure - ''; + project = stdenv.mkDerivation (drvCommon // { + inherit src; + name = "codetracer"; - buildPhase = '' - runHook preBuild - runHook postBuild - ''; + configurePhase = '' + ${buildVars} - installPhase = '' - runHook preInstall + # Copy over the Yarn cache. + rm -fr '${cacheFolder}' + mkdir -p '${cacheFolder}' + cp --reflink=auto --recursive ${cacheDrv}/* '${cacheFolder}/' - # Move the package contents to the output directory. - if grep -q '"workspaces"' package.json; then - # We can't use `yarn pack` in a workspace setup, because it only - # packages the outer workspace. - mkdir -p "$out/libexec" - mv $PWD "$out/libexec/$name" - else - # - If the package.json has a `files` field, only files matching those patterns are copied - # - Otherwise all files are copied. - yarn pack --out package.tgz - mkdir -p "$out/libexec/$name" - tar xzf package.tgz --directory "$out/libexec/$name" --strip-components=1 + # Yarn may need a writable home directory. + export yarn_global_folder="$TMP" - cp --reflink=auto .yarnrc* "$out/libexec/$name" - cp --reflink=auto ${lockfile} "$out/libexec/$name/yarn.lock" - cp --reflink=auto --recursive .yarn "$out/libexec/$name" + # Ensure global cache is disabled. Cache must be part of our output. + touch .yarnrc.yml + sed -i -e '/^enableGlobalCache/d' .yarnrc.yml + echo 'enableGlobalCache: false' >> .yarnrc.yml - # Copy the Yarn linker output into the package. - cp --reflink=auto --recursive node_modules "$out/libexec/$name" - fi + # Some node-gyp calls may call out to npm, which could fail due to an + # read-only home dir. + export HOME="$TMP" - cd "$out/libexec/$name" + # running preConfigure after the cache is populated allows for + # preConfigure to contain substituteInPlace for dependencies as well as the + # main project. This is necessary for native bindings that maybe have + # hardcoded values. + runHook preConfigure - # Invoke a plugin internal command to setup binaries. - mkdir -p "$out/bin" - yarn nixify install-bin $out/bin + # Run normal Yarn install to complete dependency installation. + yarn install --immutable --immutable-cache - # A package with node_modules doesn't need the cache - yarn cache clean + runHook postConfigure + ''; - runHook postInstall - ''; + buildPhase = '' + runHook preBuild + runHook postBuild + ''; + + installPhase = '' + runHook preInstall - passthru = { - inherit nodejs; - yarn-freestanding = yarn; - yarn = writeShellScriptBin "yarn" '' - exec '${yarn}/bin/yarn' --cwd '${overriddenProject}/libexec/${overriddenProject.name}' "$@" - ''; - }; - } - ); + # Move the package contents to the output directory. + if grep -q '"workspaces"' package.json; then + # We can't use `yarn pack` in a workspace setup, because it only + # packages the outer workspace. + mkdir -p "$out/libexec" + mv $PWD "$out/libexec/$name" + else + # - If the package.json has a `files` field, only files matching those patterns are copied + # - Otherwise all files are copied. + yarn pack --out package.tgz + mkdir -p "$out/libexec/$name" + tar xzf package.tgz --directory "$out/libexec/$name" --strip-components=1 + + cp --reflink=auto .yarnrc* "$out/libexec/$name" + cp --reflink=auto ${lockfile} "$out/libexec/$name/yarn.lock" + cp --reflink=auto --recursive .yarn "$out/libexec/$name" + + # Copy the Yarn linker output into the package. + cp --reflink=auto --recursive node_modules "$out/libexec/$name" + fi + + cd "$out/libexec/$name" + + # Invoke a plugin internal command to setup binaries. + mkdir -p "$out/bin" + yarn nixify install-bin $out/bin + + # A package with node_modules doesn't need the cache + yarn cache clean + + runHook postInstall + ''; + + passthru = { + inherit nodejs; + yarn-freestanding = yarn; + yarn = writeShellScriptBin "yarn" '' + exec '${yarn}/bin/yarn' --cwd '${overriddenProject}/libexec/${overriddenProject.name}' "$@" + ''; + }; + }); overriddenProject = optionalOverride overrideAttrs project; -in -overriddenProject +in overriddenProject