fdroid: make reproducible

Author: siddarthkay

ci/Jenkinsfile.fdroid

@@ -1,5 +1,5 @@
 #!/usr/bin/env groovy
-library 'status-jenkins-lib@v1.9.40'
+library 'status-jenkins-lib@fdroid-apk-signing'
 
 /* Options section can't access functions in objects. */
 def isPRBuild = utils.isPRBuild()
@@ -93,6 +93,12 @@ pipeline {
       }
     }
 
+    stage('Sign') {
+      steps { script {
+        status.signFdroidApk(env.STATUS_FDROID_APK)
+      } }
+    }
+
     stage('Upload') {
       steps {
         script {

fdroid/build-qt.sh

@@ -1,11 +1,64 @@
 #!/usr/bin/env bash
 set -eo pipefail
 
+# Reproducibility env, applied to every cmake / ninja / qmlcachegen / moc /
+# rcc invocation below. Defense-in-depth against the three known classes of
+# Qt non-determinism that can affect a subset of modules:
+#   LC_ALL/LANG=C      - stable locale-dependent sort order (linker symbol
+#                        ordering, CMake file globs, ninja work-item order)
+#   SOURCE_DATE_EPOCH  - honored by gcc/clang for __DATE__/__TIME__ and by
+#                        qmlcachegen / qrc for any embedded timestamp
+# The matching -DCMAKE_UNITY_BUILD=OFF and -DCMAKE_INTERPROCEDURAL_OPTIMIZATION=OFF
+# below kill the other two: Unity Build's per-target source batching, and any
+# future Qt update silently enabling IPO/LTO.
+export LC_ALL=C
+export LANG=C
+export SOURCE_DATE_EPOCH="$(git -C "$BUILD_DIR" log -1 --pretty=%ct 2>/dev/null || echo 0)"
+
 QT_VERSION="${QT_VERSION:-6.9.2}"
 
 QT_MODULES=qtbase,qtdeclarative,qt5compat,qtmultimedia,qtshadertools,qtimageformats,qtwebview,qtscxml,qtsvg,qtconnectivity,qtwebsockets,qtpositioning,qtlottie,qtwebchannel
 (cd "$QT_SRCDIR" && perl init-repository --module-subset="$QT_MODULES")
 
+# Reproducibility:
+#   add_link_options: strips .note.gnu.build-id from every .so
+#   add_compile_options: make sure that paths dont leak into final build.
+QT5_CMAKELISTS="$QT_SRCDIR/CMakeLists.txt"
+if ! grep -q 'build-id=none' "$QT5_CMAKELISTS"; then
+    sed -i '/^project(Qt$/,/^)$/{/^)$/a\
+add_link_options("LINKER:--build-id=none")\
+add_compile_options("-ffile-prefix-map=${CMAKE_SOURCE_DIR}=.")\
+add_compile_options("-ffile-prefix-map=${CMAKE_BINARY_DIR}=.")\
+add_compile_options("-ffile-prefix-map=$ENV{HOME}=.")
+}' "$QT5_CMAKELISTS"
+fi
+
+# Reproducibility: force qmlcachegen --only-bytecode for every Qt-internal QML
+# module. qmlcachegen's AOT C++ codegen (AOTCompiledContext) makes per-call
+# decisions whose heuristics walk QHash containers; iteration order depends on
+# per-process memory layout, so different runs emit different amounts of AOT
+# code for the same QML inputs. Disassembly of two back-to-back builds showed
+# +800 bytes of AOT code in one libQt6QuickControls2Fusion vs the other.
+# --only-bytecode keeps QML bytecode compilation (deterministic) and skips AOT.
+QT_QML_MACROS="$QT_SRCDIR/qtdeclarative/src/qml/Qt6QmlMacros.cmake"
+if [ -f "$QT_QML_MACROS" ] && ! grep -q 'REPRODUCIBILITY_ONLY_BYTECODE' "$QT_QML_MACROS"; then
+    sed -i '/MATCHES "-NOTFOUND\$"/,/endforeach()/{
+/endforeach()/a\
+    # REPRODUCIBILITY_ONLY_BYTECODE: pin every Qt-internal QML module to\
+    # qmlcachegen --only-bytecode so AOT C++ codegen does not introduce\
+    # per-build non-determinism in compiled .so files.\
+    get_target_property(_qmlcg_args ${target} QT_QMLCACHEGEN_ARGUMENTS)\
+    if(NOT "--only-bytecode" IN_LIST _qmlcg_args)\
+        list(APPEND _qmlcg_args "--only-bytecode")\
+        set_target_properties(${target} PROPERTIES QT_QMLCACHEGEN_ARGUMENTS "${_qmlcg_args}")\
+    endif()
+}' "$QT_QML_MACROS"
+fi
+
+# Reproducibility: rewrite absolute build/install paths embedded in compiled
+# objects (debug strings, __FILE__, etc.) so they match across rebuilders.
+QT_REPRO_CFLAGS="-ffile-prefix-map=${QT_SRCDIR}=. -ffile-prefix-map=${BUILD_DIR}=. -ffile-prefix-map=${HOME}=."
+
 # Build Qt for host (required as cross-compilation toolchain for Android)
 mkdir -p build_qt_host && cd build_qt_host
 
@@ -18,6 +71,10 @@ mkdir -p build_qt_host && cd build_qt_host
     -nomake tests \
     -- \
     -DCMAKE_BUILD_TYPE=Release \
+    -DCMAKE_C_FLAGS_INIT="$QT_REPRO_CFLAGS" \
+    -DCMAKE_CXX_FLAGS_INIT="$QT_REPRO_CFLAGS" \
+    -DCMAKE_UNITY_BUILD=OFF \
+    -DCMAKE_INTERPROCEDURAL_OPTIMIZATION=OFF \
     -DCMAKE_MESSAGE_LOG_LEVEL=WARNING \
     -Wno-dev
 
@@ -50,6 +107,10 @@ mkdir -p build_qt_android && cd build_qt_android
     -DFFMPEG_DIR="$FFMPEG_DIR" \
     -DFEATURE_ffmpeg=ON \
     -DQT_DEFAULT_MEDIA_BACKEND=ffmpeg \
+    -DCMAKE_C_FLAGS_INIT="$QT_REPRO_CFLAGS" \
+    -DCMAKE_CXX_FLAGS_INIT="$QT_REPRO_CFLAGS" \
+    -DCMAKE_UNITY_BUILD=OFF \
+    -DCMAKE_INTERPROCEDURAL_OPTIMIZATION=OFF \
     -DCMAKE_MESSAGE_LOG_LEVEL=WARNING \
     -Wno-dev
 

fdroid/generate-keystore.sh

@@ -33,6 +33,21 @@ $(STATUS_GO_LIB):
 	@echo "Building status-go mobile library"
 	@mkdir -p $(LIB_PATH)
 ifeq ($(OS),android)
+	# Reproducibility env for status-go. F-Droid byte-verifies the APK
+	# against a reference build, so libstatus.so must come out identical
+	# every time.
+	#
+	# -toolexec wraps every cgo invocation and substitutes -importpath with a
+	# content-derived value. cgo's per-package hash prefix (the X in
+	# _cgo_X_Cfunc_*) is sha256(importPath)[:6]; without this wrapper, Go's
+	# build system passes import paths that contain $WORK temp-dir components
+	# which change per invocation, cascading into a ~32-byte pclntab drift in
+	# .data.rel.ro that no -trimpath / -buildvcs / -ldflags flag can address.
+	  GOFLAGS="-trimpath -buildvcs=false -toolexec=$(abspath $(SCRIPTS_PATH)/go-toolexec-wrapper.sh)" \
+	  GOMODCACHE="$(BUILD_PATH)/.gomodcache" \
+	  CGO_CFLAGS="-ffile-prefix-map=$(HOME)=. -ffile-prefix-map=$(BUILD_PATH)=. -ffile-prefix-map=$(ROOT_DIR)=. -ffile-prefix-map=$(ANDROID_NDK_ROOT)=." \
+	  CGO_CXXFLAGS="-ffile-prefix-map=$(HOME)=. -ffile-prefix-map=$(BUILD_PATH)=. -ffile-prefix-map=$(ROOT_DIR)=. -ffile-prefix-map=$(ANDROID_NDK_ROOT)=." \
+	  CGO_LDFLAGS="-Wl,--build-id=none" \
 	  CC="$(CC)" $(MAKE) -C ../vendor/status-go statusgo-android-library \
 		ARCH=$(ARCH) \
 		ANDROID_NDK_ROOT="$(ANDROID_NDK_ROOT)" \
@@ -42,7 +57,21 @@ ifeq ($(OS),android)
 		USE_SYSTEM_NIM=$(USE_SYSTEM_NIM) \
 		GO_GENERATE_CMD="go generate" \
 		SHELL=/bin/sh
+	# Belt-and-suspenders: even with the toolexec wrapper above, run our
+	# post-link canonicalizer to rewrite any cgo hash that might still vary
+	# (and to verify reproducibility — a no-op pass should print
+	# "all N hashes already canonical").
+	@$(SCRIPTS_PATH)/canonicalize-cgo-symbols.sh \
+		../vendor/status-go/build/bin/libstatus$(LIB_EXT)
+	@$(ANDROID_NDK_ROOT)/toolchains/llvm/prebuilt/$(HOST_OS)-x86_64/bin/llvm-objcopy \
+		--remove-section=.note.gnu.build-id \
+		--remove-section=.note.go.buildid \
+		../vendor/status-go/build/bin/libstatus$(LIB_EXT)
 else ifeq ($(OS),ios)
+	GOFLAGS="-trimpath -buildvcs=false" \
+	GOMODCACHE="$(BUILD_PATH)/.gomodcache" \
+	CGO_CFLAGS="-ffile-prefix-map=$(HOME)=." \
+	CGO_CXXFLAGS="-ffile-prefix-map=$(HOME)=." \
 	$(MAKE) -C ../vendor/status-go statusgo-ios-library \
 		ARCH=$(ARCH) \
 		IPHONE_SDK="$(IPHONE_SDK)" \

fdroid/sign-apk.sh

@@ -48,13 +48,21 @@ if [[ "${OS}" == "android" ]]; then
   export BUILD_VARIANT
   export BUNDLE_IDENTIFIER
 
-  if [[ "$GRADLE_TARGETS" == *"Fdroid"* ]]; then
-    # shellcheck source=../../fdroid/generate-keystore.sh
-    source "$REPO_ROOT/fdroid/generate-keystore.sh" "$REPO_ROOT/status-fdroid.keystore"
-  fi
-
+  # Reproducibility flags for fdroid rebuilds:
+  #   --build-id=none drops the .note.gnu.build-id section
+  #   -ffile-prefix-map remaps absolute source paths so they don't depend on $HOME or $BUILD_DIR.
+  QMAKE_REPRO_ARGS=(
+    "QMAKE_LFLAGS+=-Wl,--build-id=none"
+    "QMAKE_CFLAGS+=-ffile-prefix-map=${BUILD_DIR}=."
+    "QMAKE_CFLAGS+=-ffile-prefix-map=${REPO_ROOT}=."
+    "QMAKE_CFLAGS+=-ffile-prefix-map=${HOME}=."
+    "QMAKE_CXXFLAGS+=-ffile-prefix-map=${BUILD_DIR}=."
+    "QMAKE_CXXFLAGS+=-ffile-prefix-map=${REPO_ROOT}=."
+    "QMAKE_CXXFLAGS+=-ffile-prefix-map=${HOME}=."
+  )
   "$QMAKE_BIN" "$CWD/../wrapperApp/Status.pro" "${QMAKE_CONFIG[@]}" -spec android-clang \
-    ANDROID_ABIS="${ANDROID_ABI:-arm64-v8a}" VERSION="$VERSION" "${QMAKE_DEFINES[@]}" -after
+    ANDROID_ABIS="${ANDROID_ABI:-arm64-v8a}" VERSION="$VERSION" "${QMAKE_DEFINES[@]}" \
+    -after "${QMAKE_REPRO_ARGS[@]}"
 
   make -j"$(nproc)" apk_install_target
 
@@ -117,18 +125,14 @@ if [[ "${OS}" == "android" ]]; then
   echo "APK outputs:"
   find build/outputs/apk -name '*.apk' 2>/dev/null || echo "No APKs found"
 
-  # If Gradle produced an unsigned APK (e.g. fdroid build where signing configs
-  # are stripped by fdroid's remove_signing_keys), sign it via the dedicated script.
-  if [[ ! -f "$APK_OUT" && -f "$APK_OUT_UNSIGNED" && -n "${FDROID_STORE_FILE:-}" ]]; then
-    echo "Signing unsigned APK..."
-    "$REPO_ROOT/fdroid/sign-apk.sh"
-  fi
-
   # Copy whichever artifacts were built
   BUILT=""
   if [[ -f "$APK_OUT" ]]; then
     cp "$APK_OUT" "$BIN_DIR/${OUTPUT_NAME}.apk"
     BUILT="$BIN_DIR/${OUTPUT_NAME}.apk"
+  elif [[ -f "$APK_OUT_UNSIGNED" ]]; then
+    cp "$APK_OUT_UNSIGNED" "$BIN_DIR/${OUTPUT_NAME}.apk"
+    BUILT="$BIN_DIR/${OUTPUT_NAME}.apk"
   fi
   if [[ -f "$AAB_OUT" ]]; then
     cp "$AAB_OUT" "$BIN_DIR/${OUTPUT_NAME}.aab"

mobile/Makefile

@@ -0,0 +1,108 @@
+#!/usr/bin/env bash
+# Canonicalize Go cgo's per-build symbol hashes so libstatus.so is byte-identical
+# across builds. `go build` regenerates these hashes on every invocation because
+# they derive from the build's temp dir path. No build flag fixes it; the hash
+# is baked into actual symbol names like _cgo_<hash>_Cfunc_free and
+# _cgoexp_<hash>_AcceptTerms.
+#
+# cgo emits ONE hash per cgo-using package (status-go core, libwaku, libsds,
+# ethereum bindings, etc.), so a c-shared library that aggregates many such
+# packages contains many distinct hashes. We discover them all, assign each
+# a unique deterministic canonical value based on its sorted order, then
+# rewrite both the symbol table (objcopy) and any embedded copies (perl).
+
+set -euo pipefail
+
+SO_PATH="${1:?Usage: $0 <path-to-libstatus.so>}"
+NM="${NM:-llvm-nm}"
+OBJCOPY="${OBJCOPY:-llvm-objcopy}"
+
+# Fail loudly if our toolchain isn't on PATH — silent skip masks a broken
+# build (e.g., post-link stripping leaves no .symtab, then a misconfigured
+# nm reports "no symbols" and we never canonicalize anything).
+command -v "$NM" >/dev/null      || { echo "[canonicalize-cgo] $NM not found on PATH" >&2; exit 1; }
+command -v "$OBJCOPY" >/dev/null || { echo "[canonicalize-cgo] $OBJCOPY not found on PATH" >&2; exit 1; }
+
+# Read BOTH static and dynamic symbol tables (-D includes .dynsym, needed
+# when -s stripping has removed .symtab).
+list_symbols() { "$NM" -D -P "$SO_PATH"; "$NM" -P "$SO_PATH" 2>/dev/null || true; }
+
+mapfile -t HASHES < <(
+  list_symbols | awk '{print $1}' \
+    | grep -Eo '^_cgo(exp)?_[0-9a-f]{12,16}_' \
+    | sed -E 's/^_cgo(exp)?_([0-9a-f]+)_/\2/' \
+    | sort -u
+)
+
+if [[ ${#HASHES[@]} -eq 0 ]]; then
+  echo "[canonicalize-cgo] no cgo hash symbols found in $SO_PATH" >&2
+  echo "[canonicalize-cgo] this is unexpected for a cgo build; check that nm reads .dynsym" >&2
+  exit 1
+fi
+
+# Build the mapping: every discovered hash -> c0de<sha256-prefix>, where the
+# sha256 is computed over the sorted set of symbol names that use this hash
+# (with the hash itself replaced by a placeholder so the input is content-only).
+# Same package source → same symbol set → same canonical, every build, every
+# machine. Sort-order-based assignment is NOT safe here: when two builds
+# produce different random hashes, sorting them yields different orderings
+# and the same package gets a different canonical position. Content-derivation
+# avoids that.
+MAPPING=$(mktemp)
+PERL_SCRIPT=""
+trap 'rm -f "$MAPPING"' EXIT
+
+# Pick a sha256 tool that's available on both macOS and Debian-derived fdroid
+# builders. Both ship at least one of these.
+if command -v sha256sum >/dev/null 2>&1; then
+  SHA256="sha256sum"
+else
+  SHA256="shasum -a 256"
+fi
+
+rewrites=0
+for HASH in "${HASHES[@]}"; do
+  LEN=${#HASH}
+  SHA_LEN=$((LEN - 4))
+  STABLE=$(
+    list_symbols | awk '{print $1}' \
+      | grep -E "_cgo(exp)?_${HASH}_" \
+      | sed "s/${HASH}/X/g" \
+      | sort -u \
+      | $SHA256 | awk '{print $1}'
+  )
+  CANON="c0de${STABLE:0:$SHA_LEN}"
+
+  if [[ "$HASH" == "$CANON" ]]; then
+    continue
+  fi
+  rewrites=$((rewrites + 1))
+
+  list_symbols | awk '{print $1}' \
+    | grep -E "_cgo(exp)?_${HASH}_" | sort -u \
+    | while read -r SYM; do
+        printf '%s %s\n' "$SYM" "${SYM/$HASH/$CANON}"
+      done \
+    >> "$MAPPING"
+
+  PERL_SCRIPT+="s/_cgo_${HASH}_/_cgo_${CANON}_/g;"
+  PERL_SCRIPT+="s/_cgoexp_${HASH}_/_cgoexp_${CANON}_/g;"
+done
+
+if [[ $rewrites -eq 0 ]]; then
+  echo "[canonicalize-cgo] all ${#HASHES[@]} hashes already canonical"
+  exit 0
+fi
+
+echo "[canonicalize-cgo] rewriting $rewrites of ${#HASHES[@]} hashes in $SO_PATH"
+
+# Step 1: symbol-table rename. Maintains .gnu.hash, .dynsym, .symtab, relocs.
+"$OBJCOPY" --redefine-syms="$MAPPING" "$SO_PATH"
+
+# Step 2: catch embedded copies (Go's .gopclntab, debug strings, rodata
+# literals). Same-length substitution preserves ELF layout — no relocations
+# need updating. One perl invocation handles all hashes at once to avoid
+# rewriting the 80MB file N times.
+perl -i -0777 -pe "$PERL_SCRIPT" "$SO_PATH"
+
+echo "[canonicalize-cgo] done"