fdroid: make reproducible

Author: siddarthkay

ci/Jenkinsfile.fdroid

@@ -1,5 +1,5 @@
 #!/usr/bin/env groovy
-library 'status-jenkins-lib@v1.9.40'
+library 'status-jenkins-lib@fdroid-apk-signing'
 
 /* Options section can't access functions in objects. */
 def isPRBuild = utils.isPRBuild()
@@ -93,6 +93,12 @@ pipeline {
       }
     }
 
+    stage('Sign') {
+      steps { script {
+        status.signFdroidApk(env.STATUS_FDROID_APK)
+      } }
+    }
+
     stage('Upload') {
       steps {
         script {

fdroid/build-qt.sh

@@ -1,11 +1,49 @@
 #!/usr/bin/env bash
 set -eo pipefail
 
+# Reproducibility env
+export LC_ALL=C
+export LANG=C
+export SOURCE_DATE_EPOCH="$(git -C "$BUILD_DIR" log -1 --pretty=%ct 2>/dev/null || echo 0)"
+
 QT_VERSION="${QT_VERSION:-6.9.2}"
 
 QT_MODULES=qtbase,qtdeclarative,qt5compat,qtmultimedia,qtshadertools,qtimageformats,qtwebview,qtscxml,qtsvg,qtconnectivity,qtwebsockets,qtpositioning,qtlottie,qtwebchannel
 (cd "$QT_SRCDIR" && perl init-repository --module-subset="$QT_MODULES")
 
+# Reproducibility:
+#   add_link_options: strips .note.gnu.build-id from every .so
+#   add_compile_options: make sure that paths dont leak into final build.
+QT5_CMAKELISTS="$QT_SRCDIR/CMakeLists.txt"
+if ! grep -q 'build-id=none' "$QT5_CMAKELISTS"; then
+    sed -i '/^project(Qt$/,/^)$/{/^)$/a\
+add_link_options("LINKER:--build-id=none")\
+add_compile_options("-ffile-prefix-map=${CMAKE_SOURCE_DIR}=.")\
+add_compile_options("-ffile-prefix-map=${CMAKE_BINARY_DIR}=.")\
+add_compile_options("-ffile-prefix-map=$ENV{HOME}=.")
+}' "$QT5_CMAKELISTS"
+fi
+
+# Reproducibility: force qmlcachegen --only-bytecode for every Qt-internal QML module.  
+# Otherwise we end up with +800 bytes of AOT code in one libQt6QuickControls2Fusion.
+QT_QML_MACROS="$QT_SRCDIR/qtdeclarative/src/qml/Qt6QmlMacros.cmake"
+if [ -f "$QT_QML_MACROS" ] && ! grep -q 'REPRODUCIBILITY_ONLY_BYTECODE' "$QT_QML_MACROS"; then
+    sed -i '/MATCHES "-NOTFOUND\$"/,/endforeach()/{
+/endforeach()/a\
+    # REPRODUCIBILITY_ONLY_BYTECODE: pin every Qt-internal QML module to\
+    # qmlcachegen --only-bytecode so AOT C++ codegen does not introduce\
+    # per-build non-determinism in compiled .so files.\
+    get_target_property(_qmlcg_args ${target} QT_QMLCACHEGEN_ARGUMENTS)\
+    if(NOT "--only-bytecode" IN_LIST _qmlcg_args)\
+        list(APPEND _qmlcg_args "--only-bytecode")\
+        set_target_properties(${target} PROPERTIES QT_QMLCACHEGEN_ARGUMENTS "${_qmlcg_args}")\
+    endif()
+}' "$QT_QML_MACROS"
+fi
+
+# Reproducibility: rewrite absolute build/install paths embedded in compiled objects 
+QT_REPRO_CFLAGS="-ffile-prefix-map=${QT_SRCDIR}=. -ffile-prefix-map=${BUILD_DIR}=. -ffile-prefix-map=${HOME}=."
+
 # Build Qt for host (required as cross-compilation toolchain for Android)
 mkdir -p build_qt_host && cd build_qt_host
 
@@ -18,6 +56,10 @@ mkdir -p build_qt_host && cd build_qt_host
     -nomake tests \
     -- \
     -DCMAKE_BUILD_TYPE=Release \
+    -DCMAKE_C_FLAGS_INIT="$QT_REPRO_CFLAGS" \
+    -DCMAKE_CXX_FLAGS_INIT="$QT_REPRO_CFLAGS" \
+    -DCMAKE_UNITY_BUILD=OFF \
+    -DCMAKE_INTERPROCEDURAL_OPTIMIZATION=OFF \
     -DCMAKE_MESSAGE_LOG_LEVEL=WARNING \
     -Wno-dev
 
@@ -50,6 +92,10 @@ mkdir -p build_qt_android && cd build_qt_android
     -DFFMPEG_DIR="$FFMPEG_DIR" \
     -DFEATURE_ffmpeg=ON \
     -DQT_DEFAULT_MEDIA_BACKEND=ffmpeg \
+    -DCMAKE_C_FLAGS_INIT="$QT_REPRO_CFLAGS" \
+    -DCMAKE_CXX_FLAGS_INIT="$QT_REPRO_CFLAGS" \
+    -DCMAKE_UNITY_BUILD=OFF \
+    -DCMAKE_INTERPROCEDURAL_OPTIMIZATION=OFF \
     -DCMAKE_MESSAGE_LOG_LEVEL=WARNING \
     -Wno-dev
 

fdroid/generate-keystore.sh

@@ -33,6 +33,12 @@ $(STATUS_GO_LIB):
 	@echo "Building status-go mobile library"
 	@mkdir -p $(LIB_PATH)
 ifeq ($(OS),android)
+	# Reproducibility envs for status-go
+	  GOFLAGS="-trimpath -buildvcs=false -toolexec=$(abspath $(SCRIPTS_PATH)/go-toolexec-wrapper.sh)" \
+	  GOMODCACHE="$(BUILD_PATH)/.gomodcache" \
+	  CGO_CFLAGS="-ffile-prefix-map=$(HOME)=. -ffile-prefix-map=$(BUILD_PATH)=. -ffile-prefix-map=$(ROOT_DIR)=. -ffile-prefix-map=$(ANDROID_NDK_ROOT)=." \
+	  CGO_CXXFLAGS="-ffile-prefix-map=$(HOME)=. -ffile-prefix-map=$(BUILD_PATH)=. -ffile-prefix-map=$(ROOT_DIR)=. -ffile-prefix-map=$(ANDROID_NDK_ROOT)=." \
+	  CGO_LDFLAGS="-Wl,--build-id=none" \
 	  CC="$(CC)" $(MAKE) -C ../vendor/status-go statusgo-android-library \
 		ARCH=$(ARCH) \
 		ANDROID_NDK_ROOT="$(ANDROID_NDK_ROOT)" \
@@ -42,7 +48,17 @@ ifeq ($(OS),android)
 		USE_SYSTEM_NIM=$(USE_SYSTEM_NIM) \
 		GO_GENERATE_CMD="go generate" \
 		SHELL=/bin/sh
+	@$(SCRIPTS_PATH)/canonicalize-cgo-symbols.sh \
+		../vendor/status-go/build/bin/libstatus$(LIB_EXT)
+	@$(ANDROID_NDK_ROOT)/toolchains/llvm/prebuilt/$(HOST_OS)-x86_64/bin/llvm-objcopy \
+		--remove-section=.note.gnu.build-id \
+		--remove-section=.note.go.buildid \
+		../vendor/status-go/build/bin/libstatus$(LIB_EXT)
 else ifeq ($(OS),ios)
+	GOFLAGS="-trimpath -buildvcs=false" \
+	GOMODCACHE="$(BUILD_PATH)/.gomodcache" \
+	CGO_CFLAGS="-ffile-prefix-map=$(HOME)=." \
+	CGO_CXXFLAGS="-ffile-prefix-map=$(HOME)=." \
 	$(MAKE) -C ../vendor/status-go statusgo-ios-library \
 		ARCH=$(ARCH) \
 		IPHONE_SDK="$(IPHONE_SDK)" \

fdroid/sign-apk.sh

@@ -48,13 +48,21 @@ if [[ "${OS}" == "android" ]]; then
   export BUILD_VARIANT
   export BUNDLE_IDENTIFIER
 
-  if [[ "$GRADLE_TARGETS" == *"Fdroid"* ]]; then
-    # shellcheck source=../../fdroid/generate-keystore.sh
-    source "$REPO_ROOT/fdroid/generate-keystore.sh" "$REPO_ROOT/status-fdroid.keystore"
-  fi
-
+  # Reproducibility flags for fdroid rebuilds:
+  #   --build-id=none drops the .note.gnu.build-id section
+  #   -ffile-prefix-map remaps absolute source paths so they don't depend on $HOME or $BUILD_DIR.
+  QMAKE_REPRO_ARGS=(
+    "QMAKE_LFLAGS+=-Wl,--build-id=none"
+    "QMAKE_CFLAGS+=-ffile-prefix-map=${BUILD_DIR}=."
+    "QMAKE_CFLAGS+=-ffile-prefix-map=${REPO_ROOT}=."
+    "QMAKE_CFLAGS+=-ffile-prefix-map=${HOME}=."
+    "QMAKE_CXXFLAGS+=-ffile-prefix-map=${BUILD_DIR}=."
+    "QMAKE_CXXFLAGS+=-ffile-prefix-map=${REPO_ROOT}=."
+    "QMAKE_CXXFLAGS+=-ffile-prefix-map=${HOME}=."
+  )
   "$QMAKE_BIN" "$CWD/../wrapperApp/Status.pro" "${QMAKE_CONFIG[@]}" -spec android-clang \
-    ANDROID_ABIS="${ANDROID_ABI:-arm64-v8a}" VERSION="$VERSION" "${QMAKE_DEFINES[@]}" -after
+    ANDROID_ABIS="${ANDROID_ABI:-arm64-v8a}" VERSION="$VERSION" "${QMAKE_DEFINES[@]}" \
+    -after "${QMAKE_REPRO_ARGS[@]}"
 
   make -j"$(nproc)" apk_install_target
 
@@ -117,18 +125,14 @@ if [[ "${OS}" == "android" ]]; then
   echo "APK outputs:"
   find build/outputs/apk -name '*.apk' 2>/dev/null || echo "No APKs found"
 
-  # If Gradle produced an unsigned APK (e.g. fdroid build where signing configs
-  # are stripped by fdroid's remove_signing_keys), sign it via the dedicated script.
-  if [[ ! -f "$APK_OUT" && -f "$APK_OUT_UNSIGNED" && -n "${FDROID_STORE_FILE:-}" ]]; then
-    echo "Signing unsigned APK..."
-    "$REPO_ROOT/fdroid/sign-apk.sh"
-  fi
-
   # Copy whichever artifacts were built
   BUILT=""
   if [[ -f "$APK_OUT" ]]; then
     cp "$APK_OUT" "$BIN_DIR/${OUTPUT_NAME}.apk"
     BUILT="$BIN_DIR/${OUTPUT_NAME}.apk"
+  elif [[ -f "$APK_OUT_UNSIGNED" ]]; then
+    cp "$APK_OUT_UNSIGNED" "$BIN_DIR/${OUTPUT_NAME}.apk"
+    BUILT="$BIN_DIR/${OUTPUT_NAME}.apk"
   fi
   if [[ -f "$AAB_OUT" ]]; then
     cp "$AAB_OUT" "$BIN_DIR/${OUTPUT_NAME}.aab"

mobile/Makefile

@@ -0,0 +1,79 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+SO_PATH="${1:?Usage: $0 <path-to-libstatus.so>}"
+NM="${NM:-llvm-nm}"
+OBJCOPY="${OBJCOPY:-llvm-objcopy}"
+
+command -v "$NM" >/dev/null      || { echo "[canonicalize-cgo] $NM not found on PATH" >&2; exit 1; }
+command -v "$OBJCOPY" >/dev/null || { echo "[canonicalize-cgo] $OBJCOPY not found on PATH" >&2; exit 1; }
+
+list_symbols() { "$NM" -D -P "$SO_PATH"; "$NM" -P "$SO_PATH" 2>/dev/null || true; }
+
+mapfile -t HASHES < <(
+  list_symbols | awk '{print $1}' \
+    | grep -Eo '^_cgo(exp)?_[0-9a-f]{12,16}_' \
+    | sed -E 's/^_cgo(exp)?_([0-9a-f]+)_/\2/' \
+    | sort -u
+)
+
+if [[ ${#HASHES[@]} -eq 0 ]]; then
+  echo "[canonicalize-cgo] no cgo hash symbols found in $SO_PATH" >&2
+  echo "[canonicalize-cgo] this is unexpected for a cgo build; check that nm reads .dynsym" >&2
+  exit 1
+fi
+
+MAPPING=$(mktemp)
+PERL_SCRIPT=""
+trap 'rm -f "$MAPPING"' EXIT
+
+# Pick a sha256 tool that's available on both macOS and Debian-derived fdroid
+# builders. Both ship at least one of these.
+if command -v sha256sum >/dev/null 2>&1; then
+  SHA256="sha256sum"
+else
+  SHA256="shasum -a 256"
+fi
+
+rewrites=0
+for HASH in "${HASHES[@]}"; do
+  LEN=${#HASH}
+  SHA_LEN=$((LEN - 4))
+  STABLE=$(
+    list_symbols | awk '{print $1}' \
+      | grep -E "_cgo(exp)?_${HASH}_" \
+      | sed "s/${HASH}/X/g" \
+      | sort -u \
+      | $SHA256 | awk '{print $1}'
+  )
+  CANON="c0de${STABLE:0:$SHA_LEN}"
+
+  if [[ "$HASH" == "$CANON" ]]; then
+    continue
+  fi
+  rewrites=$((rewrites + 1))
+
+  list_symbols | awk '{print $1}' \
+    | grep -E "_cgo(exp)?_${HASH}_" | sort -u \
+    | while read -r SYM; do
+        printf '%s %s\n' "$SYM" "${SYM/$HASH/$CANON}"
+      done \
+    >> "$MAPPING"
+
+  PERL_SCRIPT+="s/_cgo_${HASH}_/_cgo_${CANON}_/g;"
+  PERL_SCRIPT+="s/_cgoexp_${HASH}_/_cgoexp_${CANON}_/g;"
+done
+
+if [[ $rewrites -eq 0 ]]; then
+  echo "[canonicalize-cgo] all ${#HASHES[@]} hashes already canonical"
+  exit 0
+fi
+
+echo "[canonicalize-cgo] rewriting $rewrites of ${#HASHES[@]} hashes in $SO_PATH"
+
+"$OBJCOPY" --redefine-syms="$MAPPING" "$SO_PATH"
+
+perl -i -0777 -pe "$PERL_SCRIPT" "$SO_PATH"
+
+echo "[canonicalize-cgo] done"