use statically build libraries instead of syso files

.github/workflows/build-static-libs.yml

@@ -0,0 +1,255 @@
+name: Build Static Libraries
+
+on:
+  workflow_dispatch:  # Manual trigger
+  push:
+    paths:
+      - 'src/**'
+      - '.github/workflows/build-static-libs.yml'
+  pull_request:
+    paths:
+      - 'src/**'
+
+jobs:
+  build-linux-amd64:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y build-essential
+
+      - name: Build Linux AMD64 library
+        run: |
+          cd src
+          make clean
+          # Build with reproducible flags
+          export SOURCE_DATE_EPOCH=$(git log -1 --format=%ct)
+          make CFLAGS="-O3 -Wall -Werror -static -fno-stack-protector -fno-plt -ffile-prefix-map=$(pwd)=." ASFLAGS="-g -fpic"
+          # Copy the library to a simple location
+          cp ../build/lib/libhashtree.a ./libhashtree.a
+          # Remove non-deterministic archive metadata
+          strip --strip-debug ./libhashtree.a || true
+          # Normalize archive
+          ranlib ./libhashtree.a
+          # Generate checksum
+          sha256sum libhashtree.a > libhashtree.a.sha256
+
+      - name: Upload Linux AMD64 library
+        uses: actions/upload-artifact@v4
+        with:
+          name: libhashtree-linux-amd64
+          path: |
+            src/libhashtree.a
+            src/libhashtree.a.sha256
+          retention-days: 7
+
+  build-linux-arm64:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install ARM64 cross-compiler
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y gcc-aarch64-linux-gnu
+
+      - name: Build Linux ARM64 library
+        run: |
+          cd src
+          make clean
+          # Build with reproducible flags
+          export SOURCE_DATE_EPOCH=$(git log -1 --format=%ct)
+          make CC=aarch64-linux-gnu-gcc ARM=1 CFLAGS="-O3 -Wall -Werror -static -fno-stack-protector -fno-plt -ffile-prefix-map=$(pwd)=." ASFLAGS="-g -fpic"
+          # Copy the library to a simple location
+          cp ../build/lib/libhashtree.a ./libhashtree.a
+          # Remove non-deterministic archive metadata
+          aarch64-linux-gnu-strip --strip-debug ./libhashtree.a || true
+          # Normalize archive
+          aarch64-linux-gnu-ranlib ./libhashtree.a
+          # Generate checksum
+          sha256sum libhashtree.a > libhashtree.a.sha256
+
+      - name: Upload Linux ARM64 library
+        uses: actions/upload-artifact@v4
+        with:
+          name: libhashtree-linux-arm64
+          path: |
+            src/libhashtree.a
+            src/libhashtree.a.sha256
+          retention-days: 7
+
+
+  build-darwin-arm64:
+    runs-on: macos-14  # Apple Silicon macOS
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Build Darwin ARM64 library
+        run: |
+          cd src
+          make clean
+          # Build with reproducible flags
+          export SOURCE_DATE_EPOCH=$(git log -1 --format=%ct)
+          make CFLAGS="-O3 -Wall -Werror -mmacosx-version-min=11.0 -ffile-prefix-map=$(pwd)=." ASFLAGS="-g -fpic"
+          # Copy the library to a simple location
+          cp ../build/lib/libhashtree.a ./libhashtree.a
+          # Remove non-deterministic archive metadata
+          strip -S ./libhashtree.a || true
+          # Normalize archive
+          ranlib ./libhashtree.a
+          # Generate checksum
+          shasum -a 256 libhashtree.a > libhashtree.a.sha256
+
+      - name: Upload Darwin ARM64 library
+        uses: actions/upload-artifact@v4
+        with:
+          name: libhashtree-darwin-arm64
+          path: |
+            src/libhashtree.a
+            src/libhashtree.a.sha256
+          retention-days: 7
+
+  build-windows-amd64:
+    runs-on: windows-latest
+    defaults:
+      run:
+        shell: msys2 {0}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup MSYS2
+        uses: msys2/setup-msys2@v2
+        with:
+          msystem: MINGW64
+          update: true
+          install: >-
+            mingw-w64-x86_64-gcc
+            mingw-w64-x86_64-make
+            make
+
+      - name: Build Windows AMD64 library
+        run: |
+          cd src
+          make clean
+          # Build with reproducible flags
+          export SOURCE_DATE_EPOCH=$(git log -1 --format=%ct)
+          make CFLAGS="-O3 -Wall -Werror -static -ffile-prefix-map=$(pwd)=." ASFLAGS="-g -fpic"
+          # Copy the library (Windows might create .lib, normalize to .a)
+          if [ -f "../build/lib/libhashtree.lib" ]; then
+            cp ../build/lib/libhashtree.lib ./libhashtree.a
+          else
+            cp ../build/lib/libhashtree.a ./libhashtree.a
+          fi
+          # Remove non-deterministic archive metadata
+          strip --strip-debug ./libhashtree.a || true
+          # Normalize archive
+          ranlib ./libhashtree.a
+          # Generate checksum
+          sha256sum libhashtree.a > libhashtree.a.sha256
+
+      - name: Upload Windows AMD64 library
+        uses: actions/upload-artifact@v4
+        with:
+          name: libhashtree-windows-amd64
+          path: |
+            src/libhashtree.a
+            src/libhashtree.a.sha256
+          retention-days: 7
+
+  collect-libraries:
+    runs-on: ubuntu-latest
+    needs: [build-linux-amd64, build-linux-arm64, build-darwin-arm64, build-windows-amd64]
+    steps:
+      - name: Create lib directories
+        run: |
+          mkdir -p lib/{linux_amd64,linux_arm64,darwin_arm64,windows_amd64}
+
+      - name: Download all libraries
+        uses: actions/download-artifact@v4
+        with:
+          pattern: libhashtree-*
+          merge-multiple: false
+
+      - name: Move libraries and checksums to correct locations
+        run: |
+          cp libhashtree-linux-amd64/libhashtree.a* lib/linux_amd64/
+          cp libhashtree-linux-arm64/libhashtree.a* lib/linux_arm64/
+          cp libhashtree-darwin-arm64/libhashtree.a* lib/darwin_arm64/
+          cp libhashtree-windows-amd64/libhashtree.a* lib/windows_amd64/
+
+      - name: Verify libraries and checksums
+        run: |
+          echo "=== Library Information ==="
+          echo "Linux AMD64:" && file lib/linux_amd64/libhashtree.a
+          echo "Linux ARM64:" && file lib/linux_arm64/libhashtree.a  
+          echo "Darwin ARM64:" && file lib/darwin_arm64/libhashtree.a
+          echo "Windows AMD64:" && file lib/windows_amd64/libhashtree.a
+          echo ""
+          echo "=== Checksum Verification ==="
+          for platform in linux_amd64 linux_arm64 darwin_arm64 windows_amd64; do
+            echo "Verifying $platform..."
+            cd lib/$platform
+            if command -v sha256sum >/dev/null; then
+              sha256sum -c libhashtree.a.sha256
+            else
+              shasum -a 256 -c libhashtree.a.sha256
+            fi
+            cd ../..
+          done
+
+      - name: Create library bundle
+        run: |
+          echo "# Static Library Bundle" > lib/README.md
+          echo "" >> lib/README.md
+          echo "## Build Information" >> lib/README.md
+          echo "- **Commit:** ${{ github.sha }}" >> lib/README.md
+          echo "- **Build Date:** $(date -u +"%Y-%m-%d %H:%M:%S UTC")" >> lib/README.md
+          echo "- **Build Trigger:** ${{ github.event_name }}" >> lib/README.md
+          echo "- **Repository:** ${{ github.repository }}" >> lib/README.md
+          echo "- **Workflow:** ${{ github.workflow }}" >> lib/README.md
+          echo "- **Run ID:** ${{ github.run_id }}" >> lib/README.md
+          echo "" >> lib/README.md
+          echo "## Contents" >> lib/README.md
+          echo "Each platform directory contains:" >> lib/README.md
+          echo "- \`libhashtree.a\` - Static library with platform-specific optimizations" >> lib/README.md
+          echo "- \`libhashtree.a.sha256\` - SHA256 checksum for verification" >> lib/README.md
+          echo "" >> lib/README.md
+          echo "### Platform Details:" >> lib/README.md
+          echo "- **linux_amd64/** - Linux x86_64 with AVX2/AVX512/SHANI optimizations" >> lib/README.md
+          echo "- **linux_arm64/** - Linux ARM64 with NEON/SHA2 optimizations" >> lib/README.md
+          echo "- **darwin_arm64/** - macOS Apple Silicon with SHA2 optimizations" >> lib/README.md
+          echo "- **windows_amd64/** - Windows x86_64 with AVX2/AVX512/SHANI optimizations" >> lib/README.md
+          echo "" >> lib/README.md
+          echo "**Note:** macOS Intel (darwin_amd64) is not included - uses pure Go fallback implementation" >> lib/README.md
+          echo "" >> lib/README.md
+          echo "## Reproducible Builds" >> lib/README.md
+          echo "These libraries are built with reproducible build flags:" >> lib/README.md
+          echo "- \`SOURCE_DATE_EPOCH\` set to commit timestamp" >> lib/README.md
+          echo "- \`-ffile-prefix-map\` to normalize file paths" >> lib/README.md
+          echo "- Debug symbols stripped for determinism" >> lib/README.md
+          echo "- Archive metadata normalized" >> lib/README.md
+          echo "" >> lib/README.md
+          echo "## Verification" >> lib/README.md
+          echo "To verify library integrity:" >> lib/README.md
+          echo "\`\`\`bash" >> lib/README.md
+          echo "# Verify SHA256 checksum" >> lib/README.md
+          echo "sha256sum -c libhashtree.a.sha256" >> lib/README.md
+          echo "" >> lib/README.md
+          echo "\`\`\`" >> lib/README.md
+          echo "" >> lib/README.md
+          echo "## Reproduction" >> lib/README.md
+          echo "To reproduce these builds:" >> lib/README.md
+          echo "1. Check out commit: \`git checkout ${{ github.sha }}\`" >> lib/README.md
+          echo "2. Use the same GitHub Actions environment (see workflow file)" >> lib/README.md
+          echo "3. The build should produce identical checksums" >> lib/README.md
+
+
+      - name: Upload static library bundle
+        uses: actions/upload-artifact@v4
+        with:
+          name: hashtree-static-libs-${{ github.sha }}
+          path: lib/
+          retention-days: 30

Makefile

@@ -16,8 +16,6 @@ rust_bindings:
 	cd rust_bindings && cargo build --release
 rust_tests:
 	cd rust_bindings && cargo test
-go_bindings:
-	$(MAKE) -C src go_bindings
 all:
 	$(MAKE) -C src all
 

bindings.go

@@ -17,8 +17,7 @@ var (
 	ErrDigestsNotMultipleOf32 = errors.New("digests not multiple of 32 bytes")
 )
 
-//go:noescape
-func HashtreeHash(output *byte, input *byte, count uint64)
+// HashtreeHash is defined in assembly for supported platforms
 
 // Hash hashes the chunks two at the time and outputs the digests on the first
 // argument. It does check for lengths on the inputs.

cgo_darwin_amd64.go

@@ -0,0 +1,13 @@
+//go:build cgo && darwin && amd64
+// +build cgo,darwin,amd64
+
+package hashtree
+
+// HashtreeHash is not available on macOS Intel due to assembly syntax incompatibility.
+// This function is never called since supportedCPU is false on this platform.
+func HashtreeHash(output *byte, input *byte, count uint64) {
+	panic("HashtreeHash should not be called on macOS Intel - uses pure Go fallback")
+}
+
+// supportedCPU is false on macOS Intel, forcing pure Go implementation
+var supportedCPU = false

cgo_darwin_arm64.go

@@ -0,0 +1,27 @@
+//go:build cgo && darwin && arm64
+// +build cgo,darwin,arm64
+
+package hashtree
+
+/*
+#cgo CFLAGS: -O3 -I./src
+#cgo LDFLAGS: -L./lib/darwin_arm64 -lhashtree
+
+#include "src/hashtree.h"
+
+void call_hashtree_hash(unsigned char* output, const unsigned char* input, uint64_t count) {
+    hashtree_hash(output, input, count);
+}
+*/
+import "C"
+import (
+	"unsafe"
+	"github.com/klauspost/cpuid/v2"
+)
+
+func HashtreeHash(output *byte, input *byte, count uint64) {
+	C.call_hashtree_hash((*C.uchar)(unsafe.Pointer(output)), (*C.uchar)(unsafe.Pointer(input)), C.uint64_t(count))
+}
+
+var hasShani = cpuid.CPU.Supports(cpuid.SHA2)
+var supportedCPU = hasShani

cgo_fallback.go

@@ -0,0 +1,12 @@
+//go:build !cgo
+// +build !cgo
+
+package hashtree
+
+// HashtreeHash fallback for when CGO is disabled
+func HashtreeHash(output *byte, input *byte, count uint64) {
+	panic("HashtreeHash called with CGO disabled - this should not happen")
+}
+
+// Disable optimized CPU support when CGO is not available
+var supportedCPU = false

cgo_linux_amd64.go

@@ -0,0 +1,29 @@
+//go:build cgo && linux && amd64
+// +build cgo,linux,amd64
+
+package hashtree
+
+/*
+#cgo CFLAGS: -O3 -I./src
+#cgo LDFLAGS: -L./lib/linux_amd64 -lhashtree
+
+#include "src/hashtree.h"
+
+void call_hashtree_hash(unsigned char* output, const unsigned char* input, uint64_t count) {
+    hashtree_hash(output, input, count);
+}
+*/
+import "C"
+import (
+	"unsafe"
+	"github.com/klauspost/cpuid/v2"
+)
+
+func HashtreeHash(output *byte, input *byte, count uint64) {
+	C.call_hashtree_hash((*C.uchar)(unsafe.Pointer(output)), (*C.uchar)(unsafe.Pointer(input)), C.uint64_t(count))
+}
+
+var hasAVX512 = cpuid.CPU.Supports(cpuid.AVX512F, cpuid.AVX512VL)
+var hasAVX2 = cpuid.CPU.Supports(cpuid.AVX2, cpuid.BMI2)
+var hasShani = cpuid.CPU.Supports(cpuid.SHA, cpuid.AVX)
+var supportedCPU = hasAVX2 || hasShani || hasAVX512