final pass

Author: jakthom

.github/workflows/ci.yml

@@ -39,7 +39,14 @@ jobs:
         run: make test.static
 
       - name: Race test
-        run: go test -race ./...
+        run: |
+          set -euo pipefail
+          if [ "$(go env GOOS)" = "darwin" ]; then
+            shared="libdatafusion_go.dylib"
+          else
+            shared="libdatafusion_go.so"
+          fi
+          DATAFUSION_GO_LIBRARY="$PWD/internal/native/lib/$(go env GOOS)-$(go env GOARCH)/${shared}" go test -race ./...
 
       - name: Vet
         run: go vet ./...
@@ -106,7 +113,7 @@ jobs:
 
       - name: Race test
         shell: msys2 {0}
-        run: go test -race ./...
+        run: DATAFUSION_GO_LIBRARY="$PWD/internal/native/lib/windows-amd64/datafusion_go.dll" go test -race ./...
 
       - name: Vet
         shell: msys2 {0}

.github/workflows/release.yml

@@ -78,11 +78,11 @@ jobs:
           CARGO_BUILD_TARGET: x86_64-pc-windows-gnu
         run: make test
 
-      - name: Upload native archive
+      - name: Upload native libraries
         uses: actions/upload-artifact@v4
         with:
           name: native-${{ matrix.platform }}
-          path: internal/native/lib/${{ matrix.platform }}/libdatafusion_go.a
+          path: internal/native/lib/${{ matrix.platform }}/*datafusion_go*
           if-no-files-found: error
 
   release:
@@ -130,10 +130,12 @@ jobs:
           for dir in internal/native/lib/native-*; do
             platform="${dir##*/native-}"
             mkdir -p "internal/native/lib/${platform}"
-            mv "${dir}/libdatafusion_go.a" "internal/native/lib/${platform}/libdatafusion_go.a"
+            find "${dir}" -maxdepth 1 -type f -print | while read -r file; do
+              mv "${file}" "internal/native/lib/${platform}/$(basename "${file}")"
+            done
             rmdir "${dir}"
           done
-          find internal/native/lib -name libdatafusion_go.a -print | sort
+          find internal/native/lib -type f -print | sort
 
       - name: Generate checksums
         run: make checksums
@@ -144,6 +146,23 @@ jobs:
       - name: Verify downloaded release artifacts
         run: make verify.release.downloaded
 
+      - name: Stage release assets
+        run: |
+          set -euo pipefail
+          tag="${{ steps.version.outputs.release_tag }}"
+          rm -rf dist
+          mkdir -p dist
+          while IFS= read -r file; do
+            platform="$(basename "$(dirname "${file}")")"
+            base="$(basename "${file}")"
+            cp "${file}" "dist/datafusion-go-${tag}-${platform}-${base}"
+          done < <(find internal/native/lib -mindepth 2 -maxdepth 2 -type f \( -name 'libdatafusion_go.a' -o -name 'libdatafusion_go.so' -o -name 'libdatafusion_go.dylib' -o -name 'datafusion_go.dll' \) -print | sort)
+          cd dist
+          shasum -a 256 datafusion-go-* > SHA256SUMS
+          shasum -a 256 -c SHA256SUMS
+          cd ..
+          cp dist/SHA256SUMS internal/native/lib/SHA256SUMS
+
       - name: Prepare release notes
         id: notes
         run: |
@@ -162,6 +181,20 @@ jobs:
           fi
           echo "path=${notes_file}" >> "$GITHUB_OUTPUT"
 
+      - name: Commit release checksum manifest
+        if: ${{ inputs.publish }}
+        run: |
+          set -euo pipefail
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git add -f internal/native/lib/SHA256SUMS
+          if git diff --cached --quiet; then
+            echo "release checksum manifest is unchanged"
+          else
+            git commit -m "chore: prepare ${{ steps.version.outputs.release_tag }} native checksums"
+            git push origin "HEAD:${GITHUB_REF_NAME}"
+          fi
+
       - name: Tag release
         if: ${{ inputs.publish }}
         run: |
@@ -177,10 +210,10 @@ jobs:
           GH_TOKEN: ${{ github.token }}
         run: |
           set -euo pipefail
-          assets=(internal/native/lib/SHA256SUMS)
+          assets=(dist/SHA256SUMS)
           while IFS= read -r file; do
             assets+=("$file")
-          done < <(find internal/native/lib -name libdatafusion_go.a -print | sort)
+          done < <(find dist -type f -name 'datafusion-go-*' -print | sort)
           gh release create "${{ steps.version.outputs.release_tag }}" \
             --title "${{ steps.version.outputs.release_tag }}" \
             --notes-file "${{ steps.notes.outputs.path }}" \

.gitignore

@@ -14,7 +14,8 @@ temp/
 *.o
 *.obj
 *.wasm
-internal/native/lib/
+internal/native/lib/*
+!internal/native/lib/SHA256SUMS
 rust/target/
 
 # Go test, coverage, and profiling artifacts

CONTRIBUTING.md

@@ -67,7 +67,7 @@ changes incompatibly with the Go native wrapper.
 
 ## Native Libraries
 
-The Rust crate under `rust/` builds `libdatafusion_go.a`. The default Go build links the platform-specific archive from `internal/native/lib/<goos>-<goarch>/libdatafusion_go.a`.
+The Rust crate under `rust/` builds a static archive and a shared library. The default Go build loads the platform-specific shared library at runtime from `DATAFUSION_GO_LIBRARY`, from `internal/native/lib/<goos>-<goarch>` in source checkouts, or from the checksum-verified release-asset cache. The explicit `datafusion_use_bundled` mode links the platform-specific static archive from `internal/native/lib/<goos>-<goarch>/libdatafusion_go.a`.
 
 Use `make bundle` only when you intend to copy the current host build into `internal/native/lib`. Release verification uses `make verify.release.downloaded` so downloaded matrix artifacts are not overwritten by the release runner.
 
@@ -81,5 +81,5 @@ Go/Rust/no-cgo tests, and runs a clean consumer-module smoke test without
 tagging or creating a GitHub release.
 
 When the dry run succeeds, rerun the same workflow with `publish=true`. It tags
-the checked-out commit with the derived release tag and uploads the generated
-native archives plus checksums.
+commits the release-asset checksum manifest, tags that commit with the derived
+release tag, and uploads the generated native libraries plus checksums.

Makefile

@@ -6,6 +6,16 @@ NATIVE_LIB_DIR := internal/native/lib/$(NATIVE_PLATFORM)
 NATIVE_LIB := $(NATIVE_LIB_DIR)/libdatafusion_go.a
 CARGO_BUILD_TARGET ?=
 
+ifeq ($(GOOS),windows)
+NATIVE_SHARED_NAME := datafusion_go.dll
+else ifeq ($(GOOS),darwin)
+NATIVE_SHARED_NAME := libdatafusion_go.dylib
+else
+NATIVE_SHARED_NAME := libdatafusion_go.so
+endif
+
+NATIVE_SHARED := $(NATIVE_LIB_DIR)/$(NATIVE_SHARED_NAME)
+
 ifeq ($(GOOS)-$(GOARCH),windows-amd64)
 CARGO_BUILD_TARGET := $(or $(CARGO_BUILD_TARGET),x86_64-pc-windows-gnu)
 endif
@@ -18,11 +28,18 @@ RUST_TARGET_FLAG :=
 RUST_TARGET_RELEASE_DIR := rust/target/release
 endif
 
+RUST_SHARED_LIB := $(RUST_TARGET_RELEASE_DIR)/$(NATIVE_SHARED_NAME)
+
 ifeq ($(GOOS),darwin)
 RUST_BUILD_ENV := MACOSX_DEPLOYMENT_TARGET=$(MACOSX_DEPLOYMENT_TARGET) CFLAGS="$(strip $(CFLAGS) -mmacosx-version-min=$(MACOSX_DEPLOYMENT_TARGET))"
+STRIP_SHARED := strip -x
+else ifeq ($(GOOS),linux)
+STRIP_SHARED := strip --strip-unneeded
+else ifeq ($(GOOS),windows)
+STRIP_SHARED := strip --strip-unneeded
 endif
 
-.PHONY: generate generate.check rust bundle checksums verify.checksums test test.bundled test.source test.static consumer.smoke lint verify.release verify.release.downloaded clean
+.PHONY: generate generate.check rust bundle checksums verify.checksums test test.dynamic test.bundled test.source test.static consumer.smoke lint verify.release verify.release.downloaded clean
 
 generate:
 	go run ./internal/tools/genversions
@@ -38,20 +55,26 @@ rust: generate.check
 bundle: rust
 	mkdir -p $(NATIVE_LIB_DIR)
 	cp $(RUST_TARGET_RELEASE_DIR)/libdatafusion_go.a $(NATIVE_LIB)
+	cp $(RUST_SHARED_LIB) $(NATIVE_SHARED)
+	if [ -n "$(STRIP_SHARED)" ]; then $(STRIP_SHARED) $(NATIVE_SHARED); fi
 
 checksums:
 	mkdir -p internal/native/lib
-	cd internal/native/lib && find . -name libdatafusion_go.a -print | sed 's#^\./##' | sort | while read -r file; do shasum -a 256 "$$file"; done > SHA256SUMS
+	cd internal/native/lib && find . -type f \( -name libdatafusion_go.a -o -name libdatafusion_go.so -o -name libdatafusion_go.dylib -o -name datafusion_go.dll \) -print | sed 's#^\./##' | sort | while read -r file; do shasum -a 256 "$$file"; done > SHA256SUMS
 
 verify.checksums:
 	test -s internal/native/lib/SHA256SUMS
 	cd internal/native/lib && shasum -a 256 -c SHA256SUMS
 
 test: bundle
+	$(MAKE) test.dynamic
 	$(MAKE) test.bundled
 
+test.dynamic:
+	DATAFUSION_GO_LIBRARY=$(CURDIR)/$(NATIVE_SHARED) go test ./...
+
 test.bundled:
-	go test ./...
+	go test -tags=datafusion_use_bundled ./...
 
 test.source: rust
 	go test -tags=datafusion_use_source ./...
@@ -82,23 +105,23 @@ consumer.smoke:
 		'	if err := db.QueryRowContext(context.Background(), "select 1").Scan(&value); err != nil { panic(err) }' \
 		'	if value != 1 { panic(fmt.Sprintf("got %d, want 1", value)) }' \
 		'}' > main.go; \
-	go run .
+		DATAFUSION_GO_LIBRARY=$(CURDIR)/$(NATIVE_SHARED) go run .
 
 lint: generate.check
 	go run github.com/golangci/golangci-lint/v2/cmd/golangci-lint@latest run
 	cargo clippy --manifest-path rust/Cargo.toml --all-targets -- -D warnings
 	cargo fmt --manifest-path rust/Cargo.toml -- --check
 
 verify.release: test test.source test.static
-	go test -race ./...
+	DATAFUSION_GO_LIBRARY=$(CURDIR)/$(NATIVE_SHARED) go test -race ./...
 	go vet ./...
 	cargo test --manifest-path rust/Cargo.toml --release
 	CGO_ENABLED=0 go test ./...
 	$(MAKE) checksums
 	$(MAKE) verify.checksums
 
 verify.release.downloaded: verify.checksums test.bundled consumer.smoke test.source test.static
-	go test -race ./...
+	DATAFUSION_GO_LIBRARY=$(CURDIR)/$(NATIVE_SHARED) go test -race ./...
 	go vet ./...
 	cargo test --manifest-path rust/Cargo.toml --release
 	CGO_ENABLED=0 go test ./...

README.md

@@ -29,7 +29,7 @@ Build and bundle the Rust shim before running Go tests:
 make test
 ```
 
-That target copies the generated native archive to `internal/native/lib/<goos>-<goarch>/libdatafusion_go.a`, which is the default static-link path. Native archives are build outputs and are not committed to Git.
+That target copies the generated native archive and shared library to `internal/native/lib/<goos>-<goarch>/`. Native libraries are build outputs and are not committed to Git.
 
 To link directly from `rust/target/release` during local development:
 
@@ -52,17 +52,20 @@ db, err := sql.Open("datafusion", "")
 
 ## Linking Modes
 
-Default builds use a generated static library selected from `internal/native/lib/<goos>-<goarch>`. Source checkouts should run `make bundle` or `make test` before invoking `go test` directly.
+Default builds use cgo but do not link a DataFusion native library at Go link time. At runtime, the driver loads a shared `libdatafusion_go` from `DATAFUSION_GO_LIBRARY`, from `internal/native/lib/<goos>-<goarch>` in source checkouts, or from the checksum-verified cache populated from the matching GitHub Release asset.
+
+Source checkouts should run `make bundle` or `make test` before invoking Go tests that open the driver directly. To disable automatic release-asset downloads, set `DATAFUSION_GO_NO_DOWNLOAD=1`.
 
 Development modes:
 
 ```sh
 make rust
+go test -tags=datafusion_use_bundled ./...
 go test -tags=datafusion_use_source ./...
 go test -tags=datafusion_use_static_lib ./...
 ```
 
-`datafusion_use_source` and `datafusion_use_static_lib` link from `rust/target/release`, so run `make rust` first. On `windows-amd64`, `make rust` targets `x86_64-pc-windows-gnu` and the link path is `rust/target/x86_64-pc-windows-gnu/release`.
+`datafusion_use_bundled` links the static archive from `internal/native/lib/<goos>-<goarch>`. `datafusion_use_source` and `datafusion_use_static_lib` link from `rust/target/release`, so run `make rust` first. On `windows-amd64`, `make rust` targets `x86_64-pc-windows-gnu` and the link path is `rust/target/x86_64-pc-windows-gnu/release`.
 
 Shared-library mode links with `-ldatafusion_go` and requires `libdatafusion_go` to be on the system linker path:
 
@@ -200,6 +203,6 @@ Nested and complex Arrow values such as lists, structs, maps, unions, dictionari
 
 ## Current Distribution Status
 
-Native archives are generated by the build and intentionally kept out of Git because the static archives exceed GitHub's normal file-size limits. Release automation builds archives for `darwin-arm64`, `darwin-amd64`, `linux-amd64`, `linux-arm64`, and `windows-amd64`, writes `internal/native/lib/SHA256SUMS`, verifies it without rebundling the current runner's local archive, and can run as a dry run before publishing. When `publish` is enabled, it tags the source commit and uploads the generated archives plus checksums to the GitHub release.
+Native libraries are generated by the build and intentionally kept out of Git because the static archives exceed GitHub's normal file-size limits and Go module zips have a 500 MiB source limit. Release automation builds static archives and shared libraries for `darwin-arm64`, `darwin-amd64`, `linux-amd64`, `linux-arm64`, and `windows-amd64`, verifies them without rebundling the current runner's local output, writes the release-asset `SHA256SUMS` manifest into the tagged tree, and uploads the generated native files plus checksums to the GitHub release.
 
-Windows arm64 is not currently a bundled target. Before a public release, run the release workflow once with `publish=false`, verify that all release-runner labels are available in GitHub Actions, and confirm the generated native archives link from a clean consumer module on each target platform.
+Windows arm64 is not currently a bundled target. Before a public release, run the release workflow once with `publish=false`, verify that all release-runner labels are available in GitHub Actions, and confirm the generated native libraries load from a clean consumer module on each target platform.