SEGFAULT when using ADBC Postgres driver with Go FlightServer

[fwojciec]

What happened?

I'm building a FlightServer in Go that uses the PostgreSQL ADBC driver to fetch data from Postgres, and running into issues with SEGFAULTS (see the stack trace).

This is the code for minimal reproduction of the issue:

package main

import (
	"fmt"
	"os"

	"github.com/apache/arrow-adbc/go/adbc"
	"github.com/apache/arrow-adbc/go/adbc/drivermgr"
	"github.com/apache/arrow-go/v18/arrow/flight"
	"github.com/apache/arrow-go/v18/arrow/ipc"
)

func main() {
	if err := runServer(); err != nil {
		panic(err)
	}
}

type server struct {
	flight.BaseFlightServer
}

func (s *server) DoGet(tkt *flight.Ticket, fs flight.FlightService_DoGetServer) error {
	pgDsn := os.Getenv("POSTGRESQL_DSN")
	if pgDsn == "" {
		return fmt.Errorf("POSTGRESQL_DSN is not set")
	}

	var drv drivermgr.Driver
	db, err := drv.NewDatabase(map[string]string{
		"driver":          "adbc_driver_postgresql",
		adbc.OptionKeyURI: pgDsn,
	})
	if err != nil {
		return err
	}
	defer db.Close()
	conn, err := db.Open(fs.Context())
	if err != nil {
		return err
	}
	defer conn.Close()

	st, err := conn.NewStatement()
	if err != nil {
		return err
	}
	defer st.Close()

	if err := st.SetSqlQuery("SELECT * FROM synthetic_filter_option"); err != nil {
		return err
	}

	reader, _, err := st.ExecuteQuery(fs.Context())
	if err != nil {
		return err
	}
	defer reader.Release()

	wr := flight.NewRecordWriter(
		fs,
		ipc.WithSchema(reader.Schema()),
	)
	defer wr.Close()

	for reader.Next() {
		rec := reader.Record()
		if err := wr.Write(rec); err != nil {
			return fmt.Errorf("couldn't write record: %w", err)
		}
	}

	return nil
}

func runServer() error {
	s := flight.NewServerWithMiddleware(nil)
	s.Init("0.0.0.0:8816")
	s.RegisterFlightService(&server{})

	return s.Serve()
}

The issue doesn't seem to be data specific, the table I'm using is 365712 rows and has 9 columns with the following types:

• integer
• text
• boolean

I've provided instructions for how to create Postgres data that reproduces the issue below.

I haven't been able to reproduce when using the same exact code outside of FlightServer context - i.e. if I'm for example just printing batches to stdout instead of sending them over GRPC and exiting it works fine.

This fails only occasionally, and often works on the first try - it usually takes 3-4 tries to retrieve the table data to reproduce the SEGFAULT.

Any chance I'm just misunderstanding something about how this is supposed to work?

Stack Trace

SIGBUS: bus error
PC=0x128b09d3c m=7 sigcode=1 addr=0x40000000b1
signal arrived during cgo execution

goroutine 34 gp=0x14000186380 m=7 mp=0x14000185008 [syscall]:
runtime.cgocall(0x100ab2f1c, 0x14000072528)
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/cgocall.go:167 +0x44 fp=0x140000724f0 sp=0x140000724b0 pc=0x1004f0f14
github.com/apache/arrow-go/v18/arrow/cdata._Cfunc_ArrowArrayStreamRelease(0x600000e755f0)
	_cgo_gotypes.go:276 +0x30 fp=0x14000072520 sp=0x140000724f0 pc=0x100862830
github.com/apache/arrow-go/v18/arrow/cdata.initReader.func2.1(...)
	/Users/filip/go/pkg/mod/github.com/apache/arrow-go/v18@v18.2.0/arrow/cdata/cdata.go:926
github.com/apache/arrow-go/v18/arrow/cdata.initReader.func2(0x14000050580)
	/Users/filip/go/pkg/mod/github.com/apache/arrow-go/v18@v18.2.0/arrow/cdata/cdata.go:926 +0x60 fp=0x14000072570 sp=0x14000072520 pc=0x10086fab0
runtime.call16(0x0, 0x100d54510, 0x14000194430, 0x10, 0x10, 0x10, 0x14000072600)
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/asm_arm64.s:504 +0x78 fp=0x14000072590 sp=0x14000072570 pc=0x1004f9ff8
runtime.runfinq()
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/mfinal.go:275 +0x3d4 fp=0x140000727d0 sp=0x14000072590 pc=0x10049f5b4
runtime.goexit({})
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/asm_arm64.s:1223 +0x4 fp=0x140000727d0 sp=0x140000727d0 pc=0x1004fbff4
created by runtime.createfing in goroutine 1
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/mfinal.go:166 +0x80

goroutine 1 gp=0x140000021c0 m=nil [IO wait]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/proc.go:435 +0xc8 fp=0x14000137a90 sp=0x14000137a70 pc=0x1004f42d8
runtime.netpollblock(0x14000235b28?, 0x56ab60?, 0x1?)
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/netpoll.go:575 +0x158 fp=0x14000137ad0 sp=0x14000137a90 pc=0x1004ba538
internal/poll.runtime_pollWait(0x12813bbb8, 0x72)
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/netpoll.go:351 +0xa0 fp=0x14000137b00 sp=0x14000137ad0 pc=0x1004f34d0
internal/poll.(*pollDesc).wait(0x14000228600?, 0x10056cdb8?, 0x0)
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/internal/poll/fd_poll_runtime.go:84 +0x28 fp=0x14000137b30 sp=0x14000137b00 pc=0x100566378
internal/poll.(*pollDesc).waitRead(...)
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0x14000228600)
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/internal/poll/fd_unix.go:620 +0x24c fp=0x14000137be0 sp=0x14000137b30 pc=0x10056ac4c
net.(*netFD).accept(0x14000228600)
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/net/fd_unix.go:172 +0x28 fp=0x14000137ca0 sp=0x14000137be0 pc=0x1008b8eb8
net.(*TCPListener).accept(0x140001dab80)
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/net/tcpsock_posix.go:159 +0x24 fp=0x14000137cf0 sp=0x14000137ca0 pc=0x1008cdec4
net.(*TCPListener).Accept(0x140001dab80)
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/net/tcpsock.go:380 +0x2c fp=0x14000137d30 sp=0x14000137cf0 pc=0x1008cd1ac
google.golang.org/grpc.(*Server).Serve(0x14000246200, {0x100d5e398, 0x140001dab80})
	/Users/filip/go/pkg/mod/google.golang.org/grpc@v1.71.0/server.go:890 +0x3fc fp=0x14000137e90 sp=0x14000137d30 pc=0x100a2f43c
github.com/apache/arrow-go/v18/arrow/flight.(*server).Serve(0x140002531a0)
	/Users/filip/go/pkg/mod/github.com/apache/arrow-go/v18@v18.2.0/arrow/flight/server.go:386 +0xb4 fp=0x14000137ed0 sp=0x14000137e90 pc=0x100a62d44
main.runServer()
	/Users/filip/code/go/goadbcpgrepro/main.go:82 +0x8c fp=0x14000137f20 sp=0x14000137ed0 pc=0x100a6536c
main.main()
	/Users/filip/code/go/goadbcpgrepro/main.go:14 +0x1c fp=0x14000137f40 sp=0x14000137f20 pc=0x100a64a2c
runtime.main()
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/proc.go:283 +0x284 fp=0x14000137fd0 sp=0x14000137f40 pc=0x1004c0f84
runtime.goexit({})
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/asm_arm64.s:1223 +0x4 fp=0x14000137fd0 sp=0x14000137fd0 pc=0x1004fbff4

goroutine 2 gp=0x14000002c40 m=nil [force gc (idle)]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/proc.go:435 +0xc8 fp=0x14000072f90 sp=0x14000072f70 pc=0x1004f42d8
runtime.goparkunlock(...)
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/proc.go:441
runtime.forcegchelper()
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/proc.go:348 +0xb8 fp=0x14000072fd0 sp=0x14000072f90 pc=0x1004c12d8
runtime.goexit({})
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/asm_arm64.s:1223 +0x4 fp=0x14000072fd0 sp=0x14000072fd0 pc=0x1004fbff4
created by runtime.init.7 in goroutine 1
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/proc.go:336 +0x24

goroutine 18 gp=0x140000a2380 m=nil [GC sweep wait]:
runtime.gopark(0x1?, 0x0?, 0x0?, 0x0?, 0x0?)
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/proc.go:435 +0xc8 fp=0x1400006e760 sp=0x1400006e740 pc=0x1004f42d8
runtime.goparkunlock(...)
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/proc.go:441
runtime.bgsweep(0x1400007e000)
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/mgcsweep.go:316 +0x108 fp=0x1400006e7b0 sp=0x1400006e760 pc=0x1004ac4d8
runtime.gcenable.gowrap1()
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/mgc.go:204 +0x28 fp=0x1400006e7d0 sp=0x1400006e7b0 pc=0x1004a02e8
runtime.goexit({})
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/asm_arm64.s:1223 +0x4 fp=0x1400006e7d0 sp=0x1400006e7d0 pc=0x1004fbff4
created by runtime.gcenable in goroutine 1
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/mgc.go:204 +0x6c

goroutine 19 gp=0x140000a2540 m=nil [GC scavenge wait]:
runtime.gopark(0x10000?, 0x100bb8008?, 0x0?, 0x0?, 0x0?)
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/proc.go:435 +0xc8 fp=0x1400006ef60 sp=0x1400006ef40 pc=0x1004f42d8
runtime.goparkunlock(...)
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/proc.go:441
runtime.(*scavengerState).park(0x1012f9860)
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/mgcscavenge.go:425 +0x5c fp=0x1400006ef90 sp=0x1400006ef60 pc=0x1004a9f7c
runtime.bgscavenge(0x1400007e000)
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/mgcscavenge.go:658 +0xac fp=0x1400006efb0 sp=0x1400006ef90 pc=0x1004aa51c
runtime.gcenable.gowrap2()
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/mgc.go:205 +0x28 fp=0x1400006efd0 sp=0x1400006efb0 pc=0x1004a0288
runtime.goexit({})
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/asm_arm64.s:1223 +0x4 fp=0x1400006efd0 sp=0x1400006efd0 pc=0x1004fbff4
created by runtime.gcenable in goroutine 1
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/mgc.go:205 +0xac

goroutine 35 gp=0x140001868c0 m=nil [chan receive]:
runtime.gopark(0x1400019c640?, 0x14000160198?, 0x48?, 0x7?, 0x100884538?)
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/proc.go:435 +0xc8 fp=0x140002406f0 sp=0x140002406d0 pc=0x1004f42d8
runtime.chanrecv(0x140001820e0, 0x0, 0x1)
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/chan.go:664 +0x42c fp=0x14000240770 sp=0x140002406f0 pc=0x10049215c
runtime.chanrecv1(0x0?, 0x0?)
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/chan.go:506 +0x14 fp=0x140002407a0 sp=0x14000240770 pc=0x100491cf4
runtime.unique_runtime_registerUniqueMapCleanup.func2(...)
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/mgc.go:1796
runtime.unique_runtime_registerUniqueMapCleanup.gowrap1()
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/mgc.go:1799 +0x3c fp=0x140002407d0 sp=0x140002407a0 pc=0x1004a350c
runtime.goexit({})
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/asm_arm64.s:1223 +0x4 fp=0x140002407d0 sp=0x140002407d0 pc=0x1004fbff4
created by unique.runtime_registerUniqueMapCleanup in goroutine 1
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/mgc.go:1794 +0x78

goroutine 36 gp=0x14000186a80 m=nil [select]:
runtime.gopark(0x14000240fa8?, 0x2?, 0x0?, 0x0?, 0x14000240f9c?)
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/proc.go:435 +0xc8 fp=0x14000240e30 sp=0x14000240e10 pc=0x1004f42d8
runtime.selectgo(0x14000240fa8, 0x14000240f98, 0x0?, 0x0, 0x0?, 0x1)
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/select.go:351 +0x6c4 fp=0x14000240f60 sp=0x14000240e30 pc=0x1004d45f4
github.com/apache/arrow-go/v18/arrow/flight.(*server).Serve.func1()
	/Users/filip/go/pkg/mod/github.com/apache/arrow-go/v18@v18.2.0/arrow/flight/server.go:380 +0x54 fp=0x14000240fd0 sp=0x14000240f60 pc=0x100a62dd4
runtime.goexit({})
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/asm_arm64.s:1223 +0x4 fp=0x14000240fd0 sp=0x14000240fd0 pc=0x1004fbff4
created by github.com/apache/arrow-go/v18/arrow/flight.(*server).Serve in goroutine 1
	/Users/filip/go/pkg/mod/github.com/apache/arrow-go/v18@v18.2.0/arrow/flight/server.go:379 +0x94

goroutine 4 gp=0x140000036c0 m=nil [select]:
runtime.gopark(0x14000088e88?, 0x2?, 0x0?, 0x0?, 0x14000088e54?)
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/proc.go:435 +0xc8 fp=0x14000088ce0 sp=0x14000088cc0 pc=0x1004f42d8
runtime.selectgo(0x14000088e88, 0x14000088e50, 0x14000088e68?, 0x0, 0x14000076008?, 0x1)
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/select.go:351 +0x6c4 fp=0x14000088e10 sp=0x14000088ce0 pc=0x1004d45f4
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0x14000050040, 0x1)
	/Users/filip/go/pkg/mod/google.golang.org/grpc@v1.71.0/internal/transport/controlbuf.go:412 +0x144 fp=0x14000088eb0 sp=0x14000088e10 pc=0x1009aa604
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0x14000228700)
	/Users/filip/go/pkg/mod/google.golang.org/grpc@v1.71.0/internal/transport/controlbuf.go:575 +0x70 fp=0x14000088f10 sp=0x14000088eb0 pc=0x1009aae60
google.golang.org/grpc/internal/transport.NewServerTransport.func2()
	/Users/filip/go/pkg/mod/google.golang.org/grpc@v1.71.0/internal/transport/http2_server.go:335 +0xc4 fp=0x14000088fd0 sp=0x14000088f10 pc=0x1009bf1b4
runtime.goexit({})
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/asm_arm64.s:1223 +0x4 fp=0x14000088fd0 sp=0x14000088fd0 pc=0x1004fbff4
created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 3
	/Users/filip/go/pkg/mod/google.golang.org/grpc@v1.71.0/internal/transport/http2_server.go:333 +0x125c

goroutine 5 gp=0x14000003880 m=nil [select]:
runtime.gopark(0x140000bcf28?, 0x4?, 0x8?, 0x98?, 0x140000bcea8?)
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/proc.go:435 +0xc8 fp=0x140000bcd20 sp=0x140000bcd00 pc=0x1004f42d8
runtime.selectgo(0x140000bcf28, 0x140000bcea0, 0x0?, 0x0, 0x0?, 0x1)
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/select.go:351 +0x6c4 fp=0x140000bce50 sp=0x140000bcd20 pc=0x1004d45f4
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0x14000126000)
	/Users/filip/go/pkg/mod/google.golang.org/grpc@v1.71.0/internal/transport/http2_server.go:1180 +0x164 fp=0x140000bcfb0 sp=0x140000bce50 pc=0x1009c57b4
google.golang.org/grpc/internal/transport.NewServerTransport.gowrap1()
	/Users/filip/go/pkg/mod/google.golang.org/grpc@v1.71.0/internal/transport/http2_server.go:356 +0x28 fp=0x140000bcfd0 sp=0x140000bcfb0 pc=0x1009bf0b8
runtime.goexit({})
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/asm_arm64.s:1223 +0x4 fp=0x140000bcfd0 sp=0x140000bcfd0 pc=0x1004fbff4
created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 3
	/Users/filip/go/pkg/mod/google.golang.org/grpc@v1.71.0/internal/transport/http2_server.go:356 +0x129c

goroutine 6 gp=0x14000003a40 m=nil [IO wait]:
runtime.gopark(0xffffffffffffffff?, 0xffffffffffffffff?, 0x23?, 0x0?, 0x10054a170?)
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/proc.go:435 +0xc8 fp=0x1400008d9e0 sp=0x1400008d9c0 pc=0x1004f42d8
runtime.netpollblock(0x0?, 0x0?, 0x0?)
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/netpoll.go:575 +0x158 fp=0x1400008da20 sp=0x1400008d9e0 pc=0x1004ba538
internal/poll.runtime_pollWait(0x12813baa0, 0x72)
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/netpoll.go:351 +0xa0 fp=0x1400008da50 sp=0x1400008da20 pc=0x1004f34d0
internal/poll.(*pollDesc).wait(0x14000108000?, 0x1400010e000?, 0x0)
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/internal/poll/fd_poll_runtime.go:84 +0x28 fp=0x1400008da80 sp=0x1400008da50 pc=0x100566378
internal/poll.(*pollDesc).waitRead(...)
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0x14000108000, {0x1400010e000, 0x8000, 0x8000})
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/internal/poll/fd_unix.go:165 +0x1fc fp=0x1400008db20 sp=0x1400008da80 pc=0x10056762c
net.(*netFD).Read(0x14000108000, {0x1400010e000?, 0x1800030008dba8?, 0x128125908?})
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/net/fd_posix.go:55 +0x28 fp=0x1400008db70 sp=0x1400008db20 pc=0x1008b7488
net.(*conn).Read(0x14000076008, {0x1400010e000?, 0x1400008dc08?, 0x1004dfcc4?})
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/net/net.go:194 +0x34 fp=0x1400008dbc0 sp=0x1400008db70 pc=0x1008c5704
bufio.(*Reader).Read(0x1400010c000, {0x14000120040, 0x9, 0x1009c308c?})
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/bufio/bufio.go:245 +0x1b0 fp=0x1400008dc00 sp=0x1400008dbc0 pc=0x1006aee10
io.ReadAtLeast({0x100d5a820, 0x1400010c000}, {0x14000120040, 0x9, 0x9}, 0x9)
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/io/io.go:335 +0xa0 fp=0x1400008dc50 sp=0x1400008dc00 pc=0x10053f160
io.ReadFull(...)
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/io/io.go:354
golang.org/x/net/http2.readFrameHeader({0x14000120040, 0x9, 0x14073e51810?}, {0x100d5a820?, 0x1400010c000?})
	/Users/filip/go/pkg/mod/golang.org/x/net@v0.37.0/http2/frame.go:237 +0x58 fp=0x1400008dca0 sp=0x1400008dc50 pc=0x10099a958
golang.org/x/net/http2.(*Framer).ReadFrame(0x14000120000)
	/Users/filip/go/pkg/mod/golang.org/x/net@v0.37.0/http2/frame.go:501 +0x74 fp=0x1400008dd50 sp=0x1400008dca0 pc=0x10099b024
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0x14000126000, {0x100d5f258, 0x1400010a210}, 0x1400010a240)
	/Users/filip/go/pkg/mod/google.golang.org/grpc@v1.71.0/internal/transport/http2_server.go:640 +0xc0 fp=0x1400008de80 sp=0x1400008dd50 pc=0x1009c1a00
google.golang.org/grpc.(*Server).serveStreams(0x14000246200, {0x100d5f110?, 0x101322380?}, {0x100d5f6f0, 0x14000126000}, {0x100d64818?, 0x14000076008?})
	/Users/filip/go/pkg/mod/google.golang.org/grpc@v1.71.0/server.go:1029 +0x2f4 fp=0x1400008df60 sp=0x1400008de80 pc=0x100a305f4
google.golang.org/grpc.(*Server).handleRawConn.func1()
	/Users/filip/go/pkg/mod/google.golang.org/grpc@v1.71.0/server.go:964 +0x5c fp=0x1400008dfd0 sp=0x1400008df60 pc=0x100a2fefc
runtime.goexit({})
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/asm_arm64.s:1223 +0x4 fp=0x1400008dfd0 sp=0x1400008dfd0 pc=0x1004fbff4
created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 3
	/Users/filip/go/pkg/mod/google.golang.org/grpc@v1.71.0/server.go:963 +0x1d4

goroutine 8 gp=0x14000003c00 m=nil [GC worker (idle)]:
runtime.gopark(0x1400010a210?, 0x100a30990?, 0xf0?, 0xf6?, 0x14000126000?)
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/proc.go:435 +0xc8 fp=0x14000074710 sp=0x140000746f0 pc=0x1004f42d8
runtime.gcBgMarkWorker(0x14000124310)
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/mgc.go:1423 +0xdc fp=0x140000747b0 sp=0x14000074710 pc=0x1004a277c
runtime.gcBgMarkStartWorkers.gowrap1()
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/mgc.go:1339 +0x28 fp=0x140000747d0 sp=0x140000747b0 pc=0x1004a2668
runtime.goexit({})
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/asm_arm64.s:1223 +0x4 fp=0x140000747d0 sp=0x140000747d0 pc=0x1004fbff4
created by runtime.gcBgMarkStartWorkers in goroutine 7
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/mgc.go:1339 +0x140

goroutine 9 gp=0x14000003dc0 m=nil [GC worker (idle)]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/proc.go:435 +0xc8 fp=0x14000074f10 sp=0x14000074ef0 pc=0x1004f42d8
runtime.gcBgMarkWorker(0x14000124310)
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/mgc.go:1423 +0xdc fp=0x14000074fb0 sp=0x14000074f10 pc=0x1004a277c
runtime.gcBgMarkStartWorkers.gowrap1()
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/mgc.go:1339 +0x28 fp=0x14000074fd0 sp=0x14000074fb0 pc=0x1004a2668
runtime.goexit({})
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/asm_arm64.s:1223 +0x4 fp=0x14000074fd0 sp=0x14000074fd0 pc=0x1004fbff4
created by runtime.gcBgMarkStartWorkers in goroutine 7
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/mgc.go:1339 +0x140

goroutine 10 gp=0x1400016e000 m=nil [GC worker (idle)]:
runtime.gopark(0x5d29bb79abff?, 0x0?, 0x0?, 0x0?, 0x0?)
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/proc.go:435 +0xc8 fp=0x14000075710 sp=0x140000756f0 pc=0x1004f42d8
runtime.gcBgMarkWorker(0x14000124310)
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/mgc.go:1423 +0xdc fp=0x140000757b0 sp=0x14000075710 pc=0x1004a277c
runtime.gcBgMarkStartWorkers.gowrap1()
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/mgc.go:1339 +0x28 fp=0x140000757d0 sp=0x140000757b0 pc=0x1004a2668
runtime.goexit({})
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/asm_arm64.s:1223 +0x4 fp=0x140000757d0 sp=0x140000757d0 pc=0x1004fbff4
created by runtime.gcBgMarkStartWorkers in goroutine 7
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/mgc.go:1339 +0x140

goroutine 11 gp=0x1400016e1c0 m=nil [GC worker (idle)]:
runtime.gopark(0x5d29bb79af17?, 0x0?, 0x0?, 0x0?, 0x0?)
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/proc.go:435 +0xc8 fp=0x14000075f10 sp=0x14000075ef0 pc=0x1004f42d8
runtime.gcBgMarkWorker(0x14000124310)
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/mgc.go:1423 +0xdc fp=0x14000075fb0 sp=0x14000075f10 pc=0x1004a277c
runtime.gcBgMarkStartWorkers.gowrap1()
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/mgc.go:1339 +0x28 fp=0x14000075fd0 sp=0x14000075fb0 pc=0x1004a2668
runtime.goexit({})
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/asm_arm64.s:1223 +0x4 fp=0x14000075fd0 sp=0x14000075fd0 pc=0x1004fbff4
created by runtime.gcBgMarkStartWorkers in goroutine 7
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/mgc.go:1339 +0x140

goroutine 12 gp=0x1400016e380 m=nil [GC worker (idle)]:
runtime.gopark(0x5d2c324b7d8d?, 0x0?, 0x0?, 0x0?, 0x0?)
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/proc.go:435 +0xc8 fp=0x1400023c710 sp=0x1400023c6f0 pc=0x1004f42d8
runtime.gcBgMarkWorker(0x14000124310)
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/mgc.go:1423 +0xdc fp=0x1400023c7b0 sp=0x1400023c710 pc=0x1004a277c
runtime.gcBgMarkStartWorkers.gowrap1()
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/mgc.go:1339 +0x28 fp=0x1400023c7d0 sp=0x1400023c7b0 pc=0x1004a2668
runtime.goexit({})
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/asm_arm64.s:1223 +0x4 fp=0x1400023c7d0 sp=0x1400023c7d0 pc=0x1004fbff4
created by runtime.gcBgMarkStartWorkers in goroutine 7
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/mgc.go:1339 +0x140

goroutine 13 gp=0x1400016e540 m=nil [GC worker (idle)]:
runtime.gopark(0x5d2c324b81f2?, 0x0?, 0x0?, 0x0?, 0x0?)
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/proc.go:435 +0xc8 fp=0x1400023cf10 sp=0x1400023cef0 pc=0x1004f42d8
runtime.gcBgMarkWorker(0x14000124310)
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/mgc.go:1423 +0xdc fp=0x1400023cfb0 sp=0x1400023cf10 pc=0x1004a277c
runtime.gcBgMarkStartWorkers.gowrap1()
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/mgc.go:1339 +0x28 fp=0x1400023cfd0 sp=0x1400023cfb0 pc=0x1004a2668
runtime.goexit({})
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/asm_arm64.s:1223 +0x4 fp=0x1400023cfd0 sp=0x1400023cfd0 pc=0x1004fbff4
created by runtime.gcBgMarkStartWorkers in goroutine 7
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/mgc.go:1339 +0x140

goroutine 14 gp=0x1400016e700 m=nil [GC worker (idle)]:
runtime.gopark(0x5d2c324b87f8?, 0x3?, 0xa2?, 0x55?, 0x0?)
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/proc.go:435 +0xc8 fp=0x1400023d710 sp=0x1400023d6f0 pc=0x1004f42d8
runtime.gcBgMarkWorker(0x14000124310)
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/mgc.go:1423 +0xdc fp=0x1400023d7b0 sp=0x1400023d710 pc=0x1004a277c
runtime.gcBgMarkStartWorkers.gowrap1()
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/mgc.go:1339 +0x28 fp=0x1400023d7d0 sp=0x1400023d7b0 pc=0x1004a2668
runtime.goexit({})
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/asm_arm64.s:1223 +0x4 fp=0x1400023d7d0 sp=0x1400023d7d0 pc=0x1004fbff4
created by runtime.gcBgMarkStartWorkers in goroutine 7
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/mgc.go:1339 +0x140

goroutine 15 gp=0x1400016e8c0 m=nil [GC worker (idle)]:
runtime.gopark(0x5d2c324bc290?, 0x0?, 0x0?, 0x0?, 0x0?)
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/proc.go:435 +0xc8 fp=0x1400023df10 sp=0x1400023def0 pc=0x1004f42d8
runtime.gcBgMarkWorker(0x14000124310)
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/mgc.go:1423 +0xdc fp=0x1400023dfb0 sp=0x1400023df10 pc=0x1004a277c
runtime.gcBgMarkStartWorkers.gowrap1()
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/mgc.go:1339 +0x28 fp=0x1400023dfd0 sp=0x1400023dfb0 pc=0x1004a2668
runtime.goexit({})
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/asm_arm64.s:1223 +0x4 fp=0x1400023dfd0 sp=0x1400023dfd0 pc=0x1004fbff4
created by runtime.gcBgMarkStartWorkers in goroutine 7
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/mgc.go:1339 +0x140

goroutine 50 gp=0x14000502a80 m=nil [runnable]:
bytes.growSlice.func1()
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/bytes/buffer.go:230 +0x60 fp=0x140001352d0 sp=0x140001352d0 pc=0x100596640
bytes.growSlice({0x1400077c000?, 0x29c620?, 0x14000130a10?}, 0x100d5e0f8?)
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/bytes/buffer.go:251 +0x110 fp=0x14000135370 sp=0x140001352d0 pc=0x100592fa0
bytes.(*Buffer).grow(0x14000146608, 0x1da28b)
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/bytes/buffer.go:151 +0x12c fp=0x140001353b0 sp=0x14000135370 pc=0x10059296c
bytes.(*Buffer).Write(0x14000146608, {0x1580a8000, 0x1da28b, 0x1da28b?})
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/bytes/buffer.go:179 +0x5c fp=0x140001353e0 sp=0x140001353b0 pc=0x100592b5c
github.com/apache/arrow-go/v18/arrow/ipc.(*Payload).SerializeBody(0x140000f8000?, {0x100d597a0, 0x14000146608})
	/Users/filip/go/pkg/mod/github.com/apache/arrow-go/v18@v18.2.0/arrow/ipc/file_writer.go:192 +0x90 fp=0x14000135450 sp=0x140001353e0 pc=0x100851ed0
github.com/apache/arrow-go/v18/arrow/flight.(*flightPayloadWriter).WritePayload(0x14000146580, {0x3, 0x140000d4b40, {0x140000f6000, 0x16, 0x20}, 0xc446f0})
	/Users/filip/go/pkg/mod/github.com/apache/arrow-go/v18@v18.2.0/arrow/flight/record_batch_writer.go:47 +0x114 fp=0x140001354b0 sp=0x14000135450 pc=0x100a62024
github.com/apache/arrow-go/v18/arrow/ipc.(*Writer).Write(0x14000208000, {0x100d65fd0, 0x1400010a930})
	/Users/filip/go/pkg/mod/github.com/apache/arrow-go/v18@v18.2.0/arrow/ipc/writer.go:204 +0x3cc fp=0x140001355c0 sp=0x140001354b0 pc=0x100858aec
github.com/apache/arrow-go/v18/arrow/flight.(*Writer).Write(0x14000050180?, {0x100d65fd0?, 0x1400010a930?})
	/Users/filip/go/pkg/mod/github.com/apache/arrow-go/v18@v18.2.0/arrow/flight/record_batch_writer.go:83 +0x64 fp=0x14000135620 sp=0x140001355c0 pc=0x100a621f4
main.(*server).DoGet(0x101284750?, 0x100d0f240?, {0x100d63e90, 0x140000321a0})
	/Users/filip/code/go/goadbcpgrepro/main.go:69 +0x538 fp=0x140001359a0 sp=0x14000135620 pc=0x100a64f98
github.com/apache/arrow-go/v18/arrow/flight/gen/flight._FlightService_DoGet_Handler({0x100d0f240, 0x1400019ea20}, {0x100d62518, 0x140001201c0})
	/Users/filip/go/pkg/mod/github.com/apache/arrow-go/v18@v18.2.0/arrow/flight/gen/flight/Flight_grpc.pb.go:604 +0x11c fp=0x140001359e0 sp=0x140001359a0 pc=0x100a546fc
github.com/apache/arrow-go/v18/arrow/flight.serverAuthStreamInterceptor({0x100d0f240, 0x1400019ea20}, {0x100d62518, 0x140001201c0}, 0x1400000e1e0, 0x100d54540)
	/Users/filip/go/pkg/mod/github.com/apache/arrow-go/v18@v18.2.0/arrow/flight/server_auth.go:125 +0xfc fp=0x14000135a70 sp=0x140001359e0 pc=0x100a6373c
google.golang.org/grpc.(*Server).processStreamingRPC(0x14000246200, {0x100d5f258, 0x1400010a330}, 0x1400010c120, 0x14000253380, 0x101290960, 0x0)
	/Users/filip/go/pkg/mod/google.golang.org/grpc@v1.71.0/server.go:1702 +0xe98 fp=0x14000135d80 sp=0x14000135a70 pc=0x100a35398
google.golang.org/grpc.(*Server).handleStream(0x14000246200, {0x100d5f6f0, 0x14000126000}, 0x1400010c120)
	/Users/filip/go/pkg/mod/google.golang.org/grpc@v1.71.0/server.go:1819 +0x8d4 fp=0x14000135f60 sp=0x14000135d80 pc=0x100a36934
google.golang.org/grpc.(*Server).serveStreams.func2.1()
	/Users/filip/go/pkg/mod/google.golang.org/grpc@v1.71.0/server.go:1035 +0x84 fp=0x14000135fd0 sp=0x14000135f60 pc=0x100a30844
runtime.goexit({})
	/opt/homebrew/Cellar/go/1.24.1/libexec/src/runtime/asm_arm64.s:1223 +0x4 fp=0x14000135fd0 sp=0x14000135fd0 pc=0x1004fbff4
created by google.golang.org/grpc.(*Server).serveStreams.func2 in goroutine 6
	/Users/filip/go/pkg/mod/google.golang.org/grpc@v1.71.0/server.go:1046 +0x138

r0      0x4000000019
r1      0x1729aee88
r2      0x0
r3      0x1298082f8
r4      0x360
r5      0x14000072000
r6      0x0
r7      0x100d63500
r8      0x128ef87d0
r9      0x80000000
r10     0x0
r11     0x0
r12     0x0
r13     0x1729aef00
r14     0x0
r15     0x0
r16     0x18ec22a10
r17     0x1729aed70
r18     0x0
r19     0x4000000019
r20     0x600000e755c0
r21     0x3
r22     0x0
r23     0x0
r24     0x0
r25     0x140000532b8
r26     0x100d54510
r27     0x810
r28     0x14000186e00
r29     0x1729aee30
lr      0x128a67630
sp      0x1729aee00
pc      0x128b09d3c
fault   0x40000000b1

How can we reproduce the bug?

See this repo for full reproduction: https://github.com/fwojciec/goadbcpgrepro

Instructions for creating the data in a PostgreSQL instance (I used both an GCP AlloyDB instance and a local Postgres v14:

test=# SELECT version();
                                                           version                                                           
-----------------------------------------------------------------------------------------------------------------------------
 PostgreSQL 14.8 (Debian 14.8-1.pgdg110+1) on x86_64-pc-linux-gnu, compiled by gcc (Debian 10.2.1-6) 10.2.1 20210110, 64-bit
(1 row)

CREATE TABLE synthetic_filter_option (
  group_id    INTEGER,
  org_id      INTEGER,
  user_id     INTEGER,
  name        TEXT,
  code_type   TEXT,
  code        TEXT,
  description TEXT,
  code_id     INTEGER,
  archived    BOOLEAN
);

CREATE UNIQUE INDEX synthetic_fogmat ON synthetic_filter_option (org_id, group_id, code_id);

INSERT INTO synthetic_filter_option
SELECT
    (g % 100) AS group_id,                    -- cycles 0 to 99
    (g % 10) AS org_id,                       -- cycles 0 to 9
    (g % 1000) AS user_id,                    -- cycles 0 to 999
    'Group ' || g AS name,
    CASE WHEN g % 2 = 0 THEN 'TypeA' ELSE 'TypeB' END AS code_type,
    'Code' || g AS code,
    'Description ' || g AS description,
    g AS code_id,                             -- unique id per row
    (g % 2 = 0) AS archived                   -- even numbers true, odd false
FROM generate_series(1, 365712) AS s(g);

Then I compile the binary:

go build main.go

and run it:

POSTGRESQL_DSN=postgres://test:test@localhost:5432/test ./main

I'm using Python/ipython to request the data from the Flightserver that runs on grpc://0.0.0.0:8816:

import pyarrow.flight as fl
client = fl.connect("grpc://0.0.0.0:8816")
ticket = fl.Ticket(fl.FlightDescriptor.for_command(b"").command)

and then this until it segfaults:

reader = client.do_get(ticket)
table = reader.read_all()

For me it always fails when running reader = client.do_get(ticket).

Environment/Setup

My go.mod file from the example above:

module github.com/fwojciec/goadbcpgrepro

go 1.24.1

require (
	github.com/apache/arrow-adbc/go/adbc v1.5.0
	github.com/apache/arrow-go/v18 v18.2.0
)

require (
	github.com/goccy/go-json v0.10.5 // indirect
	github.com/google/flatbuffers v25.2.10+incompatible // indirect
	github.com/klauspost/compress v1.18.0 // indirect
	github.com/klauspost/cpuid/v2 v2.2.10 // indirect
	github.com/pierrec/lz4/v4 v4.1.22 // indirect
	github.com/zeebo/xxh3 v1.0.2 // indirect
	golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 // indirect
	golang.org/x/mod v0.24.0 // indirect
	golang.org/x/net v0.37.0 // indirect
	golang.org/x/sync v0.12.0 // indirect
	golang.org/x/sys v0.31.0 // indirect
	golang.org/x/text v0.23.0 // indirect
	golang.org/x/tools v0.31.0 // indirect
	golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
	google.golang.org/genproto/googleapis/rpc v0.0.0-20250219182151-9fdb1cabc7b2 // indirect
	google.golang.org/grpc v1.71.0 // indirect
	google.golang.org/protobuf v1.36.5 // indirect
)

MacOS: 15.3.2 (24D81)
Running on M1 Macbook Pro (so Apple Silicon)

using the libadbc_driver_postgresql.dylib renamed from libadbc_driver_postgresql.so - originally sourced by downloading the adbc_driver_postgresql-1.5.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl file from the ADBC driver Github release page.

[lidavidm]

Just to be clear: you are using the driver from

adbc_driver_postgresql-1.5.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl

This is the Linux AArch64 wheel, not the macOS one - is that intentional?

[lidavidm]

Or was that a typo? Comparing the sha256sums, it seems what you actually have is from
adbc_driver_postgresql-1.5.0-py3-none-macosx_11_0_arm64.whl, which would make more sense.

[lidavidm]

The native traceback:

* thread #4, stop reason = EXC_BAD_ACCESS (code=1, address=0x656d616e707974)
  * frame #0: 0x00006d616e707974
    frame #1: 0x00000001084cf5d0 libadbc_driver_postgresql.dylib`adbcpq::TupleReader::Release() + 60
    frame #2: 0x00000001084cf978 libadbc_driver_postgresql.dylib`adbcpq::TupleReader::ReleaseTrampoline(ArrowArrayStream*) + 84
    frame #3: 0x000000010062897c goadbcpgrepro`(anonymous namespace)::ErrorArrayStreamRelease(ArrowArrayStream*) + 56
    frame #4: 0x0000000100632d74 goadbcpgrepro`_cgo_ffc147652276_Cfunc_ArrowArrayStreamRelease + 32

@zeroshade and I were talking about this earlier and I agree with him that it looks like it's because of the finalizer (which runs asynchronously) combined with the ADBC resources being released eagerly. I think there are a few things we could consider:

• Structure the ADBC driver to keep resources alive more strongly (e.g. having the reader keep its parents alive, or at least having a weakptr) - I worry this will lead to leaking resources, though. But on the other hand having the lifetime of the stream mysteriously tied to other objects is not ideal.
• Actually implement Release/Retain on the CGo reader so that resources are not released asynchronously - this might surprise users who were accidentally depending on the prior behavior, though
• Eagerly release the ArrowArrayStream once you drain a reader, and cancel the finalizer/make the finalizer no-op in that case

BTW, not that this is related to the problem, but you probably want return reader.Err() and not return nil at the end of the RPC handler.

[paleolimbot]

I was worried this was about the fact that nanoarrow does not check alignment since that's come up with Flight + Rust before. I can prematurely add an "ensure aligned" anyway since we really should have that.

Structure the ADBC driver to keep resources alive more strongly

It's on the client side, but #2274 has a mechanism for checking the full heiarchy of reader/statement/connection/database/driver so that this "can't" happen. I tend to think it's the client's responsibility to maintain this validity (which is "guaranteed" by the C spec, so even if our own drivers provide some extra safety there's no guarantee an arbitrary driver will do the same).

The last time I thought about doing this from the driver side I thought that maybe the reader (e.g.) should have a std::shared_ptr<ParentToken<AdbcStatement>> and the statement should have a std::vector<std::shared_ptr<ParentToken<AdbcStatement>>. Then on finalize it could set all the child token references to NULL (giving the readers a mechanism to check for a valid parent before doing something that might crash). Perhaps this is what a weak_ref is doing as well 🙂

[zeroshade]

@lidavidm is there a reason why the stream from the PG driver is relying on driver/statement/connection resources for its release? Is it possible that we could ensure it doesn't rely on those resources and is fully independent or otherwise keeps what it needs alive?

[fwojciec]

Or was that a typo? Comparing the sha256sums, it seems what you actually have is from adbc_driver_postgresql-1.5.0-py3-none-macosx_11_0_arm64.whl, which would make more sense.

oh yeah, apologies - copied the wrong filename from my Makefile. I was using the lib file from adbc_driver_postgresql-1.5.0-py3-none-macosx_11_0_arm64 file when testing.

BTW, not that this is related to the problem, but you probably want return reader.Err() and not return nil at the end of the RPC handler.

good catch, of course 🤦

[lidavidm]

@lidavidm is there a reason why the stream from the PG driver is relying on driver/statement/connection resources for its release? Is it possible that we could ensure it doesn't rely on those resources and is fully independent or otherwise keeps what it needs alive?

Mostly because "the resource it needs alive" is the Postgres connection, so then the problem becomes if you close everything but the stream, does it actually close anything? Right now the answer is yes (and maybe we should be checking that you don't have any dangling streams), but maybe the answer should be no (we error, or silently keep resources alive)

[lidavidm]

Regardless, the Go reader not having a way to ensure deterministic cleanup is also not ideal IMO. Maybe at least we can ensure that if you drain and/or release the reader, we release the resources then and not in the finalizer (which we can keep as a backup)?

[lidavidm]

And I think for the Postgres driver, maybe the stream's lifetime can be made independent, but if you close the statement etc. it invalidates the stream (reading from it will error, but closing it will work) - as a sort of compromise?

[fwojciec]

out of curiosity, why would this be still segfaulting? I'm creating a slice of records in a separate scope, retaining the individual records, and then closing all DB-related resources, before passing a slice of retained records back to the DoGet handler - am I completely misunderstanding how it's supposed to work?

also, for what it's worth, not closing the Statement in getRecords "solves" it.

package main

import (
	"context"
	"fmt"
	"os"

	"github.com/apache/arrow-adbc/go/adbc"
	"github.com/apache/arrow-adbc/go/adbc/drivermgr"
	"github.com/apache/arrow-go/v18/arrow"
	"github.com/apache/arrow-go/v18/arrow/flight"
	"github.com/apache/arrow-go/v18/arrow/ipc"
)

func main() {
	if err := runServer(); err != nil {
		panic(err)
	}
}

type server struct {
	flight.BaseFlightServer
}

func getRecords(ctx context.Context) ([]arrow.Record, *arrow.Schema, error) {
	pgDsn := os.Getenv("POSTGRESQL_DSN")
	if pgDsn == "" {
		return nil, nil, fmt.Errorf("POSTGRESQL_DSN is not set")
	}

	var drv drivermgr.Driver
	db, err := drv.NewDatabase(map[string]string{
		"driver":          "adbc_driver_postgresql",
		adbc.OptionKeyURI: pgDsn,
	})
	if err != nil {
		return nil, nil, err
	}
	defer db.Close()

	conn, err := db.Open(ctx)
	if err != nil {
		return nil, nil, err
	}
	defer conn.Close()

	st, err := conn.NewStatement()
	if err != nil {
		return nil, nil, err
	}
	defer st.Close() // if I comment this out I seem to prevent segfaults

	if err := st.SetSqlQuery("SELECT * FROM synthetic_filter_option"); err != nil {
		return nil, nil, err
	}

	reader, _, err := st.ExecuteQuery(ctx)
	if err != nil {
		return nil, nil, err
	}
	defer reader.Release()

	var records []arrow.Record
	for reader.Next() {
		rec := reader.Record()
		rec.Retain()
		records = append(records, rec)
	}
	if err := reader.Err(); err != nil {
		return nil, nil, err
	}

	return records, reader.Schema(), nil
}

func (s *server) DoGet(tkt *flight.Ticket, fs flight.FlightService_DoGetServer) error {
	ctx := fs.Context()

	records, schema, err := getRecords(ctx)
	if err != nil {
		return err
	}

	wr := flight.NewRecordWriter(
		fs,
		ipc.WithSchema(schema),
	)
	defer wr.Close()

	for _, rec := range records {
		if err := wr.Write(rec); err != nil {
			return err
		}
                // need to release records here?
	}

	return nil
}

func runServer() error {
	s := flight.NewServerWithMiddleware(nil)
	s.Init("0.0.0.0:8816")
	s.RegisterFlightService(&server{})

	return s.Serve()
}

[lidavidm]

You are doing things right. What's going on is the following:

• Database, Connection, Statement, and Reader all wrap C handles
• Reader implicitly depends on Statement (we should fix this)
• On the Go side, Reader is actually closed nondeterministically via a finalizer
• You are correctly cleaning everything up
• However on the Go side when the finalizer runs to clean up the Reader it finds that everything else has been cleaned up already and the C code crashes

What I think we should do is (1) make the Reader less dependent on Statement (at least enough that you can still close it safely, while making everything else error) (2) consider making the Go side clean things up deterministically

Dewey mentioned (3) consider having the driver or the driver manager warn/error if you close things in the wrong order, but I think you aren't doing that. So this is a bug on our side :)

[acuitymd-filip]

Makes sense! Thanks for the detailed explanation!