Skip to content

Commit 008c11f

Browse files
charlieviethcharlievieth
committed
reduce CGO calls when scanning rows
This commit improves the performance of queries by at least 20% by only collecting the information needed to convert sqlite3 values to Go values and by batching CGO calls when scanning rows (the performance improvement scales with the number of columns being scanned). This commit adds a new coltype field to the SQLiteRows struct which stores the declared column type (either data/time or boolean) and the sqlite3 datatype. Previously, this library would fetch the string representation of each column, which is inefficient and rarely needed since the non-standard SQLiteRows.DeclTypes method is rarely called. It also changes the benchmark suite to use an in-memory database since we do not want the file system interfering with benchmark results. goos: darwin goarch: arm64 pkg: github.com/mattn/go-sqlite3 cpu: Apple M1 Max │ base.10.txt │ new.10.txt │ │ sec/op │ sec/op vs base │ CustomFunctions-10 3.318µ ± 2% 3.115µ ± 2% -6.10% (p=0.000 n=10) Suite/BenchmarkExec-10 1.236µ ± 1% 1.240µ ± 2% ~ (p=0.617 n=10) Suite/BenchmarkQuery-10 4.004µ ± 7% 3.363µ ± 2% -16.02% (p=0.000 n=10) Suite/BenchmarkParams-10 4.241µ ± 1% 3.758µ ± 2% -11.40% (p=0.000 n=10) Suite/BenchmarkStmt-10 2.830µ ± 0% 2.378µ ± 2% -15.97% (p=0.000 n=10) Suite/BenchmarkRows-10 126.3µ ± 1% 101.3µ ± 1% -19.79% (p=0.000 n=10) Suite/BenchmarkStmtRows-10 124.9µ ± 1% 100.5µ ± 2% -19.56% (p=0.000 n=10) Suite/BenchmarkStmt10Cols-10 10.130µ ± 0% 7.042µ ± 1% -30.48% (p=0.000 n=10) geomean 8.655µ 7.328µ -15.33% │ base.10.txt │ new.10.txt │ │ B/op │ B/op vs base │ CustomFunctions-10 568.0 ± 0% 576.0 ± 0% +1.41% (p=0.000 n=10) Suite/BenchmarkExec-10 128.0 ± 0% 128.0 ± 0% ~ (p=1.000 n=10) ¹ Suite/BenchmarkQuery-10 688.0 ± 0% 648.0 ± 0% -5.81% (p=0.000 n=10) Suite/BenchmarkParams-10 1.078Ki ± 0% 1.031Ki ± 0% -4.35% (p=0.000 n=10) Suite/BenchmarkStmt-10 920.0 ± 0% 872.0 ± 0% -5.22% (p=0.000 n=10) Suite/BenchmarkRows-10 9.305Ki ± 0% 9.188Ki ± 0% -1.26% (p=0.000 n=10) Suite/BenchmarkStmtRows-10 9.289Ki ± 0% 9.164Ki ± 0% -1.35% (p=0.000 n=10) Suite/BenchmarkStmt10Cols-10 992.0 ± 0% 696.0 ± 0% -29.84% (p=0.000 n=10) geomean 1.181Ki 1.106Ki -6.35% ¹ all samples are equal │ base.10.txt │ new.10.txt │ │ allocs/op │ allocs/op vs base │ CustomFunctions-10 18.00 ± 0% 18.00 ± 0% ~ (p=1.000 n=10) ¹ Suite/BenchmarkExec-10 7.000 ± 0% 7.000 ± 0% ~ (p=1.000 n=10) ¹ Suite/BenchmarkQuery-10 23.00 ± 0% 23.00 ± 0% ~ (p=1.000 n=10) ¹ Suite/BenchmarkParams-10 27.00 ± 0% 27.00 ± 0% ~ (p=1.000 n=10) ¹ Suite/BenchmarkStmt-10 25.00 ± 0% 25.00 ± 0% ~ (p=1.000 n=10) ¹ Suite/BenchmarkRows-10 525.0 ± 0% 519.0 ± 0% -1.14% (p=0.000 n=10) Suite/BenchmarkStmtRows-10 524.0 ± 0% 518.0 ± 0% -1.15% (p=0.000 n=10) Suite/BenchmarkStmt10Cols-10 39.00 ± 0% 19.00 ± 0% -51.28% (p=0.000 n=10) geomean 46.26 42.17 -8.86% ¹ all samples are equal
1 parent 2d793aa commit 008c11f

File tree

2 files changed

+165
-29
lines changed

2 files changed

+165
-29
lines changed

‎sqlite3.go

+93-22
Original file line numberDiff line numberDiff line change
@@ -201,6 +201,53 @@ static int sqlite3_system_errno(sqlite3 *db) {
201201
return 0;
202202
}
203203
#endif
204+
205+
#define GO_SQLITE3_DECL_DATE (1 << 7)
206+
#define GO_SQLITE3_DECL_BOOL (1 << 6)
207+
#define GO_SQLITE3_DECL_MASK (GO_SQLITE3_DECL_DATE | GO_SQLITE3_DECL_BOOL)
208+
#define GO_SQLITE3_TYPE_MASK (GO_SQLITE3_DECL_BOOL - 1)
209+
210+
// _sqlite3_column_decltypes stores the declared column type in the typs array.
211+
// This function must always be called before _sqlite3_column_types since it
212+
// overwrites the datatype.
213+
static void _sqlite3_column_decltypes(sqlite3_stmt* stmt, uint8_t *typs, int ntyps) {
214+
for (int i = 0; i < ntyps; i++) {
215+
const char *typ = sqlite3_column_decltype(stmt, i);
216+
if (typ == NULL) {
217+
typs[i] = 0;
218+
continue;
219+
}
220+
switch (typ[0]) {
221+
case 'b':
222+
case 'B':
223+
if (!sqlite3_stricmp(typ, "boolean")) {
224+
typs[i] = GO_SQLITE3_DECL_BOOL;
225+
}
226+
break;
227+
case 'd':
228+
case 'D':
229+
if (!sqlite3_stricmp(typ, "date") || !sqlite3_stricmp(typ, "datetime")) {
230+
typs[i] = GO_SQLITE3_DECL_DATE;
231+
}
232+
break;
233+
case 't':
234+
case 'T':
235+
if (!sqlite3_stricmp(typ, "timestamp")) {
236+
typs[i] = GO_SQLITE3_DECL_DATE;
237+
}
238+
break;
239+
default:
240+
typs[i] = 0;
241+
}
242+
}
243+
}
244+
245+
static void _sqlite3_column_types(sqlite3_stmt *stmt, uint8_t *typs, int ntyps) {
246+
for (int i = 0; i < ntyps; i++) {
247+
typs[i] &= GO_SQLITE3_DECL_MASK; // clear lower bits
248+
typs[i] |= (uint8_t)sqlite3_column_type(stmt, i);
249+
}
250+
}
204251
*/
205252
import "C"
206253
import (
@@ -239,12 +286,6 @@ var SQLiteTimestampFormats = []string{
239286
"2006-01-02",
240287
}
241288

242-
const (
243-
columnDate string = "date"
244-
columnDatetime string = "datetime"
245-
columnTimestamp string = "timestamp"
246-
)
247-
248289
// This variable can be replaced with -ldflags like below:
249290
// go build -ldflags="-X 'github.com/mattn/go-sqlite3.driverName=my-sqlite3'"
250291
var driverName = "sqlite3"
@@ -390,13 +431,32 @@ type SQLiteResult struct {
390431
changes int64
391432
}
392433

434+
// A columnType is a compact representation of sqlite3 columns datatype and
435+
// declared type. The first two bits store the declared type and the remaining
436+
// six bits store the sqlite3 datatype.
437+
type columnType uint8
438+
439+
// declType returns the declared type, which is currently GO_SQLITE3_DECL_DATE
440+
// or GO_SQLITE3_DECL_BOOL, since those are the only two types that we need for
441+
// converting values.
442+
func (c columnType) declType() int {
443+
return int(c) & C.GO_SQLITE3_DECL_MASK
444+
}
445+
446+
// dataType returns the sqlite3 datatype code of the column, which is the
447+
// result of sqlite3_column_type.
448+
func (c columnType) dataType() int {
449+
return int(c) & C.GO_SQLITE3_TYPE_MASK
450+
}
451+
393452
// SQLiteRows implements driver.Rows.
394453
type SQLiteRows struct {
395454
s *SQLiteStmt
396455
nc int32 // Number of columns
397456
cls bool // True if we need to close the parent statement in Close
398457
cols []string
399458
decltype []string
459+
coltype []columnType
400460
ctx context.Context // no better alternative to pass context into Next() method
401461
closemu sync.Mutex
402462
}
@@ -2148,7 +2208,10 @@ func (rc *SQLiteRows) Columns() []string {
21482208
return rc.cols
21492209
}
21502210

2151-
func (rc *SQLiteRows) declTypes() []string {
2211+
// DeclTypes return column types.
2212+
func (rc *SQLiteRows) DeclTypes() []string {
2213+
rc.s.mu.Lock()
2214+
defer rc.s.mu.Unlock()
21522215
if rc.s.s != nil && rc.decltype == nil {
21532216
rc.decltype = make([]string, rc.nc)
21542217
for i := 0; i < int(rc.nc); i++ {
@@ -2158,13 +2221,6 @@ func (rc *SQLiteRows) declTypes() []string {
21582221
return rc.decltype
21592222
}
21602223

2161-
// DeclTypes return column types.
2162-
func (rc *SQLiteRows) DeclTypes() []string {
2163-
rc.s.mu.Lock()
2164-
defer rc.s.mu.Unlock()
2165-
return rc.declTypes()
2166-
}
2167-
21682224
// Next move cursor to next. Attempts to honor context timeout from QueryContext call.
21692225
func (rc *SQLiteRows) Next(dest []driver.Value) error {
21702226
rc.s.mu.Lock()
@@ -2197,6 +2253,13 @@ func (rc *SQLiteRows) Next(dest []driver.Value) error {
21972253
}
21982254
}
21992255

2256+
func (rc *SQLiteRows) colTypePtr() *C.uint8_t {
2257+
if len(rc.coltype) == 0 {
2258+
return nil
2259+
}
2260+
return (*C.uint8_t)(unsafe.Pointer(&rc.coltype[0]))
2261+
}
2262+
22002263
// nextSyncLocked moves cursor to next; must be called with locked mutex.
22012264
func (rc *SQLiteRows) nextSyncLocked(dest []driver.Value) error {
22022265
rv := C._sqlite3_step_internal(rc.s.s)
@@ -2210,15 +2273,24 @@ func (rc *SQLiteRows) nextSyncLocked(dest []driver.Value) error {
22102273
}
22112274
return nil
22122275
}
2276+
if len(dest) == 0 {
2277+
return nil
2278+
}
22132279

2214-
rc.declTypes()
2280+
if rc.coltype == nil {
2281+
rc.coltype = make([]columnType, rc.nc)
2282+
C._sqlite3_column_decltypes(rc.s.s, rc.colTypePtr(), C.int(rc.nc))
2283+
}
2284+
// Must call this each time since sqlite3 is loosely
2285+
// typed and the column types can vary between rows.
2286+
C._sqlite3_column_types(rc.s.s, rc.colTypePtr(), C.int(rc.nc))
22152287

22162288
for i := range dest {
2217-
switch C.sqlite3_column_type(rc.s.s, C.int(i)) {
2289+
switch rc.coltype[i].dataType() {
22182290
case C.SQLITE_INTEGER:
22192291
val := int64(C.sqlite3_column_int64(rc.s.s, C.int(i)))
2220-
switch rc.decltype[i] {
2221-
case columnTimestamp, columnDatetime, columnDate:
2292+
switch rc.coltype[i].declType() {
2293+
case C.GO_SQLITE3_DECL_DATE:
22222294
var t time.Time
22232295
// Assume a millisecond unix timestamp if it's 13 digits -- too
22242296
// large to be a reasonable timestamp in seconds.
@@ -2233,7 +2305,7 @@ func (rc *SQLiteRows) nextSyncLocked(dest []driver.Value) error {
22332305
t = t.In(rc.s.c.loc)
22342306
}
22352307
dest[i] = t
2236-
case "boolean":
2308+
case C.GO_SQLITE3_DECL_BOOL:
22372309
dest[i] = val > 0
22382310
default:
22392311
dest[i] = val
@@ -2257,8 +2329,7 @@ func (rc *SQLiteRows) nextSyncLocked(dest []driver.Value) error {
22572329
n := int(C.sqlite3_column_bytes(rc.s.s, C.int(i)))
22582330
s := C.GoStringN((*C.char)(unsafe.Pointer(C.sqlite3_column_text(rc.s.s, C.int(i)))), C.int(n))
22592331

2260-
switch rc.decltype[i] {
2261-
case columnTimestamp, columnDatetime, columnDate:
2332+
if rc.coltype[i].declType() == C.GO_SQLITE3_DECL_DATE {
22622333
var t time.Time
22632334
s = strings.TrimSuffix(s, "Z")
22642335
for _, format := range SQLiteTimestampFormats {
@@ -2275,7 +2346,7 @@ func (rc *SQLiteRows) nextSyncLocked(dest []driver.Value) error {
22752346
t = t.In(rc.s.c.loc)
22762347
}
22772348
dest[i] = t
2278-
default:
2349+
} else {
22792350
dest[i] = s
22802351
}
22812352
}

‎sqlite3_test.go

+72-7
Original file line numberDiff line numberDiff line change
@@ -2030,7 +2030,7 @@ func BenchmarkCustomFunctions(b *testing.B) {
20302030
}
20312031

20322032
func TestSuite(t *testing.T) {
2033-
initializeTestDB(t)
2033+
initializeTestDB(t, false)
20342034
defer freeTestDB()
20352035

20362036
for _, test := range tests {
@@ -2039,7 +2039,7 @@ func TestSuite(t *testing.T) {
20392039
}
20402040

20412041
func BenchmarkSuite(b *testing.B) {
2042-
initializeTestDB(b)
2042+
initializeTestDB(b, true)
20432043
defer freeTestDB()
20442044

20452045
for _, benchmark := range benchmarks {
@@ -2068,8 +2068,13 @@ type TestDB struct {
20682068

20692069
var db *TestDB
20702070

2071-
func initializeTestDB(t testing.TB) {
2072-
tempFilename := TempFilename(t)
2071+
func initializeTestDB(t testing.TB, memory bool) {
2072+
var tempFilename string
2073+
if memory {
2074+
tempFilename = ":memory:"
2075+
} else {
2076+
tempFilename = TempFilename(t)
2077+
}
20732078
d, err := sql.Open("sqlite3", tempFilename+"?_busy_timeout=99999")
20742079
if err != nil {
20752080
os.Remove(tempFilename)
@@ -2084,9 +2089,11 @@ func freeTestDB() {
20842089
if err != nil {
20852090
panic(err)
20862091
}
2087-
err = os.Remove(db.tempFilename)
2088-
if err != nil {
2089-
panic(err)
2092+
if db.tempFilename != "" && db.tempFilename != ":memory:" {
2093+
err := os.Remove(db.tempFilename)
2094+
if err != nil {
2095+
panic(err)
2096+
}
20902097
}
20912098
}
20922099

@@ -2112,6 +2119,7 @@ var benchmarks = []testing.InternalBenchmark{
21122119
{Name: "BenchmarkRows", F: benchmarkRows},
21132120
{Name: "BenchmarkStmtRows", F: benchmarkStmtRows},
21142121
{Name: "BenchmarkQueryParallel", F: benchmarkQueryParallel},
2122+
{Name: "BenchmarkStmt10Cols", F: benchmarkStmt10Cols},
21152123
}
21162124

21172125
func (db *TestDB) mustExec(sql string, args ...any) sql.Result {
@@ -2586,3 +2594,60 @@ func benchmarkQueryParallel(b *testing.B) {
25862594
}
25872595
})
25882596
}
2597+
2598+
func benchmarkStmt10Cols(b *testing.B) {
2599+
db.once.Do(makeBench)
2600+
2601+
const createTableStmt = `
2602+
DROP TABLE IF EXISTS bench_cols;
2603+
VACUUM;
2604+
CREATE TABLE bench_cols (
2605+
r0 INTEGER NOT NULL,
2606+
r1 INTEGER NOT NULL,
2607+
r2 INTEGER NOT NULL,
2608+
r3 INTEGER NOT NULL,
2609+
r4 INTEGER NOT NULL,
2610+
r5 INTEGER NOT NULL,
2611+
r6 INTEGER NOT NULL,
2612+
r7 INTEGER NOT NULL,
2613+
r8 INTEGER NOT NULL,
2614+
r9 INTEGER NOT NULL
2615+
);`
2616+
if _, err := db.Exec(createTableStmt); err != nil {
2617+
b.Fatal(err)
2618+
}
2619+
for i := int64(0); i < 4; i++ {
2620+
_, err := db.Exec("INSERT INTO bench_cols VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?);",
2621+
i, i, i, i, i, i, i, i, i, i)
2622+
if err != nil {
2623+
b.Fatal(err)
2624+
}
2625+
}
2626+
2627+
stmt, err := db.Prepare("SELECT * FROM bench_cols;")
2628+
if err != nil {
2629+
b.Fatal(err)
2630+
}
2631+
defer stmt.Close()
2632+
2633+
b.ResetTimer()
2634+
var (
2635+
v0, v1, v2, v3, v4 int64
2636+
v5, v6, v7, v8, v9 int64
2637+
)
2638+
for i := 0; i < b.N; i++ {
2639+
rows, err := stmt.Query()
2640+
if err != nil {
2641+
b.Fatal(err)
2642+
}
2643+
for rows.Next() {
2644+
err := rows.Scan(&v0, &v1, &v2, &v3, &v4, &v5, &v6, &v7, &v8, &v9)
2645+
if err != nil {
2646+
b.Fatal(err)
2647+
}
2648+
}
2649+
if err := rows.Err(); err != nil {
2650+
b.Fatal(err)
2651+
}
2652+
}
2653+
}

0 commit comments

Comments
 (0)