Skip to content

Commit cf58b14

Browse files
arp242arp242
committed
Use arrays for bulkinsert if PostgreSQL is used
So turns out you can avoid building long queries on bulk insert by using arrays: insert into measurements (sensor_id, value, captured_at) select * from unnest( $1::int[], -- array of sensor ids $2::float[], -- array of values $3::timestamptz[] -- array of timestamps ); (from https://boringsql.com/posts/good-bad-arrays/#bulk-loading-with-arrays) This is ... kind of obvious, but I never realized until now 😅 This is much nicer as it won't spam pg_stat_statements etc. with "different" queries. Most of this is in zdb: arp242/zdb@1be9d14
1 parent a9be759 commit cf58b14

19 files changed

+132
-72
lines changed

‎cron/browser_stat.go‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -41,10 +41,12 @@ func updateBrowserStats(ctx context.Context, hits []goatcounter.Hit) error {
4141
grouped[k] = v
4242
}
4343

44-
var (
45-
siteID = goatcounter.MustGetSite(ctx).ID
46-
ins = goatcounter.Tables.BrowserStats.Bulk(ctx)
47-
)
44+
ins, err := goatcounter.Tables.BrowserStats.Bulk(ctx)
45+
if err != nil {
46+
return err
47+
}
48+
49+
siteID := goatcounter.MustGetSite(ctx).ID
4850
for _, v := range grouped {
4951
if v.count > 0 {
5052
ins.Values(siteID, v.pathID, v.day, v.browserID, v.count)

‎cron/campaign_stat.go‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -40,10 +40,12 @@ func updateCampaignStats(ctx context.Context, hits []goatcounter.Hit) error {
4040
grouped[k] = v
4141
}
4242

43-
var (
44-
siteID = goatcounter.MustGetSite(ctx).ID
45-
ins = goatcounter.Tables.CampaignStats.Bulk(ctx)
46-
)
43+
ins, err := goatcounter.Tables.CampaignStats.Bulk(ctx)
44+
if err != nil {
45+
return err
46+
}
47+
48+
siteID := goatcounter.MustGetSite(ctx).ID
4749
for _, v := range grouped {
4850
if v.count > 0 {
4951
ins.Values(siteID, v.pathID, v.day, v.campaignID, v.ref, v.count)

‎cron/hit_count.go‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -37,10 +37,12 @@ func updateHitCounts(ctx context.Context, hits []goatcounter.Hit) error {
3737
grouped[k] = v
3838
}
3939

40-
var (
41-
siteID = goatcounter.MustGetSite(ctx).ID
42-
ins = goatcounter.Tables.HitCounts.Bulk(ctx)
43-
)
40+
ins, err := goatcounter.Tables.HitCounts.Bulk(ctx)
41+
if err != nil {
42+
return err
43+
}
44+
45+
siteID := goatcounter.MustGetSite(ctx).ID
4446
for _, v := range grouped {
4547
if v.total > 0 {
4648
ins.Values(siteID, v.pathID, v.hour, v.total)

‎cron/hit_stat.go‎

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -53,20 +53,29 @@ func updateHitStats(ctx context.Context, hits []goatcounter.Hit) error {
5353
grouped[k] = v
5454
}
5555

56-
var (
57-
siteID = goatcounter.MustGetSite(ctx).ID
58-
ins = goatcounter.Tables.HitStats.Bulk(ctx)
59-
)
56+
ins, err := goatcounter.Tables.HitStats.Bulk(ctx)
57+
if err != nil {
58+
return err
59+
}
60+
6061
if zdb.SQLDialect(ctx) == zdb.DialectSQLite {
6162
// TODO: merge the arrays here and get rid of existingHitStats();
6263
// it's kinda tricky with SQLite :-/
63-
ins = zdb.NewBulkInsert(ctx, "hit_stats", []string{"site_id", "path_id", "day", "stats"})
64+
ins, err = zdb.NewBulkInsert(ctx, "hit_stats", []string{"site_id", "path_id", "day", "stats"})
65+
if err != nil {
66+
return err
67+
}
6468
}
69+
siteID := goatcounter.MustGetSite(ctx).ID
6570
for _, v := range grouped {
6671
if slices.Equal(v.count, empty) {
6772
continue
6873
}
69-
ins.Values(siteID, v.pathID, v.day, zjson.MustMarshal(v.count))
74+
if zdb.SQLDialect(ctx) == zdb.DialectSQLite {
75+
ins.Values(siteID, v.pathID, v.day, zjson.MustMarshal(v.count))
76+
} else {
77+
ins.Values(siteID, v.pathID, v.day, string(zjson.MustMarshal(v.count)))
78+
}
7079
}
7180
return errors.Wrap(ins.Finish(), "updateHitStats hit_stats")
7281
})

‎cron/language_stat.go‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -39,10 +39,12 @@ func updateLanguageStats(ctx context.Context, hits []goatcounter.Hit) error {
3939
grouped[k] = v
4040
}
4141

42-
var (
43-
siteID = goatcounter.MustGetSite(ctx).ID
44-
ins = goatcounter.Tables.LanguageStats.Bulk(ctx)
45-
)
42+
ins, err := goatcounter.Tables.LanguageStats.Bulk(ctx)
43+
if err != nil {
44+
return err
45+
}
46+
47+
siteID := goatcounter.MustGetSite(ctx).ID
4648
for _, v := range grouped {
4749
if v.count > 0 {
4850
ins.Values(siteID, v.pathID, v.day, v.language, v.count)

‎cron/location_stat.go‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -44,10 +44,12 @@ func updateLocationStats(ctx context.Context, hits []goatcounter.Hit) error {
4444
grouped[k] = v
4545
}
4646

47-
var (
48-
siteID = goatcounter.MustGetSite(ctx).ID
49-
ins = goatcounter.Tables.LocationStats.Bulk(ctx)
50-
)
47+
ins, err := goatcounter.Tables.LocationStats.Bulk(ctx)
48+
if err != nil {
49+
return err
50+
}
51+
52+
siteID := goatcounter.MustGetSite(ctx).ID
5153
for _, v := range grouped {
5254
if v.count > 0 {
5355
ins.Values(siteID, v.pathID, v.day, v.location, v.count)

‎cron/ref_count.go‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -39,10 +39,12 @@ func updateRefCounts(ctx context.Context, hits []goatcounter.Hit) error {
3939
grouped[k] = v
4040
}
4141

42-
var (
43-
siteID = goatcounter.MustGetSite(ctx).ID
44-
ins = goatcounter.Tables.RefCounts.Bulk(ctx)
45-
)
42+
ins, err := goatcounter.Tables.RefCounts.Bulk(ctx)
43+
if err != nil {
44+
return err
45+
}
46+
47+
siteID := goatcounter.MustGetSite(ctx).ID
4648
for _, v := range grouped {
4749
if v.total > 0 {
4850
ins.Values(siteID, v.pathID, v.hour, v.refID, v.total)

‎cron/size_stat.go‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -42,10 +42,13 @@ func updateSizeStats(ctx context.Context, hits []goatcounter.Hit) error {
4242
}
4343
grouped[k] = v
4444
}
45-
var (
46-
siteID = goatcounter.MustGetSite(ctx).ID
47-
ins = goatcounter.Tables.SizeStats.Bulk(ctx)
48-
)
45+
46+
ins, err := goatcounter.Tables.SizeStats.Bulk(ctx)
47+
if err != nil {
48+
return err
49+
}
50+
51+
siteID := goatcounter.MustGetSite(ctx).ID
4952
for _, v := range grouped {
5053
if v.count > 0 {
5154
ins.Values(siteID, v.pathID, v.day, v.width, v.count)

‎cron/system_stat.go‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -41,10 +41,12 @@ func updateSystemStats(ctx context.Context, hits []goatcounter.Hit) error {
4141
grouped[k] = v
4242
}
4343

44-
var (
45-
siteID = goatcounter.MustGetSite(ctx).ID
46-
ins = goatcounter.Tables.SystemStats.Bulk(ctx)
47-
)
44+
ins, err := goatcounter.Tables.SystemStats.Bulk(ctx)
45+
if err != nil {
46+
return err
47+
}
48+
49+
siteID := goatcounter.MustGetSite(ctx).ID
4850
for _, v := range grouped {
4951
if v.count > 0 {
5052
ins.Values(siteID, v.pathID, v.day, v.systemID, v.count)

‎db/migrate/gomig/2022-11-15-1-correct-hit-stats.go‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,10 @@ func updateHitStats(ctx context.Context, hits []Hit) error {
100100
}
101101

102102
siteID := goatcounter.MustGetSite(ctx).ID
103-
ins := zdb.NewBulkInsert(ctx, "hit_stats", []string{"site_id", "day", "path_id", "stats"})
103+
ins, err := zdb.NewBulkInsert(ctx, "hit_stats", []string{"site_id", "day", "path_id", "stats"})
104+
if err != nil {
105+
return err
106+
}
104107
if zdb.SQLDialect(ctx) == zdb.DialectPostgreSQL {
105108
ins.OnConflict(`on conflict on constraint "hit_stats#site_id#path_id#day" do update set
106109
stats = (

0 commit comments

Comments
 (0)