Skip to content

Commit 0f23a08

Browse files
authored
Merge branch 'develop' into combined-bot-prs-branch
2 parents c4e4f24 + de03fa3 commit 0f23a08

88 files changed

Lines changed: 3038 additions & 81 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

.github/workflows/test-coverage.yml

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -388,6 +388,65 @@ jobs:
388388
coverage-artifact-name: "coverage_signed_docs"
389389
coverage-path: coverage.txt
390390

391+
# These jobs run the networked tests with one node on an older release, to catch
392+
# changes that break compatibility with it. Both directions are run because they
393+
# fail differently: old-source has the older node sending, new-source receiving.
394+
#
395+
# The release binary is a public download over HTTPS, so no token is needed.
396+
test-coverage-cross-version-old-source:
397+
name: Test coverage cross version old source job
398+
399+
runs-on: runs-on=${{ github.run_id }}-${{ github.run_attempt }}-${{ strategy.job-index }}/\
400+
spot=pco/cpu=16+32/family=c6*+c7*/disk=large/extras=s3-cache
401+
402+
env:
403+
DEFRA_MULTIPLIERS: cross-version-old-source
404+
405+
steps:
406+
- name: Enable RunsOn action
407+
uses: runs-on/action@v2
408+
with:
409+
metrics: cpu,network,memory,disk,io
410+
411+
- name: Checkout code into the directory
412+
uses: actions/checkout@v6
413+
414+
- name: Setup defradb
415+
uses: ./.github/composites/setup-defradb
416+
417+
- name: Test coverage & save coverage report in an artifact
418+
uses: ./.github/composites/test-coverage-with-artifact
419+
with:
420+
coverage-artifact-name: "coverage_cross_version_old_source"
421+
coverage-path: coverage.txt
422+
423+
test-coverage-cross-version-new-source:
424+
name: Test coverage cross version new source job
425+
426+
runs-on: runs-on=${{ github.run_id }}-${{ github.run_attempt }}-${{ strategy.job-index }}/\
427+
spot=pco/cpu=16+32/family=c6*+c7*/disk=large/extras=s3-cache
428+
429+
env:
430+
DEFRA_MULTIPLIERS: cross-version-new-source
431+
432+
steps:
433+
- name: Enable RunsOn action
434+
uses: runs-on/action@v2
435+
with:
436+
metrics: cpu,network,memory,disk,io
437+
438+
- name: Checkout code into the directory
439+
uses: actions/checkout@v6
440+
441+
- name: Setup defradb
442+
uses: ./.github/composites/setup-defradb
443+
444+
- name: Test coverage & save coverage report in an artifact
445+
uses: ./.github/composites/test-coverage-with-artifact
446+
with:
447+
coverage-artifact-name: "coverage_cross_version_new_source"
448+
coverage-path: coverage.txt
449+
391450
# This job tests the leveldb datastore.
392451
test-coverage-leveldb:
393452
name: Test coverage leveldb job
@@ -431,6 +490,8 @@ jobs:
431490
- test-coverage-js # 1 test(s)
432491
- test-coverage-secondary-index # 1 test(s)
433492
- test-coverage-signed-docs # 1 test(s)
493+
- test-coverage-cross-version-old-source # 1 test(s)
494+
- test-coverage-cross-version-new-source # 1 test(s)
434495
- test-coverage-leveldb # 1 test(s)
435496

436497
# Important to know:

README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ Building DefraDB from source requires significant system resources. If you encou
7373

7474
### Prerequisites
7575

76-
- [Go](https://golang.org/) 1.24 or later
76+
- [Go](https://golang.org/) 1.26 or later
7777
- [Rust toolchain](https://www.rust-lang.org/tools/install) (for WASM lens compilation, if running tests)
7878
- Git
7979

VERSIONING.md

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,12 @@ We as a team have decided that, whilst the exact strings are user-visible via th
2222

2323
We will not make significant changes to the structure of errors returned by any of our functions. For example, if a function returns an `client.ErrValueTypeMismatch` wrapped by an `client.ErrInvalidJSONPayload` - removing or replacing either one of these will only be done as part of a `MAJOR` version increment.
2424

25+
## Go version bumps
26+
27+
Raising the minimum supported Go version can prevent developers and downstream source consumers from building DefraDB with an older Go toolchain. Although this is technically a compatibility break for those environments, we do not treat Go version bumps as breaking changes under DefraDB's semver policy, and they do not require a `MAJOR` version increment.
28+
29+
Consumers building DefraDB from source are expected to use a supported Go version. We schedule these bumps according to our [Go Version Bumping Policy](./CONTRIBUTING_INTERNAL.md#-go-version-bumping-policy).
30+
2531
## The C embedded client
2632

2733
We made a mistake when designing many of the function signatures that form the [C client](./cbindings). Many of the functions have parameters that do not pair up with their Go equivalents - they take individual formal parameters, where the Go function takes an [options](./client/options) struct.

client/db.go

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -421,8 +421,63 @@ type GQLResult struct {
421421
//
422422
// It will be nil if any errors were raised during execution.
423423
Data any `json:"data"`
424+
425+
// Extensions holds extra information about the request, such as warnings.
426+
//
427+
// It is nil when there is nothing to report, and is then left out of the response.
428+
Extensions *GQLExtensions `json:"extensions,omitempty"`
429+
}
430+
431+
// GQLExtensions sits next to data and errors in a response. It holds anything we want
432+
// to tell the caller that is neither a result nor an error.
433+
//
434+
// Callers skip anything in here they do not recognise.
435+
type GQLExtensions struct {
436+
// Warnings holds things the caller should know about a request that worked.
437+
Warnings []GQLWarning `json:"warnings,omitempty"`
438+
}
439+
440+
// IsEmpty returns true if there is nothing to send. An empty value is left out of the
441+
// response rather than sent as `{}`.
442+
//
443+
// It turns the value into JSON and looks at the result, so a field added later is
444+
// covered without changing this. A value that cannot be turned into JSON counts as
445+
// empty, so a bad warning is dropped instead of breaking the whole response.
446+
func (e *GQLExtensions) IsEmpty() bool {
447+
if e == nil {
448+
return true
449+
}
450+
451+
data, err := json.Marshal(e)
452+
return err != nil || string(data) == "{}"
424453
}
425454

455+
// GQLWarning describes something that happened during a request that still worked.
456+
type GQLWarning struct {
457+
// Code names the warning. Callers check this. It does not change once released.
458+
Code string `json:"code"`
459+
460+
// Message explains the warning to a person. The wording can change, so do not
461+
// read it in code.
462+
Message string `json:"message"`
463+
464+
// Detail holds values that belong to this warning. Optional.
465+
//
466+
// It is sent to the caller and may be logged, so keep secrets and keys out of it.
467+
// Be careful with counts too: saying how many documents were looked at can tell
468+
// the caller about documents they are not allowed to see.
469+
Detail map[string]any `json:"detail,omitempty"`
470+
}
471+
472+
// Warning codes. Callers match on these, so they do not change once released.
473+
const (
474+
// WarningCodeVectorIndexUnused means a similarity query read the whole collection even though
475+
// the field it scored has a vector index. The results are correct, but the query costs more as
476+
// the collection grows. The `reason` detail says which part of the query shape ruled the index
477+
// out.
478+
WarningCodeVectorIndexUnused = "VECTOR_INDEX_UNUSED"
479+
)
480+
426481
// gqlError represents an error that was encountered during a GQL request.
427482
//
428483
// This is only used for marshalling to keep our responses spec compliant.
@@ -439,6 +494,8 @@ type gqlResult struct {
439494
Errors []gqlError `json:"errors,omitempty"`
440495
// Data contains the result data
441496
Data any `json:"data"`
497+
// Extensions contains the result extensions
498+
Extensions *GQLExtensions `json:"extensions,omitempty"`
442499
}
443500

444501
func (res *GQLResult) UnmarshalJSON(data []byte) error {
@@ -449,6 +506,12 @@ func (res *GQLResult) UnmarshalJSON(data []byte) error {
449506
return err
450507
}
451508
res.Data = out.Data
509+
res.Extensions = out.Extensions
510+
// A peer may send `"extensions":{}`, which decodes to a non nil empty value. Callers
511+
// are told the field is nil when there is nothing to report, so make that true.
512+
if res.Extensions.IsEmpty() {
513+
res.Extensions = nil
514+
}
452515
res.Errors = make([]error, len(out.Errors))
453516
for i, e := range out.Errors {
454517
res.Errors[i] = ReviveError(e.Message)
@@ -458,6 +521,9 @@ func (res *GQLResult) UnmarshalJSON(data []byte) error {
458521

459522
func (res GQLResult) MarshalJSON() ([]byte, error) {
460523
out := gqlResult{Data: res.Data}
524+
if !res.Extensions.IsEmpty() {
525+
out.Extensions = res.Extensions
526+
}
461527
out.Errors = make([]gqlError, len(res.Errors))
462528
for i, e := range res.Errors {
463529
out.Errors[i] = gqlError{Message: e.Error()}

client/db_test.go

Lines changed: 180 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,180 @@
1+
// Copyright 2026 Democratized Data Foundation
2+
//
3+
// Use of this software is governed by the Business Source License
4+
// included in the file licenses/BSL.txt.
5+
//
6+
// As of the Change Date specified in that file, in accordance with
7+
// the Business Source License, use of this software will be governed
8+
// by the Apache License, Version 2.0, included in the file
9+
// licenses/APL.txt.
10+
11+
package client
12+
13+
import (
14+
"encoding/json"
15+
"testing"
16+
17+
"github.com/stretchr/testify/require"
18+
)
19+
20+
// GQLResult does not use the default struct marshalling. It copies itself field by
21+
// field through a private mirror, so a field can exist on the type and still be missing
22+
// from the JSON. These tests catch that.
23+
//
24+
// It matters because the Go client never serializes anything. A half applied change
25+
// passes there and fails on every other client.
26+
27+
func TestGQLResultMarshal_WithWarning_RoundTrips(t *testing.T) {
28+
input := GQLResult{
29+
Data: map[string]any{"Users": []any{}},
30+
Extensions: &GQLExtensions{
31+
Warnings: []GQLWarning{
32+
{
33+
Code: "test_warning",
34+
Message: "something worth knowing happened",
35+
Detail: map[string]any{"requested": 10, "returned": 6},
36+
},
37+
},
38+
},
39+
}
40+
41+
data, err := json.Marshal(input)
42+
require.NoError(t, err)
43+
44+
var output GQLResult
45+
err = json.Unmarshal(data, &output)
46+
require.NoError(t, err)
47+
48+
require.NotNil(t, output.Extensions)
49+
require.Len(t, output.Extensions.Warnings, 1)
50+
51+
warning := output.Extensions.Warnings[0]
52+
require.Equal(t, "test_warning", warning.Code)
53+
require.Equal(t, "something worth knowing happened", warning.Message)
54+
55+
// UnmarshalJSON calls dec.UseNumber, so numbers come back as json.Number, not
56+
// float64. Same as everything under `data`.
57+
require.Equal(t, json.Number("10"), warning.Detail["requested"])
58+
require.Equal(t, json.Number("6"), warning.Detail["returned"])
59+
}
60+
61+
func TestGQLResultMarshal_WithMultipleWarnings_PreservesOrder(t *testing.T) {
62+
input := GQLResult{
63+
Extensions: &GQLExtensions{
64+
Warnings: []GQLWarning{
65+
{Code: "first", Message: "one"},
66+
{Code: "second", Message: "two"},
67+
},
68+
},
69+
}
70+
71+
data, err := json.Marshal(input)
72+
require.NoError(t, err)
73+
74+
var output GQLResult
75+
err = json.Unmarshal(data, &output)
76+
require.NoError(t, err)
77+
78+
require.Len(t, output.Extensions.Warnings, 2)
79+
require.Equal(t, "first", output.Extensions.Warnings[0].Code)
80+
require.Equal(t, "second", output.Extensions.Warnings[1].Code)
81+
}
82+
83+
func TestGQLResultMarshal_WithoutExtensions_OmitsField(t *testing.T) {
84+
input := GQLResult{Data: map[string]any{"Users": []any{}}}
85+
86+
data, err := json.Marshal(input)
87+
require.NoError(t, err)
88+
89+
var raw map[string]json.RawMessage
90+
err = json.Unmarshal(data, &raw)
91+
require.NoError(t, err)
92+
93+
require.NotContains(t, raw, "extensions")
94+
}
95+
96+
func TestGQLResultMarshal_WithEmptyExtensions_OmitsField(t *testing.T) {
97+
// An empty value must not be sent as `"extensions":{}`. Otherwise every response
98+
// changes shape as soon as anything allocates an accumulator.
99+
input := GQLResult{
100+
Data: map[string]any{"Users": []any{}},
101+
Extensions: &GQLExtensions{},
102+
}
103+
104+
data, err := json.Marshal(input)
105+
require.NoError(t, err)
106+
107+
var raw map[string]json.RawMessage
108+
err = json.Unmarshal(data, &raw)
109+
require.NoError(t, err)
110+
111+
require.NotContains(t, raw, "extensions")
112+
}
113+
114+
func TestGQLResultUnmarshal_WithUnknownExtensionKey_IsIgnored(t *testing.T) {
115+
// An older client must ignore an entry it does not know about instead of failing
116+
// the whole response.
117+
data := []byte(`{
118+
"data": null,
119+
"extensions": {
120+
"warnings": [{"code": "known", "message": "hi", "unknownField": 1}],
121+
"unknownKey": {"anything": true}
122+
}
123+
}`)
124+
125+
var output GQLResult
126+
err := json.Unmarshal(data, &output)
127+
require.NoError(t, err)
128+
129+
require.NotNil(t, output.Extensions)
130+
require.Len(t, output.Extensions.Warnings, 1)
131+
require.Equal(t, "known", output.Extensions.Warnings[0].Code)
132+
}
133+
134+
func TestGQLResultUnmarshal_WithoutExtensions_LeavesNil(t *testing.T) {
135+
var output GQLResult
136+
err := json.Unmarshal([]byte(`{"data": null}`), &output)
137+
require.NoError(t, err)
138+
139+
require.Nil(t, output.Extensions)
140+
require.True(t, output.Extensions.IsEmpty())
141+
}
142+
143+
// The empty non nil slice is the case a field by field check gets wrong: the slice is
144+
// not the zero value, but it still encodes to nothing.
145+
func TestGQLExtensionsIsEmpty_WithEmptyWarningSlice_IsEmpty(t *testing.T) {
146+
extensions := &GQLExtensions{Warnings: []GQLWarning{}}
147+
148+
require.True(t, extensions.IsEmpty())
149+
}
150+
151+
func TestGQLExtensionsIsEmpty_WithNilReceiver_IsEmpty(t *testing.T) {
152+
var extensions *GQLExtensions
153+
154+
require.True(t, extensions.IsEmpty())
155+
}
156+
157+
func TestGQLExtensionsIsEmpty_WithAWarning_IsNotEmpty(t *testing.T) {
158+
extensions := &GQLExtensions{Warnings: []GQLWarning{{Code: "test_warning"}}}
159+
160+
require.False(t, extensions.IsEmpty())
161+
}
162+
163+
// A peer may send an empty extensions object. Callers are told the field is nil when
164+
// there is nothing to report, so decoding must make that true rather than hand back a
165+
// non nil value with nothing in it.
166+
func TestGQLResultUnmarshal_WithEmptyExtensions_LeavesNil(t *testing.T) {
167+
var output GQLResult
168+
err := json.Unmarshal([]byte(`{"data": null, "extensions": {}}`), &output)
169+
require.NoError(t, err)
170+
171+
require.Nil(t, output.Extensions)
172+
}
173+
174+
func TestGQLResultUnmarshal_WithOnlyUnknownExtensionKeys_LeavesNil(t *testing.T) {
175+
var output GQLResult
176+
err := json.Unmarshal([]byte(`{"data": null, "extensions": {"unknownKey": 1}}`), &output)
177+
require.NoError(t, err)
178+
179+
require.Nil(t, output.Extensions)
180+
}

0 commit comments

Comments
 (0)