Skip to content

Commit 4e7b9bb

Browse files
committed
Merge remote-tracking branch 'upstream/develop' into test/migrate-patch-collection-test-action
2 parents 2b21d95 + 7b48741 commit 4e7b9bb

72 files changed

Lines changed: 1412 additions & 325 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

.github/workflows/test-coverage.yml

Lines changed: 37 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -336,18 +336,48 @@ jobs:
336336
coverage-artifact-name: "coverage_js"
337337
coverage-path: coverage.txt
338338

339+
# This job runs tests with secondary indexes automatically added to schemas,
340+
# to ensure query results are consistent regardless of whether indexes are present.
341+
test-coverage-secondary-index:
342+
name: Test coverage secondary index job
343+
344+
runs-on: runs-on=${{ github.run_id }}-${{ github.run_attempt }}-${{ strategy.job-index }}/\
345+
spot=pco/cpu=16+32/family=c6*+c7*/disk=large/extras=s3-cache
346+
347+
env:
348+
DEFRA_MULTIPLIERS: secondary-index
349+
350+
steps:
351+
- name: Enable RunsOn action
352+
uses: runs-on/action@v2
353+
with:
354+
metrics: cpu,network,memory,disk,io
355+
356+
- name: Checkout code into the directory
357+
uses: actions/checkout@v6
358+
359+
- name: Setup defradb
360+
uses: ./.github/composites/setup-defradb
361+
362+
- name: Test coverage & save coverage report in an artifact
363+
uses: ./.github/composites/test-coverage-with-artifact
364+
with:
365+
coverage-artifact-name: "coverage_secondary_index"
366+
coverage-path: coverage.txt
367+
339368
## This job gathers all the coverage reports and uploads them to code-cov
340369
upload-coverage:
341370
name: Upload test code coverage job
342371

343372
needs:
344-
- test-coverage-basic # 24 test(s)
345-
- test-coverage-document-acp # 3 test(s)
346-
- test-coverage-lens # 2 test(s)
347-
- test-coverage-view # 1 test(s)
348-
- test-coverage-encryption # 1 test(s)
349-
- test-coverage-telemetry # 1 test(s)
350-
- test-coverage-js # 1 test(s)
373+
- test-coverage-basic # 24 test(s)
374+
- test-coverage-document-acp # 3 test(s)
375+
- test-coverage-lens # 2 test(s)
376+
- test-coverage-view # 1 test(s)
377+
- test-coverage-encryption # 1 test(s)
378+
- test-coverage-telemetry # 1 test(s)
379+
- test-coverage-js # 1 test(s)
380+
- test-coverage-secondary-index # 1 test(s)
351381

352382
# Important to know:
353383
# - We didn't use `if: always()` here, so this job doesn't run if we manually canceled.

cli/test/integration/test.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import (
2121
"github.com/sourcenetwork/defradb/cli/test/action"
2222
_ "github.com/sourcenetwork/defradb/cli/test/multiplier"
2323
"github.com/sourcenetwork/defradb/cli/test/state"
24+
_ "github.com/sourcenetwork/defradb/tests/multiplier"
2425
)
2526

2627
func init() {
@@ -47,12 +48,12 @@ func (test *Test) Execute(t testing.TB) {
4748
ctx, cancel := context.WithTimeout(ctx, 1*time.Second)
4849
defer cancel()
4950

50-
multiplier.Skip(t, test.Includes, test.Excludes)
51-
5251
// Prepend a start action if there is not already one present, this saves each test from
5352
// having to redeclare the same initial action.
5453
actions := prependStart(test.Actions)
5554

55+
multiplier.Skip(t, actions, test.Includes, test.Excludes)
56+
5657
actions = multiplier.Apply(actions)
5758

5859
testo.Log(t, actions)

go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ require (
5555
github.com/sourcenetwork/immutable v0.3.0
5656
github.com/sourcenetwork/lens/host-go v0.9.4
5757
github.com/sourcenetwork/sourcehub v0.3.3-0.20251204195821-6e21a42984ba
58-
github.com/sourcenetwork/testo v0.1.0
58+
github.com/sourcenetwork/testo v0.2.0
5959
github.com/spf13/cobra v1.10.1
6060
github.com/spf13/pflag v1.0.10
6161
github.com/spf13/viper v1.20.1

go.sum

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2340,8 +2340,8 @@ github.com/sourcenetwork/raccoondb/v2 v2.0.0 h1:Gb0SjsZUbrbkHCEg7PyyNs0+2IzPxjE/
23402340
github.com/sourcenetwork/raccoondb/v2 v2.0.0/go.mod h1:tPKAWHUgYcSoXNMt01iWJ+Y51AQYc2knY93IjmmbejY=
23412341
github.com/sourcenetwork/sourcehub v0.3.3-0.20251204195821-6e21a42984ba h1:fyB1/gl0jLNBCyog3HF/PmLmWqDwvTrHrzrWUovtnEU=
23422342
github.com/sourcenetwork/sourcehub v0.3.3-0.20251204195821-6e21a42984ba/go.mod h1:GUxr7ahYovbOxaPelH0cKP2osr8gyjfeGnYG3IkhujE=
2343-
github.com/sourcenetwork/testo v0.1.0 h1:mGhNDMOu6qvBTodlfPwy9NH1X4pOSc9AQ6mbOdqbmIg=
2344-
github.com/sourcenetwork/testo v0.1.0/go.mod h1:CwCk0OPI7XLc0ypBS2QVoCtwOxyybv5x8gJxDkNCd14=
2343+
github.com/sourcenetwork/testo v0.2.0 h1:jy0e/VLuDKLErHbnsjWezzChffUkx5bto7B30d1vRN0=
2344+
github.com/sourcenetwork/testo v0.2.0/go.mod h1:CwCk0OPI7XLc0ypBS2QVoCtwOxyybv5x8gJxDkNCd14=
23452345
github.com/sourcenetwork/zanzi v0.3.1-0.20251104182341-4b1bd0f7be5a h1:TlnRmk1v4+nIKGUdzMht4qnrf8AxCUAvpxSIqJ2CX94=
23462346
github.com/sourcenetwork/zanzi v0.3.1-0.20251104182341-4b1bd0f7be5a/go.mod h1:j7QmuHK+zfA0JO6FliJzQHijVNSXDuQy8hUzFC5pucA=
23472347
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=

internal/db/collection_index.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,12 @@ func (c *collection) deleteIndexedDocWithID(
137137
if err != nil {
138138
return err
139139
}
140+
if doc == nil {
141+
// If the document cannot be fetched (e.g., due to ACP restrictions),
142+
// skip index deletion. The caller (Delete) will handle the authorization
143+
// error in applyDelete.
144+
return nil
145+
}
140146
return c.deleteIndexedDoc(ctx, doc)
141147
}
142148

tests/action/get_collections.go

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
// Copyright 2025 Democratized Data Foundation
2+
//
3+
// Use of this software is governed by the Business Source License
4+
// included in the file licenses/BSL.txt.
5+
//
6+
// As of the Change Date specified in that file, in accordance with
7+
// the Business Source License, use of this software will be governed
8+
// by the Apache License, Version 2.0, included in the file
9+
// licenses/APL.txt.
10+
11+
package action
12+
13+
import (
14+
"github.com/sourcenetwork/immutable"
15+
16+
"github.com/sourcenetwork/defradb/client"
17+
"github.com/sourcenetwork/defradb/internal/db"
18+
"github.com/sourcenetwork/defradb/tests/state"
19+
)
20+
21+
// GetCollections is an action that fetches collections using the provided options.
22+
//
23+
// ID, RootID and CollectionVersionID will only be asserted on if an expected value is provided.
24+
type GetCollections struct {
25+
stateful
26+
27+
// NodeID may hold the ID (index) of a node to get collections from.
28+
//
29+
// If a value is not provided collections will be gotten from all nodes.
30+
NodeID immutable.Option[int]
31+
32+
// Used to identify the transaction for this to run against. Optional.
33+
TransactionID immutable.Option[int]
34+
35+
// The identity of this request. Optional.
36+
//
37+
// If node acp is enabled, identity will be used to check if this operation can be performed.
38+
Identity immutable.Option[state.Identity]
39+
40+
// The expected results.
41+
//
42+
// Each item will be compared individually, if CollectionID, VersionID, or FieldIDs on the
43+
// expected item are default they will not be compared with the actual.
44+
//
45+
// Assertions on Indexes and Sources will not distinguish between nil and empty (in order
46+
// to allow their omission in most cases).
47+
ExpectedResults []client.CollectionVersion
48+
49+
// An optional set of fetch options for the collections.
50+
FilterOptions client.CollectionFetchOptions
51+
52+
// Any error expected from the action. Optional.
53+
//
54+
// String can be a partial, and the test will pass if an error is returned that
55+
// contains this string.
56+
ExpectedError string
57+
}
58+
59+
var _ Action = (*GetCollections)(nil)
60+
var _ Stateful = (*GetCollections)(nil)
61+
62+
// Execute executes the get collections action.
63+
func (a *GetCollections) Execute() {
64+
// Collect transform strings from expected results for lens ID replacement
65+
transformSet := []string{}
66+
for _, col := range a.ExpectedResults {
67+
if col.PreviousVersion.HasValue() && col.PreviousVersion.Value().Transform.HasValue() {
68+
transformSet = append(transformSet, col.PreviousVersion.Value().Transform.Value())
69+
}
70+
}
71+
72+
// The lens IDs are consistent across nodes, so we can patch once for all nodes.
73+
// This will need to change if patches want to replace more than just lens IDs.
74+
if len(transformSet) > 0 {
75+
transformMap := replaceMap(a.s, 0, transformSet)
76+
77+
for i, col := range a.ExpectedResults {
78+
if col.PreviousVersion.HasValue() && col.PreviousVersion.Value().Transform.HasValue() {
79+
a.ExpectedResults[i].PreviousVersion = immutable.Some(
80+
client.CollectionSource{
81+
SourceCollectionID: a.ExpectedResults[i].PreviousVersion.Value().SourceCollectionID,
82+
Transform: immutable.Some(transformMap[col.PreviousVersion.Value().Transform.Value()]),
83+
},
84+
)
85+
}
86+
}
87+
}
88+
89+
nodeIDs, nodes := getNodesWithIDs(a.NodeID, a.s.Nodes)
90+
for index, node := range nodes {
91+
nodeID := nodeIDs[index]
92+
txn, err := a.s.GetTransaction(node, a.TransactionID)
93+
if assertError(a.s.T, err, a.ExpectedError) {
94+
return
95+
}
96+
ctx := db.InitContext(a.s.Ctx, txn)
97+
ctx = getContextWithIdentity(ctx, a.s, a.Identity, nodeID)
98+
99+
results, err := node.GetCollections(ctx, a.FilterOptions)
100+
resultDescriptions := make([]client.CollectionVersion, len(results))
101+
for i, col := range results {
102+
resultDescriptions[i] = col.Version()
103+
}
104+
105+
expectedErrorRaised := assertError(a.s.T, err, a.ExpectedError)
106+
assertExpectedErrorRaised(a.s.T, a.ExpectedError, expectedErrorRaised)
107+
108+
if !expectedErrorRaised {
109+
assertCollectionVersions(a.s, a.ExpectedResults, resultDescriptions)
110+
}
111+
}
112+
}

tests/action/get_indexes.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,7 @@ func assertIndexesListsEqual(
109109
func assertIndexesEqual(expectedIndex, actualIndex client.IndexDescription, t require.TestingT) {
110110
assert.Equal(t, expectedIndex.Name, actualIndex.Name, "index name mismatch")
111111
assert.Equal(t, expectedIndex.ID, actualIndex.ID, "index id mismatch")
112+
assert.Equal(t, expectedIndex.Unique, actualIndex.Unique, "index unique mismatch")
112113

113114
toNames := func(fields []client.IndexedFieldDescription) []string {
114115
names := make([]string, len(fields))

tests/action/results.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,7 @@ func assertCollectionVersions(
8585
require.Equal(s.T, expected.IsBranchable, actual.IsBranchable)
8686
require.Equal(s.T, expected.IsActive, actual.IsActive)
8787

88-
if expected.Indexes != nil || len(actual.Indexes) != 0 {
88+
if expected.Indexes != nil {
8989
// Dont bother asserting this if the expected is nil and the actual is nil/empty.
9090
// This is to save each test action from having to bother declaring an empty slice (if there are no indexes)
9191
require.Equal(s.T, expected.Indexes, actual.Indexes)

tests/integration/acp/dac/relationship/doc_actor/add/with_delete_test.go

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@ import (
1919

2020
func TestACP_OwnerGivesDeleteAccessToAnotherActorTwice_ShowThatTheRelationshipAlreadyExists(t *testing.T) {
2121
test := testUtils.TestCase{
22-
2322
Actions: []any{
2423
testUtils.AddDACPolicy{
2524

@@ -148,7 +147,6 @@ resources:
148147

149148
func TestACP_OwnerGivesDeleteAccessToAnotherActor_OtherActorCanDelete(t *testing.T) {
150149
test := testUtils.TestCase{
151-
152150
Actions: []any{
153151
testUtils.AddDACPolicy{
154152

tests/integration/acp/nac/collection_get_by_id_test.go

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import (
1616
"github.com/sourcenetwork/immutable"
1717

1818
"github.com/sourcenetwork/defradb/client"
19+
"github.com/sourcenetwork/defradb/tests/action"
1920
testUtils "github.com/sourcenetwork/defradb/tests/integration"
2021
)
2122

@@ -30,7 +31,7 @@ func TestNAC_GatesCollectionGetByID_AuthorizedIdentity_AllowAccess(t *testing.T)
3031
},
3132

3233
// This should work as the identity is authorized.
33-
testUtils.GetCollections{
34+
&action.GetCollections{
3435
Identity: testUtils.ClientIdentity(1),
3536
FilterOptions: client.CollectionFetchOptions{
3637
CollectionID: immutable.Some("does not exist"),
@@ -54,7 +55,7 @@ func TestNAC_GatesCollectionGetByID_NoIdentity_NotAuthorizedError(t *testing.T)
5455
},
5556

5657
// We haven't authorized non-identities. So, this should error.
57-
testUtils.GetCollections{
58+
&action.GetCollections{
5859
Identity: testUtils.NoIdentity(),
5960
FilterOptions: client.CollectionFetchOptions{
6061
CollectionID: immutable.Some("does not exist"),
@@ -78,7 +79,7 @@ func TestNAC_GatesCollectionGetByID_WrongIdentity_NotAuthorizedError(t *testing.
7879
},
7980

8081
// Wrong user/identity will also not be authorized.
81-
testUtils.GetCollections{
82+
&action.GetCollections{
8283
Identity: testUtils.ClientIdentity(2),
8384
FilterOptions: client.CollectionFetchOptions{
8485
CollectionID: immutable.Some("does not exist"),

0 commit comments

Comments
 (0)