Skip to content

Commit 7915082

Browse files
WestwoooWestwooo
committed
ING-1088: Add durability impossible tests
1 parent f16cdaa commit 7915082

3 files changed

Lines changed: 218 additions & 0 deletions

File tree

‎.github/workflows/test.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,11 @@ jobs:
8484
SGTEST_CBCONNSTR: ${{ env.CBDC_IP }}
8585
SGTEST_DINOID: ${{ env.CBDC_ID }}
8686
run: go test -tags syncWriteAmbiguous ./gateway/test -run TestGatewayOps -v -testify.m TestSyncWriteAmbiguous
87+
- name: Run DurabilityImpossible Tests
88+
env:
89+
SGTEST_CBCONNSTR: ${{ env.CBDC_IP }}
90+
SGTEST_DINOID: ${{ env.CBDC_ID }}
91+
run: go test -tags durabilityImpossible ./gateway/test -run TestGatewayOps -v -testify.m TestDurabilityImpossible
8792

8893
- name: Collect couchbase logs
8994
timeout-minutes: 10

‎gateway/test/durability_impossible_test.go‎

Lines changed: 204 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,204 @@
1+
//go:build durabilityImpossible
2+
// +build durabilityImpossible
3+
4+
package test
5+
6+
import (
7+
"context"
8+
"time"
9+
10+
"github.com/couchbase/goprotostellar/genproto/kv_v1"
11+
"github.com/couchbase/stellar-gateway/testutils"
12+
"github.com/stretchr/testify/assert"
13+
"github.com/stretchr/testify/require"
14+
"google.golang.org/grpc"
15+
"google.golang.org/grpc/codes"
16+
)
17+
18+
// This test needs to remove all but one of the cluster nodes. Therefore this test will put the cluster in a bad state and
19+
// must be run at the end or against an isolated cluster.
20+
func (s *GatewayOpsTestSuite) TestDurabilityImpossible() {
21+
testutils.SkipIfNoDinoCluster(s.T())
22+
dino := testutils.StartDinoTesting(s.T(), false)
23+
24+
nodes := testutils.GetTestNodes(s.T())
25+
assert.Len(s.T(), nodes, 3)
26+
for _, node := range nodes {
27+
if !node.IsOrchestrator {
28+
dino.RemoveNode(node.Hostname)
29+
}
30+
}
31+
32+
kvClient := kv_v1.NewKvServiceClient(s.gatewayConn)
33+
34+
type testCase struct {
35+
name string
36+
fn func() error
37+
}
38+
39+
tests := []testCase{
40+
{
41+
name: "Insert",
42+
fn: func() error {
43+
docId := s.randomDocId()
44+
_, err := kvClient.Insert(context.Background(), &kv_v1.InsertRequest{
45+
BucketName: s.bucketName,
46+
ScopeName: s.scopeName,
47+
CollectionName: s.collectionName,
48+
Key: docId,
49+
Content: &kv_v1.InsertRequest_ContentUncompressed{
50+
ContentUncompressed: TEST_CONTENT,
51+
},
52+
ContentFlags: TEST_CONTENT_FLAGS,
53+
DurabilityLevel: kv_v1.DurabilityLevel_DURABILITY_LEVEL_MAJORITY.Enum(),
54+
}, grpc.PerRPCCredentials(s.basicRpcCreds))
55+
return err
56+
},
57+
},
58+
{
59+
name: "Upsert",
60+
fn: func() error {
61+
docId := s.randomDocId()
62+
_, err := kvClient.Upsert(context.Background(), &kv_v1.UpsertRequest{
63+
BucketName: s.bucketName,
64+
ScopeName: s.scopeName,
65+
CollectionName: s.collectionName,
66+
Key: docId,
67+
Content: &kv_v1.UpsertRequest_ContentUncompressed{
68+
ContentUncompressed: TEST_CONTENT,
69+
},
70+
ContentFlags: TEST_CONTENT_FLAGS,
71+
DurabilityLevel: kv_v1.DurabilityLevel_DURABILITY_LEVEL_MAJORITY.Enum(),
72+
}, grpc.PerRPCCredentials(s.basicRpcCreds))
73+
return err
74+
},
75+
},
76+
{
77+
name: "Replace",
78+
fn: func() error {
79+
docId := s.testDocId()
80+
_, err := kvClient.Replace(context.Background(), &kv_v1.ReplaceRequest{
81+
BucketName: s.bucketName,
82+
ScopeName: s.scopeName,
83+
CollectionName: s.collectionName,
84+
Key: docId,
85+
Content: &kv_v1.ReplaceRequest_ContentUncompressed{
86+
ContentUncompressed: TEST_CONTENT,
87+
},
88+
ContentFlags: TEST_CONTENT_FLAGS,
89+
DurabilityLevel: kv_v1.DurabilityLevel_DURABILITY_LEVEL_MAJORITY.Enum(),
90+
}, grpc.PerRPCCredentials(s.basicRpcCreds))
91+
return err
92+
},
93+
},
94+
{
95+
name: "Remove",
96+
fn: func() error {
97+
docId := s.testDocId()
98+
_, err := kvClient.Remove(context.Background(), &kv_v1.RemoveRequest{
99+
BucketName: s.bucketName,
100+
ScopeName: s.scopeName,
101+
CollectionName: s.collectionName,
102+
Key: docId,
103+
DurabilityLevel: kv_v1.DurabilityLevel_DURABILITY_LEVEL_MAJORITY.Enum(),
104+
}, grpc.PerRPCCredentials(s.basicRpcCreds))
105+
return err
106+
},
107+
},
108+
{
109+
name: "Increment",
110+
fn: func() error {
111+
docId := s.binaryDocId([]byte("5"))
112+
_, err := kvClient.Increment(context.Background(), &kv_v1.IncrementRequest{
113+
BucketName: s.bucketName,
114+
ScopeName: s.scopeName,
115+
CollectionName: s.collectionName,
116+
Key: docId,
117+
Delta: 1,
118+
DurabilityLevel: kv_v1.DurabilityLevel_DURABILITY_LEVEL_MAJORITY.Enum(),
119+
}, grpc.PerRPCCredentials(s.basicRpcCreds))
120+
return err
121+
},
122+
},
123+
{
124+
name: "Decrement",
125+
fn: func() error {
126+
docId := s.binaryDocId([]byte("5"))
127+
_, err := kvClient.Decrement(context.Background(), &kv_v1.DecrementRequest{
128+
BucketName: s.bucketName,
129+
ScopeName: s.scopeName,
130+
CollectionName: s.collectionName,
131+
Key: docId,
132+
Delta: 1,
133+
DurabilityLevel: kv_v1.DurabilityLevel_DURABILITY_LEVEL_MAJORITY.Enum(),
134+
}, grpc.PerRPCCredentials(s.basicRpcCreds))
135+
return err
136+
},
137+
},
138+
{
139+
name: "Append",
140+
fn: func() error {
141+
docId := s.binaryDocId([]byte("hello"))
142+
_, err := kvClient.Append(context.Background(), &kv_v1.AppendRequest{
143+
BucketName: s.bucketName,
144+
ScopeName: s.scopeName,
145+
CollectionName: s.collectionName,
146+
Key: docId,
147+
Content: []byte("world"),
148+
DurabilityLevel: kv_v1.DurabilityLevel_DURABILITY_LEVEL_MAJORITY.Enum(),
149+
}, grpc.PerRPCCredentials(s.basicRpcCreds))
150+
return err
151+
},
152+
},
153+
{
154+
name: "Prepend",
155+
fn: func() error {
156+
docId := s.binaryDocId([]byte("hello"))
157+
_, err := kvClient.Prepend(context.Background(), &kv_v1.PrependRequest{
158+
BucketName: s.bucketName,
159+
ScopeName: s.scopeName,
160+
CollectionName: s.collectionName,
161+
Key: docId,
162+
Content: []byte("world"),
163+
DurabilityLevel: kv_v1.DurabilityLevel_DURABILITY_LEVEL_MAJORITY.Enum(),
164+
}, grpc.PerRPCCredentials(s.basicRpcCreds))
165+
return err
166+
},
167+
},
168+
{
169+
name: "MutateIn",
170+
fn: func() error {
171+
docId := s.binaryDocId([]byte(`{"foo": 14}`))
172+
_, err := kvClient.MutateIn(context.Background(), &kv_v1.MutateInRequest{
173+
BucketName: s.bucketName,
174+
ScopeName: s.scopeName,
175+
CollectionName: s.collectionName,
176+
Key: docId,
177+
DurabilityLevel: kv_v1.DurabilityLevel_DURABILITY_LEVEL_MAJORITY.Enum(),
178+
Specs: []*kv_v1.MutateInRequest_Spec{{
179+
Operation: kv_v1.MutateInRequest_Spec_OPERATION_INSERT,
180+
Path: "newfoo",
181+
Content: []byte(`"baz"`),
182+
}},
183+
}, grpc.PerRPCCredentials(s.basicRpcCreds))
184+
return err
185+
},
186+
},
187+
}
188+
189+
for _, tc := range tests {
190+
s.Run(tc.name, func() {
191+
require.Eventually(s.T(), func() bool {
192+
err := tc.fn()
193+
194+
if err != nil {
195+
assertRpcStatus(s.T(), err, codes.FailedPrecondition)
196+
assert.Contains(s.T(), err.Error(), "Not enough servers to use this durability level")
197+
return true
198+
}
199+
200+
return false
201+
}, 60*time.Second, 5*time.Second)
202+
})
203+
}
204+
}

‎testutils/dinocluster.go‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,10 @@ func runDinoAllowTraffic(node string) error {
8989
return runNoResDinoCmd([]string{"chaos", "allow-traffic", globalTestConfig.DinoId, node})
9090
}
9191

92+
func runDinoRemoveNode(node string) error {
93+
return runNoResDinoCmd([]string{"nodes", "rm", globalTestConfig.DinoId, node})
94+
}
95+
9296
type DinoController struct {
9397
t *testing.T
9498
oldFoSettings *cbmgmtx.GetAutoFailoverSettingsResponse
@@ -173,3 +177,8 @@ func (c *DinoController) AllowTraffic(node string) {
173177
c.blockedNodes = slices.Delete(c.blockedNodes, hostIdx, hostIdx+1)
174178
}
175179
}
180+
181+
func (c *DinoController) RemoveNode(node string) {
182+
err := runDinoRemoveNode(node)
183+
require.NoError(c.t, err)
184+
}

0 commit comments

Comments
 (0)