Skip to content

Commit ac1d0d9

Browse files
AloentoAloento
authored
[NEW] DIS (#458)
[NEW] DIS Closes #447 Acceptance Steps Performed === RUN TestDISWorkflow 2023/06/26 16:26:16 Create DIS App, Name: app-create-test-NLz 2023/06/26 16:26:16 Get DIS App, Name: app-create-test-NLz 2023/06/26 16:26:16 List DIS Apps 2023/06/26 16:26:17 Create DIS Stream, Name: stream-create-test-eE9 2023/06/26 16:26:18 Get DIS App status, Name: app-create-test-NLz 2023/06/26 16:26:18 Get DIS Stream, Name: stream-create-test-eE9 2023/06/26 16:26:18 Update DIS Stream partitions count, Name: stream-create-test-eE9 2023/06/26 16:26:18 List DIS Streams 2023/06/26 16:26:19 Commit DIS App Checkpoint, Name: app-create-test-NLz 2023/06/26 16:26:19 Get DIS App Checkpoint, Name: app-create-test-NLz 2023/06/26 16:26:19 Create DIS Stream Data records, Name: stream-create-test-eE9 2023/06/26 16:26:20 Create DIS Stream cursor, Name: stream-create-test-eE9 tools.go:72: { "partition_cursor": "eyJnZXRJdGVyYXRvclBhcmFtIjp7InN0cmVhbS1uYW1lIjoic3RyZWFtLWNyZWF0ZS10ZXN0LWVFOSIsInBhcnRpdGlvbi1pZCI6IjAiLCJjdXJzb3ItdHlwZSI6IkFUX1NFUVVFTkNFX05VTUJFUiIsInN0YXJ0aW5nLXNlcXVlbmNlLW51bWJlciI6IjAifSwiZ2VuZXJhdGVUaW1lc3RhbXAiOjE2ODc3ODk1ODAyNzZ9" } 2023/06/26 16:26:20 Get DIS Stream records, Name: stream-create-test-eE9 tools.go:72: { "next_partition_cursor": "eyJnZXRJdGVyYXRvclBhcmFtIjp7InN0cmVhbS1uYW1lIjoic3RyZWFtLWNyZWF0ZS10ZXN0LWVFOSIsInBhcnRpdGlvbi1pZCI6IjAiLCJjdXJzb3ItdHlwZSI6IkFUX1NFUVVFTkNFX05VTUJFUiIsInN0YXJ0aW5nLXNlcXVlbmNlLW51bWJlciI6IjAifSwiZ2VuZXJhdGVUaW1lc3RhbXAiOjE2ODc3ODk1ODI1MTF9" } 2023/06/26 16:26:22 Querying Stream Monitoring Data 2023/06/26 16:26:23 Querying Partition Monitoring Data 2023/06/26 16:26:23 Delete DIS App Checkpoint, Name: app-create-test-NLz 2023/06/26 16:26:23 Delete DIS Stream, Name: stream-create-test-eE9 2023/06/26 16:26:24 Delete DIS App, Name: app-create-test-NLz --- PASS: TestDISWorkflow (8.63s) PASS === RUN TestDISDumpWorkflow 2023/06/26 16:13:51 Create DIS App, Name: app-create-test-UIM 2023/06/26 16:13:51 Create DIS Stream, Name: stream-create-test-02e 2023/06/26 16:13:52 Delete DIS Dump task, Name: task-create-test-fVb 2023/06/26 16:13:53 Get DIS Dump task, Name: task-create-test-fVb 2023/06/26 16:13:54 Check DIS Dump task state is paused, Name: task-create-test-fVb 2023/06/26 16:13:54 List DIS Dump tasks 2023/06/26 16:13:54 Check DIS Dump task state is running, Name: task-create-test-fVb 2023/06/26 16:13:55 Delete DIS Dump task, Name: task-create-test-fVb 2023/06/26 16:13:55 Delete DIS Stream, Name: stream-create-test-02e 2023/06/26 16:13:55 Delete DIS App, Name: app-create-test-UIM --- PASS: TestDISDumpWorkflow (7.42s) PASS Process finished with the exit code 0 Reviewed-by: Anton Sidelnikov Reviewed-by: Artem Lifshits
1 parent 35930a8 commit ac1d0d9

34 files changed

+2551
-1
lines changed

‎acceptance/clients/clients.go‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -632,6 +632,16 @@ func NewDmsV1Client() (*golangsdk.ServiceClient, error) {
632632
})
633633
}
634634

635+
func NewDisV2Client() (*golangsdk.ServiceClient, error) {
636+
cc, err := CloudAndClient()
637+
if err != nil {
638+
return nil, err
639+
}
640+
return openstack.NewDISServiceV2(cc.ProviderClient, golangsdk.EndpointOpts{
641+
Region: cc.RegionName,
642+
})
643+
}
644+
635645
func NewDmsV11Client() (*golangsdk.ServiceClient, error) {
636646
cc, err := CloudAndClient()
637647
if err != nil {

‎acceptance/openstack/dis/v2/dis_test.go‎

Lines changed: 322 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,322 @@
1+
package v2
2+
3+
import (
4+
"log"
5+
"strings"
6+
"testing"
7+
"time"
8+
9+
"github.com/opentelekomcloud/gophertelekomcloud/acceptance/clients"
10+
"github.com/opentelekomcloud/gophertelekomcloud/acceptance/tools"
11+
"github.com/opentelekomcloud/gophertelekomcloud/openstack/common/pointerto"
12+
"github.com/opentelekomcloud/gophertelekomcloud/openstack/dis/v2/apps"
13+
"github.com/opentelekomcloud/gophertelekomcloud/openstack/dis/v2/checkpoints"
14+
"github.com/opentelekomcloud/gophertelekomcloud/openstack/dis/v2/data"
15+
"github.com/opentelekomcloud/gophertelekomcloud/openstack/dis/v2/dump"
16+
"github.com/opentelekomcloud/gophertelekomcloud/openstack/dis/v2/monitors"
17+
"github.com/opentelekomcloud/gophertelekomcloud/openstack/dis/v2/streams"
18+
"github.com/opentelekomcloud/gophertelekomcloud/openstack/obs"
19+
th "github.com/opentelekomcloud/gophertelekomcloud/testhelper"
20+
)
21+
22+
func TestDISWorkflow(t *testing.T) {
23+
client, err := clients.NewDisV2Client()
24+
th.AssertNoErr(t, err)
25+
now := time.Now()
26+
appName := tools.RandomString("app-create-test-", 3)
27+
log.Printf("Create DIS App, Name: %s", appName)
28+
err = apps.CreateApp(client, apps.CreateAppOpts{
29+
AppName: appName,
30+
})
31+
th.AssertNoErr(t, err)
32+
t.Cleanup(func() {
33+
log.Printf("Delete DIS App, Name: %s", appName)
34+
err = apps.DeleteApp(client, appName)
35+
th.AssertNoErr(t, err)
36+
})
37+
38+
log.Printf("Get DIS App, Name: %s", appName)
39+
app, err := apps.GetApp(client, appName)
40+
th.AssertNoErr(t, err)
41+
th.AssertEquals(t, app.AppName, appName)
42+
43+
log.Print("List DIS Apps")
44+
listApps, err := apps.ListApps(client, apps.ListAppOpts{Limit: pointerto.Int(10)})
45+
th.AssertNoErr(t, err)
46+
th.AssertEquals(t, *listApps.TotalNumber, 1)
47+
48+
streamName := tools.RandomString("stream-create-test-", 3)
49+
log.Printf("Create DIS Stream, Name: %s", streamName)
50+
err = streams.CreateStream(client, streams.CreateStreamOpts{
51+
StreamName: streamName,
52+
PartitionCount: 3,
53+
})
54+
th.AssertNoErr(t, err)
55+
t.Cleanup(func() {
56+
log.Printf("Delete DIS Stream, Name: %s", streamName)
57+
err = streams.DeleteStream(client, streamName)
58+
th.AssertNoErr(t, err)
59+
})
60+
61+
log.Printf("Get DIS App status, Name: %s", appName)
62+
appStatus, err := apps.GetAppStatus(client, apps.GetAppStatusOpts{
63+
AppName: appName,
64+
StreamName: streamName,
65+
CheckpointType: "LAST_READ",
66+
})
67+
th.AssertNoErr(t, err)
68+
th.AssertEquals(t, len(appStatus.PartitionConsumingStates), 3)
69+
70+
log.Printf("Get DIS Stream, Name: %s", streamName)
71+
getStream, err := streams.GetStream(client, streams.GetStreamOpts{
72+
StreamName: streamName,
73+
})
74+
th.AssertNoErr(t, err)
75+
th.AssertEquals(t, getStream.StreamName, streamName)
76+
77+
log.Printf("Update DIS Stream partitions count, Name: %s", streamName)
78+
err = streams.UpdatePartitionCount(client, streams.UpdatePartitionCountOpts{
79+
StreamName: streamName,
80+
TargetPartitionCount: 4,
81+
})
82+
th.AssertNoErr(t, err)
83+
84+
// "Bad request with: [PUT https://dis.eu-de.otc.t-systems.com/v2/5045c215010c440d91b2f7dce1f3753b/streams/stream-create-test-jmn],
85+
// error message: {\"errorCode\":\"DIS.4200\",\"message\":\"Invalid request. [Invalid target_partition_count null.]\"}"
86+
// https://jira.tsi-dev.otc-service.com/browse/BM-2472
87+
// err = streams.UpdateStream(client, streams.UpdateStreamOpts{
88+
// StreamName: streamName,
89+
// DataType: "JSON",
90+
// })
91+
// th.AssertNoErr(t, err)
92+
93+
// getStreamUpdated, err := streams.GetStream(client, streams.GetStreamOpts{
94+
// StreamName: streamName,
95+
// })
96+
// th.AssertNoErr(t, err)
97+
// th.AssertEquals(t, getStreamUpdated.StreamName, streamName)
98+
// th.AssertEquals(t, getStreamUpdated.DataType, "JSON")
99+
100+
// url not found: https://jira.tsi-dev.otc-service.com/browse/BM-2474
101+
// log.Printf("Create DIS Stream Policy Rule, Name: %s", streamName)
102+
// err = streams.CreatePolicyRule(client, streams.CreatePolicyRuleOpts{
103+
// StreamName: streamName,
104+
// StreamId: getStream.StreamId,
105+
// PrincipalName: client.DomainID,
106+
// ActionType: "putRecords",
107+
// Effect: "effect",
108+
// })
109+
// th.AssertNoErr(t, err)
110+
//
111+
// log.Printf("Get DIS Stream Policy Rule, Name: %s", streamName)
112+
// rule, err := streams.GetPolicyRule(client, streamName)
113+
// th.AssertNoErr(t, err)
114+
// tools.PrintResource(t, rule)
115+
116+
log.Print("List DIS Streams")
117+
listStreams, err := streams.ListStreams(client, streams.ListStreamsOpts{})
118+
th.AssertNoErr(t, err)
119+
th.AssertEquals(t, listStreams.StreamInfoList[0].DataType, "BLOB")
120+
121+
log.Printf("Commit DIS App Checkpoint, Name: %s", appName)
122+
err = checkpoints.CommitCheckpoint(client, checkpoints.CommitCheckpointOpts{
123+
AppName: appName,
124+
CheckpointType: "LAST_READ",
125+
StreamName: streamName,
126+
PartitionId: "0",
127+
SequenceNumber: "0",
128+
})
129+
th.AssertNoErr(t, err)
130+
t.Cleanup(func() {
131+
log.Printf("Delete DIS App Checkpoint, Name: %s", appName)
132+
err = checkpoints.DeleteCheckpoint(client, checkpoints.DeleteCheckpointOpts{
133+
AppName: appName,
134+
StreamName: streamName,
135+
PartitionId: "0",
136+
CheckpointType: "LAST_READ",
137+
})
138+
th.AssertNoErr(t, err)
139+
})
140+
141+
log.Printf("Get DIS App Checkpoint, Name: %s", appName)
142+
checkpoint, err := checkpoints.GetCheckpoint(client, checkpoints.GetCheckpointOpts{
143+
AppName: appName,
144+
StreamName: streamName,
145+
PartitionId: "0",
146+
CheckpointType: "LAST_READ",
147+
})
148+
th.AssertNoErr(t, err)
149+
th.AssertEquals(t, checkpoint.SequenceNumber, "0")
150+
151+
log.Printf("Create DIS Stream Data records, Name: %s", streamName)
152+
_, err = data.PutRecords(client, data.PutRecordsOpts{
153+
StreamName: streamName,
154+
StreamId: getStream.StreamId,
155+
Records: []data.PutRecordsRequestEntry{
156+
{
157+
Data: "dGVzdCBzdHJpbmc=",
158+
},
159+
},
160+
})
161+
th.AssertNoErr(t, err)
162+
163+
log.Printf("Create DIS Stream cursor, Name: %s", streamName)
164+
cursor, err := data.GetCursor(client, data.GetCursorOpts{
165+
StreamName: streamName,
166+
PartitionId: "0",
167+
})
168+
th.AssertNoErr(t, err)
169+
tools.PrintResource(t, cursor)
170+
171+
log.Printf("Get DIS Stream records, Name: %s", streamName)
172+
records, err := data.GetRecords(client, data.GetRecordsOpts{
173+
PartitionCursor: cursor.PartitionCursor,
174+
})
175+
th.AssertNoErr(t, err)
176+
tools.PrintResource(t, records)
177+
178+
log.Print("Querying Stream Monitoring Data")
179+
streamMon, err := monitors.GetStreamMonitor(client, monitors.GetStreamMonitorOpts{
180+
StreamName: streamName,
181+
Label: "total_put_bytes_per_stream",
182+
StartTime: now.Unix(),
183+
EndTime: now.Unix() + 50,
184+
})
185+
th.AssertNoErr(t, err)
186+
th.AssertEquals(t, streamMon.Metrics.Label, "total_put_bytes_per_stream")
187+
188+
log.Print("Querying Partition Monitoring Data")
189+
streamPartMon, err := monitors.GetPartitionMonitor(client, monitors.GetPartitionMonitorOpts{
190+
PartitionId: "0",
191+
StreamName: streamName,
192+
Label: "total_put_bytes_per_partition",
193+
StartTime: now.Unix(),
194+
EndTime: now.Unix() + 50,
195+
})
196+
th.AssertNoErr(t, err)
197+
th.AssertEquals(t, streamPartMon.Metrics.Label, "total_put_bytes_per_partition")
198+
}
199+
200+
func TestDISDumpWorkflow(t *testing.T) {
201+
t.Skip("Need to create dis_admin_agency first")
202+
client, err := clients.NewDisV2Client()
203+
th.AssertNoErr(t, err)
204+
205+
clientObs, err := clients.NewOBSClient()
206+
th.AssertNoErr(t, err)
207+
208+
bucketName := strings.ToLower(tools.RandomString("obs-dis-test", 5))
209+
210+
_, err = clientObs.CreateBucket(&obs.CreateBucketInput{
211+
Bucket: bucketName,
212+
})
213+
t.Cleanup(func() {
214+
_, err = clientObs.DeleteBucket(bucketName)
215+
th.AssertNoErr(t, err)
216+
})
217+
th.AssertNoErr(t, err)
218+
219+
appName := tools.RandomString("app-create-test-", 3)
220+
log.Printf("Create DIS App, Name: %s", appName)
221+
err = apps.CreateApp(client, apps.CreateAppOpts{
222+
AppName: appName,
223+
})
224+
th.AssertNoErr(t, err)
225+
t.Cleanup(func() {
226+
log.Printf("Delete DIS App, Name: %s", appName)
227+
err = apps.DeleteApp(client, appName)
228+
th.AssertNoErr(t, err)
229+
})
230+
231+
streamName := tools.RandomString("stream-create-test-", 3)
232+
log.Printf("Create DIS Stream, Name: %s", streamName)
233+
err = streams.CreateStream(client, streams.CreateStreamOpts{
234+
StreamName: streamName,
235+
PartitionCount: 3,
236+
})
237+
th.AssertNoErr(t, err)
238+
t.Cleanup(func() {
239+
log.Printf("Delete DIS Stream, Name: %s", streamName)
240+
err = streams.DeleteStream(client, streamName)
241+
th.AssertNoErr(t, err)
242+
})
243+
244+
taskName := tools.RandomString("task-create-test-", 3)
245+
log.Printf("Delete DIS Dump task, Name: %s", taskName)
246+
err = dump.CreateOBSDumpTask(client, dump.CreateOBSDumpTaskOpts{
247+
StreamName: streamName,
248+
DestinationType: "OBS",
249+
OBSDestinationDescriptor: dump.OBSDestinationDescriptorOpts{
250+
TaskName: taskName,
251+
AgencyName: "dis_admin_agency",
252+
ConsumerStrategy: "LATEST",
253+
DestinationFileType: "text",
254+
OBSBucketPath: bucketName,
255+
FilePrefix: "",
256+
PartitionFormat: "yyyy/MM/dd/HH/mm",
257+
RecordDelimiter: "|",
258+
DeliverTimeInterval: pointerto.Int(30),
259+
},
260+
})
261+
th.AssertNoErr(t, err)
262+
263+
t.Cleanup(func() {
264+
log.Printf("Delete DIS Dump task, Name: %s", taskName)
265+
err = dump.DeleteTransferTask(client, dump.DeleteTransferTaskOpts{
266+
StreamName: streamName,
267+
TaskName: taskName,
268+
})
269+
th.AssertNoErr(t, err)
270+
})
271+
272+
log.Printf("Get DIS Dump task, Name: %s", taskName)
273+
getDump, err := dump.GetTransferTask(client, dump.GetTransferTaskOpts{
274+
StreamName: streamName,
275+
TaskName: taskName,
276+
})
277+
th.AssertNoErr(t, err)
278+
th.AssertEquals(t, getDump.TaskName, taskName)
279+
280+
err = dump.TransferTaskAction(client, dump.TransferTaskActionOpts{
281+
StreamName: streamName,
282+
Action: "stop",
283+
Tasks: []dump.BatchTransferTask{
284+
{
285+
Id: getDump.TaskId,
286+
},
287+
},
288+
})
289+
th.AssertNoErr(t, err)
290+
291+
log.Printf("Check DIS Dump task state is paused, Name: %s", taskName)
292+
stateDumpStopped, err := dump.GetTransferTask(client, dump.GetTransferTaskOpts{
293+
StreamName: streamName,
294+
TaskName: taskName,
295+
})
296+
th.AssertNoErr(t, err)
297+
th.AssertEquals(t, stateDumpStopped.State, "PAUSED")
298+
299+
log.Print("List DIS Dump tasks")
300+
listTasks, err := dump.ListTransferTasks(client, streamName)
301+
th.AssertNoErr(t, err)
302+
th.AssertEquals(t, *listTasks.TotalNumber, 1)
303+
304+
err = dump.TransferTaskAction(client, dump.TransferTaskActionOpts{
305+
StreamName: streamName,
306+
Action: "start",
307+
Tasks: []dump.BatchTransferTask{
308+
{
309+
Id: getDump.TaskId,
310+
},
311+
},
312+
})
313+
th.AssertNoErr(t, err)
314+
315+
log.Printf("Check DIS Dump task state is running, Name: %s", taskName)
316+
stateDumpStarted, err := dump.GetTransferTask(client, dump.GetTransferTaskOpts{
317+
StreamName: streamName,
318+
TaskName: taskName,
319+
})
320+
th.AssertNoErr(t, err)
321+
th.AssertEquals(t, stateDumpStarted.State, "RUNNING")
322+
}

‎openstack/client.go‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -727,10 +727,16 @@ func NewDCSServiceV1(client *golangsdk.ProviderClient, eo golangsdk.EndpointOpts
727727
// NewDDSServiceV3 creates a ServiceClient that may be used to access the Document Database Service.
728728
func NewDDSServiceV3(client *golangsdk.ProviderClient, eo golangsdk.EndpointOpts) (*golangsdk.ServiceClient, error) {
729729
sc, err := initClientOpts(client, eo, "ddsv3")
730+
return sc, err
731+
}
732+
733+
func NewDISServiceV2(client *golangsdk.ProviderClient, eo golangsdk.EndpointOpts) (*golangsdk.ServiceClient, error) {
734+
sc, err := initClientOpts(client, eo, "dis")
730735
if err != nil {
731736
return nil, err
732737
}
733-
return sc, nil
738+
sc.ResourceBase = sc.Endpoint + "v2/" + client.ProjectID + "/"
739+
return sc, err
734740
}
735741

736742
// NewOBSService creates a ServiceClient that may be used to access the Object Storage Service.

‎openstack/dis/v2/apps/CreateApp.go‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
package apps
2+
3+
import (
4+
golangsdk "github.com/opentelekomcloud/gophertelekomcloud"
5+
"github.com/opentelekomcloud/gophertelekomcloud/internal/build"
6+
)
7+
8+
type CreateAppOpts struct {
9+
// Unique identifier of the consumer application to be created.
10+
// The application name contains 1 to 200 characters, including letters, digits, underscores (_), and hyphens (-).
11+
// Minimum: 1
12+
// Maximum: 200
13+
AppName string `json:"app_name"`
14+
}
15+
16+
func CreateApp(client *golangsdk.ServiceClient, opts CreateAppOpts) error {
17+
b, err := build.RequestBody(opts, "")
18+
if err != nil {
19+
return err
20+
}
21+
22+
// POST /v2/{project_id}/apps
23+
_, err = client.Post(client.ServiceURL("apps"), b, nil, &golangsdk.RequestOpts{
24+
OkCodes: []int{201},
25+
})
26+
return err
27+
}

‎openstack/dis/v2/apps/DeleteApp.go‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
package apps
2+
3+
import golangsdk "github.com/opentelekomcloud/gophertelekomcloud"
4+
5+
func DeleteApp(client *golangsdk.ServiceClient, appName string) (err error) {
6+
// DELETE /v2/{project_id}/apps/{app_name}
7+
_, err = client.Delete(client.ServiceURL("apps", appName), nil)
8+
return
9+
}

0 commit comments

Comments
 (0)