Skip to content

Commit 9aff68a

Browse files
kabachookkabachook
committed
implement datalens tables
commit_hash:e6d288d0e0d1e15eff16c4964baae575d38845ed
1 parent fd1b45d commit 9aff68a

29 files changed

Lines changed: 1872 additions & 133 deletions

‎.mapping.json‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,12 @@
2323
"client/testdata/yc_kubernetes_clusters.json":"security/cloudquery/cq-source-yc/client/testdata/yc_kubernetes_clusters.json",
2424
"client/transformers.go":"security/cloudquery/cq-source-yc/client/transformers.go",
2525
"client/transformers_test.go":"security/cloudquery/cq-source-yc/client/transformers_test.go",
26+
"client/yc/datalens/datalens.go":"security/cloudquery/cq-source-yc/client/yc/datalens/datalens.go",
27+
"client/yc/datalens/datalens_test.go":"security/cloudquery/cq-source-yc/client/yc/datalens/datalens_test.go",
28+
"client/yc/datalens/gen/main.go":"security/cloudquery/cq-source-yc/client/yc/datalens/gen/main.go",
29+
"client/yc/datalens/models.gen.go":"security/cloudquery/cq-source-yc/client/yc/datalens/models.gen.go",
30+
"client/yc/datalens/oapi-codegen.yaml":"security/cloudquery/cq-source-yc/client/yc/datalens/oapi-codegen.yaml",
31+
"client/yc/datalens/openapi.json":"security/cloudquery/cq-source-yc/client/yc/datalens/openapi.json",
2632
"client/yc/grpc.go":"security/cloudquery/cq-source-yc/client/yc/grpc.go",
2733
"client/yc/hierarchy.go":"security/cloudquery/cq-source-yc/client/yc/hierarchy.go",
2834
"client/yc/hierarchy_test.go":"security/cloudquery/cq-source-yc/client/yc/hierarchy_test.go",
@@ -85,6 +91,12 @@
8591
"docs/tables/yc_compute_instances.md":"security/cloudquery/cq-source-yc/docs/tables/yc_compute_instances.md",
8692
"docs/tables/yc_containerregistry_registries.md":"security/cloudquery/cq-source-yc/docs/tables/yc_containerregistry_registries.md",
8793
"docs/tables/yc_containerregistry_repositories.md":"security/cloudquery/cq-source-yc/docs/tables/yc_containerregistry_repositories.md",
94+
"docs/tables/yc_datalens_connections.md":"security/cloudquery/cq-source-yc/docs/tables/yc_datalens_connections.md",
95+
"docs/tables/yc_datalens_dashboards.md":"security/cloudquery/cq-source-yc/docs/tables/yc_datalens_dashboards.md",
96+
"docs/tables/yc_datalens_datasets.md":"security/cloudquery/cq-source-yc/docs/tables/yc_datalens_datasets.md",
97+
"docs/tables/yc_datalens_folders.md":"security/cloudquery/cq-source-yc/docs/tables/yc_datalens_folders.md",
98+
"docs/tables/yc_datalens_reports.md":"security/cloudquery/cq-source-yc/docs/tables/yc_datalens_reports.md",
99+
"docs/tables/yc_datalens_widgets.md":"security/cloudquery/cq-source-yc/docs/tables/yc_datalens_widgets.md",
88100
"docs/tables/yc_datasphere_communities.md":"security/cloudquery/cq-source-yc/docs/tables/yc_datasphere_communities.md",
89101
"docs/tables/yc_datasphere_projects.md":"security/cloudquery/cq-source-yc/docs/tables/yc_datasphere_projects.md",
90102
"docs/tables/yc_datatransfer_endpoints.md":"security/cloudquery/cq-source-yc/docs/tables/yc_datatransfer_endpoints.md",
@@ -226,6 +238,8 @@
226238
"resources/compute/instances.go":"security/cloudquery/cq-source-yc/resources/compute/instances.go",
227239
"resources/containerregistry/registries.go":"security/cloudquery/cq-source-yc/resources/containerregistry/registries.go",
228240
"resources/containerregistry/repositories.go":"security/cloudquery/cq-source-yc/resources/containerregistry/repositories.go",
241+
"resources/datalens/connections.go":"security/cloudquery/cq-source-yc/resources/datalens/connections.go",
242+
"resources/datalens/entries.go":"security/cloudquery/cq-source-yc/resources/datalens/entries.go",
229243
"resources/datasphere/communities.go":"security/cloudquery/cq-source-yc/resources/datasphere/communities.go",
230244
"resources/datasphere/projects.go":"security/cloudquery/cq-source-yc/resources/datasphere/projects.go",
231245
"resources/datatransfer/endpoints.go":"security/cloudquery/cq-source-yc/resources/datatransfer/endpoints.go",

‎Makefile‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,14 @@ gen-docs: build
2121
lint:
2222
@golangci-lint run --timeout 10m
2323

24+
.PHONY: gen-datalens
25+
gen-datalens:
26+
curl -sSfL https://api.datalens.tech/json/ -o client/yc/datalens/openapi.json
27+
cd client/yc/datalens && go run ./gen
28+
2429
# All gen targets
2530
.PHONY: gen
26-
gen: gen-docs
31+
gen: gen-docs gen-datalens
2732

2833
.PHONY: update-deps
2934
update-deps:

‎client/client.go‎

Lines changed: 27 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,12 @@ import (
44
"context"
55
"fmt"
66
"strings"
7+
"time"
78

89
"github.com/cloudquery/plugin-sdk/v4/state"
910
"github.com/rs/zerolog"
1011
"github.com/yandex-cloud/cq-source-yc/client/yc"
12+
"github.com/yandex-cloud/cq-source-yc/client/yc/datalens"
1113
ycsdk "github.com/yandex-cloud/go-sdk"
1214
ycsdkv2 "github.com/yandex-cloud/go-sdk/v2"
1315
)
@@ -26,10 +28,11 @@ type Client struct {
2628
MultiplexedResourceId string
2729
MultiplexedResourceType ResourceType
2830

29-
Backend state.Client
30-
Logger zerolog.Logger
31-
SDK *ycsdk.SDK
32-
SDKv2 *ycsdkv2.SDK
31+
Backend state.Client
32+
Logger zerolog.Logger
33+
SDK *ycsdk.SDK
34+
SDKv2 *ycsdkv2.SDK
35+
Datalens *datalens.Client
3336
}
3437

3538
func (c *Client) ID() string {
@@ -88,16 +91,32 @@ func New(ctx context.Context, logger zerolog.Logger, spec *Spec) (*Client, error
8891
Endpoint: spec.Endpoint,
8992
UserAgent: DefaultUserAgent,
9093
MaxRetries: spec.MaxRetries,
91-
DebugGRPC: spec.DebugGRPC,
94+
Debug: spec.Debug,
9295
})
9396
if err != nil {
9497
return nil, err
9598
}
9699

100+
// The middleware caches IAM tokens and refreshes them before expiry.
101+
iamTokens := ycsdk.NewIAMTokenMiddleware(sdk, time.Now)
102+
dl, err := datalens.New(datalens.Config{
103+
Endpoint: spec.DatalensEndpoint,
104+
UserAgent: DefaultUserAgent,
105+
MaxRetries: spec.MaxRetries,
106+
Logger: logger,
107+
Token: func(ctx context.Context) (string, error) {
108+
return iamTokens.GetIAMToken(ctx, false)
109+
},
110+
})
111+
if err != nil {
112+
return nil, fmt.Errorf("initialize DataLens client: %w", err)
113+
}
114+
97115
client := Client{
98-
SDK: sdk,
99-
SDKv2: sdkv2,
100-
Logger: logger,
116+
SDK: sdk,
117+
SDKv2: sdkv2,
118+
Datalens: dl,
119+
Logger: logger,
101120
}
102121

103122
hierarchy, err := yc.NewResourceHierarchy(ctx, logger, sdk, spec.OrganizationIDs, spec.CloudIDs, spec.FolderIDs)

‎client/spec.go‎

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
package client
22

3-
import "github.com/cloudquery/plugin-sdk/v4/scheduler"
3+
import (
4+
"github.com/cloudquery/plugin-sdk/v4/scheduler"
5+
"github.com/yandex-cloud/cq-source-yc/client/yc/datalens"
6+
)
47

58
const defaultEndpoint = "api.cloud.yandex.net:443"
69

@@ -14,15 +17,18 @@ type Spec struct {
1417
// List of all folder ids to fetch information from
1518
FolderIDs []string `json:"folder_ids"`
1619

17-
// If `true`, will log all GRPC calls (currently YC SDK only)
18-
DebugGRPC bool `json:"debug_grpc"`
20+
// If `true`, will log gRPC/HTTP calls
21+
Debug bool `json:"debug"`
1922

2023
// Defines the maximum number of times an API request will be retried.
2124
MaxRetries int `json:"max_retries,omitempty" jsonschema:"default=3"`
2225

2326
// The base URL endpoint the SDK will use
2427
Endpoint string `json:"endpoint"`
2528

29+
// The base URL of the DataLens API
30+
DatalensEndpoint string `json:"datalens_endpoint" jsonschema:"default=https://api.datalens.tech"`
31+
2632
// The best effort maximum number of Go routines to use. Lower this number to reduce memory usage.
2733
Concurrency int `json:"concurrency" jsonschema:"minimum=1,default=50000"`
2834

@@ -34,10 +40,11 @@ type Spec struct {
3440

3541
func NewDefaultSpec() *Spec {
3642
return &Spec{
37-
MaxRetries: 3,
38-
Endpoint: defaultEndpoint,
39-
Concurrency: scheduler.DefaultConcurrency,
40-
Scheduler: scheduler.StrategyShuffle,
43+
MaxRetries: 3,
44+
Endpoint: defaultEndpoint,
45+
DatalensEndpoint: datalens.DefaultEndpoint,
46+
Concurrency: scheduler.DefaultConcurrency,
47+
Scheduler: scheduler.StrategyShuffle,
4148
}
4249
}
4350

0 commit comments

Comments
 (0)