Skip to content

Commit a895e9b

Browse files
yevgenypatsyevgenypats
committed
feat!: Move spec into specs directory
1 parent 0b7dd6e commit a895e9b

21 files changed

+68
-67
lines changed

Diff for: ‎.github/PULL_REQUEST_TEMPLATE.md

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
1+
#### Summary
2+
13
<!-- 🎉 Thank you for making CloudQuery awesome by submitting a PR 🎉 -->
24

3-
#### Summary
45

56
<!--
67
Explain what problem this PR addresses

Diff for: ‎clients/destination.go

+5-5
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ import (
77
"github.com/cloudquery/cq-plugin-sdk/internal/pb"
88
"github.com/cloudquery/cq-plugin-sdk/plugins"
99
"github.com/cloudquery/cq-plugin-sdk/schema"
10-
"github.com/cloudquery/cq-plugin-sdk/spec"
10+
"github.com/cloudquery/cq-plugin-sdk/specs"
1111
"github.com/vmihailenco/msgpack/v5"
1212
"google.golang.org/grpc"
1313
"gopkg.in/yaml.v3"
@@ -31,11 +31,11 @@ func NewLocalDestinationClient(p plugins.DestinationPlugin) *DestinationClient {
3131
}
3232
}
3333

34-
func (c *DestinationClient) Configure(ctx context.Context, spec spec.DestinationSpec) error {
34+
func (c *DestinationClient) Configure(ctx context.Context, s specs.DestinationSpec) error {
3535
if c.localClient != nil {
36-
return c.localClient.Configure(ctx, spec)
36+
return c.localClient.Configure(ctx, s)
3737
}
38-
b, err := yaml.Marshal(spec)
38+
b, err := yaml.Marshal(s)
3939
if err != nil {
4040
return fmt.Errorf("failed to marshal spec: %w", err)
4141
}
@@ -53,7 +53,7 @@ func (c *DestinationClient) GetExampleConfig(ctx context.Context) (string, error
5353
if err != nil {
5454
return "", err
5555
}
56-
return string(res.Config), nil
56+
return res.Config, nil
5757
}
5858

5959
func (c *DestinationClient) Save(ctx context.Context, msg *FetchResultMessage) error {

Diff for: ‎clients/source.go

+5-5
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ import (
1111

1212
"github.com/cloudquery/cq-plugin-sdk/internal/pb"
1313
"github.com/cloudquery/cq-plugin-sdk/schema"
14-
"github.com/cloudquery/cq-plugin-sdk/spec"
14+
"github.com/cloudquery/cq-plugin-sdk/specs"
1515
"github.com/pkg/errors"
1616
"github.com/vmihailenco/msgpack/v5"
1717
"github.com/xeipuuv/gojsonschema"
@@ -53,8 +53,8 @@ func (c *SourceClient) GetTables(ctx context.Context) ([]*schema.Table, error) {
5353
return tables, nil
5454
}
5555

56-
func (c *SourceClient) Configure(ctx context.Context, s spec.SourceSpec) (*gojsonschema.Result, error) {
57-
b, err := yaml.Marshal(s)
56+
func (c *SourceClient) Configure(ctx context.Context, spec specs.SourceSpec) (*gojsonschema.Result, error) {
57+
b, err := yaml.Marshal(spec)
5858
if err != nil {
5959
return nil, errors.Wrap(err, "failed to marshal source spec")
6060
}
@@ -82,14 +82,14 @@ func (c *SourceClient) GetExampleConfig(ctx context.Context) (string, error) {
8282
if err := t.Execute(&tpl, map[string]interface{}{
8383
"Name": res.Name,
8484
"Version": res.Version,
85-
"PluginExampleConfig": string(res.Config),
85+
"PluginExampleConfig": res.Config,
8686
}); err != nil {
8787
return "", fmt.Errorf("failed to generate example config: %w", err)
8888
}
8989
return tpl.String(), nil
9090
}
9191

92-
func (c *SourceClient) Fetch(ctx context.Context, spec spec.SourceSpec, res chan<- *FetchResultMessage) error {
92+
func (c *SourceClient) Fetch(ctx context.Context, spec specs.SourceSpec, res chan<- *FetchResultMessage) error {
9393
stream, err := c.pbClient.Fetch(ctx, &pb.Fetch_Request{})
9494
if err != nil {
9595
return fmt.Errorf("failed to fetch resources: %w", err)

Diff for: ‎internal/servers/destinations.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ import (
88
"github.com/cloudquery/cq-plugin-sdk/internal/pb"
99
"github.com/cloudquery/cq-plugin-sdk/plugins"
1010
"github.com/cloudquery/cq-plugin-sdk/schema"
11-
"github.com/cloudquery/cq-plugin-sdk/spec"
11+
"github.com/cloudquery/cq-plugin-sdk/specs"
1212
"google.golang.org/grpc/codes"
1313
"google.golang.org/grpc/status"
1414
"gopkg.in/yaml.v3"
@@ -20,7 +20,7 @@ type DestinationServer struct {
2020
}
2121

2222
func (s *DestinationServer) Configure(ctx context.Context, req *pb.Configure_Request) (*pb.Configure_Response, error) {
23-
var spec spec.DestinationSpec
23+
var spec specs.DestinationSpec
2424
if err := yaml.Unmarshal(req.Config, &spec); err != nil {
2525
return nil, status.Errorf(codes.InvalidArgument, "failed to unmarshal spec: %v", err)
2626
}

Diff for: ‎internal/servers/source.go

+6-4
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import (
66
"github.com/cloudquery/cq-plugin-sdk/internal/pb"
77
"github.com/cloudquery/cq-plugin-sdk/plugins"
88
"github.com/cloudquery/cq-plugin-sdk/schema"
9-
"github.com/cloudquery/cq-plugin-sdk/spec"
9+
"github.com/cloudquery/cq-plugin-sdk/specs"
1010
"github.com/pkg/errors"
1111
"github.com/vmihailenco/msgpack/v5"
1212
"gopkg.in/yaml.v3"
@@ -35,7 +35,7 @@ func (s *SourceServer) GetExampleConfig(context.Context, *pb.GetExampleConfig_Re
3535
}
3636

3737
func (s *SourceServer) Configure(ctx context.Context, req *pb.Configure_Request) (*pb.Configure_Response, error) {
38-
var spec spec.SourceSpec
38+
var spec specs.SourceSpec
3939
if err := yaml.Unmarshal(req.Config, &spec); err != nil {
4040
return nil, errors.Wrap(err, "failed to unmarshal config")
4141
}
@@ -67,9 +67,11 @@ func (s *SourceServer) Fetch(req *pb.Fetch_Request, stream pb.Source_FetchServer
6767
if err != nil {
6868
return errors.Wrap(err, "failed to marshal resource")
6969
}
70-
stream.Send(&pb.Fetch_Response{
70+
if err := stream.Send(&pb.Fetch_Response{
7171
Resource: b,
72-
})
72+
}); err != nil {
73+
return errors.Wrap(err, "failed to send resource")
74+
}
7375
}
7476
if fetchErr != nil {
7577
return fetchErr

Diff for: ‎plugins/destination.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import (
44
"context"
55

66
"github.com/cloudquery/cq-plugin-sdk/schema"
7-
"github.com/cloudquery/cq-plugin-sdk/spec"
7+
"github.com/cloudquery/cq-plugin-sdk/specs"
88
"github.com/rs/zerolog"
99
)
1010

@@ -13,7 +13,7 @@ type DestinationPluginOptions struct {
1313
}
1414

1515
type DestinationPlugin interface {
16-
Configure(ctx context.Context, spec spec.DestinationSpec) error
16+
Configure(ctx context.Context, spec specs.DestinationSpec) error
1717
CreateTables(ctx context.Context, table []*schema.Table) error
1818
Save(ctx context.Context, resources []*schema.Resource) error
1919
GetExampleConfig(ctx context.Context) string

Diff for: ‎plugins/source.go

+11-11
Original file line numberDiff line numberDiff line change
@@ -6,16 +6,16 @@ import (
66
"sync"
77
"time"
88

9-
_ "embed"
10-
119
"github.com/cloudquery/cq-plugin-sdk/helpers"
1210
"github.com/cloudquery/cq-plugin-sdk/helpers/limit"
1311
"github.com/cloudquery/cq-plugin-sdk/schema"
14-
"github.com/cloudquery/cq-plugin-sdk/spec"
12+
"github.com/cloudquery/cq-plugin-sdk/specs"
1513
"github.com/rs/zerolog"
1614
"github.com/thoas/go-funk"
1715
"github.com/xeipuuv/gojsonschema"
1816
"golang.org/x/sync/semaphore"
17+
18+
_ "embed"
1919
)
2020

2121
//go:embed source_schema.json
@@ -38,7 +38,7 @@ type SourcePlugin struct {
3838
// Version of the plugin
3939
Version string
4040
// Called upon configure call to validate and init configuration
41-
Configure func(context.Context, *SourcePlugin, spec.SourceSpec) (schema.ClientMeta, error)
41+
Configure func(context.Context, *SourcePlugin, specs.SourceSpec) (schema.ClientMeta, error)
4242
// Tables is all tables supported by this source plugin
4343
Tables []*schema.Table
4444
// JsonSchema for specific source plugin spec
@@ -50,11 +50,11 @@ type SourcePlugin struct {
5050

5151
// Internal fields set by configure
5252
clientMeta schema.ClientMeta
53-
spec *spec.SourceSpec
53+
spec *specs.SourceSpec
5454
}
5555

56-
func (p *SourcePlugin) Init(ctx context.Context, s spec.SourceSpec) (*gojsonschema.Result, error) {
57-
res, err := spec.ValidateSpec(sourceSchema, s)
56+
func (p *SourcePlugin) Init(ctx context.Context, spec specs.SourceSpec) (*gojsonschema.Result, error) {
57+
res, err := specs.ValidateSpec(sourceSchema, spec)
5858
if err != nil {
5959
return nil, err
6060
}
@@ -64,15 +64,15 @@ func (p *SourcePlugin) Init(ctx context.Context, s spec.SourceSpec) (*gojsonsche
6464
if p.Configure == nil {
6565
return nil, fmt.Errorf("configure function not defined")
6666
}
67-
p.clientMeta, err = p.Configure(ctx, p, s)
67+
p.clientMeta, err = p.Configure(ctx, p, spec)
6868
if err != nil {
6969
return res, fmt.Errorf("failed to configure source plugin: %w", err)
7070
}
71-
p.spec = &s
71+
p.spec = &spec
7272
return res, nil
7373
}
7474

75-
// Fetch fetches data acording to source configuration and
75+
// Fetch fetches data according to source configuration and
7676
func (p *SourcePlugin) Fetch(ctx context.Context, res chan<- *schema.Resource) error {
7777
if p.spec == nil {
7878
return fmt.Errorf("source plugin not initialized")
@@ -89,7 +89,7 @@ func (p *SourcePlugin) Fetch(ctx context.Context, res chan<- *schema.Resource) e
8989
maxGoroutines = limit.GetMaxGoRoutines()
9090
}
9191
p.Logger.Info().Uint64("max_goroutines", maxGoroutines).Msg("starting fetch")
92-
goroutinesSem := semaphore.NewWeighted(helpers.Uint64ToInt64(uint64(maxGoroutines)))
92+
goroutinesSem := semaphore.NewWeighted(helpers.Uint64ToInt64(maxGoroutines))
9393

9494
w := sync.WaitGroup{}
9595
for _, table := range p.Tables {

Diff for: ‎plugins/source_test.go renamed to ‎plugins/source_test.go.backup

File renamed without changes.

Diff for: ‎plugins/source_testing.go

+4-5
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import (
55
"testing"
66

77
"github.com/cloudquery/cq-plugin-sdk/schema"
8-
"github.com/cloudquery/cq-plugin-sdk/spec"
8+
"github.com/cloudquery/cq-plugin-sdk/specs"
99
"github.com/cloudquery/faker/v3"
1010
"github.com/georgysavva/scany/pgxscan"
1111
"github.com/xeipuuv/gojsonschema"
@@ -50,11 +50,11 @@ func TestResource(t *testing.T, tc ResourceTestCase) {
5050
resources := make(chan *schema.Resource)
5151
var fetchErr error
5252
var result *gojsonschema.Result
53-
var sourceSpec spec.SourceSpec
54-
if err := yaml.Unmarshal([]byte(tc.Config), &sourceSpec); err != nil {
53+
var spec specs.SourceSpec
54+
if err := yaml.Unmarshal([]byte(tc.Config), &spec); err != nil {
5555
t.Fatal("failed to unmarshal source spec:", err)
5656
}
57-
validationResult, err := tc.Plugin.Init(context.Background(), sourceSpec)
57+
validationResult, err := tc.Plugin.Init(context.Background(), spec)
5858
if err != nil {
5959
t.Fatal("failed to init plugin:", err)
6060
}
@@ -81,7 +81,6 @@ func TestResource(t *testing.T, tc ResourceTestCase) {
8181
func validateResource(t *testing.T, resource *schema.Resource) {
8282
t.Helper()
8383
for _, columnName := range resource.Table.Columns.Names() {
84-
8584
if resource.Get(columnName) == nil && !resource.Table.Columns.Get(columnName).IgnoreInTests {
8685
t.Errorf("table: %s with unset column %s", resource.Table.Name, columnName)
8786
}

Diff for: ‎schema/column.go

+22-23
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@ import (
66
"net"
77
"reflect"
88
"runtime"
9-
"sort"
109
"strings"
1110
"time"
1211

@@ -314,14 +313,14 @@ func (c Column) Meta() *ColumnMeta {
314313
}
315314
}
316315

317-
func (c Column) signature() string {
318-
return strings.Join([]string{
319-
"c",
320-
c.Name,
321-
c.Type.String(),
322-
fmt.Sprintf("%t;%t", c.CreationOptions.Unique, c.CreationOptions.NotNull),
323-
}, "\n")
324-
}
316+
// func (c Column) signature() string {
317+
// return strings.Join([]string{
318+
// "c",
319+
// c.Name,
320+
// c.Type.String(),
321+
// fmt.Sprintf("%t;%t", c.CreationOptions.Unique, c.CreationOptions.NotNull),
322+
// }, "\n")
323+
// }
325324

326325
func SetColumnMeta(c Column, m *ColumnMeta) Column {
327326
c.meta = m
@@ -368,18 +367,18 @@ func (c ColumnList) Get(name string) *Column {
368367
return nil
369368
}
370369

371-
func (c ColumnList) signature() string {
372-
names := make([]string, len(c))
373-
nameVsColumn := make(map[string]*Column, len(c))
374-
for i := range c {
375-
names[i] = c[i].Name
376-
nameVsColumn[c[i].Name] = &c[i]
377-
}
378-
sort.Strings(names)
370+
// func (c ColumnList) signature() string {
371+
// names := make([]string, len(c))
372+
// nameVsColumn := make(map[string]*Column, len(c))
373+
// for i := range c {
374+
// names[i] = c[i].Name
375+
// nameVsColumn[c[i].Name] = &c[i]
376+
// }
377+
// sort.Strings(names)
379378

380-
sigs := make([]string, len(c))
381-
for i, colName := range names {
382-
sigs[i] = nameVsColumn[colName].signature()
383-
}
384-
return strings.Join(sigs, "\n")
385-
}
379+
// sigs := make([]string, len(c))
380+
// for i, colName := range names {
381+
// sigs[i] = nameVsColumn[colName].signature()
382+
// }
383+
// return strings.Join(sigs, "\n")
384+
// }

Diff for: ‎schema/resolvers_test.go renamed to ‎schema/resolvers_test.go.backup

File renamed without changes.

Diff for: ‎schema/table_test.go renamed to ‎schema/table_test.go.backup

File renamed without changes.

Diff for: ‎serve/enum.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,6 @@ func (a *enum) Set(p string) error {
4040
return nil
4141
}
4242

43-
func (a *enum) Type() string {
43+
func (*enum) Type() string {
4444
return "string"
4545
}

Diff for: ‎spec/connection.go renamed to ‎specs/connection.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package spec
1+
package specs
22

33
type ConnectionSpec struct {
44
Source string `yaml:"source"`

Diff for: ‎spec/destination.go renamed to ‎specs/destination.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package spec
1+
package specs
22

33
import "gopkg.in/yaml.v3"
44

Diff for: ‎spec/source.go renamed to ‎specs/source.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package spec
1+
package specs
22

33
import (
44
"strings"

Diff for: ‎spec/spec.go renamed to ‎specs/spec.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package spec
1+
package specs
22

33
import (
44
"fmt"

Diff for: ‎spec/spec_reader.go renamed to ‎specs/spec_reader.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package spec
1+
package specs
22

33
import (
44
"fmt"
@@ -80,7 +80,7 @@ func (s *SpecReader) GetConnectionByName(name string) ConnectionSpec {
8080
}
8181

8282
func (s *SpecReader) Connections() []ConnectionSpec {
83-
var connections []ConnectionSpec
83+
connections := make([]ConnectionSpec, 0, len(s.connections))
8484
for _, spec := range s.connections {
8585
connections = append(connections, spec)
8686
}

Diff for: ‎spec/spec_reader_test.go renamed to ‎specs/spec_reader_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package spec
1+
package specs
22

33
import (
44
"reflect"

Diff for: ‎spec/testdata/aws.cq.yml renamed to ‎specs/testdata/aws.cq.yml

File renamed without changes.

Diff for: ‎spec/validate.go renamed to ‎specs/validate.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package spec
1+
package specs
22

33
import "github.com/xeipuuv/gojsonschema"
44

0 commit comments

Comments
 (0)