Skip to content

Commit f10fb29

Browse files
kylemcckylemcc
authored
Merge pull request #72 from OpsHelmInc/update-data-structures
Update data structures
2 parents 075cda1 + 2c33ccb commit f10fb29

19 files changed

+120
-234
lines changed

‎codegen/recipes/elbv1.go

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,16 +8,17 @@ import (
88
"github.com/cloudquery/plugin-sdk/codegen"
99
"github.com/cloudquery/plugin-sdk/schema"
1010

11-
"github.com/OpsHelmInc/cloudquery/resources/services/elbv1/models"
11+
"github.com/OpsHelmInc/ohaws"
1212
)
1313

1414
func ELBv1Resources() []*Resource {
1515
resources := []*Resource{
1616
{
17-
SubService: "load_balancers",
18-
Struct: &models.ELBv1LoadBalancerWrapper{},
19-
Description: "https://docs.aws.amazon.com/elasticloadbalancing/2012-06-01/APIReference/API_LoadBalancerDescription.html",
20-
SkipFields: []string{},
17+
SubService: "load_balancers",
18+
Struct: &ohaws.LoadBalancerV1{},
19+
UnwrapEmbeddedStructs: true,
20+
Description: "https://docs.aws.amazon.com/elasticloadbalancing/2012-06-01/APIReference/API_LoadBalancerDescription.html",
21+
SkipFields: []string{},
2122
ExtraColumns: append(
2223
defaultRegionalColumns,
2324
[]codegen.ColumnDefinition{

‎codegen/recipes/sns.go

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -4,14 +4,14 @@ import (
44
"github.com/cloudquery/plugin-sdk/codegen"
55
"github.com/cloudquery/plugin-sdk/schema"
66

7-
"github.com/OpsHelmInc/cloudquery/resources/services/sns/models"
7+
"github.com/OpsHelmInc/ohaws"
88
)
99

1010
func SNSResources() []*Resource {
1111
resources := []*Resource{
1212
{
1313
SubService: "subscriptions",
14-
Struct: &models.Subscription{},
14+
Struct: &ohaws.Subscription{},
1515
SkipFields: []string{"SubscriptionArn", "DeliveryPolicy", "EffectiveDeliveryPolicy", "FilterPolicy", "RedrivePolicy"},
1616
PreResourceResolver: "getSnsSubscription",
1717
ExtraColumns: append(
@@ -53,7 +53,7 @@ func SNSResources() []*Resource {
5353

5454
{
5555
SubService: "topics",
56-
Struct: &models.Topic{},
56+
Struct: &ohaws.Topic{},
5757
SkipFields: []string{"Arn", "Policy", "EffectiveDeliveryPolicy", "DeliveryPolicy"},
5858
PreResourceResolver: "getTopic",
5959
ExtraColumns: append(
@@ -65,11 +65,6 @@ func SNSResources() []*Resource {
6565
Resolver: `schema.PathResolver("Arn")`,
6666
Options: schema.ColumnCreationOptions{PrimaryKey: true},
6767
},
68-
{
69-
Name: "tags",
70-
Type: schema.TypeJSON,
71-
Resolver: `resolveSnsTopicTags`,
72-
},
7368
{
7469
Name: "delivery_policy",
7570
Type: schema.TypeJSON,

‎codegen/recipes/sqs.go

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4,14 +4,14 @@ import (
44
"github.com/cloudquery/plugin-sdk/codegen"
55
"github.com/cloudquery/plugin-sdk/schema"
66

7-
"github.com/OpsHelmInc/cloudquery/resources/services/sqs/models"
7+
"github.com/OpsHelmInc/ohaws"
88
)
99

1010
func SQSResources() []*Resource {
1111
resources := []*Resource{
1212
{
1313
SubService: "queues",
14-
Struct: &models.Queue{},
14+
Struct: &ohaws.Queue{},
1515
SkipFields: []string{"Arn", "Policy", "RedriveAllowPolicy", "RedrivePolicy"},
1616
PreResourceResolver: "getQueue",
1717
ExtraColumns: append(
@@ -23,11 +23,6 @@ func SQSResources() []*Resource {
2323
Resolver: `schema.PathResolver("Arn")`,
2424
Options: schema.ColumnCreationOptions{PrimaryKey: true},
2525
},
26-
{
27-
Name: "tags",
28-
Type: schema.TypeJSON,
29-
Resolver: `resolveSqsQueueTags`,
30-
},
3126
{
3227
Name: "policy",
3328
Type: schema.TypeJSON,

‎docs/tables/aws_elbv1_load_balancers.md

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,5 +36,9 @@ The following tables depend on aws_elbv1_load_balancers:
3636
|source_security_group|JSON|
3737
|subnets|StringArray|
3838
|vpc_id|String|
39-
|tags|JSON|
40-
|attributes|JSON|
39+
|access_log|JSON|
40+
|additional_attributes|JSON|
41+
|connection_draining|JSON|
42+
|connection_settings|JSON|
43+
|cross_zone_load_balancing|JSON|
44+
|tags|JSON|

‎docs/tables/aws_sns_topics.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@ The primary key for this table is **arn**.
1616
|account_id|String|
1717
|region|String|
1818
|arn (PK)|String|
19-
|tags|JSON|
2019
|delivery_policy|JSON|
2120
|policy|JSON|
2221
|effective_delivery_policy|JSON|
@@ -29,4 +28,5 @@ The primary key for this table is **arn**.
2928
|kms_master_key_id|String|
3029
|fifo_topic|Bool|
3130
|content_based_deduplication|Bool|
31+
|tags|JSON|
3232
|unknown_fields|JSON|

‎docs/tables/aws_sqs_queues.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@ The primary key for this table is **arn**.
1616
|account_id|String|
1717
|region|String|
1818
|arn (PK)|String|
19-
|tags|JSON|
2019
|policy|JSON|
2120
|redrive_policy|JSON|
2221
|redrive_allow_policy|JSON|
@@ -39,4 +38,5 @@ The primary key for this table is **arn**.
3938
|content_based_deduplication|Bool|
4039
|deduplication_scope|String|
4140
|fifo_throughput_limit|String|
41+
|tags|JSON|
4242
|unknown_fields|JSON|

‎resources/services/ecs/task_definitions_fetch.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ import (
1010
"github.com/cloudquery/plugin-sdk/schema"
1111

1212
"github.com/OpsHelmInc/cloudquery/client"
13-
"github.com/OpsHelmInc/cloudquery/resources/services/ecs/models"
13+
"github.com/OpsHelmInc/ohaws"
1414
)
1515

1616
func fetchEcsTaskDefinitions(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error {
@@ -47,14 +47,14 @@ func getTaskDefinition(ctx context.Context, meta schema.ClientMeta, resource *sc
4747
if describeTaskDefinitionOutput.TaskDefinition == nil {
4848
return errors.New("nil TaskDefinition encountered")
4949
}
50-
resource.Item = models.TaskDefinitionWrapper{
50+
resource.Item = ohaws.TaskDefinition{
5151
TaskDefinition: describeTaskDefinitionOutput.TaskDefinition,
5252
Tags: describeTaskDefinitionOutput.Tags,
5353
}
5454
return nil
5555
}
5656

5757
func resolveEcsTaskDefinitionTags(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error {
58-
r := resource.Item.(models.TaskDefinitionWrapper)
58+
r := resource.Item.(ohaws.TaskDefinition)
5959
return resource.Set(c.Name, client.TagsToMap(r.Tags))
6060
}

‎resources/services/efs/filesystems_fetch.go

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3,27 +3,26 @@ package efs
33
import (
44
"context"
55

6-
"github.com/OpsHelmInc/cloudquery/client"
7-
"github.com/aws/aws-sdk-go-v2/aws"
86
"github.com/aws/aws-sdk-go-v2/service/efs"
97
"github.com/aws/aws-sdk-go-v2/service/efs/types"
108
"github.com/cloudquery/plugin-sdk/schema"
9+
10+
"github.com/OpsHelmInc/cloudquery/client"
1111
)
1212

1313
func fetchEfsFilesystems(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error {
1414
var config efs.DescribeFileSystemsInput
1515
c := meta.(*client.Client)
1616
svc := c.Services().Efs
17-
for {
18-
response, err := svc.DescribeFileSystems(ctx, &config)
17+
paginator := efs.NewDescribeFileSystemsPaginator(svc, &config)
18+
for paginator.HasMorePages() {
19+
page, err := paginator.NextPage(ctx, func(options *efs.Options) {
20+
options.Region = c.Region
21+
})
1922
if err != nil {
2023
return err
2124
}
22-
res <- response.FileSystems
23-
if aws.ToString(response.Marker) == "" {
24-
break
25-
}
26-
config.Marker = response.NextMarker
25+
res <- page.FileSystems
2726
}
2827
return nil
2928
}

‎resources/services/elbv1/load_balancers.go

Lines changed: 24 additions & 4 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎resources/services/elbv1/load_balancers_fetch.go

Lines changed: 23 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,12 @@ package elbv1
33
import (
44
"context"
55

6-
"github.com/OpsHelmInc/cloudquery/client"
7-
"github.com/OpsHelmInc/cloudquery/resources/services/elbv1/models"
8-
"github.com/aws/aws-sdk-go-v2/aws"
96
elbv1 "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancing"
107
"github.com/aws/aws-sdk-go-v2/service/elasticloadbalancing/types"
118
"github.com/cloudquery/plugin-sdk/schema"
9+
10+
"github.com/OpsHelmInc/cloudquery/client"
11+
"github.com/OpsHelmInc/ohaws"
1212
)
1313

1414
func fetchElbv1LoadBalancers(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error {
@@ -19,60 +19,60 @@ func fetchElbv1LoadBalancers(ctx context.Context, meta schema.ClientMeta, parent
1919
for _, lb := range loadBalancers {
2020
tagsCfg.LoadBalancerNames = append(tagsCfg.LoadBalancerNames, *lb.LoadBalancerName)
2121
}
22-
tagsResponse, err := svc.DescribeTags(ctx, tagsCfg)
22+
tagsResponse, err := svc.DescribeTags(ctx, tagsCfg, func(options *elbv1.Options) {
23+
options.Region = c.Region
24+
})
2325
if err != nil {
2426
return err
2527
}
2628
for _, lb := range loadBalancers {
27-
loadBalancerAttributes, err := svc.DescribeLoadBalancerAttributes(ctx, &elbv1.DescribeLoadBalancerAttributesInput{LoadBalancerName: lb.LoadBalancerName})
29+
loadBalancerAttributes, err := svc.DescribeLoadBalancerAttributes(ctx, &elbv1.DescribeLoadBalancerAttributesInput{LoadBalancerName: lb.LoadBalancerName}, func(options *elbv1.Options) {
30+
options.Region = c.Region
31+
})
2832
if err != nil {
2933
if c.IsNotFoundError(err) {
3034
continue
3135
}
3236
return err
3337
}
3438

35-
wrapper := models.ELBv1LoadBalancerWrapper{
39+
wrapper := ohaws.LoadBalancerV1{
3640
LoadBalancerDescription: lb,
3741
Tags: client.TagsToMap(getTagsByLoadBalancerName(*lb.LoadBalancerName, tagsResponse.TagDescriptions)),
38-
Attributes: loadBalancerAttributes.LoadBalancerAttributes,
42+
LoadBalancerAttributes: loadBalancerAttributes.LoadBalancerAttributes,
3943
}
4044

4145
res <- wrapper
4246
}
4347
return nil
4448
}
45-
46-
var config elbv1.DescribeLoadBalancersInput
47-
for {
48-
response, err := svc.DescribeLoadBalancers(ctx, &config)
49+
paginator := elbv1.NewDescribeLoadBalancersPaginator(svc, &elbv1.DescribeLoadBalancersInput{})
50+
for paginator.HasMorePages() {
51+
page, err := paginator.NextPage(ctx, func(options *elbv1.Options) {
52+
options.Region = c.Region
53+
})
4954
if err != nil {
5055
return err
5156
}
5257

53-
for i := 0; i < len(response.LoadBalancerDescriptions); i += 20 {
58+
for i := 0; i < len(page.LoadBalancerDescriptions); i += 20 {
5459
end := i + 20
5560

56-
if end > len(response.LoadBalancerDescriptions) {
57-
end = len(response.LoadBalancerDescriptions)
61+
if end > len(page.LoadBalancerDescriptions) {
62+
end = len(page.LoadBalancerDescriptions)
5863
}
59-
loadBalancers := response.LoadBalancerDescriptions[i:end]
64+
loadBalancers := page.LoadBalancerDescriptions[i:end]
6065
if err := processLoadBalancers(loadBalancers); err != nil {
6166
return err
6267
}
6368
}
64-
65-
if aws.ToString(response.NextMarker) == "" {
66-
break
67-
}
68-
config.Marker = response.NextMarker
6969
}
7070

7171
return nil
7272
}
7373

7474
func fetchElbv1LoadBalancerPolicies(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error {
75-
r := parent.Item.(models.ELBv1LoadBalancerWrapper)
75+
r := parent.Item.(ohaws.LoadBalancerV1)
7676
c := meta.(*client.Client)
7777
svc := c.Services().Elasticloadbalancing
7878
response, err := svc.DescribeLoadBalancerPolicies(ctx, &elbv1.DescribeLoadBalancerPoliciesInput{LoadBalancerName: r.LoadBalancerName})
@@ -82,6 +82,7 @@ func fetchElbv1LoadBalancerPolicies(ctx context.Context, meta schema.ClientMeta,
8282
res <- response.PolicyDescriptions
8383
return nil
8484
}
85+
8586
func resolveElbv1loadBalancerPolicyAttributeDescriptions(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error {
8687
r := resource.Item.(types.PolicyDescription)
8788

@@ -103,6 +104,6 @@ func getTagsByLoadBalancerName(id string, tagsResponse []types.TagDescription) [
103104

104105
func resolveLoadBalancerARN() schema.ColumnResolver {
105106
return client.ResolveARN(client.ElasticLoadBalancingService, func(resource *schema.Resource) ([]string, error) {
106-
return []string{"loadbalancer", *resource.Item.(models.ELBv1LoadBalancerWrapper).LoadBalancerName}, nil
107+
return []string{"loadbalancer", *resource.Item.(ohaws.LoadBalancerV1).LoadBalancerName}, nil
107108
})
108109
}

‎resources/services/elbv1/models/elbv1.go

Lines changed: 0 additions & 9 deletions
This file was deleted.

0 commit comments

Comments
 (0)