Skip to content

Commit cc0a7c7

Browse files
kylemcckylemcc
authored
Merge pull request #80 from OpsHelmInc/update-types
update custom types
2 parents 90bd547 + d0ae9c1 commit cc0a7c7

30 files changed

+362
-318
lines changed

‎client/testing.go

+12-12
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,8 @@ package client
22

33
import (
44
"context"
5-
"encoding/json"
65
"os"
76
"slices"
8-
"strings"
97
"sync"
108
"testing"
119
"time"
@@ -155,24 +153,26 @@ func validateNoEmptyColumnsExcept(t *testing.T, tables schema.Tables, messages m
155153
}
156154
}
157155

158-
func validateTagData(t *testing.T, tables schema.Tables, messages message.SyncMessages) {
156+
func validateTagData(t *testing.T, tables schema.Tables, _ message.SyncMessages) {
159157
tablesWithIssues := []string{}
160158
for _, table := range tables.FlattenTables() {
161159
index := table.Columns.Index("tags")
162160
if index == -1 {
163161
continue
164162
}
165-
records := messages.GetInserts().GetRecordsForTable(table)
166-
for _, resource := range records {
167-
arr := resource.Column(index)
168-
for i := 0; i < arr.Len(); i++ {
169-
val := arr.GetOneForMarshal(i).(json.RawMessage)
170-
if strings.HasPrefix(string(val), "[") && strings.HasSuffix(string(val), "]") {
171-
tablesWithIssues = append(tablesWithIssues, table.Name)
172-
break
163+
/*
164+
records := messages.GetInserts().GetRecordsForTable(table)
165+
for _, resource := range records {
166+
arr := resource.Column(index)
167+
for i := 0; i < arr.Len(); i++ {
168+
val := arr.GetOneForMarshal(i).(json.RawMessage)
169+
if strings.HasPrefix(string(val), "[") && strings.HasSuffix(string(val), "]") {
170+
tablesWithIssues = append(tablesWithIssues, table.Name)
171+
break
172+
}
173173
}
174174
}
175-
}
175+
*/
176176
}
177177
if len(tablesWithIssues) > 0 {
178178
t.Fatalf("found improperly structured tags in %v", tablesWithIssues)

‎docs/tables/aws_ecs_task_definitions.md

+1-24
Original file line numberDiff line numberDiff line change
@@ -17,27 +17,4 @@ The primary key for this table is **_cq_id**.
1717
|arn|`utf8`|
1818
|tags|`json`|
1919
|oh_resource_type|`utf8`|
20-
|compatibilities|`list<item: utf8, nullable>`|
21-
|container_definitions|`json`|
22-
|cpu|`utf8`|
23-
|deregistered_at|`timestamp[us, tz=UTC]`|
24-
|ephemeral_storage|`json`|
25-
|execution_role_arn|`utf8`|
26-
|family|`utf8`|
27-
|inference_accelerators|`json`|
28-
|ipc_mode|`utf8`|
29-
|memory|`utf8`|
30-
|network_mode|`utf8`|
31-
|pid_mode|`utf8`|
32-
|placement_constraints|`json`|
33-
|proxy_configuration|`json`|
34-
|registered_at|`timestamp[us, tz=UTC]`|
35-
|registered_by|`utf8`|
36-
|requires_attributes|`json`|
37-
|requires_compatibilities|`list<item: utf8, nullable>`|
38-
|revision|`int64`|
39-
|runtime_platform|`json`|
40-
|status|`utf8`|
41-
|task_definition_arn|`utf8`|
42-
|task_role_arn|`utf8`|
43-
|volumes|`json`|
20+
|task_definition|`json`|

‎docs/tables/aws_elbv1_load_balancers.md

+6-2
Original file line numberDiff line numberDiff line change
@@ -37,5 +37,9 @@ The following tables depend on aws_elbv1_load_balancers:
3737
|source_security_group|`json`|
3838
|subnets|`list<item: utf8, nullable>`|
3939
|vpc_id|`utf8`|
40-
|tags|`json`|
41-
|attributes|`json`|
40+
|access_log|`json`|
41+
|additional_attributes|`json`|
42+
|connection_draining|`json`|
43+
|connection_settings|`json`|
44+
|cross_zone_load_balancing|`json`|
45+
|tags|`json`|

‎docs/tables/aws_iam_roles.md

+2-8
Original file line numberDiff line numberDiff line change
@@ -23,11 +23,5 @@ The following tables depend on aws_iam_roles:
2323
|tags|`json`|
2424
|arn|`utf8`|
2525
|oh_resource_type|`utf8`|
26-
|create_date|`timestamp[us, tz=UTC]`|
27-
|path|`utf8`|
28-
|role_id|`utf8`|
29-
|role_name|`utf8`|
30-
|description|`utf8`|
31-
|max_session_duration|`int64`|
32-
|permissions_boundary|`json`|
33-
|role_last_used|`json`|
26+
|role|`json`|
27+
|policies|`json`|

‎docs/tables/aws_iam_user_access_keys.md

+3-3
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,9 @@ This table depends on [aws_iam_users](aws_iam_users.md).
2020
|user_arn|`utf8`|
2121
|access_key_id|`utf8`|
2222
|user_id|`utf8`|
23-
|last_used|`timestamp[us, tz=UTC]`|
24-
|last_used_service_name|`utf8`|
2523
|create_date|`timestamp[us, tz=UTC]`|
2624
|status|`utf8`|
2725
|user_name|`utf8`|
28-
|last_rotated|`timestamp[us, tz=UTC]`|
26+
|last_used_date|`timestamp[us, tz=UTC]`|
27+
|last_used_region|`utf8`|
28+
|last_used_service_name|`utf8`|

‎docs/tables/aws_iam_users.md

+9-1
Original file line numberDiff line numberDiff line change
@@ -32,4 +32,12 @@ The following tables depend on aws_iam_users:
3232
|user_id|`utf8`|
3333
|user_name|`utf8`|
3434
|password_last_used|`timestamp[us, tz=UTC]`|
35-
|permissions_boundary|`json`|
35+
|permissions_boundary|`json`|
36+
|inline_policies|`json`|
37+
|attached_policies|`json`|
38+
|groups|`json`|
39+
|mfa_devices|`json`|
40+
|login_profile|`json`|
41+
|password_set|`bool`|
42+
|mfa_active|`bool`|
43+
|access_keys|`json`|

‎docs/tables/aws_kinesis_streams.md

+2-2
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@ The primary key for this table is **_cq_id**.
1515
|account_id|`utf8`|
1616
|region|`utf8`|
1717
|arn|`utf8`|
18-
|tags|`json`|
1918
|oh_resource_type|`utf8`|
2019
|enhanced_monitoring|`json`|
2120
|open_shard_count|`int64`|
@@ -27,4 +26,5 @@ The primary key for this table is **_cq_id**.
2726
|consumer_count|`int64`|
2827
|encryption_type|`utf8`|
2928
|key_id|`utf8`|
30-
|stream_mode_details|`json`|
29+
|stream_mode_details|`json`|
30+
|tags|`json`|

‎docs/tables/aws_sqs_queues.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@ The primary key for this table is **_cq_id**.
1818
|policy|`json`|
1919
|redrive_policy|`json`|
2020
|redrive_allow_policy|`json`|
21-
|tags|`json`|
2221
|oh_resource_type|`utf8`|
2322
|url|`utf8`|
2423
|approximate_number_of_messages|`int64`|
@@ -38,4 +37,5 @@ The primary key for this table is **_cq_id**.
3837
|content_based_deduplication|`bool`|
3938
|deduplication_scope|`utf8`|
4039
|fifo_throughput_limit|`utf8`|
40+
|tags|`json`|
4141
|unknown_fields|`json`|

‎resources/services/ecs/task_definitions.go

+4-4
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ import (
1313
"github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema"
1414
"github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers"
1515
sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types"
16-
"github.com/OpsHelmInc/cloudquery/v2/resources/services/ecs/models"
16+
"github.com/OpsHelmInc/ohaws"
1717
)
1818

1919
func TaskDefinitions() *schema.Table {
@@ -24,7 +24,7 @@ func TaskDefinitions() *schema.Table {
2424
Resolver: fetchEcsTaskDefinitions,
2525
PreResourceResolver: getTaskDefinition,
2626
Multiplex: client.ServiceAccountRegionMultiplexer(tableName, "ecs"),
27-
Transform: transformers.TransformWithStruct(&types.TaskDefinition{}),
27+
Transform: transformers.TransformWithStruct(&ohaws.TaskDefinition{}),
2828
Columns: []schema.Column{
2929
client.DefaultAccountIDColumn(false),
3030
client.DefaultRegionColumn(false),
@@ -78,14 +78,14 @@ func getTaskDefinition(ctx context.Context, meta schema.ClientMeta, resource *sc
7878
if describeTaskDefinitionOutput.TaskDefinition == nil {
7979
return errors.New("nil TaskDefinition encountered")
8080
}
81-
resource.Item = models.TaskDefinitionWrapper{
81+
resource.Item = ohaws.TaskDefinition{
8282
TaskDefinition: describeTaskDefinitionOutput.TaskDefinition,
8383
Tags: describeTaskDefinitionOutput.Tags,
8484
}
8585
return nil
8686
}
8787

8888
func resolveEcsTaskDefinitionTags(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error {
89-
r := resource.Item.(models.TaskDefinitionWrapper)
89+
r := resource.Item.(ohaws.TaskDefinition)
9090
return resource.Set(c.Name, client.TagsToMap(r.Tags))
9191
}

‎resources/services/elbv1/load_balancer_policies.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ import (
1111
"github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema"
1212
"github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers"
1313
sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types"
14-
"github.com/OpsHelmInc/cloudquery/v2/resources/services/elbv1/models"
14+
"github.com/OpsHelmInc/ohaws"
1515
)
1616

1717
func loadBalancerPolicies() *schema.Table {
@@ -45,7 +45,7 @@ func loadBalancerPolicies() *schema.Table {
4545
}
4646

4747
func fetchElbv1LoadBalancerPolicies(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error {
48-
r := parent.Item.(models.ELBv1LoadBalancerWrapper)
48+
r := parent.Item.(ohaws.LoadBalancerV1)
4949
cl := meta.(*client.Client)
5050
svc := cl.Services(client.AWSServiceElasticloadbalancing).Elasticloadbalancing
5151
response, err := svc.DescribeLoadBalancerPolicies(ctx, &elbv1.DescribeLoadBalancerPoliciesInput{LoadBalancerName: r.LoadBalancerName}, func(options *elbv1.Options) {

‎resources/services/elbv1/load_balancers.go

+11-11
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ import (
1010
"github.com/OpsHelmInc/cloudquery/v2/client"
1111
"github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema"
1212
"github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers"
13-
"github.com/OpsHelmInc/cloudquery/v2/resources/services/elbv1/models"
13+
"github.com/OpsHelmInc/ohaws"
1414
)
1515

1616
func LoadBalancers() *schema.Table {
@@ -20,7 +20,7 @@ func LoadBalancers() *schema.Table {
2020
Description: `https://docs.aws.amazon.com/elasticloadbalancing/2012-06-01/APIReference/API_LoadBalancerDescription.html`,
2121
Resolver: fetchElbv1LoadBalancers,
2222
Multiplex: client.ServiceAccountRegionMultiplexer(tableName, "elasticloadbalancing"),
23-
Transform: transformers.TransformWithStruct(&models.ELBv1LoadBalancerWrapper{}, transformers.WithUnwrapAllEmbeddedStructs()),
23+
Transform: transformers.TransformWithStruct(&ohaws.LoadBalancerV1{}, transformers.WithUnwrapAllEmbeddedStructs()),
2424
Columns: []schema.Column{
2525
client.DefaultAccountIDColumn(false),
2626
client.DefaultRegionColumn(false),
@@ -39,34 +39,34 @@ func LoadBalancers() *schema.Table {
3939
}
4040

4141
func fetchElbv1LoadBalancers(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error {
42-
cl := meta.(*client.Client)
43-
svc := cl.Services(client.AWSServiceElasticloadbalancing).Elasticloadbalancing
42+
c := meta.(*client.Client)
43+
svc := c.Services(client.AWSServiceElasticloadbalancing).Elasticloadbalancing
4444
processLoadBalancers := func(loadBalancers []types.LoadBalancerDescription) error {
4545
tagsCfg := &elbv1.DescribeTagsInput{LoadBalancerNames: make([]string, 0, len(loadBalancers))}
4646
for _, lb := range loadBalancers {
4747
tagsCfg.LoadBalancerNames = append(tagsCfg.LoadBalancerNames, *lb.LoadBalancerName)
4848
}
4949
tagsResponse, err := svc.DescribeTags(ctx, tagsCfg, func(options *elbv1.Options) {
50-
options.Region = cl.Region
50+
options.Region = c.Region
5151
})
5252
if err != nil {
5353
return err
5454
}
5555
for _, lb := range loadBalancers {
5656
loadBalancerAttributes, err := svc.DescribeLoadBalancerAttributes(ctx, &elbv1.DescribeLoadBalancerAttributesInput{LoadBalancerName: lb.LoadBalancerName}, func(options *elbv1.Options) {
57-
options.Region = cl.Region
57+
options.Region = c.Region
5858
})
5959
if err != nil {
60-
if cl.IsNotFoundError(err) {
60+
if c.IsNotFoundError(err) {
6161
continue
6262
}
6363
return err
6464
}
6565

66-
wrapper := models.ELBv1LoadBalancerWrapper{
66+
wrapper := ohaws.LoadBalancerV1{
6767
LoadBalancerDescription: lb,
6868
Tags: client.TagsToMap(getTagsByLoadBalancerName(*lb.LoadBalancerName, tagsResponse.TagDescriptions)),
69-
Attributes: loadBalancerAttributes.LoadBalancerAttributes,
69+
LoadBalancerAttributes: loadBalancerAttributes.LoadBalancerAttributes,
7070
}
7171

7272
res <- wrapper
@@ -76,7 +76,7 @@ func fetchElbv1LoadBalancers(ctx context.Context, meta schema.ClientMeta, parent
7676
paginator := elbv1.NewDescribeLoadBalancersPaginator(svc, &elbv1.DescribeLoadBalancersInput{})
7777
for paginator.HasMorePages() {
7878
page, err := paginator.NextPage(ctx, func(options *elbv1.Options) {
79-
options.Region = cl.Region
79+
options.Region = c.Region
8080
})
8181
if err != nil {
8282
return err
@@ -109,6 +109,6 @@ func getTagsByLoadBalancerName(id string, tagsResponse []types.TagDescription) [
109109

110110
func resolveLoadBalancerARN() schema.ColumnResolver {
111111
return client.ResolveARN(client.ElasticLoadBalancingService, func(resource *schema.Resource) ([]string, error) {
112-
return []string{"loadbalancer", *resource.Item.(models.ELBv1LoadBalancerWrapper).LoadBalancerName}, nil
112+
return []string{"loadbalancer", *resource.Item.(ohaws.LoadBalancerV1).LoadBalancerName}, nil
113113
})
114114
}

‎resources/services/iam/mfa_devices.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
"github.com/OpsHelmInc/cloudquery/v2/client"
1111
"github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema"
1212
"github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers"
13+
"github.com/OpsHelmInc/ohaws"
1314
)
1415

1516
func mfaDevices() *schema.Table {
@@ -31,7 +32,7 @@ func mfaDevices() *schema.Table {
3132
}
3233

3334
func fetchIamMfaDevices(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error {
34-
r := parent.Item.(*types.User)
35+
r := parent.Item.(*ohaws.User)
3536
cl := meta.(*client.Client)
3637
svc := cl.Services(client.AWSServiceIam).Iam
3738
paginator := iam.NewListMFADevicesPaginator(svc, &iam.ListMFADevicesInput{

‎resources/services/iam/role_attached_policies.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
"github.com/OpsHelmInc/cloudquery/v2/client"
1111
"github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema"
1212
"github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers"
13+
"github.com/OpsHelmInc/ohaws"
1314
)
1415

1516
func roleAttachedPolicies() *schema.Table {
@@ -32,7 +33,7 @@ func roleAttachedPolicies() *schema.Table {
3233
}
3334

3435
func fetchIamRoleAttachedPolicies(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error {
35-
p := parent.Item.(*types.Role)
36+
p := parent.Item.(*ohaws.Role)
3637
cl := meta.(*client.Client)
3738
svc := cl.Services(client.AWSServiceIam).Iam
3839
config := iam.ListAttachedRolePoliciesInput{

‎resources/services/iam/role_policies.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,12 @@ import (
77

88
"github.com/apache/arrow/go/v16/arrow"
99
"github.com/aws/aws-sdk-go-v2/service/iam"
10-
"github.com/aws/aws-sdk-go-v2/service/iam/types"
1110

1211
"github.com/OpsHelmInc/cloudquery/v2/client"
1312
"github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema"
1413
"github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers"
1514
sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types"
15+
"github.com/OpsHelmInc/ohaws"
1616
)
1717

1818
func rolePolicies() *schema.Table {
@@ -43,7 +43,7 @@ func rolePolicies() *schema.Table {
4343
func fetchIamRolePolicies(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error {
4444
cl := meta.(*client.Client)
4545
svc := cl.Services(client.AWSServiceIam).Iam
46-
role := parent.Item.(*types.Role)
46+
role := parent.Item.(*ohaws.Role)
4747
paginator := iam.NewListRolePoliciesPaginator(svc, &iam.ListRolePoliciesInput{
4848
RoleName: role.RoleName,
4949
})
@@ -66,7 +66,7 @@ func getRolePolicy(ctx context.Context, meta schema.ClientMeta, resource *schema
6666
cl := meta.(*client.Client)
6767
svc := cl.Services(client.AWSServiceIam).Iam
6868
p := resource.Item.(string)
69-
role := resource.Parent.Item.(*types.Role)
69+
role := resource.Parent.Item.(*ohaws.Role)
7070

7171
policyResult, err := svc.GetRolePolicy(ctx, &iam.GetRolePolicyInput{PolicyName: &p, RoleName: role.RoleName}, func(options *iam.Options) {
7272
options.Region = cl.Region

0 commit comments

Comments
 (0)