diff --git a/providers/aws/resources/aws_config.go b/providers/aws/resources/aws_config.go index 2c56441435..2cef494d12 100644 --- a/providers/aws/resources/aws_config.go +++ b/providers/aws/resources/aws_config.go @@ -241,30 +241,36 @@ func (a *mqlAwsConfig) getRules(conn *connection.AwsConnection) []*jobpool.Job { ctx := context.Background() res := []any{} - params := &configservice.DescribeConfigRulesInput{} - rules, err := svc.DescribeConfigRules(ctx, params) - if err != nil { - return nil, err - } - for _, r := range rules.ConfigRules { - jsonSource, err := convert.JsonToDict(r.Source) + var nextToken *string + for { + rules, err := svc.DescribeConfigRules(ctx, &configservice.DescribeConfigRulesInput{NextToken: nextToken}) if err != nil { return nil, err } - mqlRule, err := CreateResource(a.MqlRuntime, "aws.config.rule", - map[string]*llx.RawData{ - "arn": llx.StringDataPtr(r.ConfigRuleArn), - "name": llx.StringDataPtr(r.ConfigRuleName), - "description": llx.StringDataPtr(r.Description), - "id": llx.StringDataPtr(r.ConfigRuleId), - "source": llx.MapData(jsonSource, types.Any), - "state": llx.StringData(string(r.ConfigRuleState)), - "region": llx.StringData(region), - }) - if err != nil { - return nil, err + for _, r := range rules.ConfigRules { + jsonSource, err := convert.JsonToDict(r.Source) + if err != nil { + return nil, err + } + mqlRule, err := CreateResource(a.MqlRuntime, "aws.config.rule", + map[string]*llx.RawData{ + "arn": llx.StringDataPtr(r.ConfigRuleArn), + "name": llx.StringDataPtr(r.ConfigRuleName), + "description": llx.StringDataPtr(r.Description), + "id": llx.StringDataPtr(r.ConfigRuleId), + "source": llx.MapData(jsonSource, types.Any), + "state": llx.StringData(string(r.ConfigRuleState)), + "region": llx.StringData(region), + }) + if err != nil { + return nil, err + } + res = append(res, mqlRule) + } + if rules.NextToken == nil { + break } - res = append(res, mqlRule) + nextToken = rules.NextToken } return jobpool.JobResult(res), nil } diff --git a/providers/aws/resources/aws_dynamodb.go b/providers/aws/resources/aws_dynamodb.go index 39eece2057..e0809e331b 100644 --- a/providers/aws/resources/aws_dynamodb.go +++ b/providers/aws/resources/aws_dynamodb.go @@ -337,11 +337,23 @@ func (a *mqlAwsDynamodbTable) backups() ([]any, error) { svc := conn.Dynamodb(region) ctx := context.Background() - listBackupsResp, err := svc.ListBackups(ctx, &dynamodb.ListBackupsInput{TableName: &tableName}) - if err != nil { - return nil, errors.Wrap(err, "could not gather aws dynamodb backups") + var allBackups []ddtypes.BackupSummary + var exclusiveStartBackupArn *string + for { + listBackupsResp, err := svc.ListBackups(ctx, &dynamodb.ListBackupsInput{ + TableName: &tableName, + ExclusiveStartBackupArn: exclusiveStartBackupArn, + }) + if err != nil { + return nil, errors.Wrap(err, "could not gather aws dynamodb backups") + } + allBackups = append(allBackups, listBackupsResp.BackupSummaries...) + if listBackupsResp.LastEvaluatedBackupArn == nil { + break + } + exclusiveStartBackupArn = listBackupsResp.LastEvaluatedBackupArn } - return convert.JsonToDictSlice(listBackupsResp.BackupSummaries) + return convert.JsonToDictSlice(allBackups) } func (a *mqlAwsDynamodbTable) tags() (map[string]any, error) { @@ -425,22 +437,28 @@ func (a *mqlAwsDynamodb) globalTables() ([]any, error) { svc := conn.Dynamodb("") ctx := context.Background() - // no pagination required - listGlobalTablesResp, err := svc.ListGlobalTables(ctx, &dynamodb.ListGlobalTablesInput{}) - if err != nil { - return nil, errors.Wrap(err, "could not gather aws dynamodb global tables") - } res := []any{} - for _, table := range listGlobalTablesResp.GlobalTables { - mqlTable, err := CreateResource(a.MqlRuntime, "aws.dynamodb.globaltable", - map[string]*llx.RawData{ - "arn": llx.StringData(fmt.Sprintf(dynamoGlobalTableArnPattern, conn.AccountId(), convert.ToValue(table.GlobalTableName))), - "name": llx.StringDataPtr(table.GlobalTableName), - }) + var exclusiveStartGlobalTableName *string + for { + listGlobalTablesResp, err := svc.ListGlobalTables(ctx, &dynamodb.ListGlobalTablesInput{ExclusiveStartGlobalTableName: exclusiveStartGlobalTableName}) if err != nil { - return nil, err + return nil, errors.Wrap(err, "could not gather aws dynamodb global tables") + } + for _, table := range listGlobalTablesResp.GlobalTables { + mqlTable, err := CreateResource(a.MqlRuntime, "aws.dynamodb.globaltable", + map[string]*llx.RawData{ + "arn": llx.StringData(fmt.Sprintf(dynamoGlobalTableArnPattern, conn.AccountId(), convert.ToValue(table.GlobalTableName))), + "name": llx.StringDataPtr(table.GlobalTableName), + }) + if err != nil { + return nil, err + } + res = append(res, mqlTable) + } + if listGlobalTablesResp.LastEvaluatedGlobalTableName == nil { + break } - res = append(res, mqlTable) + exclusiveStartGlobalTableName = listGlobalTablesResp.LastEvaluatedGlobalTableName } return res, nil } @@ -478,56 +496,62 @@ func (a *mqlAwsDynamodb) getTables(conn *connection.AwsConnection) []*jobpool.Jo ctx := context.Background() res := []any{} - // no pagination required - listTablesResp, err := svc.ListTables(ctx, &dynamodb.ListTablesInput{}) - if err != nil { - if Is400AccessDeniedError(err) { - log.Warn().Str("region", region).Msg("error accessing region for AWS API") - return res, nil - } - return nil, errors.Wrap(err, "could not gather aws dynamodb tables") - } - for _, tableName := range listTablesResp.TableNames { - // call describe table to get real info/details about the table - table, err := svc.DescribeTable(ctx, &dynamodb.DescribeTableInput{TableName: &tableName}) - if err != nil { - return nil, errors.Wrap(err, "could not get aws dynamodb table") - } - sseDict, err := convert.JsonToDict(table.Table.SSEDescription) - if err != nil { - return nil, err - } - throughputDict, err := convert.JsonToDict(table.Table.ProvisionedThroughput) + var exclusiveStartTableName *string + for { + listTablesResp, err := svc.ListTables(ctx, &dynamodb.ListTablesInput{ExclusiveStartTableName: exclusiveStartTableName}) if err != nil { - return nil, err + if Is400AccessDeniedError(err) { + log.Warn().Str("region", region).Msg("error accessing region for AWS API") + return res, nil + } + return nil, errors.Wrap(err, "could not gather aws dynamodb tables") } + for _, tableName := range listTablesResp.TableNames { + // call describe table to get real info/details about the table + table, err := svc.DescribeTable(ctx, &dynamodb.DescribeTableInput{TableName: &tableName}) + if err != nil { + return nil, errors.Wrap(err, "could not get aws dynamodb table") + } + sseDict, err := convert.JsonToDict(table.Table.SSEDescription) + if err != nil { + return nil, err + } + throughputDict, err := convert.JsonToDict(table.Table.ProvisionedThroughput) + if err != nil { + return nil, err + } - mqlTable, err := CreateResource(a.MqlRuntime, "aws.dynamodb.table", - map[string]*llx.RawData{ - "arn": llx.StringData(fmt.Sprintf(dynamoTableArnPattern, region, conn.AccountId(), tableName)), - "name": llx.StringData(tableName), - "region": llx.StringData(region), - "sseDescription": llx.DictData(sseDict), - "provisionedThroughput": llx.DictData(throughputDict), - "createdAt": llx.TimeDataPtr(table.Table.CreationDateTime), - "deletionProtectionEnabled": llx.BoolDataPtr(table.Table.DeletionProtectionEnabled), - "globalTableVersion": llx.StringDataPtr(table.Table.GlobalTableVersion), - "id": llx.StringDataPtr(table.Table.TableId), - "sizeBytes": llx.IntDataPtr(table.Table.TableSizeBytes), - "status": llx.StringData(string(table.Table.TableStatus)), - "items": llx.IntDataPtr(table.Table.ItemCount), - "latestStreamArn": llx.StringDataPtr(table.Table.LatestStreamArn), - "latestStreamLabel": llx.StringDataPtr(table.Table.LatestStreamLabel), - "tableClass": llx.StringData(tableClassFromSummary(table.Table.TableClassSummary)), - "streamEnabled": llx.BoolData(streamEnabledFromSpec(table.Table.StreamSpecification)), - "streamViewType": llx.StringData(streamViewTypeFromSpec(table.Table.StreamSpecification)), - "billingMode": llx.StringData(billingModeFromSummary(table.Table.BillingModeSummary)), - "replicaRegions": llx.ArrayData(replicaRegionsFromDescriptions(table.Table.Replicas), types.String), - }) - if err != nil { - return nil, err + mqlTable, err := CreateResource(a.MqlRuntime, "aws.dynamodb.table", + map[string]*llx.RawData{ + "arn": llx.StringData(fmt.Sprintf(dynamoTableArnPattern, region, conn.AccountId(), tableName)), + "name": llx.StringData(tableName), + "region": llx.StringData(region), + "sseDescription": llx.DictData(sseDict), + "provisionedThroughput": llx.DictData(throughputDict), + "createdAt": llx.TimeDataPtr(table.Table.CreationDateTime), + "deletionProtectionEnabled": llx.BoolDataPtr(table.Table.DeletionProtectionEnabled), + "globalTableVersion": llx.StringDataPtr(table.Table.GlobalTableVersion), + "id": llx.StringDataPtr(table.Table.TableId), + "sizeBytes": llx.IntDataPtr(table.Table.TableSizeBytes), + "status": llx.StringData(string(table.Table.TableStatus)), + "items": llx.IntDataPtr(table.Table.ItemCount), + "latestStreamArn": llx.StringDataPtr(table.Table.LatestStreamArn), + "latestStreamLabel": llx.StringDataPtr(table.Table.LatestStreamLabel), + "tableClass": llx.StringData(tableClassFromSummary(table.Table.TableClassSummary)), + "streamEnabled": llx.BoolData(streamEnabledFromSpec(table.Table.StreamSpecification)), + "streamViewType": llx.StringData(streamViewTypeFromSpec(table.Table.StreamSpecification)), + "billingMode": llx.StringData(billingModeFromSummary(table.Table.BillingModeSummary)), + "replicaRegions": llx.ArrayData(replicaRegionsFromDescriptions(table.Table.Replicas), types.String), + }) + if err != nil { + return nil, err + } + res = append(res, mqlTable) + } + if listTablesResp.LastEvaluatedTableName == nil { + break } - res = append(res, mqlTable) + exclusiveStartTableName = listTablesResp.LastEvaluatedTableName } return jobpool.JobResult(res), nil } diff --git a/providers/aws/resources/aws_ecs.go b/providers/aws/resources/aws_ecs.go index 6a0b806d6a..7fd5419fa9 100644 --- a/providers/aws/resources/aws_ecs.go +++ b/providers/aws/resources/aws_ecs.go @@ -296,12 +296,25 @@ func (a *mqlAwsEcsCluster) containerInstances() ([]any, error) { ctx := context.Background() res := []any{} - params := &ecsservice.ListContainerInstancesInput{Cluster: &clustera} - containerInstances, err := svc.ListContainerInstances(ctx, params) - if err != nil { - log.Error().Err(err).Msg("cannot list container instances") // no fail - } else if len(containerInstances.ContainerInstanceArns) > 0 { - containerInstancesDetail, err := svc.DescribeContainerInstances(ctx, &ecsservice.DescribeContainerInstancesInput{Cluster: &clustera, ContainerInstances: containerInstances.ContainerInstanceArns}) + var allContainerInstanceArns []string + var nextToken *string + for { + containerInstances, err := svc.ListContainerInstances(ctx, &ecsservice.ListContainerInstancesInput{ + Cluster: &clustera, + NextToken: nextToken, + }) + if err != nil { + log.Error().Err(err).Msg("cannot list container instances") + break + } + allContainerInstanceArns = append(allContainerInstanceArns, containerInstances.ContainerInstanceArns...) + if containerInstances.NextToken == nil { + break + } + nextToken = containerInstances.NextToken + } + if len(allContainerInstanceArns) > 0 { + containerInstancesDetail, err := svc.DescribeContainerInstances(ctx, &ecsservice.DescribeContainerInstancesInput{Cluster: &clustera, ContainerInstances: allContainerInstanceArns}) if err == nil { for _, ci := range containerInstancesDetail.ContainerInstances { // container instance assets diff --git a/providers/aws/resources/aws_redshift.go b/providers/aws/resources/aws_redshift.go index f67d42956a..4a1cbab075 100644 --- a/providers/aws/resources/aws_redshift.go +++ b/providers/aws/resources/aws_redshift.go @@ -239,11 +239,21 @@ func (a *mqlAwsRedshiftCluster) parameters() ([]any, error) { res := []redshifttypes.Parameter{} for _, name := range clusterGroupNames { stringName := name.(string) - params, err := svc.DescribeClusterParameters(ctx, &redshift.DescribeClusterParametersInput{ParameterGroupName: &stringName}) - if err != nil { - return nil, err + var marker *string + for { + params, err := svc.DescribeClusterParameters(ctx, &redshift.DescribeClusterParametersInput{ + ParameterGroupName: &stringName, + Marker: marker, + }) + if err != nil { + return nil, err + } + res = append(res, params.Parameters...) + if params.Marker == nil { + break + } + marker = params.Marker } - res = append(res, params.Parameters...) } return convert.JsonToDictSlice(res) } diff --git a/providers/aws/resources/aws_sagemaker.go b/providers/aws/resources/aws_sagemaker.go index 2a7929a7a2..c6663613b0 100644 --- a/providers/aws/resources/aws_sagemaker.go +++ b/providers/aws/resources/aws_sagemaker.go @@ -1390,21 +1390,31 @@ func (a *mqlAwsSagemakerDomain) defaultUserSettings() (map[string]any, error) { } func getSagemakerTags(ctx context.Context, svc *sagemaker.Client, arn *string) (map[string]any, error) { - resp, err := svc.ListTags(ctx, &sagemaker.ListTagsInput{ResourceArn: arn}) - var respErr *http.ResponseError - if err != nil { - if errors.As(err, &respErr) { - if respErr.HTTPStatusCode() == 404 { - return nil, nil + tags := make(map[string]any) + var nextToken *string + for { + resp, err := svc.ListTags(ctx, &sagemaker.ListTagsInput{ + ResourceArn: arn, + NextToken: nextToken, + }) + var respErr *http.ResponseError + if err != nil { + if errors.As(err, &respErr) { + if respErr.HTTPStatusCode() == 404 { + return nil, nil + } + } + return nil, err + } + for _, t := range resp.Tags { + if t.Key != nil && t.Value != nil { + tags[*t.Key] = *t.Value } } - return nil, err - } - tags := make(map[string]any) - for _, t := range resp.Tags { - if t.Key != nil && t.Value != nil { - tags[*t.Key] = *t.Value + if resp.NextToken == nil { + break } + nextToken = resp.NextToken } return tags, nil }