Skip to content

ApiGateway: bugfix to restore FilterFunc for correct mapping of resources #1332

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Mar 7, 2024
Merged

ApiGateway: bugfix to restore FilterFunc for correct mapping of resources #1332

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,7 @@ As a quick start, the following IAM policy can be used to grant the all permissi
"cloudwatch:GetMetricData",
"cloudwatch:GetMetricStatistics",
"cloudwatch:ListMetrics",
"apigateway:GET",
"aps:ListWorkspaces",
"autoscaling:DescribeAutoScalingGroups",
"dms:DescribeReplicationInstances",
Expand Down Expand Up @@ -170,6 +171,11 @@ These are the bare minimum permissions required to run Static and Discovery Jobs
"cloudwatch:ListMetrics"
```

This permission is required to discover resources for the AWS/ApiGateway namespace
```json
"apigateway:GET"
```

This permission is required to discover resources for the AWS/AutoScaling namespace
```json
"autoscaling:DescribeAutoScalingGroups"
Expand Down
2 changes: 2 additions & 0 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ require (
github.com/aws/aws-sdk-go-v2/config v1.27.4
github.com/aws/aws-sdk-go-v2/credentials v1.17.4
github.com/aws/aws-sdk-go-v2/service/amp v1.25.1
github.com/aws/aws-sdk-go-v2/service/apigateway v1.23.3
github.com/aws/aws-sdk-go-v2/service/apigatewayv2 v1.20.1
github.com/aws/aws-sdk-go-v2/service/autoscaling v1.40.2
github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.36.1
github.com/aws/aws-sdk-go-v2/service/databasemigrationservice v1.38.1
Expand Down
4 changes: 4 additions & 0 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,10 @@ github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY=
github.com/aws/aws-sdk-go-v2/service/amp v1.25.1 h1:kdytWcOvNUmqSVWSRNeAirv4e1j397UQC3d6ofS2CxQ=
github.com/aws/aws-sdk-go-v2/service/amp v1.25.1/go.mod h1:cG/8NqEsOGrLYBgIeixa7aaHzY8RvbUhMQYJBbSdD+c=
github.com/aws/aws-sdk-go-v2/service/apigateway v1.23.3 h1:STt7Loc3FYm8xoU0zF8cvXLgk0aXFEhBpw/QWoxey+8=
github.com/aws/aws-sdk-go-v2/service/apigateway v1.23.3/go.mod h1:Cz7oE8VP37eZ/JYhWoeRaqX2GoecJ2EkO7OEcjV4KQ4=
github.com/aws/aws-sdk-go-v2/service/apigatewayv2 v1.20.1 h1:nOJwQpU2wDe2qtw+vwkywJ44UhK66P6+1UIRotE7Jmo=
github.com/aws/aws-sdk-go-v2/service/apigatewayv2 v1.20.1/go.mod h1:A95FM8hxO6umoiROudoYtTmZYl7KN9nbez8deLDOCnA=
github.com/aws/aws-sdk-go-v2/service/autoscaling v1.40.2 h1:nbDDSiI4JpmOzSvtxa1jduRMmib+t/IU2mWgF3mSSBU=
github.com/aws/aws-sdk-go-v2/service/autoscaling v1.40.2/go.mod h1:g+K5WaPD3zegO3keZ8kV4dzPzU80aypAXSCXy7WL5xo=
github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.36.1 h1:mQySuI87thHtcbZvEDjwUROGWikU6fqgpHklCBXpJU4=
Expand Down
8 changes: 8 additions & 0 deletions pkg/clients/tagging/v1/client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ import (
"fmt"

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/apigateway/apigatewayiface"
"github.com/aws/aws-sdk-go/service/apigatewayv2/apigatewayv2iface"
"github.com/aws/aws-sdk-go/service/autoscaling/autoscalingiface"
"github.com/aws/aws-sdk-go/service/databasemigrationservice/databasemigrationserviceiface"
"github.com/aws/aws-sdk-go/service/ec2/ec2iface"
Expand All @@ -25,6 +27,8 @@ type client struct {
logger logging.Logger
taggingAPI resourcegroupstaggingapiiface.ResourceGroupsTaggingAPIAPI
autoscalingAPI autoscalingiface.AutoScalingAPI
apiGatewayAPI apigatewayiface.APIGatewayAPI
apiGatewayV2API apigatewayv2iface.ApiGatewayV2API
ec2API ec2iface.EC2API
dmsAPI databasemigrationserviceiface.DatabaseMigrationServiceAPI
prometheusSvcAPI prometheusserviceiface.PrometheusServiceAPI
Expand All @@ -36,6 +40,8 @@ func NewClient(
logger logging.Logger,
taggingAPI resourcegroupstaggingapiiface.ResourceGroupsTaggingAPIAPI,
autoscalingAPI autoscalingiface.AutoScalingAPI,
apiGatewayAPI apigatewayiface.APIGatewayAPI,
apiGatewayV2API apigatewayv2iface.ApiGatewayV2API,
ec2API ec2iface.EC2API,
dmsClient databasemigrationserviceiface.DatabaseMigrationServiceAPI,
prometheusClient prometheusserviceiface.PrometheusServiceAPI,
Expand All @@ -46,6 +52,8 @@ func NewClient(
logger: logger,
taggingAPI: taggingAPI,
autoscalingAPI: autoscalingAPI,
apiGatewayAPI: apiGatewayAPI,
apiGatewayV2API: apiGatewayV2API,
ec2API: ec2API,
dmsAPI: dmsClient,
prometheusSvcAPI: prometheusClient,
Expand Down
56 changes: 56 additions & 0 deletions pkg/clients/tagging/v1/filters.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,12 @@ package v1
import (
"context"
"fmt"
"strings"

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/arn"
"github.com/aws/aws-sdk-go/service/apigateway"
"github.com/aws/aws-sdk-go/service/apigatewayv2"
"github.com/aws/aws-sdk-go/service/autoscaling"
"github.com/aws/aws-sdk-go/service/databasemigrationservice"
"github.com/aws/aws-sdk-go/service/ec2"
Expand All @@ -27,6 +30,59 @@ type ServiceFilter struct {

// ServiceFilters maps a service namespace to (optional) ServiceFilter
var ServiceFilters = map[string]ServiceFilter{
"AWS/ApiGateway": {
// ApiGateway ARNs use the Id (for v1 REST APIs) and ApiId (for v2 APIs) instead of
// the ApiName (display name). See https://docs.aws.amazon.com/apigateway/latest/developerguide/arn-format-reference.html
// However, in metrics, the ApiId dimension uses the ApiName as value.
//
// Here we use the ApiGateway API to map resource correctly. For backward compatibility,
// in v1 REST APIs we change the ARN to replace the ApiId with ApiName, while for v2 APIs
// we leave the ARN as-is.
FilterFunc: func(ctx context.Context, client client, inputResources []*model.TaggedResource) ([]*model.TaggedResource, error) {
var limit int64 = 500 // max number of results per page. default=25, max=500
const maxPages = 10
input := apigateway.GetRestApisInput{Limit: &limit}
output := apigateway.GetRestApisOutput{}
var pageNum int

err := client.apiGatewayAPI.GetRestApisPagesWithContext(ctx, &input, func(page *apigateway.GetRestApisOutput, _ bool) bool {
promutil.APIGatewayAPICounter.Inc()
pageNum++
output.Items = append(output.Items, page.Items...)
return pageNum <= maxPages
})
if err != nil {
return nil, fmt.Errorf("error calling apiGatewayAPI.GetRestApisPages, %w", err)
}

outputV2, err := client.apiGatewayV2API.GetApisWithContext(ctx, &apigatewayv2.GetApisInput{})
promutil.APIGatewayAPIV2Counter.Inc()
if err != nil {
return nil, fmt.Errorf("error calling apiGatewayAPIv2.GetApis, %w", err)
}

var outputResources []*model.TaggedResource
for _, resource := range inputResources {
for i, gw := range output.Items {
if strings.HasSuffix(resource.ARN, "/restapis/"+*gw.Id) {
r := resource
r.ARN = strings.ReplaceAll(resource.ARN, *gw.Id, *gw.Name)
outputResources = append(outputResources, r)
output.Items = append(output.Items[:i], output.Items[i+1:]...)
break
}
}
for i, gw := range outputV2.Items {
if strings.HasSuffix(resource.ARN, "/apis/"+*gw.ApiId) {
outputResources = append(outputResources, resource)
outputV2.Items = append(outputV2.Items[:i], outputV2.Items[i+1:]...)
break
}
}
}
return outputResources, nil
},
},
"AWS/AutoScaling": {
ResourceFunc: func(ctx context.Context, client client, job model.DiscoveryJob, region string) ([]*model.TaggedResource, error) {
pageNum := 0
Expand Down
191 changes: 191 additions & 0 deletions pkg/clients/tagging/v1/filters_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,10 @@ import (

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/service/apigateway"
"github.com/aws/aws-sdk-go/service/apigateway/apigatewayiface"
"github.com/aws/aws-sdk-go/service/apigatewayv2"
"github.com/aws/aws-sdk-go/service/apigatewayv2/apigatewayv2iface"
"github.com/aws/aws-sdk-go/service/databasemigrationservice"
"github.com/aws/aws-sdk-go/service/databasemigrationservice/databasemigrationserviceiface"

Expand All @@ -28,6 +32,174 @@ func TestValidServiceNames(t *testing.T) {
}
}

func TestApiGatewayFilterFunc(t *testing.T) {
tests := []struct {
name string
iface client
inputResources []*model.TaggedResource
outputResources []*model.TaggedResource
}{
{
"api gateway resources skip stages",
client{
apiGatewayAPI: apiGatewayClient{
getRestApisOutput: &apigateway.GetRestApisOutput{
Items: []*apigateway.RestApi{
{
ApiKeySource: nil,
BinaryMediaTypes: nil,
CreatedDate: nil,
Description: nil,
DisableExecuteApiEndpoint: nil,
EndpointConfiguration: nil,
Id: aws.String("gwid1234"),
MinimumCompressionSize: nil,
Name: aws.String("apiname"),
Policy: nil,
Tags: nil,
Version: nil,
Warnings: nil,
},
},
Position: nil,
},
},
apiGatewayV2API: apiGatewayV2Client{
getRestApisOutput: &apigatewayv2.GetApisOutput{
Items: []*apigatewayv2.Api{},
},
},
},
[]*model.TaggedResource{
{
ARN: "arn:aws:apigateway:us-east-1::/restapis/gwid1234/stages/main",
Namespace: "apigateway",
Region: "us-east-1",
Tags: []model.Tag{
{
Key: "Test",
Value: "Value",
},
},
},
{
ARN: "arn:aws:apigateway:us-east-1::/restapis/gwid1234",
Namespace: "apigateway",
Region: "us-east-1",
Tags: []model.Tag{
{
Key: "Test",
Value: "Value 2",
},
},
},
},
[]*model.TaggedResource{
{
ARN: "arn:aws:apigateway:us-east-1::/restapis/apiname",
Namespace: "apigateway",
Region: "us-east-1",
Tags: []model.Tag{
{
Key: "Test",
Value: "Value 2",
},
},
},
},
},
{
"api gateway v2",
client{
apiGatewayAPI: apiGatewayClient{
getRestApisOutput: &apigateway.GetRestApisOutput{
Items: []*apigateway.RestApi{},
},
},
apiGatewayV2API: apiGatewayV2Client{
getRestApisOutput: &apigatewayv2.GetApisOutput{
Items: []*apigatewayv2.Api{
{
CreatedDate: nil,
Description: nil,
DisableExecuteApiEndpoint: nil,
ApiId: aws.String("gwid9876"),
Name: aws.String("apiv2name"),
Tags: nil,
Version: nil,
Warnings: nil,
},
},
NextToken: nil,
},
},
},
[]*model.TaggedResource{
{
ARN: "arn:aws:apigateway:us-east-1::/apis/gwid9876/stages/$default",
Namespace: "apigateway",
Region: "us-east-1",
Tags: []model.Tag{
{
Key: "Test",
Value: "Value",
},
},
},
{
ARN: "arn:aws:apigateway:us-east-1::/apis/gwid9876",
Namespace: "apigateway",
Region: "us-east-1",
Tags: []model.Tag{
{
Key: "Test",
Value: "Value 2",
},
},
},
},
[]*model.TaggedResource{
{
ARN: "arn:aws:apigateway:us-east-1::/apis/gwid9876",
Namespace: "apigateway",
Region: "us-east-1",
Tags: []model.Tag{
{
Key: "Test",
Value: "Value 2",
},
},
},
},
},
}

for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
apigateway := ServiceFilters["AWS/ApiGateway"]

outputResources, err := apigateway.FilterFunc(context.Background(), test.iface, test.inputResources)
if err != nil {
t.Logf("Error from FilterFunc: %v", err)
t.FailNow()
}
if len(outputResources) != len(test.outputResources) {
t.Logf("len(outputResources) = %d, want %d", len(outputResources), len(test.outputResources))
t.Fail()
}
for i, resource := range outputResources {
if len(test.outputResources) <= i {
break
}
wantResource := *test.outputResources[i]
if !reflect.DeepEqual(*resource, wantResource) {
t.Errorf("outputResources[%d] = %+v, want %+v", i, *resource, wantResource)
}
}
})
}
}

func TestDMSFilterFunc(t *testing.T) {
tests := []struct {
name string
Expand Down Expand Up @@ -243,6 +415,25 @@ func TestDMSFilterFunc(t *testing.T) {
}
}

type apiGatewayClient struct {
apigatewayiface.APIGatewayAPI
getRestApisOutput *apigateway.GetRestApisOutput
}

func (apigateway apiGatewayClient) GetRestApisPagesWithContext(_ aws.Context, _ *apigateway.GetRestApisInput, fn func(*apigateway.GetRestApisOutput, bool) bool, _ ...request.Option) error {
fn(apigateway.getRestApisOutput, true)
return nil
}

type apiGatewayV2Client struct {
apigatewayv2iface.ApiGatewayV2API
getRestApisOutput *apigatewayv2.GetApisOutput
}

func (apigateway apiGatewayV2Client) GetApisWithContext(_ aws.Context, _ *apigatewayv2.GetApisInput, _ ...request.Option) (*apigatewayv2.GetApisOutput, error) {
return apigateway.getRestApisOutput, nil
}

type dmsClient struct {
databasemigrationserviceiface.DatabaseMigrationServiceAPI
describeReplicationInstancesOutput *databasemigrationservice.DescribeReplicationInstancesOutput
Expand Down
8 changes: 8 additions & 0 deletions pkg/clients/tagging/v2/client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ import (

"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/amp"
"github.com/aws/aws-sdk-go-v2/service/apigateway"
"github.com/aws/aws-sdk-go-v2/service/apigatewayv2"
"github.com/aws/aws-sdk-go-v2/service/autoscaling"
"github.com/aws/aws-sdk-go-v2/service/databasemigrationservice"
"github.com/aws/aws-sdk-go-v2/service/ec2"
Expand All @@ -25,6 +27,8 @@ type client struct {
logger logging.Logger
taggingAPI *resourcegroupstaggingapi.Client
autoscalingAPI *autoscaling.Client
apiGatewayAPI *apigateway.Client
apiGatewayV2API *apigatewayv2.Client
ec2API *ec2.Client
dmsAPI *databasemigrationservice.Client
prometheusSvcAPI *amp.Client
Expand All @@ -36,6 +40,8 @@ func NewClient(
logger logging.Logger,
taggingAPI *resourcegroupstaggingapi.Client,
autoscalingAPI *autoscaling.Client,
apiGatewayAPI *apigateway.Client,
apiGatewayV2API *apigatewayv2.Client,
ec2API *ec2.Client,
dmsClient *databasemigrationservice.Client,
prometheusClient *amp.Client,
Expand All @@ -46,6 +52,8 @@ func NewClient(
logger: logger,
taggingAPI: taggingAPI,
autoscalingAPI: autoscalingAPI,
apiGatewayAPI: apiGatewayAPI,
apiGatewayV2API: apiGatewayV2API,
ec2API: ec2API,
dmsAPI: dmsClient,
prometheusSvcAPI: prometheusClient,
Expand Down
Loading