From 8388aa88150875c7ac1713bbb83dbf58b2b2b63d Mon Sep 17 00:00:00 2001 From: riley priddle Date: Tue, 6 May 2025 21:18:45 +0100 Subject: [PATCH 01/18] added cfn deployment process --- .github/release-drafter.yml | 34 ++++ .github/workflows/codecov.yml | 24 --- .github/workflows/deploy-prod-eu.yaml | 157 ++++++++++++++++++ .github/workflows/deploy-sandbox.yaml | 65 ++++++++ .github/workflows/deploy-staging.yaml | 157 ++++++++++++++++++ .github/workflows/draft-release.yaml | 39 +++++ .github/workflows/pull-request.yaml | 44 +++++ Makefile | 12 +- build_setup/Dockerfile | 13 ++ build_setup/build.sh | 7 + build_setup/get_regions.py | 13 ++ cfn_templates/appsync_logging_template.yaml | 72 ++++++++ .../extract_firetail_logs.go | 0 .../extract_firetail_logs_test.go | 0 {logs-handler => src}/firetail_log.go | 0 {logs-handler => src}/firetail_log_test.go | 0 go.mod => src/go.mod | 0 go.sum => src/go.sum | 0 {logs-handler => src}/handler.go | 0 {logs-handler => src}/handler_test.go | 0 {logs-handler => src}/main.go | 0 {logs-handler => src}/main_test.go | 0 {logs-handler => src}/parse_headers.go | 0 {logs-handler => src}/parse_headers_test.go | 0 {logs-handler => src}/send_to_firetail.go | 0 .../send_to_firetail_test.go | 0 template.yaml | 53 ++++++ 27 files changed, 663 insertions(+), 27 deletions(-) create mode 100644 .github/release-drafter.yml delete mode 100644 .github/workflows/codecov.yml create mode 100644 .github/workflows/deploy-prod-eu.yaml create mode 100644 .github/workflows/deploy-sandbox.yaml create mode 100644 .github/workflows/deploy-staging.yaml create mode 100644 .github/workflows/draft-release.yaml create mode 100644 .github/workflows/pull-request.yaml create mode 100644 build_setup/Dockerfile create mode 100755 build_setup/build.sh create mode 100644 build_setup/get_regions.py create mode 100644 cfn_templates/appsync_logging_template.yaml rename {logs-handler => src}/extract_firetail_logs.go (100%) rename {logs-handler => src}/extract_firetail_logs_test.go (100%) rename {logs-handler => src}/firetail_log.go (100%) rename {logs-handler => src}/firetail_log_test.go (100%) rename go.mod => src/go.mod (100%) rename go.sum => src/go.sum (100%) rename {logs-handler => src}/handler.go (100%) rename {logs-handler => src}/handler_test.go (100%) rename {logs-handler => src}/main.go (100%) rename {logs-handler => src}/main_test.go (100%) rename {logs-handler => src}/parse_headers.go (100%) rename {logs-handler => src}/parse_headers_test.go (100%) rename {logs-handler => src}/send_to_firetail.go (100%) rename {logs-handler => src}/send_to_firetail_test.go (100%) create mode 100644 template.yaml diff --git a/.github/release-drafter.yml b/.github/release-drafter.yml new file mode 100644 index 0000000..e5f247e --- /dev/null +++ b/.github/release-drafter.yml @@ -0,0 +1,34 @@ +# For the https://github.com/marketplace/actions/release-drafter Action +name-template: v$RESOLVED_VERSION Release +tag-template: v$RESOLVED_VERSION +categories: + - title: 🚀 Features + labels: + - feature + - enhancement + - title: 🐛 Bug Fixes + labels: + - fix + - bugfix + - bug + - title: 🧰 Maintenance + label: chore +change-template: '- $TITLE @$AUTHOR (#$NUMBER)' +# You can add # and @ to disable mentions, and add ` to disable code blocks. +change-title-escapes: '\<*_&' +version-resolver: + major: + labels: + - major + minor: + labels: + - minor + patch: + labels: + - patch + default: patch +template: | + ## Changes + $CHANGES +include-pre-releases: true +prerelease: true diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml deleted file mode 100644 index 67a8287..0000000 --- a/.github/workflows/codecov.yml +++ /dev/null @@ -1,24 +0,0 @@ -name: Test and coverage - -on: - pull_request: - branches: [ main, dev ] - push: - branches: [ main ] - -jobs: - build: - runs-on: ubuntu-latest - strategy: - matrix: - go-image: [ '1.18-bullseye' ] - steps: - - uses: actions/checkout@v3 - - name: Run coverage - run: docker run --rm -v ${{ github.workspace }}:/src -w /src golang:${{ matrix.go-image }} go test ./... -race -coverprofile coverage.out -covermode atomic - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 - with: - token: ${{ secrets.CODECOV_TOKEN }} - fail_ci_if_error: true - verbose: true \ No newline at end of file diff --git a/.github/workflows/deploy-prod-eu.yaml b/.github/workflows/deploy-prod-eu.yaml new file mode 100644 index 0000000..87d916b --- /dev/null +++ b/.github/workflows/deploy-prod-eu.yaml @@ -0,0 +1,157 @@ +name: Deploy To Prod EU + +on: + release: + types: + - released + +permissions: + id-token: write + contents: write + pull-requests: write +env: + GOLANG_VERSION: 1.23 + REGION_NAME: eu-west-1 + ACCOUNT_ID: 247286868737 + ROLE_TO_ASSUME: arn:aws:iam::247286868737:role/firetail-prod-github-serverless-lambda-deployment + LAMBDA_SERVERLESS_REPO: firetail-prod-eu-west-1-serverless-applications + APP_NAME: firetail-appsync-logger + CFN_TEMPLATES_BUCKET: firetail-prod-us-east-1-cf-templates + CFN_TEMPLATES_BUCKET_REGION: us-east-1 +jobs: + setup: + runs-on: ubuntu-latest + outputs: + mymatrix: ${{ steps.matrixStep.outputs.matrixItems }} + steps: + - name: Checkout Code + uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 + with: + fetch-depth: 2 + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@8c3f20df09ac63af7b3ae3d7c91f105f857d8497 # v4.0.0 + with: + aws-region: ${{ env.REGION_NAME }} + role-to-assume: ${{env.ROLE_TO_ASSUME }} + role-session-name: github + - id: matrixStep + run: | + python3 -m pip install boto3 + matrix=$(python3 build_setup/get_regions.py) + echo $matrix + echo "matrixItems=$(echo $matrix)" >> $GITHUB_ENV + download-artifact: + runs-on: ubuntu-latest + steps: + - name: Checkout Code + uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 + with: + fetch-depth: 2 + - name: Get Release + id: get_release + uses: bruceadams/get-release@74c3d60f5a28f358ccf241a00c9021ea16f0569f + env: + GITHUB_TOKEN: ${{ github.token }} + - name: Download zip from release + uses: robinraju/release-downloader@efa4cd07bd0195e6cc65e9e30c251b49ce4d3e51 + with: + releaseId: ${{ steps.get_release.outputs.id }} + fileName: lambda.zip + out-file-path: "build" + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@8c3f20df09ac63af7b3ae3d7c91f105f857d8497 # v4.0.0 + with: + aws-region: ${{ env.REGION_NAME }} + role-to-assume: ${{env.ROLE_TO_ASSUME }} + role-session-name: github + - run: + sam package --template-file template.yaml --output-template-file packaged.yaml --s3-bucket ${{ + env.LAMBDA_SERVERLESS_REPO }} --region ${{env.REGION_NAME}} + - uses: actions/upload-artifact@v3 + with: + name: packaged.yaml + path: packaged.yaml + release-application: + needs: download-artifact + runs-on: ubuntu-latest + strategy: + # matrix: ${{ fromJson(needs.setup.outputs.mymatrix) }} + fail-fast: false + matrix: + region: + [ + "us-east-2", + "us-east-1", + "us-west-1", + "us-west-2", + "af-south-1", + "ap-east-1", + "ap-south-2", + "ap-southeast-3", + "ap-southeast-4", + "ap-south-1", + "ap-northeast-3", + "ap-northeast-2", + "ap-southeast-1", + "ap-southeast-2", + "ap-northeast-1", + "ca-central-1", + "eu-central-1", + "eu-west-1", + "eu-west-2", + "eu-south-1", + "eu-west-3", + "eu-south-2", + "eu-north-1", + "eu-central-2", + "il-central-1", + "me-south-1", + "me-central-1", + "sa-east-1", + ] + steps: + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@8c3f20df09ac63af7b3ae3d7c91f105f857d8497 # v4.0.0 + with: + aws-region: ${{ env.REGION_NAME }} + role-to-assume: ${{env.ROLE_TO_ASSUME }} + role-session-name: github + - name: Get Release + id: get_release + uses: bruceadams/get-release@74c3d60f5a28f358ccf241a00c9021ea16f0569f + env: + GITHUB_TOKEN: ${{ github.token }} + - uses: actions/download-artifact@v4 + with: + name: packaged.yaml + - run: | + version=$(echo "${{ steps.get_release.outputs.tag_name }}" | cut -c 2-) + sam publish --template packaged.yaml --region ${{ matrix.region }} --semantic-version $version + aws serverlessrepo put-application-policy --region ${{ matrix.region }} --application-id arn:aws:serverlessrepo:${{ matrix.region }}:${{env.ACCOUNT_ID}}:applications/${{env.APP_NAME}} --statements Principals=*,Actions=Deploy + continue-on-error: true + release-cloudformation: + needs: release-application + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout Code + uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 + with: + fetch-depth: 2 + - name: Get Release + id: get_release + uses: bruceadams/get-release@74c3d60f5a28f358ccf241a00c9021ea16f0569f + env: + GITHUB_TOKEN: ${{ github.token }} + - run: | + version=$(echo "${{ steps.get_release.outputs.tag_name }}" | cut -c 2-) + sed -i -e "s/0.0.1/${version}/g" cfn_templates/appsync_logging_template.yaml + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@8c3f20df09ac63af7b3ae3d7c91f105f857d8497 # v4.0.0 + with: + aws-region: ${{ env.REGION_NAME }} + role-to-assume: ${{env.ROLE_TO_ASSUME }} + role-session-name: github + - name: copy new cfn to s3 + run: | + aws s3 cp ./cfn_templates/ s3://${{env.CFN_TEMPLATES_BUCKET}}/applications/cfn/ --acl public-read --region ${{env.CFN_TEMPLATES_BUCKET_REGION}} --recursive diff --git a/.github/workflows/deploy-sandbox.yaml b/.github/workflows/deploy-sandbox.yaml new file mode 100644 index 0000000..8288d62 --- /dev/null +++ b/.github/workflows/deploy-sandbox.yaml @@ -0,0 +1,65 @@ +env: + SANDBOX_AWS_ACCT_ID: 453671210445 + SANDBOX_AWS_ACCT_NAME: firetail-sandbox + AWS_REGION: eu-west-1 + ROLE_TO_ASSUME: arn:aws:iam::453671210445:role/firetail-sandbox-github-serverless-lambda-deployment + LAMBDA_SERVERLESS_REPO: firetail-sandbox-eu-west-1-serverless-applications + CFN_TEMPLATES_BUCKET: firetail-sandbox-us-east-1-cf-templates + CFN_TEMPLATES_BUCKET_REGION: us-east-1 + APP_NAME: aws-appsync-logging-lambda + SEMANTIC_VERSION: 1.1.39 +name: Deploy to Sandbox +run-name: "@${{ github.triggering_actor }}: ${{ github.ref_name }}: ${{ github.event_name }}" +on: + push: + branches: + - dev + - dev-preview +defaults: + run: + shell: bash +concurrency: + group: ${{ github.workflow }}-${{ github.ref_name }} + cancel-in-progress: true +permissions: + id-token: write + contents: read + pull-requests: read +jobs: + deploy-sandbox: + runs-on: ubuntu-latest + steps: + - name: Checkout Code + uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 + with: + fetch-depth: 2 + - name: Set Environment + run: | + git_hash="$(git rev-parse --short "${{ github.sha }}")" + RELEASE_VERSION="sandbox-${git_hash}-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" + IMAGE_TAG="${{ env.SANDBOX_ECR_HOSTNAME }}/${{ env.ECR_REPO }}:${RELEASE_VERSION}" + + cat <>"${GITHUB_ENV}" + RELEASE_VERSION=${RELEASE_VERSION} + IMAGE_TAG=${IMAGE_TAG} + EOF + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@8c3f20df09ac63af7b3ae3d7c91f105f857d8497 # v4.0.0 + with: + aws-region: ${{ env.AWS_REGION }} + role-to-assume: ${{env.ROLE_TO_ASSUME }} + role-session-name: github + - name: Run build docker image + run: docker build -t lambda-image:latest --target runtime-image -f build_setup/Dockerfile . + - name: create build dir + run: mkdir build -p + - name: extract docker build zip + run: docker run --rm --entrypoint cat lambda-image:latest /src/lambda.zip > build/lambda.zip + - name: sam + run: | + sam package --template-file template.yaml --output-template-file packaged.yaml --s3-bucket ${{ env.LAMBDA_SERVERLESS_REPO }} + sam publish --template packaged.yaml --region ${{env.AWS_REGION}} --semantic-version ${{env.SEMANTIC_VERSION}} + - name: copy new cfn to s3 + run: | + sed -i -e "s/0.0.1/${{env.SEMANTIC_VERSION}}/g" cfn_templates/appsync_logging_template.yaml + aws s3 cp ./cfn_templates/ s3://${{env.CFN_TEMPLATES_BUCKET}}/applications/cfn/ --acl public-read --region ${{env.CFN_TEMPLATES_BUCKET_REGION}} --recursive diff --git a/.github/workflows/deploy-staging.yaml b/.github/workflows/deploy-staging.yaml new file mode 100644 index 0000000..3ede09a --- /dev/null +++ b/.github/workflows/deploy-staging.yaml @@ -0,0 +1,157 @@ +name: Deploy To Staging + +on: + release: + types: [published] + +permissions: + id-token: write + contents: write + pull-requests: write +env: + GOLANG_VERSION: 1.23 + REGION_NAME: eu-west-1 + ROLE_TO_ASSUME: arn:aws:iam::261181027103:role/firetail-staging-github-serverless-lambda-deployment + LAMBDA_SERVERLESS_REPO: firetail-staging-eu-west-1-serverless-applications + APP_NAME: aws-appsync-logging-lambda + CFN_TEMPLATES_BUCKET: firetail-staging-us-east-1-cf-templates + CFN_TEMPLATES_BUCKET_REGION: us-east-1 +jobs: + setup: + if: github.event.release.prerelease + runs-on: ubuntu-latest + outputs: + mymatrix: ${{ steps.matrixStep.outputs.matrixItems }} + steps: + - name: Checkout Code + uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 + with: + fetch-depth: 2 + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@8c3f20df09ac63af7b3ae3d7c91f105f857d8497 # v4.0.0 + with: + aws-region: ${{ env.REGION_NAME }} + role-to-assume: ${{env.ROLE_TO_ASSUME }} + role-session-name: github + - id: matrixStep + run: | + python3 -m pip install boto3 + matrix=$(python3 build_setup/get_regions.py) + echo $matrix + echo "matrixItems=$(echo $matrix)" >> $GITHUB_ENV + download-artifact: + if: github.event.release.prerelease + runs-on: ubuntu-latest + steps: + - name: Checkout Code + uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 + with: + fetch-depth: 2 + - name: Get Release + id: get_release + uses: bruceadams/get-release@74c3d60f5a28f358ccf241a00c9021ea16f0569f + env: + GITHUB_TOKEN: ${{ github.token }} + - name: Download zip from release + uses: robinraju/release-downloader@efa4cd07bd0195e6cc65e9e30c251b49ce4d3e51 + with: + releaseId: ${{ steps.get_release.outputs.id }} + fileName: lambda.zip + out-file-path: "build" + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@8c3f20df09ac63af7b3ae3d7c91f105f857d8497 # v4.0.0 + with: + aws-region: ${{ env.REGION_NAME }} + role-to-assume: ${{env.ROLE_TO_ASSUME }} + role-session-name: github + - run: + sam package --template-file template.yaml --output-template-file packaged.yaml --s3-bucket ${{ + env.LAMBDA_SERVERLESS_REPO }} --region ${{env.REGION_NAME}} + - uses: actions/upload-artifact@v4 + with: + name: packaged.yaml + path: packaged.yaml + release-application: + if: github.event.release.prerelease + needs: download-artifact + runs-on: ubuntu-latest + strategy: + # matrix: ${{ fromJson(needs.setup.outputs.mymatrix) }} + fail-fast: false + matrix: + region: + [ + "us-east-2", + "us-east-1", + "us-west-1", + "us-west-2", + "af-south-1", + "ap-east-1", + "ap-south-2", + "ap-southeast-3", + "ap-southeast-4", + "ap-south-1", + "ap-northeast-3", + "ap-northeast-2", + "ap-southeast-1", + "ap-southeast-2", + "ap-northeast-1", + "ca-central-1", + "eu-central-1", + "eu-west-1", + "eu-west-2", + "eu-south-1", + "eu-west-3", + "eu-south-2", + "eu-north-1", + "eu-central-2", + "il-central-1", + "me-south-1", + "me-central-1", + "sa-east-1", + ] + steps: + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@8c3f20df09ac63af7b3ae3d7c91f105f857d8497 # v4.0.0 + with: + aws-region: ${{ env.REGION_NAME }} + role-to-assume: ${{env.ROLE_TO_ASSUME }} + role-session-name: github + - name: Get Release + id: get_release + uses: bruceadams/get-release@74c3d60f5a28f358ccf241a00c9021ea16f0569f + env: + GITHUB_TOKEN: ${{ github.token }} + - uses: actions/download-artifact@v4 + with: + name: packaged.yaml + - run: | + version=$(echo "${{ steps.get_release.outputs.tag_name }}" | cut -c 2-) + sam publish --template packaged.yaml --region ${{ matrix.region }} --semantic-version $version + continue-on-error: true + release-cloudformation: + needs: release-application + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout Code + uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 + with: + fetch-depth: 2 + - name: Get Release + id: get_release + uses: bruceadams/get-release@74c3d60f5a28f358ccf241a00c9021ea16f0569f + env: + GITHUB_TOKEN: ${{ github.token }} + - run: | + version=$(echo "${{ steps.get_release.outputs.tag_name }}" | cut -c 2-) + sed -i -e "s/0.0.1/${version}/g" cfn_templates/appsync_logging_template.yaml + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@8c3f20df09ac63af7b3ae3d7c91f105f857d8497 # v4.0.0 + with: + aws-region: ${{ env.REGION_NAME }} + role-to-assume: ${{env.ROLE_TO_ASSUME }} + role-session-name: github + - name: copy new cfn to s3 + run: | + aws s3 cp ./cfn_templates/ s3://${{env.CFN_TEMPLATES_BUCKET}}/applications/cfn/ --acl public-read --region ${{env.CFN_TEMPLATES_BUCKET_REGION}} --recursive diff --git a/.github/workflows/draft-release.yaml b/.github/workflows/draft-release.yaml new file mode 100644 index 0000000..738f0d2 --- /dev/null +++ b/.github/workflows/draft-release.yaml @@ -0,0 +1,39 @@ +# Requires a .github/release-drafter.yml file +# https://github.com/release-drafter/release-drafter#example +name: Draft Release +run-name: "@${{ github.triggering_actor }}: ${{ github.ref_name }}: ${{ github.event_name }}" +on: + push: + branches: + - main +permissions: + id-token: write + contents: write + pull-requests: write +jobs: + draft-release: + runs-on: ubuntu-latest + steps: + - name: Checkout Code + uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 + with: + fetch-depth: 2 + - name: Draft Release + id: draft_release + uses: release-drafter/release-drafter@65c5fb495d1e69aa8c08a3317bc44ff8aabe9772 # v5.24.0 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Run build docker image + run: docker build -t lambda-image:latest --target runtime-image -f build_setup/Dockerfile . + - name: create build dir + run: mkdir build -p + - name: extract docker build zip + run: docker run --rm --entrypoint cat lambda-image:latest /src/lambda.zip > build/lambda.zip + - name: Upload lambda.zip To Release + uses: shogo82148/actions-upload-release-asset@dccd6d23e64fd6a746dce6814c0bde0a04886085 # v1.7.2 + with: + upload_url: ${{ steps.draft_release.outputs.upload_url }} + asset_path: build/lambda.zip + asset_name: lambda.zip + asset_content_type: application/zip + overwrite: true diff --git a/.github/workflows/pull-request.yaml b/.github/workflows/pull-request.yaml new file mode 100644 index 0000000..3bea838 --- /dev/null +++ b/.github/workflows/pull-request.yaml @@ -0,0 +1,44 @@ +env: + DOCKER_BUILDKIT: 1 + VERSION: latest + ECR_REPO: sqs-event-processor + DOCKERFILE: build_setup/Dockerfile-bullseye +name: Pull Request +run-name: "@${{ github.triggering_actor }}: ${{ github.head_ref }}: ${{ github.event_name }}" +on: + - pull_request +permissions: + id-token: write + contents: read +defaults: + run: + shell: bash +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true +jobs: + # https://semgrep.dev/docs/semgrep-ci/sample-ci-configs/#sample-github-actions-configuration-file + # Ignores patterns in .gitignore and .semgrepignore files + sast-semgrep: + name: "Static Application Security Testing: Semgrep" + runs-on: ubuntu-latest + container: + image: returntocorp/semgrep + if: (github.actor != 'dependabot[bot]') + steps: + - name: Checkout Code + uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 + - name: Run Semgrep CI + run: semgrep ci + env: + # https://semgrep.dev/explore + SEMGREP_RULES: "p/default" + build-docker-image: + runs-on: ubuntu-latest + steps: + - name: Checkout Code + uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 + with: + fetch-depth: 2 + - name: Run build docker image + run: docker build -t lambda-image:latest -f build_setup/Dockerfile . diff --git a/Makefile b/Makefile index 614d4a3..90892cf 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,12 @@ -build: - cd logs-handler && env GOARCH=amd64 GOOS=linux go build -ldflags="-s -w" -o ../bin/logs-handler +get: + cd src && go get -u ./... + +build: get + cd src && env GOARCH=amd64 GOOS=linux go build -ldflags="-s -w" -o ../bootstrap .PHONY: test test: - go test ./... -race -coverprofile coverage.out -covermode atomic + cd src && go test ./... -race -coverprofile coverage.out -covermode atomic + +test-report: test + cd src && go tool cover -html coverage.out diff --git a/build_setup/Dockerfile b/build_setup/Dockerfile new file mode 100644 index 0000000..b650af2 --- /dev/null +++ b/build_setup/Dockerfile @@ -0,0 +1,13 @@ +ARG GO_VERSION=1.23 + +FROM golang:${GO_VERSION}-bullseye as runtime-image +WORKDIR /aws-appsync-logging-lambda +RUN apt-get update -y && apt-get -yqq install git zip -y +ADD src/ src/ +ADD Makefile Makefile +RUN make build +RUN mkdir /src +RUN zip -r /src/lambda.zip bootstrap + +FROM runtime-image as test +RUN make test diff --git a/build_setup/build.sh b/build_setup/build.sh new file mode 100755 index 0000000..45a2603 --- /dev/null +++ b/build_setup/build.sh @@ -0,0 +1,7 @@ +docker build -t lambda-image:latest -f build_setup/Dockerfile . +mkdir build -p +rm build/* || true +docker run --rm --entrypoint cat lambda-image:latest /src/lambda.zip > build/lambda.zip +sam build +sam package --template-file template.yaml --output-template-file packaged.yaml --s3-bucket firetail-sandbox-eu-west-1-serverless-manual +sam publish --template packaged.yaml --region eu-west-1 --semantic-version 1.0.15 \ No newline at end of file diff --git a/build_setup/get_regions.py b/build_setup/get_regions.py new file mode 100644 index 0000000..259f5a2 --- /dev/null +++ b/build_setup/get_regions.py @@ -0,0 +1,13 @@ +import boto3 + +account_client = boto3.client("account") + +region_res = account_client.list_regions() + +regions = [] + +for region in region_res.get("Regions"): + if region.get("RegionOptStatus") not in ("DISABLING", "DISABLED"): + regions.append(region.get("RegionName")) + +print({"regions": regions}) diff --git a/cfn_templates/appsync_logging_template.yaml b/cfn_templates/appsync_logging_template.yaml new file mode 100644 index 0000000..d21f61b --- /dev/null +++ b/cfn_templates/appsync_logging_template.yaml @@ -0,0 +1,72 @@ +AWSTemplateFormatVersion: "2010-09-09" +Transform: "AWS::Serverless-2016-10-31" +Parameters: + FTAPPKEY: + Type: String + Description: An APP Token from FireTails platform. It will start with FTA-01- + AllowedPattern: "FTA-01-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}" + NoEcho: true + FTAPI: + Type: String + Default: "https://api.logging.eu-west-1.sandbox.firetail.app" + CreateAPIGatewayRole: + Description: > + Creates the required logging role for the apigateway version 1. Gives permissions to the api gateway to send + access logs to cloudwatch log group + Default: no + Type: String + AllowedValues: + - yes + - no + ApplicationId: + Type: String + Default: "arn:aws:serverlessrepo:eu-west-1:453671210445:applications/firetail-appsync-logger" + SemanticVersion: + Type: String + Default: 0.0.1 + OrganisationUUID: + Type: "String" + Description: "The unique identifier for the organisation" + AllowedPattern: "^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$" + FireTailAccountRoleArn: + Type: "String" + Description: "DO NOT CHANGE THIS VALUE" + Default: "arn:aws:iam::453671210445:role/firetail-apigateway-assume-role" +Resources: + AppSyncLogGroup: + Type: "AWS::Logs::LogGroup" + Properties: + RetentionInDays: 7 + LogGroupName: !Sub ${AWS::StackName}-appsync-log + FiretailApp: + Type: AWS::Serverless::Application + Properties: + Location: + ApplicationId: !Ref ApplicationId + SemanticVersion: !Ref SemanticVersion + Parameters: + FTAPPKEY: !Ref FTAPPKEY + FTAPI: !Ref FTAPI + CloudWatchGroupName: !Ref AppSyncLogGroup + SemanticVersion: !Ref SemanticVersion + CustomerRole: + Type: "AWS::IAM::Role" + Properties: + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: "Allow" + Principal: + AWS: !Ref FireTailAccountRoleArn + Action: "sts:AssumeRole" + Condition: + StringEquals: + sts:ExternalId: + Ref: "OrganisationUUID" +Outputs: + FiretailCloudwatchARN: + Description: "Copy this ARN and paste into the Firetail integration form" + Value: + Fn::GetAtt: + - "AppSyncLogGroup" + - "Arn" \ No newline at end of file diff --git a/logs-handler/extract_firetail_logs.go b/src/extract_firetail_logs.go similarity index 100% rename from logs-handler/extract_firetail_logs.go rename to src/extract_firetail_logs.go diff --git a/logs-handler/extract_firetail_logs_test.go b/src/extract_firetail_logs_test.go similarity index 100% rename from logs-handler/extract_firetail_logs_test.go rename to src/extract_firetail_logs_test.go diff --git a/logs-handler/firetail_log.go b/src/firetail_log.go similarity index 100% rename from logs-handler/firetail_log.go rename to src/firetail_log.go diff --git a/logs-handler/firetail_log_test.go b/src/firetail_log_test.go similarity index 100% rename from logs-handler/firetail_log_test.go rename to src/firetail_log_test.go diff --git a/go.mod b/src/go.mod similarity index 100% rename from go.mod rename to src/go.mod diff --git a/go.sum b/src/go.sum similarity index 100% rename from go.sum rename to src/go.sum diff --git a/logs-handler/handler.go b/src/handler.go similarity index 100% rename from logs-handler/handler.go rename to src/handler.go diff --git a/logs-handler/handler_test.go b/src/handler_test.go similarity index 100% rename from logs-handler/handler_test.go rename to src/handler_test.go diff --git a/logs-handler/main.go b/src/main.go similarity index 100% rename from logs-handler/main.go rename to src/main.go diff --git a/logs-handler/main_test.go b/src/main_test.go similarity index 100% rename from logs-handler/main_test.go rename to src/main_test.go diff --git a/logs-handler/parse_headers.go b/src/parse_headers.go similarity index 100% rename from logs-handler/parse_headers.go rename to src/parse_headers.go diff --git a/logs-handler/parse_headers_test.go b/src/parse_headers_test.go similarity index 100% rename from logs-handler/parse_headers_test.go rename to src/parse_headers_test.go diff --git a/logs-handler/send_to_firetail.go b/src/send_to_firetail.go similarity index 100% rename from logs-handler/send_to_firetail.go rename to src/send_to_firetail.go diff --git a/logs-handler/send_to_firetail_test.go b/src/send_to_firetail_test.go similarity index 100% rename from logs-handler/send_to_firetail_test.go rename to src/send_to_firetail_test.go diff --git a/template.yaml b/template.yaml new file mode 100644 index 0000000..64b458d --- /dev/null +++ b/template.yaml @@ -0,0 +1,53 @@ +AWSTemplateFormatVersion: "2010-09-09" +Transform: "AWS::Serverless-2016-10-31" +Description: FireTail Logger for AppSync Access Logging +Metadata: + AWS::ServerlessRepo::Application: + Name: firetail-appsync-logger + Description: Provides a logging mechanism for aws appsync logs + Author: Riley Priddle + SpdxLicenseId: Apache-2.0 + LicenseUrl: LICENSE.txt + ReadmeUrl: README.md + Labels: ["app", "firetail"] + HomePageUrl: https://github.com/firetail-io/firetail-appsync-lambda + SemanticVersion: 1.0.11 + SourceCodeUrl: https://github.com/firetail-io/firetail-appsync-lambda +Parameters: + FTAPPKEY: + Type: String + Description: Firetail APP Key + FTAPI: + Type: String + Default: "https://api.logging.eu-west-1.prod.firetail.app" + CloudWatchGroupName: + Type: String + SemanticVersion: + Type: String + Default: 0.0.1 +Resources: + FireTailAppSyncLogger: + Type: "AWS::Serverless::Function" + Metadata: + BuildMethod: go1.x + Properties: + Handler: bootstrap + Runtime: provided.al2 + CodeUri: build/lambda.zip + Description: FireTail Logger for AppSync + MemorySize: 128 + Timeout: 60 + Events: + CWRule: + Type: CloudWatchLogs + Properties: + LogGroupName: !Ref CloudWatchGroupName + FilterPattern: "" + Environment: + Variables: + FIRETAIL_API: + Ref: FTAPI + FIRETAIL_APP_TOKEN: + Ref: FTAPPKEY + SEMANTIC_VERSION: + Ref: SemanticVersion \ No newline at end of file From 17611d001c218a55a387e874bd48b9ac5779d234 Mon Sep 17 00:00:00 2001 From: riley priddle Date: Tue, 6 May 2025 21:25:10 +0100 Subject: [PATCH 02/18] add license file --- LICENSE.txt | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 LICENSE.txt diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..e69de29 From 3ae127f13f7e8ecdb5efe6c63f77a58d087d44b2 Mon Sep 17 00:00:00 2001 From: riley priddle Date: Tue, 6 May 2025 21:35:59 +0100 Subject: [PATCH 03/18] fix template --- cfn_templates/appsync_logging_template.yaml | 31 --------------------- 1 file changed, 31 deletions(-) diff --git a/cfn_templates/appsync_logging_template.yaml b/cfn_templates/appsync_logging_template.yaml index d21f61b..35a152d 100644 --- a/cfn_templates/appsync_logging_template.yaml +++ b/cfn_templates/appsync_logging_template.yaml @@ -9,29 +9,12 @@ Parameters: FTAPI: Type: String Default: "https://api.logging.eu-west-1.sandbox.firetail.app" - CreateAPIGatewayRole: - Description: > - Creates the required logging role for the apigateway version 1. Gives permissions to the api gateway to send - access logs to cloudwatch log group - Default: no - Type: String - AllowedValues: - - yes - - no ApplicationId: Type: String Default: "arn:aws:serverlessrepo:eu-west-1:453671210445:applications/firetail-appsync-logger" SemanticVersion: Type: String Default: 0.0.1 - OrganisationUUID: - Type: "String" - Description: "The unique identifier for the organisation" - AllowedPattern: "^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$" - FireTailAccountRoleArn: - Type: "String" - Description: "DO NOT CHANGE THIS VALUE" - Default: "arn:aws:iam::453671210445:role/firetail-apigateway-assume-role" Resources: AppSyncLogGroup: Type: "AWS::Logs::LogGroup" @@ -49,20 +32,6 @@ Resources: FTAPI: !Ref FTAPI CloudWatchGroupName: !Ref AppSyncLogGroup SemanticVersion: !Ref SemanticVersion - CustomerRole: - Type: "AWS::IAM::Role" - Properties: - AssumeRolePolicyDocument: - Version: "2012-10-17" - Statement: - - Effect: "Allow" - Principal: - AWS: !Ref FireTailAccountRoleArn - Action: "sts:AssumeRole" - Condition: - StringEquals: - sts:ExternalId: - Ref: "OrganisationUUID" Outputs: FiretailCloudwatchARN: Description: "Copy this ARN and paste into the Firetail integration form" From 6c53f5a77826615a9351a175c678cb724c41e871 Mon Sep 17 00:00:00 2001 From: theteacat Date: Wed, 7 May 2025 10:02:56 +0100 Subject: [PATCH 04/18] go get github.com/aws/aws-sdk-go-v2/config github.com/aws/aws-sdk-go-v2/service/sts --- go.mod | 21 +++++++++++++++++++-- go.sum | 26 ++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index aa3c37d..b3495b5 100644 --- a/go.mod +++ b/go.mod @@ -1,14 +1,31 @@ module aws-golang-simple-http-endpoint -go 1.18 +go 1.22 + +toolchain go1.23.9 require ( github.com/aws/aws-lambda-go v1.35.0 + github.com/aws/aws-sdk-go-v2/config v1.29.14 + github.com/aws/aws-sdk-go-v2/service/sts v1.33.19 github.com/pkg/errors v0.9.1 github.com/stretchr/testify v1.8.1 ) -require github.com/hashicorp/errwrap v1.0.0 // indirect +require ( + github.com/aws/aws-sdk-go-v2 v1.36.3 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.17.67 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 // indirect + github.com/aws/smithy-go v1.22.2 // indirect + github.com/hashicorp/errwrap v1.0.0 // indirect +) require ( github.com/davecgh/go-spew v1.1.1 // indirect diff --git a/go.sum b/go.sum index 21c623c..224c997 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,31 @@ github.com/aws/aws-lambda-go v1.35.0 h1:iocVDy5Cw5SCRrKOPHwarkdFwwy48OkfmHoE6SJ3ATg= github.com/aws/aws-lambda-go v1.35.0/go.mod h1:jwFe2KmMsHmffA1X2R09hH6lFzJQxzI8qK17ewzbQMM= +github.com/aws/aws-sdk-go-v2 v1.36.3 h1:mJoei2CxPutQVxaATCzDUjcZEjVRdpsiiXi2o38yqWM= +github.com/aws/aws-sdk-go-v2 v1.36.3/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg= +github.com/aws/aws-sdk-go-v2/config v1.29.14 h1:f+eEi/2cKCg9pqKBoAIwRGzVb70MRKqWX4dg1BDcSJM= +github.com/aws/aws-sdk-go-v2/config v1.29.14/go.mod h1:wVPHWcIFv3WO89w0rE10gzf17ZYy+UVS1Geq8Iei34g= +github.com/aws/aws-sdk-go-v2/credentials v1.17.67 h1:9KxtdcIA/5xPNQyZRgUSpYOE6j9Bc4+D7nZua0KGYOM= +github.com/aws/aws-sdk-go-v2/credentials v1.17.67/go.mod h1:p3C44m+cfnbv763s52gCqrjaqyPikj9Sg47kUVaNZQQ= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 h1:x793wxmUWVDhshP8WW2mlnXuFrO4cOd3HLBroh1paFw= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30/go.mod h1:Jpne2tDnYiFascUEs2AWHJL9Yp7A5ZVy3TNyxaAjD6M= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 h1:ZK5jHhnrioRkUNOc+hOgQKlUL5JeC3S6JgLxtQ+Rm0Q= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34/go.mod h1:p4VfIceZokChbA9FzMbRGz5OV+lekcVtHlPKEO0gSZY= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 h1:SZwFm17ZUNNg5Np0ioo/gq8Mn6u9w19Mri8DnJ15Jf0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34/go.mod h1:dFZsC0BLo346mvKQLWmoJxT+Sjp+qcVR1tRVHQGOH9Q= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 h1:eAh2A4b5IzM/lum78bZ590jy36+d/aFLgKF/4Vd1xPE= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3/go.mod h1:0yKJC/kb8sAnmlYa6Zs3QVYqaC8ug2AbnNChv5Ox3uA= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 h1:dM9/92u2F1JbDaGooxTq18wmmFzbJRfXfVfy96/1CXM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15/go.mod h1:SwFBy2vjtA0vZbjjaFtfN045boopadnoVPhu4Fv66vY= +github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 h1:1Gw+9ajCV1jogloEv1RRnvfRFia2cL6c9cuKV2Ps+G8= +github.com/aws/aws-sdk-go-v2/service/sso v1.25.3/go.mod h1:qs4a9T5EMLl/Cajiw2TcbNt2UNo/Hqlyp+GiuG4CFDI= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 h1:hXmVKytPfTy5axZ+fYbR5d0cFmC3JvwLm5kM83luako= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1/go.mod h1:MlYRNmYu/fGPoxBQVvBYr9nyr948aY/WLUvwBMBJubs= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.19 h1:1XuUZ8mYJw9B6lzAkXhqHlJd/XvaX32evhproijJEZY= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.19/go.mod h1:cQnB8CUnxbMU82JvlqjKR2HBOm3fe9pWorWBza6MBJ4= +github.com/aws/smithy-go v1.22.2 h1:6D9hW43xKFrRx/tXXfAlIZc4JI+yQe6snnWcQyxSyLQ= +github.com/aws/smithy-go v1.22.2/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= From e591673b8a82ef14552da5ad3fdff8a132376c00 Mon Sep 17 00:00:00 2001 From: theteacat Date: Wed, 7 May 2025 10:03:14 +0100 Subject: [PATCH 05/18] Add RegionName and AccountID to FiretailLog struct --- logs-handler/firetail_log.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/logs-handler/firetail_log.go b/logs-handler/firetail_log.go index c7e3dc8..ed79042 100644 --- a/logs-handler/firetail_log.go +++ b/logs-handler/firetail_log.go @@ -38,6 +38,8 @@ type FiretailLog struct { RequestSummary *json.RawMessage `json:"requestSummary,omitempty"` ResponseHeaders *json.RawMessage `json:"responseHeaders,omitempty"` ResponseMappings *[]json.RawMessage `json:"responseMappings,omitempty"` + RegionName string `json:"regionName"` + AccountID string `json:"accountId"` } func (f *FiretailLog) IsPopulated() bool { From 22f561cff9dc362ac1f6522a615de5f10ac1e700 Mon Sep 17 00:00:00 2001 From: theteacat Date: Wed, 7 May 2025 10:03:44 +0100 Subject: [PATCH 06/18] Implement getting account ID and region name from STS and env vars respectively --- logs-handler/main.go | 50 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 47 insertions(+), 3 deletions(-) diff --git a/logs-handler/main.go b/logs-handler/main.go index 7aa3276..df23be5 100644 --- a/logs-handler/main.go +++ b/logs-handler/main.go @@ -1,26 +1,70 @@ package main import ( + "context" "os" "github.com/aws/aws-lambda-go/lambda" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/sts" + "github.com/pkg/errors" ) const DefaultFiretailApiUrl string = "https://api.logging.eu-west-1.prod.firetail.app/logs/aws/appsync" var firetailApiUrl string var firetailApiToken string +var regionName string -func loadEnvVars() { +func loadEnvVars() error { var firetailApiUrlSet bool firetailApiUrl, firetailApiUrlSet = os.LookupEnv("FIRETAIL_API_URL") if !firetailApiUrlSet { firetailApiUrl = DefaultFiretailApiUrl } - firetailApiToken = os.Getenv("FIRETAIL_API_TOKEN") + + var firetailApiTokenSet bool + firetailApiToken, firetailApiTokenSet = os.LookupEnv("FIRETAIL_API_TOKEN") + if !firetailApiTokenSet { + return errors.New("FIRETAIL_API_TOKEN is not set") + } + + var regionNameExists bool + regionName, regionNameExists = os.LookupEnv("AWS_REGION") + if !regionNameExists { + regionName, regionNameExists = os.LookupEnv("AWS_DEFAULT_REGION") + } + if !regionNameExists { + return errors.New("Neither AWS_REGION nor AWS_DEFAULT_REGION are set. Cannot continue.") + } + + return nil +} + +var accountID string + +func loadAccountID() error { + cfg, err := config.LoadDefaultConfig(context.TODO()) + if err != nil { + return errors.Errorf("Failed to determine account ID: %v", err) + } + client := sts.NewFromConfig(cfg) + resp, err := client.GetCallerIdentity(context.TODO(), &sts.GetCallerIdentityInput{}) + if err != nil { + return errors.Errorf("Failed to determine account ID: %v", err) + } + accountID = *resp.Account + return nil } func main() { - loadEnvVars() + err := loadAccountID() + if err != nil { + panic(err) + } + err = loadEnvVars() + if err != nil { + panic(err) + } lambda.Start(Handler) } From ab6d8559cdc3d23c3a78ca43d27204ed75150647 Mon Sep 17 00:00:00 2001 From: theteacat Date: Wed, 7 May 2025 10:04:28 +0100 Subject: [PATCH 07/18] Add RegionName and AccountID to initialisation of FireTail log structs --- logs-handler/extract_firetail_logs.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/logs-handler/extract_firetail_logs.go b/logs-handler/extract_firetail_logs.go index af90ecd..77205a8 100644 --- a/logs-handler/extract_firetail_logs.go +++ b/logs-handler/extract_firetail_logs.go @@ -44,7 +44,9 @@ func ExtractFiretailLogs(logsData *events.CloudwatchLogsData) (map[string]*Firet firetailLog, firetailLogExists := firetailLogs[requestID] if !firetailLogExists { firetailLog = &FiretailLog{ - RequestID: requestID, + RequestID: requestID, + RegionName: regionName, + AccountID: accountID, } } From 1e76d4739c0e4dbc94563bdffde416465213c9c2 Mon Sep 17 00:00:00 2001 From: theteacat Date: Wed, 7 May 2025 10:04:54 +0100 Subject: [PATCH 08/18] Remove redundant nil check --- logs-handler/handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logs-handler/handler.go b/logs-handler/handler.go index c76b36b..cb0a63b 100644 --- a/logs-handler/handler.go +++ b/logs-handler/handler.go @@ -19,7 +19,7 @@ func Handler(ctx context.Context, event events.CloudwatchLogsEvent) error { if err != nil { log.Println("Errs extracting Firetail logs:", err.Error()) } - if firetailLogs == nil || len(firetailLogs) == 0 { + if len(firetailLogs) == 0 { log.Println("Generated no Firetail logs from this batch. Exiting...") return nil } From 40e2a29d4312c4457d01ac0b65aabe6112e9226e Mon Sep 17 00:00:00 2001 From: theteacat Date: Wed, 7 May 2025 10:05:04 +0100 Subject: [PATCH 09/18] update expected logs in tests --- logs-handler/handler_test.go | 2 +- logs-handler/send_to_firetail_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/logs-handler/handler_test.go b/logs-handler/handler_test.go index 0b6de99..f0e02b6 100644 --- a/logs-handler/handler_test.go +++ b/logs-handler/handler_test.go @@ -20,7 +20,7 @@ import ( func TestHandler(t *testing.T) { testData := "H4sIAAAAAAAAAO1ae3PbNhL/Kjj9lXRECS8SpDKcjuM4ridx01Zum6uUyUAkJNHmQwVBO7bH3/0WFCVbku3Kj2smvbM9EogFFovfPgHzspWpspQTdXQ+U61e683O0c7nw71+f2d/r9VuFWe50tDNXeYJQgnm3IXutJjs66KaAaUrz8qunM3K8zyC76TsptH5l/NTqqt8MhWTeGrEeaqruDw+n8/sG61kBlOxwMwdK48LaHiuT+KxckfCi9WIc8ZGeMRlFMR8FKuIR5SNY6UkAaIngoDHGNiV1aiMdDIzSZG/TVKjdNnqDVrjRCsjk9RpBHNSmY1i6cTq1HlfTMo3spxOZR6nStvH/gaX3bSo4t+liaZAJ86ZSw05xvLs4req9anexd6pyo1d7LKVxLAZJigTAWUcY8KIH/gupQwHxIMWdFMPU9+1zYBjV1AqfOHDWLsJk4AOjMwATuLBeOxR5mHstxe6sVip8dj1RthxIzl2uJC+Eyg/doQkgquAST8eoddqkuToF/VnBfxaV+1N0ajnCkyDgLkYsGcuLOYTnwji+ZT6PsOCg/Tc8zweiDtFI/zBou1rOZv+/B79XCl93kNZZaRFGx2eHy6al8McoQhsw6ifitK8SPJZZXro0iQmVT00bO2gIxAGWeKwdfVyPgGhJLbfV8Mc/ob5n3YBYFsvNB8yUWbOMLZcpMsBfeE7YGuxw0c0cPyIeg5WnMuAEjmSZNha445QLUWzEEJpUtY8yxdpkiUgJcHXM4zKysXDNYOraynb6MNM6XrXvYWobfSb1IkcpaqEPV/dqj0BH8T3OAVV8MDzwIBcjj1BXRK4nHOPMWte1HVh+J3aY+4jDWvvi4qqWlUOsuEC/SgzUMwcafAm9DZRadz0NqA/eR/ulvu4HFqvtGINW71hq3GCQwgAST4ZttrD1kyaKdAGw1Yj2xBceQixAoS2MtfzliSgaFUW6anSOzqvaVLnPYh2vSao9FTlnMEaDundDI+9vwiCXQMylt0ate5iibK7unAt/EFcL7uNhuppUZEb9QVY9C6tsJMqsyFq/pjMeW1n/FfADPAuLVyX9qGozJ7WhbbMBp+uFqgd5HUvdI5lWiroVtejasQ1SLDUSb3lWtLJPBrs/HTQ7PFuuOrxRsu8HBc6U/GRymYphIh62uXQOtVwOGxZDMEybbNXd1BMhIOpQ3371F4OLBaedz10X5kD8NnVcSfqfD5ivsa8E1Bc9A1b/bptP7ZCdVjjWgcB+3UFj092Du8rO/kyDD55J3cnm8e7+VK62xz9BvHvdvX1pZ/d2eucBE2C/8c8uR/JfHXQuC7n5iNQXqUpuqbNYapJBF935wDrUXGiGr520n0OSzEV3LN2TnxOBfWpRz1GmB8QIjwfu4RB3eULQgLm3e2wAbvXzMtZkZfq/+nssekMJKtS84jJ1mbr0s9OWa1AvxnfSuJwm82250VueGOXz2L0d8f2VaPfKkvtLVPQMlPdKiAh9hDjczjH+Cxw4Ujm04DCUcfncA7zOAf5GQETJ5jRuwUUj/bK/2cfvOF49mRk0XlOJ7zmhWMplcdAcE+OHKteB+TljqIR86QYY6xGG7z6CvYSr7C06iohk+Qq3i2q3ApPvyFntyCHg6c4fRvZ2dvAeT17HUdAES0zaVgnUXQT1JDCs5HaqHjH1PQ7Y81DXJlt6cpfLdbcfS7fiDVx1dQ3rZ5HIaxC5dxejUBLsfpVlsnGkh7p0rUyjpImQEGApw4hDsNHhPQw67leB0pxQr0/6tEqj+8fGwiG6R9NTISKzsZHGwPqZT6Mx6WyfuVCriDtla3ygLie9adFIQgg2SeZJvFizAYfwqgI8CojwYQrrh7qWk83wbtvLB5hgvND2ZGWESD4ZNHuKT/vMT7Xp76HfbJmfI1QjytG19Tnupzxb7BGvTOca2UqnS8Jy3X+Zlu8p/S6R+FcCPEwZbfnSXhT57Z3Q92ubyV02bPhfkONK7AfvPnX10DdfxTqlHjr8f2vUW9KmU3gG8Jt2AtOXPxfx75vNIj+VfAPHpdjIQrBIcbdSgcPqO7X8IdTEWXfZNH/kGC3W0ClF82R/dsNgN/9X7V78xzx/Icqv7081dwS++aEDROAYg5TTsSzuuAa4GsqGdghn76GKsjjYiFn28XCW1XRxg/LR6AQCnuh/9B8xB9wv3FDB8QTT3CHNnmwDhicBYJ/qg6e/WD6TGcC/oAD6S3/eXmGg+dDbz5AUFOVu0VsBaHY+rq9/cgjIPf8ANc/T1fYsx7jGrTQD0rGkI976LK+QcuNk6p8YqbhgOHgUxtpNVZa6XAwNWZW9rrdZRXQgQmQzVUHaoSOzORFkUNX1oVJkX1ZZayBoXOaqDOlncjesOjzcPDje6CXKnLGykRTpwTnDAeRLsqybgPxiyOzi9xpdJfE4WCbDcHEQidwNN1e0sVaTlWChIBdbsLBzu99Z3c+3NmZzfr2JaJ64LjQZ1LHKnZmhYaBnDPoP01kOKAdjKQ/8nw4QiiiqCtcwtk4EIL74/FoRAOJO9eYdHJl0Iv6hZ639vllgwjAUUknK0ZJCph8j28FUpawQ0KYt0pNSidW5YkpZs3AcGB0ZeGcQtQBhN34gqliZI4jJWZcwSrspEhOZ7KzeCMJ6rjONWRzmCxgc6QW1mGruXAAc9IkquNy97gs8hWdZuAIsGKhy3XgdGGKRj9AklGkZmBvMp9UgH44ULmz/7oNn7/2X/0Z4k4A7brhrzGCVjgQokME69AA/tqIuB3i+h2OoXPJe1PQNrK3xN1ZKhNof9f9bgPGEqKHMadLGOtb05saCiHB7BfFJFVod6oLm0JenYbDFsECogEaturepMrW+38sTLjz/Wst87ghUW6z0cLiDcRP5ViD/6WAocTxmC+YH4wcwWLw85HgsRsHYyi9PSGk9OWG8Ma+rmM2ZP9ileucKPC/7+AnmySxTlaszl7X2rtbu7lMRh/6tWA3mP+VFiHaFTFE/3AwuUhmbRSrsY2BbTTSSz+LxvXupqf8Y5p93DkxR95v7gU9m73bOzu+iPRH/W7n6Dh+Y/59nJrZ++hg//MPb7LXUfzlon8WhsDnpqceFhdJmsquC9734tCmHlOU01foAAw1RdCBPvTRR0TwZ+J+Fi8ReHOqflejd4npukx0mIdevPvh6PB9G6XJiUL7KjopXjY67YLWOtj+or4cS500UzYQn7vrrdYy9wZwSxBWZTNz/unpOWDb+7Itc8D830U3ksBu4+Y2q4brzvMKRVOpoToKfz166/hP3822l0Fb7aa+1y+RDd5VpuIeIk+Wb9trk61LpOWbkJ+u/gM4Pc+p6CoAAA==" - expectedPayload := "{\"executionSummary\":{\"duration\":62176672,\"logType\":\"ExecutionSummary\",\"requestId\":\"0eff56b0-5caf-47a8-9e8d-7a174e93a8db\",\"startTime\":\"2022-11-30T11:03:56.035126Z\",\"endTime\":\"2022-11-30T11:03:56.097302Z\",\"parsing\":{\"startOffset\":56801,\"duration\":49156},\"version\":1,\"validation\":{\"startOffset\":132790,\"duration\":73757},\"graphQLAPIId\":\"lcyxyv2rungh7gdht7ylrudsjy\"},\"query\":\"mutation MyMutation {\\n createPost(input: {title: \\\"A Test Post\\\"}) {\\n id\\n }\\n}\\n\\nquery MyQuery {\\n getPost(id: \\\"a5422778-e5bd-4b29-8c26-0e44a921aba1\\\") {\\n id\\n title\\n }\\n listPosts(limit: 10) {\\n items {\\n id\\n }\\n }\\n}\\n, Operation: MyQuery, Variables: {}\",\"request_id\":\"0eff56b0-5caf-47a8-9e8d-7a174e93a8db\",\"requestHeaders\":{\"accept\":[\"application/json, text/plain, */*\"],\"accept-encoding\":[\"gzip, deflate, br\"],\"accept-language\":[\"en-GB,en-US;q=0.9,en;q=0.8\"],\"cloudfront-forwarded-proto\":[\"https\"],\"cloudfront-is-desktop-viewer\":[\"true\"],\"cloudfront-is-mobile-viewer\":[\"false\"],\"cloudfront-is-smarttv-viewer\":[\"false\"],\"cloudfront-is-tablet-viewer\":[\"false\"],\"cloudfront-viewer-asn\":[\"1136\"],\"cloudfront-viewer-country\":[\"NL\"],\"content-length\":[\"309\"],\"content-type\":[\"application/json\"],\"host\":[\"c5dz3eobtjce7p4emob3koivpa.appsync-api.eu-west-1.amazonaws.com\"],\"origin\":[\"https://eu-west-1.console.aws.amazon.com\"],\"referer\":[\"https://eu-west-1.console.aws.amazon.com/\"],\"sec-ch-ua\":[\"\\\"Google Chrome\\\";v=\\\"107\\\", \\\"Chromium\\\";v=\\\"107\\\", \\\"Not=A?Brand\\\";v=\\\"24\\\"\"],\"sec-ch-ua-mobile\":[\"?0\"],\"sec-ch-ua-platform\":[\"\\\"macOS\\\"\"],\"sec-fetch-dest\":[\"empty\"],\"sec-fetch-mode\":[\"cors\"],\"sec-fetch-site\":[\"cross-site\"],\"user-agent\":[\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36\"],\"via\":[\"2.0 a8b68315e1e2575143f97748ffbb29a0.cloudfront.net (CloudFront)\"],\"x-amz-cf-id\":[\"hv4XlmXAktT6V5z2wpKEwjzcrXrKATjdDtYjltpLcIG_HDmBcdxzSw==\"],\"x-amz-user-agent\":[\"AWS-Console-AppSync/\"],\"x-amzn-requestid\":[\"0eff56b0-5caf-47a8-9e8d-7a174e93a8db\"],\"x-amzn-trace-id\":[\"Root=1-6387389b-73d623b74d5d9f671677aa8a\"],\"x-api-key\":[\"****mgidri\"],\"x-forwarded-for\":[\"77.173.29.29, 15.158.40.17\"],\"x-forwarded-port\":[\"443\"],\"x-forwarded-proto\":[\"https\"]},\"requestMappings\":[{\"logType\":\"RequestMapping\",\"path\":[\"getPost\"],\"fieldName\":\"getPost\",\"resolverArn\":\"arn:aws:appsync:eu-west-1:453671210445:apis/lcyxyv2rungh7gdht7ylrudsjy/types/Query/resolvers/getPost\",\"requestId\":\"0eff56b0-5caf-47a8-9e8d-7a174e93a8db\",\"context\":{\"arguments\":{\"id\":\"a5422778-e5bd-4b29-8c26-0e44a921aba1\"},\"stash\":{},\"outErrors\":[]},\"fieldInError\":false,\"errors\":[],\"parentType\":\"Query\",\"graphQLAPIId\":\"lcyxyv2rungh7gdht7ylrudsjy\",\"transformedTemplate\":\"{\\n \\\"version\\\": \\\"2017-02-28\\\",\\n \\\"operation\\\": \\\"GetItem\\\",\\n \\\"key\\\": {\\n \\\"id\\\": {\\\"S\\\":\\\"a5422778-e5bd-4b29-8c26-0e44a921aba1\\\"},\\n },\\n}\"},{\"logType\":\"RequestMapping\",\"path\":[\"listPosts\"],\"fieldName\":\"listPosts\",\"resolverArn\":\"arn:aws:appsync:eu-west-1:453671210445:apis/lcyxyv2rungh7gdht7ylrudsjy/types/Query/resolvers/listPosts\",\"requestId\":\"0eff56b0-5caf-47a8-9e8d-7a174e93a8db\",\"context\":{\"arguments\":{\"limit\":10},\"stash\":{},\"outErrors\":[]},\"fieldInError\":false,\"errors\":[],\"parentType\":\"Query\",\"graphQLAPIId\":\"lcyxyv2rungh7gdht7ylrudsjy\",\"transformedTemplate\":\"{\\n \\\"version\\\": \\\"2017-02-28\\\",\\n \\\"operation\\\": \\\"Scan\\\",\\n \\\"filter\\\": null ,\\n \\\"limit\\\": 10,\\n \\\"nextToken\\\": null,\\n}\"}],\"requestSummary\":{\"logType\":\"RequestSummary\",\"requestId\":\"0eff56b0-5caf-47a8-9e8d-7a174e93a8db\",\"graphQLAPIId\":\"lcyxyv2rungh7gdht7ylrudsjy\",\"statusCode\":200,\"latency\":89000000},\"responseHeaders\":{\"Content-Type\":\"application/json; charset=UTF-8\"},\"responseMappings\":[{\"logType\":\"ResponseMapping\",\"path\":[\"getPost\"],\"fieldName\":\"getPost\",\"resolverArn\":\"arn:aws:appsync:eu-west-1:453671210445:apis/lcyxyv2rungh7gdht7ylrudsjy/types/Query/resolvers/getPost\",\"requestId\":\"0eff56b0-5caf-47a8-9e8d-7a174e93a8db\",\"context\":{\"arguments\":{\"id\":\"a5422778-e5bd-4b29-8c26-0e44a921aba1\"},\"result\":{\"id\":\"a5422778-e5bd-4b29-8c26-0e44a921aba1\",\"title\":\"A Test Post\"},\"stash\":{},\"outErrors\":[]},\"fieldInError\":false,\"errors\":[],\"parentType\":\"Query\",\"graphQLAPIId\":\"lcyxyv2rungh7gdht7ylrudsjy\",\"transformedTemplate\":\"{id=a5422778-e5bd-4b29-8c26-0e44a921aba1, title=A Test Post}\"},{\"logType\":\"ResponseMapping\",\"path\":[\"listPosts\"],\"fieldName\":\"listPosts\",\"resolverArn\":\"arn:aws:appsync:eu-west-1:453671210445:apis/lcyxyv2rungh7gdht7ylrudsjy/types/Query/resolvers/listPosts\",\"requestId\":\"0eff56b0-5caf-47a8-9e8d-7a174e93a8db\",\"context\":{\"arguments\":{\"limit\":10},\"result\":{\"items\":[{\"id\":\"a5422778-e5bd-4b29-8c26-0e44a921aba1\",\"title\":\"A Test Post\"},{\"id\":\"0daae63f-46ab-4631-8db4-e2c36a7f00eb\",\"title\":\"A Second Test Post\"}],\"scannedCount\":2},\"stash\":{},\"outErrors\":[]},\"fieldInError\":false,\"errors\":[],\"parentType\":\"Query\",\"graphQLAPIId\":\"lcyxyv2rungh7gdht7ylrudsjy\",\"transformedTemplate\":\"{items=[{id=a5422778-e5bd-4b29-8c26-0e44a921aba1, title=A Test Post}, {id=0daae63f-46ab-4631-8db4-e2c36a7f00eb, title=A Second Test Post}], nextToken=null, scannedCount=2, startedAt=null}\"}]}\n" + expectedPayload := "{\"executionSummary\":{\"duration\":62176672,\"logType\":\"ExecutionSummary\",\"requestId\":\"0eff56b0-5caf-47a8-9e8d-7a174e93a8db\",\"startTime\":\"2022-11-30T11:03:56.035126Z\",\"endTime\":\"2022-11-30T11:03:56.097302Z\",\"parsing\":{\"startOffset\":56801,\"duration\":49156},\"version\":1,\"validation\":{\"startOffset\":132790,\"duration\":73757},\"graphQLAPIId\":\"lcyxyv2rungh7gdht7ylrudsjy\"},\"query\":\"mutation MyMutation {\\n createPost(input: {title: \\\"A Test Post\\\"}) {\\n id\\n }\\n}\\n\\nquery MyQuery {\\n getPost(id: \\\"a5422778-e5bd-4b29-8c26-0e44a921aba1\\\") {\\n id\\n title\\n }\\n listPosts(limit: 10) {\\n items {\\n id\\n }\\n }\\n}\\n, Operation: MyQuery, Variables: {}\",\"request_id\":\"0eff56b0-5caf-47a8-9e8d-7a174e93a8db\",\"requestHeaders\":{\"accept\":[\"application/json, text/plain, */*\"],\"accept-encoding\":[\"gzip, deflate, br\"],\"accept-language\":[\"en-GB,en-US;q=0.9,en;q=0.8\"],\"cloudfront-forwarded-proto\":[\"https\"],\"cloudfront-is-desktop-viewer\":[\"true\"],\"cloudfront-is-mobile-viewer\":[\"false\"],\"cloudfront-is-smarttv-viewer\":[\"false\"],\"cloudfront-is-tablet-viewer\":[\"false\"],\"cloudfront-viewer-asn\":[\"1136\"],\"cloudfront-viewer-country\":[\"NL\"],\"content-length\":[\"309\"],\"content-type\":[\"application/json\"],\"host\":[\"c5dz3eobtjce7p4emob3koivpa.appsync-api.eu-west-1.amazonaws.com\"],\"origin\":[\"https://eu-west-1.console.aws.amazon.com\"],\"referer\":[\"https://eu-west-1.console.aws.amazon.com/\"],\"sec-ch-ua\":[\"\\\"Google Chrome\\\";v=\\\"107\\\", \\\"Chromium\\\";v=\\\"107\\\", \\\"Not=A?Brand\\\";v=\\\"24\\\"\"],\"sec-ch-ua-mobile\":[\"?0\"],\"sec-ch-ua-platform\":[\"\\\"macOS\\\"\"],\"sec-fetch-dest\":[\"empty\"],\"sec-fetch-mode\":[\"cors\"],\"sec-fetch-site\":[\"cross-site\"],\"user-agent\":[\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36\"],\"via\":[\"2.0 a8b68315e1e2575143f97748ffbb29a0.cloudfront.net (CloudFront)\"],\"x-amz-cf-id\":[\"hv4XlmXAktT6V5z2wpKEwjzcrXrKATjdDtYjltpLcIG_HDmBcdxzSw==\"],\"x-amz-user-agent\":[\"AWS-Console-AppSync/\"],\"x-amzn-requestid\":[\"0eff56b0-5caf-47a8-9e8d-7a174e93a8db\"],\"x-amzn-trace-id\":[\"Root=1-6387389b-73d623b74d5d9f671677aa8a\"],\"x-api-key\":[\"****mgidri\"],\"x-forwarded-for\":[\"77.173.29.29, 15.158.40.17\"],\"x-forwarded-port\":[\"443\"],\"x-forwarded-proto\":[\"https\"]},\"requestMappings\":[{\"logType\":\"RequestMapping\",\"path\":[\"getPost\"],\"fieldName\":\"getPost\",\"resolverArn\":\"arn:aws:appsync:eu-west-1:453671210445:apis/lcyxyv2rungh7gdht7ylrudsjy/types/Query/resolvers/getPost\",\"requestId\":\"0eff56b0-5caf-47a8-9e8d-7a174e93a8db\",\"context\":{\"arguments\":{\"id\":\"a5422778-e5bd-4b29-8c26-0e44a921aba1\"},\"stash\":{},\"outErrors\":[]},\"fieldInError\":false,\"errors\":[],\"parentType\":\"Query\",\"graphQLAPIId\":\"lcyxyv2rungh7gdht7ylrudsjy\",\"transformedTemplate\":\"{\\n \\\"version\\\": \\\"2017-02-28\\\",\\n \\\"operation\\\": \\\"GetItem\\\",\\n \\\"key\\\": {\\n \\\"id\\\": {\\\"S\\\":\\\"a5422778-e5bd-4b29-8c26-0e44a921aba1\\\"},\\n },\\n}\"},{\"logType\":\"RequestMapping\",\"path\":[\"listPosts\"],\"fieldName\":\"listPosts\",\"resolverArn\":\"arn:aws:appsync:eu-west-1:453671210445:apis/lcyxyv2rungh7gdht7ylrudsjy/types/Query/resolvers/listPosts\",\"requestId\":\"0eff56b0-5caf-47a8-9e8d-7a174e93a8db\",\"context\":{\"arguments\":{\"limit\":10},\"stash\":{},\"outErrors\":[]},\"fieldInError\":false,\"errors\":[],\"parentType\":\"Query\",\"graphQLAPIId\":\"lcyxyv2rungh7gdht7ylrudsjy\",\"transformedTemplate\":\"{\\n \\\"version\\\": \\\"2017-02-28\\\",\\n \\\"operation\\\": \\\"Scan\\\",\\n \\\"filter\\\": null ,\\n \\\"limit\\\": 10,\\n \\\"nextToken\\\": null,\\n}\"}],\"requestSummary\":{\"logType\":\"RequestSummary\",\"requestId\":\"0eff56b0-5caf-47a8-9e8d-7a174e93a8db\",\"graphQLAPIId\":\"lcyxyv2rungh7gdht7ylrudsjy\",\"statusCode\":200,\"latency\":89000000},\"responseHeaders\":{\"Content-Type\":\"application/json; charset=UTF-8\"},\"responseMappings\":[{\"logType\":\"ResponseMapping\",\"path\":[\"getPost\"],\"fieldName\":\"getPost\",\"resolverArn\":\"arn:aws:appsync:eu-west-1:453671210445:apis/lcyxyv2rungh7gdht7ylrudsjy/types/Query/resolvers/getPost\",\"requestId\":\"0eff56b0-5caf-47a8-9e8d-7a174e93a8db\",\"context\":{\"arguments\":{\"id\":\"a5422778-e5bd-4b29-8c26-0e44a921aba1\"},\"result\":{\"id\":\"a5422778-e5bd-4b29-8c26-0e44a921aba1\",\"title\":\"A Test Post\"},\"stash\":{},\"outErrors\":[]},\"fieldInError\":false,\"errors\":[],\"parentType\":\"Query\",\"graphQLAPIId\":\"lcyxyv2rungh7gdht7ylrudsjy\",\"transformedTemplate\":\"{id=a5422778-e5bd-4b29-8c26-0e44a921aba1, title=A Test Post}\"},{\"logType\":\"ResponseMapping\",\"path\":[\"listPosts\"],\"fieldName\":\"listPosts\",\"resolverArn\":\"arn:aws:appsync:eu-west-1:453671210445:apis/lcyxyv2rungh7gdht7ylrudsjy/types/Query/resolvers/listPosts\",\"requestId\":\"0eff56b0-5caf-47a8-9e8d-7a174e93a8db\",\"context\":{\"arguments\":{\"limit\":10},\"result\":{\"items\":[{\"id\":\"a5422778-e5bd-4b29-8c26-0e44a921aba1\",\"title\":\"A Test Post\"},{\"id\":\"0daae63f-46ab-4631-8db4-e2c36a7f00eb\",\"title\":\"A Second Test Post\"}],\"scannedCount\":2},\"stash\":{},\"outErrors\":[]},\"fieldInError\":false,\"errors\":[],\"parentType\":\"Query\",\"graphQLAPIId\":\"lcyxyv2rungh7gdht7ylrudsjy\",\"transformedTemplate\":\"{items=[{id=a5422778-e5bd-4b29-8c26-0e44a921aba1, title=A Test Post}, {id=0daae63f-46ab-4631-8db4-e2c36a7f00eb, title=A Second Test Post}], nextToken=null, scannedCount=2, startedAt=null}\"}],\"regionName\":\"\",\"accountId\":\"\"}\n" wg := &sync.WaitGroup{} wg.Add(1) diff --git a/logs-handler/send_to_firetail_test.go b/logs-handler/send_to_firetail_test.go index 2f20ba0..f9ee439 100644 --- a/logs-handler/send_to_firetail_test.go +++ b/logs-handler/send_to_firetail_test.go @@ -17,7 +17,7 @@ func TestSendToFiretail(t *testing.T) { testServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { requestBody, err := ioutil.ReadAll(r.Body) require.Nil(t, err) - assert.Equal(t, "{\"query\":\"TEST_QUERY\",\"request_id\":\"TEST_ID\"}\n", string(requestBody)) + assert.Equal(t, "{\"query\":\"TEST_QUERY\",\"request_id\":\"TEST_ID\",\"regionName\":\"\",\"accountId\":\"\"}\n", string(requestBody)) wg.Done() w.Write([]byte(`{"message":"success"}`)) })) @@ -40,7 +40,7 @@ func TestSendToFiretailBadServer(t *testing.T) { testServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { requestBody, err := ioutil.ReadAll(r.Body) require.Nil(t, err) - assert.Equal(t, "{\"query\":\"TEST_QUERY\",\"request_id\":\"TEST_ID\"}\n", string(requestBody)) + assert.Equal(t, "{\"query\":\"TEST_QUERY\",\"request_id\":\"TEST_ID\",\"regionName\":\"\",\"accountId\":\"\"}\n", string(requestBody)) wg.Done() w.Write([]byte(`{"message":"fail"}`)) })) From cb5c6d305c5bfd4eb1afbd1eb39b6a4d0d01eaae Mon Sep 17 00:00:00 2001 From: theteacat Date: Wed, 7 May 2025 10:20:29 +0100 Subject: [PATCH 10/18] bump semantic version in sandbox workflow --- .github/workflows/deploy-sandbox.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-sandbox.yaml b/.github/workflows/deploy-sandbox.yaml index 8288d62..e455306 100644 --- a/.github/workflows/deploy-sandbox.yaml +++ b/.github/workflows/deploy-sandbox.yaml @@ -7,7 +7,7 @@ env: CFN_TEMPLATES_BUCKET: firetail-sandbox-us-east-1-cf-templates CFN_TEMPLATES_BUCKET_REGION: us-east-1 APP_NAME: aws-appsync-logging-lambda - SEMANTIC_VERSION: 1.1.39 + SEMANTIC_VERSION: 1.2.1 name: Deploy to Sandbox run-name: "@${{ github.triggering_actor }}: ${{ github.ref_name }}: ${{ github.event_name }}" on: From 242859627e3cf519d0ca9c272f498b65c6377e2c Mon Sep 17 00:00:00 2001 From: theteacat Date: Wed, 7 May 2025 10:40:35 +0100 Subject: [PATCH 11/18] load app token from FIRETAIL_APP_TOKEN --- src/main.go | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/main.go b/src/main.go index df23be5..d4f61cc 100644 --- a/src/main.go +++ b/src/main.go @@ -13,7 +13,8 @@ import ( const DefaultFiretailApiUrl string = "https://api.logging.eu-west-1.prod.firetail.app/logs/aws/appsync" var firetailApiUrl string -var firetailApiToken string +var firetailToken string +var firetailTokenHeaderName string = "x-ft-api-key" var regionName string func loadEnvVars() error { @@ -23,10 +24,14 @@ func loadEnvVars() error { firetailApiUrl = DefaultFiretailApiUrl } - var firetailApiTokenSet bool - firetailApiToken, firetailApiTokenSet = os.LookupEnv("FIRETAIL_API_TOKEN") - if !firetailApiTokenSet { - return errors.New("FIRETAIL_API_TOKEN is not set") + var firetailTokenSet bool + firetailToken, firetailTokenSet = os.LookupEnv("FIRETAIL_API_TOKEN") + if !firetailTokenSet { + firetailToken, firetailTokenSet = os.LookupEnv("FIRETAIL_APP_TOKEN") + firetailTokenHeaderName = "x-ft-app-key" + } + if !firetailTokenSet { + return errors.New("Neither FIRETAIL_API_TOKEN or FIRETAIL_APP_TOKEN are set. Cannot continue.") } var regionNameExists bool @@ -35,7 +40,7 @@ func loadEnvVars() error { regionName, regionNameExists = os.LookupEnv("AWS_DEFAULT_REGION") } if !regionNameExists { - return errors.New("Neither AWS_REGION nor AWS_DEFAULT_REGION are set. Cannot continue.") + return errors.New("Neither AWS_REGION or AWS_DEFAULT_REGION are set. Cannot continue.") } return nil From 59ac88686d10bfdf0933452dd1c46318eb5b81f1 Mon Sep 17 00:00:00 2001 From: theteacat Date: Wed, 7 May 2025 10:41:01 +0100 Subject: [PATCH 12/18] update SendToFiretail to take header name for token as arg --- src/handler.go | 2 +- src/send_to_firetail.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/handler.go b/src/handler.go index cb0a63b..170bc63 100644 --- a/src/handler.go +++ b/src/handler.go @@ -33,5 +33,5 @@ func Handler(ctx context.Context, event events.CloudwatchLogsEvent) error { log.Println(string(logBytes)) } - return SendToFiretail(firetailLogs, firetailApiUrl, firetailApiToken) + return SendToFiretail(firetailLogs, firetailApiUrl, firetailTokenHeaderName, firetailToken) } diff --git a/src/send_to_firetail.go b/src/send_to_firetail.go index 016645f..bdff625 100644 --- a/src/send_to_firetail.go +++ b/src/send_to_firetail.go @@ -9,7 +9,7 @@ import ( "github.com/pkg/errors" ) -func SendToFiretail(firetailLogs map[string]*FiretailLog, apiUrl, apiToken string) error { +func SendToFiretail(firetailLogs map[string]*FiretailLog, apiUrl, tokenHeaderName, token string) error { reqBytes := []byte{} for _, firetailLog := range firetailLogs { logBytes, err := json.Marshal(*firetailLog) @@ -29,7 +29,7 @@ func SendToFiretail(firetailLogs map[string]*FiretailLog, apiUrl, apiToken strin return err } - req.Header.Set("x-ft-api-key", apiToken) + req.Header.Set(tokenHeaderName, token) resp, err := http.DefaultClient.Do(req) if err != nil { From 9bb6dbfa3a382340ef1d12c9c0a2653f2f418176 Mon Sep 17 00:00:00 2001 From: theteacat Date: Wed, 7 May 2025 10:41:07 +0100 Subject: [PATCH 13/18] update tests --- src/main_test.go | 6 +++--- src/send_to_firetail_test.go | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/main_test.go b/src/main_test.go index 8bd2c80..e46f9e8 100644 --- a/src/main_test.go +++ b/src/main_test.go @@ -16,7 +16,7 @@ func TestLoadEnvVars(t *testing.T) { loadEnvVars() assert.Equal(t, firetailApiUrl, MockFiretailApiUrl) - assert.Equal(t, firetailApiToken, MockFiretailApiToken) + assert.Equal(t, firetailToken, MockFiretailApiToken) } func TestLoadEnvVarsApiUrlUnset(t *testing.T) { @@ -27,5 +27,5 @@ func TestLoadEnvVarsApiUrlUnset(t *testing.T) { loadEnvVars() assert.Equal(t, firetailApiUrl, DefaultFiretailApiUrl) - assert.Equal(t, firetailApiToken, MockFiretailApiToken) -} \ No newline at end of file + assert.Equal(t, firetailToken, MockFiretailApiToken) +} diff --git a/src/send_to_firetail_test.go b/src/send_to_firetail_test.go index f9ee439..65e3a54 100644 --- a/src/send_to_firetail_test.go +++ b/src/send_to_firetail_test.go @@ -28,7 +28,7 @@ func TestSendToFiretail(t *testing.T) { Query: &testQuery, RequestID: "TEST_ID", }, - }, testServer.URL, "TEST_KEY") + }, testServer.URL, "x-ft-api-key", "TEST_KEY") require.Nil(t, err) wg.Wait() @@ -51,7 +51,7 @@ func TestSendToFiretailBadServer(t *testing.T) { Query: &testQuery, RequestID: "TEST_ID", }, - }, testServer.URL, "TEST_KEY") + }, testServer.URL, "x-ft-api-key", "TEST_KEY") require.NotNil(t, err) assert.Equal(t, "got err response from firetail api: map[message:fail]", err.Error()) @@ -65,7 +65,7 @@ func TestSendToFiretailNoServer(t *testing.T) { Query: &testQuery, RequestID: "TEST_ID", }, - }, "http://127.0.0.1:0", "TEST_KEY") + }, "http://127.0.0.1:0", "x-ft-api-key", "TEST_KEY") require.NotNil(t, err) assert.Contains(t, err.Error(), "Post \"http://127.0.0.1:0\": dial tcp 127.0.0.1:0") } @@ -77,7 +77,7 @@ func TestSendToFiretailBadUrl(t *testing.T) { Query: &testQuery, RequestID: "TEST_ID", }, - }, "\n", "TEST_KEY") + }, "\n", "x-ft-api-key", "TEST_KEY") require.NotNil(t, err) assert.Equal(t, "parse \"\\n\": net/url: invalid control character in URL", err.Error()) } From d6cd0b5ca5c4bc6c8e6595e8778604d0570f2019 Mon Sep 17 00:00:00 2001 From: riley priddle Date: Mon, 12 May 2025 16:32:32 +0100 Subject: [PATCH 14/18] removed cloudwatch group deployment --- cfn_templates/appsync_logging_template.yaml | 15 +-------------- template.yaml | 8 -------- 2 files changed, 1 insertion(+), 22 deletions(-) diff --git a/cfn_templates/appsync_logging_template.yaml b/cfn_templates/appsync_logging_template.yaml index 35a152d..200d261 100644 --- a/cfn_templates/appsync_logging_template.yaml +++ b/cfn_templates/appsync_logging_template.yaml @@ -16,11 +16,6 @@ Parameters: Type: String Default: 0.0.1 Resources: - AppSyncLogGroup: - Type: "AWS::Logs::LogGroup" - Properties: - RetentionInDays: 7 - LogGroupName: !Sub ${AWS::StackName}-appsync-log FiretailApp: Type: AWS::Serverless::Application Properties: @@ -30,12 +25,4 @@ Resources: Parameters: FTAPPKEY: !Ref FTAPPKEY FTAPI: !Ref FTAPI - CloudWatchGroupName: !Ref AppSyncLogGroup - SemanticVersion: !Ref SemanticVersion -Outputs: - FiretailCloudwatchARN: - Description: "Copy this ARN and paste into the Firetail integration form" - Value: - Fn::GetAtt: - - "AppSyncLogGroup" - - "Arn" \ No newline at end of file + SemanticVersion: !Ref SemanticVersion \ No newline at end of file diff --git a/template.yaml b/template.yaml index 64b458d..9b55e25 100644 --- a/template.yaml +++ b/template.yaml @@ -20,8 +20,6 @@ Parameters: FTAPI: Type: String Default: "https://api.logging.eu-west-1.prod.firetail.app" - CloudWatchGroupName: - Type: String SemanticVersion: Type: String Default: 0.0.1 @@ -37,12 +35,6 @@ Resources: Description: FireTail Logger for AppSync MemorySize: 128 Timeout: 60 - Events: - CWRule: - Type: CloudWatchLogs - Properties: - LogGroupName: !Ref CloudWatchGroupName - FilterPattern: "" Environment: Variables: FIRETAIL_API: From b663dd89b076417956bbc3a962182f37bc93fc4c Mon Sep 17 00:00:00 2001 From: riley priddle Date: Mon, 12 May 2025 18:06:13 +0100 Subject: [PATCH 15/18] add permissions to template --- template.yaml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/template.yaml b/template.yaml index 9b55e25..49a7238 100644 --- a/template.yaml +++ b/template.yaml @@ -42,4 +42,11 @@ Resources: FIRETAIL_APP_TOKEN: Ref: FTAPPKEY SEMANTIC_VERSION: - Ref: SemanticVersion \ No newline at end of file + Ref: SemanticVersion + LambdaInvokePermission: + Type: "AWS::Lambda::Permission" + Properties: + FunctionName: !GetAtt FireTailAppSyncLogger.Arn + Action: "lambda:InvokeFunction" + Principal: "logs.amazonaws.com" + SourceArn: !Sub "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/appsync/apis/*/*" \ No newline at end of file From b73e2d8c7be8c815653163f171113b66fc32a342 Mon Sep 17 00:00:00 2001 From: riley priddle Date: Mon, 12 May 2025 18:08:01 +0100 Subject: [PATCH 16/18] bump --- .github/workflows/deploy-sandbox.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-sandbox.yaml b/.github/workflows/deploy-sandbox.yaml index e455306..2cb8dc4 100644 --- a/.github/workflows/deploy-sandbox.yaml +++ b/.github/workflows/deploy-sandbox.yaml @@ -7,7 +7,7 @@ env: CFN_TEMPLATES_BUCKET: firetail-sandbox-us-east-1-cf-templates CFN_TEMPLATES_BUCKET_REGION: us-east-1 APP_NAME: aws-appsync-logging-lambda - SEMANTIC_VERSION: 1.2.1 + SEMANTIC_VERSION: 1.2.2 name: Deploy to Sandbox run-name: "@${{ github.triggering_actor }}: ${{ github.ref_name }}: ${{ github.event_name }}" on: From b632390a21f0eac62e28858219be9d5d68d24ae3 Mon Sep 17 00:00:00 2001 From: riley priddle Date: Mon, 12 May 2025 18:13:40 +0100 Subject: [PATCH 17/18] fix permissions --- template.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/template.yaml b/template.yaml index 49a7238..e443bd1 100644 --- a/template.yaml +++ b/template.yaml @@ -49,4 +49,4 @@ Resources: FunctionName: !GetAtt FireTailAppSyncLogger.Arn Action: "lambda:InvokeFunction" Principal: "logs.amazonaws.com" - SourceArn: !Sub "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/appsync/apis/*/*" \ No newline at end of file + SourceArn: !Sub "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/appsync/apis/*" \ No newline at end of file From 0549684f18b26b32d9fbc9abb5699b95730a4e39 Mon Sep 17 00:00:00 2001 From: riley priddle Date: Tue, 13 May 2025 10:00:02 +0100 Subject: [PATCH 18/18] nosemgrep --- .github/workflows/deploy-prod-eu.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-prod-eu.yaml b/.github/workflows/deploy-prod-eu.yaml index 87d916b..8a69eaf 100644 --- a/.github/workflows/deploy-prod-eu.yaml +++ b/.github/workflows/deploy-prod-eu.yaml @@ -12,7 +12,7 @@ permissions: env: GOLANG_VERSION: 1.23 REGION_NAME: eu-west-1 - ACCOUNT_ID: 247286868737 + ACCOUNT_ID: 247286868737 # nosemgrep ROLE_TO_ASSUME: arn:aws:iam::247286868737:role/firetail-prod-github-serverless-lambda-deployment LAMBDA_SERVERLESS_REPO: firetail-prod-eu-west-1-serverless-applications APP_NAME: firetail-appsync-logger