Skip to content

Commit a6a1bb7

Browse files
ptang-nrptang-nr
authored
chore: Add delete experiment GHA (#1288)
1 parent 0e8f48b commit a6a1bb7

File tree

5 files changed

+253
-0
lines changed

5 files changed

+253
-0
lines changed

‎.github/actions/s3-delete/action.yml‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
# This composite action is used to delete files from a S3 bucket.
2+
3+
name: 'S3 File Deletion'
4+
5+
inputs:
6+
aws_access_key_id:
7+
description: 'AWS access key id used for authentication.'
8+
required: true
9+
aws_secret_access_key:
10+
description: 'AWS secret access key used for authentication.'
11+
required: true
12+
aws_region:
13+
description: "AWS region where S3 bucket is located."
14+
required: false
15+
default: us-east-1
16+
aws_role:
17+
description: "AWS role ARN that needs to be used for authentication."
18+
required: true
19+
aws_bucket_name:
20+
description: "S3 bucket name where files need to be deleted."
21+
required: true
22+
dry_run:
23+
description: "Indicates if we should just list files to deleted without actually deleting them."
24+
required: false
25+
bucket_dir:
26+
description: 'A sub directory where the files are located within the S3 bucket.'
27+
required: false
28+
29+
outputs:
30+
results:
31+
description: "Array of objects containing information about each file uploaded"
32+
value: ${{ steps.action-script.outputs.results }}
33+
34+
runs:
35+
using: "composite"
36+
steps:
37+
- name: Install dependencies
38+
run: npm install --silent --no-progress --prefix $GITHUB_ACTION_PATH/..
39+
shell: bash
40+
- name: Run action script
41+
id: action-script
42+
env:
43+
AWS_ACCESS_KEY_ID: ${{ inputs.aws_access_key_id }}
44+
AWS_SECRET_ACCESS_KEY: ${{ inputs.aws_secret_access_key }}
45+
run: |
46+
node $GITHUB_ACTION_PATH/index.js \
47+
--region ${{ inputs.aws_region }} \
48+
--bucket ${{ inputs.aws_bucket_name }} \
49+
--role ${{ inputs.aws_role }} \
50+
${{ inputs.dry_run && '--dry' || '' }} \
51+
${{ inputs.bucket_dir && format('--dir {0}', inputs.bucket_dir) || '' }}
52+
shell: bash

‎.github/actions/s3-delete/args.js‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
import process from 'process'
2+
import yargs from 'yargs'
3+
import { hideBin } from 'yargs/helpers'
4+
5+
export const args = yargs(hideBin(process.argv))
6+
.usage('$0 [options]')
7+
8+
.string('role')
9+
.describe('role', 'S3 role ARN')
10+
11+
.string('bucket')
12+
.describe('bucket', 'S3 bucket name')
13+
14+
.string('region')
15+
.describe('region', 'AWS region location of S3 bucket. Defaults to us-east-1.')
16+
.default('region', 'us-east-1')
17+
18+
.boolean('dry')
19+
.default('dry', false)
20+
.describe('dry', 'Runs the action script without actually uploading files.')
21+
.alias('dry', 'dry-run')
22+
23+
.string('dir')
24+
.describe('dir', 'Bucket sub-directory name. Leave empty to refer to the root of the bucket.')
25+
26+
.demandOption(['bucket', 'role'])
27+
.argv

‎.github/actions/s3-delete/index.js‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
import * as core from '@actions/core'
2+
import { AssumeRoleCommand, STSClient } from '@aws-sdk/client-sts'
3+
import { S3Client, S3ServiceException, paginateListObjectsV2, DeleteObjectsCommand, waitUntilObjectNotExists } from '@aws-sdk/client-s3'
4+
import { args } from './args.js'
5+
6+
const stsClient = new STSClient({ region: args.region })
7+
const s3Credentials = await stsClient.send(new AssumeRoleCommand({
8+
RoleArn: args.role,
9+
RoleSessionName: 'deleteExperimentFromS3Session',
10+
DurationSeconds: 900
11+
}))
12+
13+
const s3Client = new S3Client({
14+
region: args.region,
15+
credentials: {
16+
accessKeyId: s3Credentials.Credentials.AccessKeyId,
17+
secretAccessKey: s3Credentials.Credentials.SecretAccessKey,
18+
sessionToken: s3Credentials.Credentials.SessionToken
19+
}
20+
})
21+
22+
async function collectKeysToDelete(bucketName, bucketDir) {
23+
const keys = []
24+
core.info("Bucket name: " + bucketName)
25+
core.info(`Looking up files with prefix '${bucketDir}'`)
26+
27+
try {
28+
const params = {
29+
Bucket: bucketName,
30+
Prefix:bucketDir
31+
}
32+
const paginator = paginateListObjectsV2({ client: s3Client }, params)
33+
for await (const page of paginator) {
34+
page.Contents?.forEach(obj => {
35+
keys.push(obj.Key)
36+
})
37+
}
38+
core.info(`Found ${keys.length} matching files`)
39+
core.info(keys.map(k => ` • ${k}`).join('\n'))
40+
return keys
41+
} catch (err) {
42+
if (err instanceof S3ServiceException) {
43+
core.setFailed(
44+
`Error from S3 while listing objects for "${bucketName}". ${err.name}: ${err.message}`,
45+
)
46+
} else {
47+
core.setFailed(`Error while listing objects for "${bucketName}". ${err.name}: ${err.message}`)
48+
}
49+
process.exit(1)
50+
}
51+
}
52+
53+
async function deleteFiles(bucketName, keys) {
54+
try {
55+
core.info('Deleting...')
56+
57+
const { Deleted } = await s3Client.send(
58+
new DeleteObjectsCommand({
59+
Bucket: bucketName,
60+
Delete: {
61+
Objects: keys.map((k) => ({ Key: k })),
62+
},
63+
}),
64+
)
65+
66+
for (const key in keys) {
67+
await waitUntilObjectNotExists(
68+
{ client: s3Client },
69+
{ Bucket: bucketName, Key: key },
70+
)
71+
}
72+
core.info(
73+
`Successfully deleted ${Deleted?.length || 0} objects.`,
74+
)
75+
core.info(Deleted?.map((d) => ` • ${d.Key}`).join("\n"))
76+
} catch (err) {
77+
if (err instanceof S3ServiceException) {
78+
core.setFailed(
79+
`Error from S3 while deleting objects for "${bucketName}". ${err.name}: ${err.message}`,
80+
)
81+
} else {
82+
core.setFailed(`Error while deleting objects for "${bucketName}". ${err.name}: ${err.message}`)
83+
}
84+
process.exit(1)
85+
}
86+
}
87+
88+
if (args.dry) {
89+
core.info('This is a dry run.')
90+
}
91+
const keysToBeDeleted = await collectKeysToDelete(args.bucket, args.dir)
92+
if (!args.dry) {
93+
if (keysToBeDeleted.length > 0) {
94+
await deleteFiles(args.bucket, keysToBeDeleted)
95+
} else {
96+
core.info('No files found to delete')
97+
}
98+
}
99+
core.info('Completed successfully')

‎.github/workflows/branch-cleanup.yml‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
# This workflow will run automatically when a branch is deleted.
2+
3+
name: Branch Cleanup
4+
5+
on: delete
6+
7+
jobs:
8+
branch-delete:
9+
if: github.event.ref_type == 'branch'
10+
runs-on: ubuntu-latest
11+
timeout-minutes: 5
12+
defaults:
13+
run:
14+
shell: bash
15+
steps:
16+
- name: Clean up for deleted branch
17+
id: cleanup-start
18+
run: echo "Clean up for branch ${{ github.event.ref }}"
19+
- name: Delete dev experiment
20+
id: s3-delete-dev
21+
uses: ./.github/actions/s3-delete
22+
with:
23+
aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
24+
aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
25+
aws_role: ${{ secrets.AWS_ROLE_ARN }}
26+
aws_bucket_name: ${{ secrets.AWS_BUCKET }}
27+
bucket_dir: experiments/dev/${{ github.event.ref }}
28+
dry_run: true
29+
- name: Delete staging experiment
30+
id: s3-delete-staging
31+
uses: ./.github/actions/s3-delete
32+
with:
33+
aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
34+
aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
35+
aws_role: ${{ secrets.AWS_ROLE_ARN }}
36+
aws_bucket_name: ${{ secrets.AWS_BUCKET }}
37+
bucket_dir: experiments/staging/${{ github.event.ref }}
38+
dry_run: true

‎.github/workflows/delete-experiment.yml‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
# This workflow can be run on any branch.
2+
# Given an environment, this workflow will delete the branch's experiment from S3
3+
4+
name: Delete Experiment
5+
6+
on:
7+
workflow_dispatch:
8+
inputs:
9+
nr_environment:
10+
description: 'Target New Relic environment'
11+
required: true
12+
type: choice
13+
options:
14+
- dev
15+
- staging
16+
17+
jobs:
18+
delete-experiment-on-s3:
19+
runs-on: ubuntu-latest
20+
timeout-minutes: 5
21+
defaults:
22+
run:
23+
shell: bash
24+
steps:
25+
- uses: actions/checkout@v4
26+
- name: Clean branch name
27+
id: clean-branch-name
28+
run: echo "results=$(echo '${{ github.ref }}' | sed 's/refs\/heads\///g' | sed 's/[^[:alnum:].-]/-/g')" >> $GITHUB_OUTPUT
29+
- name: Delete experiment
30+
id: s3-delete
31+
uses: ./.github/actions/s3-delete
32+
with:
33+
aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
34+
aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
35+
aws_role: ${{ secrets.AWS_ROLE_ARN }}
36+
aws_bucket_name: ${{ secrets.AWS_BUCKET }}
37+
bucket_dir: experiments/${{ inputs.nr_environment }}/${{ steps.clean-branch-name.outputs.results }}

0 commit comments

Comments
 (0)