From 661eb312e36de76e657dd171929859f03b2711b0 Mon Sep 17 00:00:00 2001 From: Vishal Wadhera Date: Wed, 12 Aug 2026 13:50:23 +0530 Subject: [PATCH] feat: add optional DynamoDB runner config storage --- CHANGELOG.md | 9 - lambdas/functions/control-plane/package.json | 2 + .../functions/control-plane/src/modules.d.ts | 15 + .../functions/control-plane/src/pool/pool.ts | 3 + .../src/scale-runners/github-runner.ts | 46 +- .../runner-config-storage.test.ts | 233 ++++++++++ .../scale-runners/runner-config-storage.ts | 397 ++++++++++++++++++ .../src/scale-runners/scale-up.ts | 3 + .../control-plane/src/scale-runners/types.ts | 2 + .../termination-watcher/src/lambda.ts | 7 +- .../src/termination-warning.test.ts | 47 +-- .../src/termination-warning.ts | 10 +- .../termination-watcher/src/types.d.ts | 11 - lambdas/libs/runner-providers/core/index.ts | 17 + lambdas/yarn.lock | 343 +++++++++++++++ main.tf | 12 +- modules/multi-runner/outputs.tf | 34 +- modules/multi-runner/runners.tf | 10 +- modules/multi-runner/variables.tf | 136 +++++- modules/runners/dynamodb.tf | 208 +++++++++ modules/runners/local.tf | 29 ++ modules/runners/logging.tf | 13 +- modules/runners/main.tf | 11 +- modules/runners/outputs.tf | 10 + modules/runners/policies-runner.tf | 16 +- .../policies/instance-cloudwatch-policy.json | 7 +- ...nstance-dynamodb-runner-config-policy.json | 43 ++ .../lambda-dynamodb-runner-config.json | 36 ++ modules/runners/policies/lambda-scale-up.json | 10 +- modules/runners/pool.tf | 33 +- modules/runners/pool/main.tf | 102 +++-- .../runners/pool/policies/lambda-pool.json | 22 +- modules/runners/pool/variables.tf | 45 +- modules/runners/runner-config.tf | 12 + modules/runners/scale-up.tf | 128 +++--- modules/runners/ssm-housekeeper.tf | 43 +- modules/runners/templates/start-runner-osx.sh | 108 ++++- modules/runners/templates/start-runner.ps1 | 138 ++++-- modules/runners/templates/start-runner.sh | 119 +++++- modules/runners/variables.tf | 126 ++++++ outputs.tf | 36 +- variables.tf | 128 +++++- 42 files changed, 2413 insertions(+), 347 deletions(-) create mode 100644 lambdas/functions/control-plane/src/scale-runners/runner-config-storage.test.ts create mode 100644 lambdas/functions/control-plane/src/scale-runners/runner-config-storage.ts create mode 100644 modules/runners/dynamodb.tf create mode 100644 modules/runners/policies/instance-dynamodb-runner-config-policy.json create mode 100644 modules/runners/policies/lambda-dynamodb-runner-config.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f6191e148..d1cfc8bf58 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,14 +1,5 @@ # Changelog -## [7.10.2](https://github.com/github-aws-runners/terraform-aws-github-runner/compare/v7.10.1...v7.10.2) (2026-08-10) - - -### Bug Fixes - -* **lambda:** bump postcss from 8.5.22 to 8.5.24 in /lambdas ([#5241](https://github.com/github-aws-runners/terraform-aws-github-runner/issues/5241)) ([17cd322](https://github.com/github-aws-runners/terraform-aws-github-runner/commit/17cd322aa4a43db583072cce4b87f99db494b6d0)) -* **termination-watcher:** only emit SpotInterruptionWarning metric for actual spot interruption events ([#5245](https://github.com/github-aws-runners/terraform-aws-github-runner/issues/5245)) ([7480fd5](https://github.com/github-aws-runners/terraform-aws-github-runner/commit/7480fd58d4c517e8e999a830905bbd9334dc987d)) -* wrap LOG_LEVEL env var with upper() for Powertools v2 compatibility ([#5238](https://github.com/github-aws-runners/terraform-aws-github-runner/issues/5238)) ([2394f84](https://github.com/github-aws-runners/terraform-aws-github-runner/commit/2394f8445faeba3d37998791cf51ae9dd0ab453e)) - ## [7.10.1](https://github.com/github-aws-runners/terraform-aws-github-runner/compare/v7.10.0...v7.10.1) (2026-07-31) diff --git a/lambdas/functions/control-plane/package.json b/lambdas/functions/control-plane/package.json index e886920b1f..9f7192b4d8 100644 --- a/lambdas/functions/control-plane/package.json +++ b/lambdas/functions/control-plane/package.json @@ -34,6 +34,7 @@ "@aws-github-runner/aws-ssm-util": "*", "@aws-github-runner/runner-providers": "*", "@aws-lambda-powertools/parameters": "^2.31.0", + "@aws-sdk/client-dynamodb": "^3.1009.0", "@aws-sdk/client-ec2": "^3.1009.0", "@aws-sdk/client-sqs": "^3.1009.0", "@middy/core": "^6.4.5", @@ -42,6 +43,7 @@ "@octokit/plugin-retry": "8.0.3", "@octokit/plugin-throttling": "11.0.3", "@octokit/rest": "22.0.1", + "@smithy/node-http-handler": "^4.5.0", "cron-parser": "^5.4.0" }, "nx": { diff --git a/lambdas/functions/control-plane/src/modules.d.ts b/lambdas/functions/control-plane/src/modules.d.ts index 53247cf6c6..a4a36514da 100644 --- a/lambdas/functions/control-plane/src/modules.d.ts +++ b/lambdas/functions/control-plane/src/modules.d.ts @@ -15,6 +15,21 @@ declare namespace NodeJS { PARAMETER_GITHUB_APP_CLIENT_SECRET_NAME: string; PARAMETER_GITHUB_APP_ID_NAME: string; PARAMETER_GITHUB_APP_KEY_BASE64_NAME: string; + RUNNER_CONFIG_STORAGE_BACKEND?: string; + RUNNER_CONFIG_DYNAMODB_TABLE_NAME?: string; + RUNNER_CONFIG_DYNAMODB_PARTITION_KEY_NAME?: string; + RUNNER_CONFIG_DYNAMODB_VALUE_ATTRIBUTE_NAME?: string; + RUNNER_CONFIG_DYNAMODB_CONFIG_KEY_PREFIX?: string; + RUNNER_CONFIG_DYNAMODB_CONSISTENT_READ?: string; + RUNNER_CONFIG_DYNAMODB_TOKEN_OVERWRITE_PROTECTION_ENABLED?: string; + RUNNER_CONFIG_DYNAMODB_TOKEN_KEY_PREFIX?: string; + RUNNER_CONFIG_DYNAMODB_TTL_SECONDS?: string; + RUNNER_CONFIG_DYNAMODB_TTL_ATTRIBUTE_NAME?: string; + RUNNER_CONFIG_DYNAMODB_CLIENT_MAX_ATTEMPTS?: string; + RUNNER_CONFIG_DYNAMODB_CLIENT_RETRY_MODE?: string; + RUNNER_CONFIG_DYNAMODB_CLIENT_HTTP_KEEP_ALIVE?: string; + RUNNER_CONFIG_DYNAMODB_CLIENT_HTTP_MAX_SOCKETS?: string; + RUNNER_CONFIG_DYNAMODB_CLIENT_HTTP_KEEP_ALIVE_MSECS?: string; RUNNER_OWNER: string; RUNNER_PROVIDER_TYPE?: string; SCALE_DOWN_CONFIG: string; diff --git a/lambdas/functions/control-plane/src/pool/pool.ts b/lambdas/functions/control-plane/src/pool/pool.ts index 3a6ed45be9..66d83afa21 100644 --- a/lambdas/functions/control-plane/src/pool/pool.ts +++ b/lambdas/functions/control-plane/src/pool/pool.ts @@ -6,6 +6,7 @@ import yn from 'yn'; import { createGithubAppAuth, createGithubInstallationAuth, createOctokitClient } from '../github/auth'; import { controlPlaneProviderRegistry } from '../control-plane-providers'; import { getGitHubEnterpriseApiUrl, validateSsmParameterStoreTags } from '../scale-runners/github-runner'; +import { loadRunnerConfigStorageFromEnv } from '../scale-runners/runner-config-storage'; import type { RunnerStatus } from './pool-provider'; const logger = createChildLogger('pool'); @@ -28,6 +29,7 @@ export async function adjust(event: PoolEvent): Promise { const environment = process.env.ENVIRONMENT; const ssmTokenPath = process.env.SSM_TOKEN_PATH; const ssmConfigPath = process.env.SSM_CONFIG_PATH || ''; + const runnerConfigStorage = loadRunnerConfigStorageFromEnv(); const ephemeral = yn(process.env.ENABLE_EPHEMERAL_RUNNERS, { default: false }); const enableJitConfig = yn(process.env.ENABLE_JIT_CONFIG, { default: ephemeral }); const disableAutoUpdate = yn(process.env.DISABLE_RUNNER_AUTOUPDATE, { default: false }); @@ -95,6 +97,7 @@ export async function adjust(event: PoolEvent): Promise { ssmTokenPath, ssmConfigPath, ssmParameterStoreTags, + runnerConfigStorage, }, numberOfRunners: topUp, githubInstallationClient, diff --git a/lambdas/functions/control-plane/src/scale-runners/github-runner.ts b/lambdas/functions/control-plane/src/scale-runners/github-runner.ts index cdc10b1f5d..b1feb4ab59 100644 --- a/lambdas/functions/control-plane/src/scale-runners/github-runner.ts +++ b/lambdas/functions/control-plane/src/scale-runners/github-runner.ts @@ -1,8 +1,8 @@ import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; -import { getParameter, putParameter } from '@aws-github-runner/aws-ssm-util'; import { Octokit } from '@octokit/rest'; import { metricGitHubAppRateLimit } from '../github/rate-limit'; +import { createRunnerConfigStore } from './runner-config-storage'; import { ActionRequestMessage, CreateGitHubRunnerConfig, EphemeralRunnerConfig, RunnerGroup } from './types'; const logger = createChildLogger('github-runner'); @@ -171,34 +171,28 @@ export async function getRunnerGroupId( let runnerGroupId: number | undefined = 1; if (githubRunnerConfig.runnerType === 'Org' && githubRunnerConfig.runnerGroup !== undefined) { let runnerGroup: string | undefined; - // check if runner group id is already stored in SSM Parameter Store and - // use it if it exists to avoid API call to GitHub + const runnerConfigStore = createRunnerConfigStore(githubRunnerConfig); + const runnerGroupCacheKey = `runner-group/${githubRunnerConfig.runnerGroup}`; + // check if runner group id is already cached and use it if it exists to + // avoid an API call to GitHub try { - runnerGroup = await getParameter( - `${githubRunnerConfig.ssmConfigPath}/runner-group/${githubRunnerConfig.runnerGroup}`, - ); + runnerGroup = await runnerConfigStore.getConfigValue(runnerGroupCacheKey); } catch (err) { logger.debug('Handling error:', err as Error); logger.warn( - `SSM Parameter "${githubRunnerConfig.ssmConfigPath}/runner-group/${githubRunnerConfig.runnerGroup}" - for Runner group ${githubRunnerConfig.runnerGroup} does not exist`, + `Runner group cache entry "${runnerGroupCacheKey}" for Runner group ${githubRunnerConfig.runnerGroup} does not exist`, ); } if (runnerGroup === undefined) { // get runner group id from GitHub runnerGroupId = await getRunnerGroupByName(ghClient, githubRunnerConfig); - // store runner group id in SSM + // store runner group id in the configured runner config storage try { - await putParameter( - `${githubRunnerConfig.ssmConfigPath}/runner-group/${githubRunnerConfig.runnerGroup}`, - runnerGroupId.toString(), - false, - { - tags: githubRunnerConfig.ssmParameterStoreTags, - }, - ); + await runnerConfigStore.putConfigValue(runnerGroupCacheKey, runnerGroupId.toString(), { + tags: githubRunnerConfig.ssmParameterStoreTags, + }); } catch (err) { - logger.debug('Error storing runner group id in SSM Parameter Store', err as Error); + logger.debug('Error storing runner group id', err as Error); throw err; } } else { @@ -241,10 +235,10 @@ export async function createStartRunnerConfig( } } -function addDelay(runnerIds: string[]) { +function addDelay(runnerIds: string[], enabled = true) { const delay = async (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); const ssmParameterStoreMaxThroughput = 40; - const isDelay = runnerIds.length >= ssmParameterStoreMaxThroughput; + const isDelay = enabled && runnerIds.length >= ssmParameterStoreMaxThroughput; return { isDelay, delay }; } @@ -259,7 +253,8 @@ async function createRegistrationTokenConfig( ghClient: Octokit, options: StartRunnerConfigOptions, ): Promise { - const { isDelay, delay } = addDelay(runnerIds); + const runnerConfigStore = createRunnerConfigStore(githubRunnerConfig); + const { isDelay, delay } = addDelay(runnerIds, runnerConfigStore.delayWritesForSsmThroughput); const token = await getGithubRunnerRegistrationToken(githubRunnerConfig, ghClient); const runnerServiceConfig = generateRunnerServiceConfig(githubRunnerConfig, token); @@ -268,7 +263,7 @@ async function createRegistrationTokenConfig( }); for (const runnerId of runnerIds) { - await putParameter(`${githubRunnerConfig.ssmTokenPath}/${runnerId}`, runnerServiceConfig.join(' '), true, { + await runnerConfigStore.putRunnerConfig(runnerId, runnerServiceConfig.join(' '), { tags: [...(options.getSsmParameterTags?.(runnerId) ?? []), ...githubRunnerConfig.ssmParameterStoreTags], }); if (isDelay) { @@ -293,7 +288,8 @@ async function createJitConfig( options: StartRunnerConfigOptions, ): Promise { const runnerGroupId = await getRunnerGroupId(githubRunnerConfig, ghClient); - const { isDelay, delay } = addDelay(runnerIds); + const runnerConfigStore = createRunnerConfigStore(githubRunnerConfig); + const { isDelay, delay } = addDelay(runnerIds, runnerConfigStore.delayWritesForSsmThroughput); const runnerLabels = githubRunnerConfig.runnerLabels.split(','); const failedRunnerIds: string[] = []; @@ -331,11 +327,11 @@ async function createJitConfig( runnerLabels, }); - // store jit config in ssm parameter store + // store jit config in the configured runner config storage logger.debug('Runner JIT config for ephemeral runner generated.', { instance: runnerId, }); - await putParameter(`${githubRunnerConfig.ssmTokenPath}/${runnerId}`, runnerConfig.data.encoded_jit_config, true, { + await runnerConfigStore.putRunnerConfig(runnerId, runnerConfig.data.encoded_jit_config, { tags: [...(options.getSsmParameterTags?.(runnerId) ?? []), ...githubRunnerConfig.ssmParameterStoreTags], }); if (isDelay) { diff --git a/lambdas/functions/control-plane/src/scale-runners/runner-config-storage.test.ts b/lambdas/functions/control-plane/src/scale-runners/runner-config-storage.test.ts new file mode 100644 index 0000000000..f9e5f7eaf2 --- /dev/null +++ b/lambdas/functions/control-plane/src/scale-runners/runner-config-storage.test.ts @@ -0,0 +1,233 @@ +import { DynamoDBClient, GetItemCommand, PutItemCommand } from '@aws-sdk/client-dynamodb'; +import { mockClient } from 'aws-sdk-client-mock'; +import 'aws-sdk-client-mock-jest/vitest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { createRunnerConfigStore, loadRunnerConfigStorageFromEnv, resetDynamoDbClient } from './runner-config-storage'; +import type { CreateGitHubRunnerConfig } from './types'; + +const mockDynamoDbClient = mockClient(DynamoDBClient); +const cleanEnv = process.env; + +const BASE_CONFIG: CreateGitHubRunnerConfig = { + ephemeral: true, + enableJitConfig: true, + runnerLabels: 'self-hosted,linux,x64', + runnerGroup: 'Default', + runnerNamePrefix: 'unit-test-', + runnerOwner: 'Codertocat', + runnerType: 'Org', + disableAutoUpdate: false, + ssmTokenPath: '/github-action-runners/default/runners/tokens', + ssmConfigPath: '/github-action-runners/default/runners/config', + ssmParameterStoreTags: [], +}; + +beforeEach(() => { + process.env = { ...cleanEnv, AWS_REGION: 'us-east-1' }; + mockDynamoDbClient.reset(); + resetDynamoDbClient(); + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-08-10T00:00:00Z')); +}); + +afterEach(() => { + vi.useRealTimers(); + process.env = cleanEnv; +}); + +describe('loadRunnerConfigStorageFromEnv', () => { + it('defaults to SSM storage', () => { + delete process.env.RUNNER_CONFIG_STORAGE_BACKEND; + + expect(loadRunnerConfigStorageFromEnv()).toEqual({ backend: 'ssm' }); + }); + + it('loads DynamoDB storage config', () => { + process.env.RUNNER_CONFIG_STORAGE_BACKEND = 'dynamodb'; + process.env.RUNNER_CONFIG_DYNAMODB_TABLE_NAME = 'runner-config-table'; + process.env.RUNNER_CONFIG_DYNAMODB_TOKEN_KEY_PREFIX = 'arn:aws:ec2:us-east-1:123456789012:instance/'; + process.env.RUNNER_CONFIG_DYNAMODB_TTL_SECONDS = '60'; + + expect(loadRunnerConfigStorageFromEnv()).toEqual({ + backend: 'dynamodb', + dynamodb: { + tableName: 'runner-config-table', + partitionKeyName: 'id', + valueAttributeName: 'value', + configKeyPrefix: 'config#', + consistentRead: true, + tokenOverwriteProtectionEnabled: true, + tokenKeyPrefix: 'arn:aws:ec2:us-east-1:123456789012:instance/', + tokenTtlSeconds: 60, + ttlAttributeName: 'expires_at', + }, + }); + }); + + it('loads custom DynamoDB storage schema config', () => { + process.env.RUNNER_CONFIG_STORAGE_BACKEND = 'dynamodb'; + process.env.RUNNER_CONFIG_DYNAMODB_TABLE_NAME = 'runner-config-table'; + process.env.RUNNER_CONFIG_DYNAMODB_PARTITION_KEY_NAME = 'pk'; + process.env.RUNNER_CONFIG_DYNAMODB_VALUE_ATTRIBUTE_NAME = 'payload'; + process.env.RUNNER_CONFIG_DYNAMODB_CONFIG_KEY_PREFIX = 'cfg#'; + process.env.RUNNER_CONFIG_DYNAMODB_CONSISTENT_READ = 'false'; + process.env.RUNNER_CONFIG_DYNAMODB_TOKEN_OVERWRITE_PROTECTION_ENABLED = 'false'; + process.env.RUNNER_CONFIG_DYNAMODB_TOKEN_KEY_PREFIX = 'token#'; + process.env.RUNNER_CONFIG_DYNAMODB_TTL_SECONDS = '60'; + process.env.RUNNER_CONFIG_DYNAMODB_TTL_ATTRIBUTE_NAME = 'expiresAt'; + + expect(loadRunnerConfigStorageFromEnv()).toEqual({ + backend: 'dynamodb', + dynamodb: { + tableName: 'runner-config-table', + partitionKeyName: 'pk', + valueAttributeName: 'payload', + configKeyPrefix: 'cfg#', + consistentRead: false, + tokenOverwriteProtectionEnabled: false, + tokenKeyPrefix: 'token#', + tokenTtlSeconds: 60, + ttlAttributeName: 'expiresAt', + }, + }); + }); + + it('rejects invalid storage backends', () => { + process.env.RUNNER_CONFIG_STORAGE_BACKEND = 's3'; + + expect(() => loadRunnerConfigStorageFromEnv()).toThrow("Unsupported RUNNER_CONFIG_STORAGE_BACKEND 's3'"); + }); + + it('requires DynamoDB token TTL to be provided by the environment', () => { + process.env.RUNNER_CONFIG_STORAGE_BACKEND = 'dynamodb'; + process.env.RUNNER_CONFIG_DYNAMODB_TABLE_NAME = 'runner-config-table'; + process.env.RUNNER_CONFIG_DYNAMODB_TOKEN_KEY_PREFIX = 'token#'; + delete process.env.RUNNER_CONFIG_DYNAMODB_TTL_SECONDS; + + expect(() => loadRunnerConfigStorageFromEnv()).toThrow( + 'RUNNER_CONFIG_DYNAMODB_TTL_SECONDS must be set and be a positive integer', + ); + }); +}); + +describe('DynamoDbRunnerConfigStore', () => { + function createDynamoDbStore() { + return createRunnerConfigStore({ + ...BASE_CONFIG, + runnerConfigStorage: { + backend: 'dynamodb', + dynamodb: { + tableName: 'runner-config-table', + tokenKeyPrefix: 'arn:aws:ec2:us-east-1:123456789012:instance/', + tokenTtlSeconds: 60, + }, + }, + }); + } + + it('writes one-time runner config with a token TTL and conditional put', async () => { + const store = createDynamoDbStore(); + mockDynamoDbClient.on(PutItemCommand).resolves({}); + + await store.putRunnerConfig('i-1234567890', 'encoded-jit-config', { + tags: [{ Key: 'InstanceId', Value: 'i-1234567890' }], + }); + + expect(mockDynamoDbClient).toHaveReceivedCommandWith(PutItemCommand, { + TableName: 'runner-config-table', + ConditionExpression: 'attribute_not_exists(#partition_key)', + ExpressionAttributeNames: { + '#partition_key': 'id', + }, + Item: { + id: { S: 'arn:aws:ec2:us-east-1:123456789012:instance/i-1234567890' }, + value: { S: 'encoded-jit-config' }, + expires_at: { N: '1786320060' }, + }, + }); + }); + + it('reads and writes runner group cache values under the config key prefix', async () => { + const store = createDynamoDbStore(); + mockDynamoDbClient.on(GetItemCommand).resolves({ + Item: { + value: { S: '42' }, + }, + }); + mockDynamoDbClient.on(PutItemCommand).resolves({}); + + await expect(store.getConfigValue('runner-group/Default')).resolves.toBe('42'); + await store.putConfigValue('runner-group/Default', '42'); + + expect(mockDynamoDbClient).toHaveReceivedCommandWith(GetItemCommand, { + TableName: 'runner-config-table', + Key: { + id: { S: 'config#runner-group/Default' }, + }, + ConsistentRead: true, + ProjectionExpression: '#value', + ExpressionAttributeNames: { + '#value': 'value', + }, + }); + expect(mockDynamoDbClient).toHaveReceivedCommandWith(PutItemCommand, { + TableName: 'runner-config-table', + Item: { + id: { S: 'config#runner-group/Default' }, + value: { S: '42' }, + }, + }); + }); + + it('uses custom DynamoDB key prefixes and attribute names', async () => { + const store = createRunnerConfigStore({ + ...BASE_CONFIG, + runnerConfigStorage: { + backend: 'dynamodb', + dynamodb: { + tableName: 'runner-config-table', + partitionKeyName: 'pk', + valueAttributeName: 'payload', + configKeyPrefix: 'cfg#', + consistentRead: false, + tokenOverwriteProtectionEnabled: false, + tokenKeyPrefix: 'token#', + tokenTtlSeconds: 60, + ttlAttributeName: 'expiresAt', + }, + }, + }); + mockDynamoDbClient.on(GetItemCommand).resolves({ + Item: { + payload: { S: '42' }, + }, + }); + mockDynamoDbClient.on(PutItemCommand).resolves({}); + + await expect(store.getConfigValue('runner-group/Default')).resolves.toBe('42'); + await store.putRunnerConfig('i-1234567890', 'encoded-jit-config', { + tags: [{ Key: 'InstanceId', Value: 'i-1234567890' }], + }); + + expect(mockDynamoDbClient).toHaveReceivedCommandWith(GetItemCommand, { + TableName: 'runner-config-table', + Key: { + pk: { S: 'cfg#runner-group/Default' }, + }, + ConsistentRead: false, + ProjectionExpression: '#value', + ExpressionAttributeNames: { + '#value': 'payload', + }, + }); + expect(mockDynamoDbClient).toHaveReceivedCommandWith(PutItemCommand, { + TableName: 'runner-config-table', + Item: { + pk: { S: 'token#i-1234567890' }, + payload: { S: 'encoded-jit-config' }, + expiresAt: { N: '1786320060' }, + }, + }); + }); +}); diff --git a/lambdas/functions/control-plane/src/scale-runners/runner-config-storage.ts b/lambdas/functions/control-plane/src/scale-runners/runner-config-storage.ts new file mode 100644 index 0000000000..1a0fb791d7 --- /dev/null +++ b/lambdas/functions/control-plane/src/scale-runners/runner-config-storage.ts @@ -0,0 +1,397 @@ +import { Agent as HttpsAgent, type AgentOptions as HttpsAgentOptions } from 'https'; + +import { createChildLogger, getTracedAWSV3Client } from '@aws-github-runner/aws-powertools-util'; +import { getParameter, putParameter } from '@aws-github-runner/aws-ssm-util'; +import { DynamoDBClient, GetItemCommand, PutItemCommand, type AttributeValue } from '@aws-sdk/client-dynamodb'; +import { NodeHttpHandler } from '@smithy/node-http-handler'; + +import type { CreateGitHubRunnerConfig, RunnerConfigStorage, RunnerConfigStorageBackend } from './types'; + +const logger = createChildLogger('runner-config-storage'); +const DEFAULT_DYNAMODB_PARTITION_KEY_NAME = 'id'; +const DEFAULT_DYNAMODB_VALUE_ATTRIBUTE_NAME = 'value'; +const DEFAULT_DYNAMODB_CONFIG_KEY_PREFIX = 'config#'; +const DEFAULT_DYNAMODB_CONSISTENT_READ = true; +const DEFAULT_DYNAMODB_TOKEN_OVERWRITE_PROTECTION_ENABLED = true; +const DEFAULT_DYNAMODB_TTL_ATTRIBUTE_NAME = 'expires_at'; +const DEFAULT_DYNAMODB_CLIENT_MAX_ATTEMPTS = 10; +const DEFAULT_DYNAMODB_CLIENT_RETRY_MODE = 'adaptive'; +const DEFAULT_DYNAMODB_CLIENT_HTTP_KEEP_ALIVE = true; +const DEFAULT_DYNAMODB_CLIENT_HTTP_MAX_SOCKETS = 50; + +export interface RunnerConfigStore { + backend: RunnerConfigStorageBackend; + delayWritesForSsmThroughput: boolean; + getConfigValue(key: string): Promise; + putConfigValue(key: string, value: string, options?: RunnerConfigStorePutOptions): Promise; + putRunnerConfig(runnerId: string, value: string, options?: RunnerConfigStorePutOptions): Promise; +} + +export interface RunnerConfigStorePutOptions { + tags?: { Key: string; Value: string }[]; +} + +type DynamoDbStorageConfig = NonNullable; + +interface DynamoDbRunnerConfigStoreConfig { + tableName: string; + partitionKeyName: string; + valueAttributeName: string; + configKeyPrefix: string; + consistentRead: boolean; + tokenOverwriteProtectionEnabled: boolean; + tokenKeyPrefix: string; + tokenTtlSeconds: number; + ttlAttributeName: string; +} + +interface DynamoDbClientConfig { + maxAttempts: number; + retryMode: 'standard' | 'adaptive'; + httpKeepAlive: boolean; + httpMaxSockets: number; + httpKeepAliveMsecs?: number; +} + +let memoisedDynamoDbClient: DynamoDBClient | undefined; + +export function dynamoDbClient(): DynamoDBClient { + if (memoisedDynamoDbClient) { + return memoisedDynamoDbClient; + } + + const clientConfig = loadDynamoDbClientConfigFromEnv(); + const httpsAgentOptions: HttpsAgentOptions = { + keepAlive: clientConfig.httpKeepAlive, + maxSockets: clientConfig.httpMaxSockets, + }; + + if (clientConfig.httpKeepAliveMsecs !== undefined) { + httpsAgentOptions.keepAliveMsecs = clientConfig.httpKeepAliveMsecs; + } + + memoisedDynamoDbClient = getTracedAWSV3Client( + new DynamoDBClient({ + region: process.env.AWS_REGION, + maxAttempts: clientConfig.maxAttempts, + retryMode: clientConfig.retryMode, + requestHandler: new NodeHttpHandler({ + httpsAgent: new HttpsAgent(httpsAgentOptions), + }), + }), + ); + + return memoisedDynamoDbClient; +} + +export function resetDynamoDbClient(): void { + memoisedDynamoDbClient = undefined; +} + +export function loadRunnerConfigStorageFromEnv(): RunnerConfigStorage { + const backend = parseBackend(process.env.RUNNER_CONFIG_STORAGE_BACKEND); + + if (backend === 'ssm') { + return { backend }; + } + + const tableName = requireEnv('RUNNER_CONFIG_DYNAMODB_TABLE_NAME'); + const tokenKeyPrefix = requireEnv('RUNNER_CONFIG_DYNAMODB_TOKEN_KEY_PREFIX'); + const tokenTtlSeconds = parsePositiveInteger( + process.env.RUNNER_CONFIG_DYNAMODB_TTL_SECONDS, + undefined, + 'RUNNER_CONFIG_DYNAMODB_TTL_SECONDS', + ); + + return { + backend, + dynamodb: { + tableName, + partitionKeyName: parseNonEmptyString( + process.env.RUNNER_CONFIG_DYNAMODB_PARTITION_KEY_NAME, + DEFAULT_DYNAMODB_PARTITION_KEY_NAME, + 'RUNNER_CONFIG_DYNAMODB_PARTITION_KEY_NAME', + ), + valueAttributeName: parseNonEmptyString( + process.env.RUNNER_CONFIG_DYNAMODB_VALUE_ATTRIBUTE_NAME, + DEFAULT_DYNAMODB_VALUE_ATTRIBUTE_NAME, + 'RUNNER_CONFIG_DYNAMODB_VALUE_ATTRIBUTE_NAME', + ), + configKeyPrefix: parseNonEmptyString( + process.env.RUNNER_CONFIG_DYNAMODB_CONFIG_KEY_PREFIX, + DEFAULT_DYNAMODB_CONFIG_KEY_PREFIX, + 'RUNNER_CONFIG_DYNAMODB_CONFIG_KEY_PREFIX', + ), + consistentRead: parseBoolean( + process.env.RUNNER_CONFIG_DYNAMODB_CONSISTENT_READ, + DEFAULT_DYNAMODB_CONSISTENT_READ, + 'RUNNER_CONFIG_DYNAMODB_CONSISTENT_READ', + ), + tokenOverwriteProtectionEnabled: parseBoolean( + process.env.RUNNER_CONFIG_DYNAMODB_TOKEN_OVERWRITE_PROTECTION_ENABLED, + DEFAULT_DYNAMODB_TOKEN_OVERWRITE_PROTECTION_ENABLED, + 'RUNNER_CONFIG_DYNAMODB_TOKEN_OVERWRITE_PROTECTION_ENABLED', + ), + tokenKeyPrefix, + tokenTtlSeconds, + ttlAttributeName: parseNonEmptyString( + process.env.RUNNER_CONFIG_DYNAMODB_TTL_ATTRIBUTE_NAME, + DEFAULT_DYNAMODB_TTL_ATTRIBUTE_NAME, + 'RUNNER_CONFIG_DYNAMODB_TTL_ATTRIBUTE_NAME', + ), + }, + }; +} + +export function createRunnerConfigStore(githubRunnerConfig: CreateGitHubRunnerConfig): RunnerConfigStore { + const storage = githubRunnerConfig.runnerConfigStorage ?? { backend: 'ssm' }; + + if (storage.backend === 'ssm') { + return new SsmRunnerConfigStore(githubRunnerConfig.ssmConfigPath, githubRunnerConfig.ssmTokenPath); + } + + if (!storage.dynamodb) { + throw new Error('DynamoDB runner config storage requires tableName, tokenKeyPrefix, and tokenTtlSeconds'); + } + + return new DynamoDbRunnerConfigStore(storage.dynamodb); +} + +function loadDynamoDbClientConfigFromEnv(): DynamoDbClientConfig { + return { + maxAttempts: parsePositiveInteger( + process.env.RUNNER_CONFIG_DYNAMODB_CLIENT_MAX_ATTEMPTS, + DEFAULT_DYNAMODB_CLIENT_MAX_ATTEMPTS, + 'RUNNER_CONFIG_DYNAMODB_CLIENT_MAX_ATTEMPTS', + ), + retryMode: parseRetryMode(process.env.RUNNER_CONFIG_DYNAMODB_CLIENT_RETRY_MODE), + httpKeepAlive: parseBoolean( + process.env.RUNNER_CONFIG_DYNAMODB_CLIENT_HTTP_KEEP_ALIVE, + DEFAULT_DYNAMODB_CLIENT_HTTP_KEEP_ALIVE, + 'RUNNER_CONFIG_DYNAMODB_CLIENT_HTTP_KEEP_ALIVE', + ), + httpMaxSockets: parsePositiveInteger( + process.env.RUNNER_CONFIG_DYNAMODB_CLIENT_HTTP_MAX_SOCKETS, + DEFAULT_DYNAMODB_CLIENT_HTTP_MAX_SOCKETS, + 'RUNNER_CONFIG_DYNAMODB_CLIENT_HTTP_MAX_SOCKETS', + ), + httpKeepAliveMsecs: parseOptionalNonNegativeInteger( + process.env.RUNNER_CONFIG_DYNAMODB_CLIENT_HTTP_KEEP_ALIVE_MSECS, + 'RUNNER_CONFIG_DYNAMODB_CLIENT_HTTP_KEEP_ALIVE_MSECS', + ), + }; +} + +function parseBackend(value: string | undefined): RunnerConfigStorageBackend { + const backend = (value ?? 'ssm').toLowerCase(); + if (backend === 'ssm' || backend === 'dynamodb') { + return backend; + } + + throw new Error(`Unsupported RUNNER_CONFIG_STORAGE_BACKEND '${value}'`); +} + +function requireEnv(name: string): string { + const value = process.env[name]; + if (!value || value.trim() === '') { + throw new Error(`${name} must be set when RUNNER_CONFIG_STORAGE_BACKEND is 'dynamodb'`); + } + return value; +} + +function parseNonEmptyString(value: string | undefined, defaultValue: string, name: string): string { + if (value === undefined) { + return defaultValue; + } + + if (value.trim() === '') { + throw new Error(`${name} must not be empty`); + } + + return value; +} + +function parsePositiveInteger(value: string | undefined, defaultValue: number | undefined, name: string): number { + if (!value || value.trim() === '') { + if (defaultValue === undefined) { + throw new Error(`${name} must be set and be a positive integer`); + } + + return defaultValue; + } + + const parsed = Number(value); + if (!Number.isInteger(parsed) || parsed <= 0) { + throw new Error(`${name} must be a positive integer`); + } + + return parsed; +} + +function parseOptionalNonNegativeInteger(value: string | undefined, name: string): number | undefined { + if (!value || value.trim() === '') { + return undefined; + } + + const parsed = Number(value); + if (!Number.isInteger(parsed) || parsed < 0) { + throw new Error(`${name} must be a non-negative integer`); + } + + return parsed; +} + +function parseBoolean(value: string | undefined, defaultValue: boolean, name: string): boolean { + if (!value || value.trim() === '') { + return defaultValue; + } + + const normalised = value.trim().toLowerCase(); + if (normalised === 'true') { + return true; + } + if (normalised === 'false') { + return false; + } + + throw new Error(`${name} must be either 'true' or 'false'`); +} + +function parseRetryMode(value: string | undefined): 'standard' | 'adaptive' { + const retryMode = value?.trim() ?? DEFAULT_DYNAMODB_CLIENT_RETRY_MODE; + if (retryMode === 'standard' || retryMode === 'adaptive') { + return retryMode; + } + + throw new Error(`RUNNER_CONFIG_DYNAMODB_CLIENT_RETRY_MODE must be either 'standard' or 'adaptive'`); +} + +function normaliseDynamoDbConfig(config: DynamoDbStorageConfig): DynamoDbRunnerConfigStoreConfig { + return { + tableName: config.tableName, + partitionKeyName: config.partitionKeyName ?? DEFAULT_DYNAMODB_PARTITION_KEY_NAME, + valueAttributeName: config.valueAttributeName ?? DEFAULT_DYNAMODB_VALUE_ATTRIBUTE_NAME, + configKeyPrefix: config.configKeyPrefix ?? DEFAULT_DYNAMODB_CONFIG_KEY_PREFIX, + consistentRead: config.consistentRead ?? DEFAULT_DYNAMODB_CONSISTENT_READ, + tokenOverwriteProtectionEnabled: + config.tokenOverwriteProtectionEnabled ?? DEFAULT_DYNAMODB_TOKEN_OVERWRITE_PROTECTION_ENABLED, + tokenKeyPrefix: config.tokenKeyPrefix, + tokenTtlSeconds: config.tokenTtlSeconds, + ttlAttributeName: config.ttlAttributeName ?? DEFAULT_DYNAMODB_TTL_ATTRIBUTE_NAME, + }; +} + +class SsmRunnerConfigStore implements RunnerConfigStore { + readonly backend = 'ssm'; + readonly delayWritesForSsmThroughput = true; + + constructor( + private readonly configPath: string, + private readonly tokenPath: string, + ) {} + + async getConfigValue(key: string): Promise { + return await getParameter(`${this.configPath}/${key}`); + } + + async putConfigValue(key: string, value: string, options: RunnerConfigStorePutOptions = {}): Promise { + await putParameter(`${this.configPath}/${key}`, value, false, { + tags: options.tags, + }); + } + + async putRunnerConfig(runnerId: string, value: string, options: RunnerConfigStorePutOptions = {}): Promise { + await putParameter(`${this.tokenPath}/${runnerId}`, value, true, { + tags: options.tags, + }); + } +} + +class DynamoDbRunnerConfigStore implements RunnerConfigStore { + readonly backend = 'dynamodb'; + readonly delayWritesForSsmThroughput = false; + private readonly config: DynamoDbRunnerConfigStoreConfig; + + constructor(config: DynamoDbStorageConfig) { + this.config = normaliseDynamoDbConfig(config); + } + + async getConfigValue(key: string): Promise { + return await this.getValue(`${this.config.configKeyPrefix}${key}`); + } + + async putConfigValue(key: string, value: string): Promise { + await this.putValue(`${this.config.configKeyPrefix}${key}`, value); + } + + async putRunnerConfig(runnerId: string, value: string): Promise { + await this.putValue( + `${this.config.tokenKeyPrefix}${runnerId}`, + value, + this.config.tokenOverwriteProtectionEnabled + ? { + conditionExpression: 'attribute_not_exists(#partition_key)', + expressionAttributeNames: { + '#partition_key': this.config.partitionKeyName, + }, + ttlSeconds: this.config.tokenTtlSeconds, + } + : { + ttlSeconds: this.config.tokenTtlSeconds, + }, + ); + } + + private async getValue(id: string): Promise { + const result = await dynamoDbClient().send( + new GetItemCommand({ + TableName: this.config.tableName, + Key: { + [this.config.partitionKeyName]: { S: id }, + }, + ConsistentRead: this.config.consistentRead, + ProjectionExpression: '#value', + ExpressionAttributeNames: { + '#value': this.config.valueAttributeName, + }, + }), + ); + + return result.Item?.[this.config.valueAttributeName]?.S; + } + + private async putValue( + id: string, + value: string, + options: { + conditionExpression?: string; + expressionAttributeNames?: Record; + ttlSeconds?: number; + } = {}, + ): Promise { + const item: Record = { + [this.config.partitionKeyName]: { S: id }, + [this.config.valueAttributeName]: { S: value }, + }; + + if (options.ttlSeconds) { + const expiresAt = Math.floor(Date.now() / 1000) + options.ttlSeconds; + item[this.config.ttlAttributeName] = { N: expiresAt.toString() }; + } + + logger.debug('Writing runner config value to DynamoDB', { + tableName: this.config.tableName, + id, + hasTtl: options.ttlSeconds !== undefined, + }); + + await dynamoDbClient().send( + new PutItemCommand({ + TableName: this.config.tableName, + Item: item, + ConditionExpression: options.conditionExpression, + ExpressionAttributeNames: options.expressionAttributeNames, + }), + ); + } +} diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up.ts index cdb8b1bb4a..bae7a53537 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-up.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-up.ts @@ -14,6 +14,7 @@ import { validateSsmParameterStoreTags, } from './github-runner'; import { publishRetryMessage } from './job-retry'; +import { loadRunnerConfigStorageFromEnv } from './runner-config-storage'; import type { CreateScaleUpRunnersResult } from './scale-up-provider'; import type { ActionRequestMessage, @@ -82,6 +83,7 @@ export async function scaleUp(payloads: ActionRequestMessageSQS[]): Promise { +export async function interruptionWarning( + event: SpotInterruptionWarning, + context: Context, +): Promise { setContext(context, 'lambda.ts'); logger.logEventIfEnabled(event); logger.debug('Configuration of the lambda', { config }); diff --git a/lambdas/functions/termination-watcher/src/termination-warning.test.ts b/lambdas/functions/termination-watcher/src/termination-warning.test.ts index 0e7792107b..e9dc4a05af 100644 --- a/lambdas/functions/termination-watcher/src/termination-warning.test.ts +++ b/lambdas/functions/termination-watcher/src/termination-warning.test.ts @@ -2,7 +2,7 @@ import { EC2Client, Instance } from '@aws-sdk/client-ec2'; import { mockClient } from 'aws-sdk-client-mock'; import 'aws-sdk-client-mock-jest'; import { handle } from './termination-warning'; -import { SpotInterruptionWarning, SpotTerminationDetail, InstanceStateChangeEvent } from './types'; +import { SpotInterruptionWarning, SpotTerminationDetail } from './types'; import { metricEvent } from './metric-event'; import { deregisterRunner } from './deregister'; @@ -36,7 +36,7 @@ const config = { ghesApiUrl: '', }; -const spotEvent: SpotInterruptionWarning = { +const event: SpotInterruptionWarning = { version: '0', id: '1', 'detail-type': 'EC2 Spot Instance Interruption Warning', @@ -51,23 +51,8 @@ const spotEvent: SpotInterruptionWarning = { }, }; -const stateChangeEvent: InstanceStateChangeEvent = { - version: '0', - id: '2', - 'detail-type': 'EC2 Instance State-change Notification', - source: 'aws.ec2', - account: '123456789012', - time: '2015-11-11T21:30:00Z', - region: 'us-east-1', - resources: ['arn:aws:ec2:us-east-1b:instance/i-abcd1111'], - detail: { - 'instance-id': 'i-abcd1111', - state: 'shutting-down', - }, -}; - const instance: Instance = { - InstanceId: 'i-abcd1111', + InstanceId: event.detail['instance-id'], InstanceType: 't2.micro', Tags: [ { Key: 'Name', Value: 'test-instance' }, @@ -83,27 +68,28 @@ describe('handle termination warning', () => { vi.clearAllMocks(); }); - it('should emit metric for spot interruption events', async () => { + it('should log and create an metric', async () => { vi.mocked(getInstances).mockResolvedValue([instance]); - await handle(spotEvent, config); + await handle(event, config); - expect(metricEvent).toHaveBeenCalledWith(instance, spotEvent, 'SpotInterruptionWarning', expect.anything()); + expect(metricEvent).toHaveBeenCalled(); + expect(metricEvent).toHaveBeenCalledWith(instance, event, 'SpotInterruptionWarning', expect.anything()); expect(deregisterRunner).toHaveBeenCalledWith(instance, config); }); - it('should not emit metric when createSpotWarningMetric is false', async () => { + it('should log details and not create a metric', async () => { vi.mocked(getInstances).mockResolvedValue([instance]); const noMetricConfig = { ...config, createSpotWarningMetric: false }; - await handle(spotEvent, noMetricConfig); - expect(metricEvent).toHaveBeenCalledWith(instance, spotEvent, undefined, expect.anything()); + await handle(event, noMetricConfig); + expect(metricEvent).toHaveBeenCalledWith(instance, event, undefined, expect.anything()); expect(deregisterRunner).toHaveBeenCalledWith(instance, noMetricConfig); }); - it('should not emit metric or deregister if filter not matched', async () => { + it('should not create a metric if filter not matched.', async () => { vi.mocked(getInstances).mockResolvedValue([instance]); - await handle(spotEvent, { + await handle(event, { createSpotWarningMetric: true, createSpotTerminationMetric: false, tagFilters: { 'ghr:environment': '_NO_MATCH_' }, @@ -115,13 +101,4 @@ describe('handle termination warning', () => { expect(metricEvent).not.toHaveBeenCalled(); expect(deregisterRunner).not.toHaveBeenCalled(); }); - - it('should not emit metric for instance state-change events but still deregister', async () => { - vi.mocked(getInstances).mockResolvedValue([instance]); - - await handle(stateChangeEvent, config); - - expect(metricEvent).toHaveBeenCalledWith(instance, stateChangeEvent, undefined, expect.anything()); - expect(deregisterRunner).toHaveBeenCalledWith(instance, config); - }); }); diff --git a/lambdas/functions/termination-watcher/src/termination-warning.ts b/lambdas/functions/termination-watcher/src/termination-warning.ts index b18d890a54..8e5330be25 100644 --- a/lambdas/functions/termination-watcher/src/termination-warning.ts +++ b/lambdas/functions/termination-watcher/src/termination-warning.ts @@ -1,5 +1,5 @@ import { createChildLogger, getTracedAWSV3Client } from '@aws-github-runner/aws-powertools-util'; -import { TerminationWatcherEvent } from './types'; +import { SpotInterruptionWarning, SpotTerminationDetail } from './types'; import { EC2Client, Instance } from '@aws-sdk/client-ec2'; import { Config } from './ConfigResolver'; import { tagFilter, getInstances } from './ec2'; @@ -8,7 +8,7 @@ import { deregisterRunner } from './deregister'; const logger = createChildLogger('termination-warning'); -async function handle(event: TerminationWatcherEvent, config: Config): Promise { +async function handle(event: SpotInterruptionWarning, config: Config): Promise { logger.debug('Received spot notification warning:', { event }); const ec2 = getTracedAWSV3Client(new EC2Client({ region: process.env.AWS_REGION })); const instances = await getInstances(ec2, [event.detail['instance-id']]); @@ -19,16 +19,14 @@ async function handle(event: TerminationWatcherEvent, config: Config): Promise, config: Config, ): Promise { for (const instance of instances) { const matchFilter = tagFilter(instance, config.tagFilters); if (matchFilter) { - const isSpotInterruption = event['detail-type'] === 'EC2 Spot Instance Interruption Warning'; - const metricName = isSpotInterruption && config.createSpotWarningMetric ? 'SpotInterruptionWarning' : undefined; - metricEvent(instance, event, metricName, logger); + metricEvent(instance, event, config.createSpotWarningMetric ? 'SpotInterruptionWarning' : undefined, logger); await deregisterRunner(instance, config); } else { logger.debug( diff --git a/lambdas/functions/termination-watcher/src/types.d.ts b/lambdas/functions/termination-watcher/src/types.d.ts index 2409c3c26c..d242221142 100644 --- a/lambdas/functions/termination-watcher/src/types.d.ts +++ b/lambdas/functions/termination-watcher/src/types.d.ts @@ -42,14 +42,3 @@ interface UserIdentity { interface ServiceEventDetails { instanceIdSet: string[]; } - -export interface InstanceStateChangeDetail { - 'instance-id': string; - state: string; -} - -// eslint-disable-next-line @typescript-eslint/no-empty-object-type -export interface InstanceStateChangeEvent - extends EventBridgeEvent<'EC2 Instance State-change Notification', InstanceStateChangeDetail> {} - -export type TerminationWatcherEvent = SpotInterruptionWarning | InstanceStateChangeEvent; diff --git a/lambdas/libs/runner-providers/core/index.ts b/lambdas/libs/runner-providers/core/index.ts index e0e387acdf..d8e9676732 100644 --- a/lambdas/libs/runner-providers/core/index.ts +++ b/lambdas/libs/runner-providers/core/index.ts @@ -8,6 +8,22 @@ export interface RunnerProvider { export type LambdaRunnerSource = 'scale-up-lambda' | 'pool-lambda'; export type GitHubRunnerType = 'Org' | 'Repo'; +export type RunnerConfigStorageBackend = 'ssm' | 'dynamodb'; + +export interface RunnerConfigStorage { + backend: RunnerConfigStorageBackend; + dynamodb?: { + tableName: string; + partitionKeyName?: string; + valueAttributeName?: string; + configKeyPrefix?: string; + consistentRead?: boolean; + tokenOverwriteProtectionEnabled?: boolean; + tokenKeyPrefix: string; + tokenTtlSeconds: number; + ttlAttributeName?: string; + }; +} export interface CreateGitHubRunnerConfig { ephemeral: boolean; @@ -22,6 +38,7 @@ export interface CreateGitHubRunnerConfig { ssmTokenPath: string; ssmConfigPath: string; ssmParameterStoreTags: { Key: string; Value: string }[]; + runnerConfigStorage?: RunnerConfigStorage; } export interface GitHubRunnerMetadata { diff --git a/lambdas/yarn.lock b/lambdas/yarn.lock index 38cce704ba..06f851f5b9 100644 --- a/lambdas/yarn.lock +++ b/lambdas/yarn.lock @@ -149,6 +149,7 @@ __metadata: "@aws-github-runner/aws-ssm-util": "npm:*" "@aws-github-runner/runner-providers": "npm:*" "@aws-lambda-powertools/parameters": "npm:^2.31.0" + "@aws-sdk/client-dynamodb": "npm:^3.1009.0" "@aws-sdk/client-ec2": "npm:^3.1009.0" "@aws-sdk/client-sqs": "npm:^3.1009.0" "@aws-sdk/types": "npm:^3.973.6" @@ -159,6 +160,7 @@ __metadata: "@octokit/plugin-throttling": "npm:11.0.3" "@octokit/rest": "npm:22.0.1" "@octokit/types": "npm:^16.0.0" + "@smithy/node-http-handler": "npm:^4.5.0" "@types/aws-lambda": "npm:^8.10.159" "@types/node": "npm:^22.19.3" "@vercel/ncc": "npm:^0.38.4" @@ -342,6 +344,24 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/client-dynamodb@npm:^3.1009.0": + version: 3.1101.0 + resolution: "@aws-sdk/client-dynamodb@npm:3.1101.0" + dependencies: + "@aws-sdk/core": "npm:^3.977.4" + "@aws-sdk/credential-provider-node": "npm:^3.972.76" + "@aws-sdk/dynamodb-codec": "npm:^3.973.39" + "@aws-sdk/middleware-endpoint-discovery": "npm:^3.972.27" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/fetch-http-handler": "npm:^5.6.13" + "@smithy/node-http-handler": "npm:^4.9.13" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/2377634443a17cc861b0eea5d1f9c5adf6ce93b76561b6dc89352d49fff4d255348f87c68f50a11fff31c6b53235fdc0d15b1776369981c80d8b088edd19293f + languageName: node + linkType: hard + "@aws-sdk/client-ec2@npm:^3.1009.0": version: 3.1014.0 resolution: "@aws-sdk/client-ec2@npm:3.1014.0" @@ -620,6 +640,22 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/core@npm:^3.977.4": + version: 3.977.4 + resolution: "@aws-sdk/core@npm:3.977.4" + dependencies: + "@aws-sdk/types": "npm:^3.974.2" + "@aws-sdk/xml-builder": "npm:^3.972.37" + "@aws/lambda-invoke-store": "npm:^0.3.0" + "@smithy/core": "npm:^3.31.1" + "@smithy/signature-v4": "npm:^5.6.12" + "@smithy/types": "npm:^4.16.1" + bowser: "npm:^2.11.0" + tslib: "npm:^2.6.2" + checksum: 10c0/f80b7fdf2868b457b5ee71a917e5e6b6c2dba3d334df7e4e545d31503d3fbfa686d71c66ba5df002fa0910cfca1ae7c44170dc0419ee87f9b7380cbfe9172769 + languageName: node + linkType: hard + "@aws-sdk/crc64-nvme@npm:^3.972.5": version: 3.972.5 resolution: "@aws-sdk/crc64-nvme@npm:3.972.5" @@ -643,6 +679,19 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-env@npm:^3.972.65": + version: 3.972.65 + resolution: "@aws-sdk/credential-provider-env@npm:3.972.65" + dependencies: + "@aws-sdk/core": "npm:^3.977.4" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/2d6fb7a2f4fc8482fec2d15a45ad751d23ceaac27656724305f84c4b770e4985b569927c56b647edb7775ba2e0270fd5cd89e5b7ee507b421a8306e7466473b2 + languageName: node + linkType: hard + "@aws-sdk/credential-provider-http@npm:^3.972.23": version: 3.972.23 resolution: "@aws-sdk/credential-provider-http@npm:3.972.23" @@ -661,6 +710,21 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-http@npm:^3.972.67": + version: 3.972.67 + resolution: "@aws-sdk/credential-provider-http@npm:3.972.67" + dependencies: + "@aws-sdk/core": "npm:^3.977.4" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/fetch-http-handler": "npm:^5.6.13" + "@smithy/node-http-handler": "npm:^4.9.13" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/a75ef47b924b96710fa05b5d93b5e7a2e3a0e81e7389f8183e6202b211785ccc5c12fcfbd83227f02e11f5725971013ed0124f1540e2a5d46d254558da3e809f + languageName: node + linkType: hard + "@aws-sdk/credential-provider-ini@npm:^3.972.23": version: 3.972.23 resolution: "@aws-sdk/credential-provider-ini@npm:3.972.23" @@ -683,6 +747,27 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-ini@npm:^3.973.10": + version: 3.973.10 + resolution: "@aws-sdk/credential-provider-ini@npm:3.973.10" + dependencies: + "@aws-sdk/core": "npm:^3.977.4" + "@aws-sdk/credential-provider-env": "npm:^3.972.65" + "@aws-sdk/credential-provider-http": "npm:^3.972.67" + "@aws-sdk/credential-provider-login": "npm:^3.972.72" + "@aws-sdk/credential-provider-process": "npm:^3.972.65" + "@aws-sdk/credential-provider-sso": "npm:^3.973.9" + "@aws-sdk/credential-provider-web-identity": "npm:^3.972.71" + "@aws-sdk/nested-clients": "npm:^3.997.39" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/credential-provider-imds": "npm:^4.4.16" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/a69b39c1241efc5d467ed664919eb129f4ee48b7255ccb88b9ba6e674407a2380ce553cea7508748863a2d384e658a96c0568271b28fa64f6178681b48873e1b + languageName: node + linkType: hard + "@aws-sdk/credential-provider-login@npm:^3.972.23": version: 3.972.23 resolution: "@aws-sdk/credential-provider-login@npm:3.972.23" @@ -699,6 +784,20 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-login@npm:^3.972.72": + version: 3.972.72 + resolution: "@aws-sdk/credential-provider-login@npm:3.972.72" + dependencies: + "@aws-sdk/core": "npm:^3.977.4" + "@aws-sdk/nested-clients": "npm:^3.997.39" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/2efe2b875ccbb47b4ac6b609396cb7aae58fadfface8f4d4f285652a256317c6af21fff70bbbb7d8f5ea36697e1f9549ec4fd51e5bb55d5fa2582d39019dc7d0 + languageName: node + linkType: hard + "@aws-sdk/credential-provider-node@npm:^3.972.24": version: 3.972.24 resolution: "@aws-sdk/credential-provider-node@npm:3.972.24" @@ -719,6 +818,25 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-node@npm:^3.972.76": + version: 3.972.76 + resolution: "@aws-sdk/credential-provider-node@npm:3.972.76" + dependencies: + "@aws-sdk/credential-provider-env": "npm:^3.972.65" + "@aws-sdk/credential-provider-http": "npm:^3.972.67" + "@aws-sdk/credential-provider-ini": "npm:^3.973.10" + "@aws-sdk/credential-provider-process": "npm:^3.972.65" + "@aws-sdk/credential-provider-sso": "npm:^3.973.9" + "@aws-sdk/credential-provider-web-identity": "npm:^3.972.71" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/credential-provider-imds": "npm:^4.4.16" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/f3dd3ca4cecb35a5cbe3ffd1029a4af370a27591996d50d8d2dbc08811d5472e53b10ce0cb18090d85d98cf5c46c0257ecc6a7ff1dc70e7164850a4b4bc1a5e1 + languageName: node + linkType: hard + "@aws-sdk/credential-provider-process@npm:^3.972.21": version: 3.972.21 resolution: "@aws-sdk/credential-provider-process@npm:3.972.21" @@ -733,6 +851,19 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-process@npm:^3.972.65": + version: 3.972.65 + resolution: "@aws-sdk/credential-provider-process@npm:3.972.65" + dependencies: + "@aws-sdk/core": "npm:^3.977.4" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/e9a1de343e53e9ba491137b4ae25e5516e761af8509749b896a63850872ceff4a79cab09f98bee745c86eeb4fe1dd9b8cbd71fd1a3fafd15660c6bcf1a09f838 + languageName: node + linkType: hard + "@aws-sdk/credential-provider-sso@npm:^3.972.23": version: 3.972.23 resolution: "@aws-sdk/credential-provider-sso@npm:3.972.23" @@ -749,6 +880,21 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-sso@npm:^3.973.9": + version: 3.973.9 + resolution: "@aws-sdk/credential-provider-sso@npm:3.973.9" + dependencies: + "@aws-sdk/core": "npm:^3.977.4" + "@aws-sdk/nested-clients": "npm:^3.997.39" + "@aws-sdk/token-providers": "npm:3.1100.0" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/94d66be2801d7a84cfbfac3a6c86d0b6faf4ea59dfe811134fa5fbc703c57ff9d7f084f37b3499a7ec34beaa5056e81d2cbfb49ed1faf23c1c93561d1652a572 + languageName: node + linkType: hard + "@aws-sdk/credential-provider-web-identity@npm:^3.972.23": version: 3.972.23 resolution: "@aws-sdk/credential-provider-web-identity@npm:3.972.23" @@ -764,6 +910,42 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-web-identity@npm:^3.972.71": + version: 3.972.71 + resolution: "@aws-sdk/credential-provider-web-identity@npm:3.972.71" + dependencies: + "@aws-sdk/core": "npm:^3.977.4" + "@aws-sdk/nested-clients": "npm:^3.997.39" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/340d78eb346facd2f90b21cb1ced8c51f30f1e2c877874c03b1dfb6d3d20ae0b369905f37ab66b90efb3ff8ea5b5f019945484d9ba92dfdbb6e2427bf0a41c28 + languageName: node + linkType: hard + +"@aws-sdk/dynamodb-codec@npm:^3.973.39": + version: 3.973.39 + resolution: "@aws-sdk/dynamodb-codec@npm:3.973.39" + dependencies: + "@aws-sdk/core": "npm:^3.977.4" + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/04a6c5676fd2c313762db85743bba6f51ba35ddb3e98f3aa605cca8597ba00b57aa98e346374ed549c943ba6d9379d4fe45387fb141c1e2044ba093c8905be9b + languageName: node + linkType: hard + +"@aws-sdk/endpoint-cache@npm:^3.972.9": + version: 3.972.9 + resolution: "@aws-sdk/endpoint-cache@npm:3.972.9" + dependencies: + mnemonist: "npm:0.38.3" + tslib: "npm:^2.6.2" + checksum: 10c0/cfff776b4653ddbbdfa250c17d8dbe740c61b86dcc478237ab03d7104a912e42f459d5c399c78c93d9ec5fde162913f51ebd5ebbb592de5ce15a9dfc156a895a + languageName: node + linkType: hard + "@aws-sdk/lib-storage@npm:^3.1009.0": version: 3.1014.0 resolution: "@aws-sdk/lib-storage@npm:3.1014.0" @@ -796,6 +978,19 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/middleware-endpoint-discovery@npm:^3.972.27": + version: 3.972.27 + resolution: "@aws-sdk/middleware-endpoint-discovery@npm:3.972.27" + dependencies: + "@aws-sdk/endpoint-cache": "npm:^3.972.9" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/18a42741a856a3fb52b8be3a311119bbfc4ffd080af9d4d3c8e9be4211fb7b5172806d3b555ac3f78163ecd5d6467b49b6a91b710bb5559942e066ad43318108 + languageName: node + linkType: hard + "@aws-sdk/middleware-expect-continue@npm:^3.972.8": version: 3.972.8 resolution: "@aws-sdk/middleware-expect-continue@npm:3.972.8" @@ -1002,6 +1197,22 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/nested-clients@npm:^3.997.39": + version: 3.997.39 + resolution: "@aws-sdk/nested-clients@npm:3.997.39" + dependencies: + "@aws-sdk/core": "npm:^3.977.4" + "@aws-sdk/signature-v4-multi-region": "npm:^3.996.43" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/fetch-http-handler": "npm:^5.6.13" + "@smithy/node-http-handler": "npm:^4.9.13" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/085dbcd65bd37cff080ad194aa316a915e04145c3ed92faa714ad2b20498501b45b6ed05df29c5283e1a7d8c7586c49bb27bcb0035e5ccebbb0593133a8cd988 + languageName: node + linkType: hard + "@aws-sdk/region-config-resolver@npm:^3.972.9": version: 3.972.9 resolution: "@aws-sdk/region-config-resolver@npm:3.972.9" @@ -1029,6 +1240,18 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/signature-v4-multi-region@npm:^3.996.43": + version: 3.996.43 + resolution: "@aws-sdk/signature-v4-multi-region@npm:3.996.43" + dependencies: + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/signature-v4": "npm:^5.6.12" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/268608dd5624c6377243903d588b9c13b8de3f3f3e6bea68fc684d125bc92a991fd15a67cb178d1a7a599d0415ce5283f44ba6b96d14909b185d7ff26a9d979b + languageName: node + linkType: hard + "@aws-sdk/token-providers@npm:3.1014.0": version: 3.1014.0 resolution: "@aws-sdk/token-providers@npm:3.1014.0" @@ -1044,6 +1267,20 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/token-providers@npm:3.1100.0": + version: 3.1100.0 + resolution: "@aws-sdk/token-providers@npm:3.1100.0" + dependencies: + "@aws-sdk/core": "npm:^3.977.4" + "@aws-sdk/nested-clients": "npm:^3.997.39" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/ba4848a52175f801b238dcaa1815a8b51bed1ff102e171285c73b43819395fa02458f0b2f8574e49bd062143113ba2bd93890f85c6ff18c5cbfe8c90a836314f + languageName: node + linkType: hard + "@aws-sdk/types@npm:^3.222.0, @aws-sdk/types@npm:^3.4.1, @aws-sdk/types@npm:^3.973.6": version: 3.973.6 resolution: "@aws-sdk/types@npm:3.973.6" @@ -1054,6 +1291,16 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/types@npm:^3.974.2": + version: 3.974.2 + resolution: "@aws-sdk/types@npm:3.974.2" + dependencies: + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/b5ce05e8a4160c545edce1e8527e8ac490be7a6651c736f6811190b5d31d5682699889d51186ab0600df756679bebd2df9d650a17f577523441df803c4fb5777 + languageName: node + linkType: hard + "@aws-sdk/util-arn-parser@npm:^3.972.3": version: 3.972.3 resolution: "@aws-sdk/util-arn-parser@npm:3.972.3" @@ -1139,6 +1386,16 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/xml-builder@npm:^3.972.37": + version: 3.972.37 + resolution: "@aws-sdk/xml-builder@npm:3.972.37" + dependencies: + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/738f9302f495b3b95602641166a4182244add6e9e079201dba7e8994657dd442df0e4cea3355aa8c7d7f08efb385decaaf0b543f03efdb291c118536f36ac1a1 + languageName: node + linkType: hard + "@aws/lambda-invoke-store@npm:0.2.3, @aws/lambda-invoke-store@npm:^0.2.2": version: 0.2.3 resolution: "@aws/lambda-invoke-store@npm:0.2.3" @@ -1146,6 +1403,13 @@ __metadata: languageName: node linkType: hard +"@aws/lambda-invoke-store@npm:^0.3.0": + version: 0.3.0 + resolution: "@aws/lambda-invoke-store@npm:0.3.0" + checksum: 10c0/b4a2e6b3b5397bc606053e64270d26dc5c886336f88a98cad587b1592eec17058f8fb172f1827a9f0e591f3595cf8f01575c8c9b36cde38c06456f8a65204046 + languageName: node + linkType: hard + "@babel/code-frame@npm:^7.0.0, @babel/code-frame@npm:^7.12.13, @babel/code-frame@npm:^7.23.5, @babel/code-frame@npm:^7.28.6, @babel/code-frame@npm:^7.29.0": version: 7.29.0 resolution: "@babel/code-frame@npm:7.29.0" @@ -4404,6 +4668,16 @@ __metadata: languageName: node linkType: hard +"@smithy/core@npm:^3.31.1": + version: 3.31.1 + resolution: "@smithy/core@npm:3.31.1" + dependencies: + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/b953c792dea2c13249b58c1799e4d6aaf21eb1a61e203b83e8e3a9156bebe14ca0585f0ca1ffdf65a193294dddff92a06fbe5c3fbd63ff0c174c88130b47a128 + languageName: node + linkType: hard + "@smithy/credential-provider-imds@npm:^4.2.12": version: 4.2.12 resolution: "@smithy/credential-provider-imds@npm:4.2.12" @@ -4417,6 +4691,17 @@ __metadata: languageName: node linkType: hard +"@smithy/credential-provider-imds@npm:^4.4.16": + version: 4.4.16 + resolution: "@smithy/credential-provider-imds@npm:4.4.16" + dependencies: + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/d03687efbbd1f95e77b7dcb639f24f1600671929627cd743f7acf9640238746664e91f955026f22e235603e10537d46e31fa60f231adbdf37457e53720bc80f9 + languageName: node + linkType: hard + "@smithy/eventstream-codec@npm:^4.2.12": version: 4.2.12 resolution: "@smithy/eventstream-codec@npm:4.2.12" @@ -4485,6 +4770,17 @@ __metadata: languageName: node linkType: hard +"@smithy/fetch-http-handler@npm:^5.6.13": + version: 5.6.13 + resolution: "@smithy/fetch-http-handler@npm:5.6.13" + dependencies: + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/028ba8794a6c487ebefae7f40d0124f70e51a1f4e0e465457845c1a44fd607320cd3c64d4a961f159aef59470f0fd43f0d2011b44ee5ef753b7e1dccbdf32ca3 + languageName: node + linkType: hard + "@smithy/hash-blob-browser@npm:^4.2.13": version: 4.2.13 resolution: "@smithy/hash-blob-browser@npm:4.2.13" @@ -4650,6 +4946,17 @@ __metadata: languageName: node linkType: hard +"@smithy/node-http-handler@npm:^4.9.13": + version: 4.9.13 + resolution: "@smithy/node-http-handler@npm:4.9.13" + dependencies: + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/2f1cdef7a300ad49c3bb698c2ca4773af5e9202d291cfcd855c1b21ab08b3c4ddf56f3722d3251db4e9b7ac39ec1ebc551b156abf3fa70f74c5491bec421f6b5 + languageName: node + linkType: hard + "@smithy/property-provider@npm:^4.2.12": version: 4.2.12 resolution: "@smithy/property-provider@npm:4.2.12" @@ -4735,6 +5042,17 @@ __metadata: languageName: node linkType: hard +"@smithy/signature-v4@npm:^5.6.12": + version: 5.6.12 + resolution: "@smithy/signature-v4@npm:5.6.12" + dependencies: + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/33656a41ad61dee16209703cb96b46b29014b3c4fad23bfbb90cdb5415ac06c6577b2bfff958ef9e6c19091364945135a0370b12ddc2daed557c903846e81fe7 + languageName: node + linkType: hard + "@smithy/smithy-client@npm:^4.12.7": version: 4.12.7 resolution: "@smithy/smithy-client@npm:4.12.7" @@ -4768,6 +5086,15 @@ __metadata: languageName: node linkType: hard +"@smithy/types@npm:^4.16.1": + version: 4.16.1 + resolution: "@smithy/types@npm:4.16.1" + dependencies: + tslib: "npm:^2.6.2" + checksum: 10c0/e024d9d148deca7bd21d032a9316db109bbe7cf256ffbb8d3981655b9f4f7695c08ec9b87f5a8cf1442e783ba26cb27e4f09603c5bfa3ba1e526c41b1b3e94d2 + languageName: node + linkType: hard + "@smithy/url-parser@npm:^4.2.12": version: 4.2.12 resolution: "@smithy/url-parser@npm:4.2.12" @@ -8886,6 +9213,15 @@ __metadata: languageName: node linkType: hard +"mnemonist@npm:0.38.3": + version: 0.38.3 + resolution: "mnemonist@npm:0.38.3" + dependencies: + obliterator: "npm:^1.6.1" + checksum: 10c0/064aa1ee1a89fce2754423b3617c598fd65bc34311eb3c01dc063976f6b819b073bd23532415cf8c92240157b4c8fbb7ec5d79d717f2bd4fcd95d8131cb23acb + languageName: node + linkType: hard + "moment-timezone@npm:^0.6.0": version: 0.6.0 resolution: "moment-timezone@npm:0.6.0" @@ -9288,6 +9624,13 @@ __metadata: languageName: node linkType: hard +"obliterator@npm:^1.6.1": + version: 1.6.1 + resolution: "obliterator@npm:1.6.1" + checksum: 10c0/5fad57319aae0ef6e34efa640541d41c2dd9790a7ab808f17dcb66c83a81333963fc2dfcfa6e1b62158e5cef6291cdcf15c503ad6c3de54b2227dd4c3d7e1b55 + languageName: node + linkType: hard + "obug@npm:^2.1.1": version: 2.1.1 resolution: "obug@npm:2.1.1" diff --git a/main.tf b/main.tf index ca83523285..05323292f1 100644 --- a/main.tf +++ b/main.tf @@ -169,6 +169,7 @@ module "runners" { tokens = "${var.ssm_paths.runners}/tokens" config = "${var.ssm_paths.runners}/config" } + runner_config_storage = var.runner_config_storage s3_runner_binaries = var.enable_runner_binaries_syncer ? { arn = module.runner_binaries[0].bucket.arn @@ -282,8 +283,15 @@ module "runners" { pool_lambda_reserved_concurrent_executions = var.pool_lambda_reserved_concurrent_executions pool_include_busy_runners = var.pool_include_busy_runners - ssm_housekeeper = var.runners_ssm_housekeeper - ebs_optimized = var.runners_ebs_optimized + ssm_housekeeper = { + create = var.runners_ssm_housekeeper.create + schedule_expression = var.runners_ssm_housekeeper.schedule_expression + state = var.runners_ssm_housekeeper.enabled ? "ENABLED" : "DISABLED" + lambda_memory_size = var.runners_ssm_housekeeper.lambda_memory_size + lambda_timeout = var.runners_ssm_housekeeper.lambda_timeout + config = var.runners_ssm_housekeeper.config + } + ebs_optimized = var.runners_ebs_optimized metrics = var.metrics diff --git a/modules/multi-runner/outputs.tf b/modules/multi-runner/outputs.tf index 7ce7171faf..798e44bc8b 100644 --- a/modules/multi-runner/outputs.tf +++ b/modules/multi-runner/outputs.tf @@ -1,22 +1,24 @@ output "runners_map" { value = { for runner_key, runner in module.runners : runner_key => { - launch_template_name = runner.launch_template.name - launch_template_id = runner.launch_template.id - launch_template_version = runner.launch_template.latest_version - launch_template_ami_id = runner.launch_template.image_id - lambda_up = runner.lambda_scale_up - lambda_up_log_group = runner.lambda_scale_up_log_group - lambda_down = runner.lambda_scale_down - lambda_down_log_group = runner.lambda_scale_down_log_group - lambda_pool = runner.lambda_pool - lambda_pool_log_group = runner.lambda_pool_log_group - role_runner = runner.role_runner - role_scale_up = runner.role_scale_up - role_scale_down = runner.role_scale_down - role_pool = runner.role_pool - runners_log_groups = runner.runners_log_groups - logfiles = runner.logfiles + launch_template_name = runner.launch_template.name + launch_template_id = runner.launch_template.id + launch_template_version = runner.launch_template.latest_version + launch_template_ami_id = runner.launch_template.image_id + lambda_up = runner.lambda_scale_up + lambda_up_log_group = runner.lambda_scale_up_log_group + lambda_down = runner.lambda_scale_down + lambda_down_log_group = runner.lambda_scale_down_log_group + lambda_pool = runner.lambda_pool + lambda_pool_log_group = runner.lambda_pool_log_group + role_runner = runner.role_runner + role_scale_up = runner.role_scale_up + role_scale_down = runner.role_scale_down + role_pool = runner.role_pool + runners_log_groups = runner.runners_log_groups + runner_config_storage_backend = runner.runner_config_storage_backend + runner_config_dynamodb_table = runner.runner_config_dynamodb_table + logfiles = runner.logfiles } } } diff --git a/modules/multi-runner/runners.tf b/modules/multi-runner/runners.tf index 892113dcc7..d4ac77778a 100644 --- a/modules/multi-runner/runners.tf +++ b/modules/multi-runner/runners.tf @@ -17,6 +17,7 @@ module "runners" { tokens = "${var.ssm_paths.runners}/tokens" config = "${var.ssm_paths.runners}/config" } + runner_config_storage = each.value.runner_config.runner_config_storage runner_os = each.value.runner_config.runner_os instance_types = each.value.runner_config.instance_types @@ -122,7 +123,14 @@ module "runners" { pool_lambda_reserved_concurrent_executions = var.pool_lambda_reserved_concurrent_executions associate_public_ipv4_address = var.associate_public_ipv4_address - ssm_housekeeper = var.runners_ssm_housekeeper + ssm_housekeeper = { + create = var.runners_ssm_housekeeper.create + schedule_expression = var.runners_ssm_housekeeper.schedule_expression + state = var.runners_ssm_housekeeper.enabled ? "ENABLED" : "DISABLED" + lambda_memory_size = var.runners_ssm_housekeeper.lambda_memory_size + lambda_timeout = var.runners_ssm_housekeeper.lambda_timeout + config = var.runners_ssm_housekeeper.config + } job_retry = each.value.runner_config.job_retry diff --git a/modules/multi-runner/variables.tf b/modules/multi-runner/variables.tf index df6fb77473..8a0ed8f88f 100644 --- a/modules/multi-runner/variables.tf +++ b/modules/multi-runner/variables.tf @@ -90,10 +90,58 @@ variable "multi_runner_config" { "InsufficientInstanceCapacity", "InsufficientCapacityOnHost", ]) - enable_organization_runners = optional(bool, false) - enable_runner_binaries_syncer = optional(bool, true) - enable_ssm_on_runners = optional(bool, false) - enable_userdata = optional(bool, true) + enable_organization_runners = optional(bool, false) + enable_runner_binaries_syncer = optional(bool, true) + enable_ssm_on_runners = optional(bool, false) + enable_userdata = optional(bool, true) + runner_config_storage = optional(object({ + backend = optional(string, "ssm") + dynamodb = optional(object({ + table_name = optional(string, null) + partition_key_name = optional(string, "id") + value_attribute_name = optional(string, "value") + config_key_prefix = optional(string, "config#") + token_key_prefix = optional(string, null) + consistent_read = optional(bool, true) + token_overwrite_protection_enabled = optional(bool, true) + billing_mode = optional(string, "PAY_PER_REQUEST") + read_capacity = optional(number, 5) + write_capacity = optional(number, 5) + autoscaling_enabled = optional(bool, false) + autoscaling_read_min_capacity = optional(number, 5) + autoscaling_read_max_capacity = optional(number, 100) + autoscaling_read_target_value = optional(number, 70) + autoscaling_write_min_capacity = optional(number, 5) + autoscaling_write_max_capacity = optional(number, 100) + autoscaling_write_target_value = optional(number, 70) + token_ttl_seconds = optional(number, null) + ttl_enabled = optional(bool, true) + ttl_attribute_name = optional(string, "expires_at") + client_max_attempts = optional(number, 10) + client_retry_mode = optional(string, "adaptive") + client_http_keep_alive = optional(bool, true) + client_http_max_sockets = optional(number, 50) + client_http_keep_alive_msecs = optional(number, null) + server_side_encryption_enabled = optional(bool, true) + point_in_time_recovery_enabled = optional(bool, true) + deletion_protection_enabled = optional(bool, true) + kms_key_arn = optional(string, null) + alarms_enabled = optional(bool, true) + alarm_actions = optional(list(string), []) + ok_actions = optional(list(string), []) + insufficient_data_actions = optional(list(string), []) + alarm_evaluation_periods = optional(number, 1) + alarm_datapoints_to_alarm = optional(number, 1) + alarm_period = optional(number, 300) + alarm_treat_missing_data = optional(string, "notBreaching") + read_throttle_alarm_threshold = optional(number, 0) + write_throttle_alarm_threshold = optional(number, 0) + system_errors_alarm_threshold = optional(number, 0) + user_errors_alarm_threshold = optional(number, 0) + consumed_read_capacity_threshold = optional(number, null) + consumed_write_capacity_threshold = optional(number, null) + }), {}) + }), {}) instance_allocation_strategy = optional(string, "lowest-price") instance_type_priorities = optional(map(number), null) instance_max_spot_price = optional(string, null) @@ -238,6 +286,7 @@ variable "multi_runner_config" { enable_runner_binaries_syncer: "Option to disable the lambda to sync GitHub runner distribution, useful when using a pre-build AMI." enable_ssm_on_runners: "Enable to allow access the runner instances for debugging purposes via SSM. Note that this adds additional permissions to the runner instances." enable_userdata: "Should the userdata script be enabled for the runner. Set this to false if you are using your own prebuilt AMI." + runner_config_storage: "Storage backend for label-scoped runner bootstrap config and one-time registration/JIT config. Set backend to `dynamodb` to create one table for this runner label. When `token_ttl_seconds` is null, DynamoDB token expiry uses the SSM housekeeper minimum-days retention." instance_allocation_strategy: "The allocation strategy for creating instances. For spot, AWS recommends `price-capacity-optimized`; for on-demand, use `lowest-price` or `prioritized`. The AWS default is `lowest-price`." instance_type_priorities: "A map of instance type to priority for the `prioritized` and `capacity-optimized-prioritized` allocation strategies. Lower numbers mean higher priority. If not provided, priorities are assigned based on the order of `instance_types`." instance_max_spot_price: "Max price price for spot instances per hour. This variable will be passed to the create fleet as max spot price for the fleet." @@ -292,6 +341,81 @@ variable "multi_runner_config" { redrive_build_queue: "Set options to attach (optional) a dead letter queue to the build queue, the queue between the webhook and the scale up lambda. You have the following options. 1. Disable by setting `enabled` to false. 2. Enable by setting `enabled` to `true`, `maxReceiveCount` to a number of max retries." } EOT + + validation { + condition = alltrue([for _, config in var.multi_runner_config : contains(["ssm", "dynamodb"], lower(config.runner_config.runner_config_storage.backend))]) + error_message = "`runner_config.runner_config_storage.backend` must be either `ssm` or `dynamodb`." + } + + validation { + condition = alltrue([for _, config in var.multi_runner_config : contains(["PAY_PER_REQUEST", "PROVISIONED"], config.runner_config.runner_config_storage.dynamodb.billing_mode)]) + error_message = "`runner_config.runner_config_storage.dynamodb.billing_mode` must be either `PAY_PER_REQUEST` or `PROVISIONED`." + } + + validation { + condition = alltrue([for _, config in var.multi_runner_config : config.runner_config.runner_config_storage.dynamodb.autoscaling_enabled == false || config.runner_config.runner_config_storage.dynamodb.billing_mode == "PROVISIONED"]) + error_message = "`runner_config.runner_config_storage.dynamodb.autoscaling_enabled` can only be true when billing mode is `PROVISIONED`." + } + + validation { + condition = alltrue([for _, config in var.multi_runner_config : config.runner_config.runner_config_storage.dynamodb.token_ttl_seconds == null ? true : config.runner_config.runner_config_storage.dynamodb.token_ttl_seconds > 0]) + error_message = "`runner_config.runner_config_storage.dynamodb.token_ttl_seconds` must be null or greater than zero." + } + + validation { + condition = alltrue(flatten([ + for _, config in var.multi_runner_config : [ + for value in [ + config.runner_config.runner_config_storage.dynamodb.partition_key_name, + config.runner_config.runner_config_storage.dynamodb.value_attribute_name, + config.runner_config.runner_config_storage.dynamodb.config_key_prefix, + config.runner_config.runner_config_storage.dynamodb.ttl_attribute_name, + ] : trimspace(value) != "" + ] + ])) + error_message = "`runner_config.runner_config_storage.dynamodb` attribute names and `config_key_prefix` must not be empty." + } + + validation { + condition = alltrue([ + for _, config in var.multi_runner_config : length(distinct([ + config.runner_config.runner_config_storage.dynamodb.partition_key_name, + config.runner_config.runner_config_storage.dynamodb.value_attribute_name, + config.runner_config.runner_config_storage.dynamodb.ttl_attribute_name, + ])) == 3 + ]) + error_message = "`runner_config.runner_config_storage.dynamodb` attribute names must be distinct." + } + + validation { + condition = alltrue([for _, config in var.multi_runner_config : config.runner_config.runner_config_storage.dynamodb.token_key_prefix == null ? true : trimspace(config.runner_config.runner_config_storage.dynamodb.token_key_prefix) != ""]) + error_message = "`runner_config.runner_config_storage.dynamodb.token_key_prefix` must be null or non-empty." + } + + validation { + condition = alltrue([for _, config in var.multi_runner_config : config.runner_config.runner_config_storage.dynamodb.client_max_attempts > 0]) + error_message = "`runner_config.runner_config_storage.dynamodb.client_max_attempts` must be greater than zero." + } + + validation { + condition = alltrue([for _, config in var.multi_runner_config : contains(["standard", "adaptive"], config.runner_config.runner_config_storage.dynamodb.client_retry_mode)]) + error_message = "`runner_config.runner_config_storage.dynamodb.client_retry_mode` must be either `standard` or `adaptive`." + } + + validation { + condition = alltrue([for _, config in var.multi_runner_config : config.runner_config.runner_config_storage.dynamodb.client_http_max_sockets > 0]) + error_message = "`runner_config.runner_config_storage.dynamodb.client_http_max_sockets` must be greater than zero." + } + + validation { + condition = alltrue([for _, config in var.multi_runner_config : config.runner_config.runner_config_storage.dynamodb.client_http_keep_alive_msecs == null ? true : config.runner_config.runner_config_storage.dynamodb.client_http_keep_alive_msecs >= 0]) + error_message = "`runner_config.runner_config_storage.dynamodb.client_http_keep_alive_msecs` must be null or greater than or equal to zero." + } + + validation { + condition = alltrue([for _, config in var.multi_runner_config : contains(["breaching", "notBreaching", "ignore", "missing"], config.runner_config.runner_config_storage.dynamodb.alarm_treat_missing_data)]) + error_message = "`runner_config.runner_config_storage.dynamodb.alarm_treat_missing_data` must be one of `breaching`, `notBreaching`, `ignore`, or `missing`." + } } variable "scale_up_lambda_memory_size" { @@ -704,12 +828,14 @@ variable "runners_ssm_housekeeper" { Configuration for the SSM housekeeper lambda. This lambda deletes token / JIT config from SSM. `schedule_expression`: is used to configure the schedule for the lambda. - `enabled`: enable or disable the lambda trigger via the EventBridge. + `create`: create or remove the SSM housekeeper lambda resources. + `enabled`: enable or disable only the lambda trigger via the EventBridge. `lambda_memory_size`: lambda memory size limit. `lambda_timeout`: timeout for the lambda in seconds. `config`: configuration for the lambda function. Token path will be read by default from the module. EOF type = object({ + create = optional(bool, true) schedule_expression = optional(string, "rate(1 day)") enabled = optional(bool, true) lambda_memory_size = optional(number, 512) diff --git a/modules/runners/dynamodb.tf b/modules/runners/dynamodb.tf new file mode 100644 index 0000000000..f542880988 --- /dev/null +++ b/modules/runners/dynamodb.tf @@ -0,0 +1,208 @@ +locals { + runner_config_dynamodb_items_base = { + run_as = var.runner_as_root ? "root" : var.runner_run_as + agent_mode = var.enable_ephemeral_runners ? "ephemeral" : "persistent" + disable_default_labels = tostring(var.runner_disable_default_labels) + enable_jit_config = tostring(var.enable_jit_config == null ? var.enable_ephemeral_runners : var.enable_jit_config) + enable_cloudwatch = tostring(var.enable_cloudwatch_agent) + } + + runner_config_dynamodb_items = merge( + local.runner_config_dynamodb_items_base, + var.enable_cloudwatch_agent ? { + cloudwatch_agent_config_runner = local.cloudwatch_agent_config_runner + } : {} + ) + + runner_config_dynamodb_alarm_dimensions = { + TableName = local.runner_config_dynamodb_table_name + } + + runner_config_dynamodb_base_alarms = { + read_throttle_events = { + alarm_name = "${var.prefix}-runner-config-dynamodb-read-throttles" + alarm_description = "DynamoDB read throttles on the runner config table." + metric_name = "ReadThrottleEvents" + statistic = "Sum" + comparison_operator = "GreaterThanThreshold" + threshold = var.runner_config_storage.dynamodb.read_throttle_alarm_threshold + } + write_throttle_events = { + alarm_name = "${var.prefix}-runner-config-dynamodb-write-throttles" + alarm_description = "DynamoDB write throttles on the runner config table." + metric_name = "WriteThrottleEvents" + statistic = "Sum" + comparison_operator = "GreaterThanThreshold" + threshold = var.runner_config_storage.dynamodb.write_throttle_alarm_threshold + } + system_errors = { + alarm_name = "${var.prefix}-runner-config-dynamodb-system-errors" + alarm_description = "DynamoDB system errors on the runner config table." + metric_name = "SystemErrors" + statistic = "Sum" + comparison_operator = "GreaterThanThreshold" + threshold = var.runner_config_storage.dynamodb.system_errors_alarm_threshold + } + user_errors = { + alarm_name = "${var.prefix}-runner-config-dynamodb-user-errors" + alarm_description = "DynamoDB user errors on the runner config table." + metric_name = "UserErrors" + statistic = "Sum" + comparison_operator = "GreaterThanThreshold" + threshold = var.runner_config_storage.dynamodb.user_errors_alarm_threshold + } + } + + runner_config_dynamodb_capacity_alarms = merge( + var.runner_config_storage.dynamodb.consumed_read_capacity_threshold == null ? {} : { + consumed_read_capacity = { + alarm_name = "${var.prefix}-runner-config-dynamodb-consumed-read-capacity" + alarm_description = "DynamoDB consumed read capacity on the runner config table." + metric_name = "ConsumedReadCapacityUnits" + statistic = "Sum" + comparison_operator = "GreaterThanThreshold" + threshold = var.runner_config_storage.dynamodb.consumed_read_capacity_threshold + } + }, + var.runner_config_storage.dynamodb.consumed_write_capacity_threshold == null ? {} : { + consumed_write_capacity = { + alarm_name = "${var.prefix}-runner-config-dynamodb-consumed-write-capacity" + alarm_description = "DynamoDB consumed write capacity on the runner config table." + metric_name = "ConsumedWriteCapacityUnits" + statistic = "Sum" + comparison_operator = "GreaterThanThreshold" + threshold = var.runner_config_storage.dynamodb.consumed_write_capacity_threshold + } + } + ) + + runner_config_dynamodb_alarms = local.runner_config_storage_dynamodb && var.runner_config_storage.dynamodb.alarms_enabled ? merge( + local.runner_config_dynamodb_base_alarms, + local.runner_config_dynamodb_capacity_alarms + ) : {} +} + +resource "aws_dynamodb_table" "runner_config" { + count = local.runner_config_storage_dynamodb ? 1 : 0 + + name = coalesce(var.runner_config_storage.dynamodb.table_name, "${var.prefix}-runner-config") + billing_mode = var.runner_config_storage.dynamodb.billing_mode + hash_key = local.runner_config_dynamodb_partition_key_name + read_capacity = var.runner_config_storage.dynamodb.billing_mode == "PROVISIONED" ? var.runner_config_storage.dynamodb.read_capacity : null + write_capacity = var.runner_config_storage.dynamodb.billing_mode == "PROVISIONED" ? var.runner_config_storage.dynamodb.write_capacity : null + deletion_protection_enabled = var.runner_config_storage.dynamodb.deletion_protection_enabled + + attribute { + name = local.runner_config_dynamodb_partition_key_name + type = "S" + } + + ttl { + enabled = var.runner_config_storage.dynamodb.ttl_enabled + attribute_name = var.runner_config_storage.dynamodb.ttl_attribute_name + } + + point_in_time_recovery { + enabled = var.runner_config_storage.dynamodb.point_in_time_recovery_enabled + } + + server_side_encryption { + enabled = var.runner_config_storage.dynamodb.server_side_encryption_enabled + kms_key_arn = var.runner_config_storage.dynamodb.server_side_encryption_enabled ? var.runner_config_storage.dynamodb.kms_key_arn : null + } + + tags = local.tags +} + +resource "aws_dynamodb_table_item" "runner_config" { + for_each = local.runner_config_storage_dynamodb ? local.runner_config_dynamodb_items : {} + + table_name = aws_dynamodb_table.runner_config[0].name + hash_key = aws_dynamodb_table.runner_config[0].hash_key + + item = jsonencode({ + (local.runner_config_dynamodb_partition_key_name) = { + S = "${local.runner_config_dynamodb_config_key_prefix}${each.key}" + } + (local.runner_config_dynamodb_value_attribute_name) = { + S = each.value + } + }) +} + +resource "aws_appautoscaling_target" "runner_config_dynamodb_read" { + count = local.runner_config_storage_dynamodb && var.runner_config_storage.dynamodb.billing_mode == "PROVISIONED" && var.runner_config_storage.dynamodb.autoscaling_enabled ? 1 : 0 + + max_capacity = var.runner_config_storage.dynamodb.autoscaling_read_max_capacity + min_capacity = var.runner_config_storage.dynamodb.autoscaling_read_min_capacity + resource_id = "table/${aws_dynamodb_table.runner_config[0].name}" + scalable_dimension = "dynamodb:table:ReadCapacityUnits" + service_namespace = "dynamodb" +} + +resource "aws_appautoscaling_policy" "runner_config_dynamodb_read" { + count = local.runner_config_storage_dynamodb && var.runner_config_storage.dynamodb.billing_mode == "PROVISIONED" && var.runner_config_storage.dynamodb.autoscaling_enabled ? 1 : 0 + + name = "${var.prefix}-runner-config-dynamodb-read" + policy_type = "TargetTrackingScaling" + resource_id = aws_appautoscaling_target.runner_config_dynamodb_read[0].resource_id + scalable_dimension = aws_appautoscaling_target.runner_config_dynamodb_read[0].scalable_dimension + service_namespace = aws_appautoscaling_target.runner_config_dynamodb_read[0].service_namespace + + target_tracking_scaling_policy_configuration { + predefined_metric_specification { + predefined_metric_type = "DynamoDBReadCapacityUtilization" + } + + target_value = var.runner_config_storage.dynamodb.autoscaling_read_target_value + } +} + +resource "aws_appautoscaling_target" "runner_config_dynamodb_write" { + count = local.runner_config_storage_dynamodb && var.runner_config_storage.dynamodb.billing_mode == "PROVISIONED" && var.runner_config_storage.dynamodb.autoscaling_enabled ? 1 : 0 + + max_capacity = var.runner_config_storage.dynamodb.autoscaling_write_max_capacity + min_capacity = var.runner_config_storage.dynamodb.autoscaling_write_min_capacity + resource_id = "table/${aws_dynamodb_table.runner_config[0].name}" + scalable_dimension = "dynamodb:table:WriteCapacityUnits" + service_namespace = "dynamodb" +} + +resource "aws_appautoscaling_policy" "runner_config_dynamodb_write" { + count = local.runner_config_storage_dynamodb && var.runner_config_storage.dynamodb.billing_mode == "PROVISIONED" && var.runner_config_storage.dynamodb.autoscaling_enabled ? 1 : 0 + + name = "${var.prefix}-runner-config-dynamodb-write" + policy_type = "TargetTrackingScaling" + resource_id = aws_appautoscaling_target.runner_config_dynamodb_write[0].resource_id + scalable_dimension = aws_appautoscaling_target.runner_config_dynamodb_write[0].scalable_dimension + service_namespace = aws_appautoscaling_target.runner_config_dynamodb_write[0].service_namespace + + target_tracking_scaling_policy_configuration { + predefined_metric_specification { + predefined_metric_type = "DynamoDBWriteCapacityUtilization" + } + + target_value = var.runner_config_storage.dynamodb.autoscaling_write_target_value + } +} + +resource "aws_cloudwatch_metric_alarm" "runner_config_dynamodb" { + for_each = local.runner_config_dynamodb_alarms + + alarm_name = each.value.alarm_name + alarm_description = each.value.alarm_description + namespace = "AWS/DynamoDB" + metric_name = each.value.metric_name + dimensions = local.runner_config_dynamodb_alarm_dimensions + statistic = each.value.statistic + comparison_operator = each.value.comparison_operator + threshold = each.value.threshold + period = var.runner_config_storage.dynamodb.alarm_period + evaluation_periods = var.runner_config_storage.dynamodb.alarm_evaluation_periods + datapoints_to_alarm = var.runner_config_storage.dynamodb.alarm_datapoints_to_alarm + treat_missing_data = var.runner_config_storage.dynamodb.alarm_treat_missing_data + alarm_actions = var.runner_config_storage.dynamodb.alarm_actions + ok_actions = var.runner_config_storage.dynamodb.ok_actions + insufficient_data_actions = var.runner_config_storage.dynamodb.insufficient_data_actions + tags = local.tags +} diff --git a/modules/runners/local.tf b/modules/runners/local.tf index 7cdc3a0a0c..8dd0fd4867 100644 --- a/modules/runners/local.tf +++ b/modules/runners/local.tf @@ -5,4 +5,33 @@ locals { Value = value } ]) + + runner_config_storage_backend = lower(var.runner_config_storage.backend) + runner_config_storage_ssm = local.runner_config_storage_backend == "ssm" + runner_config_storage_dynamodb = ( + local.runner_config_storage_backend == "dynamodb" + ) + runner_config_dynamodb_table_name = ( + local.runner_config_storage_dynamodb + ? aws_dynamodb_table.runner_config[0].name + : "" + ) + runner_config_dynamodb_partition_key_name = var.runner_config_storage.dynamodb.partition_key_name + runner_config_dynamodb_value_attribute_name = var.runner_config_storage.dynamodb.value_attribute_name + runner_config_dynamodb_config_key_prefix = var.runner_config_storage.dynamodb.config_key_prefix + runner_config_dynamodb_consistent_read = var.runner_config_storage.dynamodb.consistent_read + runner_config_dynamodb_token_overwrite_protection_enabled = var.runner_config_storage.dynamodb.token_overwrite_protection_enabled + runner_config_dynamodb_token_key_prefix = coalesce( + var.runner_config_storage.dynamodb.token_key_prefix, + "arn:${var.aws_partition}:ec2:${var.aws_region}:${data.aws_caller_identity.current.account_id}:instance/", + ) + runner_config_dynamodb_token_leading_keys = ( + var.runner_config_storage.dynamodb.token_key_prefix == null + ? ["$${ec2:SourceInstanceARN}"] + : ["${var.runner_config_storage.dynamodb.token_key_prefix}*"] + ) + runner_config_dynamodb_ttl_seconds = coalesce( + var.runner_config_storage.dynamodb.token_ttl_seconds, + local.ssm_housekeeper.config.minimumDaysOld * 86400, + ) } diff --git a/modules/runners/logging.tf b/modules/runners/logging.tf index 71fd3f274e..aec5bba58d 100644 --- a/modules/runners/logging.tf +++ b/modules/runners/logging.tf @@ -51,17 +51,18 @@ locals { ][0] ] + cloudwatch_agent_config_runner = var.cloudwatch_config != null ? var.cloudwatch_config : templatefile("${path.module}/templates/cloudwatch_config.json", { + logfiles = jsonencode(local.logfiles) + }) } resource "aws_ssm_parameter" "cloudwatch_agent_config_runner" { - count = var.enable_cloudwatch_agent ? 1 : 0 + count = var.enable_cloudwatch_agent && local.runner_config_storage_ssm ? 1 : 0 name = "${var.ssm_paths.root}/${var.ssm_paths.config}/cloudwatch_agent_config_runner" type = "String" - value = var.cloudwatch_config != null ? var.cloudwatch_config : templatefile("${path.module}/templates/cloudwatch_config.json", { - logfiles = jsonencode(local.logfiles) - }) - tags = local.tags + value = local.cloudwatch_agent_config_runner + tags = local.tags } resource "aws_cloudwatch_log_group" "gh_runners" { @@ -79,7 +80,7 @@ resource "aws_iam_role_policy" "cloudwatch" { role = aws_iam_role.runner[0].name policy = templatefile("${path.module}/policies/instance-cloudwatch-policy.json", { - ssm_parameter_arn = aws_ssm_parameter.cloudwatch_agent_config_runner[0].arn + ssm_parameter_arn = local.runner_config_storage_ssm ? aws_ssm_parameter.cloudwatch_agent_config_runner[0].arn : "" } ) } diff --git a/modules/runners/main.tf b/modules/runners/main.tf index 547f320bda..b3b5a24da6 100644 --- a/modules/runners/main.tf +++ b/modules/runners/main.tf @@ -72,7 +72,14 @@ locals { hook_job_started = var.runner_hook_job_started hook_job_completed = var.runner_hook_job_completed start_runner = templatefile(local.userdata_start_runner[var.runner_os], { - metadata_tags = var.metadata_options != null ? var.metadata_options.instance_metadata_tags : "enabled" + metadata_tags = var.metadata_options != null ? var.metadata_options.instance_metadata_tags : "enabled" + runner_config_storage_backend = local.runner_config_storage_backend + runner_config_dynamodb_table_name = local.runner_config_dynamodb_table_name + runner_config_dynamodb_partition_key_name = local.runner_config_dynamodb_partition_key_name + runner_config_dynamodb_value_attribute_name = local.runner_config_dynamodb_value_attribute_name + runner_config_dynamodb_config_key_prefix = local.runner_config_dynamodb_config_key_prefix + runner_config_dynamodb_consistent_read = local.runner_config_dynamodb_consistent_read + runner_config_dynamodb_token_key_prefix = local.runner_config_dynamodb_token_key_prefix }) ghes_url = var.ghes_url ghes_ssl_verify = var.ghes_ssl_verify @@ -80,7 +87,7 @@ locals { ## retain these for backwards compatibility environment = var.prefix enable_cloudwatch_agent = var.enable_cloudwatch_agent - ssm_key_cloudwatch_agent_config = var.enable_cloudwatch_agent ? aws_ssm_parameter.cloudwatch_agent_config_runner[0].name : "" + ssm_key_cloudwatch_agent_config = var.enable_cloudwatch_agent && local.runner_config_storage_ssm ? aws_ssm_parameter.cloudwatch_agent_config_runner[0].name : "" }) : var.userdata_content) : "" encoded_user_data = ( diff --git a/modules/runners/outputs.tf b/modules/runners/outputs.tf index 8f366dce90..90d246681b 100644 --- a/modules/runners/outputs.tf +++ b/modules/runners/outputs.tf @@ -42,6 +42,16 @@ output "role_pool" { value = try(module.pool[0].role_pool, null) } +output "runner_config_storage_backend" { + description = "Storage backend used for label-scoped runner bootstrap config and one-time registration/JIT config." + value = local.runner_config_storage_backend +} + +output "runner_config_dynamodb_table" { + description = "DynamoDB table used for label-scoped runner bootstrap config and one-time registration/JIT config when the DynamoDB backend is enabled." + value = try(aws_dynamodb_table.runner_config[0], null) +} + output "runners_log_groups" { description = "List of log groups from different log files of runner machine." value = try(aws_cloudwatch_log_group.gh_runners, []) diff --git a/modules/runners/policies-runner.tf b/modules/runners/policies-runner.tf index 0c08487a0f..2cc2f74998 100644 --- a/modules/runners/policies-runner.tf +++ b/modules/runners/policies-runner.tf @@ -25,7 +25,7 @@ resource "aws_iam_role_policy" "runner_session_manager_aws_managed" { } resource "aws_iam_role_policy" "ssm_parameters" { - count = (var.iam_overrides["override_runner_role"] || var.iam_overrides["override_instance_profile"]) ? 0 : 1 + count = (var.iam_overrides["override_runner_role"] || var.iam_overrides["override_instance_profile"]) ? 0 : (local.runner_config_storage_ssm ? 1 : 0) name = "runner-ssm-parameters" role = aws_iam_role.runner[0].name policy = templatefile("${path.module}/policies/instance-ssm-parameters-policy.json", @@ -36,6 +36,20 @@ resource "aws_iam_role_policy" "ssm_parameters" { ) } +resource "aws_iam_role_policy" "dynamodb_runner_config" { + count = (var.iam_overrides["override_runner_role"] || var.iam_overrides["override_instance_profile"]) ? 0 : (local.runner_config_storage_dynamodb ? 1 : 0) + name = "runner-dynamodb-config" + role = aws_iam_role.runner[0].name + policy = templatefile("${path.module}/policies/instance-dynamodb-runner-config-policy.json", + { + config_key_prefix = local.runner_config_dynamodb_config_key_prefix + table_arn = aws_dynamodb_table.runner_config[0].arn + token_leading_keys_json = jsonencode(local.runner_config_dynamodb_token_leading_keys) + kms_key_arn = var.runner_config_storage.dynamodb.kms_key_arn != null ? var.runner_config_storage.dynamodb.kms_key_arn : "" + } + ) +} + resource "aws_iam_role_policy" "dist_bucket" { count = (var.iam_overrides["override_runner_role"] || var.iam_overrides["override_instance_profile"]) ? 0 : (var.enable_runner_binaries_syncer ? 1 : 0) diff --git a/modules/runners/policies/instance-cloudwatch-policy.json b/modules/runners/policies/instance-cloudwatch-policy.json index 308dcaad36..6ac1b0e4f5 100644 --- a/modules/runners/policies/instance-cloudwatch-policy.json +++ b/modules/runners/policies/instance-cloudwatch-policy.json @@ -13,13 +13,14 @@ "logs:CreateLogStream" ], "Resource": "*" - }, + }%{ if ssm_parameter_arn != "" ~}, { "Effect": "Allow", "Action": [ "ssm:GetParameter" ], - "Resource": "${ssm_parameter_arn}/*" + "Resource": "${ssm_parameter_arn}" } +%{ endif ~} ] -} \ No newline at end of file +} diff --git a/modules/runners/policies/instance-dynamodb-runner-config-policy.json b/modules/runners/policies/instance-dynamodb-runner-config-policy.json new file mode 100644 index 0000000000..03c2e2c959 --- /dev/null +++ b/modules/runners/policies/instance-dynamodb-runner-config-policy.json @@ -0,0 +1,43 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "dynamodb:GetItem" + ], + "Resource": "${table_arn}", + "Condition": { + "ForAllValues:StringLike": { + "dynamodb:LeadingKeys": [ + "${config_key_prefix}*" + ] + } + } + }, + { + "Effect": "Allow", + "Action": [ + "dynamodb:DeleteItem", + "dynamodb:GetItem" + ], + "Resource": "${table_arn}", + "Condition": { + "ForAllValues:StringLike": { + "dynamodb:LeadingKeys": ${token_leading_keys_json} + } + } + }%{ if kms_key_arn != "" ~}, + { + "Effect": "Allow", + "Action": [ + "kms:Decrypt", + "kms:DescribeKey", + "kms:Encrypt", + "kms:GenerateDataKey" + ], + "Resource": "${kms_key_arn}" + } +%{ endif ~} + ] +} diff --git a/modules/runners/policies/lambda-dynamodb-runner-config.json b/modules/runners/policies/lambda-dynamodb-runner-config.json new file mode 100644 index 0000000000..6132c428aa --- /dev/null +++ b/modules/runners/policies/lambda-dynamodb-runner-config.json @@ -0,0 +1,36 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "dynamodb:DescribeTable" + ], + "Resource": "${table_arn}" + }, + { + "Effect": "Allow", + "Action": [ + "dynamodb:GetItem", + "dynamodb:PutItem" + ], + "Resource": "${table_arn}", + "Condition": { + "ForAllValues:StringLike": { + "dynamodb:LeadingKeys": ${leading_keys_json} + } + } + }%{ if kms_key_arn != "" ~}, + { + "Effect": "Allow", + "Action": [ + "kms:Decrypt", + "kms:DescribeKey", + "kms:Encrypt", + "kms:GenerateDataKey" + ], + "Resource": "${kms_key_arn}" + } +%{ endif ~} + ] +} diff --git a/modules/runners/policies/lambda-scale-up.json b/modules/runners/policies/lambda-scale-up.json index 4a38d23f6a..d820763940 100644 --- a/modules/runners/policies/lambda-scale-up.json +++ b/modules/runners/policies/lambda-scale-up.json @@ -48,6 +48,7 @@ "Action": "iam:PassRole", "Resource": "${arn_runner_instance_role}" }, +%{ if runner_config_storage_backend == "ssm" ~} { "Effect": "Allow", "Action": [ @@ -56,6 +57,7 @@ ], "Resource": "*" }, +%{ endif ~} { "Effect": "Allow", "Action": [ @@ -65,19 +67,21 @@ "Resource": [ "${github_app_key_base64_arn}", "${github_app_id_arn}", - "${ssm_config_path}/*", "${ssm_ami_id_parameter_arn}" ] }, +%{ if runner_config_storage_backend == "ssm" ~} { "Effect": "Allow", "Action": [ - "ssm:GetParameters" + "ssm:GetParameter", + "ssm:GetParameters" ], "Resource": [ - "${ssm_ami_id_parameter_arn}" + "${ssm_config_path}/*" ] }, +%{ endif ~} { "Effect": "Allow", "Action": [ diff --git a/modules/runners/pool.tf b/modules/runners/pool.tf index 11840a4638..e5601682b1 100644 --- a/modules/runners/pool.tf +++ b/modules/runners/pool.tf @@ -57,14 +57,31 @@ module "pool" { role = { arn = var.iam_overrides["override_runner_role"] ? var.iam_overrides["runner_role_arn"] : aws_iam_role.runner[0].arn } use_dedicated_host = var.use_dedicated_host } - subnet_ids = var.subnet_ids - ssm_token_path = "${var.ssm_paths.root}/${var.ssm_paths.tokens}" - ssm_config_path = "${var.ssm_paths.root}/${var.ssm_paths.config}" - ami_id_ssm_parameter_name = local.ami_id_ssm_parameter_name - ami_id_ssm_parameter_read_policy_arn = local.ami_id_ssm_parameter_name != null ? aws_iam_policy.ami_id_ssm_parameter_read[0].arn : null - tags = local.tags - lambda_tags = var.lambda_tags - arn_ssm_parameters_path_config = local.arn_ssm_parameters_path_config + subnet_ids = var.subnet_ids + ssm_token_path = "${var.ssm_paths.root}/${var.ssm_paths.tokens}" + ssm_config_path = "${var.ssm_paths.root}/${var.ssm_paths.config}" + runner_config_storage_backend = local.runner_config_storage_backend + runner_config_dynamodb_table_name = local.runner_config_dynamodb_table_name + runner_config_dynamodb_table_arn = local.runner_config_storage_dynamodb ? aws_dynamodb_table.runner_config[0].arn : "" + runner_config_dynamodb_partition_key_name = local.runner_config_dynamodb_partition_key_name + runner_config_dynamodb_value_attribute_name = local.runner_config_dynamodb_value_attribute_name + runner_config_dynamodb_config_key_prefix = local.runner_config_dynamodb_config_key_prefix + runner_config_dynamodb_consistent_read = local.runner_config_dynamodb_consistent_read + runner_config_dynamodb_token_overwrite_protection_enabled = local.runner_config_dynamodb_token_overwrite_protection_enabled + runner_config_dynamodb_token_key_prefix = local.runner_config_dynamodb_token_key_prefix + runner_config_dynamodb_ttl_seconds = local.runner_config_dynamodb_ttl_seconds + runner_config_dynamodb_ttl_attribute_name = var.runner_config_storage.dynamodb.ttl_attribute_name + runner_config_dynamodb_client_max_attempts = var.runner_config_storage.dynamodb.client_max_attempts + runner_config_dynamodb_client_retry_mode = var.runner_config_storage.dynamodb.client_retry_mode + runner_config_dynamodb_client_http_keep_alive = var.runner_config_storage.dynamodb.client_http_keep_alive + runner_config_dynamodb_client_http_max_sockets = var.runner_config_storage.dynamodb.client_http_max_sockets + runner_config_dynamodb_client_http_keep_alive_msecs = var.runner_config_storage.dynamodb.client_http_keep_alive_msecs == null ? "" : var.runner_config_storage.dynamodb.client_http_keep_alive_msecs + runner_config_dynamodb_kms_key_arn = var.runner_config_storage.dynamodb.kms_key_arn != null ? var.runner_config_storage.dynamodb.kms_key_arn : "" + ami_id_ssm_parameter_name = local.ami_id_ssm_parameter_name + ami_id_ssm_parameter_read_policy_arn = local.ami_id_ssm_parameter_name != null ? aws_iam_policy.ami_id_ssm_parameter_read[0].arn : null + tags = local.tags + lambda_tags = var.lambda_tags + arn_ssm_parameters_path_config = local.arn_ssm_parameters_path_config } aws_partition = var.aws_partition diff --git a/modules/runners/pool/main.tf b/modules/runners/pool/main.tf index e4f2485ccd..34e4359f15 100644 --- a/modules/runners/pool/main.tf +++ b/modules/runners/pool/main.tf @@ -25,42 +25,57 @@ resource "aws_lambda_function" "pool" { environment { variables = { - AMI_ID_SSM_PARAMETER_NAME = var.config.ami_id_ssm_parameter_name - DISABLE_RUNNER_AUTOUPDATE = var.config.runner.disable_runner_autoupdate - ENABLE_EPHEMERAL_RUNNERS = var.config.runner.ephemeral - ENABLE_JIT_CONFIG = var.config.runner.enable_jit_config - ENVIRONMENT = var.config.prefix - GHES_URL = var.config.ghes.url - USER_AGENT = var.config.user_agent - INSTANCE_ALLOCATION_STRATEGY = var.config.instance_allocation_strategy - INSTANCE_MAX_SPOT_PRICE = var.config.instance_max_spot_price - INSTANCE_TARGET_CAPACITY_TYPE = var.config.instance_target_capacity_type - INSTANCE_TYPE_PRIORITIES = var.config.instance_type_priorities != null ? jsonencode(var.config.instance_type_priorities) : "" - INSTANCE_TYPES = join(",", var.config.instance_types) - LAUNCH_TEMPLATE_NAME = var.config.runner.launch_template.name - LOG_LEVEL = upper(var.config.lambda.log_level) - NODE_TLS_REJECT_UNAUTHORIZED = var.config.ghes.url != null && !var.config.ghes.ssl_verify ? 0 : 1 - PARAMETER_GITHUB_APP_ID_NAME = var.config.github_app_parameters.id.name - PARAMETER_GITHUB_APP_KEY_BASE64_NAME = var.config.github_app_parameters.key_base64.name - POWERTOOLS_LOGGER_LOG_EVENT = var.config.lambda.log_level == "debug" ? "true" : "false" - RUNNER_BOOT_TIME_IN_MINUTES = var.config.runner.boot_time_in_minutes - RUNNER_LABELS = lower(join(",", var.config.runner.labels)) - RUNNER_GROUP_NAME = var.config.runner.group_name - RUNNER_NAME_PREFIX = var.config.runner.name_prefix - RUNNER_OWNER = var.config.runner.pool_owner - RUNNERS_MAXIMUM_COUNT = var.config.runners_maximum_count - SSM_TOKEN_PATH = var.config.ssm_token_path - SSM_CONFIG_PATH = var.config.ssm_config_path - SUBNET_IDS = join(",", var.config.subnet_ids) - POWERTOOLS_SERVICE_NAME = "${var.config.prefix}-pool" - POWERTOOLS_TRACE_ENABLED = var.tracing_config.mode != null ? true : false - POWERTOOLS_TRACER_CAPTURE_HTTPS_REQUESTS = var.tracing_config.capture_http_requests - POWERTOOLS_TRACER_CAPTURE_ERROR = var.tracing_config.capture_error - ENABLE_ON_DEMAND_FAILOVER_FOR_ERRORS = jsonencode(var.config.runner.enable_on_demand_failover_for_errors) - SSM_PARAMETER_STORE_TAGS = var.config.lambda.parameter_store_tags - SCALE_ERRORS = jsonencode(var.config.runner.scale_errors) - USE_DEDICATED_HOST = var.config.runner.use_dedicated_host - INCLUDE_BUSY_RUNNERS = var.config.include_busy_runners + AMI_ID_SSM_PARAMETER_NAME = var.config.ami_id_ssm_parameter_name + DISABLE_RUNNER_AUTOUPDATE = var.config.runner.disable_runner_autoupdate + ENABLE_EPHEMERAL_RUNNERS = var.config.runner.ephemeral + ENABLE_JIT_CONFIG = var.config.runner.enable_jit_config + ENVIRONMENT = var.config.prefix + GHES_URL = var.config.ghes.url + USER_AGENT = var.config.user_agent + INSTANCE_ALLOCATION_STRATEGY = var.config.instance_allocation_strategy + INSTANCE_MAX_SPOT_PRICE = var.config.instance_max_spot_price + INSTANCE_TARGET_CAPACITY_TYPE = var.config.instance_target_capacity_type + INSTANCE_TYPE_PRIORITIES = var.config.instance_type_priorities != null ? jsonencode(var.config.instance_type_priorities) : "" + INSTANCE_TYPES = join(",", var.config.instance_types) + LAUNCH_TEMPLATE_NAME = var.config.runner.launch_template.name + LOG_LEVEL = upper(var.config.lambda.log_level) + NODE_TLS_REJECT_UNAUTHORIZED = var.config.ghes.url != null && !var.config.ghes.ssl_verify ? 0 : 1 + PARAMETER_GITHUB_APP_ID_NAME = var.config.github_app_parameters.id.name + PARAMETER_GITHUB_APP_KEY_BASE64_NAME = var.config.github_app_parameters.key_base64.name + POWERTOOLS_LOGGER_LOG_EVENT = var.config.lambda.log_level == "debug" ? "true" : "false" + RUNNER_BOOT_TIME_IN_MINUTES = var.config.runner.boot_time_in_minutes + RUNNER_LABELS = lower(join(",", var.config.runner.labels)) + RUNNER_GROUP_NAME = var.config.runner.group_name + RUNNER_NAME_PREFIX = var.config.runner.name_prefix + RUNNER_OWNER = var.config.runner.pool_owner + RUNNER_CONFIG_STORAGE_BACKEND = var.config.runner_config_storage_backend + RUNNER_CONFIG_DYNAMODB_TABLE_NAME = var.config.runner_config_dynamodb_table_name + RUNNER_CONFIG_DYNAMODB_PARTITION_KEY_NAME = var.config.runner_config_dynamodb_partition_key_name + RUNNER_CONFIG_DYNAMODB_VALUE_ATTRIBUTE_NAME = var.config.runner_config_dynamodb_value_attribute_name + RUNNER_CONFIG_DYNAMODB_CONFIG_KEY_PREFIX = var.config.runner_config_dynamodb_config_key_prefix + RUNNER_CONFIG_DYNAMODB_CONSISTENT_READ = var.config.runner_config_dynamodb_consistent_read + RUNNER_CONFIG_DYNAMODB_TOKEN_OVERWRITE_PROTECTION_ENABLED = var.config.runner_config_dynamodb_token_overwrite_protection_enabled + RUNNER_CONFIG_DYNAMODB_TOKEN_KEY_PREFIX = var.config.runner_config_dynamodb_token_key_prefix + RUNNER_CONFIG_DYNAMODB_TTL_SECONDS = var.config.runner_config_dynamodb_ttl_seconds + RUNNER_CONFIG_DYNAMODB_TTL_ATTRIBUTE_NAME = var.config.runner_config_dynamodb_ttl_attribute_name + RUNNER_CONFIG_DYNAMODB_CLIENT_MAX_ATTEMPTS = var.config.runner_config_dynamodb_client_max_attempts + RUNNER_CONFIG_DYNAMODB_CLIENT_RETRY_MODE = var.config.runner_config_dynamodb_client_retry_mode + RUNNER_CONFIG_DYNAMODB_CLIENT_HTTP_KEEP_ALIVE = var.config.runner_config_dynamodb_client_http_keep_alive + RUNNER_CONFIG_DYNAMODB_CLIENT_HTTP_MAX_SOCKETS = var.config.runner_config_dynamodb_client_http_max_sockets + RUNNER_CONFIG_DYNAMODB_CLIENT_HTTP_KEEP_ALIVE_MSECS = var.config.runner_config_dynamodb_client_http_keep_alive_msecs + RUNNERS_MAXIMUM_COUNT = var.config.runners_maximum_count + SSM_TOKEN_PATH = var.config.ssm_token_path + SSM_CONFIG_PATH = var.config.ssm_config_path + SUBNET_IDS = join(",", var.config.subnet_ids) + POWERTOOLS_SERVICE_NAME = "${var.config.prefix}-pool" + POWERTOOLS_TRACE_ENABLED = var.tracing_config.mode != null ? true : false + POWERTOOLS_TRACER_CAPTURE_HTTPS_REQUESTS = var.tracing_config.capture_http_requests + POWERTOOLS_TRACER_CAPTURE_ERROR = var.tracing_config.capture_error + ENABLE_ON_DEMAND_FAILOVER_FOR_ERRORS = jsonencode(var.config.runner.enable_on_demand_failover_for_errors) + SSM_PARAMETER_STORE_TAGS = var.config.lambda.parameter_store_tags + SCALE_ERRORS = jsonencode(var.config.runner.scale_errors) + USE_DEDICATED_HOST = var.config.runner.use_dedicated_host + INCLUDE_BUSY_RUNNERS = var.config.include_busy_runners } } @@ -107,6 +122,21 @@ resource "aws_iam_role_policy" "pool" { kms_key_arn = var.config.kms_key_arn ami_kms_key_arn = var.config.ami_kms_key_arn ssm_ami_id_parameter_arn = var.config.ami_id_ssm_parameter_arn + runner_config_storage_backend = var.config.runner_config_storage_backend + }) +} + +resource "aws_iam_role_policy" "pool_runner_config_dynamodb" { + count = var.config.runner_config_storage_backend == "dynamodb" ? 1 : 0 + name = "dynamodb-runner-config-policy" + role = aws_iam_role.pool.name + policy = templatefile("${path.module}/../policies/lambda-dynamodb-runner-config.json", { + table_arn = var.config.runner_config_dynamodb_table_arn + leading_keys_json = jsonencode([ + "${var.config.runner_config_dynamodb_config_key_prefix}*", + "${var.config.runner_config_dynamodb_token_key_prefix}*", + ]) + kms_key_arn = var.config.runner_config_dynamodb_kms_key_arn }) } diff --git a/modules/runners/pool/policies/lambda-pool.json b/modules/runners/pool/policies/lambda-pool.json index 91c9997ce4..31a70863f7 100644 --- a/modules/runners/pool/policies/lambda-pool.json +++ b/modules/runners/pool/policies/lambda-pool.json @@ -19,6 +19,7 @@ "Action": "iam:PassRole", "Resource": "${arn_runner_instance_role}" }, +%{ if runner_config_storage_backend == "ssm" ~} { "Effect": "Allow", "Action": [ @@ -28,17 +29,18 @@ "Resource": "*" }, { - "Effect": "Allow", - "Action": [ - "ssm:GetParameter", - "ssm:GetParameters", - "ssm:GetParametersByPath" - ], - "Resource": [ - "${arn_ssm_parameters_path_config}", - "${arn_ssm_parameters_path_config}/*" - ] + "Effect": "Allow", + "Action": [ + "ssm:GetParameter", + "ssm:GetParameters", + "ssm:GetParametersByPath" + ], + "Resource": [ + "${arn_ssm_parameters_path_config}", + "${arn_ssm_parameters_path_config}/*" + ] }, +%{ endif ~} { "Effect": "Allow", "Action": [ diff --git a/modules/runners/pool/variables.tf b/modules/runners/pool/variables.tf index adf5ad571c..64af865cb1 100644 --- a/modules/runners/pool/variables.tf +++ b/modules/runners/pool/variables.tf @@ -60,19 +60,36 @@ variable "config" { schedule_expression_timezone = string size = number })) - include_busy_runners = bool - role_permissions_boundary = string - kms_key_arn = string - ami_kms_key_arn = string - ami_id_ssm_parameter_arn = string - role_path = string - ssm_token_path = string - ssm_config_path = string - ami_id_ssm_parameter_name = string - ami_id_ssm_parameter_read_policy_arn = string - arn_ssm_parameters_path_config = string - lambda_tags = map(string) - user_agent = string + include_busy_runners = bool + role_permissions_boundary = string + kms_key_arn = string + ami_kms_key_arn = string + ami_id_ssm_parameter_arn = string + role_path = string + ssm_token_path = string + ssm_config_path = string + runner_config_storage_backend = string + runner_config_dynamodb_table_name = string + runner_config_dynamodb_table_arn = string + runner_config_dynamodb_partition_key_name = string + runner_config_dynamodb_value_attribute_name = string + runner_config_dynamodb_config_key_prefix = string + runner_config_dynamodb_consistent_read = bool + runner_config_dynamodb_token_overwrite_protection_enabled = bool + runner_config_dynamodb_token_key_prefix = string + runner_config_dynamodb_ttl_seconds = number + runner_config_dynamodb_ttl_attribute_name = string + runner_config_dynamodb_client_max_attempts = number + runner_config_dynamodb_client_retry_mode = string + runner_config_dynamodb_client_http_keep_alive = bool + runner_config_dynamodb_client_http_max_sockets = number + runner_config_dynamodb_client_http_keep_alive_msecs = string + runner_config_dynamodb_kms_key_arn = string + ami_id_ssm_parameter_name = string + ami_id_ssm_parameter_read_policy_arn = string + arn_ssm_parameters_path_config = string + lambda_tags = map(string) + user_agent = string }) } @@ -91,5 +108,3 @@ variable "tracing_config" { }) default = {} } - - diff --git a/modules/runners/runner-config.tf b/modules/runners/runner-config.tf index d381b7679c..4de9a01727 100644 --- a/modules/runners/runner-config.tf +++ b/modules/runners/runner-config.tf @@ -1,4 +1,6 @@ resource "aws_ssm_parameter" "runner_config_run_as" { + count = local.runner_config_storage_ssm ? 1 : 0 + name = "${var.ssm_paths.root}/${var.ssm_paths.config}/run_as" type = "String" value = var.runner_as_root ? "root" : var.runner_run_as @@ -6,6 +8,8 @@ resource "aws_ssm_parameter" "runner_config_run_as" { } resource "aws_ssm_parameter" "runner_agent_mode" { + count = local.runner_config_storage_ssm ? 1 : 0 + name = "${var.ssm_paths.root}/${var.ssm_paths.config}/agent_mode" type = "String" value = var.enable_ephemeral_runners ? "ephemeral" : "persistent" @@ -13,6 +17,8 @@ resource "aws_ssm_parameter" "runner_agent_mode" { } resource "aws_ssm_parameter" "disable_default_labels" { + count = local.runner_config_storage_ssm ? 1 : 0 + name = "${var.ssm_paths.root}/${var.ssm_paths.config}/disable_default_labels" type = "String" value = var.runner_disable_default_labels @@ -20,6 +26,8 @@ resource "aws_ssm_parameter" "disable_default_labels" { } resource "aws_ssm_parameter" "jit_config_enabled" { + count = local.runner_config_storage_ssm ? 1 : 0 + name = "${var.ssm_paths.root}/${var.ssm_paths.config}/enable_jit_config" type = "String" value = var.enable_jit_config == null ? var.enable_ephemeral_runners : var.enable_jit_config @@ -27,6 +35,8 @@ resource "aws_ssm_parameter" "jit_config_enabled" { } resource "aws_ssm_parameter" "runner_enable_cloudwatch" { + count = local.runner_config_storage_ssm ? 1 : 0 + name = "${var.ssm_paths.root}/${var.ssm_paths.config}/enable_cloudwatch" type = "String" value = var.enable_cloudwatch_agent @@ -34,6 +44,8 @@ resource "aws_ssm_parameter" "runner_enable_cloudwatch" { } resource "aws_ssm_parameter" "token_path" { + count = local.runner_config_storage_ssm ? 1 : 0 + name = "${var.ssm_paths.root}/${var.ssm_paths.config}/token_path" type = "String" value = "${var.ssm_paths.root}/${var.ssm_paths.tokens}" diff --git a/modules/runners/scale-up.tf b/modules/runners/scale-up.tf index bdda3c070f..7970681407 100644 --- a/modules/runners/scale-up.tf +++ b/modules/runners/scale-up.tf @@ -25,46 +25,61 @@ resource "aws_lambda_function" "scale_up" { architectures = [var.lambda_architecture] environment { variables = { - AMI_ID_SSM_PARAMETER_NAME = local.ami_id_ssm_parameter_name - DISABLE_RUNNER_AUTOUPDATE = var.disable_runner_autoupdate - ENABLE_EPHEMERAL_RUNNERS = var.enable_ephemeral_runners - ENABLE_JIT_CONFIG = var.enable_jit_config - ENABLE_JOB_QUEUED_CHECK = local.enable_job_queued_check - ENABLE_METRIC_GITHUB_APP_RATE_LIMIT = var.metrics.enable && var.metrics.metric.enable_github_app_rate_limit - ENABLE_ORGANIZATION_RUNNERS = var.enable_organization_runners - ENVIRONMENT = var.prefix - GHES_URL = var.ghes_url - USER_AGENT = var.user_agent - INSTANCE_ALLOCATION_STRATEGY = var.instance_allocation_strategy - INSTANCE_MAX_SPOT_PRICE = var.instance_max_spot_price - INSTANCE_TARGET_CAPACITY_TYPE = var.instance_target_capacity_type - INSTANCE_TYPE_PRIORITIES = var.instance_type_priorities != null ? jsonencode(var.instance_type_priorities) : "" - INSTANCE_TYPES = join(",", var.instance_types) - LAUNCH_TEMPLATE_NAME = aws_launch_template.runner.name - LOG_LEVEL = upper(var.log_level) - MINIMUM_RUNNING_TIME_IN_MINUTES = coalesce(var.minimum_running_time_in_minutes, local.min_runtime_defaults[var.runner_os]) - NODE_TLS_REJECT_UNAUTHORIZED = var.ghes_url != null && !var.ghes_ssl_verify ? 0 : 1 - PARAMETER_GITHUB_APP_ID_NAME = var.github_app_parameters.id.name - PARAMETER_GITHUB_APP_KEY_BASE64_NAME = var.github_app_parameters.key_base64.name - POWERTOOLS_LOGGER_LOG_EVENT = var.log_level == "debug" ? "true" : "false" - POWERTOOLS_METRICS_NAMESPACE = var.metrics.namespace - POWERTOOLS_TRACE_ENABLED = var.tracing_config.mode != null ? true : false - POWERTOOLS_TRACER_CAPTURE_HTTPS_REQUESTS = var.tracing_config.capture_http_requests - POWERTOOLS_TRACER_CAPTURE_ERROR = var.tracing_config.capture_error - RUNNER_LABELS = lower(join(",", var.runner_labels)) - RUNNER_GROUP_NAME = var.runner_group_name - RUNNER_NAME_PREFIX = var.runner_name_prefix - RUNNER_PROVIDER_TYPE = "ec2" - RUNNERS_MAXIMUM_COUNT = var.runners_maximum_count - POWERTOOLS_SERVICE_NAME = "${var.prefix}-scale-up" - SSM_TOKEN_PATH = local.token_path - SSM_CONFIG_PATH = "${var.ssm_paths.root}/${var.ssm_paths.config}" - SSM_PARAMETER_STORE_TAGS = local.parameter_store_tags - SUBNET_IDS = join(",", var.subnet_ids) - ENABLE_ON_DEMAND_FAILOVER_FOR_ERRORS = jsonencode(var.enable_on_demand_failover_for_errors) - SCALE_ERRORS = jsonencode(var.scale_errors) - JOB_RETRY_CONFIG = jsonencode(local.job_retry_config) - USE_DEDICATED_HOST = var.use_dedicated_host + AMI_ID_SSM_PARAMETER_NAME = local.ami_id_ssm_parameter_name + DISABLE_RUNNER_AUTOUPDATE = var.disable_runner_autoupdate + ENABLE_EPHEMERAL_RUNNERS = var.enable_ephemeral_runners + ENABLE_JIT_CONFIG = var.enable_jit_config + ENABLE_JOB_QUEUED_CHECK = local.enable_job_queued_check + ENABLE_METRIC_GITHUB_APP_RATE_LIMIT = var.metrics.enable && var.metrics.metric.enable_github_app_rate_limit + ENABLE_ORGANIZATION_RUNNERS = var.enable_organization_runners + ENVIRONMENT = var.prefix + GHES_URL = var.ghes_url + USER_AGENT = var.user_agent + INSTANCE_ALLOCATION_STRATEGY = var.instance_allocation_strategy + INSTANCE_MAX_SPOT_PRICE = var.instance_max_spot_price + INSTANCE_TARGET_CAPACITY_TYPE = var.instance_target_capacity_type + INSTANCE_TYPE_PRIORITIES = var.instance_type_priorities != null ? jsonencode(var.instance_type_priorities) : "" + INSTANCE_TYPES = join(",", var.instance_types) + LAUNCH_TEMPLATE_NAME = aws_launch_template.runner.name + LOG_LEVEL = upper(var.log_level) + MINIMUM_RUNNING_TIME_IN_MINUTES = coalesce(var.minimum_running_time_in_minutes, local.min_runtime_defaults[var.runner_os]) + NODE_TLS_REJECT_UNAUTHORIZED = var.ghes_url != null && !var.ghes_ssl_verify ? 0 : 1 + PARAMETER_GITHUB_APP_ID_NAME = var.github_app_parameters.id.name + PARAMETER_GITHUB_APP_KEY_BASE64_NAME = var.github_app_parameters.key_base64.name + POWERTOOLS_LOGGER_LOG_EVENT = var.log_level == "debug" ? "true" : "false" + POWERTOOLS_METRICS_NAMESPACE = var.metrics.namespace + POWERTOOLS_TRACE_ENABLED = var.tracing_config.mode != null ? true : false + POWERTOOLS_TRACER_CAPTURE_HTTPS_REQUESTS = var.tracing_config.capture_http_requests + POWERTOOLS_TRACER_CAPTURE_ERROR = var.tracing_config.capture_error + RUNNER_LABELS = lower(join(",", var.runner_labels)) + RUNNER_GROUP_NAME = var.runner_group_name + RUNNER_NAME_PREFIX = var.runner_name_prefix + RUNNER_PROVIDER_TYPE = "ec2" + RUNNER_CONFIG_STORAGE_BACKEND = local.runner_config_storage_backend + RUNNER_CONFIG_DYNAMODB_TABLE_NAME = local.runner_config_dynamodb_table_name + RUNNER_CONFIG_DYNAMODB_PARTITION_KEY_NAME = local.runner_config_dynamodb_partition_key_name + RUNNER_CONFIG_DYNAMODB_VALUE_ATTRIBUTE_NAME = local.runner_config_dynamodb_value_attribute_name + RUNNER_CONFIG_DYNAMODB_CONFIG_KEY_PREFIX = local.runner_config_dynamodb_config_key_prefix + RUNNER_CONFIG_DYNAMODB_CONSISTENT_READ = local.runner_config_dynamodb_consistent_read + RUNNER_CONFIG_DYNAMODB_TOKEN_OVERWRITE_PROTECTION_ENABLED = local.runner_config_dynamodb_token_overwrite_protection_enabled + RUNNER_CONFIG_DYNAMODB_TOKEN_KEY_PREFIX = local.runner_config_dynamodb_token_key_prefix + RUNNER_CONFIG_DYNAMODB_TTL_SECONDS = local.runner_config_dynamodb_ttl_seconds + RUNNER_CONFIG_DYNAMODB_TTL_ATTRIBUTE_NAME = var.runner_config_storage.dynamodb.ttl_attribute_name + RUNNER_CONFIG_DYNAMODB_CLIENT_MAX_ATTEMPTS = var.runner_config_storage.dynamodb.client_max_attempts + RUNNER_CONFIG_DYNAMODB_CLIENT_RETRY_MODE = var.runner_config_storage.dynamodb.client_retry_mode + RUNNER_CONFIG_DYNAMODB_CLIENT_HTTP_KEEP_ALIVE = var.runner_config_storage.dynamodb.client_http_keep_alive + RUNNER_CONFIG_DYNAMODB_CLIENT_HTTP_MAX_SOCKETS = var.runner_config_storage.dynamodb.client_http_max_sockets + RUNNER_CONFIG_DYNAMODB_CLIENT_HTTP_KEEP_ALIVE_MSECS = var.runner_config_storage.dynamodb.client_http_keep_alive_msecs == null ? "" : var.runner_config_storage.dynamodb.client_http_keep_alive_msecs + RUNNERS_MAXIMUM_COUNT = var.runners_maximum_count + POWERTOOLS_SERVICE_NAME = "${var.prefix}-scale-up" + SSM_TOKEN_PATH = local.token_path + SSM_CONFIG_PATH = "${var.ssm_paths.root}/${var.ssm_paths.config}" + SSM_PARAMETER_STORE_TAGS = local.parameter_store_tags + SUBNET_IDS = join(",", var.subnet_ids) + ENABLE_ON_DEMAND_FAILOVER_FOR_ERRORS = jsonencode(var.enable_on_demand_failover_for_errors) + SCALE_ERRORS = jsonencode(var.scale_errors) + JOB_RETRY_CONFIG = jsonencode(local.job_retry_config) + USE_DEDICATED_HOST = var.use_dedicated_host } } @@ -121,15 +136,30 @@ resource "aws_iam_role_policy" "scale_up" { name = "scale-up-policy" role = aws_iam_role.scale_up.name policy = templatefile("${path.module}/policies/lambda-scale-up.json", { - arn_runner_instance_role = var.iam_overrides["override_runner_role"] ? var.iam_overrides["runner_role_arn"] : aws_iam_role.runner[0].arn - environment = var.prefix - sqs_arn = var.sqs_build_queue.arn - github_app_id_arn = var.github_app_parameters.id.arn - github_app_key_base64_arn = var.github_app_parameters.key_base64.arn - ssm_config_path = "arn:${var.aws_partition}:ssm:${var.aws_region}:${data.aws_caller_identity.current.account_id}:parameter${var.ssm_paths.root}/${var.ssm_paths.config}" - kms_key_arn = local.kms_key_arn - ami_kms_key_arn = local.ami_kms_key_arn - ssm_ami_id_parameter_arn = local.ami_id_ssm_module_managed ? aws_ssm_parameter.runner_ami_id[0].arn : var.ami.id_ssm_parameter_arn + arn_runner_instance_role = var.iam_overrides["override_runner_role"] ? var.iam_overrides["runner_role_arn"] : aws_iam_role.runner[0].arn + environment = var.prefix + sqs_arn = var.sqs_build_queue.arn + github_app_id_arn = var.github_app_parameters.id.arn + github_app_key_base64_arn = var.github_app_parameters.key_base64.arn + ssm_config_path = "arn:${var.aws_partition}:ssm:${var.aws_region}:${data.aws_caller_identity.current.account_id}:parameter${var.ssm_paths.root}/${var.ssm_paths.config}" + kms_key_arn = local.kms_key_arn + ami_kms_key_arn = local.ami_kms_key_arn + ssm_ami_id_parameter_arn = local.ami_id_ssm_module_managed ? aws_ssm_parameter.runner_ami_id[0].arn : var.ami.id_ssm_parameter_arn + runner_config_storage_backend = local.runner_config_storage_backend + }) +} + +resource "aws_iam_role_policy" "scale_up_runner_config_dynamodb" { + count = local.runner_config_storage_dynamodb ? 1 : 0 + name = "dynamodb-runner-config-policy" + role = aws_iam_role.scale_up.name + policy = templatefile("${path.module}/policies/lambda-dynamodb-runner-config.json", { + table_arn = aws_dynamodb_table.runner_config[0].arn + leading_keys_json = jsonencode([ + "${local.runner_config_dynamodb_config_key_prefix}*", + "${local.runner_config_dynamodb_token_key_prefix}*", + ]) + kms_key_arn = var.runner_config_storage.dynamodb.kms_key_arn != null ? var.runner_config_storage.dynamodb.kms_key_arn : "" }) } diff --git a/modules/runners/ssm-housekeeper.tf b/modules/runners/ssm-housekeeper.tf index ab2e014232..8da82b1fc6 100644 --- a/modules/runners/ssm-housekeeper.tf +++ b/modules/runners/ssm-housekeeper.tf @@ -1,5 +1,6 @@ locals { ssm_housekeeper = { + create = var.ssm_housekeeper.create schedule_expression = var.ssm_housekeeper.schedule_expression state = var.ssm_housekeeper.state lambda_timeout = var.ssm_housekeeper.lambda_timeout @@ -13,13 +14,15 @@ locals { } resource "aws_lambda_function" "ssm_housekeeper" { + count = local.ssm_housekeeper.create ? 1 : 0 + s3_bucket = var.lambda_s3_bucket != null ? var.lambda_s3_bucket : null s3_key = var.runners_lambda_s3_key != null ? var.runners_lambda_s3_key : null s3_object_version = var.runners_lambda_s3_object_version != null ? var.runners_lambda_s3_object_version : null filename = var.lambda_s3_bucket == null ? local.lambda_zip : null source_code_hash = var.lambda_s3_bucket == null ? filebase64sha256(local.lambda_zip) : null function_name = "${var.prefix}-ssm-housekeeper" - role = aws_iam_role.ssm_housekeeper.arn + role = aws_iam_role.ssm_housekeeper[0].arn handler = "index.ssmHousekeeper" runtime = var.lambda_runtime timeout = local.ssm_housekeeper.lambda_timeout @@ -56,7 +59,9 @@ resource "aws_lambda_function" "ssm_housekeeper" { } resource "aws_cloudwatch_log_group" "ssm_housekeeper" { - name = "/aws/lambda/${aws_lambda_function.ssm_housekeeper.function_name}" + count = local.ssm_housekeeper.create ? 1 : 0 + + name = "/aws/lambda/${aws_lambda_function.ssm_housekeeper[0].function_name}" retention_in_days = var.logging_retention_in_days kms_key_id = var.logging_kms_key_id log_group_class = var.log_class @@ -64,6 +69,8 @@ resource "aws_cloudwatch_log_group" "ssm_housekeeper" { } resource "aws_cloudwatch_event_rule" "ssm_housekeeper" { + count = local.ssm_housekeeper.create ? 1 : 0 + name = "${var.prefix}-ssm-housekeeper" schedule_expression = local.ssm_housekeeper.schedule_expression tags = var.tags @@ -71,19 +78,25 @@ resource "aws_cloudwatch_event_rule" "ssm_housekeeper" { } resource "aws_cloudwatch_event_target" "ssm_housekeeper" { - rule = aws_cloudwatch_event_rule.ssm_housekeeper.name - arn = aws_lambda_function.ssm_housekeeper.arn + count = local.ssm_housekeeper.create ? 1 : 0 + + rule = aws_cloudwatch_event_rule.ssm_housekeeper[0].name + arn = aws_lambda_function.ssm_housekeeper[0].arn } resource "aws_lambda_permission" "ssm_housekeeper" { + count = local.ssm_housekeeper.create ? 1 : 0 + statement_id = "AllowExecutionFromCloudWatch" action = "lambda:InvokeFunction" - function_name = aws_lambda_function.ssm_housekeeper.function_name + function_name = aws_lambda_function.ssm_housekeeper[0].function_name principal = "events.amazonaws.com" - source_arn = aws_cloudwatch_event_rule.ssm_housekeeper.arn + source_arn = aws_cloudwatch_event_rule.ssm_housekeeper[0].arn } resource "aws_iam_role" "ssm_housekeeper" { + count = local.ssm_housekeeper.create ? 1 : 0 + name = "${substr("${var.prefix}-ssm-hk-lambda", 0, 54)}-${substr(md5("${var.prefix}-ssm-hk-lambda"), 0, 8)}" description = "Lambda role for SSM Housekeeper (${var.prefix})" assume_role_policy = data.aws_iam_policy_document.lambda_assume_role_policy.json @@ -93,30 +106,34 @@ resource "aws_iam_role" "ssm_housekeeper" { } resource "aws_iam_role_policy" "ssm_housekeeper" { + count = local.ssm_housekeeper.create ? 1 : 0 + name = "ssm-policy" - role = aws_iam_role.ssm_housekeeper.name + role = aws_iam_role.ssm_housekeeper[0].name policy = templatefile("${path.module}/policies/lambda-ssm-housekeeper.json", { ssm_token_path = "arn:${var.aws_partition}:ssm:${var.aws_region}:${data.aws_caller_identity.current.account_id}:parameter${local.token_path}" }) } resource "aws_iam_role_policy" "ssm_housekeeper_logging" { + count = local.ssm_housekeeper.create ? 1 : 0 + name = "logging-policy" - role = aws_iam_role.ssm_housekeeper.name + role = aws_iam_role.ssm_housekeeper[0].name policy = templatefile("${path.module}/policies/lambda-cloudwatch.json", { - log_group_arn = aws_cloudwatch_log_group.ssm_housekeeper.arn + log_group_arn = aws_cloudwatch_log_group.ssm_housekeeper[0].arn }) } resource "aws_iam_role_policy_attachment" "ssm_housekeeper_vpc_execution_role" { - count = length(var.lambda_subnet_ids) > 0 ? 1 : 0 - role = aws_iam_role.ssm_housekeeper.name + count = local.ssm_housekeeper.create && length(var.lambda_subnet_ids) > 0 ? 1 : 0 + role = aws_iam_role.ssm_housekeeper[0].name policy_arn = "arn:${var.aws_partition}:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole" } resource "aws_iam_role_policy" "ssm_housekeeper_xray" { - count = var.tracing_config.mode != null ? 1 : 0 + count = local.ssm_housekeeper.create && var.tracing_config.mode != null ? 1 : 0 name = "xray-policy" policy = data.aws_iam_policy_document.lambda_xray[0].json - role = aws_iam_role.ssm_housekeeper.name + role = aws_iam_role.ssm_housekeeper[0].name } diff --git a/modules/runners/templates/start-runner-osx.sh b/modules/runners/templates/start-runner-osx.sh index a6da66116d..82384a13be 100644 --- a/modules/runners/templates/start-runner-osx.sh +++ b/modules/runners/templates/start-runner-osx.sh @@ -49,6 +49,48 @@ cleanup() { trap 'cleanup $?' EXIT +runner_config_storage_backend="${runner_config_storage_backend}" +runner_config_dynamodb_table_name="${runner_config_dynamodb_table_name}" +runner_config_dynamodb_partition_key_name="${runner_config_dynamodb_partition_key_name}" +runner_config_dynamodb_value_attribute_name="${runner_config_dynamodb_value_attribute_name}" +runner_config_dynamodb_config_key_prefix="${runner_config_dynamodb_config_key_prefix}" +runner_config_dynamodb_consistent_read="${runner_config_dynamodb_consistent_read}" +runner_config_dynamodb_token_key_prefix="${runner_config_dynamodb_token_key_prefix}" + +get_dynamodb_item_value() { + local item_id="$1" + local key_json + local expression_attribute_names + local item_json + local consistent_read_args=() + local value + key_json=$(jq -cn --arg key_attr "$runner_config_dynamodb_partition_key_name" --arg id "$item_id" '{($key_attr):{S:$id}}') + expression_attribute_names=$(jq -cn --arg value_attr "$runner_config_dynamodb_value_attribute_name" '{"#value":$value_attr}') + if [[ "$runner_config_dynamodb_consistent_read" == "true" ]]; then + consistent_read_args=(--consistent-read) + fi + item_json=$(aws dynamodb get-item \ + --table-name "$runner_config_dynamodb_table_name" \ + --key "$key_json" \ + "$${consistent_read_args[@]}" \ + --projection-expression "#value" \ + --expression-attribute-names "$expression_attribute_names" \ + --region "$region" \ + --output json 2>/dev/null || true) + value=$(jq -r --arg value_attr "$runner_config_dynamodb_value_attribute_name" '.Item[$value_attr].S // ""' <<< "$item_json" 2>/dev/null || true) + printf '%s' "$value" +} + +delete_dynamodb_item() { + local item_id="$1" + local key_json + key_json=$(jq -cn --arg key_attr "$runner_config_dynamodb_partition_key_name" --arg id "$item_id" '{($key_attr):{S:$id}}') + aws dynamodb delete-item \ + --table-name "$runner_config_dynamodb_table_name" \ + --key "$key_json" \ + --region "$region" +} + echo "Retrieving TOKEN from AWS API" token=$(curl -f -X PUT "http://169.254.169.254/latest/api/token" \ -H "X-aws-ec2-metadata-token-ttl-seconds: 180" || true) @@ -88,37 +130,57 @@ echo "Retrieved ghr:environment tag - ($environment)" echo "Retrieved ghr:ssm_config_path tag - ($ssm_config_path)" echo "Retrieved ghr:runner_name_prefix tag - ($runner_name_prefix)" -parameters=$(aws ssm get-parameters-by-path \ - --path "$ssm_config_path" \ - --region "$region" \ - --query "Parameters[*].{Name:Name,Value:Value}") -echo "Retrieved parameters from AWS SSM ($parameters)" +if [[ "$runner_config_storage_backend" == "dynamodb" ]]; then + echo "Retrieving runner bootstrap config from AWS DynamoDB ($runner_config_dynamodb_table_name)" + run_as=$(get_dynamodb_item_value "$${runner_config_dynamodb_config_key_prefix}run_as") + agent_mode=$(get_dynamodb_item_value "$${runner_config_dynamodb_config_key_prefix}agent_mode") + disable_default_labels=$(get_dynamodb_item_value "$${runner_config_dynamodb_config_key_prefix}disable_default_labels") + enable_jit_config=$(get_dynamodb_item_value "$${runner_config_dynamodb_config_key_prefix}enable_jit_config") + + echo "Get GH Runner config from AWS DynamoDB" + runner_config_key="$runner_config_dynamodb_token_key_prefix$instance_id" + config=$(get_dynamodb_item_value "$runner_config_key") + while [[ -z "$config" ]]; do + echo "Waiting for GH Runner config to become available in AWS DynamoDB" + sleep 1 + config=$(get_dynamodb_item_value "$runner_config_key") + done + + echo "Delete GH Runner token from AWS DynamoDB" + delete_dynamodb_item "$runner_config_key" +else + parameters=$(aws ssm get-parameters-by-path \ + --path "$ssm_config_path" \ + --region "$region" \ + --query "Parameters[*].{Name:Name,Value:Value}") + echo "Retrieved parameters from AWS SSM ($parameters)" -run_as=$(echo "$parameters" | jq -r '.[] | select(.Name == "'$ssm_config_path'/run_as") | .Value') -echo "Retrieved /$ssm_config_path/run_as parameter - ($run_as)" + run_as=$(echo "$parameters" | jq -r '.[] | select(.Name == "'$ssm_config_path'/run_as") | .Value') + echo "Retrieved /$ssm_config_path/run_as parameter - ($run_as)" -agent_mode=$(echo "$parameters" | jq -r '.[] | select(.Name == "'$ssm_config_path'/agent_mode") | .Value') -echo "Retrieved /$ssm_config_path/agent_mode parameter - ($agent_mode)" + agent_mode=$(echo "$parameters" | jq -r '.[] | select(.Name == "'$ssm_config_path'/agent_mode") | .Value') + echo "Retrieved /$ssm_config_path/agent_mode parameter - ($agent_mode)" -disable_default_labels=$(echo "$parameters" | jq -r '.[] | select(.Name == "'$ssm_config_path'/disable_default_labels") | .Value') -echo "Retrieved /$ssm_config_path/disable_default_labels parameter - ($disable_default_labels)" + disable_default_labels=$(echo "$parameters" | jq -r '.[] | select(.Name == "'$ssm_config_path'/disable_default_labels") | .Value') + echo "Retrieved /$ssm_config_path/disable_default_labels parameter - ($disable_default_labels)" -enable_jit_config=$(echo "$parameters" | jq -r '.[] | select(.Name == "'$ssm_config_path'/enable_jit_config") | .Value') -echo "Retrieved /$ssm_config_path/enable_jit_config parameter - ($enable_jit_config)" + enable_jit_config=$(echo "$parameters" | jq -r '.[] | select(.Name == "'$ssm_config_path'/enable_jit_config") | .Value') + echo "Retrieved /$ssm_config_path/enable_jit_config parameter - ($enable_jit_config)" -token_path=$(echo "$parameters" | jq -r '.[] | select(.Name == "'$ssm_config_path'/token_path") | .Value') -echo "Retrieved /$ssm_config_path/token_path parameter - ($token_path)" + token_path=$(echo "$parameters" | jq -r '.[] | select(.Name == "'$ssm_config_path'/token_path") | .Value') + echo "Retrieved /$ssm_config_path/token_path parameter - ($token_path)" -echo "Get GH Runner config from AWS SSM" -config=$(aws ssm get-parameter --name "$token_path"/"$instance_id" --with-decryption --region "$region" | jq -r ".Parameter | .Value") -while [[ -z "$config" ]]; do - echo "Waiting for GH Runner config to become available in AWS SSM" - sleep 1 + echo "Get GH Runner config from AWS SSM" config=$(aws ssm get-parameter --name "$token_path"/"$instance_id" --with-decryption --region "$region" | jq -r ".Parameter | .Value") -done + while [[ -z "$config" ]]; do + echo "Waiting for GH Runner config to become available in AWS SSM" + sleep 1 + config=$(aws ssm get-parameter --name "$token_path"/"$instance_id" --with-decryption --region "$region" | jq -r ".Parameter | .Value") + done -echo "Delete GH Runner token from AWS SSM" -aws ssm delete-parameter --name "$token_path"/"$instance_id" --region "$region" + echo "Delete GH Runner token from AWS SSM" + aws ssm delete-parameter --name "$token_path"/"$instance_id" --region "$region" +fi if [ -z "$run_as" ]; then echo "No user specified, using default ec2-user account" diff --git a/modules/runners/templates/start-runner.ps1 b/modules/runners/templates/start-runner.ps1 index ae2eeff3c9..f141a57c1f 100644 --- a/modules/runners/templates/start-runner.ps1 +++ b/modules/runners/templates/start-runner.ps1 @@ -37,6 +37,55 @@ function Tag-InstanceWithRunnerId { } } +$RunnerConfigStorageBackend = "${runner_config_storage_backend}" +$RunnerConfigDynamodbTableName = "${runner_config_dynamodb_table_name}" +$RunnerConfigDynamodbPartitionKeyName = "${runner_config_dynamodb_partition_key_name}" +$RunnerConfigDynamodbValueAttributeName = "${runner_config_dynamodb_value_attribute_name}" +$RunnerConfigDynamodbConfigKeyPrefix = "${runner_config_dynamodb_config_key_prefix}" +$RunnerConfigDynamodbConsistentRead = "${runner_config_dynamodb_consistent_read}" +$RunnerConfigDynamodbTokenKeyPrefix = "${runner_config_dynamodb_token_key_prefix}" +$cloudwatchAgentConfig = $null + +function Get-DynamodbItemValue { + param ( + [string]$Id + ) + + $key = @{} + $key[$RunnerConfigDynamodbPartitionKeyName] = @{ S = $Id } + $keyJson = $key | ConvertTo-Json -Compress -Depth 5 + + $expressionAttributeNames = @{ "#value" = $RunnerConfigDynamodbValueAttributeName } | ConvertTo-Json -Compress -Depth 5 + $consistentReadArgs = @() + if ($RunnerConfigDynamodbConsistentRead -eq "true") { + $consistentReadArgs += "--consistent-read" + } + + $item = aws dynamodb get-item --table-name "$RunnerConfigDynamodbTableName" --key "$keyJson" @consistentReadArgs --projection-expression "#value" --expression-attribute-names "$expressionAttributeNames" --region "$Region" --output json 2>$null | ConvertFrom-Json + + if ($LASTEXITCODE -ne 0 -or $null -eq $item.Item) { + return $null + } + + $valueAttribute = $item.Item.PSObject.Properties[$RunnerConfigDynamodbValueAttributeName].Value + if ($null -eq $valueAttribute) { + return $null + } + + return $valueAttribute.S +} + +function Remove-DynamodbItem { + param ( + [string]$Id + ) + + $key = @{} + $key[$RunnerConfigDynamodbPartitionKeyName] = @{ S = $Id } + $keyJson = $key | ConvertTo-Json -Compress -Depth 5 + aws dynamodb delete-item --table-name "$RunnerConfigDynamodbTableName" --key "$keyJson" --region "$Region" +} + ## Retrieve instance metadata Write-Host "Retrieving TOKEN from AWS API" @@ -77,48 +126,87 @@ Write-Host "Retrieved ghr:runner_name_prefix tag - ($runner_name_prefix)" $ssm_config_path=$tags.Tags.where( {$_.Key -eq 'ghr:ssm_config_path'}).value Write-Host "Retrieved ghr:ssm_config_path tag - ($ssm_config_path)" -$parameters=$(aws ssm get-parameters-by-path --path "$ssm_config_path" --region "$Region" --query "Parameters[*].{Name:Name,Value:Value}") | ConvertFrom-Json -Write-Host "Retrieved parameters from AWS SSM" +if ($RunnerConfigStorageBackend -eq "dynamodb") +{ + Write-Host "Retrieving runner bootstrap config from AWS DynamoDB ($RunnerConfigDynamodbTableName)" + $run_as=Get-DynamodbItemValue -Id "$${RunnerConfigDynamodbConfigKeyPrefix}run_as" + $enable_cloudwatch_agent=Get-DynamodbItemValue -Id "$${RunnerConfigDynamodbConfigKeyPrefix}enable_cloudwatch" + $agent_mode=Get-DynamodbItemValue -Id "$${RunnerConfigDynamodbConfigKeyPrefix}agent_mode" + $disable_default_labels=Get-DynamodbItemValue -Id "$${RunnerConfigDynamodbConfigKeyPrefix}disable_default_labels" + $enable_jit_config=Get-DynamodbItemValue -Id "$${RunnerConfigDynamodbConfigKeyPrefix}enable_jit_config" + $cloudwatchAgentConfig=Get-DynamodbItemValue -Id "$${RunnerConfigDynamodbConfigKeyPrefix}cloudwatch_agent_config_runner" +} +else +{ + $parameters=$(aws ssm get-parameters-by-path --path "$ssm_config_path" --region "$Region" --query "Parameters[*].{Name:Name,Value:Value}") | ConvertFrom-Json + Write-Host "Retrieved parameters from AWS SSM" -$run_as=$parameters.where( {$_.Name -eq "$ssm_config_path/run_as"}).value -Write-Host "Retrieved $ssm_config_path/run_as parameter - ($run_as)" + $run_as=$parameters.where( {$_.Name -eq "$ssm_config_path/run_as"}).value + Write-Host "Retrieved $ssm_config_path/run_as parameter - ($run_as)" -$enable_cloudwatch_agent=$parameters.where( {$_.Name -eq "$ssm_config_path/enable_cloudwatch"}).value -Write-Host "Retrieved $ssm_config_path/enable_cloudwatch parameter - ($enable_cloudwatch_agent)" + $enable_cloudwatch_agent=$parameters.where( {$_.Name -eq "$ssm_config_path/enable_cloudwatch"}).value + Write-Host "Retrieved $ssm_config_path/enable_cloudwatch parameter - ($enable_cloudwatch_agent)" -$agent_mode=$parameters.where( {$_.Name -eq "$ssm_config_path/agent_mode"}).value -Write-Host "Retrieved $ssm_config_path/agent_mode parameter - ($agent_mode)" + $agent_mode=$parameters.where( {$_.Name -eq "$ssm_config_path/agent_mode"}).value + Write-Host "Retrieved $ssm_config_path/agent_mode parameter - ($agent_mode)" -$disable_default_labels=$parameters.where( {$_.Name -eq "$ssm_config_path/disable_default_labels"}).value -Write-Host "Retrieved $ssm_config_path/disable_default_labels parameter - ($disable_default_labels)" + $disable_default_labels=$parameters.where( {$_.Name -eq "$ssm_config_path/disable_default_labels"}).value + Write-Host "Retrieved $ssm_config_path/disable_default_labels parameter - ($disable_default_labels)" -$enable_jit_config=$parameters.where( {$_.Name -eq "$ssm_config_path/enable_jit_config"}).value -Write-Host "Retrieved $ssm_config_path/enable_jit_config parameter - ($enable_jit_config)" + $enable_jit_config=$parameters.where( {$_.Name -eq "$ssm_config_path/enable_jit_config"}).value + Write-Host "Retrieved $ssm_config_path/enable_jit_config parameter - ($enable_jit_config)" -$token_path=$parameters.where( {$_.Name -eq "$ssm_config_path/token_path"}).value -Write-Host "Retrieved $ssm_config_path/token_path parameter - ($token_path)" + $token_path=$parameters.where( {$_.Name -eq "$ssm_config_path/token_path"}).value + Write-Host "Retrieved $ssm_config_path/token_path parameter - ($token_path)" +} if ($enable_cloudwatch_agent -eq "true") { Write-Host "Enabling CloudWatch Agent" - & 'C:\Program Files\Amazon\AmazonCloudWatchAgent\amazon-cloudwatch-agent-ctl.ps1' -a fetch-config -m ec2 -s -c "ssm:$ssm_config_path/cloudwatch_agent_config_runner" + if ($RunnerConfigStorageBackend -eq "dynamodb") + { + $cloudwatchAgentConfigPath = "$env:TEMP\github-runner-cloudwatch-agent.json" + Set-Content -Path "$cloudwatchAgentConfigPath" -Value "$cloudwatchAgentConfig" + & 'C:\Program Files\Amazon\AmazonCloudWatchAgent\amazon-cloudwatch-agent-ctl.ps1' -a fetch-config -m ec2 -s -c "file:$cloudwatchAgentConfigPath" + } + else + { + & 'C:\Program Files\Amazon\AmazonCloudWatchAgent\amazon-cloudwatch-agent-ctl.ps1' -a fetch-config -m ec2 -s -c "ssm:$ssm_config_path/cloudwatch_agent_config_runner" + } } ## Configure the runner -Write-Host "Get GH Runner config from AWS SSM" $config = $null $i = 0 -do { - $config = (aws ssm get-parameters --names "$token_path/$InstanceId" --with-decryption --region $Region --query "Parameters[*].{Name:Name,Value:Value}" | ConvertFrom-Json)[0].value - Write-Host "Waiting for GH Runner config to become available in AWS SSM ($i/30)" - Start-Sleep 1 - $i++ -} while (($null -eq $config) -and ($i -lt 30)) - -Write-Host "Delete GH Runner token from AWS SSM" -aws ssm delete-parameter --name "$token_path/$InstanceId" --region $Region +if ($RunnerConfigStorageBackend -eq "dynamodb") +{ + Write-Host "Get GH Runner config from AWS DynamoDB" + $runnerConfigKey = "$RunnerConfigDynamodbTokenKeyPrefix$InstanceId" + do { + $config = Get-DynamodbItemValue -Id "$runnerConfigKey" + Write-Host "Waiting for GH Runner config to become available in AWS DynamoDB ($i/30)" + Start-Sleep 1 + $i++ + } while (($null -eq $config) -and ($i -lt 30)) + + Write-Host "Delete GH Runner token from AWS DynamoDB" + Remove-DynamodbItem -Id "$runnerConfigKey" +} +else +{ + Write-Host "Get GH Runner config from AWS SSM" + do { + $config = (aws ssm get-parameters --names "$token_path/$InstanceId" --with-decryption --region $Region --query "Parameters[*].{Name:Name,Value:Value}" | ConvertFrom-Json)[0].value + Write-Host "Waiting for GH Runner config to become available in AWS SSM ($i/30)" + Start-Sleep 1 + $i++ + } while (($null -eq $config) -and ($i -lt 30)) + + Write-Host "Delete GH Runner token from AWS SSM" + aws ssm delete-parameter --name "$token_path/$InstanceId" --region $Region +} # Create or update user if (-not($run_as)) { diff --git a/modules/runners/templates/start-runner.sh b/modules/runners/templates/start-runner.sh index 7f2c0f82c5..821e9f1e4e 100644 --- a/modules/runners/templates/start-runner.sh +++ b/modules/runners/templates/start-runner.sh @@ -112,6 +112,49 @@ cleanup() { trap 'cleanup $? $LINENO $BASH_LINENO' EXIT +runner_config_storage_backend="${runner_config_storage_backend}" +runner_config_dynamodb_table_name="${runner_config_dynamodb_table_name}" +runner_config_dynamodb_partition_key_name="${runner_config_dynamodb_partition_key_name}" +runner_config_dynamodb_value_attribute_name="${runner_config_dynamodb_value_attribute_name}" +runner_config_dynamodb_config_key_prefix="${runner_config_dynamodb_config_key_prefix}" +runner_config_dynamodb_consistent_read="${runner_config_dynamodb_consistent_read}" +runner_config_dynamodb_token_key_prefix="${runner_config_dynamodb_token_key_prefix}" +cloudwatch_agent_config="" + +get_dynamodb_item_value() { + local item_id="$1" + local key_json + local expression_attribute_names + local item_json + local consistent_read_args=() + local value + key_json=$(jq -cn --arg key_attr "$runner_config_dynamodb_partition_key_name" --arg id "$item_id" '{($key_attr):{S:$id}}') + expression_attribute_names=$(jq -cn --arg value_attr "$runner_config_dynamodb_value_attribute_name" '{"#value":$value_attr}') + if [[ "$runner_config_dynamodb_consistent_read" == "true" ]]; then + consistent_read_args=(--consistent-read) + fi + item_json=$(aws dynamodb get-item \ + --table-name "$runner_config_dynamodb_table_name" \ + --key "$key_json" \ + "$${consistent_read_args[@]}" \ + --projection-expression "#value" \ + --expression-attribute-names "$expression_attribute_names" \ + --region "$region" \ + --output json 2>/dev/null || true) + value=$(jq -r --arg value_attr "$runner_config_dynamodb_value_attribute_name" '.Item[$value_attr].S // ""' <<< "$item_json" 2>/dev/null || true) + printf '%s' "$value" +} + +delete_dynamodb_item() { + local item_id="$1" + local key_json + key_json=$(jq -cn --arg key_attr "$runner_config_dynamodb_partition_key_name" --arg id "$item_id" '{($key_attr):{S:$id}}') + aws dynamodb delete-item \ + --table-name "$runner_config_dynamodb_table_name" \ + --key "$key_json" \ + --region "$region" +} + echo "Retrieving TOKEN from AWS API" token=$(curl -f -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 180" || true) if [ -z "$token" ]; then @@ -159,26 +202,36 @@ echo "Retrieved ghr:environment tag - ($environment)" echo "Retrieved ghr:ssm_config_path tag - ($ssm_config_path)" echo "Retrieved ghr:runner_name_prefix tag - ($runner_name_prefix)" -parameters=$(aws ssm get-parameters-by-path --path "$ssm_config_path" --region "$region" --query "Parameters[*].{Name:Name,Value:Value}") -echo "Retrieved parameters from AWS SSM ($parameters)" +if [[ "$runner_config_storage_backend" == "dynamodb" ]]; then + echo "Retrieving runner bootstrap config from AWS DynamoDB ($runner_config_dynamodb_table_name)" + run_as=$(get_dynamodb_item_value "$${runner_config_dynamodb_config_key_prefix}run_as") + enable_cloudwatch_agent=$(get_dynamodb_item_value "$${runner_config_dynamodb_config_key_prefix}enable_cloudwatch") + agent_mode=$(get_dynamodb_item_value "$${runner_config_dynamodb_config_key_prefix}agent_mode") + disable_default_labels=$(get_dynamodb_item_value "$${runner_config_dynamodb_config_key_prefix}disable_default_labels") + enable_jit_config=$(get_dynamodb_item_value "$${runner_config_dynamodb_config_key_prefix}enable_jit_config") + cloudwatch_agent_config=$(get_dynamodb_item_value "$${runner_config_dynamodb_config_key_prefix}cloudwatch_agent_config_runner") +else + parameters=$(aws ssm get-parameters-by-path --path "$ssm_config_path" --region "$region" --query "Parameters[*].{Name:Name,Value:Value}") + echo "Retrieved parameters from AWS SSM ($parameters)" -run_as=$(echo "$parameters" | jq -r '.[] | select(.Name == "'$ssm_config_path'/run_as") | .Value') -echo "Retrieved /$ssm_config_path/run_as parameter - ($run_as)" + run_as=$(echo "$parameters" | jq -r '.[] | select(.Name == "'$ssm_config_path'/run_as") | .Value') + echo "Retrieved /$ssm_config_path/run_as parameter - ($run_as)" -enable_cloudwatch_agent=$(echo "$parameters" | jq --arg ssm_config_path "$ssm_config_path" -r '.[] | select(.Name == "'$ssm_config_path'/enable_cloudwatch") | .Value') -echo "Retrieved /$ssm_config_path/enable_cloudwatch parameter - ($enable_cloudwatch_agent)" + enable_cloudwatch_agent=$(echo "$parameters" | jq --arg ssm_config_path "$ssm_config_path" -r '.[] | select(.Name == "'$ssm_config_path'/enable_cloudwatch") | .Value') + echo "Retrieved /$ssm_config_path/enable_cloudwatch parameter - ($enable_cloudwatch_agent)" -agent_mode=$(echo "$parameters" | jq --arg ssm_config_path "$ssm_config_path" -r '.[] | select(.Name == "'$ssm_config_path'/agent_mode") | .Value') -echo "Retrieved /$ssm_config_path/agent_mode parameter - ($agent_mode)" + agent_mode=$(echo "$parameters" | jq --arg ssm_config_path "$ssm_config_path" -r '.[] | select(.Name == "'$ssm_config_path'/agent_mode") | .Value') + echo "Retrieved /$ssm_config_path/agent_mode parameter - ($agent_mode)" -disable_default_labels=$(echo "$parameters" | jq --arg ssm_config_path "$ssm_config_path" -r '.[] | select(.Name == "'$ssm_config_path'/disable_default_labels") | .Value') -echo "Retrieved /$ssm_config_path/disable_default_labels parameter - ($disable_default_labels)" + disable_default_labels=$(echo "$parameters" | jq --arg ssm_config_path "$ssm_config_path" -r '.[] | select(.Name == "'$ssm_config_path'/disable_default_labels") | .Value') + echo "Retrieved /$ssm_config_path/disable_default_labels parameter - ($disable_default_labels)" -enable_jit_config=$(echo "$parameters" | jq --arg ssm_config_path "$ssm_config_path" -r '.[] | select(.Name == "'$ssm_config_path'/enable_jit_config") | .Value') -echo "Retrieved /$ssm_config_path/enable_jit_config parameter - ($enable_jit_config)" + enable_jit_config=$(echo "$parameters" | jq --arg ssm_config_path "$ssm_config_path" -r '.[] | select(.Name == "'$ssm_config_path'/enable_jit_config") | .Value') + echo "Retrieved /$ssm_config_path/enable_jit_config parameter - ($enable_jit_config)" -token_path=$(echo "$parameters" | jq --arg ssm_config_path "$ssm_config_path" -r '.[] | select(.Name == "'$ssm_config_path'/token_path") | .Value') -echo "Retrieved /$ssm_config_path/token_path parameter - ($token_path)" + token_path=$(echo "$parameters" | jq --arg ssm_config_path "$ssm_config_path" -r '.[] | select(.Name == "'$ssm_config_path'/token_path") | .Value') + echo "Retrieved /$ssm_config_path/token_path parameter - ($token_path)" +fi if [[ "$xray_trace_id" != "" ]]; then # run xray service @@ -194,21 +247,41 @@ fi if [[ "$enable_cloudwatch_agent" == "true" ]]; then echo "Cloudwatch is enabled" - amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c "ssm:$ssm_config_path/cloudwatch_agent_config_runner" + if [[ "$runner_config_storage_backend" == "dynamodb" ]]; then + cloudwatch_agent_config_path="/opt/aws/amazon-cloudwatch-agent/etc/github-runner-cloudwatch-agent.json" + printf '%s' "$cloudwatch_agent_config" > "$cloudwatch_agent_config_path" + amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c "file:$cloudwatch_agent_config_path" + else + amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c "ssm:$ssm_config_path/cloudwatch_agent_config_runner" + fi fi ## Configure the runner -echo "Get GH Runner config from AWS SSM" -config=$(aws ssm get-parameter --name "$token_path"/"$instance_id" --with-decryption --region "$region" | jq -r ".Parameter | .Value") -while [[ -z "$config" ]]; do - echo "Waiting for GH Runner config to become available in AWS SSM" - sleep 1 +if [[ "$runner_config_storage_backend" == "dynamodb" ]]; then + echo "Get GH Runner config from AWS DynamoDB" + runner_config_key="$runner_config_dynamodb_token_key_prefix$instance_id" + config=$(get_dynamodb_item_value "$runner_config_key") + while [[ -z "$config" ]]; do + echo "Waiting for GH Runner config to become available in AWS DynamoDB" + sleep 1 + config=$(get_dynamodb_item_value "$runner_config_key") + done + + echo "Delete GH Runner token from AWS DynamoDB" + delete_dynamodb_item "$runner_config_key" +else + echo "Get GH Runner config from AWS SSM" config=$(aws ssm get-parameter --name "$token_path"/"$instance_id" --with-decryption --region "$region" | jq -r ".Parameter | .Value") -done + while [[ -z "$config" ]]; do + echo "Waiting for GH Runner config to become available in AWS SSM" + sleep 1 + config=$(aws ssm get-parameter --name "$token_path"/"$instance_id" --with-decryption --region "$region" | jq -r ".Parameter | .Value") + done -echo "Delete GH Runner token from AWS SSM" -aws ssm delete-parameter --name "$token_path"/"$instance_id" --region "$region" + echo "Delete GH Runner token from AWS SSM" + aws ssm delete-parameter --name "$token_path"/"$instance_id" --region "$region" +fi if [ -z "$run_as" ]; then echo "No user specified, using default ec2-user account" diff --git a/modules/runners/variables.tf b/modules/runners/variables.tf index 08283ce65c..fa867187b6 100644 --- a/modules/runners/variables.tf +++ b/modules/runners/variables.tf @@ -654,6 +654,130 @@ variable "ssm_paths" { }) } +variable "runner_config_storage" { + description = "Storage backend for label-scoped runner bootstrap config and one-time registration/JIT config. Common GitHub App secrets and webhook routing config continue to use SSM Parameter Store. Set `backend` to `dynamodb` to create and use a DynamoDB table for this runner label. When `token_ttl_seconds` is null, DynamoDB token expiry uses the SSM housekeeper minimum-days retention." + type = object({ + backend = optional(string, "ssm") + dynamodb = optional(object({ + table_name = optional(string, null) + partition_key_name = optional(string, "id") + value_attribute_name = optional(string, "value") + config_key_prefix = optional(string, "config#") + token_key_prefix = optional(string, null) + consistent_read = optional(bool, true) + token_overwrite_protection_enabled = optional(bool, true) + billing_mode = optional(string, "PAY_PER_REQUEST") + read_capacity = optional(number, 5) + write_capacity = optional(number, 5) + autoscaling_enabled = optional(bool, false) + autoscaling_read_min_capacity = optional(number, 5) + autoscaling_read_max_capacity = optional(number, 100) + autoscaling_read_target_value = optional(number, 70) + autoscaling_write_min_capacity = optional(number, 5) + autoscaling_write_max_capacity = optional(number, 100) + autoscaling_write_target_value = optional(number, 70) + token_ttl_seconds = optional(number, null) + ttl_enabled = optional(bool, true) + ttl_attribute_name = optional(string, "expires_at") + client_max_attempts = optional(number, 10) + client_retry_mode = optional(string, "adaptive") + client_http_keep_alive = optional(bool, true) + client_http_max_sockets = optional(number, 50) + client_http_keep_alive_msecs = optional(number, null) + server_side_encryption_enabled = optional(bool, true) + point_in_time_recovery_enabled = optional(bool, true) + deletion_protection_enabled = optional(bool, true) + kms_key_arn = optional(string, null) + alarms_enabled = optional(bool, true) + alarm_actions = optional(list(string), []) + ok_actions = optional(list(string), []) + insufficient_data_actions = optional(list(string), []) + alarm_evaluation_periods = optional(number, 1) + alarm_datapoints_to_alarm = optional(number, 1) + alarm_period = optional(number, 300) + alarm_treat_missing_data = optional(string, "notBreaching") + read_throttle_alarm_threshold = optional(number, 0) + write_throttle_alarm_threshold = optional(number, 0) + system_errors_alarm_threshold = optional(number, 0) + user_errors_alarm_threshold = optional(number, 0) + consumed_read_capacity_threshold = optional(number, null) + consumed_write_capacity_threshold = optional(number, null) + }), {}) + }) + default = {} + + validation { + condition = contains(["ssm", "dynamodb"], lower(var.runner_config_storage.backend)) + error_message = "`runner_config_storage.backend` must be either `ssm` or `dynamodb`." + } + + validation { + condition = contains(["PAY_PER_REQUEST", "PROVISIONED"], var.runner_config_storage.dynamodb.billing_mode) + error_message = "`runner_config_storage.dynamodb.billing_mode` must be either `PAY_PER_REQUEST` or `PROVISIONED`." + } + + validation { + condition = var.runner_config_storage.dynamodb.autoscaling_enabled == false || var.runner_config_storage.dynamodb.billing_mode == "PROVISIONED" + error_message = "`runner_config_storage.dynamodb.autoscaling_enabled` can only be true when billing mode is `PROVISIONED`." + } + + validation { + condition = var.runner_config_storage.dynamodb.token_ttl_seconds == null ? true : var.runner_config_storage.dynamodb.token_ttl_seconds > 0 + error_message = "`runner_config_storage.dynamodb.token_ttl_seconds` must be null or greater than zero." + } + + validation { + condition = alltrue([ + for value in [ + var.runner_config_storage.dynamodb.partition_key_name, + var.runner_config_storage.dynamodb.value_attribute_name, + var.runner_config_storage.dynamodb.config_key_prefix, + var.runner_config_storage.dynamodb.ttl_attribute_name, + ] : trimspace(value) != "" + ]) + error_message = "`runner_config_storage.dynamodb` attribute names and `config_key_prefix` must not be empty." + } + + validation { + condition = length(distinct([ + var.runner_config_storage.dynamodb.partition_key_name, + var.runner_config_storage.dynamodb.value_attribute_name, + var.runner_config_storage.dynamodb.ttl_attribute_name, + ])) == 3 + error_message = "`runner_config_storage.dynamodb` attribute names must be distinct." + } + + validation { + condition = var.runner_config_storage.dynamodb.token_key_prefix == null ? true : trimspace(var.runner_config_storage.dynamodb.token_key_prefix) != "" + error_message = "`runner_config_storage.dynamodb.token_key_prefix` must be null or non-empty." + } + + validation { + condition = var.runner_config_storage.dynamodb.client_max_attempts > 0 + error_message = "`runner_config_storage.dynamodb.client_max_attempts` must be greater than zero." + } + + validation { + condition = contains(["standard", "adaptive"], var.runner_config_storage.dynamodb.client_retry_mode) + error_message = "`runner_config_storage.dynamodb.client_retry_mode` must be either `standard` or `adaptive`." + } + + validation { + condition = var.runner_config_storage.dynamodb.client_http_max_sockets > 0 + error_message = "`runner_config_storage.dynamodb.client_http_max_sockets` must be greater than zero." + } + + validation { + condition = var.runner_config_storage.dynamodb.client_http_keep_alive_msecs == null ? true : var.runner_config_storage.dynamodb.client_http_keep_alive_msecs >= 0 + error_message = "`runner_config_storage.dynamodb.client_http_keep_alive_msecs` must be null or greater than or equal to zero." + } + + validation { + condition = contains(["breaching", "notBreaching", "ignore", "missing"], var.runner_config_storage.dynamodb.alarm_treat_missing_data) + error_message = "`runner_config_storage.dynamodb.alarm_treat_missing_data` must be one of `breaching`, `notBreaching`, `ignore`, or `missing`." + } +} + variable "runner_name_prefix" { description = "The prefix used for the GitHub runner name. The prefix will be used in the default start script to prefix the instance name when register the runner in GitHub. The value is available via an EC2 tag 'ghr:runner_name_prefix'." type = string @@ -746,12 +870,14 @@ variable "ssm_housekeeper" { Configuration for the SSM housekeeper lambda. This lambda deletes token / JIT config from SSM. `schedule_expression`: is used to configure the schedule for the lambda. + `create`: create or remove the SSM housekeeper lambda resources. `state`: state of the cloudwatch event rule. Valid values are `DISABLED`, `ENABLED`, and `ENABLED_WITH_ALL_CLOUDTRAIL_MANAGEMENT_EVENTS`. `lambda_memory_size`: lambda memory size limit. `lambda_timeout`: timeout for the lambda in seconds. `config`: configuration for the lambda function. Token path will be read by default from the module. EOF type = object({ + create = optional(bool, true) schedule_expression = optional(string, "rate(1 day)") state = optional(string, "ENABLED") lambda_memory_size = optional(number, 512) diff --git a/outputs.tf b/outputs.tf index fdf4a37801..3f3b89fc40 100644 --- a/outputs.tf +++ b/outputs.tf @@ -1,22 +1,24 @@ output "runners" { value = { - launch_template_name = module.runners.launch_template.name - launch_template_id = module.runners.launch_template.id - launch_template_version = module.runners.launch_template.latest_version - launch_template_ami_id = module.runners.launch_template.image_id - lambda_up = module.runners.lambda_scale_up - lambda_up_log_group = module.runners.lambda_scale_up_log_group - lambda_down = module.runners.lambda_scale_down - lambda_down_log_group = module.runners.lambda_scale_down_log_group - lambda_pool = module.runners.lambda_pool - lambda_pool_log_group = module.runners.lambda_pool_log_group - role_runner = module.runners.role_runner - role_scale_up = module.runners.role_scale_up - role_scale_down = module.runners.role_scale_down - role_pool = module.runners.role_pool - runners_log_groups = module.runners.runners_log_groups - labels = local.runner_labels - logfiles = module.runners.logfiles + launch_template_name = module.runners.launch_template.name + launch_template_id = module.runners.launch_template.id + launch_template_version = module.runners.launch_template.latest_version + launch_template_ami_id = module.runners.launch_template.image_id + lambda_up = module.runners.lambda_scale_up + lambda_up_log_group = module.runners.lambda_scale_up_log_group + lambda_down = module.runners.lambda_scale_down + lambda_down_log_group = module.runners.lambda_scale_down_log_group + lambda_pool = module.runners.lambda_pool + lambda_pool_log_group = module.runners.lambda_pool_log_group + role_runner = module.runners.role_runner + role_scale_up = module.runners.role_scale_up + role_scale_down = module.runners.role_scale_down + role_pool = module.runners.role_pool + runners_log_groups = module.runners.runners_log_groups + runner_config_storage_backend = module.runners.runner_config_storage_backend + runner_config_dynamodb_table = module.runners.runner_config_dynamodb_table + labels = local.runner_labels + logfiles = module.runners.logfiles } } diff --git a/variables.tf b/variables.tf index 4af2ab4cd1..6df62414f9 100644 --- a/variables.tf +++ b/variables.tf @@ -935,6 +935,130 @@ variable "ssm_paths" { default = {} } +variable "runner_config_storage" { + description = "Storage backend for label-scoped runner bootstrap config and one-time registration/JIT config. Common GitHub App secrets and webhook routing config continue to use SSM Parameter Store. Set `backend` to `dynamodb` to create and use a DynamoDB table for the runner label. When `token_ttl_seconds` is null, DynamoDB token expiry uses the SSM housekeeper minimum-days retention." + type = object({ + backend = optional(string, "ssm") + dynamodb = optional(object({ + table_name = optional(string, null) + partition_key_name = optional(string, "id") + value_attribute_name = optional(string, "value") + config_key_prefix = optional(string, "config#") + token_key_prefix = optional(string, null) + consistent_read = optional(bool, true) + token_overwrite_protection_enabled = optional(bool, true) + billing_mode = optional(string, "PAY_PER_REQUEST") + read_capacity = optional(number, 5) + write_capacity = optional(number, 5) + autoscaling_enabled = optional(bool, false) + autoscaling_read_min_capacity = optional(number, 5) + autoscaling_read_max_capacity = optional(number, 100) + autoscaling_read_target_value = optional(number, 70) + autoscaling_write_min_capacity = optional(number, 5) + autoscaling_write_max_capacity = optional(number, 100) + autoscaling_write_target_value = optional(number, 70) + token_ttl_seconds = optional(number, null) + ttl_enabled = optional(bool, true) + ttl_attribute_name = optional(string, "expires_at") + client_max_attempts = optional(number, 10) + client_retry_mode = optional(string, "adaptive") + client_http_keep_alive = optional(bool, true) + client_http_max_sockets = optional(number, 50) + client_http_keep_alive_msecs = optional(number, null) + server_side_encryption_enabled = optional(bool, true) + point_in_time_recovery_enabled = optional(bool, true) + deletion_protection_enabled = optional(bool, true) + kms_key_arn = optional(string, null) + alarms_enabled = optional(bool, true) + alarm_actions = optional(list(string), []) + ok_actions = optional(list(string), []) + insufficient_data_actions = optional(list(string), []) + alarm_evaluation_periods = optional(number, 1) + alarm_datapoints_to_alarm = optional(number, 1) + alarm_period = optional(number, 300) + alarm_treat_missing_data = optional(string, "notBreaching") + read_throttle_alarm_threshold = optional(number, 0) + write_throttle_alarm_threshold = optional(number, 0) + system_errors_alarm_threshold = optional(number, 0) + user_errors_alarm_threshold = optional(number, 0) + consumed_read_capacity_threshold = optional(number, null) + consumed_write_capacity_threshold = optional(number, null) + }), {}) + }) + default = {} + + validation { + condition = contains(["ssm", "dynamodb"], lower(var.runner_config_storage.backend)) + error_message = "`runner_config_storage.backend` must be either `ssm` or `dynamodb`." + } + + validation { + condition = contains(["PAY_PER_REQUEST", "PROVISIONED"], var.runner_config_storage.dynamodb.billing_mode) + error_message = "`runner_config_storage.dynamodb.billing_mode` must be either `PAY_PER_REQUEST` or `PROVISIONED`." + } + + validation { + condition = var.runner_config_storage.dynamodb.autoscaling_enabled == false || var.runner_config_storage.dynamodb.billing_mode == "PROVISIONED" + error_message = "`runner_config_storage.dynamodb.autoscaling_enabled` can only be true when billing mode is `PROVISIONED`." + } + + validation { + condition = var.runner_config_storage.dynamodb.token_ttl_seconds == null ? true : var.runner_config_storage.dynamodb.token_ttl_seconds > 0 + error_message = "`runner_config_storage.dynamodb.token_ttl_seconds` must be null or greater than zero." + } + + validation { + condition = alltrue([ + for value in [ + var.runner_config_storage.dynamodb.partition_key_name, + var.runner_config_storage.dynamodb.value_attribute_name, + var.runner_config_storage.dynamodb.config_key_prefix, + var.runner_config_storage.dynamodb.ttl_attribute_name, + ] : trimspace(value) != "" + ]) + error_message = "`runner_config_storage.dynamodb` attribute names and `config_key_prefix` must not be empty." + } + + validation { + condition = length(distinct([ + var.runner_config_storage.dynamodb.partition_key_name, + var.runner_config_storage.dynamodb.value_attribute_name, + var.runner_config_storage.dynamodb.ttl_attribute_name, + ])) == 3 + error_message = "`runner_config_storage.dynamodb` attribute names must be distinct." + } + + validation { + condition = var.runner_config_storage.dynamodb.token_key_prefix == null ? true : trimspace(var.runner_config_storage.dynamodb.token_key_prefix) != "" + error_message = "`runner_config_storage.dynamodb.token_key_prefix` must be null or non-empty." + } + + validation { + condition = var.runner_config_storage.dynamodb.client_max_attempts > 0 + error_message = "`runner_config_storage.dynamodb.client_max_attempts` must be greater than zero." + } + + validation { + condition = contains(["standard", "adaptive"], var.runner_config_storage.dynamodb.client_retry_mode) + error_message = "`runner_config_storage.dynamodb.client_retry_mode` must be either `standard` or `adaptive`." + } + + validation { + condition = var.runner_config_storage.dynamodb.client_http_max_sockets > 0 + error_message = "`runner_config_storage.dynamodb.client_http_max_sockets` must be greater than zero." + } + + validation { + condition = var.runner_config_storage.dynamodb.client_http_keep_alive_msecs == null ? true : var.runner_config_storage.dynamodb.client_http_keep_alive_msecs >= 0 + error_message = "`runner_config_storage.dynamodb.client_http_keep_alive_msecs` must be null or greater than or equal to zero." + } + + validation { + condition = contains(["breaching", "notBreaching", "ignore", "missing"], var.runner_config_storage.dynamodb.alarm_treat_missing_data) + error_message = "`runner_config_storage.dynamodb.alarm_treat_missing_data` must be one of `breaching`, `notBreaching`, `ignore`, or `missing`." + } +} + variable "runner_name_prefix" { description = "The prefix used for the GitHub runner name. The prefix will be used in the default start script to prefix the instance name when register the runner in GitHub. The value is available via an EC2 tag 'ghr:runner_name_prefix'." type = string @@ -1026,12 +1150,14 @@ variable "runners_ssm_housekeeper" { Configuration for the SSM housekeeper lambda. This lambda deletes token / JIT config from SSM. `schedule_expression`: is used to configure the schedule for the lambda. - `enabled`: enable or disable the lambda trigger via the EventBridge. + `create`: create or remove the SSM housekeeper lambda resources. + `enabled`: enable or disable only the lambda trigger via the EventBridge. `lambda_memory_size`: lambda memory size limit. `lambda_timeout`: timeout for the lambda in seconds. `config`: configuration for the lambda function. Token path will be read by default from the module. EOF type = object({ + create = optional(bool, true) schedule_expression = optional(string, "rate(1 day)") enabled = optional(bool, true) lambda_memory_size = optional(number, 512)