Skip to content

feat: add optional DynamoDB runner config storage #5266

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wadherv wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

feat: add optional DynamoDB runner config storage #5266

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 0 additions & 9 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,14 +1,5 @@
# Changelog

## [7.10.2](https://github.com/github-aws-runners/terraform-aws-github-runner/compare/v7.10.1...v7.10.2) (2026-08-10)


### Bug Fixes

* **lambda:** bump postcss from 8.5.22 to 8.5.24 in /lambdas ([#5241](https://github.com/github-aws-runners/terraform-aws-github-runner/issues/5241)) ([17cd322](https://github.com/github-aws-runners/terraform-aws-github-runner/commit/17cd322aa4a43db583072cce4b87f99db494b6d0))
* **termination-watcher:** only emit SpotInterruptionWarning metric for actual spot interruption events ([#5245](https://github.com/github-aws-runners/terraform-aws-github-runner/issues/5245)) ([7480fd5](https://github.com/github-aws-runners/terraform-aws-github-runner/commit/7480fd58d4c517e8e999a830905bbd9334dc987d))
* wrap LOG_LEVEL env var with upper() for Powertools v2 compatibility ([#5238](https://github.com/github-aws-runners/terraform-aws-github-runner/issues/5238)) ([2394f84](https://github.com/github-aws-runners/terraform-aws-github-runner/commit/2394f8445faeba3d37998791cf51ae9dd0ab453e))

## [7.10.1](https://github.com/github-aws-runners/terraform-aws-github-runner/compare/v7.10.0...v7.10.1) (2026-07-31)


Expand Down
2 changes: 2 additions & 0 deletions lambdas/functions/control-plane/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
"@aws-github-runner/aws-ssm-util": "*",
"@aws-github-runner/runner-providers": "*",
"@aws-lambda-powertools/parameters": "^2.31.0",
"@aws-sdk/client-dynamodb": "^3.1009.0",
"@aws-sdk/client-ec2": "^3.1009.0",
"@aws-sdk/client-sqs": "^3.1009.0",
"@middy/core": "^6.4.5",
Expand All @@ -42,6 +43,7 @@
"@octokit/plugin-retry": "8.0.3",
"@octokit/plugin-throttling": "11.0.3",
"@octokit/rest": "22.0.1",
"@smithy/node-http-handler": "^4.5.0",
"cron-parser": "^5.4.0"
},
"nx": {
Expand Down
15 changes: 15 additions & 0 deletions lambdas/functions/control-plane/src/modules.d.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,21 @@ declare namespace NodeJS {
PARAMETER_GITHUB_APP_CLIENT_SECRET_NAME: string;
PARAMETER_GITHUB_APP_ID_NAME: string;
PARAMETER_GITHUB_APP_KEY_BASE64_NAME: string;
RUNNER_CONFIG_STORAGE_BACKEND?: string;
RUNNER_CONFIG_DYNAMODB_TABLE_NAME?: string;
RUNNER_CONFIG_DYNAMODB_PARTITION_KEY_NAME?: string;
RUNNER_CONFIG_DYNAMODB_VALUE_ATTRIBUTE_NAME?: string;
RUNNER_CONFIG_DYNAMODB_CONFIG_KEY_PREFIX?: string;
RUNNER_CONFIG_DYNAMODB_CONSISTENT_READ?: string;
RUNNER_CONFIG_DYNAMODB_TOKEN_OVERWRITE_PROTECTION_ENABLED?: string;
RUNNER_CONFIG_DYNAMODB_TOKEN_KEY_PREFIX?: string;
RUNNER_CONFIG_DYNAMODB_TTL_SECONDS?: string;
RUNNER_CONFIG_DYNAMODB_TTL_ATTRIBUTE_NAME?: string;
RUNNER_CONFIG_DYNAMODB_CLIENT_MAX_ATTEMPTS?: string;
RUNNER_CONFIG_DYNAMODB_CLIENT_RETRY_MODE?: string;
RUNNER_CONFIG_DYNAMODB_CLIENT_HTTP_KEEP_ALIVE?: string;
RUNNER_CONFIG_DYNAMODB_CLIENT_HTTP_MAX_SOCKETS?: string;
RUNNER_CONFIG_DYNAMODB_CLIENT_HTTP_KEEP_ALIVE_MSECS?: string;
RUNNER_OWNER: string;
RUNNER_PROVIDER_TYPE?: string;
SCALE_DOWN_CONFIG: string;
Expand Down
3 changes: 3 additions & 0 deletions lambdas/functions/control-plane/src/pool/pool.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import yn from 'yn';
import { createGithubAppAuth, createGithubInstallationAuth, createOctokitClient } from '../github/auth';
import { controlPlaneProviderRegistry } from '../control-plane-providers';
import { getGitHubEnterpriseApiUrl, validateSsmParameterStoreTags } from '../scale-runners/github-runner';
import { loadRunnerConfigStorageFromEnv } from '../scale-runners/runner-config-storage';
import type { RunnerStatus } from './pool-provider';

const logger = createChildLogger('pool');
Expand All @@ -28,6 +29,7 @@ export async function adjust(event: PoolEvent): Promise<void> {
const environment = process.env.ENVIRONMENT;
const ssmTokenPath = process.env.SSM_TOKEN_PATH;
const ssmConfigPath = process.env.SSM_CONFIG_PATH || '';
const runnerConfigStorage = loadRunnerConfigStorageFromEnv();
const ephemeral = yn(process.env.ENABLE_EPHEMERAL_RUNNERS, { default: false });
const enableJitConfig = yn(process.env.ENABLE_JIT_CONFIG, { default: ephemeral });
const disableAutoUpdate = yn(process.env.DISABLE_RUNNER_AUTOUPDATE, { default: false });
Expand Down Expand Up @@ -95,6 +97,7 @@ export async function adjust(event: PoolEvent): Promise<void> {
ssmTokenPath,
ssmConfigPath,
ssmParameterStoreTags,
runnerConfigStorage,
},
numberOfRunners: topUp,
githubInstallationClient,
Expand Down
46 changes: 21 additions & 25 deletions lambdas/functions/control-plane/src/scale-runners/github-runner.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
import { createChildLogger } from '@aws-github-runner/aws-powertools-util';
import { getParameter, putParameter } from '@aws-github-runner/aws-ssm-util';
import { Octokit } from '@octokit/rest';

import { metricGitHubAppRateLimit } from '../github/rate-limit';
import { createRunnerConfigStore } from './runner-config-storage';
import { ActionRequestMessage, CreateGitHubRunnerConfig, EphemeralRunnerConfig, RunnerGroup } from './types';

const logger = createChildLogger('github-runner');
Expand Down Expand Up @@ -171,34 +171,28 @@ export async function getRunnerGroupId(
let runnerGroupId: number | undefined = 1;
if (githubRunnerConfig.runnerType === 'Org' && githubRunnerConfig.runnerGroup !== undefined) {
let runnerGroup: string | undefined;
// check if runner group id is already stored in SSM Parameter Store and
// use it if it exists to avoid API call to GitHub
const runnerConfigStore = createRunnerConfigStore(githubRunnerConfig);
const runnerGroupCacheKey = `runner-group/${githubRunnerConfig.runnerGroup}`;
// check if runner group id is already cached and use it if it exists to
// avoid an API call to GitHub
try {
runnerGroup = await getParameter(
`${githubRunnerConfig.ssmConfigPath}/runner-group/${githubRunnerConfig.runnerGroup}`,
);
runnerGroup = await runnerConfigStore.getConfigValue(runnerGroupCacheKey);
} catch (err) {
logger.debug('Handling error:', err as Error);
logger.warn(
`SSM Parameter "${githubRunnerConfig.ssmConfigPath}/runner-group/${githubRunnerConfig.runnerGroup}"
for Runner group ${githubRunnerConfig.runnerGroup} does not exist`,
`Runner group cache entry "${runnerGroupCacheKey}" for Runner group ${githubRunnerConfig.runnerGroup} does not exist`,
);
}
if (runnerGroup === undefined) {
// get runner group id from GitHub
runnerGroupId = await getRunnerGroupByName(ghClient, githubRunnerConfig);
// store runner group id in SSM
// store runner group id in the configured runner config storage
try {
await putParameter(
`${githubRunnerConfig.ssmConfigPath}/runner-group/${githubRunnerConfig.runnerGroup}`,
runnerGroupId.toString(),
false,
{
tags: githubRunnerConfig.ssmParameterStoreTags,
},
);
await runnerConfigStore.putConfigValue(runnerGroupCacheKey, runnerGroupId.toString(), {
tags: githubRunnerConfig.ssmParameterStoreTags,
});
} catch (err) {
logger.debug('Error storing runner group id in SSM Parameter Store', err as Error);
logger.debug('Error storing runner group id', err as Error);
throw err;
}
} else {
Expand Down Expand Up @@ -241,10 +235,10 @@ export async function createStartRunnerConfig(
}
}

function addDelay(runnerIds: string[]) {
function addDelay(runnerIds: string[], enabled = true) {
const delay = async (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
const ssmParameterStoreMaxThroughput = 40;
const isDelay = runnerIds.length >= ssmParameterStoreMaxThroughput;
const isDelay = enabled && runnerIds.length >= ssmParameterStoreMaxThroughput;
return { isDelay, delay };
}

Expand All @@ -259,7 +253,8 @@ async function createRegistrationTokenConfig(
ghClient: Octokit,
options: StartRunnerConfigOptions,
): Promise<string[]> {
const { isDelay, delay } = addDelay(runnerIds);
const runnerConfigStore = createRunnerConfigStore(githubRunnerConfig);
const { isDelay, delay } = addDelay(runnerIds, runnerConfigStore.delayWritesForSsmThroughput);
const token = await getGithubRunnerRegistrationToken(githubRunnerConfig, ghClient);
const runnerServiceConfig = generateRunnerServiceConfig(githubRunnerConfig, token);

Expand All @@ -268,7 +263,7 @@ async function createRegistrationTokenConfig(
});

for (const runnerId of runnerIds) {
await putParameter(`${githubRunnerConfig.ssmTokenPath}/${runnerId}`, runnerServiceConfig.join(' '), true, {
await runnerConfigStore.putRunnerConfig(runnerId, runnerServiceConfig.join(' '), {
tags: [...(options.getSsmParameterTags?.(runnerId) ?? []), ...githubRunnerConfig.ssmParameterStoreTags],
});
if (isDelay) {
Expand All @@ -293,7 +288,8 @@ async function createJitConfig(
options: StartRunnerConfigOptions,
): Promise<string[]> {
const runnerGroupId = await getRunnerGroupId(githubRunnerConfig, ghClient);
const { isDelay, delay } = addDelay(runnerIds);
const runnerConfigStore = createRunnerConfigStore(githubRunnerConfig);
const { isDelay, delay } = addDelay(runnerIds, runnerConfigStore.delayWritesForSsmThroughput);
const runnerLabels = githubRunnerConfig.runnerLabels.split(',');
const failedRunnerIds: string[] = [];

Expand Down Expand Up @@ -331,11 +327,11 @@ async function createJitConfig(
runnerLabels,
});

// store jit config in ssm parameter store
// store jit config in the configured runner config storage
logger.debug('Runner JIT config for ephemeral runner generated.', {
instance: runnerId,
});
await putParameter(`${githubRunnerConfig.ssmTokenPath}/${runnerId}`, runnerConfig.data.encoded_jit_config, true, {
await runnerConfigStore.putRunnerConfig(runnerId, runnerConfig.data.encoded_jit_config, {
tags: [...(options.getSsmParameterTags?.(runnerId) ?? []), ...githubRunnerConfig.ssmParameterStoreTags],
});
if (isDelay) {
Expand Down
Loading
Loading