Adds scaleUpHealing cron

terraform-aws-github-runner/modules/runners/lambdas/runners/src/scale-runners/config.ts

@@ -40,7 +40,7 @@ export class Config {
   readonly scaleConfigRepo: string;
   readonly scaleConfigRepoPath: string;
   readonly scaleUpMinQueueTimeMinutes: number;
-  readonly scaleUpRecordQueueUrl: string | undefined;
+  readonly scaleUpCronRecordQueueUrl: string | undefined;
   readonly secretsManagerSecretsId: string | undefined;
   readonly sSMParamCleanupAgeDays: number;
   readonly sSMParamMaxCleanupAllowance: number;
@@ -96,8 +96,8 @@ export class Config {
     this.retryScaleUpRecordDelayS = Number(process.env.RETRY_SCALE_UP_RECORD_DELAY_S || '0');
     /* istanbul ignore next */
     this.retryScaleUpRecordJitterPct = Number(process.env.RETRY_SCALE_UP_RECORD_JITTER_PCT || '0');
-    this.retryScaleUpRecordQueueUrl = process.env.RETRY_SCALE_UP_RECORD_QUEUE_URL;
-    this.scaleUpRecordQueueUrl = process.env.SCALE_UP_RECORD_QUEUE_URL;
+    this.retryScaleUpRecordQueueUrl = process.env.RETRY_SCALE_UP_CRON_RECORD_QUEUE_URL;
+    this.scaleUpCronRecordQueueUrl = process.env.SCALE_UP_CRON_HUD_QUERY_URL;
     this.scaleUpMinQueueTimeMinutes = process.env.SCALE_UP_MIN_QUEUE_TIME_MINUTES
       ? Number(process.env.SCALE_UP_MIN_QUEUE_TIME_MINUTES)
       : 30;

terraform-aws-github-runner/modules/runners/lambdas/runners/src/scale-runners/metrics.ts

@@ -913,8 +913,8 @@ export class Metrics {
 }
 
 export class ScaleUpMetrics extends Metrics {
-  constructor() {
-    super('scaleUp');
+  constructor(lambdaName: string | undefined = undefined) {
+    super(lambdaName || 'scaleUp');
   }
 
   /* istanbul ignore next */
@@ -1632,8 +1632,9 @@ export class ScaleDownMetrics extends Metrics {
 
 export class ScaleUpChronMetrics extends ScaleUpMetrics {
   constructor() {
-    super();
+    super('scaleUpChron');
   }
+
   queuedRunnerStats(org: string, runnerType: string, numQueuedJobs: number) {
     const dimensions = new Map([
       ['Org', org],
@@ -1642,10 +1643,12 @@ export class ScaleUpChronMetrics extends ScaleUpMetrics {
     ]);
     this.addEntry('gh.scaleupchron.queuedRunners', 3, dimensions);
   }
+
   queuedRunnerFailure(error: string) {
     const dimensions = new Map([['error', error]]);
     this.countEntry('gh.scaleupchron.queuedRunners.failure', 1, dimensions);
   }
+
   /* istanbul ignore next */
   getQueuedJobsEndpointSuccess(ms: number) {
     this.countEntry(`gh.calls.total`, 1);
@@ -1666,17 +1669,20 @@ export class ScaleUpChronMetrics extends ScaleUpMetrics {
     this.scaleUpSuccess();
     this.countEntry('run.scaleupchron.success');
   }
+
   scaleUpInstanceFailureNonRetryable(error: string) {
     const dimensions = new Map([['error', error]]);
     // should we add more information about this or do we not care since  it'll be requeued?
     this.countEntry('run.scaleupchron.failure.nonRetryable', 1, dimensions);
   }
+
   scaleUpInstanceFailureRetryable(error: string) {
     const dimensions = new Map([['error', error]]);
 
     // should we add more information about this or do we not care since  it'll be requeued?
     this.countEntry('run.scaleupchron.failure.retryable', 1, dimensions);
   }
+
   scaleUpInstanceNoOp() {
     this.countEntry('run.scaleupchron.noop');
   }

terraform-aws-github-runner/modules/runners/lambdas/runners/src/scale-runners/scale-up-chron.test.ts

@@ -9,6 +9,7 @@ import { scaleUp } from './scale-up';
 
 import * as MetricsModule from './metrics';
 import { RunnerType } from './runners';
+import nock from 'nock';
 
 jest.mock('./runners');
 jest.mock('./gh-runners');
@@ -81,34 +82,36 @@ const runnerTypeInvalid = 'runner_type_invalid';
 const baseCfg = {
   scaleConfigOrg: 'test_org1',
   scaleUpMinQueueTimeMinutes: 30,
-  scaleUpRecordQueueUrl: 'url',
+  scaleUpCronRecordQueueUrl: 'url',
 } as unknown as Config;
 
 const metrics = new MetricsModule.ScaleUpChronMetrics();
 beforeEach(() => {
   jest.resetModules();
   jest.clearAllMocks();
   jest.restoreAllMocks();
+
+  nock.disableNetConnect();
 });
 
 describe('scaleUpChron', () => {
-  it('invalid scaleUpRecordQueueUrl', async () => {
+  it('invalid scaleUpCronRecordQueueUrl', async () => {
     const scaleUpChron = jest.requireActual('./scale-up-chron').scaleUpChron;
 
     jest.clearAllMocks();
     jest.spyOn(Config, 'Instance', 'get').mockImplementation(
       () =>
         ({
           ...baseCfg,
-          scaleUpRecordQueueUrl: null,
+          scaleUpCronRecordQueueUrl: null,
         } as unknown as Config),
     );
 
     mocked(getRepo).mockReturnValue({ owner: 'owner', repo: 'repo' });
     mocked(getRunnerTypes).mockResolvedValue(new Map([[runnerTypeValid, { is_ephemeral: false } as RunnerType]]));
 
     await expect(scaleUpChron(metrics)).rejects.toThrow(
-      new Error('scaleUpRecordQueueUrl is not set. Cannot send queued scale up requests'),
+      new Error('scaleUpCronRecordQueueUrl is not set. Cannot send queued scale up requests'),
     );
   });
 

terraform-aws-github-runner/modules/runners/lambdas/runners/src/scale-runners/scale-up-chron.ts

@@ -11,23 +11,23 @@ export async function scaleUpChron(metrics: ScaleUpChronMetrics): Promise<void>
   // 1. Queries for queued runners via HUD
   // 2. Polls scale-config to filter the list to ones that are self-hosted by this fleet and
   //    are ephemeral and nonephemeral
-  // 3. Sends a SQS request to the scale-up lambda to provision more of those instances
+  // 3. For each runner queued for longer than the minimum delay, try to scale it up
 
   const scaleConfigRepo = getRepo(Config.Instance.scaleConfigOrg, Config.Instance.scaleConfigRepo);
 
   const validRunnerTypes = await getRunnerTypes(scaleConfigRepo, metrics);
 
   const minAutoScaleupDelayMinutes = Config.Instance.scaleUpMinQueueTimeMinutes;
-  if (!Config.Instance.scaleUpRecordQueueUrl) {
+  if (!Config.Instance.scaleUpCronRecordQueueUrl) {
     metrics.scaleUpInstanceFailureNonRetryable(
-      'scaleUpRecordQueueUrl is not set. Cannot send queued scale up requests',
+      'scaleUpCronRecordQueueUrl is not set. Cannot send queued scale up requests',
     );
-    throw new Error('scaleUpRecordQueueUrl is not set. Cannot send queued scale up requests');
+    throw new Error('scaleUpCronRecordQueueUrl is not set. Cannot send queued scale up requests');
   }
-  const scaleUpRecordQueueUrl = Config.Instance.scaleUpRecordQueueUrl;
+  const scaleUpCronRecordQueueUrl = Config.Instance.scaleUpCronRecordQueueUrl;
   // Only proactively scale up the jobs that have been queued for longer than normal
   // Filter out the queued jobs that are do not correspond to a valid runner type
-  const queuedJobs = (await getQueuedJobs(metrics, scaleUpRecordQueueUrl))
+  const queuedJobs = (await getQueuedJobs(metrics, scaleUpCronRecordQueueUrl))
     .filter((runner) => {
       return (
         runner.min_queue_time_minutes >= minAutoScaleupDelayMinutes && runner.org === Config.Instance.scaleConfigOrg
@@ -76,12 +76,12 @@ interface QueuedJobsForRunner {
 
 export async function getQueuedJobs(
   metrics: ScaleUpChronMetrics,
-  scaleUpRecordQueueUrl: string,
+  scaleUpCronRecordQueueUrl: string,
 ): Promise<QueuedJobsForRunner[]> {
   // This function queries the HUD for queued runners
   // and returns a list of them
 
-  const url = scaleUpRecordQueueUrl;
+  const url = scaleUpCronRecordQueueUrl;
 
   try {
     const response = await expBackOff(() => {
@@ -93,9 +93,25 @@ export async function getQueuedJobs(
 
     // Map the response to the class
     if (response && response.data) {
-      return JSON.parse(response.data) as QueuedJobsForRunner[];
+      const data = JSON.parse(response.data);
+      // check if data is valid as QueuedJobsForRunner[]
+      if (data && Array.isArray(data)) {
+        return data.filter(
+          (runner) =>
+            runner.runner_label &&
+            runner.org &&
+            runner.repo &&
+            typeof runner.num_queued_jobs == 'number' &&
+            runner.num_queued_jobs > 0 &&
+            typeof runner.min_queue_time_minutes == 'number' &&
+            typeof runner.max_queue_time_minutes == 'number',
+        );
+      } else {
+        /* istanbul ignore next */
+        throw new Error(`Invalid data returned from axios get request with url: ${url} - Not an array`);
+      }
     } else {
-      throw new Error('No data returned from axios get request with url: ' + url);
+      throw new Error(`No data returned from axios get request with url: ${url}`);
     }
   } catch (error) {
     metrics.queuedRunnerFailure((error as Error).message);

terraform-aws-github-runner/modules/runners/scale-up-chron.tf

@@ -47,7 +47,7 @@ resource "aws_lambda_function" "scale_up_chron" {
       SCALE_CONFIG_REPO               = var.scale_config_repo
       SCALE_CONFIG_REPO_PATH          = var.scale_config_repo_path
       SCALE_UP_MIN_QUEUE_TIME_MINUTES = 30
-      SCALE_UP_RECORD_QUEUE_URL       = var.hud_query_url
+      SCALE_UP_CRON_HUD_QUERY_URL       = var.retry_scale_up_cron_hud_query_url
       scale_up_chron_CONFIG               = jsonencode(var.idle_config)
       SECRETSMANAGER_SECRETS_ID       = var.secretsmanager_secrets_id
       AWS_REGIONS_TO_VPC_IDS = join(

terraform-aws-github-runner/modules/runners/scale-up.tf

@@ -65,7 +65,7 @@ resource "aws_lambda_function" "scale_up" {
       REDIS_LOGIN                          = var.redis_login
       RETRY_SCALE_UP_RECORD_DELAY_S        = "60"
       RETRY_SCALE_UP_RECORD_JITTER_PCT     = "0.5"
-      RETRY_SCALE_UP_RECORD_QUEUE_URL      = var.sqs_build_queue_retry.url
+      RETRY_SCALE_UP_CRON_RECORD_QUEUE_URL      = var.sqs_build_queue_retry.url
       RUNNER_EXTRA_LABELS                  = var.runner_extra_labels
       SCALE_CONFIG_ORG                     = var.scale_config_org
       SCALE_CONFIG_REPO                    = var.scale_config_repo

terraform-aws-github-runner/modules/runners/variables.tf

@@ -121,7 +121,7 @@ variable "lambda_timeout_scale_up" {
 variable "lambda_timeout_scale_up_chron" {
   description = "Time out for the scale up chron lambda in seconds."
   type        = number
-  default     = 60
+  default     = 900
 }
 
 variable "role_permissions_boundary" {
@@ -319,7 +319,7 @@ variable "min_available_runners" {
   type        = number
 }
 
-variable "hud_query_url" {
+variable "retry_scale_up_cron_hud_query_url" {
   description = "URL used in scale-up-chron to query HUD for queued jobs"
   type        = string
   default     = ""