Adds scaleUpHealing cron

terraform-aws-github-runner/main.tf

@@ -104,6 +104,7 @@ module "runners" {
   environment          = var.environment
   tags                 = local.tags
 
+  scale_config_org       = var.scale_config_org
   scale_config_repo      = var.scale_config_repo
   scale_config_repo_path = var.scale_config_repo_path
 

terraform-aws-github-runner/modules/runners/lambdas/runners/jest.config.js

@@ -11,5 +11,8 @@ module.exports = {
       lines: 93,
       statements: 94
     }
-  }
+  },
+  moduleNameMapper: {
+    axios: 'axios/dist/node/axios.cjs', // Allow axios to work in tests
+  },
 };

terraform-aws-github-runner/modules/runners/lambdas/runners/package.json

@@ -42,6 +42,7 @@
     "@types/uuid": "^9.0.1",
     "async-mutex": "^0.4.0",
     "aws-sdk": "^2.863.0",
+    "axios": "^1.7.7",
     "cron-parser": "^3.3.0",
     "generic-pool": "^3.9.0",
     "lru-cache": "^6.0.0",

terraform-aws-github-runner/modules/runners/lambdas/runners/src/lambda.test.ts

@@ -1,11 +1,12 @@
-import { scaleDown as scaleDownL, scaleUp as scaleUpL } from './lambda';
+import { scaleDown as scaleDownL, scaleUp as scaleUpL, scaleUpChron as scaleUpChronL } from './lambda';
 
 import nock from 'nock';
 import { Config } from './scale-runners/config';
 import { Context, SQSEvent, ScheduledEvent } from 'aws-lambda';
 import { mocked } from 'ts-jest/utils';
 import { scaleDown } from './scale-runners/scale-down';
 import { scaleUp, RetryableScalingError } from './scale-runners/scale-up';
+import { scaleUpChron } from './scale-runners/scale-up-chron';
 import { sqsSendMessages, sqsDeleteMessageBatch } from './scale-runners/sqs';
 import * as MetricsModule from './scale-runners/metrics';
 
@@ -21,8 +22,10 @@ jest.mock('aws-sdk', () => ({
 jest.mock('./scale-runners/scale-down');
 jest.mock('./scale-runners/scale-up');
 jest.mock('./scale-runners/sqs');
+jest.mock('./scale-runners/scale-up-chron');
 
-const metrics = new MetricsModule.ScaleUpMetrics();
+const mockScaleUpMetrics = new MetricsModule.ScaleUpMetrics();
+const mockScaleUpChronMetrics = new MetricsModule.ScaleUpChronMetrics();
 
 beforeEach(() => {
   jest.resetModules();
@@ -34,7 +37,7 @@ beforeEach(() => {
 describe('scaleUp', () => {
   beforeEach(() => {
     jest.spyOn(global.Math, 'random').mockReturnValue(1.0);
-    jest.spyOn(MetricsModule, 'ScaleUpMetrics').mockReturnValue(metrics);
+    jest.spyOn(MetricsModule, 'ScaleUpMetrics').mockReturnValue(mockScaleUpMetrics);
   });
 
   afterEach(() => {
@@ -55,8 +58,8 @@ describe('scaleUp', () => {
       callback,
     );
     expect(mockedScaleUp).toBeCalledTimes(2);
-    expect(mockedScaleUp).toBeCalledWith('aws:sqs', { id: 1 }, metrics);
-    expect(mockedScaleUp).toBeCalledWith('aws:sqs', { id: 2 }, metrics);
+    expect(mockedScaleUp).toBeCalledWith('aws:sqs', { id: 1 }, mockScaleUpMetrics);
+    expect(mockedScaleUp).toBeCalledWith('aws:sqs', { id: 2 }, mockScaleUpMetrics);
     expect(callback).toBeCalledTimes(1);
     expect(callback).toBeCalledWith(null);
   });
@@ -88,12 +91,12 @@ describe('scaleUp', () => {
       callback,
     );
     expect(mockedScaleUp).toBeCalledTimes(1);
-    expect(mockedScaleUp).toBeCalledWith('aws:sqs', { id: 1 }, metrics);
+    expect(mockedScaleUp).toBeCalledWith('aws:sqs', { id: 1 }, mockScaleUpMetrics);
     expect(callback).toBeCalledTimes(1);
     expect(callback).toBeCalledWith('Failed handling SQS event');
 
     expect(sqsDeleteMessageBatch).toBeCalledTimes(1);
-    expect(sqsDeleteMessageBatch).toBeCalledWith(metrics, evts);
+    expect(sqsDeleteMessageBatch).toBeCalledWith(mockScaleUpMetrics, evts);
   });
 
   it('stochasticOvershoot when retryCount > 5', async () => {
@@ -137,7 +140,7 @@ describe('scaleUp', () => {
       },
     ];
     expect(sqsSendMessages).toBeCalledTimes(1);
-    expect(sqsSendMessages).toBeCalledWith(metrics, expected, 'asdf');
+    expect(sqsSendMessages).toBeCalledWith(mockScaleUpMetrics, expected, 'asdf');
 
     expect(sqsDeleteMessageBatch).toBeCalledTimes(0);
   });
@@ -205,10 +208,10 @@ describe('scaleUp', () => {
       },
     ];
     expect(sqsSendMessages).toBeCalledTimes(1);
-    expect(sqsSendMessages).toBeCalledWith(metrics, expected, 'asdf');
+    expect(sqsSendMessages).toBeCalledWith(mockScaleUpMetrics, expected, 'asdf');
 
     expect(sqsDeleteMessageBatch).toBeCalledTimes(1);
-    expect(sqsDeleteMessageBatch).toBeCalledWith(metrics, records);
+    expect(sqsDeleteMessageBatch).toBeCalledWith(mockScaleUpMetrics, records);
   });
 });
 
@@ -231,3 +234,28 @@ describe('scaleDown', () => {
     expect(callback).toBeCalledWith('Failed');
   });
 });
+
+describe('scaleUpChron', () => {
+  beforeEach(() => {
+    jest.spyOn(MetricsModule, 'ScaleUpChronMetrics').mockReturnValue(mockScaleUpChronMetrics);
+  });
+
+  it('succeeds', async () => {
+    const mockedScaleUpChron = mocked(scaleUpChron).mockResolvedValue(undefined);
+    const callback = jest.fn();
+    await scaleUpChronL({} as unknown as ScheduledEvent, {} as unknown as Context, callback);
+    expect(mockedScaleUpChron).toBeCalledTimes(1);
+    expect(mockedScaleUpChron).toBeCalledWith(mockScaleUpChronMetrics);
+    expect(callback).toBeCalledTimes(1);
+    expect(callback).toBeCalledWith(null);
+  });
+
+  it('fails', async () => {
+    const mockedScaleUpChron = mocked(scaleUpChron).mockRejectedValue(Error('error'));
+    const callback = jest.fn();
+    await scaleUpChronL({} as unknown as ScheduledEvent, {} as unknown as Context, callback);
+    expect(mockedScaleUpChron).toBeCalledTimes(1);
+    expect(callback).toBeCalledTimes(1);
+    expect(callback).toBeCalledWith('Failed');
+  });
+});

terraform-aws-github-runner/modules/runners/lambdas/runners/src/lambda.ts

@@ -2,9 +2,15 @@ import { ActionRequestMessage, RetryableScalingError, scaleUp as scaleUpR } from
 import { Context, SQSEvent, SQSRecord, ScheduledEvent } from 'aws-lambda';
 
 import { Config } from './scale-runners/config';
-import { ScaleUpMetrics, sendMetricsAtTimeout, sendMetricsTimeoutVars } from './scale-runners/metrics';
+import {
+  ScaleUpMetrics,
+  ScaleUpChronMetrics,
+  sendMetricsAtTimeout,
+  sendMetricsTimeoutVars,
+} from './scale-runners/metrics';
 import { getDelayWithJitterRetryCount, stochaticRunOvershoot } from './scale-runners/utils';
 import { scaleDown as scaleDownR } from './scale-runners/scale-down';
+import { scaleUpChron as scaleUpChronR } from './scale-runners/scale-up-chron';
 import { sqsSendMessages, sqsDeleteMessageBatch } from './scale-runners/sqs';
 
 async function sendRetryEvents(evtFailed: Array<[SQSRecord, boolean, number]>, metrics: ScaleUpMetrics) {
@@ -155,3 +161,35 @@ export async function scaleDown(event: ScheduledEvent, context: Context, callbac
     return callback('Failed');
   }
 }
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export async function scaleUpChron(event: ScheduledEvent, context: Context, callback: any) {
+  // we mantain open connections to redis, so the event pool is only cleaned when the SIGTERM is sent
+  context.callbackWaitsForEmptyEventLoop = false;
+
+  const metrics = new ScaleUpChronMetrics();
+  const sndMetricsTimout: sendMetricsTimeoutVars = {
+    metrics: metrics,
+  };
+  sndMetricsTimout.setTimeout = setTimeout(
+    sendMetricsAtTimeout(sndMetricsTimout),
+    (Config.Instance.lambdaTimeout - 10) * 1000,
+  );
+
+  try {
+    await scaleUpChronR(metrics);
+    return callback(null);
+  } catch (e) {
+    console.error(e);
+    return callback('Failed');
+  } finally {
+    try {
+      clearTimeout(sndMetricsTimout.setTimeout);
+      sndMetricsTimout.metrics = undefined;
+      sndMetricsTimout.setTimeout = undefined;
+      await metrics.sendMetrics();
+    } catch (e) {
+      console.error(`Error sending metrics: ${e}`);
+    }
+  }
+}

terraform-aws-github-runner/modules/runners/lambdas/runners/src/scale-runners/config.ts

@@ -36,8 +36,11 @@ export class Config {
   readonly retryScaleUpRecordQueueUrl: string | undefined;
   readonly runnerGroupName: string | undefined;
   readonly runnersExtraLabels: undefined | string;
+  readonly scaleConfigOrg: string;
   readonly scaleConfigRepo: string;
   readonly scaleConfigRepoPath: string;
+  readonly scaleUpMinQueueTimeMinutes: number;
+  readonly scaleUpCronRecordQueueUrl: string | undefined;
   readonly secretsManagerSecretsId: string | undefined;
   readonly sSMParamCleanupAgeDays: number;
   readonly sSMParamMaxCleanupAllowance: number;
@@ -93,9 +96,14 @@ export class Config {
     this.retryScaleUpRecordDelayS = Number(process.env.RETRY_SCALE_UP_RECORD_DELAY_S || '0');
     /* istanbul ignore next */
     this.retryScaleUpRecordJitterPct = Number(process.env.RETRY_SCALE_UP_RECORD_JITTER_PCT || '0');
-    this.retryScaleUpRecordQueueUrl = process.env.RETRY_SCALE_UP_RECORD_QUEUE_URL;
+    this.retryScaleUpRecordQueueUrl = process.env.RETRY_SCALE_UP_CRON_RECORD_QUEUE_URL;
+    this.scaleUpCronRecordQueueUrl = process.env.SCALE_UP_CRON_HUD_QUERY_URL;
+    this.scaleUpMinQueueTimeMinutes = process.env.SCALE_UP_MIN_QUEUE_TIME_MINUTES
+      ? Number(process.env.SCALE_UP_MIN_QUEUE_TIME_MINUTES)
+      : 30;
     this.runnerGroupName = process.env.RUNNER_GROUP_NAME;
     this.runnersExtraLabels = process.env.RUNNER_EXTRA_LABELS;
+    this.scaleConfigOrg = process.env.SCALE_CONFIG_ORG || '';
     /* istanbul ignore next */
     this.scaleConfigRepo = process.env.SCALE_CONFIG_REPO || '';
     if (this.enableOrganizationRunners && !this.scaleConfigRepo) {

terraform-aws-github-runner/modules/runners/lambdas/runners/src/scale-runners/metrics.ts

@@ -913,8 +913,8 @@ export class Metrics {
 }
 
 export class ScaleUpMetrics extends Metrics {
-  constructor() {
-    super('scaleUp');
+  constructor(lambdaName: string | undefined = undefined) {
+    super(lambdaName || 'scaleUp');
   }
 
   /* istanbul ignore next */
@@ -1630,6 +1630,64 @@ export class ScaleDownMetrics extends Metrics {
   }
 }
 
+export class ScaleUpChronMetrics extends ScaleUpMetrics {
+  constructor() {
+    super('scaleUpChron');
+  }
+
+  queuedRunnerStats(org: string, runnerType: string, numQueuedJobs: number) {
+    const dimensions = new Map([
+      ['Org', org],
+      ['RunnerType', runnerType],
+      ['numQueuedJobs', numQueuedJobs.toString()],
+    ]);
+    this.addEntry('gh.scaleupchron.queuedRunners', 3, dimensions);
+  }
+
+  queuedRunnerFailure(error: string) {
+    const dimensions = new Map([['error', error]]);
+    this.countEntry('gh.scaleupchron.queuedRunners.failure', 1, dimensions);
+  }
+
+  /* istanbul ignore next */
+  getQueuedJobsEndpointSuccess(ms: number) {
+    this.countEntry(`gh.calls.total`, 1);
+    this.countEntry(`gh.calls.getQueuedJobsEndpoint.count`, 1);
+    this.countEntry(`gh.calls.getQueuedJobsEndpoint.success`, 1);
+    this.addEntry(`gh.calls.getQueuedJobsEndpoint.wallclock`, ms);
+  }
+
+  /* istanbul ignore next */
+  getQueuedJobsEndpointFailure(ms: number) {
+    this.countEntry(`gh.calls.total`, 1);
+    this.countEntry(`gh.calls.getQueuedJobsEndpoint.count`, 1);
+    this.countEntry(`gh.calls.getQueuedJobsEndpoint.failure`, 1);
+    this.addEntry(`gh.calls.getQueuedJobsEndpoint.wallclock`, ms);
+  }
+
+  scaleUpInstanceSuccess() {
+    this.scaleUpSuccess();
+    this.countEntry('run.scaleupchron.success');
+  }
+
+  scaleUpInstanceFailureNonRetryable(error: string) {
+    const dimensions = new Map([['error', error]]);
+    // should we add more information about this or do we not care since  it'll be requeued?
+    this.countEntry('run.scaleupchron.failure.nonRetryable', 1, dimensions);
+  }
+
+  scaleUpInstanceFailureRetryable(error: string) {
+    const dimensions = new Map([['error', error]]);
+
+    // should we add more information about this or do we not care since  it'll be requeued?
+    this.countEntry('run.scaleupchron.failure.retryable', 1, dimensions);
+  }
+
+  scaleUpInstanceNoOp() {
+    this.countEntry('run.scaleupchron.noop');
+  }
+}
+
 export interface sendMetricsTimeoutVars {
   metrics?: Metrics;
   setTimeout?: ReturnType<typeof setTimeout>;