Throttle CRF to two concurrent deploys

src/commands/functions-delete.ts

@@ -90,6 +90,8 @@
     try {
       const fab = new fabricator.Fabricator({
         functionExecutor,
+        // Note: we don't need the temporary concurrency reduction of 2, because that quota limit is for deploys
+        runFunctionExecutor: functionExecutor,
         appEngineLocation,
         executor: new executor.QueueExecutor({}),
         sources: {},
@@ -99,7 +101,7 @@
       const summary = await fab.applyPlan(plan);
       await reporter.logAndTrackDeployStats(summary);
       reporter.printErrors(summary);
     } catch (err: any) {
       throw new FirebaseError("Failed to delete functions", {
         original: err as Error,
         exit: 1,

src/deploy/functions/release/fabricator.spec.ts

@@ -115,6 +115,7 @@
   const ctorArgs: fabricator.FabricatorArgs = {
     executor: new executor.InlineExecutor(),
     functionExecutor: new executor.InlineExecutor(),
+    runFunctionExecutor: new executor.InlineExecutor(),
     sources: {
       default: {
         sourceUrl: "https://example.com",
@@ -446,7 +447,7 @@
     it("handles topics that already exist", async () => {
       pubsub.createTopic.callsFake(() => {
         const err = new Error("Already exists");
         (err as any).status = 409;
         return Promise.reject(err);
       });
       gcfv2.createFunction.resolves({ name: "op", done: false });
@@ -521,7 +522,7 @@
       eventarc.createChannel.callsFake(({ name }) => {
         expect(name).to.equal("channel");
         const err = new Error("Already exists");
         (err as any).status = 409;
         return Promise.reject(err);
       });
       gcfv2.createFunction.resolves({ name: "op", done: false });
@@ -587,7 +588,7 @@
       eventarc.getChannel.resolves(undefined);
       eventarc.createChannel.callsFake(() => {
         const err = new Error("🤷‍♂️");
         (err as any).status = 400;
         return Promise.reject(err);
       });
 
@@ -1715,7 +1716,7 @@
 
   describe("createRunFunction", () => {
     it("creates a Cloud Run service with correct configuration", async () => {
       runv2.createService.resolves({ uri: "https://service", name: "service" } as any);
       run.setInvokerUpdate.resolves();
 
       const ep = endpoint(
@@ -1758,7 +1759,7 @@
 
   describe("updateRunFunction", () => {
     it("updates a Cloud Run service with correct configuration", async () => {
       runv2.updateService.resolves({ uri: "https://service", name: "service" } as any);
       run.setInvokerUpdate.resolves();
 
       const ep = endpoint(

src/deploy/functions/release/fabricator.ts

@@ -58,6 +58,7 @@ const CLOUD_RUN_RESOURCE_EXHAUSTED_CODE = 8;
 export interface FabricatorArgs {
   executor: Executor;
   functionExecutor: Executor;
+  runFunctionExecutor: Executor;
   appEngineLocation: string;
   sources: Record<string, args.Source>;
   projectNumber: string;
@@ -74,13 +75,15 @@ const rethrowAs =
 export class Fabricator {
   executor: Executor;
   functionExecutor: Executor;
+  runFunctionExecutor: Executor;
   sources: Record<string, args.Source>;
   appEngineLocation: string;
   projectNumber: string;
 
   constructor(args: FabricatorArgs) {
     this.executor = args.executor;
     this.functionExecutor = args.functionExecutor;
+    this.runFunctionExecutor = args.runFunctionExecutor;
     this.sources = args.sources;
     this.appEngineLocation = args.appEngineLocation;
     this.projectNumber = args.projectNumber;
@@ -634,7 +637,7 @@ export class Fabricator {
       },
     };
 
-    await this.executor
+    await this.runFunctionExecutor
       .run(async () => {
         const op = await runV2.createService(
           endpoint.project,
@@ -669,7 +672,7 @@ export class Fabricator {
       },
     };
 
-    await this.executor
+    await this.runFunctionExecutor
       .run(async () => {
         const op = await runV2.updateService(service);
         endpoint.uri = op.uri;
@@ -681,7 +684,7 @@ export class Fabricator {
   }
 
   async deleteRunFunction(endpoint: backend.Endpoint): Promise<void> {
-    await this.executor
+    await this.runFunctionExecutor
       .run(async () => {
         try {
           await runV2.deleteService(endpoint.project, endpoint.region, endpoint.id);

src/deploy/functions/release/index.ts

@@ -84,9 +84,18 @@ export async function release(
     maxBackoff: 100000,
   };
 
+  // N.B. THIS IS TEMPORARY
+  // This will limit concurrent deploys of run functions to two while zip deploy capacity
+  // is low.
+  const runThrottlerOptions = {
+    ...throttlerOptions,
+    concurrency: 2,
+  };
+
   const projectNumber = options.projectNumber || (await getProjectNumber(context.projectId));
   const fab = new fabricator.Fabricator({
     functionExecutor: new executor.QueueExecutor(throttlerOptions),
+    runFunctionExecutor: new executor.QueueExecutor(runThrottlerOptions),
     executor: new executor.QueueExecutor(throttlerOptions),
     sources: context.sources,
     appEngineLocation: getAppEngineLocation(context.firebaseConfig),