fix(aws): restore PAYG production routing

Use pay-as-you-go CloudFront, cap the worker fleet at 50, accept ALB hostnames at the TLS proxy, and prevent ineligible jobs from driving autoscaling.

Author: richiemcilroy

.github/workflows/deploy-aws.yml

@@ -268,14 +268,13 @@ jobs:
             aws ecs wait services-stable --cluster "$cluster" --services "$service"
           done
 
-      - name: Verify public health and flat-rate gate
+      - name: Verify public media pipeline
         if: ${{ inputs.apply && inputs.services_enabled }}
         working-directory: infra/aws/platform
         env:
           REND_READINESS_API_KEY: ${{ secrets.REND_READINESS_API_KEY }}
         run: |
           set -euo pipefail
-          test "$(terraform output -raw cloudfront_flat_rate_plan_verified)" = true
           api_url="$(terraform output -raw api_url)"
           playback_url="$(terraform output -raw playback_url)"
           curl --fail --silent --show-error "$api_url/readyz" >/dev/null

docs/deployment-aws.md

@@ -170,14 +170,11 @@ must activate this tag when Rend runs in a linked account. Service, worker,
 storage, and organization hard ceilings remain active while the Budget is
 pending; set the flag true and apply again as soon as the tag is active.
 
-Production planning is also gated by
-`cloudfront_flat_rate_plan_verified = true`. Set it only after AWS shows the
-Business flat-rate plan active in account `211125561119`; VPC Origins require
-Business or Premium. Terraform cannot activate the commercial plan itself. A
-brand-new distribution therefore has one explicit bootstrap apply: set
-`cloudfront_flat_rate_plan_bootstrap = true`, apply, subscribe the distribution
-to Business in the CloudFront console, then immediately set bootstrap false and
-verification true before the next apply.
+CloudFront remains on pay-as-you-go pricing. Playback is authenticated and WAF
+protected, while cost and abuse exposure is controlled by server-side quotas,
+queue admission, and hard ECS autoscaling ceilings. Production keeps one media
+worker warm and can run at most 50 workers globally; no quota or infrastructure
+limit needs to be rendered in the dashboard.
 
 The source and media buckets remain external to AWS but their full contract is
 owned by Terraform. During apply, an idempotent reconciler reads Tigris

infra/aws/README.md

@@ -132,14 +132,9 @@ Before the first platform apply:
 
    Delete local plaintext copies after storing the secret and your encrypted
    recovery copy.
-4. Activate the CloudFront Business flat-rate plan in account `211125561119`,
-   then set `cloudfront_flat_rate_plan_verified = true`. VPC Origins require at
-   least Business. Terraform blocks production while verification is false.
-   For the first distribution only, set
-   `cloudfront_flat_rate_plan_bootstrap = true` for one apply, subscribe the new
-   distribution immediately in the CloudFront console, then set bootstrap back
-   to false and verification to true. This escape hatch avoids an IaC/console
-   creation deadlock and must never remain enabled.
+4. Keep CloudFront on pay-as-you-go pricing. Rend controls exposure through
+   authenticated playback, WAF rules, backend quotas, and hard ECS autoscaling
+   ceilings rather than a fixed monthly CloudFront plan.
 5. Obtain the production PlanetScale PrivateLink endpoint service name, set it
    in tfvars, and ensure the TLS database URL uses PlanetScale's private DNS.
 6. Add the two Route53 delegation record sets emitted by bootstrap at the
@@ -153,11 +148,11 @@ Before the first platform apply:
    `Application=rend`; an account-wide budget is intentionally forbidden in
    this shared account.
 
-The Business plan is $200/month with no traffic overage charges. The remaining
-default cost envelope is guarded by a $400 monthly AWS alert budget, 50 videos
-and 10 open uploads per organization, two active media jobs per organization,
-and at most ten 4-vCPU/8-GiB workers. ClickHouse starts with 100 GiB encrypted
-gp3.
+CloudFront is pay-as-you-go. The default cost envelope is guarded by a $400
+monthly AWS alert budget, 50 videos and 10 open uploads per organization, two
+active media jobs per organization, and at most fifty 4-vCPU/8-GiB workers.
+Only one worker is kept warm; the fleet scales from queue demand. ClickHouse
+starts with 100 GiB encrypted gp3.
 
 ClickHouse has two independent nightly recovery layers: a native database
 backup copied to a versioned KMS-encrypted S3 bucket at 02:15 UTC and an AWS

infra/aws/environments/production.tfvars.example

@@ -65,15 +65,9 @@ api_max_tasks    = 6
 edge_min_tasks   = 2
 edge_max_tasks   = 6
 worker_min_tasks = 1
-worker_max_tasks = 10
+worker_max_tasks = 50
 
 cloudfront_price_class = "PriceClass_All"
-# VPC Origins require Business or Premium. For a brand-new distribution only,
-# set bootstrap=true for its first apply, immediately subscribe it to Business
-# in the CloudFront console, then set verified=true and bootstrap=false.
-cloudfront_flat_rate_plan_tier     = "Business"
-cloudfront_flat_rate_plan_bootstrap = false
-cloudfront_flat_rate_plan_verified  = false
 monthly_budget_usd                  = 400
 # Linked accounts cannot activate this tag themselves. Leave false for the
 # initial migration, then enable it after the Organizations management account

infra/aws/platform/compute.tf

@@ -209,8 +209,7 @@ resource "aws_ecs_task_definition" "api" {
       essential  = true
       entryPoint = ["/bin/sh", "-c"]
       command = [<<-EOT
-        proxy_ip="$(hostname -i | awk '{print $1}')"
-        printf '{\n  auto_https disable_redirects\n  admin off\n}\nhttps://%s:8443 {\n  tls internal\n  reverse_proxy 127.0.0.1:4000\n}\n' "$proxy_ip" >/tmp/Caddyfile
+        printf '{\n  auto_https disable_redirects\n  admin off\n}\nhttps://:8443 {\n  tls internal\n  reverse_proxy 127.0.0.1:4000\n}\n' >/tmp/Caddyfile
         exec caddy run --config /tmp/Caddyfile --adapter caddyfile
       EOT
       ]
@@ -352,8 +351,7 @@ resource "aws_ecs_task_definition" "edge" {
       essential  = true
       entryPoint = ["/bin/sh", "-c"]
       command = [<<-EOT
-        proxy_ip="$(hostname -i | awk '{print $1}')"
-        printf '{\n  auto_https disable_redirects\n  admin off\n}\nhttps://%s:8443 {\n  tls internal\n  reverse_proxy 127.0.0.1:4100\n}\n' "$proxy_ip" >/tmp/Caddyfile
+        printf '{\n  auto_https disable_redirects\n  admin off\n}\nhttps://:8443 {\n  tls internal\n  reverse_proxy 127.0.0.1:4100\n}\n' >/tmp/Caddyfile
         exec caddy run --config /tmp/Caddyfile --adapter caddyfile
       EOT
       ]

infra/aws/platform/delivery.tf

@@ -152,21 +152,6 @@ resource "aws_cloudfront_key_group" "playback" {
   items   = [aws_cloudfront_public_key.playback.id]
 }
 
-resource "terraform_data" "cloudfront_flat_rate_plan_guard" {
-  input = {
-    bootstrap = var.cloudfront_flat_rate_plan_bootstrap
-    tier      = var.cloudfront_flat_rate_plan_tier
-    verified  = var.cloudfront_flat_rate_plan_verified
-  }
-
-  lifecycle {
-    precondition {
-      condition     = var.cloudfront_flat_rate_plan_verified || var.cloudfront_flat_rate_plan_bootstrap
-      error_message = "Production is blocked until the CloudFront flat-rate plan is active in account 211125561119. For the first distribution apply only, explicitly set cloudfront_flat_rate_plan_bootstrap=true, subscribe immediately, then set verified=true and bootstrap=false."
-    }
-  }
-}
-
 resource "aws_cloudfront_response_headers_policy" "public" {
   name = "${local.resource_prefix}-security"
 
@@ -499,7 +484,6 @@ resource "aws_cloudfront_distribution" "this" {
   depends_on = [
     aws_s3_bucket_acl.cloudfront_logs,
     aws_s3_bucket_policy.cloudfront_logs,
-    terraform_data.cloudfront_flat_rate_plan_guard,
   ]
 }
 

infra/aws/platform/outputs.tf

@@ -11,14 +11,6 @@ output "cloudfront_playback_key_pair_id" {
   value       = aws_cloudfront_public_key.playback.id
 }
 
-output "cloudfront_flat_rate_plan_verified" {
-  value = var.cloudfront_flat_rate_plan_verified
-}
-
-output "cloudfront_flat_rate_plan_tier" {
-  value = var.cloudfront_flat_rate_plan_tier
-}
-
 output "api_url" {
   value = local.api_base_url
 }

infra/aws/platform/variables.tf

@@ -372,12 +372,12 @@ variable "worker_min_tasks" {
 
 variable "worker_max_tasks" {
   type    = number
-  default = 10
+  default = 50
 
 
   validation {
-    condition     = var.worker_max_tasks >= 1 && var.worker_max_tasks <= 10
-    error_message = "worker_max_tasks must be between 1 and the hard ceiling of 10."
+    condition     = var.worker_max_tasks >= 1 && var.worker_max_tasks <= 50
+    error_message = "worker_max_tasks must be between 1 and the hard global ceiling of 50."
   }
 }
 
@@ -425,29 +425,6 @@ variable "cloudfront_price_class" {
   }
 }
 
-variable "cloudfront_flat_rate_plan_verified" {
-  description = "Explicit production gate confirming the AWS account has an active CloudFront flat-rate plan."
-  type        = bool
-  default     = false
-}
-
-variable "cloudfront_flat_rate_plan_tier" {
-  description = "Selected CloudFront flat-rate tier. VPC Origins require Business or Premium."
-  type        = string
-  default     = "Business"
-
-  validation {
-    condition     = contains(["Business", "Premium"], var.cloudfront_flat_rate_plan_tier)
-    error_message = "The Rend VPC Origin requires the Business or Premium CloudFront flat-rate tier."
-  }
-}
-
-variable "cloudfront_flat_rate_plan_bootstrap" {
-  description = "One-apply escape hatch used only to create a new distribution before subscribing it to a plan in the CloudFront console."
-  type        = bool
-  default     = false
-}
-
 variable "tls_proxy_image" {
   description = "Pinned Caddy image used only for self-signed TLS between the internal ALB and ECS tasks."
   type        = string

services/rend-api/src/main.rs

@@ -1703,8 +1703,17 @@ async fn publish_media_queue_metrics(state: &AppState) -> Result<()> {
           GREATEST(count(DISTINCT locked_by) FILTER (
             WHERE status = 'running' AND lease_expires_at > now()
           ), 1)
-        FROM rend.media_jobs
-        WHERE job_type = 'process_media'
+        FROM rend.media_jobs job
+        JOIN rend.assets asset
+          ON asset.id = job.asset_id
+         AND asset.deleted_at IS NULL
+         AND asset.suspended_at IS NULL
+        JOIN rend_auth.organization org
+          ON org.id = asset.organization_id
+         AND org.suspended_at IS NULL
+        JOIN rend.organization_storage_usage quota
+          ON quota.organization_id = asset.organization_id
+        WHERE job.job_type = 'process_media'
         ",
     )
     .fetch_one(&state.db)