fix(file-service): restore configured S3 uploads

infrastructure/helm/file-service/values.yaml

@@ -53,22 +53,12 @@ serviceAccount:
 # Non-sensitive service configuration rendered into {{ .Release.Name }}-config
 # and injected via envFrom. Populate at deploy time (e.g. from Terraform outputs).
 config:
-  # Permanently force every upload to fail (file-service writes to a nonexistent
-  # S3 bucket and returns 5xx). OFF on the golden app (`main`); ON here because
-  # this is the demo-coggtm variant branch, whose whole point is a broken upload
-  # path that survives Redis restarts and TTL expiry. What actually reaches the
-  # tenant is the branch's file-service *image*, which sets the variable in its
-  # Dockerfile -- the deploy runner renders charts from its own bundled tree, so
-  # this value only applies when someone deploys from this checkout -- and then
-  # it is not inert: deploy-tenant.sh renders without --reuse-values, so it also
-  # discards a manual --set-string of false. Kept in agreement with the image so
-  # the two can never contradict each other. To turn the failure off for this
-  # tenant without reverting the branch, set it explicitly (an explicit value
-  # beats the image's, until the next redeploy re-renders the chart):
+  # Disabled by default. Enable only as an explicit per-tenant override when
+  # reproducing an upload failure:
   #   helm upgrade file-service infrastructure/helm/file-service \
-  #     -n otterworks-coggtm --reuse-values \
-  #     --set-string config.FILE_UPLOAD_ALWAYS_FAIL=false
-  FILE_UPLOAD_ALWAYS_FAIL: "true"
+  #     -n otterworks-<ID> --reuse-values \
+  #     --set-string config.FILE_UPLOAD_ALWAYS_FAIL=true
+  FILE_UPLOAD_ALWAYS_FAIL: "false"
   # Same demo-variant convention as above (values kept in agreement with the
   # image ENVs): every Share click publishes file_shared to a nonexistent SNS
   # topic (real AWS NotFound -> red banner + incident + Slack alert), and users

scripts/bug-catalog.yaml

@@ -58,16 +58,12 @@ scenarios:
     service: file-service
     set: "config.FILE_UPLOAD_ALWAYS_FAIL=true"
     description: >
-      Permanent version of file-upload-fails: the FILE_UPLOAD_ALWAYS_FAIL env
-      var makes file-service route every upload to a nonexistent S3 bucket, so
-      uploads keep returning 5xx across Redis restarts and TTL expiry (the chaos
-      flag is transient). Downloads, listing, folders and sharing are unaffected,
-      and no metadata row is written for the failed upload. Already ON for this
-      tenant: the demo-coggtm branch bakes the variable into the file-service
-      image's Dockerfile, which is what the tenant actually runs (the runner
-      renders charts from its own bundled tree, so a chart value on the branch
-      may never reach the pod). Turning it off therefore means setting the value
-      to false explicitly -- the pod template's ConfigMap checksum rolls the pod.
+      Explicitly setting FILE_UPLOAD_ALWAYS_FAIL makes file-service route every
+      upload to a nonexistent S3 bucket, so uploads keep returning 5xx across
+      Redis restarts and TTL expiry (the Redis flag is transient). Downloads,
+      listing, folders and sharing are unaffected, and no metadata row is written
+      for the failed upload. The image and chart default to false; set this value
+      to false to restore uploads, and the ConfigMap checksum will roll the pod.
 
   notification-queue-fail:
     mechanism: config

scripts/inject-bug.sh

@@ -85,15 +85,11 @@ if [ "${SCENARIO}" = "file-upload-always-fails" ]; then
   log "Injecting config bug 'file-upload-always-fails' (file-service uploads always 5xx)..."
   helm upgrade file-service "${REPO_ROOT}/infrastructure/helm/file-service" -n "${NS}" --reuse-values \
     --set-string config.FILE_UPLOAD_ALWAYS_FAIL=true
-  log "Applied. A value change rolls the pod on its own (ConfigMap checksum annotation);"
-  log "if the env was already true nothing restarts and the tenant is already failing"
-  log "uploads -- tenant coggtm is in that state, because the demo-coggtm branch bakes"
-  log "the variable into the file-service image itself. Turn it off with:"
+  log "Applied. A value change rolls the pod on its own (ConfigMap checksum annotation)."
+  log "Turn it off with:"
   log "  helm upgrade file-service infrastructure/helm/file-service -n ${NS} --reuse-values --set-string config.FILE_UPLOAD_ALWAYS_FAIL=false"
-  log "(whether that false survives a redeploy depends on which tree renders the chart:"
-  log "deploy-tenant.sh from this branch drops it and re-applies the chart's true, while"
-  log "the ops-dashboard runner's bundled tree sets no such key, so the image default"
-  log "true applies again. Either way a redeploy re-enables the failure.)"
+  log "The image and chart default to false; uploads remain healthy unless the release"
+  log "explicitly overrides FILE_UPLOAD_ALWAYS_FAIL to true."
   exit 0
 fi
 

services/file-service/Dockerfile

@@ -16,18 +16,7 @@ COPY --from=builder /app/target/release/file-service /app/file-service
 RUN useradd -r -u 1001 appuser
 USER appuser
 
-# demo-coggtm tenant variant -- NEVER merge this to `main`. The tenant runs the
-# image this branch builds (tag `tenant-coggtm`), so baking the switch in here is
-# the only way to make its uploads fail permanently: it survives pod restarts,
-# Redis restarts, TTL expiry and idle-suspend/wake, and needs no chart override:
-# the deploy runner renders charts from its own bundled tree, whose file-service
-# values are `config: {}` and whose build_helm_args sets a fixed list of
-# config.* keys that does not include this one, so nothing overrides the image.
-# Anything that does set the variable explicitly still wins -- docker-compose.yml
-# passes false, so local stacks behave like the golden app.
-ENV FILE_UPLOAD_ALWAYS_FAIL=true
-
-# Click-triggered notification failure (same bake rationale as above): sharing
+# Click-triggered notification failure: sharing
 # a file publishes the file_shared event to a nonexistent SNS topic, so every
 # share click fails with a real AWS SNS error and fires the
 # NotificationEventPublishFailure alert at admin-service (incident + Devin
@@ -37,9 +26,8 @@ ENV FILE_UPLOAD_ALWAYS_FAIL=true
 # stacks behave like the golden app.
 ENV FILE_SHARE_EVENT_ALWAYS_FAIL=true
 
-# With uploads failing permanently (above), users would have no files to share.
 # Seeding gives every user a few demo documents on their first file listing,
-# so the share-notification failure is demoable alongside the upload failure.
+# so the share flow is demoable on an account that has uploaded nothing yet.
 # docker-compose.yml passes false, so local stacks behave like the golden app.
 ENV FILE_SEED_DEMO_DOCS=true
 

services/file-service/src/config.rs

@@ -155,4 +155,22 @@ mod tests {
         }
         assert!(!super::ServerConfig::from_env().upload_always_fail);
     }
+
+    #[test]
+    fn production_deployments_do_not_force_upload_failures() {
+        let upload_failure_enabled = [
+            include_str!("../Dockerfile"),
+            include_str!("../../../infrastructure/helm/file-service/values.yaml"),
+        ]
+        .iter()
+        .flat_map(|contents| contents.lines())
+        .any(|line| {
+            let line = line.trim();
+            line.strip_prefix("ENV FILE_UPLOAD_ALWAYS_FAIL=")
+                .or_else(|| line.strip_prefix("FILE_UPLOAD_ALWAYS_FAIL:"))
+                .is_some_and(|value| parse_bool(value.trim().trim_matches('"'), false))
+        });
+
+        assert!(!upload_failure_enabled);
+    }
 }