diff --git a/infrastructure/helm/file-service/values.yaml b/infrastructure/helm/file-service/values.yaml index 69d258e0a..add847f13 100644 --- a/infrastructure/helm/file-service/values.yaml +++ b/infrastructure/helm/file-service/values.yaml @@ -53,22 +53,12 @@ serviceAccount: # Non-sensitive service configuration rendered into {{ .Release.Name }}-config # and injected via envFrom. Populate at deploy time (e.g. from Terraform outputs). config: - # Permanently force every upload to fail (file-service writes to a nonexistent - # S3 bucket and returns 5xx). OFF on the golden app (`main`); ON here because - # this is the demo-coggtm variant branch, whose whole point is a broken upload - # path that survives Redis restarts and TTL expiry. What actually reaches the - # tenant is the branch's file-service *image*, which sets the variable in its - # Dockerfile -- the deploy runner renders charts from its own bundled tree, so - # this value only applies when someone deploys from this checkout -- and then - # it is not inert: deploy-tenant.sh renders without --reuse-values, so it also - # discards a manual --set-string of false. Kept in agreement with the image so - # the two can never contradict each other. To turn the failure off for this - # tenant without reverting the branch, set it explicitly (an explicit value - # beats the image's, until the next redeploy re-renders the chart): + # Disabled by default. Enable only as an explicit per-tenant override when + # reproducing an upload failure: # helm upgrade file-service infrastructure/helm/file-service \ - # -n otterworks-coggtm --reuse-values \ - # --set-string config.FILE_UPLOAD_ALWAYS_FAIL=false - FILE_UPLOAD_ALWAYS_FAIL: "true" + # -n otterworks- --reuse-values \ + # --set-string config.FILE_UPLOAD_ALWAYS_FAIL=true + FILE_UPLOAD_ALWAYS_FAIL: "false" # Same demo-variant convention as above (values kept in agreement with the # image ENVs): every Share click publishes file_shared to a nonexistent SNS # topic (real AWS NotFound -> red banner + incident + Slack alert), and users diff --git a/scripts/bug-catalog.yaml b/scripts/bug-catalog.yaml index c6d2cc1bf..032076049 100644 --- a/scripts/bug-catalog.yaml +++ b/scripts/bug-catalog.yaml @@ -58,16 +58,12 @@ scenarios: service: file-service set: "config.FILE_UPLOAD_ALWAYS_FAIL=true" description: > - Permanent version of file-upload-fails: the FILE_UPLOAD_ALWAYS_FAIL env - var makes file-service route every upload to a nonexistent S3 bucket, so - uploads keep returning 5xx across Redis restarts and TTL expiry (the chaos - flag is transient). Downloads, listing, folders and sharing are unaffected, - and no metadata row is written for the failed upload. Already ON for this - tenant: the demo-coggtm branch bakes the variable into the file-service - image's Dockerfile, which is what the tenant actually runs (the runner - renders charts from its own bundled tree, so a chart value on the branch - may never reach the pod). Turning it off therefore means setting the value - to false explicitly -- the pod template's ConfigMap checksum rolls the pod. + Explicitly setting FILE_UPLOAD_ALWAYS_FAIL makes file-service route every + upload to a nonexistent S3 bucket, so uploads keep returning 5xx across + Redis restarts and TTL expiry (the Redis flag is transient). Downloads, + listing, folders and sharing are unaffected, and no metadata row is written + for the failed upload. The image and chart default to false; set this value + to false to restore uploads, and the ConfigMap checksum will roll the pod. notification-queue-fail: mechanism: config diff --git a/scripts/inject-bug.sh b/scripts/inject-bug.sh index 2d517807b..b314daad0 100755 --- a/scripts/inject-bug.sh +++ b/scripts/inject-bug.sh @@ -85,15 +85,11 @@ if [ "${SCENARIO}" = "file-upload-always-fails" ]; then log "Injecting config bug 'file-upload-always-fails' (file-service uploads always 5xx)..." helm upgrade file-service "${REPO_ROOT}/infrastructure/helm/file-service" -n "${NS}" --reuse-values \ --set-string config.FILE_UPLOAD_ALWAYS_FAIL=true - log "Applied. A value change rolls the pod on its own (ConfigMap checksum annotation);" - log "if the env was already true nothing restarts and the tenant is already failing" - log "uploads -- tenant coggtm is in that state, because the demo-coggtm branch bakes" - log "the variable into the file-service image itself. Turn it off with:" + log "Applied. A value change rolls the pod on its own (ConfigMap checksum annotation)." + log "Turn it off with:" log " helm upgrade file-service infrastructure/helm/file-service -n ${NS} --reuse-values --set-string config.FILE_UPLOAD_ALWAYS_FAIL=false" - log "(whether that false survives a redeploy depends on which tree renders the chart:" - log "deploy-tenant.sh from this branch drops it and re-applies the chart's true, while" - log "the ops-dashboard runner's bundled tree sets no such key, so the image default" - log "true applies again. Either way a redeploy re-enables the failure.)" + log "The image and chart default to false; uploads remain healthy unless the release" + log "explicitly overrides FILE_UPLOAD_ALWAYS_FAIL to true." exit 0 fi diff --git a/services/file-service/Dockerfile b/services/file-service/Dockerfile index 15501b37b..5b54569f5 100644 --- a/services/file-service/Dockerfile +++ b/services/file-service/Dockerfile @@ -16,18 +16,7 @@ COPY --from=builder /app/target/release/file-service /app/file-service RUN useradd -r -u 1001 appuser USER appuser -# demo-coggtm tenant variant -- NEVER merge this to `main`. The tenant runs the -# image this branch builds (tag `tenant-coggtm`), so baking the switch in here is -# the only way to make its uploads fail permanently: it survives pod restarts, -# Redis restarts, TTL expiry and idle-suspend/wake, and needs no chart override: -# the deploy runner renders charts from its own bundled tree, whose file-service -# values are `config: {}` and whose build_helm_args sets a fixed list of -# config.* keys that does not include this one, so nothing overrides the image. -# Anything that does set the variable explicitly still wins -- docker-compose.yml -# passes false, so local stacks behave like the golden app. -ENV FILE_UPLOAD_ALWAYS_FAIL=true - -# Click-triggered notification failure (same bake rationale as above): sharing +# Click-triggered notification failure: sharing # a file publishes the file_shared event to a nonexistent SNS topic, so every # share click fails with a real AWS SNS error and fires the # NotificationEventPublishFailure alert at admin-service (incident + Devin @@ -37,9 +26,8 @@ ENV FILE_UPLOAD_ALWAYS_FAIL=true # stacks behave like the golden app. ENV FILE_SHARE_EVENT_ALWAYS_FAIL=true -# With uploads failing permanently (above), users would have no files to share. # Seeding gives every user a few demo documents on their first file listing, -# so the share-notification failure is demoable alongside the upload failure. +# so the share flow is demoable on an account that has uploaded nothing yet. # docker-compose.yml passes false, so local stacks behave like the golden app. ENV FILE_SEED_DEMO_DOCS=true diff --git a/services/file-service/src/config.rs b/services/file-service/src/config.rs index bb10898fd..6dddb9ccc 100644 --- a/services/file-service/src/config.rs +++ b/services/file-service/src/config.rs @@ -155,4 +155,22 @@ mod tests { } assert!(!super::ServerConfig::from_env().upload_always_fail); } + + #[test] + fn production_deployments_do_not_force_upload_failures() { + let upload_failure_enabled = [ + include_str!("../Dockerfile"), + include_str!("../../../infrastructure/helm/file-service/values.yaml"), + ] + .iter() + .flat_map(|contents| contents.lines()) + .any(|line| { + let line = line.trim(); + line.strip_prefix("ENV FILE_UPLOAD_ALWAYS_FAIL=") + .or_else(|| line.strip_prefix("FILE_UPLOAD_ALWAYS_FAIL:")) + .is_some_and(|value| parse_bool(value.trim().trim_matches('"'), false)) + }); + + assert!(!upload_failure_enabled); + } }