otterworks/scripts/bug-catalog.yaml at d831497a288e3e0225e1795191a087f5436c38e4 · COG-GTM/otterworks · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
# ------------------------------------------------------------------------------
# OtterWorks Bug-Injection Catalog
#
# Maps a demo scenario -> a per-tenant variant. Every scenario is scoped to ONE
# tenant namespace and NEVER mutates the golden app (main) or other tenants.
# Applied with:  ./scripts/inject-bug.sh <ATTENDEE_ID> <scenario>
# Cleared with:  ./scripts/inject-bug.sh <ATTENDEE_ID> reset
#
# Three mechanisms, lightest to heaviest (see docs/MULTI-TENANT-DEMO-PLAN.md §6):
#   chaos  - a Redis flag in the tenant's OWN in-cluster Redis (instant, no redeploy,
#            auto-expires). Ideal for "find & fix the incident" demos.
#   config - helm upgrade of one release in the tenant namespace with a bad value,
#            then a rollout restart (seconds).
#   image  - helm upgrade --set image.tag=<variant> for one service (a code bug
#            built on a variant branch/patch off the golden app).
# ------------------------------------------------------------------------------

scenarios:
  file-upload-fails:
    mechanism: chaos
    service: file-service
    redis_key: "chaos:file-service:upload_s3_error"
    description: >
      File uploads fail: file-service writes to a non-existent S3 bucket and
      returns 5xx. Classic "uploads are broken" incident.

  search-suggest-500:
    mechanism: chaos
    service: search-service
    redis_key: "chaos:search-service:suggest_500"
    description: Search autocomplete/suggest endpoint returns HTTP 500.

  document-slow:
    mechanism: chaos
    service: document-service
    redis_key: "chaos:document-service:slow_queries"
    description: >
      Document reads get a 3-5s injected latency (P95 latency-alert demo).

  notification-schema:
    mechanism: chaos
    service: notification-service
    redis_key: "chaos:notification-service:consumer_strict_schema"
    description: >
      Notification consumer switches to a strict schema and rejects messages
      with epoch-int timestamps (dead-letter / consumer-error demo).

  file-bad-bucket:
    mechanism: config
    service: file-service
    set: "config.S3_BUCKET=otterworks-does-not-exist"
    description: >
      Misconfiguration bug: point file-service at a bucket that does not exist
      via a config override (requires a rollout restart to take effect).

  file-upload-always-fails:
    mechanism: config
    service: file-service
    set: "config.FILE_UPLOAD_ALWAYS_FAIL=true"
    description: >
      Explicitly setting FILE_UPLOAD_ALWAYS_FAIL makes file-service route every
      upload to a nonexistent S3 bucket, so uploads keep returning 5xx across
      Redis restarts and TTL expiry (the Redis flag is transient). Downloads,
      listing, folders and sharing are unaffected, and no metadata row is written
      for the failed upload. The image and chart default to false; set this value
      to false to restore uploads, and the ConfigMap checksum will roll the pod.

  notification-queue-fail:
    mechanism: config
    service: notification-service
    set: "config.NOTIFICATION_SQS_ALWAYS_FAIL=true"
    description: >
      Permanent SQS failure point: the NOTIFICATION_SQS_ALWAYS_FAIL env var
      points the notification consumer at a nonexistent SQS queue, so every
      ReceiveMessage poll fails with a real AWS SQS error
      (QueueDoesNotExist). The consumer fires the
      NotificationConsumerProcessingErrors alert at admin-service's ingest
      endpoint (rate-limited; dedup keeps one open incident per service), so
      a Devin session is created even on tenants without Grafana. OFF by
      default (image Dockerfile sets false); the click-triggered
      share-notify-fail scenario below is the demo's primary notification
      failure. NOTIFICATION_STRICT_SCHEMA is the durable env equivalent of
      the notification-schema chaos flag (deserialization failures; needs
      live message traffic).

  share-notify-fail:
    mechanism: config
    service: file-service
    set: "config.FILE_SHARE_EVENT_ALWAYS_FAIL=true"
    description: >
      Click-triggered notification failure: with FILE_SHARE_EVENT_ALWAYS_FAIL
      set, a user clicking "Share" makes file-service publish the file_shared
      notification event to a nonexistent SNS topic (the real topic name with
      a plausible "-v2" suffix), so the publish fails with a real AWS SNS
      NotFound error. The share record is persisted, but the request returns
      the error so the UI shows a red failure banner, and file-service
      fires the NotificationEventPublishFailure alert at admin-service's
      ingest endpoint (dedup=false: one incident + Devin session + Slack
      message per share, attributed to the sharing user via reporter_email).
      Already ON for the coggtm tenant: the demo branch bakes the variable
      into the file-service image's Dockerfile.

  # Example of a code-level variant. Build a variant image from a bug branch/patch
  # off the golden app, push it to ECR, then inject by tag. Immediate rollback =
  # re-deploy the tenant with the golden tag.
  code-variant:
    mechanism: image
    service: file-service
    image_tag: "<variant-tag>"
    description: >
      Swap ONE service to a variant image containing a planted code bug. Scoped
      to the tenant; other tenants keep the golden image.