Skip to content

Commit b6276ed

Browse files
authored
fix: dedupe TFC run-event publishes and mark locked-workspace errors (#66)
* fix: dedupe TFC run-event publishes and mark locked-workspace errors permanent Signed-off-by: Ihor Horak <ihor.horak@zapier.com>
1 parent 3d10e84 commit b6276ed

11 files changed

Lines changed: 391 additions & 31 deletions

File tree

docs/usage.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@ The full list of supported environment variables and flags is described below:
7272
|`TFBUDDY_WORKSPACE_JETSTREAM_REPLICAS`|`--workspace-jetstream-replicas`|JetStream replica count for the workspace-trigger stream. Use 1 for single-node NATS or local dev; set to your NATS cluster size (often 3) in production for durability.|`1`|
7373
|`TFBUDDY_TFC_RATE_LIMIT_RPS`|`--tfc-rate-limit-rps`|Client-side rate limit (requests per second) for the Terraform Cloud API. Tuned to match TFC's documented per-token limit and prevent 429s when many workspaces are triggered concurrently.|`30`|
7474
|`TFBUDDY_TFC_RATE_LIMIT_BURST`|`--tfc-rate-limit-burst`|Burst capacity for the TFC API token-bucket rate limiter.|`30`|
75+
|`TFBUDDY_JETSTREAM_DEDUP_WINDOW`|`--jetstream-dedup-window`|Window during which JetStream remembers a Nats-Msg-Id to dedupe republishes. Applied to RUN_EVENTS and TFBUDDY_WORKSPACE_TRIGGERS streams. Accepts a Go duration string (e.g. 30m, 1h).|`30m0s`|
7576
<!-- END GENERATED CONFIGURATION -->
7677

7778
For sensitive environment variables use `secrets.envs` which can contain a list of key/value pairs

internal/config/config.go

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"reflect"
55
"strconv"
66
"strings"
7+
"time"
78

89
"github.com/go-viper/mapstructure/v2"
910
"github.com/spf13/pflag"
@@ -34,6 +35,7 @@ const (
3435
KeyWorkspaceJetStreamReplicas = "workspace-jetstream-replicas"
3536
KeyTFCRateLimitRPS = "tfc-rate-limit-rps"
3637
KeyTFCRateLimitBurst = "tfc-rate-limit-burst"
38+
KeyJetStreamDedupWindow = "jetstream-dedup-window"
3739
)
3840

3941
type Config struct {
@@ -58,8 +60,9 @@ type Config struct {
5860
GitlabCloneDepth int `mapstructure:"gitlab-clone-depth"`
5961
WorkspaceFanoutEnabled bool `mapstructure:"workspace-fanout-enabled"`
6062
WorkspaceJetStreamReplicas int `mapstructure:"workspace-jetstream-replicas"`
61-
TFCRateLimitRPS int `mapstructure:"tfc-rate-limit-rps"`
62-
TFCRateLimitBurst int `mapstructure:"tfc-rate-limit-burst"`
63+
TFCRateLimitRPS int `mapstructure:"tfc-rate-limit-rps"`
64+
TFCRateLimitBurst int `mapstructure:"tfc-rate-limit-burst"`
65+
JetStreamDedupWindow time.Duration `mapstructure:"jetstream-dedup-window"`
6366
}
6467

6568
var C Config
@@ -95,6 +98,13 @@ var bindings = []binding{
9598
{key: KeyWorkspaceJetStreamReplicas, defaultValue: 1, description: "JetStream replica count for the workspace-trigger stream. Use 1 for single-node NATS or local dev; set to your NATS cluster size (often 3) in production for durability."},
9699
{key: KeyTFCRateLimitRPS, defaultValue: 30, description: "Client-side rate limit (requests per second) for the Terraform Cloud API. Tuned to match TFC's documented per-token limit and prevent 429s when many workspaces are triggered concurrently."},
97100
{key: KeyTFCRateLimitBurst, defaultValue: 30, description: "Burst capacity for the TFC API token-bucket rate limiter."},
101+
// Dedup window for NATS JetStream-backed streams that use Nats-Msg-Id. Applied
102+
// to RUN_EVENTS (TFC notification publishes, dedup key runID:status) and
103+
// TFBUDDY_WORKSPACE_TRIGGERS (per-workspace MR fan-out, dedup key
104+
// deliveryID/workspace/org). Long enough to cover the slowest plausible same-key
105+
// re-fire (e.g. TFC firing multiple notification triggers across a long
106+
// planning phase), short enough that JetStream's interest store stays bounded.
107+
{key: KeyJetStreamDedupWindow, defaultValue: 30 * time.Minute, description: "Window during which JetStream remembers a Nats-Msg-Id to dedupe republishes. Applied to RUN_EVENTS and TFBUDDY_WORKSPACE_TRIGGERS streams. Accepts a Go duration string (e.g. 30m, 1h)."},
98108
}
99109

100110
func init() {
@@ -152,6 +162,8 @@ func RegisterFlags(fs *pflag.FlagSet) error {
152162
fs.Int(item.key, def, item.description)
153163
case []string:
154164
fs.StringSlice(item.key, def, item.description)
165+
case time.Duration:
166+
fs.Duration(item.key, def, item.description)
155167
default:
156168
continue
157169
}
@@ -261,6 +273,8 @@ func defaultValueString(value any) string {
261273
return ""
262274
}
263275
return strings.Join(v, ",")
276+
case time.Duration:
277+
return v.String()
264278
default:
265279
return ""
266280
}

internal/config/config_test.go

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package config
33
import (
44
"reflect"
55
"testing"
6+
"time"
67

78
"github.com/spf13/pflag"
89
"github.com/spf13/viper"
@@ -79,6 +80,23 @@ func TestDevModeDefaultsToFalse(t *testing.T) {
7980
}
8081
}
8182

83+
func TestJetStreamDedupWindowDefaultsTo30Minutes(t *testing.T) {
84+
resetViperForTest(t)
85+
86+
if got, want := C.JetStreamDedupWindow, 30*time.Minute; got != want {
87+
t.Fatalf("C.JetStreamDedupWindow = %s, want %s", got, want)
88+
}
89+
}
90+
91+
func TestJetStreamDedupWindowCanBeSetFromEnv(t *testing.T) {
92+
t.Setenv("TFBUDDY_JETSTREAM_DEDUP_WINDOW", "5m")
93+
resetViperForTest(t)
94+
95+
if got, want := C.JetStreamDedupWindow, 5*time.Minute; got != want {
96+
t.Fatalf("C.JetStreamDedupWindow = %s, want %s", got, want)
97+
}
98+
}
99+
82100
func TestStringAccessorsReadConfiguredValues(t *testing.T) {
83101
t.Setenv("TFBUDDY_LOG_LEVEL", "debug")
84102
t.Setenv("TFBUDDY_NATS_SERVICE_URL", "nats://example:4222")

pkg/hooks/server.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ func StartServer(cfg config.Config) {
4646
}
4747

4848
hs := hooks_stream.NewHooksStream(nc)
49-
rs := runstream.NewStream(js)
49+
rs := runstream.NewStream(js, cfg.JetStreamDedupWindow)
5050
health.AddReadinessCheck("nats-connection", tfnats.HealthcheckFn(nc))
5151
health.AddLivenessCheck("nats-connection", tfnats.HealthcheckFn(nc))
5252
health.AddLivenessCheck("runstream-streams", rs.HealthCheck)
@@ -61,7 +61,7 @@ func StartServer(cfg config.Config) {
6161
// legacy inline path during rollout.
6262
var workspaceStream tfc_trigger.WorkspacePublisher
6363
if cfg.WorkspaceFanoutEnabled {
64-
ws, err := tfc_trigger.NewWorkspaceStream(js, cfg.WorkspaceJetStreamReplicas)
64+
ws, err := tfc_trigger.NewWorkspaceStream(js, cfg.WorkspaceJetStreamReplicas, cfg.JetStreamDedupWindow)
6565
if err != nil {
6666
log.Fatal().Err(err).Msg("could not configure workspace trigger stream")
6767
}

pkg/runstream/run_event.go

Lines changed: 29 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -64,11 +64,33 @@ func (s *Stream) PublishTFRunEvent(ctx context.Context, re RunEvent) error {
6464
if err != nil {
6565
return err
6666
}
67-
_, err = s.js.Publish(fmt.Sprintf("%s.%s", RunEventsStreamName, rmd.GetVcsProvider()), b)
67+
68+
// Anchor JetStream dedup to (runID, newStatus). TFC fires a webhook for
69+
// every configured notification trigger, and several triggers resolve to
70+
// the same RunStatus over the run's lifecycle (e.g. plan_queued, planning,
71+
// cost_estimating, policy_checking can all surface as `planning` in the
72+
// notification payload). Without this header, each redundant webhook
73+
// produced its own TFRunEvent and a duplicate GitLab reply.
74+
// Combined with the Duplicates window on the stream (see
75+
// configureTFRunEventsStream), this collapses retries and same-status
76+
// redeliveries server-side.
77+
msgID := tfRunEventMsgID(re.GetRunID(), re.GetNewStatus())
78+
_, err = s.js.Publish(
79+
fmt.Sprintf("%s.%s", RunEventsStreamName, rmd.GetVcsProvider()),
80+
b,
81+
nats.MsgId(msgID),
82+
)
6883

6984
return err
7085
}
7186

87+
// tfRunEventMsgID is the JetStream dedup key for a TFRunEvent. Package-level
88+
// helper so the publish path and the in-package tests stay in lockstep — keep
89+
// it unexported until another package actually needs to compute the same key.
90+
func tfRunEventMsgID(runID, newStatus string) string {
91+
return runID + ":" + newStatus
92+
}
93+
7294
func (s *Stream) SubscribeTFRunEvents(vcsProvider string, cb func(run RunEvent) bool) (closer func(), err error) {
7395
sub, err := s.js.QueueSubscribe(
7496
fmt.Sprintf("%s.%s", RunEventsStreamName, vcsProvider),
@@ -132,7 +154,11 @@ func (s *Stream) SubscribeTFRunEvents(vcsProvider string, cb func(run RunEvent)
132154
return closer, nil
133155
}
134156

135-
func configureTFRunEventsStream(js nats.JetStreamContext) {
157+
// configureTFRunEventsStream provisions the RUN_EVENTS stream. dedupWindow
158+
// sets the JetStream Duplicates window used in tandem with the Nats-Msg-Id
159+
// stamped by PublishTFRunEvent. Operators tune the window via the
160+
// TFBUDDY_JETSTREAM_DEDUP_WINDOW env var; tests pass an explicit value.
161+
func configureTFRunEventsStream(js nats.JetStreamContext, dedupWindow time.Duration) {
136162
sCfg := &nats.StreamConfig{
137163
Name: RunEventsStreamName,
138164
Description: "Terraform Cloud Run Notifications",
@@ -141,6 +167,7 @@ func configureTFRunEventsStream(js nats.JetStreamContext) {
141167
MaxMsgs: 10240,
142168
MaxAge: time.Hour * 6,
143169
Replicas: 1,
170+
Duplicates: dedupWindow,
144171
}
145172

146173
addOrUpdateStream(js, sCfg)
Lines changed: 202 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,202 @@
1+
package runstream
2+
3+
import (
4+
"context"
5+
"testing"
6+
"time"
7+
8+
"github.com/nats-io/nats.go"
9+
)
10+
11+
// Test_PublishTFRunEvent_DedupesSameRunIDAndStatus verifies that two
12+
// publishes for the same (runID, newStatus) result in a single message on
13+
// the RUN_EVENTS stream. This is the core fix for duplicate GitLab reply
14+
// comments — TFC fires a webhook for every notification trigger, and several
15+
// triggers can resolve to the same RunStatus over the run's lifecycle.
16+
func Test_PublishTFRunEvent_DedupesSameRunIDAndStatus(t *testing.T) {
17+
_, url := startTestNATS(t)
18+
nc := testConnect(t, url)
19+
defer nc.Close()
20+
js := testGetJetstreamContext(t, nc)
21+
22+
s := newTestStream(t, js)
23+
seedRunMetadata(t, s, "run-DEDUP", "gitlab")
24+
25+
publishOnce(t, s, "run-DEDUP", "planning")
26+
publishOnce(t, s, "run-DEDUP", "planning")
27+
publishOnce(t, s, "run-DEDUP", "planning")
28+
29+
if got := streamMsgs(t, js); got != 1 {
30+
t.Fatalf("expected 1 message on %s after 3 same-status publishes, got %d",
31+
RunEventsStreamName, got)
32+
}
33+
}
34+
35+
// Test_PublishTFRunEvent_KeepsDistinctStatuses verifies that the dedup is
36+
// scoped to (runID, newStatus). Different status values for the same run
37+
// must all land on the stream — that's how real state transitions reach the
38+
// GitLab reply pipeline.
39+
func Test_PublishTFRunEvent_KeepsDistinctStatuses(t *testing.T) {
40+
_, url := startTestNATS(t)
41+
nc := testConnect(t, url)
42+
defer nc.Close()
43+
js := testGetJetstreamContext(t, nc)
44+
45+
s := newTestStream(t, js)
46+
seedRunMetadata(t, s, "run-DISTINCT", "gitlab")
47+
48+
for _, status := range []string{"planning", "planned", "applying", "applied"} {
49+
publishOnce(t, s, "run-DISTINCT", status)
50+
}
51+
52+
if got := streamMsgs(t, js); got != 4 {
53+
t.Fatalf("expected 4 messages on %s for 4 distinct statuses, got %d",
54+
RunEventsStreamName, got)
55+
}
56+
}
57+
58+
// Test_PublishTFRunEvent_DedupesAcrossRuns verifies that the dedup is keyed
59+
// to runID so different runs at the same status are not collapsed.
60+
func Test_PublishTFRunEvent_DedupesAcrossRuns(t *testing.T) {
61+
_, url := startTestNATS(t)
62+
nc := testConnect(t, url)
63+
defer nc.Close()
64+
js := testGetJetstreamContext(t, nc)
65+
66+
s := newTestStream(t, js)
67+
seedRunMetadata(t, s, "run-A", "gitlab")
68+
seedRunMetadata(t, s, "run-B", "gitlab")
69+
70+
publishOnce(t, s, "run-A", "planning")
71+
publishOnce(t, s, "run-A", "planning") // dedup
72+
publishOnce(t, s, "run-B", "planning") // distinct run, must land
73+
publishOnce(t, s, "run-B", "planning") // dedup
74+
75+
if got := streamMsgs(t, js); got != 2 {
76+
t.Fatalf("expected 2 messages (one per runID) on %s, got %d",
77+
RunEventsStreamName, got)
78+
}
79+
}
80+
81+
// Test_PublishTFRunEvent_PlanThenApplyFlow simulates the manual
82+
// plan-then-`tfc apply` workflow. Plan and apply are two separate TFC runs
83+
// with distinct runIDs, but both pass through the planning -> planned
84+
// status transitions before the apply run progresses to applying -> applied.
85+
// Each run must own its own set of dedup messages — none collide.
86+
func Test_PublishTFRunEvent_PlanThenApplyFlow(t *testing.T) {
87+
_, url := startTestNATS(t)
88+
nc := testConnect(t, url)
89+
defer nc.Close()
90+
js := testGetJetstreamContext(t, nc)
91+
92+
s := newTestStream(t, js)
93+
seedRunMetadata(t, s, "run-PLAN", "gitlab")
94+
seedRunMetadata(t, s, "run-APPLY", "gitlab")
95+
96+
// Plan run: TFC walks pending -> planning -> planned -> planned_and_finished.
97+
// Each redundant webhook within a status (e.g. plan_queued + planning
98+
// both surface as `planning`) is collapsed by dedup.
99+
planStatuses := []string{"pending", "planning", "planning", "planned", "planned_and_finished"}
100+
for _, status := range planStatuses {
101+
publishOnce(t, s, "run-PLAN", status)
102+
}
103+
104+
// Apply run: separate runID. TFC runs always plan first, so this run
105+
// also passes through `planning` and `planned` before reaching the apply
106+
// phase. Same status names, different runID = distinct dedup keys.
107+
applyStatuses := []string{"pending", "planning", "planned", "applying", "applying", "applied", "applied"}
108+
for _, status := range applyStatuses {
109+
publishOnce(t, s, "run-APPLY", status)
110+
}
111+
112+
// Expected unique (runID, status) pairs:
113+
// run-PLAN: pending, planning, planned, planned_and_finished (4)
114+
// run-APPLY: pending, planning, planned, applying, applied (5)
115+
const want = 9
116+
if got := streamMsgs(t, js); got != want {
117+
t.Fatalf("plan+apply flow: expected %d distinct messages, got %d", want, got)
118+
}
119+
}
120+
121+
// Test_tfRunEventMsgID locks in the exact dedup-key shape so a refactor
122+
// can't accidentally widen or narrow the scope.
123+
func Test_tfRunEventMsgID(t *testing.T) {
124+
tests := []struct {
125+
runID, status, want string
126+
}{
127+
{"run-rKys4r9a19W7Dk8Q", "planning", "run-rKys4r9a19W7Dk8Q:planning"},
128+
{"run-rKys4r9a19W7Dk8Q", "applied", "run-rKys4r9a19W7Dk8Q:applied"},
129+
{"run-OTHER", "planning", "run-OTHER:planning"},
130+
}
131+
for _, tc := range tests {
132+
if got := tfRunEventMsgID(tc.runID, tc.status); got != tc.want {
133+
t.Errorf("tfRunEventMsgID(%q, %q) = %q, want %q",
134+
tc.runID, tc.status, got, tc.want)
135+
}
136+
}
137+
}
138+
139+
// newTestStream wires up a Stream wired to the test JetStream context. We
140+
// skip NewStream() because it spins up a polling task dispatcher goroutine
141+
// that we don't need for these tests.
142+
//
143+
// JetStream uses on-disk storage by default and natstest reuses os.TempDir()
144+
// across instances, so the stream can survive a NATS server shutdown. Purge
145+
// it on entry so each test starts at zero messages and zero dedup history.
146+
func newTestStream(t *testing.T, js nats.JetStreamContext) *Stream {
147+
t.Helper()
148+
configureTFRunEventsStream(js, testDedupWindow)
149+
if err := js.PurgeStream(RunEventsStreamName); err != nil {
150+
t.Fatalf("could not purge %s before test: %v", RunEventsStreamName, err)
151+
}
152+
kv, err := configureTFRunMetadataKVStore(js)
153+
if err != nil {
154+
t.Fatalf("could not configure metadata KV: %v", err)
155+
}
156+
return &Stream{js: js, metadataKV: kv}
157+
}
158+
159+
func seedRunMetadata(t *testing.T, s *Stream, runID, vcs string) {
160+
t.Helper()
161+
// KV is persistent across test runs (file storage under os.TempDir).
162+
// Clear any prior entry so AddRunMeta's Create() doesn't fail with
163+
// "key exists" when tests are re-run.
164+
_ = s.metadataKV.Delete(runID)
165+
rmd := &TFRunMetadata{
166+
RunID: runID,
167+
Organization: "zapier",
168+
Workspace: "test-ws",
169+
Action: "plan",
170+
CommitSHA: "deadbeef",
171+
MergeRequestProjectNameWithNamespace: "zapier/terraform-services",
172+
MergeRequestIID: 1,
173+
VcsProvider: vcs,
174+
}
175+
if err := s.AddRunMeta(rmd); err != nil {
176+
t.Fatalf("could not seed RunMetadata for %q: %v", runID, err)
177+
}
178+
}
179+
180+
func publishOnce(t *testing.T, s *Stream, runID, status string) {
181+
t.Helper()
182+
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
183+
defer cancel()
184+
re := &TFRunEvent{
185+
RunID: runID,
186+
Organization: "zapier",
187+
Workspace: "test-ws",
188+
NewStatus: status,
189+
}
190+
if err := s.PublishTFRunEvent(ctx, re); err != nil {
191+
t.Fatalf("PublishTFRunEvent(%q, %q) error: %v", runID, status, err)
192+
}
193+
}
194+
195+
func streamMsgs(t *testing.T, js nats.JetStreamContext) uint64 {
196+
t.Helper()
197+
info, err := js.StreamInfo(RunEventsStreamName)
198+
if err != nil {
199+
t.Fatalf("could not read %s info: %v", RunEventsStreamName, err)
200+
}
201+
return info.State.Msgs
202+
}

pkg/runstream/stream.go

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,13 @@ type Stream struct {
1717
pollingKV nats.KeyValue
1818
}
1919

20-
func NewStream(js nats.JetStreamContext) StreamClient {
20+
// NewStream wires up the runstream JetStream client. dedupWindow is the
21+
// JetStream Duplicates window applied to the RUN_EVENTS stream so that
22+
// Nats-Msg-Id-based dedup actually fires server-side. Operators tune it via
23+
// TFBUDDY_JETSTREAM_DEDUP_WINDOW.
24+
func NewStream(js nats.JetStreamContext, dedupWindow time.Duration) StreamClient {
2125

22-
configureTFRunEventsStream(js)
26+
configureTFRunEventsStream(js, dedupWindow)
2327
configureTFRunPollingTaskStream(js)
2428
kv, _ := configureTFRunMetadataKVStore(js)
2529
pollingKV, _ := configureRunPollingKVStore(js)

0 commit comments

Comments
 (0)