Skip to content

Commit da5f19b

Browse files
adityathebemoshloop
adityathebe
authored and
moshloop
committed
fix: error count on reconciliation
1 parent 9419b85 commit da5f19b

File tree

2 files changed

+41
-29
lines changed

2 files changed

+41
-29
lines changed

‎tests/upstream_test.go

Lines changed: 22 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -66,9 +66,10 @@ var _ = ginkgo.Describe("Reconcile Test", ginkgo.Ordered, func() {
6666
})
6767

6868
ginkgo.It("should push config items first to satisfy foreign keys for changes & analyses", func() {
69-
count, err := upstream.ReconcileSome(DefaultContext, upstreamConf, 100, "config_items")
69+
count, fkFailed, err := upstream.ReconcileSome(DefaultContext, upstreamConf, 100, "config_items")
7070
Expect(err).To(BeNil())
7171
Expect(count).To(Not(BeZero()))
72+
Expect(fkFailed).To(BeZero())
7273
})
7374

7475
ginkgo.It("should sync config_changes to upstream", func() {
@@ -84,8 +85,9 @@ var _ = ginkgo.Describe("Reconcile Test", ginkgo.Ordered, func() {
8485
Expect(err).ToNot(HaveOccurred())
8586
Expect(changes).To(BeZero())
8687

87-
count, err := upstream.ReconcileSome(DefaultContext, upstreamConf, 10, "config_changes")
88+
count, fkFailed, err := upstream.ReconcileSome(DefaultContext, upstreamConf, 10, "config_changes")
8889
Expect(err).ToNot(HaveOccurred())
90+
Expect(fkFailed).To(BeZero())
8991

9092
err = upstreamCtx.DB().Select("COUNT(*)").Model(&models.ConfigChange{}).Scan(&changes).Error
9193
Expect(err).ToNot(HaveOccurred())
@@ -112,8 +114,9 @@ var _ = ginkgo.Describe("Reconcile Test", ginkgo.Ordered, func() {
112114
Expect(err).ToNot(HaveOccurred())
113115
Expect(analyses).To(BeZero())
114116

115-
count, err := upstream.ReconcileSome(DefaultContext, upstreamConf, 10, "config_analysis")
117+
count, fkFailed, err := upstream.ReconcileSome(DefaultContext, upstreamConf, 10, "config_analysis")
116118
Expect(err).ToNot(HaveOccurred())
119+
Expect(fkFailed).To(BeZero())
117120

118121
err = upstreamCtx.DB().Select("COUNT(*)").Model(&models.ConfigAnalysis{}).Scan(&analyses).Error
119122
Expect(err).ToNot(HaveOccurred())
@@ -138,8 +141,9 @@ var _ = ginkgo.Describe("Reconcile Test", ginkgo.Ordered, func() {
138141
Expect(err).ToNot(HaveOccurred())
139142
Expect(artifacts).To(BeZero())
140143

141-
count, err := upstream.ReconcileSome(DefaultContext, upstreamConf, 10, "artifacts")
144+
count, fkFailed, err := upstream.ReconcileSome(DefaultContext, upstreamConf, 10, "artifacts")
142145
Expect(err).ToNot(HaveOccurred())
146+
Expect(fkFailed).To(BeZero())
143147

144148
err = upstreamCtx.DB().Select("COUNT(*)").Model(&models.Artifact{}).Scan(&artifacts).Error
145149
Expect(err).ToNot(HaveOccurred())
@@ -162,8 +166,9 @@ var _ = ginkgo.Describe("Reconcile Test", ginkgo.Ordered, func() {
162166
Expect(err).ToNot(HaveOccurred())
163167
Expect(upstreamCount).To(BeZero())
164168

165-
count, err := upstream.ReconcileSome(DefaultContext, upstreamConf, 10, "job_history")
169+
count, fkFailed, err := upstream.ReconcileSome(DefaultContext, upstreamConf, 10, "job_history")
166170
Expect(err).ToNot(HaveOccurred())
171+
Expect(fkFailed).To(BeZero())
167172

168173
err = upstreamCtx.DB().Select("COUNT(*)").Model(&models.JobHistory{}).Scan(&upstreamCount).Error
169174
Expect(err).ToNot(HaveOccurred())
@@ -257,9 +262,9 @@ var _ = ginkgo.Describe("Reconcile Test", ginkgo.Ordered, func() {
257262
Where("id IN ?", []uuid.UUID{deployment.ID, pod.ID}).UpdateColumn("is_pushed", true).Error
258263
Expect(err).To(BeNil())
259264

260-
count, err := upstream.ReconcileSome(DefaultContext, upstreamConf, 10, t)
265+
_, fkFailed, err := upstream.ReconcileSome(DefaultContext, upstreamConf, 10, t)
261266
Expect(err).To(HaveOccurred())
262-
Expect(count).To(Equal(0))
267+
Expect(fkFailed).To(BeNumerically(">", 0))
263268

264269
// After reconciliation, those config items should have been marked as unpushed.
265270
var unpushed int
@@ -317,8 +322,9 @@ var _ = ginkgo.Describe("Reconcile Test", ginkgo.Ordered, func() {
317322
})
318323

319324
ginkgo.It("should reconcile the above canary & checks", func() {
320-
_, err := upstream.ReconcileSome(DefaultContext, upstreamConf, 10, "canaries", "checks")
325+
_, fkFailed, err := upstream.ReconcileSome(DefaultContext, upstreamConf, 10, "canaries", "checks")
321326
Expect(err).To(BeNil())
327+
Expect(fkFailed).To(BeZero())
322328

323329
var canaryCount int
324330
err = DefaultContext.DB().Model(&models.Canary{}).Select("Count(*)").Where("id IN ?", []uuid.UUID{httpCanary.ID, tcpCanary.ID}).Where("is_pushed = ?", true).Scan(&canaryCount).Error
@@ -339,8 +345,9 @@ var _ = ginkgo.Describe("Reconcile Test", ginkgo.Ordered, func() {
339345
err = DefaultContext.DB().Model(&models.Check{}).Where("id IN ?", []uuid.UUID{httpChecks.ID, tcpCheck.ID}).Update("is_pushed", false).Error
340346
Expect(err).To(BeNil())
341347

342-
_, err = upstream.ReconcileSome(DefaultContext, upstreamConf, 100, "checks")
348+
_, fkFailed, err := upstream.ReconcileSome(DefaultContext, upstreamConf, 100, "checks")
343349
Expect(err).To(Not(BeNil()))
350+
Expect(fkFailed).To(BeNumerically(">", 0))
344351

345352
// We expect the http check to have been marked as pushed
346353
// while the tcp check & its canary to have been marked as unpushed
@@ -361,17 +368,19 @@ var _ = ginkgo.Describe("Reconcile Test", ginkgo.Ordered, func() {
361368
})
362369

363370
ginkgo.It("The next round of reconciliation should have no error", func() {
364-
_, err := upstream.ReconcileAll(DefaultContext, upstreamConf, 100)
371+
_, fkFailed, err := upstream.ReconcileAll(DefaultContext, upstreamConf, 100)
365372
Expect(err).To(BeNil())
373+
Expect(fkFailed).To(BeZero())
366374
})
367375
})
368376
})
369377
})
370378

371379
ginkgo.Context("should handle updates", func() {
372380
ginkgo.It("ensure all the topologies & canaries have been pushed", func() {
373-
_, err := upstream.ReconcileSome(DefaultContext, upstreamConf, 100, "topologies", "canaries")
381+
_, fkFailed, err := upstream.ReconcileSome(DefaultContext, upstreamConf, 100, "topologies", "canaries")
374382
Expect(err).To(BeNil())
383+
Expect(fkFailed).To(BeZero())
375384

376385
var unpushedCanaries int
377386
err = DefaultContext.DB().Select("COUNT(*)").Where("is_pushed = false").Model(&models.Canary{}).Scan(&unpushedCanaries).Error
@@ -392,9 +401,10 @@ var _ = ginkgo.Describe("Reconcile Test", ginkgo.Ordered, func() {
392401
err = DefaultContext.DB().Model(&models.Canary{}).Where("is_pushed = ?", true).Update("is_pushed", false).Error
393402
Expect(err).To(BeNil())
394403

395-
count, err := upstream.ReconcileSome(DefaultContext, upstreamConf, 100, "topologies", "canaries")
404+
count, fkFailed, err := upstream.ReconcileSome(DefaultContext, upstreamConf, 100, "topologies", "canaries")
396405
Expect(err).To(BeNil())
397406
Expect(count).To(Not(BeZero()))
407+
Expect(fkFailed).To(BeZero())
398408

399409
var unpushedCanaries int
400410
err = DefaultContext.DB().Select("COUNT(*)").Where("is_pushed = false").Model(&models.Canary{}).Scan(&unpushedCanaries).Error

‎upstream/jobs.go

Lines changed: 19 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -67,53 +67,54 @@ var reconciledTables = []pushableTable{
6767
models.JobHistory{},
6868
}
6969

70-
func ReconcileAll(ctx context.Context, config UpstreamConfig, batchSize int) (int, error) {
70+
func ReconcileAll(ctx context.Context, config UpstreamConfig, batchSize int) (int, int, error) {
7171
return ReconcileSome(ctx, config, batchSize)
7272
}
7373

74-
func ReconcileSome(ctx context.Context, config UpstreamConfig, batchSize int, runOnly ...string) (int, error) {
75-
var count int
74+
func ReconcileSome(ctx context.Context, config UpstreamConfig, batchSize int, runOnly ...string) (int, int, error) {
75+
var count, fkFailed int
7676
for _, table := range reconciledTables {
7777
if len(runOnly) > 0 && !lo.Contains(runOnly, table.TableName()) {
7878
continue
7979
}
8080

81-
if c, err := reconcileTable(ctx, config, table, batchSize); err != nil {
82-
return count, fmt.Errorf("failed to reconcile table %s: %w", table.TableName(), err)
83-
} else {
84-
count += c
81+
success, failed, err := reconcileTable(ctx, config, table, batchSize)
82+
count += success
83+
fkFailed += failed
84+
if err != nil {
85+
return count, fkFailed, fmt.Errorf("failed to reconcile table %s: %w", table.TableName(), err)
8586
}
8687
}
8788

88-
return count, nil
89+
return count, fkFailed, nil
8990
}
9091

9192
// ReconcileTable pushes all unpushed items in a table to upstream.
92-
func reconcileTable(ctx context.Context, config UpstreamConfig, table pushableTable, batchSize int) (int, error) {
93+
func reconcileTable(ctx context.Context, config UpstreamConfig, table pushableTable, batchSize int) (int, int, error) {
9394
client := NewUpstreamClient(config)
9495

95-
var count int
96+
var count, fkFailed int
9697
for {
9798
items, err := table.GetUnpushed(ctx.DB().Limit(batchSize))
9899
if err != nil {
99-
return 0, fmt.Errorf("failed to fetch unpushed items for table %s: %w", table, err)
100+
return count, fkFailed, fmt.Errorf("failed to fetch unpushed items for table %s: %w", table, err)
100101
}
101102

102103
if len(items) == 0 {
103-
return count, nil
104+
return count, fkFailed, nil
104105
}
105106

106107
ctx.Tracef("pushing %s %d to upstream", table.TableName(), len(items))
107108
pushError := client.Push(ctx, NewPushData(items))
108109
if pushError != nil {
109110
httpError := api.HTTPErrorFromErr(pushError)
110111
if httpError == nil || httpError.Data == "" {
111-
return 0, fmt.Errorf("failed to push %s to upstream: %w", table.TableName(), pushError)
112+
return count, fkFailed, fmt.Errorf("failed to push %s to upstream: %w", table.TableName(), pushError)
112113
}
113114

114115
var foreignKeyErr PushFKError
115116
if err := json.Unmarshal([]byte(httpError.Data), &foreignKeyErr); err != nil {
116-
return 0, fmt.Errorf("failed to push %s to upstream (could not decode api error: %w): %w", table.TableName(), err, pushError)
117+
return count, fkFailed, fmt.Errorf("failed to push %s to upstream (could not decode api error: %w): %w", table.TableName(), err, pushError)
117118
}
118119

119120
failedOnes := lo.SliceToMap(foreignKeyErr.IDs, func(item string) (string, struct{}) {
@@ -123,10 +124,11 @@ func reconcileTable(ctx context.Context, config UpstreamConfig, table pushableTa
123124
_, ok := failedOnes[item.PK()]
124125
return ok
125126
})
127+
fkFailed += len(failedItems)
126128

127129
if c, ok := table.(parentIsPushedUpdater); ok && len(failedItems) > 0 {
128130
if err := c.UpdateParentsIsPushed(ctx.DB(), failedItems); err != nil {
129-
return 0, fmt.Errorf("failed to mark parents as unpushed: %w", err)
131+
return count, fkFailed, fmt.Errorf("failed to mark parents as unpushed: %w", err)
130132
}
131133
}
132134

@@ -166,12 +168,12 @@ func reconcileTable(ctx context.Context, config UpstreamConfig, table pushableTa
166168
return nil
167169
})
168170
if err != nil {
169-
return count, err
171+
return count, fkFailed, err
170172
}
171173
}
172174

173175
if pushError != nil {
174-
return count, pushError
176+
return count, fkFailed, pushError
175177
}
176178
}
177179
}

0 commit comments

Comments
 (0)