Skip to content

Commit 77f3994

Browse files
build: deploy companion charts in parallel (#6428)
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
1 parent 5b06bc0 commit 77f3994

7 files changed

Lines changed: 198 additions & 21 deletions

File tree

scripts/camunda-deployer/go.mod

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ go 1.25.0
44

55
require (
66
github.com/spf13/cobra v1.10.2
7+
golang.org/x/sync v0.21.0
78
gopkg.in/yaml.v3 v3.0.1
89
scripts/camunda-core v0.0.0
910
)

scripts/camunda-deployer/go.sum

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,8 @@ golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT
117117
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
118118
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
119119
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
120+
golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM=
121+
golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
120122
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
121123
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
122124
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

scripts/camunda-deployer/pkg/deployer/deployer.go

Lines changed: 4 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -114,19 +114,10 @@ func Deploy(ctx context.Context, o types.Options) error {
114114
}
115115
}
116116

117-
// Deploy companion charts (e.g., OpenSearch) as separate Helm releases
118-
// in the same namespace. Each chart is deployed with --wait to ensure
119-
// it is ready before the main Camunda chart deployment begins.
120-
for i, cc := range o.CompanionCharts {
121-
logging.Logger.Info().
122-
Str("chart", cc.ChartRef).
123-
Str("version", cc.Version).
124-
Str("release", cc.ReleaseName).
125-
Str("namespace", o.Namespace).
126-
Msg("Deploying companion chart")
127-
if err := deployCompanionChart(ctx, cc, o); err != nil {
128-
return fmt.Errorf("companion chart [%d] %q failed: %w", i, cc.ReleaseName, err)
129-
}
117+
// Deploy companion charts before the post-infra hooks and main chart.
118+
// deployCompanionCharts blocks until all companions are ready.
119+
if err := deployCompanionCharts(ctx, o); err != nil {
120+
return err
130121
}
131122

132123
// Run post-infra hooks after the companion charts (external infrastructure)

scripts/camunda-deployer/pkg/deployer/helm.go

Lines changed: 42 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,10 @@ import (
88
"scripts/camunda-core/pkg/logging"
99
"scripts/camunda-deployer/pkg/types"
1010
"strings"
11+
"sync"
1112
"time"
13+
14+
"golang.org/x/sync/errgroup"
1215
)
1316

1417
// Package-level function variables for helm operations. These default to the
@@ -209,17 +212,51 @@ func formatArgs(args []string) string {
209212
return strings.Join(parts, " ")
210213
}
211214

215+
// companionRepoMu serializes helm repo add/update — those commands rewrite the
216+
// shared repositories.yaml and must not run concurrently.
217+
var companionRepoMu sync.Mutex
218+
219+
// deployCompanionCharts deploys all configured companion charts concurrently,
220+
// blocking until every companion is ready or the first failure cancels the rest.
221+
// Returns the first error encountered; nil means all companions are up.
222+
func deployCompanionCharts(ctx context.Context, o types.Options) error {
223+
g, gCtx := errgroup.WithContext(ctx)
224+
for i, cc := range o.CompanionCharts {
225+
g.Go(func() error {
226+
logging.Logger.Info().
227+
Str("chart", cc.ChartRef).
228+
Str("version", cc.Version).
229+
Str("release", cc.ReleaseName).
230+
Str("namespace", o.Namespace).
231+
Msg("Deploying companion chart")
232+
if err := deployCompanionChart(gCtx, cc, o); err != nil {
233+
return fmt.Errorf("companion chart [%d] %q failed: %w", i, cc.ReleaseName, err)
234+
}
235+
return nil
236+
})
237+
}
238+
return g.Wait()
239+
}
240+
212241
// deployCompanionChart deploys a single companion chart as its own Helm release
213242
// in the same namespace as the main Camunda chart. It uses helm upgrade --install
214243
// with --wait to ensure the chart is fully ready before returning.
215244
func deployCompanionChart(ctx context.Context, cc types.CompanionChart, o types.Options) error {
216245
// Ensure the Helm repo is registered when a repo-style chart ref is used.
217246
if cc.RepoName != "" && cc.RepoURL != "" {
218-
if err := helmRepoAdd(ctx, cc.RepoName, cc.RepoURL); err != nil {
219-
return fmt.Errorf("companion chart %q: repo add failed: %w", cc.ReleaseName, err)
220-
}
221-
if err := helmRepoUpdate(ctx); err != nil {
222-
return fmt.Errorf("companion chart %q: repo update failed: %w", cc.ReleaseName, err)
247+
err := func() error {
248+
companionRepoMu.Lock()
249+
defer companionRepoMu.Unlock()
250+
if err := helmRepoAdd(ctx, cc.RepoName, cc.RepoURL); err != nil {
251+
return fmt.Errorf("companion chart %q: repo add failed: %w", cc.ReleaseName, err)
252+
}
253+
if err := helmRepoUpdate(ctx); err != nil {
254+
return fmt.Errorf("companion chart %q: repo update failed: %w", cc.ReleaseName, err)
255+
}
256+
return nil
257+
}()
258+
if err != nil {
259+
return err
223260
}
224261
}
225262

scripts/camunda-deployer/pkg/deployer/helm_test.go

Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66
"fmt"
77
"scripts/camunda-deployer/pkg/types"
88
"strings"
9+
"sync/atomic"
910
"testing"
1011
"time"
1112
)
@@ -505,3 +506,148 @@ func TestUpgradeInstall_NoRetryOnNonTransient(t *testing.T) {
505506
t.Errorf("expected exactly 1 helm invocation for non-transient error, got %d", attempts)
506507
}
507508
}
509+
510+
func TestDeployCompanionCharts_DeploysInParallel(t *testing.T) {
511+
restore := stubHelm(
512+
func(ctx context.Context, args []string, workDir string) error { return nil },
513+
func(ctx context.Context, name, url string) error { return nil },
514+
func(ctx context.Context) error { return nil },
515+
)
516+
defer restore()
517+
518+
var inFlight, maxInFlight, total int32
519+
helmRunCapturing = func(ctx context.Context, args []string, workDir string) (string, error) {
520+
atomic.AddInt32(&total, 1)
521+
cur := atomic.AddInt32(&inFlight, 1)
522+
for {
523+
prev := atomic.LoadInt32(&maxInFlight)
524+
if cur <= prev || atomic.CompareAndSwapInt32(&maxInFlight, prev, cur) {
525+
break
526+
}
527+
}
528+
time.Sleep(50 * time.Millisecond)
529+
atomic.AddInt32(&inFlight, -1)
530+
return "", nil
531+
}
532+
533+
err := deployCompanionCharts(context.Background(), types.Options{
534+
Namespace: "ns",
535+
CompanionCharts: []types.CompanionChart{
536+
{ChartRef: "/charts/a", ReleaseName: "a"},
537+
{ChartRef: "/charts/b", ReleaseName: "b"},
538+
},
539+
})
540+
if err != nil {
541+
t.Fatalf("unexpected error: %v", err)
542+
}
543+
if total != 2 {
544+
t.Errorf("expected both companions to deploy, got %d invocations", total)
545+
}
546+
if maxInFlight < 2 {
547+
t.Errorf("expected companions to overlap (maxInFlight>=2), got %d — deploys ran serially", maxInFlight)
548+
}
549+
}
550+
551+
func TestDeployCompanionCharts_ErrorCancelsSiblings(t *testing.T) {
552+
restore := stubHelm(
553+
func(ctx context.Context, args []string, workDir string) error { return nil },
554+
func(ctx context.Context, name, url string) error { return nil },
555+
func(ctx context.Context) error { return nil },
556+
)
557+
defer restore()
558+
559+
var siblingCancelled atomic.Bool
560+
helmRunCapturing = func(ctx context.Context, args []string, workDir string) (string, error) {
561+
if containsArg(args, "fails") {
562+
return "", fmt.Errorf("boom")
563+
}
564+
// Slow sibling: block until the failing companion cancels the group ctx.
565+
select {
566+
case <-ctx.Done():
567+
siblingCancelled.Store(true)
568+
return "", ctx.Err()
569+
case <-time.After(2 * time.Second):
570+
return "", nil
571+
}
572+
}
573+
574+
err := deployCompanionCharts(context.Background(), types.Options{
575+
Namespace: "ns",
576+
CompanionCharts: []types.CompanionChart{
577+
{ChartRef: "/charts/fails", ReleaseName: "fails"},
578+
{ChartRef: "/charts/slow", ReleaseName: "slow"},
579+
},
580+
})
581+
if err == nil {
582+
t.Fatal("expected error from failing companion, got nil")
583+
}
584+
if !strings.Contains(err.Error(), "companion chart") {
585+
t.Errorf("error = %q, want it to mention the failing companion chart", err.Error())
586+
}
587+
if !siblingCancelled.Load() {
588+
t.Error("expected the in-flight sibling to observe context cancellation")
589+
}
590+
}
591+
592+
func TestDeployCompanionCharts_SingleCompanionSucceeds(t *testing.T) {
593+
restore := stubHelm(
594+
func(ctx context.Context, args []string, workDir string) error { return nil },
595+
func(ctx context.Context, name, url string) error { return nil },
596+
func(ctx context.Context) error { return nil },
597+
)
598+
defer restore()
599+
600+
var calls int32
601+
helmRunCapturing = func(ctx context.Context, args []string, workDir string) (string, error) {
602+
atomic.AddInt32(&calls, 1)
603+
return "", nil
604+
}
605+
606+
err := deployCompanionCharts(context.Background(), types.Options{
607+
Namespace: "ns",
608+
CompanionCharts: []types.CompanionChart{
609+
{ChartRef: "/charts/only", ReleaseName: "only"},
610+
},
611+
})
612+
if err != nil {
613+
t.Fatalf("unexpected error: %v", err)
614+
}
615+
if calls != 1 {
616+
t.Errorf("expected exactly 1 companion deploy, got %d", calls)
617+
}
618+
}
619+
620+
func TestDeployCompanionCharts_RepoRegistrationSerialized(t *testing.T) {
621+
var inFlight, maxRepoInFlight int32
622+
repoOp := func() {
623+
cur := atomic.AddInt32(&inFlight, 1)
624+
for {
625+
prev := atomic.LoadInt32(&maxRepoInFlight)
626+
if cur <= prev || atomic.CompareAndSwapInt32(&maxRepoInFlight, prev, cur) {
627+
break
628+
}
629+
}
630+
time.Sleep(30 * time.Millisecond)
631+
atomic.AddInt32(&inFlight, -1)
632+
}
633+
restore := stubHelm(
634+
func(ctx context.Context, args []string, workDir string) error { return nil },
635+
func(ctx context.Context, name, url string) error { repoOp(); return nil },
636+
func(ctx context.Context) error { return nil },
637+
)
638+
defer restore()
639+
640+
err := deployCompanionCharts(context.Background(), types.Options{
641+
Namespace: "ns",
642+
CompanionCharts: []types.CompanionChart{
643+
{ChartRef: "x/a", ReleaseName: "a", RepoName: "x", RepoURL: "https://x"},
644+
{ChartRef: "y/b", ReleaseName: "b", RepoName: "y", RepoURL: "https://y"},
645+
},
646+
})
647+
if err != nil {
648+
t.Fatalf("unexpected error: %v", err)
649+
}
650+
if maxRepoInFlight != 1 {
651+
t.Errorf("repo registration must be serialized (maxRepoInFlight=1), got %d", maxRepoInFlight)
652+
}
653+
}

scripts/deploy-camunda/go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ require (
5959
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
6060
golang.org/x/net v0.38.0 // indirect
6161
golang.org/x/oauth2 v0.27.0 // indirect
62-
golang.org/x/sync v0.20.0 // indirect
62+
golang.org/x/sync v0.21.0 // indirect
6363
golang.org/x/sys v0.46.0 // indirect
6464
golang.org/x/text v0.23.0 // indirect
6565
golang.org/x/time v0.3.0 // indirect

scripts/deploy-camunda/go.sum

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -156,8 +156,8 @@ golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT
156156
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
157157
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
158158
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
159-
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
160-
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
159+
golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM=
160+
golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
161161
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
162162
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
163163
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

0 commit comments

Comments
 (0)