tests: history finalize on multi-node

Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com>

Author: crazy-max

tests/history.go

@@ -19,6 +19,8 @@ import (
 var historyTests = []func(t *testing.T, sb integration.Sandbox){
 	testHistoryExport,
 	testHistoryExportFinalize,
+	testHistoryExportFinalizeMultiNodeRef,
+	testHistoryExportFinalizeMultiNodeAll,
 	testHistoryInspect,
 	testHistoryLs,
 	testHistoryRm,
@@ -29,21 +31,55 @@ var historyTests = []func(t *testing.T, sb integration.Sandbox){
 func testHistoryExport(t *testing.T, sb integration.Sandbox) {
 	ref := buildTestProject(t, sb)
 	require.NotEmpty(t, ref.Ref)
+	requireHistoryRef(t, sb, ref.Ref)
 
 	outFile := path.Join(t.TempDir(), "export.dockerbuild")
 	cmd := buildxCmd(sb, withArgs("history", "export", ref.Ref, "--output", outFile))
-	out, err := cmd.Output()
+	out, err := cmd.CombinedOutput()
 	require.NoError(t, err, string(out))
 	require.FileExists(t, outFile)
 }
 
 func testHistoryExportFinalize(t *testing.T, sb integration.Sandbox) {
 	ref := buildTestProject(t, sb)
 	require.NotEmpty(t, ref.Ref)
+	requireHistoryRef(t, sb, ref.Ref)
 
 	outFile := path.Join(t.TempDir(), "export.dockerbuild")
 	cmd := buildxCmd(sb, withArgs("history", "export", ref.Ref, "--finalize", "--output", outFile))
-	out, err := cmd.Output()
+	out, err := cmd.CombinedOutput()
+	require.NoError(t, err, string(out))
+	require.FileExists(t, outFile)
+}
+
+func testHistoryExportFinalizeMultiNodeRef(t *testing.T, sb integration.Sandbox) {
+	if !isRemoteMultiNodeWorker(sb) {
+		t.Skip("only testing with remote multi-node worker")
+	}
+
+	ref := buildTestProject(t, sb, withArgs("--platform=linux/amd64,linux/arm64", "--output=type=cacheonly"))
+	require.NotEmpty(t, ref.Ref)
+	requireHistoryRef(t, sb, ref.Ref)
+
+	outFile := path.Join(t.TempDir(), "export.dockerbuild")
+	cmd := buildxCmd(sb, withArgs("history", "export", ref.Ref, "--finalize", "--output", outFile))
+	out, err := cmd.CombinedOutput()
+	require.NoError(t, err, string(out))
+	require.FileExists(t, outFile)
+}
+
+func testHistoryExportFinalizeMultiNodeAll(t *testing.T, sb integration.Sandbox) {
+	if !isRemoteMultiNodeWorker(sb) {
+		t.Skip("only testing with remote multi-node worker")
+	}
+
+	ref := buildTestProject(t, sb, withArgs("--platform=linux/amd64,linux/arm64", "--output=type=cacheonly"))
+	require.NotEmpty(t, ref.Ref)
+	requireHistoryRef(t, sb, ref.Ref)
+
+	outFile := path.Join(t.TempDir(), "export.dockerbuild")
+	cmd := buildxCmd(sb, withArgs("history", "export", "--finalize", "--all", "--output", outFile))
+	out, err := cmd.CombinedOutput()
 	require.NoError(t, err, string(out))
 	require.FileExists(t, outFile)
 }
@@ -53,7 +89,7 @@ func testHistoryInspect(t *testing.T, sb integration.Sandbox) {
 	require.NotEmpty(t, ref.Ref)
 
 	cmd := buildxCmd(sb, withArgs("history", "inspect", ref.Ref, "--format=json"))
-	out, err := cmd.Output()
+	out, err := cmd.CombinedOutput()
 	require.NoError(t, err, string(out))
 
 	type recT struct {
@@ -77,31 +113,10 @@ func testHistoryInspect(t *testing.T, sb integration.Sandbox) {
 }
 
 func testHistoryLs(t *testing.T, sb integration.Sandbox) {
-	if isRemoteMultiNodeWorker(sb) {
-		// FIXME: "history ls" fails on multi nodes
-		t.Skip("fails with multi nodes")
-	}
-
 	ref := buildTestProject(t, sb)
 	require.NotEmpty(t, ref.Ref)
 
-	cmd := buildxCmd(sb, withArgs("history", "ls", "--filter=ref="+ref.Ref, "--format=json"))
-	out, err := cmd.Output()
-	require.NoError(t, err, string(out))
-
-	type recT struct {
-		Ref            string     `json:"ref"`
-		Name           string     `json:"name"`
-		Status         string     `json:"status"`
-		CreatedAt      *time.Time `json:"created_at"`
-		CompletedAt    *time.Time `json:"completed_at"`
-		TotalSteps     int32      `json:"total_steps"`
-		CompletedSteps int32      `json:"completed_steps"`
-		CachedSteps    int32      `json:"cached_steps"`
-	}
-	var rec recT
-	err = json.Unmarshal(out, &rec)
-	require.NoError(t, err)
+	rec := requireHistoryRecord(t, sb, ref.String())
 	require.Equal(t, ref.String(), rec.Ref)
 	require.NotEmpty(t, rec.Name)
 }
@@ -146,15 +161,10 @@ func testHistoryLsStoppedBuilder(t *testing.T, sb integration.Sandbox) {
 }
 
 func testHistoryBuildName(t *testing.T, sb integration.Sandbox) {
-	if isRemoteMultiNodeWorker(sb) {
-		// FIXME: "history ls" fails on multi nodes
-		t.Skip("fails with multi nodes")
-	}
-
 	t.Run("override", func(t *testing.T) {
 		dir := createTestProject(t)
 		out, err := buildCmd(sb, withArgs("--build-arg=BUILDKIT_BUILD_NAME=foobar", "--metadata-file", filepath.Join(dir, "md.json"), dir))
-		require.NoError(t, err, string(out))
+		require.NoError(t, err, out)
 
 		dt, err := os.ReadFile(filepath.Join(dir, "md.json"))
 		require.NoError(t, err)
@@ -169,23 +179,7 @@ func testHistoryBuildName(t *testing.T, sb integration.Sandbox) {
 		refParts := strings.Split(md.BuildRef, "/")
 		require.Len(t, refParts, 3)
 
-		cmd := buildxCmd(sb, withArgs("history", "ls", "--filter=ref="+refParts[2], "--format=json"))
-		bout, err := cmd.Output()
-		require.NoError(t, err, string(bout))
-
-		type recT struct {
-			Ref            string     `json:"ref"`
-			Name           string     `json:"name"`
-			Status         string     `json:"status"`
-			CreatedAt      *time.Time `json:"created_at"`
-			CompletedAt    *time.Time `json:"completed_at"`
-			TotalSteps     int32      `json:"total_steps"`
-			CompletedSteps int32      `json:"completed_steps"`
-			CachedSteps    int32      `json:"cached_steps"`
-		}
-		var rec recT
-		err = json.Unmarshal(bout, &rec)
-		require.NoError(t, err)
+		rec := requireHistoryRecord(t, sb, md.BuildRef)
 		require.Equal(t, md.BuildRef, rec.Ref)
 		require.Equal(t, "foobar", rec.Name)
 	})
@@ -230,23 +224,7 @@ COPY foo /foo
 		refParts := strings.Split(md.BuildRef, "/")
 		require.Len(t, refParts, 3)
 
-		cmd := buildxCmd(sb, withArgs("history", "ls", "--filter=ref="+refParts[2], "--format=json"))
-		bout, err := cmd.Output()
-		require.NoError(t, err, string(bout))
-
-		type recT struct {
-			Ref            string     `json:"ref"`
-			Name           string     `json:"name"`
-			Status         string     `json:"status"`
-			CreatedAt      *time.Time `json:"created_at"`
-			CompletedAt    *time.Time `json:"completed_at"`
-			TotalSteps     int32      `json:"total_steps"`
-			CompletedSteps int32      `json:"completed_steps"`
-			CachedSteps    int32      `json:"cached_steps"`
-		}
-		var rec recT
-		err = json.Unmarshal(bout, &rec)
-		require.NoError(t, err)
+		rec := requireHistoryRecord(t, sb, md.BuildRef)
 		require.Equal(t, md.BuildRef, rec.Ref)
 		require.Equal(t, addr+"#main", rec.Name)
 	})
@@ -296,23 +274,7 @@ EOT
 		refParts := strings.Split(md.Default.BuildRef, "/")
 		require.Len(t, refParts, 3)
 
-		cmd := buildxCmd(sb, withArgs("history", "ls", "--filter=ref="+refParts[2], "--format=json"))
-		bout, err := cmd.Output()
-		require.NoError(t, err, string(bout))
-
-		type recT struct {
-			Ref            string     `json:"ref"`
-			Name           string     `json:"name"`
-			Status         string     `json:"status"`
-			CreatedAt      *time.Time `json:"created_at"`
-			CompletedAt    *time.Time `json:"completed_at"`
-			TotalSteps     int32      `json:"total_steps"`
-			CompletedSteps int32      `json:"completed_steps"`
-			CachedSteps    int32      `json:"cached_steps"`
-		}
-		var rec recT
-		err = json.Unmarshal(bout, &rec)
-		require.NoError(t, err)
+		rec := requireHistoryRecord(t, sb, md.Default.BuildRef)
 		require.Equal(t, md.Default.BuildRef, rec.Ref)
 		require.Equal(t, addr, rec.Name)
 	})
@@ -328,10 +290,11 @@ func (b buildRef) String() string {
 	return b.Builder + "/" + b.Node + "/" + b.Ref
 }
 
-func buildTestProject(t *testing.T, sb integration.Sandbox) buildRef {
+func buildTestProject(t *testing.T, sb integration.Sandbox, opts ...cmdOpt) buildRef {
 	dir := createTestProject(t)
-	out, err := buildCmd(sb, withArgs("--metadata-file", filepath.Join(dir, "md.json"), dir))
-	require.NoError(t, err, string(out))
+	opts = append(opts, withArgs("--metadata-file", filepath.Join(dir, "md.json"), dir))
+	out, err := buildCmd(sb, opts...)
+	require.NoError(t, err, out)
 
 	dt, err := os.ReadFile(filepath.Join(dir, "md.json"))
 	require.NoError(t, err)
@@ -352,3 +315,52 @@ func buildTestProject(t *testing.T, sb integration.Sandbox) buildRef {
 		Ref:     refParts[2],
 	}
 }
+
+func requireHistoryRef(t *testing.T, sb integration.Sandbox, ref string) {
+	cmd := buildxCmd(sb, withArgs("history", "ls", "--format={{.Ref}}"))
+	out, err := cmd.CombinedOutput()
+	require.NoError(t, err, string(out))
+	var matches int
+	for line := range strings.SplitSeq(strings.TrimSpace(string(out)), "\n") {
+		if strings.TrimSpace(line) == ref {
+			matches++
+		}
+	}
+	require.GreaterOrEqual(t, matches, 1)
+}
+
+type historyLsRecord struct {
+	Ref            string     `json:"ref"`
+	Name           string     `json:"name"`
+	Status         string     `json:"status"`
+	CreatedAt      *time.Time `json:"created_at"`
+	CompletedAt    *time.Time `json:"completed_at"`
+	TotalSteps     int32      `json:"total_steps"`
+	CompletedSteps int32      `json:"completed_steps"`
+	CachedSteps    int32      `json:"cached_steps"`
+}
+
+func requireHistoryRecord(t *testing.T, sb integration.Sandbox, ref string, opts ...cmdOpt) historyLsRecord {
+	cmd := buildxCmd(sb, append(opts, withArgs("history", "ls", "--format=json"))...)
+	out, err := cmd.CombinedOutput()
+	require.NoError(t, err, string(out))
+
+	for line := range strings.SplitSeq(strings.TrimSpace(string(out)), "\n") {
+		line = strings.TrimSpace(line)
+		if line == "" {
+			continue
+		}
+		if !strings.HasPrefix(line, "{") {
+			continue
+		}
+		var rec historyLsRecord
+		err := json.Unmarshal([]byte(line), &rec)
+		require.NoError(t, err)
+		if rec.Ref == ref {
+			return rec
+		}
+	}
+
+	require.Failf(t, "history record not found", "ref %q was not found in history ls output:\n%s", ref, string(out))
+	return historyLsRecord{}
+}