sources: verify git refs with checksum

.github/workflows/ci.yml

@@ -111,8 +111,6 @@ jobs:
           - suite: other
             skip: Azlinux3|Bookworm|Bullseye|Bionic|Focal|Jammy|Noble|Windows|Almalinux8|Almalinux9|Rockylinux8|Rockylinux9|Trixie
 
-    # TODO: support diff/merge
-    # Right now this is handled by the e2e suite, but we can migrate that here.
     steps:
       - name: Harden Runner
         uses: step-security/harden-runner@fe104658747b27e96e4f7e80cd0a94068e53901d # v2.16.1
@@ -127,6 +125,18 @@ jobs:
       - name: checkout
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
 
+      - name: Update Docker
+        run: |
+          set -e
+          docker version
+          docker info
+
+          sudo apt update
+          sudo apt install -y moby-engine=29.4.* moby-buildx=0.33.* moby-cli=29.4.* moby-containerd=2.3.*
+
+          docker version
+          docker info
+
       - uses: ./.github/actions/enable-containerd
 
       - uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0

docs/spec.schema.json

@@ -1820,22 +1820,33 @@
 			],
 			"properties": {
 				"auth": {
-					"$ref": "#/$defs/GitAuth"
+					"$ref": "#/$defs/GitAuth",
+					"description": "Auth can be used to add instructions on how to authenticate with the git repository."
+				},
+				"checksum": {
+					"type": [
+						"string",
+						"null"
+					],
+					"description": "Checksum is the expected commit hash for the resolved ref.\nIt is useful when \"Commit\" refers to a mutable ref, such as a tag."
 				},
 				"commit": {
 					"type": [
 						"string"
-					]
+					],
+					"description": "Commit is the ref, which may be a commit, a tag, or even a full ref.\nNOTE: When using a commit ref, tag info is *not* fetched."
 				},
 				"keepGitDir": {
 					"type": [
 						"boolean"
-					]
+					],
+					"description": "KeepGitDir includes the .git directory in the source.\nNOTE: Not all git metadata may be available. The available metadata depends\non the ref specified by \"Commit\"."
 				},
 				"url": {
 					"type": [
 						"string"
-					]
+					],
+					"description": "URL is the URL of the git repository."
 				}
 			},
 			"additionalProperties": {

load_test.go

@@ -41,6 +41,16 @@ func TestSourceValidation(t *testing.T) {
 			},
 			expectErr: true,
 		},
+		{
+			title: "git source checksum accepts hex",
+			src: Source{
+				Git: &SourceGit{
+					URL:      "https://example.com/repo.git",
+					Commit:   "v1.2.3",
+					Checksum: "0123456789abcdef",
+				},
+			},
+		},
 		{
 			title:     "has multiple source types in docker-image command mount",
 			expectErr: true,
@@ -666,6 +676,7 @@ func TestSpec_SubstituteBuildArgs(t *testing.T) {
 	const (
 		foo            = "foo"
 		bar            = "bar"
+		checksum       = "baddecaf"
 		argWithDefault = "some default value"
 		plainOleValue  = "some plain old value"
 	)
@@ -701,6 +712,13 @@ func TestSpec_SubstituteBuildArgs(t *testing.T) {
 		Excludes: []string{"foo/${BAR}"},
 		Inline:   &SourceInline{},
 	}
+	spec.Sources["git"] = Source{
+		Git: &SourceGit{
+			URL:      "https://example.com/foo/${BAR}.git",
+			Commit:   "$FOO",
+			Checksum: "$CHECKSUM",
+		},
+	}
 
 	spec.Patches = map[string][]PatchSpec{
 		"src": {
@@ -795,13 +813,18 @@ func TestSpec_SubstituteBuildArgs(t *testing.T) {
 	env["BAR"] = bar
 
 	spec.Args["BAR"] = ""
+	spec.Args["CHECKSUM"] = ""
 	spec.Args["VAR_WITH_DEFAULT"] = argWithDefault
+	env["CHECKSUM"] = checksum
 
 	assert.NilError(t, spec.SubstituteArgs(env))
 
 	assert.Check(t, cmp.Equal(spec.Sources["patch"].Path, "foo/"+bar))
 	assert.Check(t, cmp.Equal(spec.Sources["patch"].Includes[0], "foo/"+bar))
 	assert.Check(t, cmp.Equal(spec.Sources["patch"].Excludes[0], "foo/"+bar))
+	assert.Check(t, cmp.Equal(spec.Sources["git"].Git.URL, "https://example.com/foo/"+bar+".git"))
+	assert.Check(t, cmp.Equal(spec.Sources["git"].Git.Commit, foo))
+	assert.Check(t, cmp.Equal(spec.Sources["git"].Git.Checksum, checksum))
 	assert.Check(t, cmp.Equal(spec.Patches["src"][0].Path, foo))
 
 	// Base package config
@@ -1193,6 +1216,26 @@ targets:
 		assert.Check(t, cmp.Equal(target.PackageConfig.Signer.Args["FOO"], "test"))
 	})
 
+	t.Run("git checksum build arg loaded from yaml", func(t *testing.T) {
+		dt := []byte(`
+args:
+  COMMIT:
+sources:
+  test:
+    git:
+      url: https://example.com/repo.git
+      commit: v1.2.3
+      checksum: ${COMMIT}
+`)
+
+		spec, err := LoadSpec(dt)
+		assert.NilError(t, err)
+
+		err = spec.SubstituteArgs(map[string]string{"COMMIT": "0123456789abcdef"})
+		assert.NilError(t, err)
+		assert.Check(t, cmp.Equal(spec.Sources["test"].Git.Checksum, "0123456789abcdef"))
+	})
+
 	t.Run("default value", func(t *testing.T) {
 		dt := []byte(`
 args:

source_git.go

@@ -13,10 +13,20 @@ import (
 )
 
 type SourceGit struct {
-	URL        string  `yaml:"url" json:"url"`
-	Commit     string  `yaml:"commit" json:"commit"`
-	KeepGitDir bool    `yaml:"keepGitDir,omitempty" json:"keepGitDir,omitempty"`
-	Auth       GitAuth `yaml:"auth,omitempty" json:"auth,omitempty"`
+	// URL is the URL of the git repository.
+	URL string `yaml:"url" json:"url"`
+	// Commit is the ref, which may be a commit, a tag, or even a full ref.
+	// NOTE: When using a commit ref, tag info is *not* fetched.
+	Commit string `yaml:"commit" json:"commit"`
+	// Checksum is the expected commit hash for the resolved ref.
+	// It is useful when "Commit" refers to a mutable ref, such as a tag.
+	Checksum string `yaml:"checksum,omitempty" json:"checksum,omitempty"`
+	// KeepGitDir includes the .git directory in the source.
+	// NOTE: Not all git metadata may be available. The available metadata depends
+	// on the ref specified by "Commit".
+	KeepGitDir bool `yaml:"keepGitDir,omitempty" json:"keepGitDir,omitempty"`
+	// Auth can be used to add instructions on how to authenticate with the git repository.
+	Auth GitAuth `yaml:"auth,omitempty" json:"auth,omitempty"`
 
 	_sourceMap *sourceMap `yaml:"-" json:"-"`
 }
@@ -118,6 +128,9 @@ func (src *SourceGit) baseState(opts fetchOptions) llb.State {
 	if src.KeepGitDir {
 		gOpts = append(gOpts, llb.KeepGitDir())
 	}
+	if src.Checksum != "" {
+		gOpts = append(gOpts, llb.GitChecksum(src.Checksum))
+	}
 	gOpts = append(gOpts, WithConstraints(opts.Constraints...))
 	gOpts = append(gOpts, &src.Auth)
 	gOpts = append(gOpts, src._sourceMap.GetRootLocation())
@@ -162,6 +175,12 @@ func (src *SourceGit) processBuildArgs(lex *shell.Lex, args map[string]string, a
 	if err != nil {
 		errs = append(errs, err)
 	}
+
+	updated, err = expandArgs(lex, src.Checksum, args, allowArg)
+	src.Checksum = updated
+	if err != nil {
+		errs = append(errs, err)
+	}
 	if len(errs) > 0 {
 		err := fmt.Errorf("failed to process build args for git source: %w", stderrors.Join(errs...))
 		err = errdefs.WithSource(err, src._sourceMap.GetErrdefsSource())
@@ -185,4 +204,7 @@ func (src *SourceGit) doc(w io.Writer, name string) {
 	printDocLn(w, "Generated from a git repository:")
 	printDocLn(w, "	Remote:", ref.Remote)
 	printDocLn(w, "	Ref:", src.Commit)
+	if src.Checksum != "" {
+		printDocLn(w, "	Checksum:", src.Checksum)
+	}
 }

source_test.go

@@ -175,6 +175,20 @@ func TestSourceGitHTTP(t *testing.T) {
 		checkGitOp(t, ops, &src)
 	})
 
+	t.Run("with checksum and git dir", func(t *testing.T) {
+		src := Source{
+			Git: &SourceGit{
+				URL:        "https://localhost/test.git",
+				Commit:     "v1.2.3",
+				Checksum:   "0123456789abcdef",
+				KeepGitDir: true,
+			},
+		}
+
+		ops := getSourceOp(ctx, t, src)
+		checkGitOp(t, ops, &src)
+	})
+
 	t.Run("gomod auth", func(t *testing.T) {
 		const (
 			numSecrets = 2
@@ -1032,6 +1046,18 @@ func checkGitOp(t *testing.T, ops []*pb.Op, src *Source) {
 		t.Errorf("expected git.fullurl %q, got %q", src.Git.URL, op.Attrs["git.fullurl"])
 	}
 
+	if src.Git.Checksum != "" {
+		assert.Check(t, cmp.Equal(op.Attrs[pb.AttrGitChecksum], src.Git.Checksum), op.Attrs)
+	} else {
+		assert.Check(t, cmp.Equal(op.Attrs[pb.AttrGitChecksum], ""), op.Attrs)
+	}
+
+	if src.Git.KeepGitDir {
+		assert.Check(t, cmp.Equal(op.Attrs[pb.AttrKeepGitDir], "true"), op.Attrs)
+	} else {
+		assert.Check(t, cmp.Equal(op.Attrs[pb.AttrKeepGitDir], ""), op.Attrs)
+	}
+
 	const (
 		defaultAuthHeader = "GIT_AUTH_HEADER"
 		defaultAuthToken  = "GIT_AUTH_TOKEN"

test/source_test.go

@@ -13,6 +13,8 @@ import (
 	"github.com/opencontainers/go-digest"
 	"github.com/project-dalec/dalec"
 	"github.com/project-dalec/dalec/frontend/pkg/bkfs"
+	gitservices "github.com/project-dalec/dalec/test/git_services"
+	"github.com/project-dalec/dalec/test/testenv"
 	"gotest.tools/v3/assert"
 )
 
@@ -453,6 +455,123 @@ func TestSourceHTTP(t *testing.T) {
 	})
 }
 
+func TestSourceGitChecksumPreservesTagMetadata(t *testing.T) {
+	t.Parallel()
+
+	parentCtx := startTestSpan(baseCtx, t)
+
+	const (
+		secretName = "super-secret"
+		sourceName = "repo"
+		tagName    = "v1.2.3"
+	)
+
+	runGitServerTest := func(ctx context.Context, t *testing.T, f func(context.Context, gwclient.Client, llb.State, string, func(string) *dalec.Spec)) {
+		t.Helper()
+
+		testEnv.RunTest(ctx, t, func(ctx context.Context, gwc gwclient.Client) {
+			t.Helper()
+
+			attr := gitservices.Attributes{
+				ServerRoot:      "/",
+				PrivateRepoPath: "username/private",
+				HTTPServerPath:  "/usr/local/bin/git_http_server",
+				HTTPPort:        "8080",
+			}
+
+			testState := gitservices.NewTestState(t, gwc, &attr)
+			worker := initWorker(gwc)
+			repo := llb.Scratch().File(llb.Mkfile("foo", 0o644, []byte("bar\n")))
+			repo = worker.Dir(attr.PrivateRepoAbsPath()).Run(dalec.ShArgs(`
+			set -eux
+			export GIT_CONFIG_NOGLOBAL=true
+			git init
+			git config user.name foo
+			git config user.email foo@bar.com
+			git add -A
+			git commit -m commit --no-gpg-sign
+			git tag -a `+tagName+` -m `+tagName+`
+		`)).AddMount(attr.RepoAbsDir(), repo)
+
+			commitSt := worker.Dir(attr.PrivateRepoAbsPath()).Run(dalec.ShArgs(`
+			set -eu
+			git rev-parse HEAD > /out/commit
+		`), llb.AddMount(attr.RepoAbsDir(), repo, llb.Readonly)).AddMount("/out", llb.Scratch())
+			commitDef, err := commitSt.Marshal(ctx)
+			assert.NilError(t, err)
+
+			commitRes, err := gwc.Solve(ctx, gwclient.SolveRequest{Definition: commitDef.ToPB(), Evaluate: true})
+			assert.NilError(t, err)
+			commit := strings.TrimSpace(string(readFile(ctx, t, "commit", commitRes)))
+
+			gitHost := worker.With(hostedRepo(repo, attr.RepoAbsDir()))
+			httpServer := testState.StartHTTPGitServer(ctx, gitHost)
+
+			newSpec := func(checksum string) *dalec.Spec {
+				return &dalec.Spec{
+					Sources: map[string]dalec.Source{
+						sourceName: {
+							Git: &dalec.SourceGit{
+								URL:        "http://" + httpServer.IP + ":" + httpServer.Port + "/" + attr.PrivateRepoPath,
+								Commit:     tagName,
+								Checksum:   checksum,
+								KeepGitDir: true,
+								Auth: dalec.GitAuth{
+									Token: secretName,
+								},
+							},
+						},
+					},
+				}
+			}
+
+			f(ctx, gwc, worker, commit, newSpec)
+		}, testenv.WithSecrets(secretName, "password"))
+	}
+
+	t.Run("preserves tag metadata", func(t *testing.T) {
+		t.Parallel()
+		ctx := startTestSpan(parentCtx, t)
+
+		runGitServerTest(ctx, t, func(ctx context.Context, gwc gwclient.Client, worker llb.State, commit string, newSpec func(string) *dalec.Spec) {
+			req := newSolveRequest(withBuildTarget("debug/sources"), withSpec(ctx, t, newSpec(commit)))
+			sourceRes := solveT(ctx, t, gwc, req)
+			verifySt := worker.Run(dalec.ShArgs(`
+			set -eu
+			git -C /src/`+sourceName+` tag --points-at HEAD > /out/tag
+			git -C /src/`+sourceName+` rev-parse HEAD > /out/head
+			git -C /src/`+sourceName+` rev-parse `+tagName+`^{} > /out/tag-commit
+			git -C /src/`+sourceName+` cat-file -t `+tagName+` > /out/tag-type
+		`), llb.AddMount("/src", resultToState(t, sourceRes), llb.Readonly)).AddMount("/out", llb.Scratch())
+			verifyDef, err := verifySt.Marshal(ctx)
+			assert.NilError(t, err)
+
+			verifyRes, err := gwc.Solve(ctx, gwclient.SolveRequest{Definition: verifyDef.ToPB(), Evaluate: true})
+			assert.NilError(t, err)
+
+			checkFile(ctx, t, "tag", verifyRes, []byte(tagName+"\n"))
+			checkFile(ctx, t, "head", verifyRes, []byte(commit+"\n"))
+			checkFile(ctx, t, "tag-commit", verifyRes, []byte(commit+"\n"))
+			checkFile(ctx, t, "tag-type", verifyRes, []byte("tag\n"))
+		})
+	})
+
+	t.Run("rejects checksum mismatch", func(t *testing.T) {
+		t.Parallel()
+		ctx := startTestSpan(parentCtx, t)
+
+		runGitServerTest(ctx, t, func(ctx context.Context, gwc gwclient.Client, worker llb.State, commit string, newSpec func(string) *dalec.Spec) {
+			_, err := gwc.Solve(ctx, newSolveRequest(withBuildTarget("debug/sources"), withSpec(ctx, t, newSpec("0000000000000000000000000000000000000000"))))
+			if err == nil {
+				t.Fatal("expected git checksum mismatch, but received none")
+			}
+			if !strings.Contains(err.Error(), "expected checksum to match") {
+				t.Fatalf("expected git checksum mismatch, got: %v", err)
+			}
+		})
+	})
+}
+
 // Create a very simple fake module with a limited dependency tree just to
 // keep the test as fast/reliable as possible.
 const gomodFixtureMain = `package main