Consolidate go modules for bridged providers

Author: blampe

.github/workflows/update-workflows.yml

@@ -76,10 +76,9 @@ jobs:
           rm pulumi-${{ inputs.provider_name }}/.github/workflows/master.yml || echo "not found"
       - name: Generate workflow files into pulumi-${{ inputs.provider_name }}
         if: inputs.bridged
-        run: |
-          cd ci-mgmt/provider-ci && go run ./... generate \
-                --config ../../pulumi-${{ inputs.provider_name }}/.ci-mgmt.yaml \
-                --out ../../pulumi-${{ inputs.provider_name }}
+        run: go run github.com/pulumi/ci-mgmt/provider-ci@${{ github.sha }} generate
+        shell: bash
+        working-directory: pulumi-${{ inputs.provider_name }}
       - name: Copy files from ci-mgmt to pulumi-${{ inputs.provider_name }}
         if: ${{ inputs.bridged != true }}
         run: |

provider-ci/Makefile

@@ -7,7 +7,7 @@ ACTIONLINT_VERSION := 1.6.24
 ACTIONLINT := bin/actionlint-$(ACTIONLINT_VERSION)
 
 .PHONY: all test gen ensure
-all: ensure test format lint
+all: ensure test format lint unit
 test: test-providers
 gen: test-providers
 ensure:: bin/provider-ci $(ACTIONLINT)
@@ -20,7 +20,7 @@ $(ACTIONLINT):
 	mv bin/actionlint $(ACTIONLINT)
 
 # Basic helper targets.
-.PHONY: clean lint format
+.PHONY: clean lint format unit
 clean:
 	rm -rf bin
 
@@ -30,6 +30,9 @@ lint:
 format:
 	go fmt ./...
 
+unit:
+	go test -v ./...
+
 # We check in a subset of provider workflows so template changes are visible in PR diffs.
 #
 # This provides an example of generated providers for PR reviewers. This target

provider-ci/go.mod

@@ -1,10 +1,11 @@
 module github.com/pulumi/ci-mgmt/provider-ci
 
-go 1.21
+go 1.23.3
 
 require (
 	github.com/Masterminds/sprig v2.22.0+incompatible
 	github.com/spf13/cobra v1.7.0
+	github.com/stretchr/testify v1.7.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 
@@ -19,8 +20,8 @@ require (
 	github.com/kr/pretty v0.1.0 // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/stretchr/testify v1.7.0 // indirect
 	golang.org/x/crypto v0.17.0 // indirect
 	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
 )

provider-ci/internal/pkg/migrations/consolidate_modules.go

@@ -0,0 +1,213 @@
+package migrations
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+)
+
+// consolidateModules moves ./provider/go.mod to the repository root (./go.mod)
+// and consolidates it with ./examples/go.mod and ./tests/go.mod (if they
+// exist). The SDK module is untouched, so SDK consumers are unaffected.
+//
+// The migration simplifies dependency management; eliminates the need for
+// `replace` directives (except for shims); ensures consistent package
+// versioning between provider logic and tests; makes it easier to share code;
+// yields better IDE integration; and is all-around easier to work with.
+//
+// This was initially motivated by work to shard our integration tests. Our old
+// module structure sometimes forced us to put integration tests alongside unit
+// tests under ./provider. We also had integration tests under ./examples.
+// Being able to shard both of those things concurrently (as part of a single
+// `go test` command) wasn't possible due to them existing in separate modules.
+//
+// See also: https://go.dev/wiki/Modules#should-i-have-multiple-modules-in-a-single-repository
+type consolidateModules struct{}
+
+func (consolidateModules) Name() string {
+	return "Consolidate Go modules"
+}
+
+func (consolidateModules) ShouldRun(_ string) bool {
+	_, err := os.Stat("provider/go.mod")
+	return err == nil // Exists.
+}
+
+func (consolidateModules) Migrate(_, outDir string) error {
+	run := func(args ...string) ([]byte, error) {
+		cmd := exec.Command(args[0], args[1:]...)
+		cmd.Dir = outDir
+		cmd.Stderr = os.Stderr
+		return cmd.Output()
+	}
+
+	// Move provider's module down.
+	if _, err := run("git", "mv", "-f", "provider/go.mod", "go.mod"); err != nil {
+		return fmt.Errorf("moving provider/go.mod: %w", err)
+	}
+	if _, err := run("git", "mv", "-f", "provider/go.sum", "go.sum"); err != nil {
+		return fmt.Errorf("moving provider/go.sum: %w", err)
+	}
+
+	// Load the module as JSON.
+	out, err := run("go", "mod", "edit", "-json", "go.mod")
+	if err != nil {
+		return fmt.Errorf("exporting go.mod: %w", err)
+	}
+	var mod gomod
+	err = json.Unmarshal(out, &mod)
+	if err != nil {
+		return fmt.Errorf("reading go.mod: %w", err)
+	}
+
+	// Move relative `replace` paths up or down a directory.
+	for idx, r := range mod.Replace {
+		if strings.HasPrefix(r.New.Path, "../") {
+			r.New.Path = strings.Replace(r.New.Path, "../", "./", 1)
+		} else if strings.HasPrefix(r.New.Path, "./") {
+			r.New.Path = strings.Replace(r.New.Path, "./", "./provider/", 1)
+		}
+		if r.New.Path == mod.Replace[idx].New.Path {
+			continue // Unchanged.
+		}
+
+		// Commit the changes.
+		old := r.Old.Path
+		if r.Old.Version != "" {
+			old += "@" + r.Old.Version
+		}
+		_, err = run("go", "mod", "edit", fmt.Sprintf("-replace=%s=%s", old, r.New.Path))
+		if err != nil {
+			return fmt.Errorf("replacing %q: %w", old, err)
+		}
+	}
+
+	// Remove examples/tests modules. We'll recover their requirements with a
+	// `tidy` at the end. It's OK if these don't exist.
+	_, _ = run("git", "rm", "examples/go.mod")
+	_, _ = run("git", "rm", "examples/go.sum")
+	_, _ = run("git", "rm", "tests/go.mod")
+	_, _ = run("git", "rm", "tests/go.sum")
+
+	// Rewrite our module path and determine our new import, if it's changed.
+	//
+	// The module `github.com/pulumi/pulumi-foo/provider/v6` becomes
+	// `github.com/pulumi/pulumi-foo/v6` and existing code should be imported
+	// as `github.com/pulumi/pulumi-foo/v6/provider`.
+	//
+	// For v1 modules, `github.com/pulumi/pulumi-foo/provider` becomes
+	// `github.com/pulumi/pulumi-foo` and existing imports are unchanged.
+
+	oldImport := mod.Module.Path
+	newModule := filepath.Dir(oldImport) // Strip "/vN" or "/provider".
+	newImport := oldImport
+
+	// Handle major version.
+	if base := filepath.Base(oldImport); base != "provider" {
+		if !strings.HasPrefix(base, "v") {
+			return fmt.Errorf("expected a major version, got %q", base)
+		}
+		newModule = filepath.Join(filepath.Dir(newModule), base)
+		newImport = filepath.Join(newModule, "provider")
+	}
+
+	// Update our module name.
+	_, err = run("go", "mod", "edit", "-module="+newModule)
+	if err != nil {
+		return fmt.Errorf("rewriting module name: %w", err)
+	}
+
+	// Re-write imports for our provider, examples, and tests modules.
+	rewriteImport := func(oldImport, newImport string) error {
+		if oldImport == newImport {
+			return nil // Nothing to do.
+		}
+		_, err := run("find", ".",
+			"-type", "f",
+			"-not", "-path", "./sdk/*",
+			"-not", "-path", "./upstream/*",
+			"-not", "-path", "./.git/*",
+			"-not", "-path", "./.pulumi/*",
+			"-exec", "sed", "-i.bak",
+			fmt.Sprintf("s/%s/%s/g",
+				strings.Replace(oldImport, "/", `\/`, -1),
+				strings.Replace(newImport, "/", `\/`, -1),
+			), "{}", ";")
+		if err != nil {
+			return fmt.Errorf("rewriting %q to %q: %w", oldImport, newImport, err)
+		}
+		_, err = run("find", ".", "-name", "*.bak", "-exec", "rm", "{}", "+")
+		if err != nil {
+			return fmt.Errorf("cleaning up: %w", err)
+		}
+		return nil
+
+	}
+	if err := rewriteImport(oldImport, newImport); err != nil {
+		return err
+	}
+	if err := rewriteImport(
+		strings.Replace(oldImport, "provider", "examples", 1),
+		strings.Replace(newImport, "provider", "examples", 1),
+	); err != nil {
+		return err
+	}
+	if err := rewriteImport(
+		strings.Replace(oldImport, "provider", "tests", 1),
+		strings.Replace(newImport, "provider", "tests", 1),
+	); err != nil {
+		return err
+	}
+
+	// Tidy up.
+	_, err = run("go", "mod", "tidy")
+	if err != nil {
+		return fmt.Errorf("tidying up: %w", err)
+	}
+
+	return nil
+
+}
+
+// The types below are for loading the module as JSON and are copied from `go
+// help mod edit`.
+
+type module struct {
+	Path    string
+	Version string
+}
+
+type gomod struct {
+	Module    modpath
+	Go        string
+	Toolchain string
+	Require   []requirement
+	Exclude   []module
+	Replace   []replace
+	Retract   []retract
+}
+
+type modpath struct {
+	Path       string
+	Deprecated string
+}
+
+type requirement struct {
+	Path     string
+	Version  string
+	Indirect bool
+}
+
+type replace struct {
+	Old module
+	New module
+}
+
+type retract struct {
+	Low       string
+	High      string
+	Rationale string
+}

provider-ci/internal/pkg/migrations/migrations.go

@@ -14,6 +14,7 @@ type Migration interface {
 
 func Migrate(templateName, outDir string) error {
 	migrations := []Migration{
+		consolidateModules{},
 		fixupBridgeImports{},
 		removeExplicitSDKDependency{},
 		ignoreMakeDir{},

provider-ci/internal/pkg/migrations/migrations_test.go

@@ -0,0 +1,85 @@
+package migrations
+
+import (
+	"os"
+	"os/exec"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestConsolidateModules(t *testing.T) {
+	tests, err := os.ReadDir("testdata/modules")
+	require.NoError(t, err)
+
+	for _, tt := range tests {
+		t.Run(tt.Name(), func(t *testing.T) {
+			if !tt.IsDir() {
+				return
+			}
+
+			tmp := t.TempDir()
+
+			// Copy GIVEN to a temp directory so we can mutate it.
+			given := filepath.Join("testdata/modules", tt.Name(), "GIVEN")
+			err = os.CopyFS(tmp, os.DirFS(given))
+
+			// We need to operate on a git repo, so initialize one.
+			out, err := exec.Command("git", "init", tmp).CombinedOutput()
+			require.NoError(t, err, string(out))
+			out, err = exec.Command("git", "-C", tmp, "add", ".").CombinedOutput()
+			require.NoError(t, err, string(out))
+			out, err = exec.Command("git", "-C", tmp,
+				"-c", "user.name=pulumi-bot", "-c", "[email protected]",
+				"commit", "-m", "Initial commit").CombinedOutput()
+			require.NoError(t, err, string(out))
+
+			// Do the migration.
+			m := consolidateModules{}
+			err = m.Migrate("", tmp)
+			require.NoError(t, err)
+
+			// Make sure we got the expected output.
+			want := filepath.Join("testdata/modules", tt.Name(), "WANT")
+			assertDirectoryContains(t, want, tmp)
+			assertDirectoryContains(t, tmp, want)
+		})
+	}
+}
+
+// assertDirectoryContains asserts that dir1 contains all of the files in dir2
+// with exactly the same context. The .git directory is ignored.
+func assertDirectoryContains(t *testing.T, dir1, dir2 string) {
+	t.Helper()
+
+	entries, err := os.ReadDir(dir2)
+	require.NoError(t, err)
+
+	for _, entry := range entries {
+		if entry.Name() == ".git" {
+			continue
+		}
+
+		stat, err := os.Stat(filepath.Join(dir1, entry.Name()))
+		assert.NoError(t, err)
+		assert.Equal(t, entry.IsDir(), stat.IsDir())
+
+		subPath1 := filepath.Join(dir1, entry.Name())
+		subPath2 := filepath.Join(dir2, entry.Name())
+
+		if entry.IsDir() {
+			assertDirectoryContains(t, subPath1, subPath2)
+			continue
+		}
+
+		content1, err := os.ReadFile(subPath1)
+		assert.NoError(t, err)
+
+		content2, err := os.ReadFile(subPath2)
+		assert.NoError(t, err)
+
+		assert.Equal(t, string(content1), string(content2), subPath1)
+	}
+}

provider-ci/internal/pkg/migrations/testdata/modules/v1/GIVEN/examples/examples_test.go

@@ -0,0 +1,6 @@
+package examples
+
+import (
+	_ "github.com/pulumi/pulumi-foo/examples/some-package"
+	_ "github.com/pulumi/pulumi-foo/provider"
+)

provider-ci/internal/pkg/migrations/testdata/modules/v1/GIVEN/examples/go.mod

@@ -0,0 +1,12 @@
+module github.com/pulumi/pulumi-foo/examples
+
+go 1.22.7
+
+replace (
+	github.com/pulumi/pulumi-foo/provider => ../provider
+	github.com/terraform-providers/terraform-provider-foo/shim => ../provider/shim
+)
+
+require github.com/pulumi/pulumi-foo/provider v1.0.0-20230306191832-8c7659ab0229
+
+require github.com/terraform-providers/terraform-provider-foo/shim v0.0.0 // indirect

provider-ci/internal/pkg/migrations/testdata/modules/v1/GIVEN/examples/go.sum

@@ -0,0 +1 @@
+package bar