test: Enable mocking http provider sources in tests

internal/command/init_test.go

@@ -3,8 +3,10 @@ package command
 import (
 	"bytes"
 	"context"
+	"crypto/tls"
 	"encoding/json"
 	"fmt"
+	"io"
 	"log"
 	"maps"
 	"net/http"
@@ -38,6 +40,7 @@ import (
 	"github.com/hashicorp/terraform/internal/configs/configschema"
 	"github.com/hashicorp/terraform/internal/depsfile"
 	"github.com/hashicorp/terraform/internal/getproviders"
+	"github.com/hashicorp/terraform/internal/httpclient"
 	"github.com/hashicorp/terraform/internal/providercache"
 	"github.com/hashicorp/terraform/internal/providers"
 	testing_provider "github.com/hashicorp/terraform/internal/providers/testing"
@@ -3450,6 +3453,73 @@ func TestInit_testsWithModule(t *testing.T) {
 
 // Testing init's behaviors with `state_store` when run in an empty working directory
 func TestInit_stateStore_newWorkingDir(t *testing.T) {
+	t.Run("temporary: test showing use of HTTP server in mock provider source", func(t *testing.T) {
+		// Create a temporary, uninitialized working directory with configuration including a state store
+		td := t.TempDir()
+		testCopyDir(t, testFixturePath("init-with-state-store"), td)
+		t.Chdir(td)
+
+		// Mock provider still needs to be supplied via testingOverrides despite the mock HTTP source
+		mockProvider := mockPluggableStateStorageProvider()
+		mockProviderVersion := getproviders.MustParseVersion("1.2.3")
+		mockProviderAddress := addrs.NewDefaultProvider("test")
+
+		// Set up mock provider source that mocks out downloading hashicorp/test v1.2.3 via HTTP.
+		// This stops Terraform auto-approving the provider installation.
+		source, close := newMockProviderSourceUsingTestHttpServer(t, mockProviderAddress, mockProviderVersion)
+		t.Cleanup(close)
+
+		ui := new(cli.MockUi)
+		view, done := testView(t)
+		meta := Meta{
+			Ui:                        ui,
+			View:                      view,
+			AllowExperimentalFeatures: true,
+			testingOverrides: &testingOverrides{
+				Providers: map[addrs.Provider]providers.Factory{
+					mockProviderAddress: providers.FactoryFixed(mockProvider),
+				},
+			},
+			ProviderSource: source,
+		}
+		c := &InitCommand{
+			Meta: meta,
+		}
+
+		args := []string{"-enable-pluggable-state-storage-experiment=true"}
+		code := c.Run(args)
+		testOutput := done(t)
+		if code != 0 {
+			t.Fatalf("expected code 0 exit code, got %d, output: \n%s", code, testOutput.All())
+		}
+
+		// Check output
+		output := testOutput.All()
+		expectedOutputs := []string{
+			"Initializing the state store...",
+			"Terraform created an empty state file for the default workspace",
+			"Terraform has been successfully initialized!",
+		}
+		for _, expected := range expectedOutputs {
+			if !strings.Contains(output, expected) {
+				t.Fatalf("expected output to include %q, but got':\n %s", expected, output)
+			}
+		}
+
+		// Assert how the mock provider source was called.
+		gotLog := source.CallLog()
+		wantLog := [][]interface{}{
+			{"PackageMeta", mockProviderAddress, mockProviderVersion, getproviders.CurrentPlatform},
+			{"AvailableVersions", mockProviderAddress},
+			{"PackageMeta", mockProviderAddress, mockProviderVersion, getproviders.CurrentPlatform},
+			// The client was retrieved to be used during installation.
+			{"Client"},
+		}
+		if diff := cmp.Diff(wantLog, gotLog); diff != "" {
+			t.Fatalf("unexpected call log\n%s", diff)
+		}
+	})
+
 	t.Run("the init command creates a backend state file, and creates the default workspace by default", func(t *testing.T) {
 		// Create a temporary, uninitialized working directory with configuration including a state store
 		td := t.TempDir()
@@ -5827,6 +5897,134 @@ func newMockProviderSource(t *testing.T, availableProviderVersions map[string][]
 	return getproviders.NewMockSource(packages, nil), close
 }
 
+// newMockProviderSourceViaHTTP is similar to newMockProviderSource except that the metadata (PackageMeta) for each provider
+// reports that the provider is going to be accessed via HTTP
+//
+// Provider binaries are not available via the mock HTTP provider source. This source is sufficient only to allow Terraform
+// to complete the provider installation process while believing it's installing providers over HTTP.
+// This method is not sufficient to enable Terraform to use providers with those names.
+//
+// When using `newMockProviderSourceViaHTTP` to set a value for `(Meta).ProviderSource` in a test, also set up `testOverrides`
+// in the same Meta. That way the provider source will allow the download process to complete, and when Terraform attempts to use
+// those binaries it will instead use the testOverride providers.
+func newMockProviderSourceViaHTTP(t *testing.T, availableProviderVersions map[string][]string, address string, client *http.Client) (source *getproviders.MockSource, close func()) {
+	t.Helper()
+	var packages []getproviders.PackageMeta
+	var closes []func()
+	close = func() {
+		for _, f := range closes {
+			f()
+		}
+	}
+	for source, versions := range availableProviderVersions {
+		addr := addrs.MustParseProviderSourceString(source)
+		for _, versionStr := range versions {
+			version, err := getproviders.ParseVersion(versionStr)
+			if err != nil {
+				close()
+				t.Fatalf("failed to parse %q as a version number for %q: %s", versionStr, addr.ForDisplay(), err)
+			}
+			meta, close, err := getproviders.FakePackageMetaViaHTTP(addr, version, getproviders.VersionList{getproviders.MustParseVersion("5.0")}, getproviders.CurrentPlatform, address, "")
+			if err != nil {
+				close()
+				t.Fatalf("failed to prepare fake package for %s %s: %s", addr.ForDisplay(), versionStr, err)
+			}
+			closes = append(closes, close)
+			packages = append(packages, meta)
+		}
+	}
+
+	return getproviders.NewMockSourceWithClient(packages, nil, client), close
+}
+
+// newMockProviderSourceUsingTestHttpServer is a helper that makes it easier to use newMockProviderSourceViaHTTP.
+// This helper sets up a test HTTP server for use with newMockProviderSourceViaHTTP, and configures a handler that will respond when
+// Terraform attempts to download provider binaries during installation. The mock source is returned an ready to use, and calling code
+// is expected to defer/t.Cleanup the close function.
+//
+// This source is not sufficient for providers to be available to use during a test; when using this helper, also set up testOverrides in
+// the same Meta to provide the actual provider implementations for use during the test.
+//
+// Currently this helper only allows one provider/version to be mocked. In future we could extend it to allow multiple providers/versions.
+func newMockProviderSourceUsingTestHttpServer(t *testing.T, p addrs.Provider, v getproviders.Version) (*getproviders.MockSource, func()) {
+	var closes []func() // See final return values.
+
+	// Get un-started server so we can obtain the port it'll run on.
+	server := httptest.NewUnstartedServer(nil)
+
+	// Prepare a client that ignores TLS errors, since the test server uses a self-signed cert when started with StartTLS.
+	client := httpclient.New()
+	client.Transport = &http.Transport{
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+	}
+
+	// Set up mock provider source that mocks installation via HTTP.
+	source, close := newMockProviderSourceViaHTTP(
+		t,
+		map[string][]string{
+			fmt.Sprintf("%s/%s", p.Namespace, p.Type): {v.String()},
+		},
+		server.Listener.Addr().String(),
+		client,
+	)
+	closes = append(closes, close)
+
+	// Supply a download location so that the installation completes ok
+	// while Terraform still believes it's downloading a provider via HTTP.
+	providerMetadata, err := source.PackageMeta(
+		context.Background(),
+		p,
+		v,
+		getproviders.CurrentPlatform,
+	)
+	if err != nil {
+		t.Fatalf("failed to get provider metadata: %s", err)
+	}
+
+	// Make Terraform believe it's downloading the provider.
+	// Any requests to the test server that aren't for that purpose will cause the test to fail.
+	server.Config = &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		providerLocationPath := strings.ReplaceAll(
+			providerMetadata.Location.String(),
+			"https://"+server.Listener.Addr().String(),
+			"",
+		)
+		// This is the URL that the init command will hit to download the provider, so we return a valid provider archive.
+		if r.URL.Path == providerLocationPath {
+			// This code returns data in the temporary file that's created by the mock provider source.
+			// This 'downloaded' is not used when Terraform uses the provider after the mock installation completes;
+			// Terraform will look for will use testOverrides in the Meta set up for this test.
+			//
+			// Although it's not used later we need to use this file (versus empty or made-up bytes) to enable installation
+			// logic to receive data with the correct checksum.
+			f, err := os.Open(providerMetadata.Filename)
+			if err != nil {
+				t.Fatalf("failed to open mock source file: %s", err)
+			}
+			defer f.Close()
+			archiveBytes, err := io.ReadAll(f)
+			if err != nil {
+				t.Fatalf("failed to read mock source file: %s", err)
+			}
+			w.WriteHeader(http.StatusOK)
+			w.Write(archiveBytes)
+			return
+		} else {
+			t.Fatalf("unexpected URL path: %s", r.URL.Path)
+		}
+	})}
+
+	server.StartTLS()
+	closes = append(closes, server.Close)
+
+	allCloses := func() {
+		for _, f := range closes {
+			f()
+		}
+	}
+	return source, allCloses
+}
+
 // installFakeProviderPackages installs a fake package for the given provider
 // names (interpreted as a "default" provider address) and versions into the
 // local plugin cache for the given "meta".
@@ -5882,7 +6080,7 @@ func installFakeProviderPackagesElsewhere(t *testing.T, cacheDir *providercache.
 			if err != nil {
 				t.Fatalf("failed to prepare fake package for %s %s: %s", name, versionStr, err)
 			}
-			_, err = cacheDir.InstallPackage(context.Background(), meta, nil)
+			_, err = cacheDir.InstallPackage(context.Background(), meta, nil, nil)
 			if err != nil {
 				t.Fatalf("failed to install fake package for %s %s: %s", name, versionStr, err)
 			}

internal/getproviders/memoize_source_test.go

@@ -88,7 +88,6 @@ func TestMemoizeSource(t *testing.T) {
 		if warns[0] != "WARNING!" {
 			t.Fatalf("wrong result! Got %s, expected \"WARNING!\"", warns[0])
 		}
-
 	})
 	t.Run("PackageMeta for existing provider", func(t *testing.T) {
 		mock := NewMockSource([]PackageMeta{meta}, nil)

internal/getproviders/mock_source.go

@@ -6,7 +6,7 @@ import (
 	"crypto/sha256"
 	"fmt"
 	"io"
-	"io/ioutil"
+	"net/http"
 	"os"
 
 	"github.com/hashicorp/terraform/internal/addrs"
@@ -23,9 +23,17 @@ type MockSource struct {
 	packages []PackageMeta
 	warnings map[addrs.Provider]Warnings
 	calls    [][]interface{}
+
+	// Client is set during test-setup; it's a client created from a test http server
+	// When we set a client here the we intend to specifically test Terraform's behaviour
+	// when downloading providers via HTTP.
+	client *http.Client
 }
 
-var _ Source = (*MockSource)(nil)
+var (
+	_ Source               = (*MockSource)(nil)
+	_ MockSourceWithClient = (*MockSource)(nil)
+)
 
 // NewMockSource creates and returns a MockSource with the given packages.
 //
@@ -40,6 +48,21 @@ func NewMockSource(packages []PackageMeta, warns map[addrs.Provider]Warnings) *M
 	}
 }
 
+// NewMockSourceWithClient is used when the mock source is intended to resemble downloading a provider
+// via HTTP. See the newMockProviderSourceUsingTestHttpServer helper, which creates the test server needed with this mock.
+func NewMockSourceWithClient(packages []PackageMeta, warns map[addrs.Provider]Warnings, client *http.Client) *MockSource {
+	return &MockSource{
+		packages: packages,
+		warnings: warns,
+		client:   client,
+	}
+}
+
+func (s *MockSource) Client() *http.Client {
+	s.calls = append(s.calls, []interface{}{"Client"})
+	return s.client
+}
+
 // AvailableVersions returns all of the versions of the given provider that
 // are available in the fixed set of packages that were passed to
 // NewMockSource when creating the receiving source.
@@ -153,7 +176,7 @@ func FakePackageMeta(provider addrs.Provider, version Version, protocols Version
 // should call the callback even if this function returns an error, because
 // some error conditions leave a partially-created file on disk.
 func FakeInstallablePackageMeta(provider addrs.Provider, version Version, protocols VersionList, target Platform, execFilename string) (PackageMeta, func(), error) {
-	f, err := ioutil.TempFile("", "terraform-getproviders-fake-package-")
+	f, err := os.CreateTemp("", "terraform-getproviders-fake-package-")
 	if err != nil {
 		return PackageMeta{}, func() {}, err
 	}
@@ -212,6 +235,79 @@ func FakeInstallablePackageMeta(provider addrs.Provider, version Version, protoc
 	return meta, close, nil
 }
 
+// This is basically the same as FakePackageMeta, except that we'll use a PackageHTTPURL instead of a PackageLocalArchive when creating metadata for the provider.
+// By doing so, we create a mock source that makes Terraform believe it's downloading the provider via HTTP, instead of from a local archive.
+//
+// The caller is responsible for calling the close callback to clean up the temporary file.
+// The temporary file is only used to calculate checksums and isn't actually used to install the provider in the test.
+func FakePackageMetaViaHTTP(provider addrs.Provider, version Version, protocols VersionList, target Platform, locationBaseUrl string, execFilename string) (PackageMeta, func(), error) {
+	f, err := os.CreateTemp("", "terraform-getproviders-fake-package-")
+	if err != nil {
+		return PackageMeta{}, func() {}, err
+	}
+
+	// After this point, all of our return paths should include this as the
+	// close callback.
+	close := func() {
+		f.Close()
+		os.Remove(f.Name())
+	}
+
+	if execFilename == "" {
+		execFilename = fmt.Sprintf("terraform-provider-%s_%s", provider.Type, version.String())
+		if target.OS == "windows" {
+			// For a little more (technically unnecessary) realism...
+			execFilename += ".exe"
+		}
+	}
+
+	zw := zip.NewWriter(f)
+	fw, err := zw.Create(execFilename)
+	if err != nil {
+		return PackageMeta{}, close, fmt.Errorf("failed to add %s to mock zip file: %s", execFilename, err)
+	}
+	fmt.Fprintf(fw, "This is a fake provider package for %s %s, not a real provider.\n", provider, version)
+	err = zw.Close()
+	if err != nil {
+		return PackageMeta{}, close, fmt.Errorf("failed to close the mock zip file: %s", err)
+	}
+
+	// Compute the SHA256 checksum of the generated file, to allow package
+	// authentication code to be exercised.
+	f.Seek(0, io.SeekStart)
+	h := sha256.New()
+	io.Copy(h, f)
+	checksum := [32]byte{}
+	h.Sum(checksum[:0])
+
+	meta := PackageMeta{
+		Provider:         provider,
+		Version:          version,
+		ProtocolVersions: protocols,
+		TargetPlatform:   target,
+
+		Location: PackageHTTPURL(
+			fmt.Sprintf(
+				"https://%[1]s/terraform-provider-%[2]s/%[3]s/terraform-provider-%[2]s_%[3]s_%[4]s.zip",
+				locationBaseUrl,
+				provider.Type,
+				version.String(),
+				target.String(),
+			),
+		),
+
+		// This is a fake filename that mimics what a real registry might
+		// indicate as a good filename for this package, in case some caller
+		// intends to use it to name a local copy of the temporary file.
+		// (At the time of writing, no caller actually does that, but who
+		// knows what the future holds?)
+		Filename: f.Name(),
+
+		Authentication: NewArchiveChecksumAuthentication(target, checksum),
+	}
+	return meta, close, nil
+}
+
 func (s *MockSource) ForDisplay(provider addrs.Provider) string {
 	return "mock source"
 }

internal/getproviders/multi_source_test.go

@@ -182,7 +182,6 @@ func TestMultiSourceAvailableVersions(t *testing.T) {
 		if err.Error() != wantErr {
 			t.Fatalf("wrong error.\ngot:  %s\nwant: %s\n", err, wantErr)
 		}
-
 	})
 
 	t.Run("merging with warnings", func(t *testing.T) {