[PF] Property based Configure tests

Author: iwahbe

go.mod

@@ -22,7 +22,7 @@ require (
 	github.com/hashicorp/hcl v1.0.0
 	github.com/hashicorp/hcl/v2 v2.19.1
 	github.com/hashicorp/hil v0.0.0-20190212132231-97b3a9cdfa93
-	github.com/hashicorp/terraform-plugin-framework v1.7.0
+	github.com/hashicorp/terraform-plugin-framework v1.12.0
 	github.com/hashicorp/terraform-plugin-framework-validators v0.12.0
 	github.com/hashicorp/terraform-plugin-mux v0.16.0
 	github.com/hashicorp/terraform-plugin-sdk v1.7.0

go.sum

@@ -1666,8 +1666,8 @@ github.com/hashicorp/terraform-json v0.4.0/go.mod h1:eAbqb4w0pSlRmdvl8fOyHAi/+8j
 github.com/hashicorp/terraform-json v0.19.0/go.mod h1:qdeBs11ovMzo5puhrRibdD6d2Dq6TyE/28JiU4tIQxk=
 github.com/hashicorp/terraform-json v0.21.0 h1:9NQxbLNqPbEMze+S6+YluEdXgJmhQykRyRNd+zTI05U=
 github.com/hashicorp/terraform-json v0.21.0/go.mod h1:qdeBs11ovMzo5puhrRibdD6d2Dq6TyE/28JiU4tIQxk=
-github.com/hashicorp/terraform-plugin-framework v1.7.0 h1:wOULbVmfONnJo9iq7/q+iBOBJul5vRovaYJIu2cY/Pw=
-github.com/hashicorp/terraform-plugin-framework v1.7.0/go.mod h1:jY9Id+3KbZ17OMpulgnWLSfwxNVYSoYBQFTgsx044CI=
+github.com/hashicorp/terraform-plugin-framework v1.12.0 h1:7HKaueHPaikX5/7cbC1r9d1m12iYHY+FlNZEGxQ42CQ=
+github.com/hashicorp/terraform-plugin-framework v1.12.0/go.mod h1:N/IOQ2uYjW60Jp39Cp3mw7I/OpC/GfZ0385R0YibmkE=
 github.com/hashicorp/terraform-plugin-framework-validators v0.12.0 h1:HOjBuMbOEzl7snOdOoUfE2Jgeto6JOjLVQ39Ls2nksc=
 github.com/hashicorp/terraform-plugin-framework-validators v0.12.0/go.mod h1:jfHGE/gzjxYz6XoUwi/aYiiKrJDeutQNUtGQXkaHklg=
 github.com/hashicorp/terraform-plugin-go v0.22.0/go.mod h1:mPULV91VKss7sik6KFEcEu7HuTogMLLO/EvWCuFkRVE=

pf/go.mod

@@ -41,7 +41,7 @@ require (
 	github.com/google/s2a-go v0.1.7 // indirect
 	github.com/hashicorp/go-hclog v1.6.3 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
-	github.com/hashicorp/terraform-plugin-framework v1.11.0 // indirect
+	github.com/hashicorp/terraform-plugin-framework v1.12.0 // indirect
 	github.com/hashicorp/terraform-plugin-go v0.24.0 // indirect
 	github.com/hashicorp/terraform-plugin-log v0.9.0 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect

pf/go.sum

@@ -567,8 +567,8 @@ github.com/hashicorp/hil v0.0.0-20190212132231-97b3a9cdfa93 h1:T1Q6ag9tCwun16AW+
 github.com/hashicorp/hil v0.0.0-20190212132231-97b3a9cdfa93/go.mod h1:n2TSygSNwsLJ76m8qFXTSc7beTb+auJxYdqrnoqwZWE=
 github.com/hashicorp/logutils v1.0.0 h1:dLEQVugN8vlakKOUE3ihGLTZJRB4j+M2cdTm/ORI65Y=
 github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
-github.com/hashicorp/terraform-plugin-framework v1.11.0 h1:M7+9zBArexHFXDx/pKTxjE6n/2UCXY6b8FIq9ZYhwfE=
-github.com/hashicorp/terraform-plugin-framework v1.11.0/go.mod h1:qBXLDn69kM97NNVi/MQ9qgd1uWWsVftGSnygYG1tImM=
+github.com/hashicorp/terraform-plugin-framework v1.12.0 h1:7HKaueHPaikX5/7cbC1r9d1m12iYHY+FlNZEGxQ42CQ=
+github.com/hashicorp/terraform-plugin-framework v1.12.0/go.mod h1:N/IOQ2uYjW60Jp39Cp3mw7I/OpC/GfZ0385R0YibmkE=
 github.com/hashicorp/terraform-plugin-framework-validators v0.12.0 h1:HOjBuMbOEzl7snOdOoUfE2Jgeto6JOjLVQ39Ls2nksc=
 github.com/hashicorp/terraform-plugin-framework-validators v0.12.0/go.mod h1:jfHGE/gzjxYz6XoUwi/aYiiKrJDeutQNUtGQXkaHklg=
 github.com/hashicorp/terraform-plugin-go v0.24.0 h1:2WpHhginCdVhFIrWHxDEg6RBn3YaWzR2o6qUeIEat2U=

pkg/pf/tests/internal/cross-tests/configure.go

@@ -30,6 +30,7 @@ import (
 	pb "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/pf/tests/internal/providerbuilder"
 	"github.com/pulumi/pulumi-terraform-bridge/v3/pkg/pf/tfbridge"
 	"github.com/pulumi/pulumi-terraform-bridge/v3/pkg/pf/tfgen"
+	"github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tests/assume"
 	crosstests "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tests/cross-tests"
 	"github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tests/tfcheck"
 	"github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfbridge/info"
@@ -50,10 +51,13 @@ import (
 //	}
 //
 // For details on the test itself, see [Configure].
-func MakeConfigure(schema schema.Schema, tfConfig map[string]cty.Value, puConfig resource.PropertyMap) func(t *testing.T) {
+func MakeConfigure(
+	schema schema.Schema, tfConfig map[string]cty.Value, puConfig resource.PropertyMap,
+	options ...ConfigureOption,
+) func(t *testing.T) {
 	return func(t *testing.T) {
 		t.Parallel()
-		Configure(t, schema, tfConfig, puConfig)
+		Configure(t, schema, tfConfig, puConfig, options...)
 	}
 }
 
@@ -81,12 +85,17 @@ func MakeConfigure(schema schema.Schema, tfConfig map[string]cty.Value, puConfig
 //	+--------------------+                      +---------------------+
 //
 // Configure should be safe to run in parallel.
-func Configure(t *testing.T, schema schema.Schema, tfConfig map[string]cty.Value, puConfig resource.PropertyMap) {
-	skipUnlessLinux(t)
+func Configure(
+	t TestingT, schema schema.Schema, tfConfig map[string]cty.Value, puConfig resource.PropertyMap,
+	options ...ConfigureOption,
+) {
+	assume.TerraformCLI(t)
+
+	var opts configureOptions
+	for _, o := range options {
+		o(&opts)
+	}
 
-	// By default, logs only show when they are on a failed test. By logging to
-	// topLevelT, we can log items to be shown if downstream tests fail.
-	topLevelT := t
 	const providerName = "test"
 
 	prov := func(config *tfsdk.Config) *pb.Provider {
@@ -103,8 +112,11 @@ func Configure(t *testing.T, schema schema.Schema, tfConfig map[string]cty.Value
 	}
 
 	var tfOutput, puOutput tfsdk.Config
-	t.Run("tf", func(t *testing.T) {
-		defer propageteSkip(topLevelT, t)
+	var runOnFail []func(t TestingT)
+
+	logf := func(msg string, a ...any) { runOnFail = append(runOnFail, func(t TestingT) { t.Logf(msg, a...) }) }
+
+	withAugmentedT(t, func(t *augmentedT) { // --- Run Terraform Provider ---
 		var hcl bytes.Buffer
 		err := crosstests.WritePF(&hcl).Provider(schema, providerName, tfConfig)
 		require.NoError(t, err)
@@ -120,13 +132,12 @@ resource "` + providerName + `_res" "res" {}
 
 		driver.Write(t, hcl.String())
 		plan, err := driver.Plan(t)
-		require.NoError(t, err)
+		require.NoError(t, err, "failed to generate TF plan")
 		err = driver.Apply(t, plan)
 		require.NoError(t, err)
 	})
 
-	t.Run("bridged", func(t *testing.T) {
-		defer propageteSkip(topLevelT, t)
+	withAugmentedT(t, func(t *augmentedT) { // --- Run Pulumi Provider ---
 		dir := t.TempDir()
 
 		pulumiYaml := map[string]any{
@@ -145,7 +156,7 @@ resource "` + providerName + `_res" "res" {}
 
 		bytes, err := yaml.Marshal(pulumiYaml)
 		require.NoError(t, err)
-		topLevelT.Logf("Pulumi.yaml:\n%s", string(bytes))
+		logf("Pulumi.yaml:\n%s", string(bytes))
 		err = os.WriteFile(filepath.Join(dir, "Pulumi.yaml"), bytes, 0600)
 		require.NoError(t, err)
 
@@ -187,11 +198,49 @@ resource "` + providerName + `_res" "res" {}
 		contract.Ignore(test.Up(t))      // Assert that the update succeeded, but not the result.
 	})
 
-	skipCompare := t.Failed() || t.Skipped()
-	t.Run("compare", func(t *testing.T) {
-		if skipCompare {
-			t.Skipf("skipping since earlier steps did not complete")
-		}
+	// --- Compare results -----------------------------
+	if opts.testEqual != nil {
+		opts.testEqual(t, tfOutput, puOutput)
+	} else {
 		assert.Equal(t, tfOutput, puOutput)
-	})
+	}
+
+	if t.Failed() {
+		for _, f := range runOnFail {
+			f(t)
+		}
+	}
+}
+
+// An option for configuring [Configure] or [MakeConfigure].
+//
+// Existing options are:
+// - [WithConfigureEquals]
+type ConfigureOption func(*configureOptions)
+
+type configureOptions struct {
+	testEqual func(t TestingT, tfOutput, puOutput tfsdk.Config)
+}
+
+// WithConfigureEqual defines a comparison function for the cross-test.
+//
+// This function is called after both the Terraform and Pulumi portions have run, and is
+// responsible for asserting that the results match.
+//
+// Here are 2 examples:
+//
+//	// Assert that both Terraform and Pulumi ran, but do not assert anything about their behavior.
+//	WithConfigureEqual(func(t TestingT, tfOutput, puOutput tfsdk.Config) {})
+//
+//	// Assert that the underlying provider witnessed saw could not distinguish between
+//	// the direct and bridged call (the default behavior).
+//	WithConfigureEqual(func(t TestingT, tfOutput, puOutput tfsdk.Config) {
+//		assert.Equal(t, tfOutput, puOutput)
+//	})
+//
+// WithConfigureEqual should be used only when the direct and bridged providers don't
+// agree, to limit the scope of the test so it can be checked in. In general, usage should
+// be accompanied by a bridge issue to track the discrepancy.
+func WithConfigureEqual(equal func(t TestingT, tfOutput, puOutput tfsdk.Config)) ConfigureOption {
+	return func(opts *configureOptions) { opts.testEqual = equal }
 }

pkg/pf/tests/internal/cross-tests/util.go

@@ -17,22 +17,30 @@ package crosstests
 import (
 	"context"
 	"os"
-	"runtime"
 	"strings"
-	"testing"
+	"time"
 
 	"github.com/pulumi/pulumi/sdk/v3/go/common/diag"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/resource"
 	"github.com/stretchr/testify/require"
 )
 
-func propageteSkip(parent, child *testing.T) {
-	if child.Skipped() {
-		parent.Skipf("skipping due to skipped child test")
-	}
+// TestingT describers what crosstests needs to run a test with.
+//
+// TestingT should be compatible with [pgregory.net/rapid.T].
+type TestingT interface {
+	Skip(args ...any)
+	Failed() bool
+	Errorf(format string, args ...any)
+	Name() string
+	Log(...any)
+	Logf(string, ...any)
+	Fail()
+	FailNow()
+	Helper()
 }
 
-type testLogSink struct{ t *testing.T }
+type testLogSink struct{ t TestingT }
 
 func (s testLogSink) Log(_ context.Context, sev diag.Severity, urn resource.URN, msg string) error {
 	return s.log("LOG", sev, urn, msg)
@@ -50,7 +58,7 @@ func (s testLogSink) log(kind string, sev diag.Severity, urn resource.URN, msg s
 	return nil
 }
 
-func convertResourceValue(t *testing.T, properties resource.PropertyMap) map[string]any {
+func convertResourceValue(t TestingT, properties resource.PropertyMap) map[string]any {
 	var convertValue func(resource.PropertyValue) (any, bool)
 	convertValue = func(v resource.PropertyValue) (any, bool) {
 		if v.IsComputed() {
@@ -81,8 +89,67 @@ func convertResourceValue(t *testing.T, properties resource.PropertyMap) map[str
 	return properties.MapRepl(nil, convertValue)
 }
 
-func skipUnlessLinux(t *testing.T) {
-	if ci, ok := os.LookupEnv("CI"); ok && ci == "true" && !strings.Contains(strings.ToLower(runtime.GOOS), "linux") {
-		t.Skip("Skipping on non-Linux platforms as our CI does not yet install Terraform CLI required for these tests")
+func withAugmentedT(t TestingT, f func(t *augmentedT)) {
+	c := augmentedT{TestingT: t}
+	defer c.cleanup()
+	f(&c)
+}
+
+// augmentedT augments
+type augmentedT struct {
+	TestingT
+	tasks []func()
+}
+
+// TempDir returns a temporary directory for the test to use.
+// The directory is automatically removed when the test and
+// all its subtests complete.
+// Each subsequent call to t.TempDir returns a unique directory;
+// if the directory creation fails, TempDir terminates the test by calling Fatal.
+func (t *augmentedT) TempDir() string {
+	// If the underlying TestingT actually implements TempDir, then just call that.
+	if t, ok := t.TestingT.(interface{ TempDir() string }); ok {
+		return t.TempDir()
+	}
+
+	// Re-implement TempDir:
+
+	name := t.Name()
+	name = strings.ReplaceAll(name, "#", "")
+	name = strings.ReplaceAll(name, string(os.PathSeparator), "")
+	dir, err := os.MkdirTemp("", name)
+	require.NoError(t, err)
+	return dir
+}
+
+func (t *augmentedT) Cleanup(f func()) {
+	// If the underlying TestingT actually implements Cleanup, then just call that.
+	if t, ok := t.TestingT.(interface{ Cleanup(f func()) }); ok {
+		t.Cleanup(f)
+		return
+	}
+
+	// Add f to the set of tasks to be cleaned up later. Cleanup is only valid when
+	// called in a context where t.cleanup() will be called, such as [withAugmentedT].
+	t.tasks = append(t.tasks, f)
+}
+
+func (t *augmentedT) Deadline() (time.Time, bool) {
+	// If the underlying TestingT actually implements Deadline, then just call that.
+	if t, ok := t.TestingT.(interface{ Deadline() (time.Time, bool) }); ok {
+		return t.Deadline()
+	}
+
+	// Otherwise the test has no deadline.
+
+	return time.Time{}, false
+}
+
+func (t *augmentedT) cleanup() {
+	for i := len(t.tasks) - 1; i >= 0; i-- {
+		v := t.tasks[i]
+		if v != nil {
+			v()
+		}
 	}
 }

pkg/pf/tests/internal/cross-tests/util_test.go

@@ -0,0 +1,46 @@
+// Copyright 2016-2024, Pulumi Corporation.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crosstests
+
+import (
+	"testing"
+
+	"github.com/pulumi/pulumi/sdk/v3/go/common/resource"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestConvertResourceValue(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		input    resource.PropertyMap
+		expected map[string]any
+	}{
+		{
+			input: resource.PropertyMap{
+				"a": resource.NewProperty(resource.PropertyMap{}),
+			},
+			expected: map[string]any{
+				"a": map[string]any{},
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run("", func(t *testing.T) {
+			actual := convertResourceValue(t, tt.input)
+			assert.Equal(t, tt.expected, actual)
+		})
+	}
+}