From d1e9379c53311787d38c7de76e6010f9c78a112e Mon Sep 17 00:00:00 2001 From: Venelin Date: Wed, 14 Aug 2024 19:36:28 +0300 Subject: [PATCH 1/4] TF Schema dump --- pf/internal/schemashim/attr_schema.go | 12 ++ pf/internal/schemashim/block_schema.go | 12 ++ pf/internal/schemashim/type_schema.go | 12 ++ pf/proto/attribute.go | 4 +- pf/proto/block.go | 3 + pf/proto/element.go | 10 +- pkg/tfbridge/info/info.go | 226 ++++++++++++++++++++- pkg/tfbridge/info/marshal_test.go | 23 +++ pkg/tfbridge/main.go | 29 +++ pkg/tfshim/schema/schema.go | 12 ++ pkg/tfshim/sdk-v1/schema.go | 12 ++ pkg/tfshim/sdk-v2/provider_test.go | 263 +++++++++++++++++++++++++ pkg/tfshim/sdk-v2/schema.go | 12 ++ pkg/tfshim/shim.go | 34 ++++ pkg/tfshim/tfplugin5/schema.go | 9 + 15 files changed, 660 insertions(+), 13 deletions(-) create mode 100644 pkg/tfbridge/info/marshal_test.go diff --git a/pf/internal/schemashim/attr_schema.go b/pf/internal/schemashim/attr_schema.go index 613f988a8..b429d2b21 100644 --- a/pf/internal/schemashim/attr_schema.go +++ b/pf/internal/schemashim/attr_schema.go @@ -124,6 +124,18 @@ func (*attrSchema) ExactlyOneOf() []string { panic("ExactlyOneOf() should not be called during schema generation") } +func (*attrSchema) AtLeastOneOf() []string { + panic("AtLeastOneOf() should not be called during schema generation") +} + +func (*attrSchema) RequiredWith() []string { + panic("RequiredWith() should not be called during schema generation") +} + +func (*attrSchema) ConfigMode() shim.ConfigModeType { + return 0 +} + func (s *attrSchema) Deprecated() string { return s.attr.GetDeprecationMessage() } diff --git a/pf/internal/schemashim/block_schema.go b/pf/internal/schemashim/block_schema.go index 88797bca1..e6ccf1002 100644 --- a/pf/internal/schemashim/block_schema.go +++ b/pf/internal/schemashim/block_schema.go @@ -139,6 +139,18 @@ func (*blockSchema) ExactlyOneOf() []string { panic("ExactlyOneOf() should not be called during schema generation") } +func (*blockSchema) AtLeastOneOf() []string { + panic("ConflictsWith() should not be called during schema generation") +} + +func (*blockSchema) RequiredWith() []string { + panic("RequiredWith() should not be called during schema generation") +} + +func (*blockSchema) ConfigMode() shim.ConfigModeType { + return 0 +} + func (*blockSchema) SetElement(config interface{}) (interface{}, error) { panic("SetElement() should not be called during schema generation") } diff --git a/pf/internal/schemashim/type_schema.go b/pf/internal/schemashim/type_schema.go index 8a68aceaa..ca19e910e 100644 --- a/pf/internal/schemashim/type_schema.go +++ b/pf/internal/schemashim/type_schema.go @@ -109,6 +109,18 @@ func (*typeSchema) ExactlyOneOf() []string { panic("ExactlyOneOf() should not be called during schema generation") } +func (*typeSchema) AtLeastOneOf() []string { + panic("AtLeastOneOf() should not be called during schema generation") +} + +func (*typeSchema) RequiredWith() []string { + panic("RequiredWith() should not be called during schema generation") +} + +func (*typeSchema) ConfigMode() shim.ConfigModeType { + return 0 +} + func (*typeSchema) Removed() string { panic("Removed() should not be called during schema generation") } func (*typeSchema) SetElement(config interface{}) (interface{}, error) { diff --git a/pf/proto/attribute.go b/pf/proto/attribute.go index 99a5946a5..b880a3b74 100644 --- a/pf/proto/attribute.go +++ b/pf/proto/attribute.go @@ -82,7 +82,6 @@ func (a attribute) Elem() interface{} { } } return element{a.attr.ValueType(), a.Optional()}.Elem() - } // Defaults are applied in the provider binary, not here @@ -94,6 +93,9 @@ func (a attribute) DefaultValue() (interface{}, error) { return nil, nil } func (a attribute) StateFunc() shim.SchemaStateFunc { return nil } func (a attribute) ConflictsWith() []string { return nil } func (a attribute) ExactlyOneOf() []string { return nil } +func (a attribute) AtLeastOneOf() []string { return nil } +func (a attribute) RequiredWith() []string { return nil } +func (a attribute) ConfigMode() shim.ConfigModeType { return 0 } func (a attribute) SetElement(config interface{}) (interface{}, error) { panic("UNIMPLIMENTED") diff --git a/pf/proto/block.go b/pf/proto/block.go index 45f919a04..4e07e4cd4 100644 --- a/pf/proto/block.go +++ b/pf/proto/block.go @@ -127,6 +127,9 @@ func (m blockSchema) MaxItems() int { return int(m.block.M func (m blockSchema) MinItems() int { return int(m.block.MinItems) } func (m blockSchema) ConflictsWith() []string { return nil } func (m blockSchema) ExactlyOneOf() []string { return nil } +func (a blockSchema) AtLeastOneOf() []string { return nil } +func (a blockSchema) RequiredWith() []string { return nil } +func (a blockSchema) ConfigMode() shim.ConfigModeType { return 0 } func (m blockSchema) Removed() string { return "" } func (m blockSchema) Sensitive() bool { return false } func (m blockSchema) Deprecated() string { return deprecated(m.block.Block.Deprecated) } diff --git a/pf/proto/element.go b/pf/proto/element.go index cd1929aca..49315de53 100644 --- a/pf/proto/element.go +++ b/pf/proto/element.go @@ -22,8 +22,10 @@ import ( shim "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfshim" ) -var _ = shim.Schema(element{}) -var _ = shim.Resource(elementObject{}) +var ( + _ = shim.Schema(element{}) + _ = shim.Resource(elementObject{}) +) type element struct { typ tftypes.Type @@ -56,7 +58,6 @@ func (e element) Type() shim.ValueType { panic(fmt.Sprintf("UNKNOWN TYPE of %#v", t)) // TODO: Remove for release // return shim.TypeInvalid } - } func (e element) Elem() interface{} { @@ -90,6 +91,9 @@ func (e element) MaxItems() int { return 0 } func (e element) MinItems() int { return 0 } func (e element) ConflictsWith() []string { return nil } func (e element) ExactlyOneOf() []string { return nil } +func (e element) AtLeastOneOf() []string { return nil } +func (e element) RequiredWith() []string { return nil } +func (e element) ConfigMode() shim.ConfigModeType { return 0 } func (e element) Deprecated() string { return "" } func (e element) Removed() string { return "" } func (e element) Sensitive() bool { return false } diff --git a/pkg/tfbridge/info/info.go b/pkg/tfbridge/info/info.go index 8328bd72a..8d8a768b8 100644 --- a/pkg/tfbridge/info/info.go +++ b/pkg/tfbridge/info/info.go @@ -21,7 +21,12 @@ package info import ( + "bytes" "context" + "encoding/csv" + "fmt" + "strconv" + "strings" pschema "github.com/pulumi/pulumi/pkg/v3/codegen/schema" "github.com/pulumi/pulumi/pkg/v3/resource/provider" @@ -823,6 +828,20 @@ type MarshallableSchemaShim struct { MaxItems int `json:"maxItems,omitempty"` MinItems int `json:"minItems,omitempty"` DeprecationMessage string `json:"deprecated,omitempty"` + Default interface{} `json:"default,omitempty"` + // SDK specific fields + ConfigMode shim.ConfigModeType `json:"configMode,omitempty"` + // TODO + // DefaultFunc + // Set + ConflictsWith []string `json:"conflictsWith,omitempty"` + ExactlyOneOf []string `json:"exactlyOneOf,omitempty"` + AtLeastOneOf []string `json:"atLeastOneOf,omitempty"` + RequiredWith []string `json:"requiredWith,omitempty"` + + // PF specific fields + // TODO + // PlanModifiers } // MarshalSchemaShim converts a Terraform schema into a MarshallableSchema. @@ -837,6 +856,12 @@ func MarshalSchemaShim(s shim.Schema) *MarshallableSchemaShim { MaxItems: s.MaxItems(), MinItems: s.MinItems(), DeprecationMessage: s.Deprecated(), + Default: s.Default(), + ConfigMode: s.ConfigMode(), + ConflictsWith: s.ConflictsWith(), + ExactlyOneOf: s.ExactlyOneOf(), + AtLeastOneOf: s.AtLeastOneOf(), + RequiredWith: s.RequiredWith(), } } @@ -855,35 +880,115 @@ func (m *MarshallableSchemaShim) Unmarshal() shim.Schema { }).Shim() } +func (m *MarshallableSchemaShim) GetFlattenedSchema( + provider, version, path string, schemaRows *[][]string, resourceRows *[][]string, +) { + if m.Elem != nil { + m.Elem.GetFlattenedSchema(provider, version, path+".Elem", schemaRows, resourceRows) + } + + row := []string{ + provider, + version, + path, + fmt.Sprintf("%v", m.Type), + strconv.FormatBool(m.Optional), + strconv.FormatBool(m.Required), + strconv.FormatBool(m.Computed), + strconv.FormatBool(m.ForceNew), + strconv.Itoa(m.MaxItems), + strconv.Itoa(m.MinItems), + m.DeprecationMessage, + fmt.Sprintf("%v", m.Default), + fmt.Sprintf("%v", m.ConfigMode), + strings.Join(m.ConflictsWith, ","), + strings.Join(m.ExactlyOneOf, ","), + strings.Join(m.AtLeastOneOf, ","), + strings.Join(m.RequiredWith, ","), + } + + *schemaRows = append(*schemaRows, row) +} + // MarshallableResourceShim is the JSON-marshallable form of a Terraform resource schema. -type MarshallableResourceShim map[string]*MarshallableSchemaShim +type MarshallableResourceShim struct { + Schema map[string]*MarshallableSchemaShim `json:"schema,omitempty"` + SchemaVersion int `json:"schemaVersion,omitempty"` + // StateUpgraders + // UseJSONNumber +} // MarshalResourceShim converts a Terraform resource schema into a MarshallableResourceShim. func MarshalResourceShim(r shim.Resource) MarshallableResourceShim { - m := make(MarshallableResourceShim) + m := make(map[string]*MarshallableSchemaShim) if r.Schema() == nil { - return m + return MarshallableResourceShim{Schema: m} } r.Schema().Range(func(k string, v shim.Schema) bool { m[k] = MarshalSchemaShim(v) return true }) - return m + return MarshallableResourceShim{ + Schema: m, + SchemaVersion: r.SchemaVersion(), + } } // Unmarshal creates a mostly-initialized Terraform resource schema from the given MarshallableResourceShim. func (m MarshallableResourceShim) Unmarshal() shim.Resource { s := schema.SchemaMap{} - for k, v := range m { + for k, v := range m.Schema { s[k] = v.Unmarshal() } - return (&schema.Resource{Schema: s}).Shim() + return (&schema.Resource{ + Schema: s, + SchemaVersion: m.SchemaVersion, + }).Shim() +} + +type ResourceType int + +const ( + ResourceTypeResource ResourceType = iota + ResourceTypeDataSource + ResourceTypeNested +) + +func (r ResourceType) String() string { + switch r { + case ResourceTypeResource: + return "resource" + case ResourceTypeDataSource: + return "dataSource" + case ResourceTypeNested: + return "nested" + default: + return "unknown" + } +} + +func (m *MarshallableResourceShim) GetFlattenedSchema( + provider, version, path string, resourceType ResourceType, schemaRows *[][]string, resourceRows *[][]string, +) { + for k, v := range m.Schema { + v.GetFlattenedSchema(provider, version, path+"."+k, schemaRows, resourceRows) + } + + row := []string{ + provider, + version, + path, + strconv.Itoa(m.SchemaVersion), + resourceType.String(), + } + + *resourceRows = append(*resourceRows, row) } // MarshallableElemShim is the JSON-marshallable form of a Terraform schema's element field. type MarshallableElemShim struct { - Schema *MarshallableSchemaShim `json:"schema,omitempty"` - Resource MarshallableResourceShim `json:"resource,omitempty"` + Schema *MarshallableSchemaShim `json:"schema,omitempty"` + Resource *MarshallableResourceShim `json:"resource,omitempty"` } // MarshalElemShim converts a Terraform schema's element field into a MarshallableElemShim. @@ -892,7 +997,8 @@ func MarshalElemShim(e interface{}) *MarshallableElemShim { case shim.Schema: return &MarshallableElemShim{Schema: MarshalSchemaShim(v)} case shim.Resource: - return &MarshallableElemShim{Resource: MarshalResourceShim(v)} + res := MarshalResourceShim(v) + return &MarshallableElemShim{Resource: &res} default: contract.Assertf(e == nil, "unexpected schema element of type %T", e) return nil @@ -913,6 +1019,17 @@ func (m *MarshallableElemShim) Unmarshal() interface{} { } } +func (m *MarshallableElemShim) GetFlattenedSchema( + provider, version, path string, schemaRows *[][]string, resourceRows *[][]string, +) { + if m.Schema != nil { + m.Schema.GetFlattenedSchema(provider, version, path, schemaRows, resourceRows) + } + if m.Resource != nil { + m.Resource.GetFlattenedSchema(provider, version, path, ResourceTypeNested, schemaRows, resourceRows) + } +} + // MarshallableProviderShim is the JSON-marshallable form of a Terraform provider schema. type MarshallableProviderShim struct { Schema map[string]*MarshallableSchemaShim `json:"schema,omitempty"` @@ -973,6 +1090,81 @@ func (m *MarshallableProviderShim) Unmarshal() shim.Provider { }).Shim() } +func (m *MarshallableProviderShim) GetFlattenedSchema(provider, version string) ( + schemaRows [][]string, resourceRows [][]string, +) { + for k, v := range m.Schema { + v.GetFlattenedSchema(provider, version, provider+"."+k, &schemaRows, &resourceRows) + } + for k, v := range m.Resources { + v.GetFlattenedSchema(provider, version, provider+"."+k, ResourceTypeResource, &schemaRows, &resourceRows) + } + for k, v := range m.DataSources { + v.GetFlattenedSchema(provider, version, provider+"."+k, ResourceTypeDataSource, &schemaRows, &resourceRows) + } + return +} + +//nolint:errcheck +func (m *MarshallableProviderShim) GetCSVSchema( + provider, version string, tokenMapping map[string]string, schemaOut, resourceOut *bytes.Buffer, +) error { + schemaRows, resourceRows := m.GetFlattenedSchema(provider, version) + schemaWriter := csv.NewWriter(schemaOut) + resourceWriter := csv.NewWriter(resourceOut) + + schemaWriter.Write([]string{ + "provider", + "version", + "path", + "type", + "optional", + "required", + "computed", + "forceNew", + "maxItems", + "minItems", + "deprecated", + "default", + "configMode", + "conflictsWith", + "exactlyOneOf", + "atLeastOneOf", + "requiredWith", + }) + + resourceWriter.Write([]string{ + "provider", + "version", + "path", + "schemaVersion", + "token", + }) + + for _, row := range schemaRows { + schemaWriter.Write(row) + } + for _, row := range resourceRows { + // Find the token for the resource + path := row[2] + resourceName := strings.Split(path, ".")[1] + token := tokenMapping[resourceName] + row = append(row, token) + resourceWriter.Write(row) + } + + schemaWriter.Flush() + resourceWriter.Flush() + + if err := schemaWriter.Error(); err != nil { + return fmt.Errorf("error writing schema CSV: %w", err) + } + if err := resourceWriter.Error(); err != nil { + return fmt.Errorf("error writing resource CSV: %w", err) + } + return nil +} + // MarshallableSchema is the JSON-marshallable form of a Pulumi SchemaInfo value. type MarshallableSchema struct { Name string `json:"name,omitempty"` @@ -1191,6 +1383,22 @@ func MarshalProvider(p *Provider) *MarshallableProvider { return &info } +func (info *Provider) GetCSVSchema( + provider, version string, schemaOut, resourceOut *bytes.Buffer, +) error { + tokenMapping := make(map[string]string) + + // Create a map of resource names to tokens + for k, v := range info.Resources { + tokenMapping[k] = v.Tok.String() + } + for k, v := range info.DataSources { + tokenMapping[k] = v.Tok.String() + } + + return MarshalProviderShim(info.P).GetCSVSchema(provider, version, tokenMapping, schemaOut, resourceOut) +} + // Unmarshal creates a mostly-=initialized Pulumi ProviderInfo value from the given MarshallableProviderInfo. func (m *MarshallableProvider) Unmarshal() *Provider { config := make(map[string]*Schema) diff --git a/pkg/tfbridge/info/marshal_test.go b/pkg/tfbridge/info/marshal_test.go new file mode 100644 index 000000000..c4ec6750e --- /dev/null +++ b/pkg/tfbridge/info/marshal_test.go @@ -0,0 +1,23 @@ +package info + +import ( + "testing" + + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" + shim "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfshim" + sdkv2 "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfshim/sdk-v2" + "github.com/stretchr/testify/require" +) + +func TestSchema(t *testing.T) { + sch := schema.Schema{ + Type: schema.TypeString, + Default: "default", + } + shimSchema := sdkv2.NewSchema(&sch) + + marshalled := MarshalSchemaShim(shimSchema) + unmarshalled := marshalled.Unmarshal() + + require.Equal(t, shim.ValueType(sch.Type), unmarshalled.Type()) +} diff --git a/pkg/tfbridge/main.go b/pkg/tfbridge/main.go index 78138c56a..e2034a8b1 100644 --- a/pkg/tfbridge/main.go +++ b/pkg/tfbridge/main.go @@ -15,6 +15,7 @@ package tfbridge import ( + "bytes" "context" "encoding/json" "flag" @@ -41,6 +42,8 @@ func Main(pkg string, version string, prov ProviderInfo, pulumiSchema []byte) { flags.SetOutput(io.Discard) dumpInfo := flags.Bool("get-provider-info", false, "dump provider info as JSON to stdout") + dumpTFSchemaCSVPath := flags.String( + "dump-tf-resource-csv-path", "", "dump Terraform resource CSV to the specified path") providerVersion := flags.Bool("version", false, "get built provider version") err := flags.Parse(os.Args[1:]) @@ -62,6 +65,32 @@ func Main(pkg string, version string, prov ProviderInfo, pulumiSchema []byte) { os.Exit(0) } + if *dumpTFSchemaCSVPath != "" { + resBuf := &bytes.Buffer{} + schBuf := &bytes.Buffer{} + err := prov.GetCSVSchema(prov.Name, prov.Version, schBuf, resBuf) + if err != nil { + cmdutil.ExitError(err.Error()) + } + + resPath := fmt.Sprintf("%s/resources.csv", *dumpTFSchemaCSVPath) + schPath := fmt.Sprintf("%s/schemas.csv", *dumpTFSchemaCSVPath) + + if err := os.MkdirAll(*dumpTFSchemaCSVPath, 0o700); err != nil { + cmdutil.ExitError(err.Error()) + } + + if err := os.WriteFile(resPath, resBuf.Bytes(), 0o600); err != nil { + cmdutil.ExitError(err.Error()) + } + + if err := os.WriteFile(schPath, schBuf.Bytes(), 0o600); err != nil { + cmdutil.ExitError(err.Error()) + } + + os.Exit(0) + } + if *providerVersion { fmt.Println(version) os.Exit(0) diff --git a/pkg/tfshim/schema/schema.go b/pkg/tfshim/schema/schema.go index 6992419f9..c394ebbf7 100644 --- a/pkg/tfshim/schema/schema.go +++ b/pkg/tfshim/schema/schema.go @@ -107,6 +107,18 @@ func (s SchemaShim) ExactlyOneOf() []string { return s.V.ExactlyOneOf } +func (s SchemaShim) AtLeastOneOf() []string { + return nil +} + +func (s SchemaShim) RequiredWith() []string { + return nil +} + +func (s SchemaShim) ConfigMode() shim.ConfigModeType { + return 0 +} + func (s SchemaShim) Removed() string { return s.V.Removed } diff --git a/pkg/tfshim/sdk-v1/schema.go b/pkg/tfshim/sdk-v1/schema.go index a9286d26a..29c57655f 100644 --- a/pkg/tfshim/sdk-v1/schema.go +++ b/pkg/tfshim/sdk-v1/schema.go @@ -141,6 +141,18 @@ func (s v1Schema) SetHash(v interface{}) int { return code } +func (s v1Schema) AtLeastOneOf() []string { + return s.tf.AtLeastOneOf +} + +func (s v1Schema) RequiredWith() []string { + return nil +} + +func (s v1Schema) ConfigMode() shim.ConfigModeType { + return shim.ConfigModeType(s.tf.ConfigMode) +} + type v1SchemaMap map[string]*schema.Schema func NewSchemaMap(m map[string]*schema.Schema) shim.SchemaMap { diff --git a/pkg/tfshim/sdk-v2/provider_test.go b/pkg/tfshim/sdk-v2/provider_test.go index 5f8216663..2e2a36a7e 100644 --- a/pkg/tfshim/sdk-v2/provider_test.go +++ b/pkg/tfshim/sdk-v2/provider_test.go @@ -1,12 +1,16 @@ package sdkv2 import ( + "bytes" "context" + "encoding/json" "testing" "github.com/hashicorp/go-cty/cty" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" "github.com/hashicorp/terraform-plugin-sdk/v2/terraform" + "github.com/hexops/autogold/v2" + "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfbridge/info" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -104,3 +108,262 @@ func TestProvider1UpgradeResourceState(t *testing.T) { }) } } + +//nolint:lll +func TestMarshalProviderShim(t *testing.T) { + prov := NewProvider(&schema.Provider{ + ResourcesMap: map[string]*schema.Resource{ + "test_resource": { + Schema: map[string]*schema.Schema{ + "foo": { + Type: schema.TypeString, + Optional: true, + }, + "bar": { + Type: schema.TypeInt, + Optional: true, + }, + "nested_prop": { + Type: schema.TypeList, + Optional: true, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "nested_foo": { + Type: schema.TypeString, + Optional: true, + }, + }, + }, + }, + "max_item_one_prop": { + Type: schema.TypeList, + Required: true, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "nested_foo": { + Type: schema.TypeString, + Optional: true, + }, + }, + }, + MaxItems: 1, + }, + "map_prop": { + Type: schema.TypeMap, + Optional: true, + Elem: &schema.Schema{ + Type: schema.TypeString, + }, + }, + "config_mode_prop": { + Type: schema.TypeList, + Optional: true, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "prop": { + Type: schema.TypeString, + Optional: true, + }, + }, + }, + ConfigMode: schema.SchemaConfigModeAttr, + }, + "default_prop": { + Type: schema.TypeString, + Default: "default", + Optional: true, + }, + "conflicting_prop": { + Type: schema.TypeString, + ExactlyOneOf: []string{"foo", "conflicting_prop"}, + ConflictsWith: []string{"bar"}, + RequiredWith: []string{"map_prop"}, + AtLeastOneOf: []string{"nested_prop", "conflicting_prop"}, + Optional: true, + }, + }, + }, + }, + DataSourcesMap: map[string]*schema.Resource{ + "test_data_source": { + Schema: map[string]*schema.Schema{ + "foo": { + Type: schema.TypeString, + Optional: true, + }, + "bar": { + Type: schema.TypeInt, + Optional: true, + }, + }, + }, + }, + Schema: map[string]*schema.Schema{ + "test_schema": { + Type: schema.TypeString, + Optional: true, + }, + }, + }) + + err := prov.InternalValidate() + require.NoError(t, err) + + marshallableProv := info.MarshalProviderShim(prov) + + jsonArr, err := json.Marshal(marshallableProv) + require.NoError(t, err) + var out bytes.Buffer + err = json.Indent(&out, jsonArr, "", " ") + require.NoError(t, err) + + autogold.Expect(`{ + "schema": { + "test_schema": { + "type": "String", + "optional": true + } + }, + "resources": { + "test_resource": { + "schema": { + "bar": { + "type": "Int", + "optional": true + }, + "config_mode_prop": { + "type": "List", + "optional": true, + "element": { + "resource": { + "schema": { + "prop": { + "type": "String", + "optional": true + } + } + } + }, + "configMode": "attr" + }, + "conflicting_prop": { + "type": "String", + "optional": true, + "conflictsWith": [ + "bar" + ], + "exactlyOneOf": [ + "foo", + "conflicting_prop" + ], + "atLeastOneOf": [ + "nested_prop", + "conflicting_prop" + ], + "requiredWith": [ + "map_prop" + ] + }, + "default_prop": { + "type": "String", + "optional": true, + "default": "default" + }, + "foo": { + "type": "String", + "optional": true + }, + "map_prop": { + "type": "Map", + "optional": true, + "element": { + "schema": { + "type": "String" + } + } + }, + "max_item_one_prop": { + "type": "List", + "required": true, + "element": { + "resource": { + "schema": { + "nested_foo": { + "type": "String", + "optional": true + } + } + } + }, + "maxItems": 1 + }, + "nested_prop": { + "type": "List", + "optional": true, + "element": { + "resource": { + "schema": { + "nested_foo": { + "type": "String", + "optional": true + } + } + } + } + } + } + } + }, + "dataSources": { + "test_data_source": { + "schema": { + "bar": { + "type": "Int", + "optional": true + }, + "foo": { + "type": "String", + "optional": true + } + } + } + } +}`).Equal(t, out.String()) + + tokenMapping := map[string]string{ + "test_resource": "testResource", + "test_data_source": "testDataSource", + } + + resBuf := &bytes.Buffer{} + schBuf := &bytes.Buffer{} + + err = marshallableProv.GetCSVSchema("myProvider", "0.0.1", tokenMapping, schBuf, resBuf) + require.NoError(t, err) + + autogold.Expect(`provider,version,path,type,optional,required,computed,forceNew,maxItems,minItems,deprecated,default,configMode,conflictsWith,exactlyOneOf,atLeastOneOf,requiredWith +myProvider,0.0.1,myProvider.test_schema,String,true,false,false,false,0,0,,,auto,,,, +myProvider,0.0.1,myProvider.test_resource.bar,Int,true,false,false,false,0,0,,,auto,,,, +myProvider,0.0.1,myProvider.test_resource.nested_prop.Elem.nested_foo,String,true,false,false,false,0,0,,,auto,,,, +myProvider,0.0.1,myProvider.test_resource.nested_prop,List,true,false,false,false,0,0,,,auto,,,, +myProvider,0.0.1,myProvider.test_resource.max_item_one_prop.Elem.nested_foo,String,true,false,false,false,0,0,,,auto,,,, +myProvider,0.0.1,myProvider.test_resource.max_item_one_prop,List,false,true,false,false,1,0,,,auto,,,, +myProvider,0.0.1,myProvider.test_resource.map_prop.Elem,String,false,false,false,false,0,0,,,auto,,,, +myProvider,0.0.1,myProvider.test_resource.map_prop,Map,true,false,false,false,0,0,,,auto,,,, +myProvider,0.0.1,myProvider.test_resource.config_mode_prop.Elem.prop,String,true,false,false,false,0,0,,,auto,,,, +myProvider,0.0.1,myProvider.test_resource.config_mode_prop,List,true,false,false,false,0,0,,,attr,,,, +myProvider,0.0.1,myProvider.test_resource.default_prop,String,true,false,false,false,0,0,,default,auto,,,, +myProvider,0.0.1,myProvider.test_resource.conflicting_prop,String,true,false,false,false,0,0,,,auto,bar,"foo,conflicting_prop","nested_prop,conflicting_prop",map_prop +myProvider,0.0.1,myProvider.test_resource.foo,String,true,false,false,false,0,0,,,auto,,,, +myProvider,0.0.1,myProvider.test_data_source.bar,Int,true,false,false,false,0,0,,,auto,,,, +myProvider,0.0.1,myProvider.test_data_source.foo,String,true,false,false,false,0,0,,,auto,,,, +`).Equal(t, schBuf.String()) + + autogold.Expect(`provider,version,path,schemaVersion,token +myProvider,0.0.1,myProvider.test_resource.nested_prop.Elem,0,nested,testResource +myProvider,0.0.1,myProvider.test_resource.max_item_one_prop.Elem,0,nested,testResource +myProvider,0.0.1,myProvider.test_resource.config_mode_prop.Elem,0,nested,testResource +myProvider,0.0.1,myProvider.test_resource,0,resource,testResource +myProvider,0.0.1,myProvider.test_data_source,0,dataSource,testDataSource +`).Equal(t, resBuf.String()) +} diff --git a/pkg/tfshim/sdk-v2/schema.go b/pkg/tfshim/sdk-v2/schema.go index a2631db50..d54cb84aa 100644 --- a/pkg/tfshim/sdk-v2/schema.go +++ b/pkg/tfshim/sdk-v2/schema.go @@ -108,6 +108,18 @@ func (s v2Schema) ExactlyOneOf() []string { return s.tf.ExactlyOneOf } +func (s v2Schema) AtLeastOneOf() []string { + return s.tf.AtLeastOneOf +} + +func (s v2Schema) RequiredWith() []string { + return s.tf.RequiredWith +} + +func (s v2Schema) ConfigMode() shim.ConfigModeType { + return shim.ConfigModeType(s.tf.ConfigMode) +} + func (s v2Schema) Removed() string { return "" } diff --git a/pkg/tfshim/shim.go b/pkg/tfshim/shim.go index ff600bb26..2fcac448f 100644 --- a/pkg/tfshim/shim.go +++ b/pkg/tfshim/shim.go @@ -4,6 +4,8 @@ import ( "context" "fmt" "time" + + sdkv2schema "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" ) type ResourceConfig interface { @@ -87,6 +89,35 @@ func (i ValueType) String() string { } } +func (i ValueType) MarshalText() ([]byte, error) { + return []byte(i.String()), nil +} + +type ConfigModeType int + +func (c ConfigModeType) String() string { + switch c { + case ConfigModeType(sdkv2schema.SchemaConfigModeAuto): + return "auto" + case ConfigModeType(sdkv2schema.SchemaConfigModeAttr): + return "attr" + case ConfigModeType(sdkv2schema.SchemaConfigModeBlock): + return "block" + default: + return "unknown" + } +} + +func (c ConfigModeType) MarshalText() ([]byte, error) { + return []byte(c.String()), nil +} + +type SchemaExtraFields struct { + ConfigMode ConfigModeType + AtLeastOneOf []string + RequiredWith []string +} + type SchemaDefaultFunc func() (interface{}, error) type SchemaStateFunc func(interface{}) string @@ -139,9 +170,12 @@ type Schema interface { MinItems() int ConflictsWith() []string ExactlyOneOf() []string + AtLeastOneOf() []string + RequiredWith() []string Deprecated() string Removed() string Sensitive() bool + ConfigMode() ConfigModeType SetElement(config interface{}) (interface{}, error) SetHash(v interface{}) int diff --git a/pkg/tfshim/tfplugin5/schema.go b/pkg/tfshim/tfplugin5/schema.go index 1d3b4656f..50ee85d76 100644 --- a/pkg/tfshim/tfplugin5/schema.go +++ b/pkg/tfshim/tfplugin5/schema.go @@ -92,6 +92,15 @@ func (s *attributeSchema) AtLeastOneOf() []string { return nil } +func (s attributeSchema) RequiredWith() []string { + return nil +} + +func (s attributeSchema) ConfigMode() shim.ConfigModeType { + return 0 +} + + func (s *attributeSchema) Removed() string { return "" } From 26b1cbdacf1c5aedb3d9065049753f396b9eaf31 Mon Sep 17 00:00:00 2001 From: Venelin Date: Wed, 14 Aug 2024 20:24:13 +0300 Subject: [PATCH 2/4] fix --- pf/internal/schemashim/attr_schema.go | 12 ++-- pf/internal/schemashim/block_schema.go | 12 ++-- pf/internal/schemashim/datasource.go | 4 +- .../schemashim/object_pseudoresource.go | 3 +- pf/internal/schemashim/resource.go | 4 +- pf/internal/schemashim/type_schema.go | 10 ++-- pf/tfbridge/main.go | 38 ++++++++++++ pkg/tfbridge/info/info.go | 60 +++++++++++-------- pkg/tfbridge/main.go | 11 +++- pkg/tfshim/sdk-v2/provider_test.go | 42 ++++++------- 10 files changed, 125 insertions(+), 71 deletions(-) diff --git a/pf/internal/schemashim/attr_schema.go b/pf/internal/schemashim/attr_schema.go index b429d2b21..3dd623d06 100644 --- a/pf/internal/schemashim/attr_schema.go +++ b/pf/internal/schemashim/attr_schema.go @@ -47,8 +47,8 @@ func (s *attrSchema) Required() bool { return s.attr.IsRequired() } -func (*attrSchema) Default() interface{} { - panic("Default() should not be called during schema generation") +func (s *attrSchema) Default() interface{} { + return nil } func (*attrSchema) DefaultFunc() shim.SchemaDefaultFunc { panic("DefaultFunc() should not be called during schema generation") @@ -117,19 +117,19 @@ func (*attrSchema) MinItems() int { } func (*attrSchema) ConflictsWith() []string { - panic("ConflictsWith() should not be called during schema generation") + return nil } func (*attrSchema) ExactlyOneOf() []string { - panic("ExactlyOneOf() should not be called during schema generation") + return nil } func (*attrSchema) AtLeastOneOf() []string { - panic("AtLeastOneOf() should not be called during schema generation") + return nil } func (*attrSchema) RequiredWith() []string { - panic("RequiredWith() should not be called during schema generation") + return nil } func (*attrSchema) ConfigMode() shim.ConfigModeType { diff --git a/pf/internal/schemashim/block_schema.go b/pf/internal/schemashim/block_schema.go index e6ccf1002..98cb19c1a 100644 --- a/pf/internal/schemashim/block_schema.go +++ b/pf/internal/schemashim/block_schema.go @@ -118,11 +118,11 @@ func (s *blockSchema) MaxItems() int { return int(s.block.GetMaxItems()) } func (s *blockSchema) MinItems() int { return int(s.block.GetMinItems()) } func (*blockSchema) ConflictsWith() []string { - panic("ConflictsWith() should not be called during schema generation") + return nil } -func (*blockSchema) Default() interface{} { - panic("Default() should not be called during schema generation") +func (s *blockSchema) Default() interface{} { + return nil } func (*blockSchema) DefaultFunc() shim.SchemaDefaultFunc { @@ -136,15 +136,15 @@ func (*blockSchema) DefaultValue() (interface{}, error) { } func (*blockSchema) ExactlyOneOf() []string { - panic("ExactlyOneOf() should not be called during schema generation") + return nil } func (*blockSchema) AtLeastOneOf() []string { - panic("ConflictsWith() should not be called during schema generation") + return nil } func (*blockSchema) RequiredWith() []string { - panic("RequiredWith() should not be called during schema generation") + return nil } func (*blockSchema) ConfigMode() shim.ConfigModeType { diff --git a/pf/internal/schemashim/datasource.go b/pf/internal/schemashim/datasource.go index d60476549..b7b58d931 100644 --- a/pf/internal/schemashim/datasource.go +++ b/pf/internal/schemashim/datasource.go @@ -29,8 +29,8 @@ func (r *schemaOnlyDataSource) Schema() shim.SchemaMap { return r.tf.Shim() } -func (*schemaOnlyDataSource) SchemaVersion() int { - panic("DataSource SchemaVersion() should not be called during schema generation") +func (r *schemaOnlyDataSource) SchemaVersion() int { + return int(r.tf.ResourceSchemaVersion()) } func (r *schemaOnlyDataSource) DeprecationMessage() string { diff --git a/pf/internal/schemashim/object_pseudoresource.go b/pf/internal/schemashim/object_pseudoresource.go index 6cbbd78ba..7550dba6b 100644 --- a/pf/internal/schemashim/object_pseudoresource.go +++ b/pf/internal/schemashim/object_pseudoresource.go @@ -73,8 +73,7 @@ func (r *objectPseudoResource) Schema() shim.SchemaMap { } func (*objectPseudoResource) SchemaVersion() int { - panic("This is an Object type encoded as a shim.Resource, and " + - "SchemaVersion() should not be called on this entity during schema generation") + return 0 } func (*objectPseudoResource) DeprecationMessage() string { diff --git a/pf/internal/schemashim/resource.go b/pf/internal/schemashim/resource.go index 45ee49fac..97d5bb908 100644 --- a/pf/internal/schemashim/resource.go +++ b/pf/internal/schemashim/resource.go @@ -29,8 +29,8 @@ func (r *schemaOnlyResource) Schema() shim.SchemaMap { return r.tf.Shim() } -func (*schemaOnlyResource) SchemaVersion() int { - panic("SchemaVersion() should not be called on a Resource during schema generation") +func (r *schemaOnlyResource) SchemaVersion() int { + return int(r.tf.ResourceSchemaVersion()) } func (r *schemaOnlyResource) DeprecationMessage() string { diff --git a/pf/internal/schemashim/type_schema.go b/pf/internal/schemashim/type_schema.go index ca19e910e..f0c34974d 100644 --- a/pf/internal/schemashim/type_schema.go +++ b/pf/internal/schemashim/type_schema.go @@ -79,7 +79,7 @@ func (*typeSchema) MinItems() int { return 0 } func (*typeSchema) Deprecated() string { return "" } func (*typeSchema) Default() interface{} { - panic("Default() should not be called during schema generation") + return nil } func (*typeSchema) DefaultFunc() shim.SchemaDefaultFunc { @@ -102,19 +102,19 @@ func (*typeSchema) StateFunc() shim.SchemaStateFunc { } func (*typeSchema) ConflictsWith() []string { - panic("ConflictsWith() should not be called during schema generation") + return nil } func (*typeSchema) ExactlyOneOf() []string { - panic("ExactlyOneOf() should not be called during schema generation") + return nil } func (*typeSchema) AtLeastOneOf() []string { - panic("AtLeastOneOf() should not be called during schema generation") + return nil } func (*typeSchema) RequiredWith() []string { - panic("RequiredWith() should not be called during schema generation") + return nil } func (*typeSchema) ConfigMode() shim.ConfigModeType { diff --git a/pf/tfbridge/main.go b/pf/tfbridge/main.go index 71fbc5df2..fda4e9586 100644 --- a/pf/tfbridge/main.go +++ b/pf/tfbridge/main.go @@ -15,12 +15,14 @@ package tfbridge import ( + "bytes" "context" "encoding/json" "flag" "fmt" "io" "os" + "time" rprovider "github.com/pulumi/pulumi/pkg/v3/resource/provider" "github.com/pulumi/pulumi/sdk/v3/go/common/util/cmdutil" @@ -70,6 +72,8 @@ func handleFlags( dumpInfo := flags.Bool("get-provider-info", false, "dump provider info as JSON to stdout") providerVersion := flags.Bool("version", false, "get built provider version") + dumpTFSchemaCSVPath := flags.String( + "dump-tf-resource-csv-path", "", "dump Terraform resource CSV to the specified path") err := flags.Parse(os.Args[1:]) contract.IgnoreError(err) @@ -94,6 +98,40 @@ func handleFlags( os.Exit(0) } + if *dumpTFSchemaCSVPath != "" { + prov, err := getProviderInfo() + if err != nil { + cmdutil.ExitError(err.Error()) + } + resBuf := &bytes.Buffer{} + schBuf := &bytes.Buffer{} + err = prov.GetCSVSchema(prov.Name, prov.Version, schBuf, resBuf) + if err != nil { + cmdutil.ExitError(err.Error()) + } + + // get_current date + currentTime := time.Now() + formattedTime := currentTime.Format("150405") + + resPath := fmt.Sprintf("%s/%s_%s_resources_%s.csv", *dumpTFSchemaCSVPath, prov.Name, prov.Version, formattedTime) + schPath := fmt.Sprintf("%s/%s_%s_schemas_%s.csv", *dumpTFSchemaCSVPath, prov.Name, prov.Version, formattedTime) + + if err := os.MkdirAll(*dumpTFSchemaCSVPath, 0o700); err != nil { + cmdutil.ExitError(err.Error()) + } + + if err := os.WriteFile(resPath, resBuf.Bytes(), 0o600); err != nil { + cmdutil.ExitError(err.Error()) + } + + if err := os.WriteFile(schPath, schBuf.Bytes(), 0o600); err != nil { + cmdutil.ExitError(err.Error()) + } + + os.Exit(0) + } + if *providerVersion { fmt.Println(version) os.Exit(0) diff --git a/pkg/tfbridge/info/info.go b/pkg/tfbridge/info/info.go index 8d8a768b8..8f469e88b 100644 --- a/pkg/tfbridge/info/info.go +++ b/pkg/tfbridge/info/info.go @@ -25,6 +25,7 @@ import ( "context" "encoding/csv" "fmt" + "sort" "strconv" "strings" @@ -881,10 +882,10 @@ func (m *MarshallableSchemaShim) Unmarshal() shim.Schema { } func (m *MarshallableSchemaShim) GetFlattenedSchema( - provider, version, path string, schemaRows *[][]string, resourceRows *[][]string, + provider, version, path string, tokenMapping map[string]string, schemaRows *[][]string, resourceRows *[][]string, ) { if m.Elem != nil { - m.Elem.GetFlattenedSchema(provider, version, path+".Elem", schemaRows, resourceRows) + m.Elem.GetFlattenedSchema(provider, version, path+".Elem", tokenMapping, schemaRows, resourceRows) } row := []string{ @@ -968,18 +969,23 @@ func (r ResourceType) String() string { } func (m *MarshallableResourceShim) GetFlattenedSchema( - provider, version, path string, resourceType ResourceType, schemaRows *[][]string, resourceRows *[][]string, + provider, version, path string, resourceType ResourceType, tokenMapping map[string]string, schemaRows *[][]string, resourceRows *[][]string, ) { for k, v := range m.Schema { - v.GetFlattenedSchema(provider, version, path+"."+k, schemaRows, resourceRows) + v.GetFlattenedSchema(provider, version, path+"."+k, tokenMapping, schemaRows, resourceRows) } + // Find the token for the resource + resourceName := strings.Split(path, ".")[1] + token := tokenMapping[resourceName] + row := []string{ provider, version, path, strconv.Itoa(m.SchemaVersion), resourceType.String(), + token, } *resourceRows = append(*resourceRows, row) @@ -1020,13 +1026,13 @@ func (m *MarshallableElemShim) Unmarshal() interface{} { } func (m *MarshallableElemShim) GetFlattenedSchema( - provider, version, path string, schemaRows *[][]string, resourceRows *[][]string, + provider, version, path string, tokenMapping map[string]string, schemaRows *[][]string, resourceRows *[][]string, ) { if m.Schema != nil { - m.Schema.GetFlattenedSchema(provider, version, path, schemaRows, resourceRows) + m.Schema.GetFlattenedSchema(provider, version, path, tokenMapping, schemaRows, resourceRows) } if m.Resource != nil { - m.Resource.GetFlattenedSchema(provider, version, path, ResourceTypeNested, schemaRows, resourceRows) + m.Resource.GetFlattenedSchema(provider, version, path, ResourceTypeNested, tokenMapping, schemaRows, resourceRows) } } @@ -1090,26 +1096,34 @@ func (m *MarshallableProviderShim) Unmarshal() shim.Provider { }).Shim() } -func (m *MarshallableProviderShim) GetFlattenedSchema(provider, version string) ( +func (m *MarshallableProviderShim) GetFlattenedSchema(provider, version string, resourceTokenMapping, datasourceTokenMapping map[string]string) ( schemaRows [][]string, resourceRows [][]string, ) { for k, v := range m.Schema { - v.GetFlattenedSchema(provider, version, provider+"."+k, &schemaRows, &resourceRows) + v.GetFlattenedSchema(provider, version, provider+"."+k, nil, &schemaRows, &resourceRows) } for k, v := range m.Resources { - v.GetFlattenedSchema(provider, version, provider+"."+k, ResourceTypeResource, &schemaRows, &resourceRows) + v.GetFlattenedSchema(provider, version, provider+"."+k, ResourceTypeResource, resourceTokenMapping, &schemaRows, &resourceRows) } for k, v := range m.DataSources { - v.GetFlattenedSchema(provider, version, provider+"."+k, ResourceTypeDataSource, &schemaRows, &resourceRows) + v.GetFlattenedSchema(provider, version, provider+"."+k, ResourceTypeDataSource, datasourceTokenMapping, &schemaRows, &resourceRows) } return } //nolint:errcheck func (m *MarshallableProviderShim) GetCSVSchema( - provider, version string, tokenMapping map[string]string, schemaOut, resourceOut *bytes.Buffer, + provider, version string, + resourceTokenMapping, datasourceTokenMapping map[string]string, schemaOut, resourceOut *bytes.Buffer, ) error { - schemaRows, resourceRows := m.GetFlattenedSchema(provider, version) + schemaRows, resourceRows := m.GetFlattenedSchema(provider, version, resourceTokenMapping, datasourceTokenMapping) + // Make the order deterministic + pathCmp := func(i, j int) bool { + return schemaRows[i][2] < schemaRows[j][2] + } + sort.Slice(schemaRows, pathCmp) + sort.Slice(resourceRows, pathCmp) + schemaWriter := csv.NewWriter(schemaOut) resourceWriter := csv.NewWriter(resourceOut) @@ -1145,11 +1159,6 @@ func (m *MarshallableProviderShim) GetCSVSchema( schemaWriter.Write(row) } for _, row := range resourceRows { - // Find the token for the resource - path := row[2] - resourceName := strings.Split(path, ".")[1] - token := tokenMapping[resourceName] - row = append(row, token) resourceWriter.Write(row) } @@ -1383,20 +1392,21 @@ func MarshalProvider(p *Provider) *MarshallableProvider { return &info } -func (info *Provider) GetCSVSchema( +func (m *MarshallableProvider) GetCSVSchema( provider, version string, schemaOut, resourceOut *bytes.Buffer, ) error { - tokenMapping := make(map[string]string) + resourceTokenMapping := make(map[string]string) + datasourceTokenMapping := make(map[string]string) // Create a map of resource names to tokens - for k, v := range info.Resources { - tokenMapping[k] = v.Tok.String() + for k, v := range m.Resources { + resourceTokenMapping[k] = v.Tok.String() } - for k, v := range info.DataSources { - tokenMapping[k] = v.Tok.String() + for k, v := range m.DataSources { + datasourceTokenMapping[k] = v.Tok.String() } - return MarshalProviderShim(info.P).GetCSVSchema(provider, version, tokenMapping, schemaOut, resourceOut) + return m.Provider.GetCSVSchema(provider, version, resourceTokenMapping, datasourceTokenMapping, schemaOut, resourceOut) } // Unmarshal creates a mostly-=initialized Pulumi ProviderInfo value from the given MarshallableProviderInfo. diff --git a/pkg/tfbridge/main.go b/pkg/tfbridge/main.go index e2034a8b1..8f1556a1d 100644 --- a/pkg/tfbridge/main.go +++ b/pkg/tfbridge/main.go @@ -22,6 +22,7 @@ import ( "fmt" "io" "os" + "time" "github.com/pulumi/pulumi/sdk/v3/go/common/util/cmdutil" "github.com/pulumi/pulumi/sdk/v3/go/common/util/contract" @@ -68,13 +69,17 @@ func Main(pkg string, version string, prov ProviderInfo, pulumiSchema []byte) { if *dumpTFSchemaCSVPath != "" { resBuf := &bytes.Buffer{} schBuf := &bytes.Buffer{} - err := prov.GetCSVSchema(prov.Name, prov.Version, schBuf, resBuf) + err := MarshalProviderInfo(&prov).GetCSVSchema(prov.Name, prov.Version, schBuf, resBuf) if err != nil { cmdutil.ExitError(err.Error()) } - resPath := fmt.Sprintf("%s/resources.csv", *dumpTFSchemaCSVPath) - schPath := fmt.Sprintf("%s/schemas.csv", *dumpTFSchemaCSVPath) + // get_current date + currentTime := time.Now() + formattedTime := currentTime.Format("150405") + + resPath := fmt.Sprintf("%s/%s_%s_resources_%s.csv", *dumpTFSchemaCSVPath, prov.Name, prov.Version, formattedTime) + schPath := fmt.Sprintf("%s/%s_%s_schemas_%s.csv", *dumpTFSchemaCSVPath, prov.Name, prov.Version, formattedTime) if err := os.MkdirAll(*dumpTFSchemaCSVPath, 0o700); err != nil { cmdutil.ExitError(err.Error()) diff --git a/pkg/tfshim/sdk-v2/provider_test.go b/pkg/tfshim/sdk-v2/provider_test.go index 2e2a36a7e..0afebb1ef 100644 --- a/pkg/tfshim/sdk-v2/provider_test.go +++ b/pkg/tfshim/sdk-v2/provider_test.go @@ -330,40 +330,42 @@ func TestMarshalProviderShim(t *testing.T) { } }`).Equal(t, out.String()) - tokenMapping := map[string]string{ - "test_resource": "testResource", - "test_data_source": "testDataSource", + resTokenMapping := map[string]string{ + "test_resource": "myProvider:index:testResource", + } + datasourceTokenMapping := map[string]string{ + "test_data_source": "myProvider:index:testDataSource", } resBuf := &bytes.Buffer{} schBuf := &bytes.Buffer{} - err = marshallableProv.GetCSVSchema("myProvider", "0.0.1", tokenMapping, schBuf, resBuf) + err = marshallableProv.GetCSVSchema("myProvider", "0.0.1", resTokenMapping, datasourceTokenMapping, schBuf, resBuf) require.NoError(t, err) autogold.Expect(`provider,version,path,type,optional,required,computed,forceNew,maxItems,minItems,deprecated,default,configMode,conflictsWith,exactlyOneOf,atLeastOneOf,requiredWith -myProvider,0.0.1,myProvider.test_schema,String,true,false,false,false,0,0,,,auto,,,, +myProvider,0.0.1,myProvider.test_data_source.bar,Int,true,false,false,false,0,0,,,auto,,,, +myProvider,0.0.1,myProvider.test_data_source.foo,String,true,false,false,false,0,0,,,auto,,,, myProvider,0.0.1,myProvider.test_resource.bar,Int,true,false,false,false,0,0,,,auto,,,, -myProvider,0.0.1,myProvider.test_resource.nested_prop.Elem.nested_foo,String,true,false,false,false,0,0,,,auto,,,, -myProvider,0.0.1,myProvider.test_resource.nested_prop,List,true,false,false,false,0,0,,,auto,,,, -myProvider,0.0.1,myProvider.test_resource.max_item_one_prop.Elem.nested_foo,String,true,false,false,false,0,0,,,auto,,,, -myProvider,0.0.1,myProvider.test_resource.max_item_one_prop,List,false,true,false,false,1,0,,,auto,,,, -myProvider,0.0.1,myProvider.test_resource.map_prop.Elem,String,false,false,false,false,0,0,,,auto,,,, -myProvider,0.0.1,myProvider.test_resource.map_prop,Map,true,false,false,false,0,0,,,auto,,,, -myProvider,0.0.1,myProvider.test_resource.config_mode_prop.Elem.prop,String,true,false,false,false,0,0,,,auto,,,, myProvider,0.0.1,myProvider.test_resource.config_mode_prop,List,true,false,false,false,0,0,,,attr,,,, -myProvider,0.0.1,myProvider.test_resource.default_prop,String,true,false,false,false,0,0,,default,auto,,,, +myProvider,0.0.1,myProvider.test_resource.config_mode_prop.Elem.prop,String,true,false,false,false,0,0,,,auto,,,, myProvider,0.0.1,myProvider.test_resource.conflicting_prop,String,true,false,false,false,0,0,,,auto,bar,"foo,conflicting_prop","nested_prop,conflicting_prop",map_prop +myProvider,0.0.1,myProvider.test_resource.default_prop,String,true,false,false,false,0,0,,default,auto,,,, myProvider,0.0.1,myProvider.test_resource.foo,String,true,false,false,false,0,0,,,auto,,,, -myProvider,0.0.1,myProvider.test_data_source.bar,Int,true,false,false,false,0,0,,,auto,,,, -myProvider,0.0.1,myProvider.test_data_source.foo,String,true,false,false,false,0,0,,,auto,,,, +myProvider,0.0.1,myProvider.test_resource.map_prop,Map,true,false,false,false,0,0,,,auto,,,, +myProvider,0.0.1,myProvider.test_resource.map_prop.Elem,String,false,false,false,false,0,0,,,auto,,,, +myProvider,0.0.1,myProvider.test_resource.max_item_one_prop,List,false,true,false,false,1,0,,,auto,,,, +myProvider,0.0.1,myProvider.test_resource.max_item_one_prop.Elem.nested_foo,String,true,false,false,false,0,0,,,auto,,,, +myProvider,0.0.1,myProvider.test_resource.nested_prop,List,true,false,false,false,0,0,,,auto,,,, +myProvider,0.0.1,myProvider.test_resource.nested_prop.Elem.nested_foo,String,true,false,false,false,0,0,,,auto,,,, +myProvider,0.0.1,myProvider.test_schema,String,true,false,false,false,0,0,,,auto,,,, `).Equal(t, schBuf.String()) autogold.Expect(`provider,version,path,schemaVersion,token -myProvider,0.0.1,myProvider.test_resource.nested_prop.Elem,0,nested,testResource -myProvider,0.0.1,myProvider.test_resource.max_item_one_prop.Elem,0,nested,testResource -myProvider,0.0.1,myProvider.test_resource.config_mode_prop.Elem,0,nested,testResource -myProvider,0.0.1,myProvider.test_resource,0,resource,testResource -myProvider,0.0.1,myProvider.test_data_source,0,dataSource,testDataSource +myProvider,0.0.1,myProvider.test_resource.nested_prop.Elem,0,nested,myProvider:index:testResource +myProvider,0.0.1,myProvider.test_resource.max_item_one_prop.Elem,0,nested,myProvider:index:testResource +myProvider,0.0.1,myProvider.test_resource.config_mode_prop.Elem,0,nested,myProvider:index:testResource +myProvider,0.0.1,myProvider.test_resource,0,resource,myProvider:index:testResource +myProvider,0.0.1,myProvider.test_data_source,0,dataSource,myProvider:index:testDataSource `).Equal(t, resBuf.String()) } From 9dc029c5a8787dffdfb7418eb73a1deea4380e2b Mon Sep 17 00:00:00 2001 From: Venelin Date: Thu, 15 Aug 2024 11:29:18 +0100 Subject: [PATCH 3/4] add default func, implementation and usejsonnumber --- pf/internal/schemashim/attr_schema.go | 4 + pf/internal/schemashim/block_schema.go | 5 +- pf/internal/schemashim/datasource.go | 8 ++ .../schemashim/object_pseudoresource.go | 40 +++++++-- pf/internal/schemashim/resource.go | 8 ++ pf/internal/schemashim/type_schema.go | 4 + pf/proto/attribute.go | 2 + pf/proto/block.go | 3 + pf/proto/element.go | 2 + pf/proto/object.go | 3 + pf/proto/resource.go | 9 ++ pf/proto/schema.go | 6 +- pkg/tfbridge/info/info.go | 63 +++++++++----- pkg/tfshim/schema/resource.go | 8 ++ pkg/tfshim/schema/schema.go | 4 + pkg/tfshim/sdk-v1/resource.go | 14 +++- pkg/tfshim/sdk-v1/schema.go | 10 ++- pkg/tfshim/sdk-v2/provider_test.go | 84 ++++++++++++++----- pkg/tfshim/sdk-v2/resource.go | 17 +++- pkg/tfshim/sdk-v2/schema.go | 4 + pkg/tfshim/shim.go | 5 ++ pkg/tfshim/tfplugin5/resource.go | 14 +++- pkg/tfshim/tfplugin5/schema.go | 7 +- 23 files changed, 262 insertions(+), 62 deletions(-) diff --git a/pf/internal/schemashim/attr_schema.go b/pf/internal/schemashim/attr_schema.go index 3dd623d06..8c6b3b98a 100644 --- a/pf/internal/schemashim/attr_schema.go +++ b/pf/internal/schemashim/attr_schema.go @@ -30,6 +30,10 @@ type attrSchema struct { var _ shim.Schema = (*attrSchema)(nil) +func (s *attrSchema) Implementation() string { + return "pf" +} + func (s *attrSchema) Type() shim.ValueType { ty := s.attr.GetType() vt, err := convertType(ty) diff --git a/pf/internal/schemashim/block_schema.go b/pf/internal/schemashim/block_schema.go index 98cb19c1a..0ddd34f06 100644 --- a/pf/internal/schemashim/block_schema.go +++ b/pf/internal/schemashim/block_schema.go @@ -45,6 +45,10 @@ func (s *blockSchema) Type() shim.ValueType { return vt } +func (*blockSchema) Implementation() string { + return "pf" +} + func (s *blockSchema) Description() string { if desc := s.block.GetMarkdownDescription(); desc != "" { return desc @@ -54,7 +58,6 @@ func (s *blockSchema) Description() string { // Needs to return a shim.Schema, a shim.Resource, or nil. See docstring on shim.Schema.Elem(). func (s *blockSchema) Elem() interface{} { - asObjectType := func(typ any) (shim.Resource, bool) { if tt, ok := typ.(basetypes.ObjectTypable); ok { var res shim.Resource = newObjectPseudoResource(tt, diff --git a/pf/internal/schemashim/datasource.go b/pf/internal/schemashim/datasource.go index b7b58d931..6af435aa1 100644 --- a/pf/internal/schemashim/datasource.go +++ b/pf/internal/schemashim/datasource.go @@ -29,6 +29,14 @@ func (r *schemaOnlyDataSource) Schema() shim.SchemaMap { return r.tf.Shim() } +func (*schemaOnlyDataSource) Implementation() string { + return "pf" +} + +func (*schemaOnlyDataSource) UseJSONNumber() bool { + return false +} + func (r *schemaOnlyDataSource) SchemaVersion() int { return int(r.tf.ResourceSchemaVersion()) } diff --git a/pf/internal/schemashim/object_pseudoresource.go b/pf/internal/schemashim/object_pseudoresource.go index 7550dba6b..171e5696a 100644 --- a/pf/internal/schemashim/object_pseudoresource.go +++ b/pf/internal/schemashim/object_pseudoresource.go @@ -42,7 +42,8 @@ type objectPseudoResource struct { func newObjectPseudoResource(t basetypes.ObjectTypable, nestedAttrs map[string]pfutils.Attr, - nestedBlocks map[string]pfutils.Block) *objectPseudoResource { + nestedBlocks map[string]pfutils.Block, +) *objectPseudoResource { lowerType := t.TerraformType(context.Background()) objType, ok := lowerType.(tftypes.Object) contract.Assertf(ok, "t basetypes.ObjectTypable should produce a tftypes.Object "+ @@ -61,13 +62,19 @@ func newObjectPseudoResource(t basetypes.ObjectTypable, } } -var _ shim.Resource = (*objectPseudoResource)(nil) -var _ shim.SchemaMap = (*objectPseudoResource)(nil) +var ( + _ shim.Resource = (*objectPseudoResource)(nil) + _ shim.SchemaMap = (*objectPseudoResource)(nil) +) func (r *objectPseudoResource) Validate() error { return nil } +func (r *objectPseudoResource) Implementation() string { + return "pf" +} + func (r *objectPseudoResource) Schema() shim.SchemaMap { return r } @@ -76,6 +83,10 @@ func (*objectPseudoResource) SchemaVersion() int { return 0 } +func (*objectPseudoResource) UseJSONNumber() bool { + return false +} + func (*objectPseudoResource) DeprecationMessage() string { return "" } @@ -91,13 +102,15 @@ func (*objectPseudoResource) Timeouts() *shim.ResourceTimeout { } func (*objectPseudoResource) InstanceState(id string, object, - meta map[string]interface{}) (shim.InstanceState, error) { + meta map[string]interface{}, +) (shim.InstanceState, error) { panic("This is an Object type encoded as a shim.Resource, and " + "InstanceState() should not be called on this entity during schema generation") } func (*objectPseudoResource) DecodeTimeouts( - config shim.ResourceConfig) (*shim.ResourceTimeout, error) { + config shim.ResourceConfig, +) (*shim.ResourceTimeout, error) { panic("This is an Object type encoded as a shim.Resource, and " + "DecodeTimeouts() should not be called on this entity during schema generation") } @@ -186,7 +199,8 @@ func newTuplePseudoResource(t attr.TypeWithElementTypes) shim.Resource { return &tuplePseudoResource{ schemaOnly: schemaOnly{"tuplePseudoResource"}, attrs: attrs, - tuple: t} + tuple: t, + } } func (*tuplePseudoResource) SchemaVersion() int { panic("TODO") } @@ -196,6 +210,14 @@ func (r *tuplePseudoResource) Validate() error { return nil } +func (*tuplePseudoResource) Implementation() string { + return "pf" +} + +func (*tuplePseudoResource) UseJSONNumber() bool { + return false +} + func (r *tuplePseudoResource) Schema() shim.SchemaMap { return r } @@ -250,7 +272,8 @@ func (s *schemaOnly) Timeouts() *shim.ResourceTimeout { } func (s *schemaOnly) InstanceState(id string, object, - meta map[string]interface{}) (shim.InstanceState, error) { + meta map[string]interface{}, +) (shim.InstanceState, error) { m := "type" if s != nil || s.typ != "" { m = s.typ @@ -259,7 +282,8 @@ func (s *schemaOnly) InstanceState(id string, object, } func (s *schemaOnly) DecodeTimeouts( - config shim.ResourceConfig) (*shim.ResourceTimeout, error) { + config shim.ResourceConfig, +) (*shim.ResourceTimeout, error) { m := "type" if s != nil || s.typ != "" { m = s.typ diff --git a/pf/internal/schemashim/resource.go b/pf/internal/schemashim/resource.go index 97d5bb908..23f7a7708 100644 --- a/pf/internal/schemashim/resource.go +++ b/pf/internal/schemashim/resource.go @@ -29,6 +29,14 @@ func (r *schemaOnlyResource) Schema() shim.SchemaMap { return r.tf.Shim() } +func (*schemaOnlyResource) Implementation() string { + return "pf" +} + +func (*schemaOnlyResource) UseJSONNumber() bool { + return false +} + func (r *schemaOnlyResource) SchemaVersion() int { return int(r.tf.ResourceSchemaVersion()) } diff --git a/pf/internal/schemashim/type_schema.go b/pf/internal/schemashim/type_schema.go index f0c34974d..4c6e6cc30 100644 --- a/pf/internal/schemashim/type_schema.go +++ b/pf/internal/schemashim/type_schema.go @@ -40,6 +40,10 @@ func newTypeSchema(t pfattr.Type, nested map[string]pfutils.Attr) *typeSchema { } } +func (*typeSchema) Implementation() string { + return "pf" +} + func (s *typeSchema) Type() shim.ValueType { vt, err := convertType(s.t) if err != nil { diff --git a/pf/proto/attribute.go b/pf/proto/attribute.go index b880a3b74..87aa1beb0 100644 --- a/pf/proto/attribute.go +++ b/pf/proto/attribute.go @@ -25,6 +25,8 @@ var _ = shim.Schema(attribute{}) type attribute struct{ attr tfprotov6.SchemaAttribute } +func (a attribute) Implementation() string { return "pf" } + // Simple schema options func (a attribute) Optional() bool { return a.attr.Optional } diff --git a/pf/proto/block.go b/pf/proto/block.go index 4e07e4cd4..b1a2103a1 100644 --- a/pf/proto/block.go +++ b/pf/proto/block.go @@ -44,6 +44,8 @@ type blockResource struct { block *tfprotov6.SchemaBlock } +func (b blockResource) Implementation() string { return "pf" } +func (b blockResource) UseJSONNumber() bool { return false } func (b blockResource) Schema() shim.SchemaMap { return blockMap{b.block} } func (b blockResource) DeprecationMessage() string { return deprecated(b.block.Deprecated) } @@ -114,6 +116,7 @@ func (m blockSchema) Elem() interface{} { } } +func (m blockSchema) Implementation() string { return "pf" } func (m blockSchema) Optional() bool { return false } func (m blockSchema) Required() bool { return false } func (m blockSchema) Default() interface{} { return nil } diff --git a/pf/proto/element.go b/pf/proto/element.go index 49315de53..67aa76408 100644 --- a/pf/proto/element.go +++ b/pf/proto/element.go @@ -79,6 +79,8 @@ func (e element) Optional() bool { return e.optional } // Methods that are not available to tftypes.Type +func (e element) Implementation() string { return "pf" } +func (e element) UseJSONNumber() bool { return false } func (e element) Required() bool { return false } func (e element) Default() interface{} { return nil } func (e element) DefaultFunc() shim.SchemaDefaultFunc { return nil } diff --git a/pf/proto/object.go b/pf/proto/object.go index c1550e6da..46dee55bd 100644 --- a/pf/proto/object.go +++ b/pf/proto/object.go @@ -31,6 +31,9 @@ type object struct { obj tfprotov6.SchemaObject } +func (o object) Implementation() string { return "pf" } +func (o object) UseJSONNumber() bool { return false } + func (o object) Schema() shim.SchemaMap { contract.Assertf(o.obj.Nesting != tfprotov6.SchemaObjectNestingModeMap, "%T cannot be a map, since that would require `o` to represent a Map type", o) diff --git a/pf/proto/resource.go b/pf/proto/resource.go index 33c4fd056..580769826 100644 --- a/pf/proto/resource.go +++ b/pf/proto/resource.go @@ -62,6 +62,14 @@ func (m resourceMap) Set(key string, value shim.Resource) { type resource struct{ r *tfprotov6.Schema } +func (r resource) Implementation() string { + return "pf" +} + +func (r resource) UseJSONNumber() bool { + return false +} + func (r resource) Schema() shim.SchemaMap { return blockMap{r.r.Block} } @@ -81,6 +89,7 @@ func (r resource) Timeouts() *shim.ResourceTimeout { func (r resource) InstanceState(id string, object, meta map[string]interface{}) (shim.InstanceState, error) { panic("Cannot call InstanceState for a schema only resource") } + func (r resource) DecodeTimeouts(config shim.ResourceConfig) (*shim.ResourceTimeout, error) { panic("Cannot call DecodeTimeouts for a schema only resource") } diff --git a/pf/proto/schema.go b/pf/proto/schema.go index c9be2237a..3fc5e924f 100644 --- a/pf/proto/schema.go +++ b/pf/proto/schema.go @@ -23,19 +23,23 @@ import ( // pseudoResource represents a type that must pretent to be a [shim.Resource], but does not represent a resource. type pseudoResource struct{} +func (pseudoResource) Implementation() string { return "pf" } +func (pseudoResource) UseJSONNumber() bool { return false } func (pseudoResource) SchemaVersion() int { return 0 } func (pseudoResource) Importer() shim.ImportFunc { return nil } func (pseudoResource) Timeouts() *shim.ResourceTimeout { return nil } func (pseudoResource) InstanceState(id string, object, meta map[string]interface{}) (shim.InstanceState, error) { panic("Cannot invoke InstanceState on a pseudoResource") } + func (pseudoResource) DecodeTimeouts(config shim.ResourceConfig) (*shim.ResourceTimeout, error) { panic("Cannot invoke DecodeTimeouts on a pseudoResource") } func getSchemaMap[T any](m interface { GetOk(string) (T, bool) -}, key string) T { +}, key string, +) T { v, _ := m.GetOk(key) // Some functions (such as terraformToPulumiName; link: [1]) do not correctly use // GetOk, so we can't panic on Get for a missing value, even though that is the diff --git a/pkg/tfbridge/info/info.go b/pkg/tfbridge/info/info.go index 8f469e88b..ae739502e 100644 --- a/pkg/tfbridge/info/info.go +++ b/pkg/tfbridge/info/info.go @@ -25,6 +25,8 @@ import ( "context" "encoding/csv" "fmt" + "reflect" + "runtime" "sort" "strconv" "strings" @@ -814,12 +816,17 @@ type PreConfigureCallbackWithLogger func( config shim.ResourceConfig, ) error +func getFunctionName(i interface{}) string { + return runtime.FuncForPC(reflect.ValueOf(i).Pointer()).Name() +} + // The types below are marshallable versions of the schema descriptions associated with a provider. These are used when // marshalling a provider info as JSON; Note that these types only represent a subset of the information associated // with a ProviderInfo; thus, a ProviderInfo cannot be round-tripped through JSON. // MarshallableSchemaShim is the JSON-marshallable form of a Terraform schema. type MarshallableSchemaShim struct { + Implementation string `json:"implementation,omitempty"` Type shim.ValueType `json:"type"` Optional bool `json:"optional,omitempty"` Required bool `json:"required,omitempty"` @@ -831,23 +838,21 @@ type MarshallableSchemaShim struct { DeprecationMessage string `json:"deprecated,omitempty"` Default interface{} `json:"default,omitempty"` // SDK specific fields - ConfigMode shim.ConfigModeType `json:"configMode,omitempty"` - // TODO - // DefaultFunc - // Set - ConflictsWith []string `json:"conflictsWith,omitempty"` - ExactlyOneOf []string `json:"exactlyOneOf,omitempty"` - AtLeastOneOf []string `json:"atLeastOneOf,omitempty"` - RequiredWith []string `json:"requiredWith,omitempty"` + ConfigMode shim.ConfigModeType `json:"configMode,omitempty"` + DefaultFunc string `json:"defaultFunc,omitempty"` + ConflictsWith []string `json:"conflictsWith,omitempty"` + ExactlyOneOf []string `json:"exactlyOneOf,omitempty"` + AtLeastOneOf []string `json:"atLeastOneOf,omitempty"` + RequiredWith []string `json:"requiredWith,omitempty"` // PF specific fields - // TODO - // PlanModifiers + // PlanModifierDescriptions []string `json:"planModifiers,omitempty"` } // MarshalSchemaShim converts a Terraform schema into a MarshallableSchema. func MarshalSchemaShim(s shim.Schema) *MarshallableSchemaShim { return &MarshallableSchemaShim{ + Implementation: s.Implementation(), Type: s.Type(), Optional: s.Optional(), Required: s.Required(), @@ -863,6 +868,8 @@ func MarshalSchemaShim(s shim.Schema) *MarshallableSchemaShim { ExactlyOneOf: s.ExactlyOneOf(), AtLeastOneOf: s.AtLeastOneOf(), RequiredWith: s.RequiredWith(), + DefaultFunc: getFunctionName(s.DefaultFunc()), + // PlanModifierDescriptions: s.PlanModifierDescriptions(), } } @@ -892,6 +899,7 @@ func (m *MarshallableSchemaShim) GetFlattenedSchema( provider, version, path, + m.Implementation, fmt.Sprintf("%v", m.Type), strconv.FormatBool(m.Optional), strconv.FormatBool(m.Required), @@ -906,6 +914,8 @@ func (m *MarshallableSchemaShim) GetFlattenedSchema( strings.Join(m.ExactlyOneOf, ","), strings.Join(m.AtLeastOneOf, ","), strings.Join(m.RequiredWith, ","), + m.DefaultFunc, + // strings.Join(m.PlanModifierDescriptions, ","), } *schemaRows = append(*schemaRows, row) @@ -913,10 +923,12 @@ func (m *MarshallableSchemaShim) GetFlattenedSchema( // MarshallableResourceShim is the JSON-marshallable form of a Terraform resource schema. type MarshallableResourceShim struct { - Schema map[string]*MarshallableSchemaShim `json:"schema,omitempty"` - SchemaVersion int `json:"schemaVersion,omitempty"` - // StateUpgraders - // UseJSONNumber + Implementation string `json:"implementation,omitempty"` + Schema map[string]*MarshallableSchemaShim `json:"schema,omitempty"` + SchemaVersion int `json:"schemaVersion,omitempty"` + UseJSONNumber bool `json:"useJSONNumber,omitempty"` + // TODO + // StateUpgraders types? } // MarshalResourceShim converts a Terraform resource schema into a MarshallableResourceShim. @@ -930,8 +942,10 @@ func MarshalResourceShim(r shim.Resource) MarshallableResourceShim { return true }) return MarshallableResourceShim{ - Schema: m, - SchemaVersion: r.SchemaVersion(), + Schema: m, + Implementation: r.Implementation(), + SchemaVersion: r.SchemaVersion(), + UseJSONNumber: r.UseJSONNumber(), } } @@ -983,8 +997,10 @@ func (m *MarshallableResourceShim) GetFlattenedSchema( provider, version, path, + m.Implementation, strconv.Itoa(m.SchemaVersion), resourceType.String(), + strconv.FormatBool(m.UseJSONNumber), token, } @@ -1096,17 +1112,20 @@ func (m *MarshallableProviderShim) Unmarshal() shim.Provider { }).Shim() } -func (m *MarshallableProviderShim) GetFlattenedSchema(provider, version string, resourceTokenMapping, datasourceTokenMapping map[string]string) ( +func (m *MarshallableProviderShim) GetFlattenedSchema( + provider, version string, resourceTokenMapping, datasourceTokenMapping map[string]string) ( schemaRows [][]string, resourceRows [][]string, ) { for k, v := range m.Schema { v.GetFlattenedSchema(provider, version, provider+"."+k, nil, &schemaRows, &resourceRows) } for k, v := range m.Resources { - v.GetFlattenedSchema(provider, version, provider+"."+k, ResourceTypeResource, resourceTokenMapping, &schemaRows, &resourceRows) + v.GetFlattenedSchema( + provider, version, provider+"."+k, ResourceTypeResource, resourceTokenMapping, &schemaRows, &resourceRows) } for k, v := range m.DataSources { - v.GetFlattenedSchema(provider, version, provider+"."+k, ResourceTypeDataSource, datasourceTokenMapping, &schemaRows, &resourceRows) + v.GetFlattenedSchema( + provider, version, provider+"."+k, ResourceTypeDataSource, datasourceTokenMapping, &schemaRows, &resourceRows) } return } @@ -1131,6 +1150,7 @@ func (m *MarshallableProviderShim) GetCSVSchema( "provider", "version", "path", + "implementation", "type", "optional", "required", @@ -1145,13 +1165,18 @@ func (m *MarshallableProviderShim) GetCSVSchema( "exactlyOneOf", "atLeastOneOf", "requiredWith", + "defaultFunc", + // "planModifiers", }) resourceWriter.Write([]string{ "provider", "version", "path", + "implementation", "schemaVersion", + "resourceType", + "useJSONNumber", "token", }) diff --git a/pkg/tfshim/schema/resource.go b/pkg/tfshim/schema/resource.go index bdeab637f..1a7f38785 100644 --- a/pkg/tfshim/schema/resource.go +++ b/pkg/tfshim/schema/resource.go @@ -24,6 +24,10 @@ type ResourceShim struct { V *Resource } +func (r ResourceShim) Implementation() string { + return "tfshim" +} + func (r ResourceShim) Schema() shim.SchemaMap { return r.V.Schema } @@ -32,6 +36,10 @@ func (r ResourceShim) SchemaVersion() int { return r.V.SchemaVersion } +func (r ResourceShim) UseJSONNumber() bool { + return false +} + func (r ResourceShim) Importer() shim.ImportFunc { return r.V.Importer } diff --git a/pkg/tfshim/schema/schema.go b/pkg/tfshim/schema/schema.go index c394ebbf7..088c67279 100644 --- a/pkg/tfshim/schema/schema.go +++ b/pkg/tfshim/schema/schema.go @@ -35,6 +35,10 @@ type SchemaShim struct { V *Schema } +func (s SchemaShim) Implementation() string { + return "tfshim" +} + func (s SchemaShim) Type() shim.ValueType { return s.V.Type } diff --git a/pkg/tfshim/sdk-v1/resource.go b/pkg/tfshim/sdk-v1/resource.go index 8eea854ac..0c149bafc 100644 --- a/pkg/tfshim/sdk-v1/resource.go +++ b/pkg/tfshim/sdk-v1/resource.go @@ -9,8 +9,10 @@ import ( shim "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfshim" ) -var _ = shim.Resource(v1Resource{}) -var _ = shim.ResourceMap(v1ResourceMap{}) +var ( + _ = shim.Resource(v1Resource{}) + _ = shim.ResourceMap(v1ResourceMap{}) +) type v1Resource struct { tf *schema.Resource @@ -20,6 +22,10 @@ func NewResource(r *schema.Resource) shim.Resource { return v1Resource{r} } +func (s v1Resource) Implementation() string { + return "sdkv1" +} + func (r v1Resource) Schema() shim.SchemaMap { return v1SchemaMap(r.tf.Schema) } @@ -116,6 +122,10 @@ func (r v1Resource) DecodeTimeouts(config shim.ResourceConfig) (*shim.ResourceTi }, nil } +func (r v1Resource) UseJSONNumber() bool { + return false +} + type v1ResourceMap map[string]*schema.Resource func (m v1ResourceMap) Len() int { diff --git a/pkg/tfshim/sdk-v1/schema.go b/pkg/tfshim/sdk-v1/schema.go index 29c57655f..b59806b20 100644 --- a/pkg/tfshim/sdk-v1/schema.go +++ b/pkg/tfshim/sdk-v1/schema.go @@ -7,8 +7,10 @@ import ( shim "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfshim" ) -var _ = shim.Schema(v1Schema{}) -var _ = shim.SchemaMap(v1SchemaMap{}) +var ( + _ = shim.Schema(v1Schema{}) + _ = shim.SchemaMap(v1SchemaMap{}) +) // UnknownVariableValue is the sentinal defined in github.com/hashicorp/terraform/configs/hcl2shim, // representing a variable whose value is not known at some particular time. The value is duplicated here in @@ -24,6 +26,10 @@ func NewSchema(s *schema.Schema) shim.Schema { return v1Schema{s} } +func (s v1Schema) Implementation() string { + return "sdkv1" +} + func (s v1Schema) Type() shim.ValueType { switch s.tf.Type { case schema.TypeBool: diff --git a/pkg/tfshim/sdk-v2/provider_test.go b/pkg/tfshim/sdk-v2/provider_test.go index 0afebb1ef..b30a3462b 100644 --- a/pkg/tfshim/sdk-v2/provider_test.go +++ b/pkg/tfshim/sdk-v2/provider_test.go @@ -109,6 +109,10 @@ func TestProvider1UpgradeResourceState(t *testing.T) { } } +func barDefaultFunc() (interface{}, error) { + return "bar", nil +} + //nolint:lll func TestMarshalProviderShim(t *testing.T) { prov := NewProvider(&schema.Provider{ @@ -181,7 +185,13 @@ func TestMarshalProviderShim(t *testing.T) { AtLeastOneOf: []string{"nested_prop", "conflicting_prop"}, Optional: true, }, + "default_func_prop": { + Type: schema.TypeString, + Optional: true, + DefaultFunc: barDefaultFunc, + }, }, + UseJSONNumber: true, }, }, DataSourcesMap: map[string]*schema.Resource{ @@ -220,24 +230,30 @@ func TestMarshalProviderShim(t *testing.T) { autogold.Expect(`{ "schema": { "test_schema": { + "implementation": "sdkv2", "type": "String", "optional": true } }, "resources": { "test_resource": { + "implementation": "sdkv2", "schema": { "bar": { + "implementation": "sdkv2", "type": "Int", "optional": true }, "config_mode_prop": { + "implementation": "sdkv2", "type": "List", "optional": true, "element": { "resource": { + "implementation": "sdkv2", "schema": { "prop": { + "implementation": "sdkv2", "type": "String", "optional": true } @@ -247,6 +263,7 @@ func TestMarshalProviderShim(t *testing.T) { "configMode": "attr" }, "conflicting_prop": { + "implementation": "sdkv2", "type": "String", "optional": true, "conflictsWith": [ @@ -264,31 +281,44 @@ func TestMarshalProviderShim(t *testing.T) { "map_prop" ] }, + "default_func_prop": { + "implementation": "sdkv2", + "type": "String", + "optional": true, + "defaultFunc": "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfshim/sdk-v2.barDefaultFunc" + }, "default_prop": { + "implementation": "sdkv2", "type": "String", "optional": true, "default": "default" }, "foo": { + "implementation": "sdkv2", "type": "String", "optional": true }, "map_prop": { + "implementation": "sdkv2", "type": "Map", "optional": true, "element": { "schema": { + "implementation": "sdkv2", "type": "String" } } }, "max_item_one_prop": { + "implementation": "sdkv2", "type": "List", "required": true, "element": { "resource": { + "implementation": "sdkv2", "schema": { "nested_foo": { + "implementation": "sdkv2", "type": "String", "optional": true } @@ -298,12 +328,15 @@ func TestMarshalProviderShim(t *testing.T) { "maxItems": 1 }, "nested_prop": { + "implementation": "sdkv2", "type": "List", "optional": true, "element": { "resource": { + "implementation": "sdkv2", "schema": { "nested_foo": { + "implementation": "sdkv2", "type": "String", "optional": true } @@ -311,17 +344,21 @@ func TestMarshalProviderShim(t *testing.T) { } } } - } + }, + "useJSONNumber": true } }, "dataSources": { "test_data_source": { + "implementation": "sdkv2", "schema": { "bar": { + "implementation": "sdkv2", "type": "Int", "optional": true }, "foo": { + "implementation": "sdkv2", "type": "String", "optional": true } @@ -343,29 +380,30 @@ func TestMarshalProviderShim(t *testing.T) { err = marshallableProv.GetCSVSchema("myProvider", "0.0.1", resTokenMapping, datasourceTokenMapping, schBuf, resBuf) require.NoError(t, err) - autogold.Expect(`provider,version,path,type,optional,required,computed,forceNew,maxItems,minItems,deprecated,default,configMode,conflictsWith,exactlyOneOf,atLeastOneOf,requiredWith -myProvider,0.0.1,myProvider.test_data_source.bar,Int,true,false,false,false,0,0,,,auto,,,, -myProvider,0.0.1,myProvider.test_data_source.foo,String,true,false,false,false,0,0,,,auto,,,, -myProvider,0.0.1,myProvider.test_resource.bar,Int,true,false,false,false,0,0,,,auto,,,, -myProvider,0.0.1,myProvider.test_resource.config_mode_prop,List,true,false,false,false,0,0,,,attr,,,, -myProvider,0.0.1,myProvider.test_resource.config_mode_prop.Elem.prop,String,true,false,false,false,0,0,,,auto,,,, -myProvider,0.0.1,myProvider.test_resource.conflicting_prop,String,true,false,false,false,0,0,,,auto,bar,"foo,conflicting_prop","nested_prop,conflicting_prop",map_prop -myProvider,0.0.1,myProvider.test_resource.default_prop,String,true,false,false,false,0,0,,default,auto,,,, -myProvider,0.0.1,myProvider.test_resource.foo,String,true,false,false,false,0,0,,,auto,,,, -myProvider,0.0.1,myProvider.test_resource.map_prop,Map,true,false,false,false,0,0,,,auto,,,, -myProvider,0.0.1,myProvider.test_resource.map_prop.Elem,String,false,false,false,false,0,0,,,auto,,,, -myProvider,0.0.1,myProvider.test_resource.max_item_one_prop,List,false,true,false,false,1,0,,,auto,,,, -myProvider,0.0.1,myProvider.test_resource.max_item_one_prop.Elem.nested_foo,String,true,false,false,false,0,0,,,auto,,,, -myProvider,0.0.1,myProvider.test_resource.nested_prop,List,true,false,false,false,0,0,,,auto,,,, -myProvider,0.0.1,myProvider.test_resource.nested_prop.Elem.nested_foo,String,true,false,false,false,0,0,,,auto,,,, -myProvider,0.0.1,myProvider.test_schema,String,true,false,false,false,0,0,,,auto,,,, + autogold.Expect(`provider,version,path,implementation,type,optional,required,computed,forceNew,maxItems,minItems,deprecated,default,configMode,conflictsWith,exactlyOneOf,atLeastOneOf,requiredWith,defaultFunc +myProvider,0.0.1,myProvider.test_data_source.bar,sdkv2,Int,true,false,false,false,0,0,,,auto,,,,, +myProvider,0.0.1,myProvider.test_data_source.foo,sdkv2,String,true,false,false,false,0,0,,,auto,,,,, +myProvider,0.0.1,myProvider.test_resource.bar,sdkv2,Int,true,false,false,false,0,0,,,auto,,,,, +myProvider,0.0.1,myProvider.test_resource.config_mode_prop,sdkv2,List,true,false,false,false,0,0,,,attr,,,,, +myProvider,0.0.1,myProvider.test_resource.config_mode_prop.Elem.prop,sdkv2,String,true,false,false,false,0,0,,,auto,,,,, +myProvider,0.0.1,myProvider.test_resource.conflicting_prop,sdkv2,String,true,false,false,false,0,0,,,auto,bar,"foo,conflicting_prop","nested_prop,conflicting_prop",map_prop, +myProvider,0.0.1,myProvider.test_resource.default_func_prop,sdkv2,String,true,false,false,false,0,0,,,auto,,,,,github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfshim/sdk-v2.barDefaultFunc +myProvider,0.0.1,myProvider.test_resource.default_prop,sdkv2,String,true,false,false,false,0,0,,default,auto,,,,, +myProvider,0.0.1,myProvider.test_resource.foo,sdkv2,String,true,false,false,false,0,0,,,auto,,,,, +myProvider,0.0.1,myProvider.test_resource.map_prop,sdkv2,Map,true,false,false,false,0,0,,,auto,,,,, +myProvider,0.0.1,myProvider.test_resource.map_prop.Elem,sdkv2,String,false,false,false,false,0,0,,,auto,,,,, +myProvider,0.0.1,myProvider.test_resource.max_item_one_prop,sdkv2,List,false,true,false,false,1,0,,,auto,,,,, +myProvider,0.0.1,myProvider.test_resource.max_item_one_prop.Elem.nested_foo,sdkv2,String,true,false,false,false,0,0,,,auto,,,,, +myProvider,0.0.1,myProvider.test_resource.nested_prop,sdkv2,List,true,false,false,false,0,0,,,auto,,,,, +myProvider,0.0.1,myProvider.test_resource.nested_prop.Elem.nested_foo,sdkv2,String,true,false,false,false,0,0,,,auto,,,,, +myProvider,0.0.1,myProvider.test_schema,sdkv2,String,true,false,false,false,0,0,,,auto,,,,, `).Equal(t, schBuf.String()) - autogold.Expect(`provider,version,path,schemaVersion,token -myProvider,0.0.1,myProvider.test_resource.nested_prop.Elem,0,nested,myProvider:index:testResource -myProvider,0.0.1,myProvider.test_resource.max_item_one_prop.Elem,0,nested,myProvider:index:testResource -myProvider,0.0.1,myProvider.test_resource.config_mode_prop.Elem,0,nested,myProvider:index:testResource -myProvider,0.0.1,myProvider.test_resource,0,resource,myProvider:index:testResource -myProvider,0.0.1,myProvider.test_data_source,0,dataSource,myProvider:index:testDataSource + autogold.Expect(`provider,version,path,implementation,schemaVersion,resourceType,useJSONNumber,token +myProvider,0.0.1,myProvider.test_resource.config_mode_prop.Elem,sdkv2,0,nested,false,myProvider:index:testResource +myProvider,0.0.1,myProvider.test_resource.max_item_one_prop.Elem,sdkv2,0,nested,false,myProvider:index:testResource +myProvider,0.0.1,myProvider.test_resource.nested_prop.Elem,sdkv2,0,nested,false,myProvider:index:testResource +myProvider,0.0.1,myProvider.test_resource,sdkv2,0,resource,true,myProvider:index:testResource +myProvider,0.0.1,myProvider.test_data_source,sdkv2,0,dataSource,false,myProvider:index:testDataSource `).Equal(t, resBuf.String()) } diff --git a/pkg/tfshim/sdk-v2/resource.go b/pkg/tfshim/sdk-v2/resource.go index ded02c2a1..189816d26 100644 --- a/pkg/tfshim/sdk-v2/resource.go +++ b/pkg/tfshim/sdk-v2/resource.go @@ -10,8 +10,10 @@ import ( shim "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfshim" ) -var _ = shim.Resource(v2Resource{}) -var _ = shim.ResourceMap(v2ResourceMap{}) +var ( + _ = shim.Resource(v2Resource{}) + _ = shim.ResourceMap(v2ResourceMap{}) +) type v2Resource struct { tf *schema.Resource @@ -21,6 +23,10 @@ func NewResource(r *schema.Resource) shim.Resource { return v2Resource{r} } +func (r v2Resource) Implementation() string { + return "sdkv2" +} + func (r v2Resource) Schema() shim.SchemaMap { return v2SchemaMap(r.tf.SchemaMap()) } @@ -29,6 +35,10 @@ func (r v2Resource) SchemaVersion() int { return r.tf.SchemaVersion } +func (r v2Resource) UseJSONNumber() bool { + return r.tf.UseJSONNumber +} + //nolint:staticcheck func (r v2Resource) Importer() shim.ImportFunc { if r.tf.Importer == nil { @@ -109,7 +119,8 @@ func (r v2Resource) InstanceState(id string, object, meta map[string]interface{} ID: id, Attributes: attributes, Meta: meta, - }, nil}, nil + }, nil, + }, nil } func (r v2Resource) DecodeTimeouts(config shim.ResourceConfig) (*shim.ResourceTimeout, error) { diff --git a/pkg/tfshim/sdk-v2/schema.go b/pkg/tfshim/sdk-v2/schema.go index d54cb84aa..dc3bbe2ef 100644 --- a/pkg/tfshim/sdk-v2/schema.go +++ b/pkg/tfshim/sdk-v2/schema.go @@ -24,6 +24,10 @@ func NewSchema(s *schema.Schema) shim.Schema { return v2Schema{s} } +func (s v2Schema) Implementation() string { + return "sdkv2" +} + func (s v2Schema) Type() shim.ValueType { switch s.tf.Type { case schema.TypeBool: diff --git a/pkg/tfshim/shim.go b/pkg/tfshim/shim.go index 2fcac448f..40e47b9cd 100644 --- a/pkg/tfshim/shim.go +++ b/pkg/tfshim/shim.go @@ -123,6 +123,7 @@ type SchemaDefaultFunc func() (interface{}, error) type SchemaStateFunc func(interface{}) string type Schema interface { + Implementation() string Type() ValueType Optional() bool Required() bool @@ -179,6 +180,8 @@ type Schema interface { SetElement(config interface{}) (interface{}, error) SetHash(v interface{}) int + // TODO + // PlanModifierDescriptions() []string } type SchemaMap interface { @@ -208,7 +211,9 @@ type ResourceTimeout struct { type Resource interface { Schema() SchemaMap + Implementation() string SchemaVersion() int + UseJSONNumber() bool Importer() ImportFunc DeprecationMessage() string Timeouts() *ResourceTimeout diff --git a/pkg/tfshim/tfplugin5/resource.go b/pkg/tfshim/tfplugin5/resource.go index 42529710a..33bcaf130 100644 --- a/pkg/tfshim/tfplugin5/resource.go +++ b/pkg/tfshim/tfplugin5/resource.go @@ -10,8 +10,10 @@ import ( "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfshim/schema" ) -var _ = shim.Resource((*resource)(nil)) -var _ = shim.ResourceMap(resourceMap{}) +var ( + _ = shim.Resource((*resource)(nil)) + _ = shim.ResourceMap(resourceMap{}) +) type resource struct { provider *provider @@ -22,6 +24,10 @@ type resource struct { schemaVersion int } +func (r *resource) Implementation() string { + return "tfplugin5" +} + func (r *resource) Schema() shim.SchemaMap { return r.schema } @@ -30,6 +36,10 @@ func (r *resource) SchemaVersion() int { return r.schemaVersion } +func (r *resource) UseJSONNumber() bool { + return false +} + func (r *resource) Importer() shim.ImportFunc { if r.provider == nil { return nil diff --git a/pkg/tfshim/tfplugin5/schema.go b/pkg/tfshim/tfplugin5/schema.go index 50ee85d76..1a9d47dd6 100644 --- a/pkg/tfshim/tfplugin5/schema.go +++ b/pkg/tfshim/tfplugin5/schema.go @@ -13,6 +13,8 @@ import ( // rather a lot of things. const UnknownVariableValue = "74D93920-ED26-11E3-AC10-0800200C9A66" +var _ = shim.Schema((*attributeSchema)(nil)) + type attributeSchema struct { ctyType cty.Type valueType shim.ValueType @@ -28,6 +30,10 @@ type attributeSchema struct { sensitive bool } +func (s *attributeSchema) Implementation() string { + return "tfplugin5" +} + func (s *attributeSchema) Type() shim.ValueType { return s.valueType } @@ -100,7 +106,6 @@ func (s attributeSchema) ConfigMode() shim.ConfigModeType { return 0 } - func (s *attributeSchema) Removed() string { return "" } From 24dc9b578a4d86a16d7ac3146786862b0ff831ef Mon Sep 17 00:00:00 2001 From: Venelin Date: Thu, 15 Aug 2024 11:41:18 +0100 Subject: [PATCH 4/4] fix panics --- pf/internal/schemashim/attr_schema.go | 2 +- pf/internal/schemashim/block_schema.go | 2 +- pf/internal/schemashim/type_schema.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pf/internal/schemashim/attr_schema.go b/pf/internal/schemashim/attr_schema.go index 8c6b3b98a..33a039ebb 100644 --- a/pf/internal/schemashim/attr_schema.go +++ b/pf/internal/schemashim/attr_schema.go @@ -55,7 +55,7 @@ func (s *attrSchema) Default() interface{} { return nil } func (*attrSchema) DefaultFunc() shim.SchemaDefaultFunc { - panic("DefaultFunc() should not be called during schema generation") + return nil } func (*attrSchema) DefaultValue() (interface{}, error) { // DefaultValue() should not be called by tfgen, but it currently may be called by ExtractInputsFromOutputs, so diff --git a/pf/internal/schemashim/block_schema.go b/pf/internal/schemashim/block_schema.go index 0ddd34f06..bb8563ea9 100644 --- a/pf/internal/schemashim/block_schema.go +++ b/pf/internal/schemashim/block_schema.go @@ -129,7 +129,7 @@ func (s *blockSchema) Default() interface{} { } func (*blockSchema) DefaultFunc() shim.SchemaDefaultFunc { - panic("DefaultFunc() should not be called during schema generation") + return nil } func (*blockSchema) DefaultValue() (interface{}, error) { diff --git a/pf/internal/schemashim/type_schema.go b/pf/internal/schemashim/type_schema.go index 4c6e6cc30..ff34ea413 100644 --- a/pf/internal/schemashim/type_schema.go +++ b/pf/internal/schemashim/type_schema.go @@ -87,7 +87,7 @@ func (*typeSchema) Default() interface{} { } func (*typeSchema) DefaultFunc() shim.SchemaDefaultFunc { - panic("DefaultFunc() should not be called during schema generation") + return nil } func (*typeSchema) DefaultValue() (interface{}, error) {