Generate SDKs via pulumi package gen-sdk

pkg/tests/schema_generation_test.go

@@ -4,6 +4,8 @@ import (
 	"bytes"
 	"encoding/json"
 	"io"
+	"os"
+	"path/filepath"
 	"runtime"
 	"testing"
 
@@ -12,9 +14,12 @@ import (
 	pschema "github.com/pulumi/pulumi/pkg/v3/codegen/schema"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/diag"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/diag/colors"
+	"github.com/pulumi/pulumi/sdk/v3/go/common/tokens"
+	"github.com/spf13/afero"
 	"github.com/stretchr/testify/require"
 
 	"github.com/pulumi/pulumi-terraform-bridge/v3/pkg/internal/tests/pulcheck"
+	"github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfbridge/info"
 	"github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfgen"
 )
 
@@ -115,3 +120,177 @@ func TestRequiredInputWithDefaultFlagDisabled(t *testing.T) {
 	resourceSchema := schema.Resources["testprovider:index/res:Res"]
 	require.Contains(t, resourceSchema.RequiredInputs, "name")
 }
+
+func skipOnWindows(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skipf("Skipping on windows - tests cases need to be made robust to newline handling")
+	}
+}
+
+func Test_Generate(t *testing.T) {
+	t.Parallel()
+	skipOnWindows(t)
+
+	p := pulcheck.BridgedProvider(t, "prov", &schema.Provider{
+		ResourcesMap: map[string]*schema.Resource{
+			"prov_test": {Schema: map[string]*schema.Schema{
+				"test": {Type: schema.TypeString, Optional: true},
+			}},
+		},
+	})
+
+	sink := diag.DefaultSink(os.Stdout, os.Stderr, diag.FormatOptions{
+		Color: colors.Never,
+	})
+
+	outDir := t.TempDir()
+
+	var root afero.Fs
+	if outDir != "" {
+		absOutDir, err := filepath.Abs(outDir)
+		require.NoError(t, err)
+		require.NoError(t, os.MkdirAll(absOutDir, 0o700))
+		root = afero.NewBasePathFs(afero.NewOsFs(), absOutDir)
+	}
+
+	gen, err := tfgen.NewGenerator(tfgen.GeneratorOptions{
+		Package:       "prov",
+		Version:       "0.0.1",
+		ProviderInfo:  p,
+		Root:          root,
+		Language:      tfgen.NodeJS,
+		XInMemoryDocs: true,
+		SkipDocs:      true,
+		SkipExamples:  true,
+		Sink:          sink,
+		Debug:         true,
+	})
+	require.NoError(t, err)
+
+	_, err = gen.Generate()
+	require.NoError(t, err)
+
+	_, err = afero.ReadFile(root, "test.ts")
+	require.NoError(t, err)
+	_, err = afero.ReadFile(root, "package.json")
+	require.NoError(t, err)
+
+	err = afero.Walk(root, ".", func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if info.IsDir() {
+			return nil
+		}
+		content, err := afero.ReadFile(root, path)
+		if err != nil {
+			return err
+		}
+		t.Logf("file: %s", path)
+		t.Logf("content: %s", string(content))
+		return nil
+	})
+	require.NoError(t, err)
+}
+
+func Test_GenerateWithOverlay(t *testing.T) {
+	t.Parallel()
+	skipOnWindows(t)
+
+	p := pulcheck.BridgedProvider(t, "prov", &schema.Provider{
+		ResourcesMap: map[string]*schema.Resource{
+			"prov_test": {Schema: map[string]*schema.Schema{
+				"test": {Type: schema.TypeString, Optional: true},
+			}},
+			"prov_test_mod": {Schema: map[string]*schema.Schema{
+				"test": {Type: schema.TypeString, Optional: true},
+			}},
+		},
+	})
+
+	moduleName := "mod"
+	// overwrite the token to ensure the module is generated
+	p.Resources["prov_test_mod"].Tok = tokens.Type("prov:mod/Mod:Test")
+
+	sink := diag.DefaultSink(os.Stdout, os.Stderr, diag.FormatOptions{
+		Color: colors.Never,
+	})
+
+	language := tfgen.NodeJS
+
+	overlayFileName := "hello.ts"
+	overlayFileContent := []byte(`
+	export const hello = "world";
+	`)
+
+	overlayModFileName := "helloMod.ts"
+	overlayModFileContent := []byte(`
+	export const helloMod = "worldMod";
+	`)
+
+	if p.JavaScript == nil {
+		p.JavaScript = &info.JavaScript{}
+	}
+	p.JavaScript.Overlay = &info.Overlay{
+		DestFiles: []string{
+			overlayFileName,
+		},
+		Modules: map[string]*info.Overlay{
+			moduleName: {
+				DestFiles: []string{
+					overlayModFileName,
+				},
+			},
+		},
+	}
+
+	root := afero.NewMemMapFs()
+
+	err := afero.WriteFile(root, overlayFileName, overlayFileContent, 0o600)
+	require.NoError(t, err)
+
+	err = afero.WriteFile(root, filepath.Join(moduleName, overlayModFileName), overlayModFileContent, 0o600)
+	require.NoError(t, err)
+
+	gen, err := tfgen.NewGenerator(tfgen.GeneratorOptions{
+		Package:       "prov",
+		Version:       "0.0.1",
+		ProviderInfo:  p,
+		Root:          root,
+		Language:      language,
+		XInMemoryDocs: true,
+		SkipDocs:      true,
+		SkipExamples:  true,
+		Sink:          sink,
+		Debug:         true,
+	})
+	require.NoError(t, err)
+
+	_, err = gen.Generate()
+	require.NoError(t, err)
+
+	content, err := afero.ReadFile(root, overlayFileName)
+	require.NoError(t, err)
+	require.Equal(t, overlayFileContent, content)
+
+	content, err = afero.ReadFile(root, filepath.Join(moduleName, overlayModFileName))
+	require.NoError(t, err)
+	require.Equal(t, overlayModFileContent, content)
+
+	err = afero.Walk(root, ".", func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if info.IsDir() {
+			return nil
+		}
+		content, err := afero.ReadFile(root, path)
+		if err != nil {
+			return err
+		}
+		t.Logf("file: %s", path)
+		t.Logf("content: %s", string(content))
+		return nil
+	})
+	require.NoError(t, err)
+}

pkg/tfgen/generate.go

@@ -23,6 +23,7 @@ import (
 	"io"
 	"io/fs"
 	"os"
+	"os/exec"
 	"path"
 	"path/filepath"
 	"sort"
@@ -33,11 +34,7 @@ import (
 	"github.com/hashicorp/go-multierror"
 	pkgerrors "github.com/pkg/errors"
 	"github.com/pulumi/pulumi/pkg/v3/codegen"
-	dotnetgen "github.com/pulumi/pulumi/pkg/v3/codegen/dotnet"
-	gogen "github.com/pulumi/pulumi/pkg/v3/codegen/go"
-	nodejsgen "github.com/pulumi/pulumi/pkg/v3/codegen/nodejs"
 	"github.com/pulumi/pulumi/pkg/v3/codegen/pcl"
-	pygen "github.com/pulumi/pulumi/pkg/v3/codegen/python"
 	pschema "github.com/pulumi/pulumi/pkg/v3/codegen/schema"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/diag"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/resource/plugin"
@@ -115,8 +112,120 @@ func (l Language) shouldConvertExamples() bool {
 	return false
 }
 
+func dirToBytesMap(fs afero.Fs, dir string) (map[string][]byte, error) {
+	result := make(map[string][]byte)
+	err := afero.Walk(fs, dir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if info.IsDir() {
+			return nil
+		}
+		relPath, err := filepath.Rel(dir, path)
+		if err != nil {
+			return err
+		}
+		if relPath == "." {
+			return nil
+		}
+		content, err := afero.ReadFile(fs, path)
+		if err != nil {
+			return err
+		}
+		result[relPath] = content
+		return nil
+	})
+	return result, err
+}
+
+func writeBytesMapToDir(fs afero.Fs, dir string, files map[string][]byte) error {
+	err := fs.MkdirAll(dir, 0o755)
+	if err != nil {
+		return pkgerrors.Wrap(err, "failed to create dir")
+	}
+	for name, content := range files {
+		srcDir := filepath.Dir(name)
+		if srcDir != "." {
+			err = fs.MkdirAll(filepath.Join(dir, srcDir), 0o755)
+			if err != nil {
+				return pkgerrors.Wrap(err, "failed to create dir")
+			}
+		}
+		err := afero.WriteFile(fs, filepath.Join(dir, name), content, 0o600)
+		if err != nil {
+			return pkgerrors.Wrap(err, "failed to write file")
+		}
+	}
+	return nil
+}
+
+func runPulumiPackageGenSDK(l Language, pkg *pschema.Package, extraFiles map[string][]byte) (map[string][]byte, error) {
+	var err error
+
+	fs := afero.NewOsFs()
+	outDir, err := afero.TempDir(fs, "", "pulumi-package-gen-sdk")
+	if err != nil {
+		return nil, pkgerrors.Wrap(err, "failed to create temp dir")
+	}
+	defer func() {
+		rmErr := fs.RemoveAll(outDir)
+		if rmErr != nil {
+			err = multierror.Append(err, rmErr)
+		}
+	}()
+
+	args := []string{"package", "gen-sdk", "--language", string(l), "--out", outDir}
+
+	if len(extraFiles) > 0 {
+		overlayDir, err := afero.TempDir(fs, "", "pulumi-package-gen-sdk-overlays")
+		if err != nil {
+			return nil, pkgerrors.Wrap(err, "failed to create temp dir")
+		}
+		defer func() {
+			rmErr := fs.RemoveAll(overlayDir)
+			if rmErr != nil {
+				err = multierror.Append(err, rmErr)
+			}
+		}()
+		dest := filepath.Join(overlayDir, string(l))
+		err = writeBytesMapToDir(fs, dest, extraFiles)
+		if err != nil {
+			return nil, pkgerrors.Wrap(err, "failed to write overlay files")
+		}
+
+		args = append(args, "--overlays", overlayDir)
+	}
+
+	schemaDir, err := afero.TempDir(fs, "", "schema")
+	if err != nil {
+		return nil, pkgerrors.Wrap(err, "failed to create temp dir")
+	}
+	schemaFile := filepath.Join(schemaDir, "schema.json")
+	schemaBytes, err := pkg.MarshalJSON()
+	if err != nil {
+		return nil, pkgerrors.Wrap(err, "failed to marshal schema")
+	}
+	err = afero.WriteFile(fs, schemaFile, schemaBytes, 0o600)
+	if err != nil {
+		return nil, pkgerrors.Wrap(err, "failed to write schema")
+	}
+
+	args = append(args, schemaFile)
+
+	cmd := exec.Command("pulumi", args...)
+	out, err := cmd.Output()
+	if err != nil {
+		stderr := ""
+		if exitErr, ok := err.(*exec.ExitError); ok {
+			stderr = string(exitErr.Stderr)
+		}
+		return nil, pkgerrors.New(string(out) + "\n" + stderr + "\n" + err.Error())
+	}
+
+	return dirToBytesMap(fs, filepath.Join(outDir, string(l)))
+}
+
 func (l Language) emitSDK(pkg *pschema.Package, info tfbridge.ProviderInfo, root afero.Fs,
-	loader pschema.ReferenceLoader,
 ) (map[string][]byte, error) {
 	var extraFiles map[string][]byte
 	var err error
@@ -135,7 +244,7 @@ func (l Language) emitSDK(pkg *pschema.Package, info tfbridge.ProviderInfo, root
 			return nil, err
 		}
 
-		m, err := gogen.GeneratePackage(tfgen, pkg, nil)
+		m, err := runPulumiPackageGenSDK(l, pkg, extraFiles)
 		if err != nil {
 			return nil, err
 		}
@@ -167,7 +276,7 @@ func (l Language) emitSDK(pkg *pschema.Package, info tfbridge.ProviderInfo, root
 		if err != nil && !os.IsNotExist(err) {
 			return nil, err
 		}
-		return nodejsgen.GeneratePackage(tfgen, pkg, extraFiles, nil, false, loader)
+		return runPulumiPackageGenSDK(l, pkg, extraFiles)
 	case Python:
 		if psi := info.Python; psi != nil && psi.Overlay != nil {
 			extraFiles, err = getOverlayFiles(psi.Overlay, ".py", root)
@@ -182,7 +291,7 @@ func (l Language) emitSDK(pkg *pschema.Package, info tfbridge.ProviderInfo, root
 		if err != nil && !os.IsNotExist(err) {
 			return nil, err
 		}
-		return pygen.GeneratePackage(tfgen, pkg, extraFiles, loader)
+		return runPulumiPackageGenSDK(l, pkg, extraFiles)
 	case CSharp:
 		if psi := info.CSharp; psi != nil && psi.Overlay != nil {
 			extraFiles, err = getOverlayFiles(psi.Overlay, ".cs", root)
@@ -194,7 +303,7 @@ func (l Language) emitSDK(pkg *pschema.Package, info tfbridge.ProviderInfo, root
 		if err != nil && !os.IsNotExist(err) {
 			return nil, err
 		}
-		return dotnetgen.GeneratePackage(tfgen, pkg, extraFiles, nil)
+		return runPulumiPackageGenSDK(l, pkg, extraFiles)
 	default:
 		return nil, fmt.Errorf("%v does not support SDK generation", l)
 	}
@@ -1049,8 +1158,7 @@ func (g *Generator) UnstableGenerateFromSchema(genSchemaResult *GenerateSchemaRe
 		if diags.HasErrors() {
 			return nil, err
 		}
-		loader := pschema.NewPluginLoader(g.pluginHost)
-		if files, err = g.language.emitSDK(pulumiPackage, g.info, g.root, loader); err != nil {
+		if files, err = g.language.emitSDK(pulumiPackage, g.info, g.root); err != nil {
 			return nil, pkgerrors.Wrapf(err, "failed to generate package")
 		}
 	}