importer-data-plane - add first pass Data Plane importing

.github/workflows/unit-test-end-to-end-data-plane.yaml

@@ -0,0 +1,42 @@
+---
+name: Verify All Resource Manager Services can be Imported and Generated
+on:
+  pull_request:
+    types: ['opened', 'synchronize']
+    paths:
+      - '.github/workflows/unit-test-end-to-end-data-plane.yaml'
+      - 'config/data-plane.hcl'
+      - 'tools/generator-go-sdk/**'
+      - 'tools/importer-rest-api-specs/**'
+
+jobs:
+  test:
+    runs-on: custom-linux-xl
+    steps:
+      - name: mount a ramdisk to working dir
+        run: |
+          sudo mount -t tmpfs -o size=32G tmpfs $GITHUB_WORKSPACE
+
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+        with:
+          submodules: recursive
+
+      - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0
+        with:
+          go-version-file: ./.go-version
+
+      - name: Install tools
+        run: |
+          go install golang.org/x/tools/cmd/goimports@latest
+
+      - name: "Build and Run importer-rest-api-specs"
+        id: import-data
+        run: |
+          cd ./tools/importer-rest-api-specs
+          make tools
+          make build
+          make import-data-plane
+# Uncomment after the first data-plane files are present in go-azure-sdk
+#      - name: "Run the Go SDK Generator"
+#        run: |
+#          ./scripts/automation-generate-go-sdk.sh data-plane

.github/workflows/unit-test-rest-api-specs-importer.yaml

@@ -30,6 +30,3 @@ jobs:
         run: |
           cd ./tools/importer-rest-api-specs
           make test
-
-.github/workflows/unit-test-rest-api-specs-importer:
-  yaml:

config/data-plane.hcl

@@ -0,0 +1,37 @@
+# Copyright IBM Corp. 2021, 2025
+# SPDX-License-Identifier: MPL-2.0
+
+service "appconfiguration" {
+  name      = "AppConfiguration"
+  available = ["1.0"]
+}
+
+service "attestation" {
+  name      = "Attestation"
+  available = ["2020-10-01", "2022-08-01"]
+}
+
+service "batch" {
+  name = "Batch"
+  available = ["2022-01-01.15.0", "2020-03-01.11.0"]
+}
+
+service "iotcentral"  {
+  name = "IoTCentral"
+  available = ["2022-10-31-preview"]
+}
+
+service "keyvault" {
+  name = "KeyVault"
+  available = ["7.4", "7.5-preview.1", "2025-07-01"]
+}
+
+service "search" {
+  name      = "Search"
+  available = ["2025-09-01"]
+}
+
+service "synapse" {
+  name = "Synapse"
+  available = ["2019-06-01-preview", "2020-08-01-preview", "2021-06-01-preview"]
+}

scripts/automation-generate-and-commit-go-sdk.sh

@@ -11,12 +11,12 @@ function buildAndInstallDependencies {
   echo "Outputting Go Version.."
   go version
 
-  echo "Installing the Data API V2 onto the GOBIN.."
+  echo "Installing the Data API V2 onto the GOBIN"
   cd "${DIR}/tools/data-api"
   go install .
   cd "${DIR}"
 
-  echo "Installing the Go SDK Generator into the GOBIN.."
+  echo "Installing the Go SDK Generator into the GOBIN"
   cd "${DIR}/tools/generator-go-sdk"
   go install .
   cd "${DIR}"
@@ -31,40 +31,43 @@ function runWrapper {
   local apiDefinitionsDirectory=$1
   local outputDirectory=$2
 
-  echo "Running Wrapper for Resource Manager.."
+  echo "Running Wrapper for Resource Manager"
   cd "${DIR}/tools/wrapper-automation"
   ./wrapper-automation resource-manager go-sdk \
     --api-definitions-dir="../../$apiDefinitionsDirectory"\
     --output-dir="../../$outputDirectory"
 
-  echo "Running Wrapper for Microsoft Graph.."
+  echo "Running Wrapper for Data Plane"
+  cd "${DIR}/tools/wrapper-automation"
+  ./wrapper-automation microsoft-graph go-sdk \
+    --api-definitions-dir="../../$apiDefinitionsDirectory"\
+    --output-dir="../../$outputDirectory"
+
+  echo "Running Wrapper for Microsoft Graph"
   cd "${DIR}/tools/wrapper-automation"
   ./wrapper-automation microsoft-graph go-sdk \
     --api-definitions-dir="../../$apiDefinitionsDirectory"\
     --output-dir="../../$outputDirectory"
 
   cd "${DIR}"
 
-  echo "Running 'make tools' within the SDK codebase.."
+  echo "Running 'make tools' within the SDK codebase"
   cd "${outputDirectory}"
   make tools
 
-  echo "Running 'make fmt' on the generated code.."
-  make fmt
-
-  echo "Running 'make imports' on the generated code.."
+  echo "Running 'make imports' on the generated code"
   make imports
 }
 
 function prepareGoSdk {
   local workingDirectory=$1
   local sdkRepo=$2
 
-  echo "Removing any existing working directory.."
+  echo "Removing any existing working directory"
   cd "${DIR}"
   rm -rf "$workingDirectory"
 
-  echo "Cloning SDK Repository into $workingDirectory.."
+  echo "Cloning SDK Repository into $workingDirectory"
   git clone "$sdkRepo" "$workingDirectory"
 
   echo "Preparing the repository for generation"

scripts/automation-generate-go-sdk.sh

@@ -12,6 +12,7 @@ usage() {
   echo "Usage: automation-generate-go-sdk.sh sdkToGenerate" >&2
   echo "" >&2
   echo "sdkToGenerate is required and should be one of:" >&2
+  echo "  data-plane" >&2
   echo "  microsoft-graph" >&2
   echo "  resource-manager" >&2
   echo "" >&2

tools/data-api-repository/repository/directory_for_service.go

@@ -54,4 +54,7 @@ var defaultDataDirectories = map[sdkModels.SourceDataType]map[sdkModels.SourceDa
 		sdkModels.AzureRestAPISpecsSourceDataOrigin: "resource-manager",
 		sdkModels.HandWrittenSourceDataOrigin:       "handwritten-resource-manager",
 	},
+	sdkModels.DataPlaneSourceDataType: {
+		sdkModels.AzureRestAPISpecsSourceDataOrigin: "data-plane",
+	},
 }

tools/data-api-repository/repository/internal/models/metadata.go

@@ -27,6 +27,9 @@ const (
 	// AzureResourceManagerDataSource specifies that this Data is related to Azure Resource Manager.
 	AzureResourceManagerDataSource DataSource = "AzureResourceManager"
 
+	// AzureDataPlaneDataSource specifies that this Data is related to Azure Resource Manager.
+	AzureDataPlaneDataSource DataSource = "AzureDataPlane"
+
 	// MicrosoftGraphDataSource specifies that this Data is for Microsoft Graph.
 	MicrosoftGraphDataSource DataSource = "MicrosoftGraph"
 )

tools/data-api-repository/repository/internal/transforms/source_data_type.go

@@ -11,9 +11,11 @@ import (
 var sourceDataTypesFromRepository = map[repositoryModels.DataSource]sdkModels.SourceDataType{
 	repositoryModels.AzureResourceManagerDataSource: sdkModels.ResourceManagerSourceDataType,
 	repositoryModels.MicrosoftGraphDataSource:       sdkModels.MicrosoftGraphSourceDataType,
+	repositoryModels.AzureDataPlaneDataSource:       sdkModels.DataPlaneSourceDataType,
 }
 
 var sourceDataTypesToRepository = map[sdkModels.SourceDataType]repositoryModels.DataSource{
 	sdkModels.ResourceManagerSourceDataType: repositoryModels.AzureResourceManagerDataSource,
 	sdkModels.MicrosoftGraphSourceDataType:  repositoryModels.MicrosoftGraphDataSource,
+	sdkModels.DataPlaneSourceDataType:       repositoryModels.AzureDataPlaneDataSource,
 }

tools/data-api-sdk/v1/available_source_data_types.go

@@ -12,5 +12,6 @@ func AvailableSourceDataTypes() []models.SourceDataType {
 	return []models.SourceDataType{
 		models.MicrosoftGraphSourceDataType,
 		models.ResourceManagerSourceDataType,
+		models.DataPlaneSourceDataType,
 	}
 }

tools/data-api-sdk/v1/models/resource_id_segment_type.go

@@ -28,4 +28,6 @@ const (
 	// UserSpecifiedResourceIDSegmentType specifies that the ResourceIDSegment is a User-Specified/Provided value.
 	// Typically (but importantly not always) these represent the Name components for a given Resource.
 	UserSpecifiedResourceIDSegmentType ResourceIDSegmentType = "UserSpecified"
+
+	DataPlaneBaseURLResourceIDSegmentType ResourceIDSegmentType = "DataPlaneBaseUrl"
 )

tools/data-api-sdk/v1/models/source_data_types.go

@@ -12,6 +12,9 @@ const (
 
 	// ResourceManagerSourceDataType defines that this Data is related to Azure Resource Manager.
 	ResourceManagerSourceDataType SourceDataType = "resource-manager"
+
+	// DataPlaneSourceDataType defines that this Data is related to Azure Resource Manager.
+	DataPlaneSourceDataType SourceDataType = "data-plane"
 )
 
 func SourceDataTypeIsDataPlane(sourceDataType SourceDataType) bool {

tools/data-api/go.mod

@@ -38,8 +38,6 @@ require (
 	golang.org/x/sys v0.40.0 // indirect
 )
 
-replace github.com/hashicorp/pandora/tools/sdk => ../sdk
-
 replace github.com/hashicorp/pandora/tools/data-api-repository => ../data-api-repository
 
 replace github.com/hashicorp/pandora/tools/data-api-sdk => ../data-api-sdk

tools/data-api/internal/endpoints/routing.go

@@ -37,6 +37,17 @@ func Router(workingDirectory string, serviceNames *[]string) func(chi.Router) {
 			}
 			v1.Router(r, opts, serviceRepo)
 		})
+		router.Route("/v1/data-plane/", func(r chi.Router) {
+			opts := v1.Options{
+				ServiceType: sdkModels.DataPlaneSourceDataType,
+				UriPrefix:   "/v1/data-plane",
+			}
+			serviceRepo, err := repository.NewRepository(workingDirectory, opts.ServiceType, serviceNames, logging.Log)
+			if err != nil {
+				logging.Fatalf("Error: %+v", err)
+			}
+			v1.Router(r, opts, serviceRepo)
+		})
 		router.Get("/", HomePage(router))
 	}
 }

tools/generator-go-sdk/GNUmakefile

@@ -1,19 +1,19 @@
 default: build
 
-build:
+build: tools
 	go build .
 
 fmt:
 	find . -name '*.go' | grep -v vendor | xargs gofmt -s -w
 
 run: build
-	./generator-go-sdk resource-manager generate
+	@./generator-go-sdk resource-manager generate
 
 test: build
 	go test -v ./...
 
 tools:
-	@echo "==> no tools required at this time."
+	go install golang.org/x/tools/cmd/goimports@latest
 
 .PHONY: build fmt run test tools
 

tools/generator-go-sdk/internal/cmd/generate.go

@@ -58,44 +58,57 @@ func (g GenerateCommand) Run(args []string) int {
 		},
 	}
 
-	if g.sourceDataType == models.MicrosoftGraphSourceDataType {
-		input.settings.AllowOmittingDiscriminatedValue = true
-		input.settings.DeleteExistingResourcesForVersion = true
-		input.settings.GenerateDescriptionsForModels = true
-		input.settings.RecurseParentModels = false
-
-		input.settings.CanonicalApiVersions = map[string]string{
-			"stable": "v1.0",
-		}
+	switch {
+	case g.sourceDataType == models.MicrosoftGraphSourceDataType:
+		{
+			input.settings.AllowOmittingDiscriminatedValue = true
+			input.settings.DeleteExistingResourcesForVersion = true
+			input.settings.GenerateDescriptionsForModels = true
+			input.settings.RecurseParentModels = false
+
+			input.settings.CanonicalApiVersions = map[string]string{
+				"stable": "v1.0",
+			}
 
-		input.settings.VersionsToGenerateCommonTypes = map[string]models.SourceDataOrigin{
-			"stable": models.MicrosoftGraphMetaDataSourceDataOrigin,
-			"beta":   models.MicrosoftGraphMetaDataSourceDataOrigin,
+			input.settings.VersionsToGenerateCommonTypes = map[string]models.SourceDataOrigin{
+				"stable": models.MicrosoftGraphMetaDataSourceDataOrigin,
+				"beta":   models.MicrosoftGraphMetaDataSourceDataOrigin,
+			}
+		}
+	case g.sourceDataType == models.ResourceManagerSourceDataType:
+		{
+			input.settings.AllowOmittingDiscriminatedValue = false
+			input.settings.DeleteExistingResourcesForVersion = false
+			input.settings.GenerateDescriptionsForModels = false
+			input.settings.RecurseParentModels = true
+
+			input.settings.UseOldBaseLayerFor(
+				// @tombuildsstuff: New Services should now use the `hashicorp/go-azure-sdk` base layer by default
+				// instead of the base layer from `Azure/go-autorest` - as such this list is for compatibility purposes
+				// with services already used in `terraform-provider-azurerm`. These services will be gradually removed
+				// from this list to ensure they're migrated across to using `hashicorp/go-azure-sdk`s base layer.
+
+				"FrontDoor",
+				"RecoveryServicesBackup", // error: generating Service "RecoveryServicesBackup" / Version "2023-04-01" / Resource "Operation": generating methods: templating methods (using hashicorp/go-azure-sdk): templating: building methods: building response struct template: existing model "ValidateOperationResponse" conflicts with the operation response model for "Validate"
+				"Subscription",
+
+				// @tombuildsstuff: The Key Vault API has an issue where it requires that the EXACT casing returned in the Response
+				// is sent in the Request to update or remove a Key Vault Access Policy - and using other casings mean the update
+				// or removal fails - which is tracked in https://github.com/hashicorp/pandora/issues/3229.
+				//
+				// After testing, it appears that `2023-07-01` doesn't suffer from this problem - as such we're going to leave
+				// `2023-02-01` on the older base layer and use the newer API Version as a divide to give us a clear migration path.
+				"KeyVault@2023-02-01",
+			)
+		}
+	case g.sourceDataType == models.DataPlaneSourceDataType:
+		{
+			input.settings.AllowOmittingDiscriminatedValue = false
+			input.settings.DeleteExistingResourcesForVersion = false
+			input.settings.GenerateDescriptionsForModels = false
+			input.settings.RecurseParentModels = true
+			// input.settings.UseOldBaseLayerFor("KeyVault")
 		}
-	} else if g.sourceDataType == models.ResourceManagerSourceDataType {
-		input.settings.AllowOmittingDiscriminatedValue = false
-		input.settings.DeleteExistingResourcesForVersion = false
-		input.settings.GenerateDescriptionsForModels = false
-		input.settings.RecurseParentModels = true
-
-		input.settings.UseOldBaseLayerFor(
-			// @tombuildsstuff: New Services should now use the `hashicorp/go-azure-sdk` base layer by default
-			// instead of the base layer from `Azure/go-autorest` - as such this list is for compatibility purposes
-			// with services already used in `terraform-provider-azurerm`. These services will be gradually removed
-			// from this list to ensure they're migrated across to using `hashicorp/go-azure-sdk`s base layer.
-
-			"FrontDoor",
-			"RecoveryServicesBackup", // error: generating Service "RecoveryServicesBackup" / Version "2023-04-01" / Resource "Operation": generating methods: templating methods (using hashicorp/go-azure-sdk): templating: building methods: building response struct template: existing model "ValidateOperationResponse" conflicts with the operation response model for "Validate"
-			"Subscription",
-
-			// @tombuildsstuff: The Key Vault API has an issue where it requires that the EXACT casing returned in the Response
-			// is sent in the Request to update or remove a Key Vault Access Policy - and using other casings mean the update
-			// or removal fails - which is tracked in https://github.com/hashicorp/pandora/issues/3229.
-			//
-			// After testing, it appears that `2023-07-01` doesn't suffer from this problem - as such we're going to leave
-			// `2023-02-01` on the older base layer and use the newer API Version as a divide to give us a clear migration path.
-			"KeyVault@2023-02-01",
-		)
 	}
 
 	var serviceNames string

tools/generator-go-sdk/internal/generator/data.go

@@ -96,10 +96,11 @@ func (i ServiceGeneratorInput) generatorData(settings Settings) GeneratorData {
 	servicePackageName := strings.ToLower(i.ServiceName)
 	versionDirectoryName := strings.ToLower(i.VersionName)
 
-	versionOutputPath := filepath.Join(i.OutputDirectory, servicePackageName, versionDirectoryName)
+	versionOutputPath := filepath.Join(i.OutputDirectory, servicePackageName, strings.ReplaceAll(versionDirectoryName, ".", "-"))
 	resourceOutputPath := filepath.Join(versionOutputPath, resourcePackageName)
 
 	versionPackageName := strings.ToLower(strings.ReplaceAll(i.VersionName, "-", "_"))
+	versionPackageName = strings.ToLower(strings.ReplaceAll(versionPackageName, ".", "_"))
 	if regexp.MustCompile(`^[0-9]+`).MatchString(versionPackageName) {
 		versionPackageName = fmt.Sprintf("v%s", versionPackageName)
 	}
@@ -157,9 +158,10 @@ func (i VersionGeneratorInput) generatorData(settings Settings) VersionGenerator
 	versionDirectoryName := strings.ToLower(i.VersionName)
 
 	commonTypesOutputPath := filepath.Join(i.OutputDirectory, settings.CommonTypesPackageName, versionDirectoryName)
-	versionOutputPath := filepath.Join(i.OutputDirectory, servicePackageName, versionDirectoryName)
+	versionOutputPath := filepath.Join(i.OutputDirectory, servicePackageName, strings.ReplaceAll(versionDirectoryName, ".", "-"))
 
 	versionPackageName := strings.ToLower(strings.ReplaceAll(i.VersionName, "-", "_"))
+	versionPackageName = strings.ReplaceAll(versionPackageName, ".", "_")
 	if regexp.MustCompile(`^[0-9]+`).MatchString(versionPackageName) {
 		versionPackageName = fmt.Sprintf("v%s", versionPackageName)
 	}
@@ -183,7 +185,7 @@ func (i VersionGeneratorInput) generatorData(settings Settings) VersionGenerator
 			source:                          i.Source,
 			sourceType:                      i.Type,
 			useNewBaseLayer:                 useNewBaseLayer,
-			versionDirectoryName:            versionDirectoryName,
+			versionDirectoryName:            strings.ReplaceAll(versionDirectoryName, ".", "-"),
 			versionPackageName:              versionPackageName,
 		},
 		commonTypesOutputPath: commonTypesOutputPath,