Skip to content

Commit 11fee6e

Browse files
jackofallopsjackofallops
committed
add very basic thread tracing for parallelised imports
1 parent 2bc5517 commit 11fee6e

10 files changed

Lines changed: 33 additions & 84 deletions

File tree

‎tools/data-api-repository/repository/interface.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,5 +37,5 @@ type Repository interface {
3737
SaveCommonTypes(opts SaveCommonTypesOptions) error
3838

3939
// SaveService persists the API Definitions for the Service specified in opts.
40-
SaveService(opts SaveServiceOptions) error
40+
SaveService(opts SaveServiceOptions, threadId int) error
4141
}

‎tools/data-api-repository/repository/save_service.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,8 @@ type SaveServiceOptions struct {
3333
}
3434

3535
// SaveService persists the API Definitions for the Service specified in opts.
36-
func (r *repositoryImpl) SaveService(opts SaveServiceOptions) error {
37-
r.logger.Info(fmt.Sprintf("Processing Service %q", opts.ServiceName))
36+
func (r *repositoryImpl) SaveService(opts SaveServiceOptions, threadId int) error {
37+
r.logger.Info(fmt.Sprintf("ThreadID: %d - Processing Service %q", threadId, opts.ServiceName))
3838

3939
if opts.ServiceName == helpers.CommonTypesDirectoryName {
4040
return fmt.Errorf("`ServiceName` cannot be %q since that's reserved for storing Common Types", opts.ServiceName)
@@ -50,7 +50,7 @@ func (r *repositoryImpl) SaveService(opts SaveServiceOptions) error {
5050
}
5151

5252
for apiVersion, apiVersionDetails := range opts.Service.APIVersions {
53-
r.logger.Info(fmt.Sprintf("Processing Service %q / API Version %q..", opts.ServiceName, apiVersion))
53+
r.logger.Info(fmt.Sprintf("ThreadID: %d - Processing Service %q / API Version %q", threadId, opts.ServiceName, apiVersion))
5454
items = append(items, stages.APIVersionStage{
5555
APIVersion: apiVersionDetails,
5656
})

‎tools/importer-msgraph-metadata/internal/pipeline/task_persist_service.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ func (p pipelineForService) persistApiDefinitions(sdkService sdkModels.Service,
2323
SourceDataOrigin: sdkModels.MicrosoftGraphMetaDataSourceDataOrigin,
2424
}
2525

26-
if err := p.repo.SaveService(opts); err != nil {
26+
if err := p.repo.SaveService(opts, 0); err != nil {
2727
return fmt.Errorf("persisting Data API Definitions for Service %q: %+v", sdkService.Name, err)
2828
}
2929

‎tools/importer-rest-api-specs/go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ require (
1717
github.com/hashicorp/pandora/tools/sdk v0.0.0-00010101000000-000000000000
1818
github.com/mitchellh/cli v1.1.4
1919
github.com/zclconf/go-cty v1.13.1
20+
golang.org/x/sync v0.18.0
2021
golang.org/x/text v0.31.0
2122
)
2223

@@ -71,7 +72,6 @@ require (
7172
golang.org/x/crypto v0.45.0 // indirect
7273
golang.org/x/mod v0.29.0 // indirect
7374
golang.org/x/net v0.47.0 // indirect
74-
golang.org/x/sync v0.18.0 // indirect
7575
golang.org/x/sys v0.38.0 // indirect
7676
golang.org/x/tools v0.38.0 // indirect
7777
gopkg.in/warnings.v0 v0.1.2 // indirect

‎tools/importer-rest-api-specs/internal/components/apidefinitions/parse_service.go‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -12,28 +12,28 @@ import (
1212
"github.com/hashicorp/pandora/tools/importer-rest-api-specs/internal/logging"
1313
)
1414

15-
func ParseService(input discoveryModels.AvailableDataSet) (*sdkModels.Service, error) {
15+
func ParseService(input discoveryModels.AvailableDataSet, threadId int) (*sdkModels.Service, error) {
1616
// Some Services have been deprecated or should otherwise be ignored - check before proceeding
1717
if ignore.Services(input.ServiceName) {
18-
logging.Debugf("Service %q should be ignored - skipping", input.ServiceName)
18+
logging.Debugf("ThreadID: %d - Service %q should be ignored - skipping", threadId, input.ServiceName)
1919
return nil, nil
2020
}
2121

22-
logging.Infof("Parsing Data for Service %q..", input.ServiceName)
22+
logging.Infof("ThreadID: %d - Parsing Data for Service %q..", threadId, input.ServiceName)
2323
apiVersions := make(map[string]sdkModels.APIVersion)
2424

2525
for apiVersionName, dataSet := range input.DataSetsForAPIVersions {
26-
logging.Infof("Parsing Data for %q API Version %q..", input.ServiceName, apiVersionName)
26+
logging.Infof("ThreadID: %d - Parsing Data for %q API Version %q..", threadId, input.ServiceName, apiVersionName)
2727
parsed, err := parseAPIVersion(input.ServiceName, dataSet, input.ResourceProvider)
2828
if err != nil {
2929
return nil, fmt.Errorf("parsing %q API Version %q: %+v", input.ServiceName, apiVersionName, err)
3030
}
3131

3232
apiVersions[apiVersionName] = *parsed
33-
logging.Infof("Parsing Data for API Version %q - Completed", apiVersionName)
33+
logging.Infof("ThreadID: %d - Parsing Data for API Version %q - Completed", threadId, apiVersionName)
3434
}
3535

36-
logging.Infof("Parsing Data for Service %q - Completed", input.ServiceName)
36+
logging.Infof("ThreadID: %d - Parsing Data for Service %q - Completed", threadId, input.ServiceName)
3737
return &sdkModels.Service{
3838
APIVersions: apiVersions,
3939
Generate: true,

‎tools/importer-rest-api-specs/internal/components/apidefinitions/parser/dataworkarounds/workaround_network_ProvisioningState.go‎

Lines changed: 0 additions & 51 deletions
This file was deleted.

‎tools/importer-rest-api-specs/internal/components/apidefinitions/parser/testhelpers/parse_swagger_file.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ func ParseSwaggerFileForTesting(t *testing.T, filePath string, serviceName *stri
3434
}
3535

3636
func ParseDataSetForTesting(t *testing.T, input discoveryModels.AvailableDataSet, apiVersion string) (*sdkModels.APIVersion, error) {
37-
result, err := apidefinitions.ParseService(input)
37+
result, err := apidefinitions.ParseService(input, 0)
3838
if err != nil {
3939
t.Fatal(err.Error())
4040
}

‎tools/importer-rest-api-specs/internal/pipeline/import.go‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,10 @@ import (
88
"fmt"
99
"runtime"
1010

11-
"golang.org/x/sync/errgroup"
1211
"github.com/hashicorp/pandora/tools/data-api-repository/repository"
1312
"github.com/hashicorp/pandora/tools/importer-rest-api-specs/internal/components/terraform"
1413
"github.com/hashicorp/pandora/tools/importer-rest-api-specs/internal/logging"
14+
"golang.org/x/sync/errgroup"
1515
)
1616

1717
func RunImporter(opts Options) error {
@@ -54,7 +54,7 @@ func RunImporter(opts Options) error {
5454
g, ctx := errgroup.WithContext(context.Background())
5555
semaphore := make(chan struct{}, runtime.NumCPU())
5656

57-
for _, service := range p.servicesFromConfigurationFiles {
57+
for threadId, service := range p.servicesFromConfigurationFiles {
5858
service := service
5959

6060
if len(opts.ServiceNamesToLimitTo) > 0 {
@@ -66,7 +66,7 @@ func RunImporter(opts Options) error {
6666
}
6767
}
6868
if !processThisService {
69-
logging.Infof("Skipping the Service %q..", service.Name)
69+
logging.Infof("ThreadID: %d - Skipping the Service %q..", threadId, service.Name)
7070
continue
7171
}
7272
}
@@ -79,37 +79,37 @@ func RunImporter(opts Options) error {
7979
}
8080
defer func() { <-semaphore }()
8181

82-
logging.Infof("Discovering the Data for Service %q..", service.Name)
83-
data, err := p.parseDataForService(service)
82+
logging.Infof("ThreadID: %d - Discovering the Data for Service %q..", threadId, service.Name)
83+
data, err := p.parseDataForService(service, threadId)
8484
if err != nil {
8585
return fmt.Errorf("parsing Data for the Service %q: %+v", service.Name, err)
8686
}
87-
logging.Debugf("Completed - Discovering the Data for Service %q.", service.Name)
87+
logging.Debugf("ThreadID: %d - Completed - Discovering the Data for Service %q.", threadId, service.Name)
8888

8989
terraformDetails, ok := p.servicesToTerraformDetails[service.Name]
9090
if ok {
91-
logging.Infof("Building the Terraform Data for the Service %q..", service.Name)
91+
logging.Infof("ThreadID: %d - Building the Terraform Data for the Service %q..", threadId, service.Name)
9292
data, err = terraform.BuildForService(*data, terraformDetails.resourceLabelToResourceDefinitions, p.opts.ProviderPrefix, terraformDetails.terraformPackageName)
9393
if err != nil {
9494
return fmt.Errorf("building the Terraform Data for Service %q: %+v", service.Name, err)
9595
}
96-
logging.Debugf("Completed - Building the Terraform Data for the Service %q.", service.Name)
96+
logging.Debugf("ThreadID: %d - Completed - Building the Terraform Data for the Service %q.", threadId, service.Name)
9797
} else {
98-
logging.Debugf("Skipping - no Terraform Definitions for the Service %q..", service.Name)
98+
logging.Debugf("ThreadID: %d - Skipping - no Terraform Definitions for the Service %q..", threadId, service.Name)
9999
}
100100

101-
logging.Infof("Writing Data for Service %q..", service.Name)
101+
logging.Infof("ThreadID: %d - Writing Data for Service %q..", threadId, service.Name)
102102
saveServiceOpts := repository.SaveServiceOptions{
103103
ResourceProvider: service.ResourceProvider,
104104
Service: *data,
105105
ServiceName: service.Name,
106106
SourceCommitSHA: restAPISpecsCommitSHA,
107107
SourceDataOrigin: p.opts.SourceDataOrigin,
108108
}
109-
if err := p.repository.SaveService(saveServiceOpts); err != nil {
109+
if err := p.repository.SaveService(saveServiceOpts, threadId); err != nil {
110110
return fmt.Errorf("saving the Service %q: %+v", service.Name, err)
111111
}
112-
logging.Debugf("Completed - writing Data for Service %q.", service.Name)
112+
logging.Debugf("ThreadID: %d - Completed - writing Data for Service %q.", threadId, service.Name)
113113

114114
return nil
115115
})

‎tools/importer-rest-api-specs/internal/pipeline/stage_parse_data_for_service.go‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -13,21 +13,21 @@ import (
1313
"github.com/hashicorp/pandora/tools/sdk/config/services"
1414
)
1515

16-
func (p *Pipeline) parseDataForService(input services.Service) (*sdkModels.Service, error) {
17-
logging.Debugf("Discovering Data for Service %q in %q..", input.Name, p.opts.RestAPISpecsDirectory)
16+
func (p *Pipeline) parseDataForService(input services.Service, threadId int) (*sdkModels.Service, error) {
17+
logging.Debugf("ThreadID: %d - Discovering Data for Service %q in %q..", threadId, input.Name, p.opts.RestAPISpecsDirectory)
1818
skipDataPlane := p.opts.SourceDataType != sdkModels.DataPlaneSourceDataType
1919
data, err := discovery.DiscoverForService(input, p.opts.RestAPISpecsDirectory, skipDataPlane)
2020
if err != nil {
2121
return nil, fmt.Errorf("discovering for Service %q: %+v", input.Name, err)
2222
}
23-
logging.Tracef("Resource Provider is %q for the Service %q", *data.ResourceProvider, input.Name)
24-
logging.Debugf("Discovering Data for Service %q in %q - Completed", input.Name, p.opts.RestAPISpecsDirectory)
23+
logging.Tracef("ThreadID: %d - Resource Provider is %q for the Service %q", threadId, *data.ResourceProvider, input.Name)
24+
logging.Debugf("ThreadID: %d - Discovering Data for Service %q in %q - Completed", threadId, input.Name, p.opts.RestAPISpecsDirectory)
2525

26-
logging.Debugf("Parsing Data for Service %q..", input.Name)
27-
service, err := apidefinitions.ParseService(*data)
26+
logging.Debugf("ThreadID: %d - Parsing Data for Service %q..", threadId, input.Name)
27+
service, err := apidefinitions.ParseService(*data, threadId)
2828
if err != nil {
2929
return nil, fmt.Errorf("parsing Data for Service %q: %+v", input.Name, err)
3030
}
31-
logging.Debugf("Parsing Data for Service %q - Completed", input.Name)
31+
logging.Debugf("ThreadID: %d - Parsing Data for Service %q - Completed", threadId, input.Name)
3232
return service, nil
3333
}

‎tools/importer-rest-api-specs/internal/pipeline/validate.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ func RunValidate(opts Options) error {
3535
serviceNamesToResults := make(map[string]validationResult)
3636
for _, service := range p.servicesFromConfigurationFiles {
3737
logging.Infof("Parsing the Data for Service %q..", service.Name)
38-
data, err := p.parseDataForService(service)
38+
data, err := p.parseDataForService(service, 0)
3939
if err != nil {
4040
serviceNamesToResults[service.Name] = validationResult{
4141
succeeded: false,

0 commit comments

Comments
 (0)