Skip to content

Commit 2142f47

Browse files
stephybunstephybun
authored
Data API V2: extending NewServicesRepository to build a complete list of services (#3392)
* auto discover api definition directories with metadata.json * handle and return errors in NewServiceRepository * return error when processing service definitions * use log.Fatal, refactor mappings into a separate file and add discovery functions for finding service type directories and services * load services from cache in GetAll and GetByName, call discovery functions to populate all available services and their file paths * fix nit
1 parent c2d93e2 commit 2142f47

File tree

6 files changed

+406
-197
lines changed

6 files changed

+406
-197
lines changed

‎tools/data-api/internal/endpoints/routing.go

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
package endpoints
22

33
import (
4+
"log"
5+
46
"github.com/go-chi/chi/v5"
57
"github.com/hashicorp/pandora/tools/data-api/internal/endpoints/infrastructure"
68
"github.com/hashicorp/pandora/tools/data-api/internal/endpoints/v1"
@@ -16,7 +18,11 @@ func Router(directory string, serviceNames *[]string) func(chi.Router) {
1618
UriPrefix: "/v1/microsoft-graph/beta",
1719
UsesCommonTypes: true,
1820
}
19-
serviceRepo := repositories.NewServicesRepository(directory, opts.ServiceType, serviceNames)
21+
serviceRepo, err := repositories.NewServicesRepository(directory, opts.ServiceType, serviceNames)
22+
if err != nil {
23+
// TODO logging
24+
log.Fatalf("Error: %+v", err)
25+
}
2026
v1.Router(r, opts, serviceRepo)
2127
})
2228
router.Route("/v1/microsoft-graph/stable-v1", func(r chi.Router) {
@@ -25,7 +31,11 @@ func Router(directory string, serviceNames *[]string) func(chi.Router) {
2531
UriPrefix: "/v1/microsoft-graph/stable-v1",
2632
UsesCommonTypes: true,
2733
}
28-
serviceRepo := repositories.NewServicesRepository(directory, opts.ServiceType, serviceNames)
34+
serviceRepo, err := repositories.NewServicesRepository(directory, opts.ServiceType, serviceNames)
35+
if err != nil {
36+
// TODO logging
37+
log.Fatalf("Error: %+v", err)
38+
}
2939
v1.Router(r, opts, serviceRepo)
3040
})
3141
router.Route("/v1/resource-manager", func(r chi.Router) {
@@ -34,7 +44,11 @@ func Router(directory string, serviceNames *[]string) func(chi.Router) {
3444
UriPrefix: "/v1/resource-manager",
3545
UsesCommonTypes: false,
3646
}
37-
serviceRepo := repositories.NewServicesRepository(directory, opts.ServiceType, serviceNames)
47+
serviceRepo, err := repositories.NewServicesRepository(directory, opts.ServiceType, serviceNames)
48+
if err != nil {
49+
// TODO logging
50+
log.Fatalf("Error: %+v", err)
51+
}
3852
v1.Router(r, opts, serviceRepo)
3953
})
4054
router.Get("/", HomePage(router))

‎tools/data-api/internal/repositories/discovery.go

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
package repositories
2+
3+
import (
4+
"encoding/json"
5+
"errors"
6+
"fmt"
7+
"os"
8+
"path"
9+
10+
"github.com/hashicorp/pandora/tools/sdk/dataapimodels"
11+
)
12+
13+
func (s *ServicesRepositoryImpl) discoverServiceTypeDirectories() (*[]string, error) {
14+
// discoverServiceTypeDirectories finds all directories under the root directory that contain api definitions for a given
15+
// service type by checking for a metadata.json and comparing the Data Source value defined within it to the Data Source
16+
// value we're expecting for the Services Repository
17+
dirs, err := listSubDirectories(s.rootDirectory)
18+
if err != nil {
19+
return nil, fmt.Errorf("listing directories under %q: %+v", s.rootDirectory, err)
20+
}
21+
22+
serviceTypeDirectories := make([]string, 0)
23+
24+
for _, d := range *dirs {
25+
serviceTypeDir := path.Join(s.rootDirectory, d)
26+
27+
// check whether directory contains a metadata.json
28+
var metadata dataapimodels.MetaData
29+
contents, err := loadJson(path.Join(serviceTypeDir, "metadata.json"))
30+
if err != nil {
31+
var pathError *os.PathError
32+
if errors.As(err, &pathError) {
33+
// this folder has no metadata.json, so we skip it
34+
continue
35+
}
36+
return nil, fmt.Errorf("loading metadata.json: %+v", err)
37+
}
38+
39+
if err := json.Unmarshal(*contents, &metadata); err != nil {
40+
return nil, fmt.Errorf("unmarshaling metadata.json: %+v", err)
41+
}
42+
43+
if metadata.DataSource != s.expectedDataSource {
44+
// this folder contains definitions not belonging to this service type, so we skip it
45+
continue
46+
}
47+
serviceTypeDirectories = append(serviceTypeDirectories, serviceTypeDir)
48+
}
49+
50+
return &serviceTypeDirectories, nil
51+
}
52+
53+
func (s *ServicesRepositoryImpl) discoverSubsetOfServices() error {
54+
// discoverSubsetOfServices populates the serviceNamesToDirectory attribute of the ServicesRepositoryImpl.
55+
// This function is called if we're spinning up the data API for a subset of services and avoids iterating over
56+
// all available services.
57+
dirs, err := s.discoverServiceTypeDirectories()
58+
if err != nil {
59+
return fmt.Errorf("discovering service type directories for service type %q: %+v", s.serviceType, err)
60+
}
61+
62+
services := make(map[string]string, 0)
63+
for _, d := range *dirs {
64+
for _, service := range *s.serviceNames {
65+
serviceDir := path.Join(d, service)
66+
if _, err := os.Stat(serviceDir); os.IsNotExist(err) {
67+
// we continue here since the service we're looking for could exist in another source directory e.g. under handwritten definitions
68+
continue
69+
}
70+
if _, ok := services[service]; ok {
71+
return fmt.Errorf("duplicate definitions for service %q", service)
72+
}
73+
services[service] = serviceDir
74+
}
75+
}
76+
77+
// this checks if all services have been found if we're running the data API for a subset
78+
for _, service := range *s.serviceNames {
79+
if _, ok := services[service]; !ok {
80+
return fmt.Errorf("service %q was not found", service)
81+
}
82+
}
83+
84+
s.serviceNamesToDirectory = &services
85+
86+
return nil
87+
}
88+
89+
func (s *ServicesRepositoryImpl) discoverAllServices() error {
90+
// discoverAllServices populates the serviceNamesToDirectory attribute of the ServicesRepositoryImpl.
91+
// It iterates through all available services to build a complete list of available services for a given
92+
// service type and checks if there are duplicate definitions for a service.
93+
dirs, err := s.discoverServiceTypeDirectories()
94+
if err != nil {
95+
return fmt.Errorf("discovering service type directories for service type %q: %+v", s.serviceType, err)
96+
}
97+
98+
allServices := make(map[string]string, 0)
99+
for _, d := range *dirs {
100+
files, err := os.ReadDir(d)
101+
if err != nil {
102+
return fmt.Errorf("getting all services: %+v", err)
103+
}
104+
105+
for _, f := range files {
106+
if f.IsDir() {
107+
if _, ok := allServices[f.Name()]; ok {
108+
return fmt.Errorf("duplicate definitions for service %q", f.Name())
109+
}
110+
allServices[f.Name()] = path.Join(d, f.Name())
111+
}
112+
}
113+
}
114+
115+
s.serviceNamesToDirectory = &allServices
116+
117+
return nil
118+
}

‎tools/data-api/internal/repositories/mappings.go

Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
package repositories
2+
3+
import (
4+
"fmt"
5+
6+
"github.com/hashicorp/go-azure-helpers/lang/pointer"
7+
"github.com/hashicorp/pandora/tools/sdk/dataapimodels"
8+
)
9+
10+
func mapObjectDefinition(input *dataapimodels.ObjectDefinition) (*ObjectDefinition, error) {
11+
if input == nil {
12+
return nil, nil
13+
}
14+
15+
objectDefinitionType, err := mapObjectDefinitionType(input.Type)
16+
if err != nil {
17+
return nil, err
18+
}
19+
20+
output := ObjectDefinition{
21+
ReferenceName: input.ReferenceName,
22+
Type: pointer.From(objectDefinitionType),
23+
}
24+
25+
if input.NestedItem != nil {
26+
nestedItem, err := mapObjectDefinition(input.NestedItem)
27+
if err != nil {
28+
return nil, fmt.Errorf("mapping Nested Item for Object Definition: %+v", err)
29+
}
30+
output.NestedItem = nestedItem
31+
}
32+
33+
return &output, nil
34+
}
35+
36+
func mapOptionObjectDefinition(input *dataapimodels.OptionObjectDefinition, constants map[string]ConstantDetails, apiModels map[string]ModelDetails) (*OptionObjectDefinition, error) {
37+
optionObjectType, err := mapOptionObjectDefinitionType(input.Type)
38+
if err != nil {
39+
return nil, err
40+
}
41+
42+
output := OptionObjectDefinition{
43+
ReferenceName: input.ReferenceName,
44+
Type: pointer.From(optionObjectType),
45+
}
46+
47+
if input.NestedItem != nil {
48+
nestedItem, err := mapOptionObjectDefinition(input.NestedItem, constants, apiModels)
49+
if err != nil {
50+
return nil, fmt.Errorf("mapping Nested Item for Option Object Definition: %+v", err)
51+
}
52+
output.NestedItem = nestedItem
53+
}
54+
55+
if err := validateOptionObjectDefinition(output, constants, apiModels); err != nil {
56+
return nil, fmt.Errorf("validating mapped Option Object Definition: %+v", err)
57+
}
58+
59+
return &output, nil
60+
}
61+
62+
func mapDateFormatType(input dataapimodels.DateFormat) (*DateFormat, error) {
63+
mappings := map[dataapimodels.DateFormat]DateFormat{
64+
dataapimodels.RFC3339DateFormat: RFC3339DateFormat,
65+
}
66+
if v, ok := mappings[input]; ok {
67+
return &v, nil
68+
}
69+
70+
return nil, fmt.Errorf("unmapped Date Format Type %q", string(input))
71+
}
72+
73+
func mapObjectDefinitionType(input dataapimodels.ObjectDefinitionType) (*ObjectDefinitionType, error) {
74+
mappings := map[dataapimodels.ObjectDefinitionType]ObjectDefinitionType{
75+
dataapimodels.BooleanObjectDefinitionType: BooleanObjectDefinitionType,
76+
dataapimodels.DateTimeObjectDefinitionType: DateTimeObjectDefinitionType,
77+
dataapimodels.IntegerObjectDefinitionType: IntegerObjectDefinitionType,
78+
dataapimodels.FloatObjectDefinitionType: FloatObjectDefinitionType,
79+
dataapimodels.RawFileObjectDefinitionType: RawFileObjectDefinitionType,
80+
dataapimodels.RawObjectObjectDefinitionType: RawObjectObjectDefinitionType,
81+
dataapimodels.ReferenceObjectDefinitionType: ReferenceObjectDefinitionType,
82+
dataapimodels.StringObjectDefinitionType: StringObjectDefinitionType,
83+
dataapimodels.CsvObjectDefinitionType: CsvObjectDefinitionType,
84+
dataapimodels.DictionaryObjectDefinitionType: DictionaryObjectDefinitionType,
85+
dataapimodels.ListObjectDefinitionType: ListObjectDefinitionType,
86+
87+
dataapimodels.EdgeZoneObjectDefinitionType: EdgeZoneObjectDefinitionType,
88+
dataapimodels.LocationObjectDefinitionType: LocationObjectDefinitionType,
89+
dataapimodels.TagsObjectDefinitionType: TagsObjectDefinitionType,
90+
dataapimodels.SystemAssignedIdentityObjectDefinitionType: SystemAssignedIdentityObjectDefinitionType,
91+
dataapimodels.SystemAndUserAssignedIdentityListObjectDefinitionType: SystemAndUserAssignedIdentityListObjectDefinitionType,
92+
dataapimodels.SystemAndUserAssignedIdentityMapObjectDefinitionType: SystemAndUserAssignedIdentityMapObjectDefinitionType,
93+
dataapimodels.LegacySystemAndUserAssignedIdentityListObjectDefinitionType: LegacySystemAndUserAssignedIdentityListObjectDefinitionType,
94+
dataapimodels.LegacySystemAndUserAssignedIdentityMapObjectDefinitionType: LegacySystemAndUserAssignedIdentityMapObjectDefinitionType,
95+
dataapimodels.SystemOrUserAssignedIdentityListObjectDefinitionType: SystemOrUserAssignedIdentityListObjectDefinitionType,
96+
dataapimodels.SystemOrUserAssignedIdentityMapObjectDefinitionType: SystemOrUserAssignedIdentityMapObjectDefinitionType,
97+
dataapimodels.UserAssignedIdentityListObjectDefinitionType: UserAssignedIdentityListObjectDefinitionType,
98+
dataapimodels.UserAssignedIdentityMapObjectDefinitionType: UserAssignedIdentityMapObjectDefinitionType,
99+
dataapimodels.SystemDataObjectDefinitionType: SystemDataObjectDefinitionType,
100+
dataapimodels.ZoneObjectDefinitionType: ZoneObjectDefinitionType,
101+
dataapimodels.ZonesObjectDefinitionType: ZonesObjectDefinitionType,
102+
}
103+
if v, ok := mappings[input]; ok {
104+
return &v, nil
105+
}
106+
107+
return nil, fmt.Errorf("unmapped Object Definition Type %q", string(input))
108+
}
109+
110+
func mapOptionObjectDefinitionType(input dataapimodels.OptionObjectDefinitionType) (*OptionObjectDefinitionType, error) {
111+
mappings := map[dataapimodels.OptionObjectDefinitionType]OptionObjectDefinitionType{
112+
dataapimodels.BooleanOptionObjectDefinitionType: BooleanOptionObjectDefinition,
113+
dataapimodels.IntegerOptionObjectDefinitionType: IntegerOptionObjectDefinition,
114+
dataapimodels.FloatOptionObjectDefinitionType: FloatOptionObjectDefinitionType,
115+
dataapimodels.StringOptionObjectDefinitionType: StringOptionObjectDefinitionType,
116+
dataapimodels.CsvOptionObjectDefinitionType: CsvOptionObjectDefinitionType,
117+
dataapimodels.ListOptionObjectDefinitionType: ListOptionObjectDefinitionType,
118+
dataapimodels.ReferenceOptionObjectDefinitionType: ReferenceOptionObjectDefinitionType,
119+
}
120+
if v, ok := mappings[input]; ok {
121+
return &v, nil
122+
}
123+
124+
return nil, fmt.Errorf("unmapped Options Object Definition Type %q", string(input))
125+
}
126+
127+
func mapConstantFieldType(input dataapimodels.ConstantType) (*ConstantType, error) {
128+
mappings := map[dataapimodels.ConstantType]ConstantType{
129+
dataapimodels.FloatConstant: FloatConstant,
130+
dataapimodels.IntegerConstant: IntegerConstant,
131+
dataapimodels.StringConstant: StringConstant,
132+
}
133+
if v, ok := mappings[input]; ok {
134+
return &v, nil
135+
}
136+
137+
return nil, fmt.Errorf("unmapped Constant Type %q", string(input))
138+
}
139+
140+
func mapResourceIdSegmentType(input dataapimodels.ResourceIdSegmentType) (*ResourceIdSegmentType, error) {
141+
mappings := map[dataapimodels.ResourceIdSegmentType]ResourceIdSegmentType{
142+
dataapimodels.ConstantResourceIdSegmentType: ConstantResourceIdSegmentType,
143+
dataapimodels.ResourceGroupResourceIdSegmentType: ResourceGroupResourceIdSegmentType,
144+
dataapimodels.ResourceProviderResourceIdSegmentType: ResourceProviderResourceIdSegmentType,
145+
dataapimodels.ScopeResourceIdSegmentType: ScopeResourceIdSegmentType,
146+
dataapimodels.StaticResourceIdSegmentType: StaticResourceIdSegmentType,
147+
dataapimodels.SubscriptionIdResourceIdSegmentType: SubscriptionIdResourceIdSegmentType,
148+
dataapimodels.UserSpecifiedResourceIdSegmentType: UserSpecifiedResourceIdSegmentType,
149+
}
150+
if v, ok := mappings[input]; ok {
151+
return &v, nil
152+
}
153+
154+
return nil, fmt.Errorf("unmapped Resource Id Segment Type %q", string(input))
155+
}

‎tools/data-api/internal/repositories/models.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ type ServiceDetails struct {
7575
Name string
7676
ApiVersions map[string]*ServiceApiVersionDetails
7777
Generate bool
78-
ResourceProvider string
78+
ResourceProvider *string
7979
TerraformPackageName *string
8080
TerraformDetails TerraformDetails
8181
}

0 commit comments

Comments
 (0)