diff --git a/bulk-import-tools/README.md b/bulk-import-tools/README.md index 4e1900765..7d6b3bea0 100644 --- a/bulk-import-tools/README.md +++ b/bulk-import-tools/README.md @@ -101,18 +101,29 @@ csv(`preflight_error_timestamp_filename`) to be generated with error messages co #### Bulk import tool ```bash - Import host data from input file into the Edge Orchestrator. +Import host data from input file into the Edge Orchestrator. - Usage: orch-host-bulk-import COMMAND +Usage: orch-host-bulk-import COMMAND + + +COMMANDS: + +import [OPTIONS] Import data from given CSV file to orchestrator URL + file Required source CSV file to read data from + url Required Edge Orchestrator URL +version Display version information +help Show this help message + +OPTIONS: + +--onboard If set, hosts will be automatically onboarded when connected +--project Optional project name in Edge Orchestrator. Alternatively, set env variable EDGEORCH_PROJECT +--os-profile Optional operating system profile name/id to configure for hosts. Alternatively, set env variable EDGEORCH_OSPROFILE +--site Optional site name/id to configure for hosts. Alternatively, set env variable EDGEORCH_SITE +--secure Optional security feature to configure for hosts. Alternatively, set env variable EDGEORCH_SECURE. Valid values: true, false +--remote-user Optional remote user name/id to configure for hosts. Alternatively, set env variable EDGEORCH_REMOTEUSER +--metadata Optional metadata to configure for hosts. Alternatively, set env variable EDGEORCH_METADATA. Metadata format: key=value&key=value - Commands: - import [--onboard] Import data from given CSV file to orchestrator URL - --onboard If set, hosts will be automatically onboarded when connected - file Required source CSV file to read data from - url Required Edge Orchestrator URL - project Optional project name in Edge Orchestrator. Alternatively, set env variable EDGEORCH_PROJECT - version Display version information - help Show this help message ``` Before running the bulk import tool, project name can be optionally set in envioronment variable or can be passed diff --git a/bulk-import-tools/VERSION b/bulk-import-tools/VERSION index ec802c662..26ca59460 100644 --- a/bulk-import-tools/VERSION +++ b/bulk-import-tools/VERSION @@ -1 +1 @@ -1.5.1-dev +1.5.1 diff --git a/bulk-import-tools/cmd/orch-host-bulk-import/orch_host_bulk_import.go b/bulk-import-tools/cmd/orch-host-bulk-import/orch_host_bulk_import.go index 288e47734..49a10a91d 100644 --- a/bulk-import-tools/cmd/orch-host-bulk-import/orch_host_bulk_import.go +++ b/bulk-import-tools/cmd/orch-host-bulk-import/orch_host_bulk_import.go @@ -50,11 +50,18 @@ func main() { func handleImportCommand() { importCmd := flag.NewFlagSet("import", flag.ExitOnError) - onboardFlag := importCmd.Bool("onboard", false, "Enable onboarding") + importCmd.Usage = displayHelp + onboardFlag := importCmd.Bool("onboard", false, "") + projectNameIn := importCmd.String("project", "", "") + osProfileIn := importCmd.String("os-profile", "", "") + siteIn := importCmd.String("site", "", "") + secureFlag := importCmd.String("secure", string(types.SecureUnspecified), "") + remoteUserIn := importCmd.String("remote-user", "", "") + metadataIn := importCmd.String("metadata", "", "") err := importCmd.Parse(os.Args[idxAfterFlags:]) // Check for the correct number of arguments after flags - if err != nil || importCmd.NArg() < numArgs { + if err != nil || importCmd.NArg() < numArgs-1 { fmt.Println("error: Filename & url required as arguments") displayHelp() os.Exit(1) @@ -62,12 +69,9 @@ func handleImportCommand() { filePath := importCmd.Arg(0) serverURL := importCmd.Arg(1) - //nolint:mnd // 2 is the index of the project name - projectName := importCmd.Arg(2) - //nolint:mnd // 3 is the index of the optional osprofile - osProfile := importCmd.Arg(3) // Check if project name is not provided, use the environment variable EDGEORCH_PROJECT + projectName := *projectNameIn if projectName == "" { projectName = os.Getenv("EDGEORCH_PROJECT") } @@ -79,38 +83,90 @@ func handleImportCommand() { } // Check if osprofile is not provided, use the environment variable EDGEORCH_OSPROFILE + osProfile := *osProfileIn if osProfile == "" { osProfile = os.Getenv("EDGEORCH_OSPROFILE") } + // Check if site is not provided, use the environment variable EDGEORCH_SITE + site := *siteIn + if site == "" { + site = os.Getenv("EDGEORCH_SITE") + } + + // Check if secure flag is not provided, use the environment variable EDGEORCH_SECURE + secure := getGlobalSecureAttr(secureFlag) + + // Check if remote user is not provided, use the environment variable EDGEORCH_REMOTEUSER + remoteUser := *remoteUserIn + if remoteUser == "" { + remoteUser = os.Getenv("EDGEORCH_REMOTEUSER") + } + + // Check if metadata is not provided, use the environment variable EDGEORCH_METADATA + metadata := *metadataIn + if metadata == "" { + metadata = os.Getenv("EDGEORCH_METADATA") + } + + globalAttr := &types.HostRecord{ + OSProfile: osProfile, + Site: site, + Secure: types.StringToRecordSecure(secure), + RemoteUser: remoteUser, + Metadata: metadata, + } + fmt.Printf("Importing hosts from file: %s to server: %s\n", filePath, serverURL) // Implement the import functionality here - if err := doImport(*onboardFlag, filePath, serverURL, projectName, osProfile); err != nil { + if err := doImport(*onboardFlag, filePath, serverURL, projectName, globalAttr); err != nil { fmt.Printf("error: %v\n\n", err.Error()) os.Exit(1) } fmt.Print("CSV import successful\n\n") } +func getGlobalSecureAttr(secureFlag *string) string { + secure := *secureFlag + if secure == "" { + secureEnv := os.Getenv("EDGEORCH_SECURE") + if secureEnv == "" { + secure = string(types.SecureUnspecified) + } else { + secure = secureEnv + } + } + return secure +} + // displayHelp prints the help information for the utility. func displayHelp() { - fmt.Print("Import host data from input file into the Edge Orchestrator.\n\n") + fmt.Print("\n\nImport host data from input file into the Edge Orchestrator.\n\n") fmt.Print("Usage: orch-host-bulk-import COMMAND\n\n") - fmt.Println("Commands:") - fmt.Println("\timport [--onboard] Import data from given CSV file to orchestrator URL") - fmt.Println("\t --onboard If set, hosts will be automatically onboarded when connected") - fmt.Println("\t file Required source CSV file to read data from") - fmt.Println("\t url Required Edge Orchestrator URL") - fmt.Println("\t project Optional project name in Edge Orchestrator.", + fmt.Print("COMMANDS:\n\n") + fmt.Println("import [OPTIONS] Import data from given CSV file to orchestrator URL") + fmt.Println(" file Required source CSV file to read data from") + fmt.Println(" url Required Edge Orchestrator URL") + fmt.Println("version Display version information") + fmt.Print("help Show this help message\n\n") + fmt.Print("OPTIONS:\n\n") + fmt.Println("--onboard If set, hosts will be automatically onboarded when connected") + fmt.Println("--project Optional project name in Edge Orchestrator.", "Alternatively, set env variable EDGEORCH_PROJECT") - fmt.Println("\t osprofile Optional operating system profile name/id to configure for hosts.", + fmt.Println("--os-profile Optional operating system profile name/id to configure for hosts.", "Alternatively, set env variable EDGEORCH_OSPROFILE") - fmt.Println("\tversion Display version information") - fmt.Print("\thelp Show this help message\n\n") + fmt.Println("--site Optional site name/id to configure for hosts.", + "Alternatively, set env variable EDGEORCH_SITE") + fmt.Println("--secure Optional security feature to configure for hosts.", + "Alternatively, set env variable EDGEORCH_SECURE. Valid values: true, false") + fmt.Println("--remote-user Optional remote user name/id to configure for hosts.", + "Alternatively, set env variable EDGEORCH_REMOTEUSER") + fmt.Print("--metadata Optional metadata to configure for hosts.", + "Alternatively, set env variable EDGEORCH_METADATA. Metadata format: key=value&key=value\n\n") } -func doImport(autoOnboard bool, filePath, serverURL, projectName, osProfile string) error { +func doImport(autoOnboard bool, filePath, serverURL, projectName string, globalAttr *types.HostRecord) error { ctx, cancel := context.WithCancelCause(context.Background()) defer cancel(nil) erringRecords := []types.HostRecord{} @@ -130,18 +186,10 @@ func doImport(autoOnboard bool, filePath, serverURL, projectName, osProfile stri return err } - var osProfileID string - // if osProfile is provided globally, verify if its valid or get - // the id from the server if profilename is provided - if osProfile != "" { - if osProfileID, err = oClient.GetOsProfileID(ctx, osProfile); err != nil { - return err - } - } // registerHost // iterate over all entries available for _, record := range validated { - doRegister(ctx, oClient, autoOnboard, osProfileID, record, &erringRecords) + doRegister(ctx, oClient, autoOnboard, globalAttr, record, &erringRecords) } // write import error to import_error__ // if there is any error record after header @@ -158,83 +206,152 @@ func doImport(autoOnboard bool, filePath, serverURL, projectName, osProfile stri } func doRegister(ctx context.Context, oClient *orchcli.OrchCli, autoOnboard bool, - osProfileID string, record types.HostRecord, erringRecords *[]types.HostRecord, + globalAttr *types.HostRecord, rIn types.HostRecord, erringRecords *[]types.HostRecord, ) { // get the required fields from the record - sNo := record.Serial - uuid := record.UUID + sNo := rIn.Serial + uuid := rIn.UUID - // try to register - hostID, err := oClient.RegisterHost(ctx, "", sNo, uuid, autoOnboard) + rOut, err := sanitizeProvisioningFields(ctx, oClient, rIn, erringRecords, globalAttr) if err != nil { + return + } + + // Register host + hostID, err := oClient.RegisterHost(ctx, "", sNo, uuid, autoOnboard) + if err != nil && !e.Is(e.ErrAlreadyRegistered, err) { // add to reject list if failed - record.Error = err.Error() - *erringRecords = append(*erringRecords, record) - } else { - // print host_id from response if successful - fmt.Printf("Host Serial number : %s UUID : %s registered. Name : %s\n", sNo, uuid, hostID) + rIn.Error = err.Error() + *erringRecords = append(*erringRecords, rIn) + return + } + // Create instance if osProfileID is available else append to error list + // Need not notify user of instance ID. Unnecessary detail for user. + _, err = oClient.CreateInstance(ctx, hostID, rOut) + if err != nil { + rIn.Error = err.Error() + *erringRecords = append(*erringRecords, rIn) + return + } - if err := createInstanceAndUpdateHost(ctx, oClient, record, erringRecords, osProfileID, hostID); err != nil { - return - } + if err := oClient.AllocateHostToSiteAndAddMetadata(ctx, hostID, rOut.Site, rOut.Metadata); err != nil { + rIn.Error = err.Error() + *erringRecords = append(*erringRecords, rIn) + return } + // Print host_id from response if successful + fmt.Printf("✔ Host Serial number : %s UUID : %s registered. Name : %s\n", sNo, uuid, hostID) } -func createInstanceAndUpdateHost(ctx context.Context, oClient *orchcli.OrchCli, record types.HostRecord, - erringRecords *[]types.HostRecord, osProfileID, hostID string, -) error { - var siteID, laID string - isSecure := record.Secure +func sanitizeProvisioningFields(ctx context.Context, oClient *orchcli.OrchCli, record types.HostRecord, + erringRecords *[]types.HostRecord, globalAttr *types.HostRecord, +) (*types.HostRecord, error) { + isSecure := resolveSecure(record.Secure, globalAttr.Secure) + osProfileID, err := resolveOSProfile(ctx, oClient, record.OSProfile, globalAttr.OSProfile, record, erringRecords) + if err != nil { + return nil, err + } - // if osProfile is provided in command line, that takes precedence & - // osProfileID is already set. If not, check if osProfile is provided - // in the csv file. - if osProfileID == "" { - osProfileID = record.OSProfile + if valErr := validateSecurityFeature(oClient, osProfileID, isSecure, record, erringRecords); valErr != nil { + return nil, valErr } - var err error + siteID, err := resolveSite(ctx, oClient, record.Site, globalAttr.Site, record, erringRecords) + if err != nil { + return nil, err + } - if osProfileID, err = oClient.GetOsProfileID(ctx, osProfileID); err != nil { + laID, err := resolveRemoteUser(ctx, oClient, record.RemoteUser, globalAttr.RemoteUser, record, erringRecords) + if err != nil { + return nil, err + } + + metadataToUse := resolveMetadata(record.Metadata, globalAttr.Metadata) + + return &types.HostRecord{ + OSProfile: osProfileID, + RemoteUser: laID, + Site: siteID, + Secure: isSecure, + UUID: record.UUID, + Serial: record.Serial, + Metadata: metadataToUse, + }, nil +} + +func resolveSecure(recordSecure, globalSecure types.RecordSecure) types.RecordSecure { + if globalSecure != recordSecure && globalSecure != types.SecureUnspecified { + return globalSecure + } + return recordSecure +} + +func resolveOSProfile(ctx context.Context, oClient *orchcli.OrchCli, recordOSProfile, globalOSProfile string, + record types.HostRecord, erringRecords *[]types.HostRecord, +) (string, error) { + osProfileID := recordOSProfile + if globalOSProfile != "" { + osProfileID = globalOSProfile + } + + osProfileID, err := oClient.GetOsProfileID(ctx, osProfileID) + if err != nil { record.Error = err.Error() *erringRecords = append(*erringRecords, record) - return err + return "", err } + return osProfileID, nil +} + +func validateSecurityFeature(oClient *orchcli.OrchCli, osProfileID string, isSecure types.RecordSecure, + record types.HostRecord, erringRecords *[]types.HostRecord, +) error { osProfile, ok := oClient.OSProfileCache[osProfileID] - if !ok || (*osProfile.SecurityFeature != api.SECURITYFEATURESECUREBOOTANDFULLDISKENCRYPTION && isSecure) { + if !ok || (*osProfile.SecurityFeature != api.SECURITYFEATURESECUREBOOTANDFULLDISKENCRYPTION && isSecure == types.SecureTrue) { record.Error = e.NewCustomError(e.ErrOSSecurityMismatch).Error() *erringRecords = append(*erringRecords, record) - return err + return e.NewCustomError(e.ErrOSSecurityMismatch) + } + return nil +} + +func resolveSite(ctx context.Context, oClient *orchcli.OrchCli, recordSite, globalSite string, + record types.HostRecord, erringRecords *[]types.HostRecord, +) (string, error) { + siteToQuery := recordSite + if globalSite != "" { + siteToQuery = globalSite } - // site is an optional field. If not provided, instance will be created - // but site will not be updated. Can be updated later from UI. - if siteID, err = oClient.GetSiteID(ctx, record.Site); err != nil { + siteID, err := oClient.GetSiteID(ctx, siteToQuery) + if err != nil { record.Error = err.Error() *erringRecords = append(*erringRecords, record) - return err + return "", err } + return siteID, nil +} - // local account is a optional field, instance will be created irrespective - if laID, err = oClient.GetLocalAccountID(ctx, record.RemoteUser); err != nil { - record.Error = err.Error() - *erringRecords = append(*erringRecords, record) - return err +func resolveRemoteUser(ctx context.Context, oClient *orchcli.OrchCli, recordRemoteUser, globalRemoteUser string, + record types.HostRecord, erringRecords *[]types.HostRecord, +) (string, error) { + remoteUserToQuery := recordRemoteUser + if globalRemoteUser != "" { + remoteUserToQuery = globalRemoteUser } - // create instance if osProfileID is available else append to error list - // Need not notify user of instance ID. Unnecessary detail for user. - _, err = oClient.CreateInstance(ctx, hostID, osProfileID, laID, isSecure) + laID, err := oClient.GetLocalAccountID(ctx, remoteUserToQuery) if err != nil { record.Error = err.Error() *erringRecords = append(*erringRecords, record) - return err + return "", err } + return laID, nil +} - if err := oClient.AllocateHostToSiteAndAddMetadata(ctx, hostID, siteID, record.Metadata); err != nil { - record.Error = err.Error() - *erringRecords = append(*erringRecords, record) - return err +func resolveMetadata(recordMetadata, globalMetadata string) string { + if globalMetadata != "" { + return globalMetadata } - return nil + return recordMetadata } diff --git a/bulk-import-tools/cmd/orch-host-bulk-import/orch_host_bulk_import_test.go b/bulk-import-tools/cmd/orch-host-bulk-import/orch_host_bulk_import_test.go index e92aea99b..4d6f60f57 100644 --- a/bulk-import-tools/cmd/orch-host-bulk-import/orch_host_bulk_import_test.go +++ b/bulk-import-tools/cmd/orch-host-bulk-import/orch_host_bulk_import_test.go @@ -33,10 +33,37 @@ func TestBinaryImportCommand(t *testing.T) { }, { name: "import with invalid url", - args: []string{"import", "--onboard", "input.csv", "https://xyz.com", "test"}, + args: []string{"import", "--onboard", "--project", "test", "input.csv", "https://xyz.com"}, expectOutput: "Importing hosts from file: input.csv to server: https://xyz.com\nOnboarding is enabled\n", expectErr: true, }, + { + name: "import with all flags", + args: []string{ + "import", "--onboard", "--project", "test", "--os-profile", "test", "--site", + "test", "--secure", "test", "--remote-user", "test", "--metadata", "test", "input.csv", "https://xyz.com", + }, + expectOutput: "Importing hosts from file: input.csv to server: https://xyz.com\nOnboarding is enabled\n", + expectErr: true, + }, + { + name: "import with all flags with equals sign", + args: []string{ + "import", "--onboard", "--project=test", "--os-profile=test", "--site=test", "--secure=true", + "--remote-user=test", "--metadata=test", "input.csv", "https://xyz.com", + }, + expectOutput: "Importing hosts from file: input.csv to server: https://xyz.com\nOnboarding is enabled\n", + expectErr: true, + }, + { + name: "import with invalid flag", + args: []string{ + "import", "--onboard", "--project", "test", "--osprofile", "test", "--site", + "test", "--secure", "test", "--remote-user", "test", "--metadata", "test", "input.csv", "https://xyz.com", + }, + expectOutput: "flag provided but not defined: -osprofile\n", + expectErr: true, + }, { name: "help", args: []string{"help"}, diff --git a/bulk-import-tools/internal/errors/errors.go b/bulk-import-tools/internal/errors/errors.go index baf52bb82..6d8a02fb0 100644 --- a/bulk-import-tools/internal/errors/errors.go +++ b/bulk-import-tools/internal/errors/errors.go @@ -3,7 +3,10 @@ package errors -import "fmt" +import ( + "errors" + "fmt" +) type ErrorCode int @@ -56,7 +59,7 @@ var errorMessages = map[ErrorCode]string{ ErrHostSiteMetadataFailed: "Failed to allocate site or metadata", ErrAuthNFailed: "Failed to authenticate with server", ErrURL: "Malformed server URL", - ErrAlreadyRegistered: "Host UUID already registered", + ErrAlreadyRegistered: "Host already registered", ErrHTTPReq: "HTTP request error", ErrOSSecurityMismatch: "OS Profile and Security feature mismatch", } @@ -80,3 +83,11 @@ func NewCustomError(code ErrorCode) error { Message: msg, } } + +func Is(code ErrorCode, err error) bool { + customErr := new(CustomError) + if errors.As(err, &customErr) { + return customErr.Code == code + } + return false +} diff --git a/bulk-import-tools/internal/files/files.go b/bulk-import-tools/internal/files/files.go index f45c3aa2b..417fe2a31 100644 --- a/bulk-import-tools/internal/files/files.go +++ b/bulk-import-tools/internal/files/files.go @@ -8,7 +8,6 @@ import ( "io" "os" "path/filepath" - "strconv" "strings" e "github.com/open-edge-platform/infra-core/bulk-import-tools/internal/errors" @@ -103,7 +102,7 @@ func ReadHostRecords(filePath string) ([]types.HostRecord, error) { UUID: getField(record, 1), OSProfile: getField(record, 2), Site: getField(record, 3), - Secure: getField(record, 4) == "true", + Secure: types.StringToRecordSecure(getField(record, 4)), RemoteUser: getField(record, 5), Metadata: getField(record, 6), Error: getField(record, 7), @@ -151,7 +150,7 @@ func WriteHostRecords(filePath string, records []types.HostRecord) error { record.UUID, record.OSProfile, record.Site, - strconv.FormatBool(record.Secure), + string(record.Secure), record.RemoteUser, record.Metadata, record.Error, diff --git a/bulk-import-tools/internal/files/files_test.go b/bulk-import-tools/internal/files/files_test.go index a3074ec73..92266961d 100644 --- a/bulk-import-tools/internal/files/files_test.go +++ b/bulk-import-tools/internal/files/files_test.go @@ -126,7 +126,7 @@ func TestReadHostRecords(t *testing.T) { UUID: "uuid-1234", OSProfile: "profile1", Site: "site1", - Secure: true, + Secure: types.SecureTrue, RemoteUser: "user1", Metadata: "cluster-name=test&app-id=testApp", Error: "", @@ -137,7 +137,7 @@ func TestReadHostRecords(t *testing.T) { UUID: "uuid-5678", OSProfile: "profile2", Site: "site2", - Secure: false, + Secure: types.SecureUnspecified, RemoteUser: "user2", Metadata: "meta2", Error: "", @@ -202,7 +202,7 @@ func TestWriteHostRecords(t *testing.T) { UUID: "uuid-1234", OSProfile: "profile1", Site: "site1", - Secure: true, + Secure: types.SecureTrue, RemoteUser: "user1", Metadata: "meta1", Error: "error1", @@ -223,7 +223,7 @@ func TestWriteHostRecords(t *testing.T) { UUID: "uuid-1234", OSProfile: "profile1", Site: "site1", - Secure: true, + Secure: types.SecureTrue, RemoteUser: "user1", Metadata: "meta1", Error: "error1", @@ -233,7 +233,7 @@ func TestWriteHostRecords(t *testing.T) { UUID: "uuid-5678", OSProfile: "profile2", Site: "site2", - Secure: false, + Secure: types.SecureFalse, RemoteUser: "user2", Metadata: "meta2", Error: "error2", diff --git a/bulk-import-tools/internal/orchcli/orchcli.go b/bulk-import-tools/internal/orchcli/orchcli.go index 520f1bf57..a122bdbf0 100644 --- a/bulk-import-tools/internal/orchcli/orchcli.go +++ b/bulk-import-tools/internal/orchcli/orchcli.go @@ -20,6 +20,7 @@ import ( "github.com/open-edge-platform/infra-core/api/pkg/api/v0" "github.com/open-edge-platform/infra-core/bulk-import-tools/internal/authn" e "github.com/open-edge-platform/infra-core/bulk-import-tools/internal/errors" + "github.com/open-edge-platform/infra-core/bulk-import-tools/internal/types" "github.com/open-edge-platform/infra-core/bulk-import-tools/internal/validator" ) @@ -112,44 +113,26 @@ func (oC *OrchCli) RegisterHost(ctx context.Context, host, sNo, uuid string, aut return *hostInfo.ResourceId, nil } -func (oC *OrchCli) CreateInstance(ctx context.Context, hostID, oSResourceID, laID string, secure bool) (string, error) { - osRe := regexp.MustCompile(validator.OSPIDPATTERN) - if !osRe.MatchString(oSResourceID) { - return "", e.NewCustomError(e.ErrInvalidOSProfile) - } - - uParsed := *oC.SvcURL - uParsed.Path = path.Join(uParsed.Path, fmt.Sprintf("/v1/projects/%s/compute/instances", oC.Project)) - - // Prepare the form data - payload := &api.Instance{ - HostID: &hostID, - OsID: &oSResourceID, - SecurityFeature: new(api.SecurityFeature), - Kind: new(api.InstanceKind), - } - - if laID != "" { - payload.LocalAccountID = &laID - } - *payload.Kind = api.INSTANCEKINDUNSPECIFIED - osResource, ok := oC.OSProfileCache[oSResourceID] - if !ok { - return "", e.NewCustomError(e.ErrInternal) +func (oC *OrchCli) CreateInstance(ctx context.Context, hostID string, r *types.HostRecord) (string, error) { + if exists, err := oC.InstanceExists(ctx, r.Serial, r.UUID); exists { + return "", e.NewCustomError(e.ErrAlreadyRegistered) + } else if err != nil { + return "", err } - *payload.SecurityFeature = *osResource.SecurityFeature - if !secure { - *payload.SecurityFeature = api.SECURITYFEATURENONE + if err := validateOSProfile(r.OSProfile); err != nil { + return "", err } - jsonData, err := json.Marshal(payload) + payload, err := oC.prepareInstancePayload(hostID, r) if err != nil { - return "", e.NewCustomError(e.ErrInternal) + return "", err } - // Create the HTTP client and make request - resp, err := oC.doRequest(ctx, uParsed.String(), http.MethodPost, bytes.NewBuffer(jsonData)) + uParsed := *oC.SvcURL + uParsed.Path = path.Join(uParsed.Path, fmt.Sprintf("/v1/projects/%s/compute/instances", oC.Project)) + + resp, err := oC.doRequest(ctx, uParsed.String(), http.MethodPost, bytes.NewBuffer(payload)) if err != nil { return "", err } @@ -160,7 +143,6 @@ func (oC *OrchCli) CreateInstance(ctx context.Context, hostID, oSResourceID, laI } var instanceInfo api.Instance - if err := json.NewDecoder(resp.Body).Decode(&instanceInfo); err != nil { return "", e.NewCustomError(e.ErrInternal) } @@ -168,6 +150,40 @@ func (oC *OrchCli) CreateInstance(ctx context.Context, hostID, oSResourceID, laI return *instanceInfo.ResourceId, nil } +func validateOSProfile(osProfile string) error { + osRe := regexp.MustCompile(validator.OSPIDPATTERN) + if !osRe.MatchString(osProfile) { + return e.NewCustomError(e.ErrInvalidOSProfile) + } + return nil +} + +func (oC *OrchCli) prepareInstancePayload(hostID string, r *types.HostRecord) ([]byte, error) { + payload := &api.Instance{ + HostID: &hostID, + OsID: &r.OSProfile, + SecurityFeature: new(api.SecurityFeature), + Kind: new(api.InstanceKind), + } + + if r.RemoteUser != "" { + payload.LocalAccountID = &r.RemoteUser + } + *payload.Kind = api.INSTANCEKINDUNSPECIFIED + + osResource, ok := oC.OSProfileCache[r.OSProfile] + if !ok { + return nil, e.NewCustomError(e.ErrInternal) + } + + *payload.SecurityFeature = *osResource.SecurityFeature + if r.Secure != types.SecureTrue { + *payload.SecurityFeature = api.SECURITYFEATURENONE + } + + return json.Marshal(payload) +} + func obtainRequestPath(oC *OrchCli, input, pattern, pathByID, pathByName, filter string) (url.URL, *regexp.Regexp) { uParsed := *oC.SvcURL // match os to id pattern @@ -185,6 +201,48 @@ func obtainRequestPath(oC *OrchCli, input, pattern, pathByID, pathByName, filter return uParsed, re } +func (oC *OrchCli) InstanceExists(ctx context.Context, sn, uuid string) (bool, error) { + pathByName := "/v1/projects/%s/compute/instances" + uParsed := *oC.SvcURL + uParsed.Path = path.Join(uParsed.Path, fmt.Sprintf(pathByName, oC.Project)) + query := uParsed.Query() + switch { + case sn != "" && uuid != "": + query.Set("filter", fmt.Sprintf("%s=%q AND %s=%q", "host.serialNumber", sn, "host.uuid", uuid)) + case sn != "": + query.Set("filter", fmt.Sprintf("%s=%q", "host.serialNumber", sn)) + case uuid != "": + query.Set("filter", fmt.Sprintf("%s=%q", "host.uuid", uuid)) + default: + return false, nil + } + uParsed.RawQuery = query.Encode() + + // Create the HTTP client and make request + resp, err := oC.doRequest(ctx, uParsed.String(), http.MethodGet, http.NoBody) + if err != nil { + return false, e.NewCustomError(e.ErrInternal) + } + + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return false, e.NewCustomError(e.ErrInternal) + } + + var instances api.InstanceList + + if err := json.NewDecoder(resp.Body).Decode(&instances); err != nil { + return false, e.NewCustomError(e.ErrInternal) + } + + if *instances.TotalElements > 0 { + return true, nil + } + + return false, nil +} + func (oC *OrchCli) GetOsProfileID(ctx context.Context, os string) (string, error) { if os == "" { return "", e.NewCustomError(e.ErrInvalidOSProfile) @@ -225,11 +283,12 @@ func (oC *OrchCli) GetOsProfileID(ctx context.Context, os string) (string, error return "", e.NewCustomError(e.ErrInternal) } - // Matches substrings as well. Hence will pick up 1st result only for complete match - if *osResources.TotalElements >= 1 && *(*osResources.OperatingSystemResources)[0].ProfileName == os { - oC.OSProfileCache[os] = (*osResources.OperatingSystemResources)[0] - oC.OSProfileCache[*(*osResources.OperatingSystemResources)[0].ResourceId] = (*osResources.OperatingSystemResources)[0] - return *(*osResources.OperatingSystemResources)[0].ResourceId, nil + for _, osResource := range *osResources.OperatingSystemResources { + if *osResource.ProfileName == os { + oC.OSProfileCache[os] = osResource + oC.OSProfileCache[*osResource.ResourceId] = osResource + return *osResource.ResourceId, nil + } } return "", e.NewCustomError(e.ErrInvalidOSProfile) @@ -256,7 +315,7 @@ func (oC *OrchCli) GetSiteID(ctx context.Context, site string) (string, error) { defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return "", e.NewCustomError(e.ErrInvalidOSProfile) + return "", e.NewCustomError(e.ErrInvalidSite) } if siteRe.MatchString(site) { @@ -275,14 +334,15 @@ func (oC *OrchCli) GetSiteID(ctx context.Context, site string) (string, error) { return "", e.NewCustomError(e.ErrInternal) } - // Matches substrings as well. Hence will pick up 1st result only for complete match - if *sites.TotalElements >= 1 && *(*sites.Sites)[0].Name == site { - oC.SiteCache[site] = (*sites.Sites)[0] - oC.SiteCache[*(*sites.Sites)[0].ResourceId] = (*sites.Sites)[0] - return *(*sites.Sites)[0].ResourceId, nil + for _, siteItem := range *sites.Sites { + if *siteItem.Name == site { + oC.SiteCache[site] = siteItem + oC.SiteCache[*siteItem.ResourceId] = siteItem + return *siteItem.ResourceId, nil + } } - return "", e.NewCustomError(e.ErrInvalidOSProfile) + return "", e.NewCustomError(e.ErrInvalidSite) } func (oC *OrchCli) GetLocalAccountID(ctx context.Context, lAName string) (string, error) { @@ -325,12 +385,14 @@ func (oC *OrchCli) GetLocalAccountID(ctx context.Context, lAName string) (string return "", e.NewCustomError(e.ErrInternal) } - // Matches substrings as well. Hence will pick up 1st result only for complete match - if *lAs.TotalElements >= 1 && (*lAs.LocalAccounts)[0].Username == lAName { - oC.LACache[lAName] = (*lAs.LocalAccounts)[0] - oC.LACache[*(*lAs.LocalAccounts)[0].ResourceId] = (*lAs.LocalAccounts)[0] - return *(*lAs.LocalAccounts)[0].ResourceId, nil + for _, la := range *lAs.LocalAccounts { + if la.Username == lAName { + oC.LACache[lAName] = la + oC.LACache[*la.ResourceId] = la + return *la.ResourceId, nil + } } + return "", e.NewCustomError(e.ErrInvalidLocalAccount) } diff --git a/bulk-import-tools/internal/orchcli/orchcli_test.go b/bulk-import-tools/internal/orchcli/orchcli_test.go index 9f6a4c31d..c2b7be483 100644 --- a/bulk-import-tools/internal/orchcli/orchcli_test.go +++ b/bulk-import-tools/internal/orchcli/orchcli_test.go @@ -19,6 +19,7 @@ import ( "github.com/open-edge-platform/infra-core/api/pkg/api/v0" e "github.com/open-edge-platform/infra-core/bulk-import-tools/internal/errors" "github.com/open-edge-platform/infra-core/bulk-import-tools/internal/orchcli" + "github.com/open-edge-platform/infra-core/bulk-import-tools/internal/types" ) var ( @@ -312,7 +313,13 @@ func TestCreateInstance(t *testing.T) { SecurityFeature: new(api.SecurityFeature), } - resp, err := oc.CreateInstance(context.Background(), hostID, oSResourceID, "", true) + hr := &types.HostRecord{ + OSProfile: oSResourceID, + RemoteUser: "", + Secure: types.SecureTrue, + } + + resp, err := oc.CreateInstance(context.Background(), hostID, hr) assert.NoError(t, err) assert.Equal(t, resourceID, resp) } @@ -323,7 +330,13 @@ func TestCreateInstanceInvalidOSProfile(t *testing.T) { oc := newOrchCli(t, mockServer.URL, project, jwt) - resp, err := oc.CreateInstance(context.Background(), uuid, "invalid-os-id", "", true) + hr := &types.HostRecord{ + OSProfile: "invalid-os-profile", + RemoteUser: "", + Secure: types.SecureTrue, + } + + resp, err := oc.CreateInstance(context.Background(), hostID, hr) assert.Error(t, err) assert.Empty(t, resp) } @@ -334,14 +347,21 @@ func TestCreateInstanceInternalError(t *testing.T) { oc := newOrchCli(t, mockServer.URL, project, jwt) - resp, err := oc.CreateInstance(context.Background(), hostID, oSResourceID, "", true) + hr := &types.HostRecord{ + OSProfile: oSResourceID, + RemoteUser: "", + Secure: types.SecureTrue, + } + + resp, err := oc.CreateInstance(context.Background(), hostID, hr) assert.Error(t, err) assert.Empty(t, resp) oc.OSProfileCache[oSResourceID] = api.OperatingSystemResource{ SecurityFeature: new(api.SecurityFeature), } - resp, err = oc.CreateInstance(context.Background(), hostID, oSResourceID, "", true) + + resp, err = oc.CreateInstance(context.Background(), hostID, hr) assert.Error(t, err) assert.Empty(t, resp) } @@ -361,7 +381,13 @@ func TestCreateInstanceWithLocalAccount(t *testing.T) { SecurityFeature: new(api.SecurityFeature), } - resp, err := oc.CreateInstance(context.Background(), hostID, oSResourceID, localAccountID, true) + hr := &types.HostRecord{ + OSProfile: oSResourceID, + RemoteUser: localAccountID, + Secure: types.SecureTrue, + } + + resp, err := oc.CreateInstance(context.Background(), hostID, hr) assert.NoError(t, err) assert.Equal(t, resourceID, resp) } @@ -379,8 +405,13 @@ func TestCreateInstanceSecurityFeatureNone(t *testing.T) { oc.OSProfileCache[oSResourceID] = api.OperatingSystemResource{ SecurityFeature: new(api.SecurityFeature), } + hr := &types.HostRecord{ + OSProfile: oSResourceID, + RemoteUser: "", + Secure: types.SecureFalse, + } - resp, err := oc.CreateInstance(context.Background(), hostID, oSResourceID, "", false) + resp, err := oc.CreateInstance(context.Background(), hostID, hr) assert.NoError(t, err) assert.Equal(t, resourceID, resp) } @@ -782,3 +813,85 @@ func TestAllocateHostToSiteAndAddMetadata(t *testing.T) { assert.Equal(t, e.NewCustomError(e.ErrHostSiteMetadataFailed), err) }) } + +func TestInstanceExists(t *testing.T) { + instanceList := api.InstanceList{ + TotalElements: new(int), + } + *instanceList.TotalElements = 1 + + t.Run("Instance Exists with Serial Number and UUID", func(t *testing.T) { + mockServer := setupMockServerForGetResources(t, http.StatusOK, instanceList) + defer mockServer.Close() + + oc := newOrchCli(t, mockServer.URL, project, jwt) + + exists, err := oc.InstanceExists(context.Background(), sn, uuid) + assert.NoError(t, err) + assert.True(t, exists) + }) + + t.Run("Instance Exists with Serial Number Only", func(t *testing.T) { + mockServer := setupMockServerForGetResources(t, http.StatusOK, instanceList) + defer mockServer.Close() + + oc := newOrchCli(t, mockServer.URL, project, jwt) + + exists, err := oc.InstanceExists(context.Background(), sn, "") + assert.NoError(t, err) + assert.True(t, exists) + }) + + t.Run("Instance Exists with UUID Only", func(t *testing.T) { + mockServer := setupMockServerForGetResources(t, http.StatusOK, instanceList) + defer mockServer.Close() + + oc := newOrchCli(t, mockServer.URL, project, jwt) + + exists, err := oc.InstanceExists(context.Background(), "", uuid) + assert.NoError(t, err) + assert.True(t, exists) + }) + + t.Run("Instance Does Not Exist", func(t *testing.T) { + instanceList.TotalElements = new(int) // Set TotalElements to 0 + mockServer := setupMockServerForGetResources(t, http.StatusOK, instanceList) + defer mockServer.Close() + + oc := newOrchCli(t, mockServer.URL, project, jwt) + + exists, err := oc.InstanceExists(context.Background(), sn, uuid) + assert.NoError(t, err) + assert.False(t, exists) + }) + + t.Run("Empty Serial Number and UUID", func(t *testing.T) { + oc := newOrchCli(t, "", project, jwt) + + exists, err := oc.InstanceExists(context.Background(), "", "") + assert.NoError(t, err) + assert.False(t, exists) + }) + + t.Run("Internal Server Error", func(t *testing.T) { + mockServer := setupMockServerForGetResources(t, http.StatusInternalServerError, nil) + defer mockServer.Close() + + oc := newOrchCli(t, mockServer.URL, project, jwt) + + exists, err := oc.InstanceExists(context.Background(), sn, uuid) + assert.Error(t, err) + assert.False(t, exists) + }) + + t.Run("Invalid Response Format", func(t *testing.T) { + mockServer := setupMockServerForGetResources(t, http.StatusOK, "invalid-response") + defer mockServer.Close() + + oc := newOrchCli(t, mockServer.URL, project, jwt) + + exists, err := oc.InstanceExists(context.Background(), sn, uuid) + assert.Error(t, err) + assert.False(t, exists) + }) +} diff --git a/bulk-import-tools/internal/types/HostRecord.go b/bulk-import-tools/internal/types/HostRecord.go index cc18c65f7..42194a93f 100644 --- a/bulk-import-tools/internal/types/HostRecord.go +++ b/bulk-import-tools/internal/types/HostRecord.go @@ -8,7 +8,7 @@ type HostRecord struct { UUID string OSProfile string Site string - Secure bool + Secure RecordSecure RemoteUser string // Metadata is a set of key-value pairs (key=value) separated by '&' rather than a // JSON string to simplify the input data for the user and to avoid handling commas @@ -19,3 +19,28 @@ type HostRecord struct { Error string RawRecord string } + +type RecordSecure string + +const ( + SecureTrue RecordSecure = "true" + SecureFalse RecordSecure = "false" + SecureUnspecified RecordSecure = "" +) + +// StringToRecordSecure converts a string to a RecordSecure enum value. +func StringToRecordSecure(value string) RecordSecure { + switch value { + case string(SecureTrue): + return SecureTrue + case string(SecureFalse): + return SecureFalse + default: + return SecureUnspecified + } +} + +// RecordSecureToString converts a RecordSecure enum value to a string. +func RecordSecureToString(value RecordSecure) string { + return string(value) +} diff --git a/bulk-import-tools/internal/validator/validator_test.go b/bulk-import-tools/internal/validator/validator_test.go index 531e3ed44..be7e43c12 100644 --- a/bulk-import-tools/internal/validator/validator_test.go +++ b/bulk-import-tools/internal/validator/validator_test.go @@ -389,11 +389,11 @@ func TestCheckCSV(t *testing.T) { expectStr: []types.HostRecord{ { Serial: "ABCD123", UUID: "4c4c4c4c-0000-1111-2222-333333333333", OSProfile: "", - RawRecord: "ABCD123,4c4c4c4c-0000-1111-2222-333333333333,,,false,,,", + RawRecord: "ABCD123,4c4c4c4c-0000-1111-2222-333333333333,,,,,,", }, { Serial: "QWERTY123", UUID: "1c1c1c1c-0000-1111-2222-333333333333", OSProfile: "os2", - RawRecord: "QWERTY123,1c1c1c1c-0000-1111-2222-333333333333,os2,,false,,,", + RawRecord: "QWERTY123,1c1c1c1c-0000-1111-2222-333333333333,os2,,,,,", }, }, }, @@ -406,7 +406,7 @@ func TestCheckCSV(t *testing.T) { expectStr: []types.HostRecord{ { Serial: "ABCD-123", UUID: "4c4c4c4c-0000-1111-2222-333333333333", OSProfile: "os1", - Error: "Invalid Serial number;", RawRecord: "ABCD-123,4c4c4c4c-0000-1111-2222-333333333333,os1,,false,,,", + Error: "Invalid Serial number;", RawRecord: "ABCD-123,4c4c4c4c-0000-1111-2222-333333333333,os1,,,,,", }, }, expectErrStr: "Pre-flight check failed", @@ -421,11 +421,11 @@ func TestCheckCSV(t *testing.T) { expectStr: []types.HostRecord{ { Serial: "ABCD123", UUID: "4c4c4c4c-0000-1111-2222-333333333333", OSProfile: "os1", - RawRecord: "ABCD123,4c4c4c4c-0000-1111-2222-333333333333,os1,,false,,,", + RawRecord: "ABCD123,4c4c4c4c-0000-1111-2222-333333333333,os1,,,,,", }, { Serial: "QWERTY123", UUID: "4c4c4c4c-0000-1111-2222-333333333333", Error: "Duplicate UUID : Row 1;", - RawRecord: "QWERTY123,4c4c4c4c-0000-1111-2222-333333333333,,,false,,,", + RawRecord: "QWERTY123,4c4c4c4c-0000-1111-2222-333333333333,,,,,,", }, }, expectErrStr: "Pre-flight check failed",