Make imports idempotent

bulk-import-tools/README.md

@@ -101,18 +101,29 @@ csv(`preflight_error_timestamp_filename`) to be generated with error messages co
 #### Bulk import tool
 
 ```bash
-  Import host data from input file into the Edge Orchestrator.
+Import host data from input file into the Edge Orchestrator.
 
-  Usage: orch-host-bulk-import COMMAND
+Usage: orch-host-bulk-import COMMAND
+
+
+COMMANDS:
+
+import [OPTIONS] <file> <url>  Import data from given CSV file to orchestrator URL
+        file                   Required source CSV file to read data from
+        url                    Required Edge Orchestrator URL
+version                        Display version information
+help                           Show this help message
+
+OPTIONS:
+
+--onboard                      If set, hosts will be automatically onboarded when connected
+--project <name>               Optional project name in Edge Orchestrator. Alternatively, set env variable EDGEORCH_PROJECT
+--os-profile <name/id>         Optional operating system profile name/id to configure for hosts. Alternatively, set env variable EDGEORCH_OSPROFILE
+--site <name/id>               Optional site name/id to configure for hosts. Alternatively, set env variable EDGEORCH_SITE
+--secure <value>               Optional security feature to configure for hosts. Alternatively, set env variable EDGEORCH_SECURE. Valid values: true, false
+--remote-user <name/id>        Optional remote user name/id to configure for hosts. Alternatively, set env variable EDGEORCH_REMOTEUSER
+--metadata <data>              Optional metadata to configure for hosts. Alternatively, set env variable EDGEORCH_METADATA. Metadata format: key=value&key=value
 
-  Commands:
-    import [--onboard] <file> <url> <project> Import data from given CSV file to orchestrator URL
-            --onboard  If set, hosts will be automatically onboarded when connected
-            file       Required source CSV file to read data from
-            url        Required Edge Orchestrator URL
-            project    Optional project name in Edge Orchestrator. Alternatively, set env variable EDGEORCH_PROJECT
-    version Display version information
-    help    Show this help message
 ```
 
 Before running the bulk import tool, project name can be optionally set in envioronment variable or can be passed

bulk-import-tools/VERSION

@@ -1 +1 @@
-1.5.1-dev
+1.5.1

bulk-import-tools/cmd/orch-host-bulk-import/orch_host_bulk_import.go

@@ -50,24 +50,28 @@ func main() {
 
 func handleImportCommand() {
 	importCmd := flag.NewFlagSet("import", flag.ExitOnError)
-	onboardFlag := importCmd.Bool("onboard", false, "Enable onboarding")
+	importCmd.Usage = displayHelp
+	onboardFlag := importCmd.Bool("onboard", false, "")
+	projectNameIn := importCmd.String("project", "", "")
+	osProfileIn := importCmd.String("os-profile", "", "")
+	siteIn := importCmd.String("site", "", "")
+	secureFlag := importCmd.String("secure", string(types.SecureUnspecified), "")
+	remoteUserIn := importCmd.String("remote-user", "", "")
+	metadataIn := importCmd.String("metadata", "", "")
 	err := importCmd.Parse(os.Args[idxAfterFlags:])
 
 	// Check for the correct number of arguments after flags
-	if err != nil || importCmd.NArg() < numArgs {
+	if err != nil || importCmd.NArg() < numArgs-1 {
 		fmt.Println("error: Filename & url required as arguments")
 		displayHelp()
 		os.Exit(1)
 	}
 
 	filePath := importCmd.Arg(0)
 	serverURL := importCmd.Arg(1)
-	//nolint:mnd // 2 is the index of the project name
-	projectName := importCmd.Arg(2)
-	//nolint:mnd // 3 is the index of the optional osprofile
-	osProfile := importCmd.Arg(3)
 
 	// Check if project name is not provided, use the environment variable EDGEORCH_PROJECT
+	projectName := *projectNameIn
 	if projectName == "" {
 		projectName = os.Getenv("EDGEORCH_PROJECT")
 	}
@@ -79,38 +83,90 @@ func handleImportCommand() {
 	}
 
 	// Check if osprofile is not provided, use the environment variable EDGEORCH_OSPROFILE
+	osProfile := *osProfileIn
 	if osProfile == "" {
 		osProfile = os.Getenv("EDGEORCH_OSPROFILE")
 	}
 
+	// Check if site is not provided, use the environment variable EDGEORCH_SITE
+	site := *siteIn
+	if site == "" {
+		site = os.Getenv("EDGEORCH_SITE")
+	}
+
+	// Check if secure flag is not provided, use the environment variable EDGEORCH_SECURE
+	secure := getGlobalSecureAttr(secureFlag)
+
+	// Check if remote user is not provided, use the environment variable EDGEORCH_REMOTEUSER
+	remoteUser := *remoteUserIn
+	if remoteUser == "" {
+		remoteUser = os.Getenv("EDGEORCH_REMOTEUSER")
+	}
+
+	// Check if metadata is not provided, use the environment variable EDGEORCH_METADATA
+	metadata := *metadataIn
+	if metadata == "" {
+		metadata = os.Getenv("EDGEORCH_METADATA")
+	}
+
+	globalAttr := &types.HostRecord{
+		OSProfile:  osProfile,
+		Site:       site,
+		Secure:     types.StringToRecordSecure(secure),
+		RemoteUser: remoteUser,
+		Metadata:   metadata,
+	}
+
 	fmt.Printf("Importing hosts from file: %s to server: %s\n", filePath, serverURL)
 
 	// Implement the import functionality here
-	if err := doImport(*onboardFlag, filePath, serverURL, projectName, osProfile); err != nil {
+	if err := doImport(*onboardFlag, filePath, serverURL, projectName, globalAttr); err != nil {
 		fmt.Printf("error: %v\n\n", err.Error())
 		os.Exit(1)
 	}
 	fmt.Print("CSV import successful\n\n")
 }
 
+func getGlobalSecureAttr(secureFlag *string) string {
+	secure := *secureFlag
+	if secure == "" {
+		secureEnv := os.Getenv("EDGEORCH_SECURE")
+		if secureEnv == "" {
+			secure = string(types.SecureUnspecified)
+		} else {
+			secure = secureEnv
+		}
+	}
+	return secure
+}
+
 // displayHelp prints the help information for the utility.
 func displayHelp() {
-	fmt.Print("Import host data from input file into the Edge Orchestrator.\n\n")
+	fmt.Print("\n\nImport host data from input file into the Edge Orchestrator.\n\n")
 	fmt.Print("Usage: orch-host-bulk-import COMMAND\n\n")
-	fmt.Println("Commands:")
-	fmt.Println("\timport [--onboard] <file> <url> <project> <osprofile> Import data from given CSV file to orchestrator URL")
-	fmt.Println("\t        --onboard  If set, hosts will be automatically onboarded when connected")
-	fmt.Println("\t        file       Required source CSV file to read data from")
-	fmt.Println("\t        url        Required Edge Orchestrator URL")
-	fmt.Println("\t        project    Optional project name in Edge Orchestrator.",
+	fmt.Print("COMMANDS:\n\n")
+	fmt.Println("import [OPTIONS] <file> <url>  Import data from given CSV file to orchestrator URL")
+	fmt.Println("        file                   Required source CSV file to read data from")
+	fmt.Println("        url                    Required Edge Orchestrator URL")
+	fmt.Println("version                        Display version information")
+	fmt.Print("help                           Show this help message\n\n")
+	fmt.Print("OPTIONS:\n\n")
+	fmt.Println("--onboard                      If set, hosts will be automatically onboarded when connected")
+	fmt.Println("--project <name>               Optional project name in Edge Orchestrator.",
 		"Alternatively, set env variable EDGEORCH_PROJECT")
-	fmt.Println("\t        osprofile  Optional operating system profile name/id to configure for hosts.",
+	fmt.Println("--os-profile <name/id>         Optional operating system profile name/id to configure for hosts.",
 		"Alternatively, set env variable EDGEORCH_OSPROFILE")
-	fmt.Println("\tversion Display version information")
-	fmt.Print("\thelp    Show this help message\n\n")
+	fmt.Println("--site <name/id>               Optional site name/id to configure for hosts.",
+		"Alternatively, set env variable EDGEORCH_SITE")
+	fmt.Println("--secure <value>               Optional security feature to configure for hosts.",
+		"Alternatively, set env variable EDGEORCH_SECURE. Valid values: true, false")
+	fmt.Println("--remote-user <name/id>        Optional remote user name/id to configure for hosts.",
+		"Alternatively, set env variable EDGEORCH_REMOTEUSER")
+	fmt.Print("--metadata <data>              Optional metadata to configure for hosts.",
+		"Alternatively, set env variable EDGEORCH_METADATA. Metadata format: key=value&key=value\n\n")
 }
 
-func doImport(autoOnboard bool, filePath, serverURL, projectName, osProfile string) error {
+func doImport(autoOnboard bool, filePath, serverURL, projectName string, globalAttr *types.HostRecord) error {
 	ctx, cancel := context.WithCancelCause(context.Background())
 	defer cancel(nil)
 	erringRecords := []types.HostRecord{}
@@ -130,18 +186,10 @@ func doImport(autoOnboard bool, filePath, serverURL, projectName, osProfile stri
 		return err
 	}
 
-	var osProfileID string
-	// if osProfile is provided globally, verify if its valid or get
-	// the id from the server if profilename is provided
-	if osProfile != "" {
-		if osProfileID, err = oClient.GetOsProfileID(ctx, osProfile); err != nil {
-			return err
-		}
-	}
 	// registerHost
 	// iterate over all entries available
 	for _, record := range validated {
-		doRegister(ctx, oClient, autoOnboard, osProfileID, record, &erringRecords)
+		doRegister(ctx, oClient, autoOnboard, globalAttr, record, &erringRecords)
 	}
 	// write import error to import_error_<rfc3339_timestamp>_<filename>
 	// if there is any error record after header
@@ -158,83 +206,152 @@ func doImport(autoOnboard bool, filePath, serverURL, projectName, osProfile stri
 }
 
 func doRegister(ctx context.Context, oClient *orchcli.OrchCli, autoOnboard bool,
-	osProfileID string, record types.HostRecord, erringRecords *[]types.HostRecord,
+	globalAttr *types.HostRecord, rIn types.HostRecord, erringRecords *[]types.HostRecord,
 ) {
 	// get the required fields from the record
-	sNo := record.Serial
-	uuid := record.UUID
+	sNo := rIn.Serial
+	uuid := rIn.UUID
 
-	// try to register
-	hostID, err := oClient.RegisterHost(ctx, "", sNo, uuid, autoOnboard)
+	rOut, err := sanitizeProvisioningFields(ctx, oClient, rIn, erringRecords, globalAttr)
 	if err != nil {
+		return
+	}
+
+	// Register host
+	hostID, err := oClient.RegisterHost(ctx, "", sNo, uuid, autoOnboard)
+	if err != nil && !e.Is(e.ErrAlreadyRegistered, err) {
 		// add to reject list if failed
-		record.Error = err.Error()
-		*erringRecords = append(*erringRecords, record)
-	} else {
-		// print host_id from response if successful
-		fmt.Printf("Host Serial number : %s  UUID : %s registered. Name : %s\n", sNo, uuid, hostID)
+		rIn.Error = err.Error()
+		*erringRecords = append(*erringRecords, rIn)
+		return
+	}
+	// Create instance if osProfileID is available else append to error list
+	// Need not notify user of instance ID. Unnecessary detail for user.
+	_, err = oClient.CreateInstance(ctx, hostID, rOut)
+	if err != nil {
+		rIn.Error = err.Error()
+		*erringRecords = append(*erringRecords, rIn)
+		return
+	}
 
-		if err := createInstanceAndUpdateHost(ctx, oClient, record, erringRecords, osProfileID, hostID); err != nil {
-			return
-		}
+	if err := oClient.AllocateHostToSiteAndAddMetadata(ctx, hostID, rOut.Site, rOut.Metadata); err != nil {
+		rIn.Error = err.Error()
+		*erringRecords = append(*erringRecords, rIn)
+		return
 	}
+	// Print host_id from response if successful
+	fmt.Printf("✔ Host Serial number : %s  UUID : %s registered. Name : %s\n", sNo, uuid, hostID)
 }
 
-func createInstanceAndUpdateHost(ctx context.Context, oClient *orchcli.OrchCli, record types.HostRecord,
-	erringRecords *[]types.HostRecord, osProfileID, hostID string,
-) error {
-	var siteID, laID string
-	isSecure := record.Secure
+func sanitizeProvisioningFields(ctx context.Context, oClient *orchcli.OrchCli, record types.HostRecord,
+	erringRecords *[]types.HostRecord, globalAttr *types.HostRecord,
+) (*types.HostRecord, error) {
+	isSecure := resolveSecure(record.Secure, globalAttr.Secure)
+	osProfileID, err := resolveOSProfile(ctx, oClient, record.OSProfile, globalAttr.OSProfile, record, erringRecords)
+	if err != nil {
+		return nil, err
+	}
 
-	// if osProfile is provided in command line, that takes precedence &
-	// osProfileID is already set. If not, check if osProfile is provided
-	// in the csv file.
-	if osProfileID == "" {
-		osProfileID = record.OSProfile
+	if valErr := validateSecurityFeature(oClient, osProfileID, isSecure, record, erringRecords); valErr != nil {
+		return nil, valErr
 	}
 
-	var err error
+	siteID, err := resolveSite(ctx, oClient, record.Site, globalAttr.Site, record, erringRecords)
+	if err != nil {
+		return nil, err
+	}
 
-	if osProfileID, err = oClient.GetOsProfileID(ctx, osProfileID); err != nil {
+	laID, err := resolveRemoteUser(ctx, oClient, record.RemoteUser, globalAttr.RemoteUser, record, erringRecords)
+	if err != nil {
+		return nil, err
+	}
+
+	metadataToUse := resolveMetadata(record.Metadata, globalAttr.Metadata)
+
+	return &types.HostRecord{
+		OSProfile:  osProfileID,
+		RemoteUser: laID,
+		Site:       siteID,
+		Secure:     isSecure,
+		UUID:       record.UUID,
+		Serial:     record.Serial,
+		Metadata:   metadataToUse,
+	}, nil
+}
+
+func resolveSecure(recordSecure, globalSecure types.RecordSecure) types.RecordSecure {
+	if globalSecure != recordSecure && globalSecure != types.SecureUnspecified {
+		return globalSecure
+	}
+	return recordSecure
+}
+
+func resolveOSProfile(ctx context.Context, oClient *orchcli.OrchCli, recordOSProfile, globalOSProfile string,
+	record types.HostRecord, erringRecords *[]types.HostRecord,
+) (string, error) {
+	osProfileID := recordOSProfile
+	if globalOSProfile != "" {
+		osProfileID = globalOSProfile
+	}
+
+	osProfileID, err := oClient.GetOsProfileID(ctx, osProfileID)
+	if err != nil {
 		record.Error = err.Error()
 		*erringRecords = append(*erringRecords, record)
-		return err
+		return "", err
 	}
+	return osProfileID, nil
+}
+
+func validateSecurityFeature(oClient *orchcli.OrchCli, osProfileID string, isSecure types.RecordSecure,
+	record types.HostRecord, erringRecords *[]types.HostRecord,
+) error {
 	osProfile, ok := oClient.OSProfileCache[osProfileID]
-	if !ok || (*osProfile.SecurityFeature != api.SECURITYFEATURESECUREBOOTANDFULLDISKENCRYPTION && isSecure) {
+	if !ok || (*osProfile.SecurityFeature != api.SECURITYFEATURESECUREBOOTANDFULLDISKENCRYPTION && isSecure == types.SecureTrue) {
 		record.Error = e.NewCustomError(e.ErrOSSecurityMismatch).Error()
 		*erringRecords = append(*erringRecords, record)
-		return err
+		return e.NewCustomError(e.ErrOSSecurityMismatch)
+	}
+	return nil
+}
+
+func resolveSite(ctx context.Context, oClient *orchcli.OrchCli, recordSite, globalSite string,
+	record types.HostRecord, erringRecords *[]types.HostRecord,
+) (string, error) {
+	siteToQuery := recordSite
+	if globalSite != "" {
+		siteToQuery = globalSite
 	}
 
-	// site is an optional field. If not provided, instance will be created
-	// but site will not be updated. Can be updated later from UI.
-	if siteID, err = oClient.GetSiteID(ctx, record.Site); err != nil {
+	siteID, err := oClient.GetSiteID(ctx, siteToQuery)
+	if err != nil {
 		record.Error = err.Error()
 		*erringRecords = append(*erringRecords, record)
-		return err
+		return "", err
 	}
+	return siteID, nil
+}
 
-	// local account is a optional field, instance will be created irrespective
-	if laID, err = oClient.GetLocalAccountID(ctx, record.RemoteUser); err != nil {
-		record.Error = err.Error()
-		*erringRecords = append(*erringRecords, record)
-		return err
+func resolveRemoteUser(ctx context.Context, oClient *orchcli.OrchCli, recordRemoteUser, globalRemoteUser string,
+	record types.HostRecord, erringRecords *[]types.HostRecord,
+) (string, error) {
+	remoteUserToQuery := recordRemoteUser
+	if globalRemoteUser != "" {
+		remoteUserToQuery = globalRemoteUser
 	}
 
-	// create instance if osProfileID is available else append to error list
-	// Need not notify user of instance ID. Unnecessary detail for user.
-	_, err = oClient.CreateInstance(ctx, hostID, osProfileID, laID, isSecure)
+	laID, err := oClient.GetLocalAccountID(ctx, remoteUserToQuery)
 	if err != nil {
 		record.Error = err.Error()
 		*erringRecords = append(*erringRecords, record)
-		return err
+		return "", err
 	}
+	return laID, nil
+}
 
-	if err := oClient.AllocateHostToSiteAndAddMetadata(ctx, hostID, siteID, record.Metadata); err != nil {
-		record.Error = err.Error()
-		*erringRecords = append(*erringRecords, record)
-		return err
+func resolveMetadata(recordMetadata, globalMetadata string) string {
+	if globalMetadata != "" {
+		return globalMetadata
 	}
-	return nil
+	return recordMetadata
 }