diff --git a/cmd/credentialUtil.go b/cmd/credentialUtil.go index 6b09c48b6..7f068a042 100644 --- a/cmd/credentialUtil.go +++ b/cmd/credentialUtil.go @@ -70,11 +70,13 @@ func GetUserOAuthTokenManagerInstance() *common.UserOAuthTokenManager { if common.AzcopyJobPlanFolder == "" { panic("invalid state, AzcopyJobPlanFolder should not be an empty string") } + cacheName := common.GetEnvironmentVariable(common.EEnvironmentVariable.LoginCacheName()) + currentUserOAuthTokenManager = common.NewUserOAuthTokenManagerInstance(common.CredCacheOptions{ DPAPIFilePath: common.AzcopyJobPlanFolder, - KeyName: oauthLoginSessionCacheKeyName, + KeyName: common.Iff(cacheName != "", cacheName, oauthLoginSessionCacheKeyName), ServiceName: oauthLoginSessionCacheServiceName, - AccountName: oauthLoginSessionCacheAccountName, + AccountName: common.Iff(cacheName != "", cacheName, oauthLoginSessionCacheAccountName), }) }) diff --git a/cmd/login.go b/cmd/login.go index c40cd9379..e4ea6ee17 100644 --- a/cmd/login.go +++ b/cmd/login.go @@ -158,12 +158,12 @@ func (lca loginCmdArgs) process() error { // For MSI login, info success message to user. glcm.Info("Login with identity succeeded.") case common.EAutoLoginType.AzCLI().String(): - if err := uotm.AzCliLogin(lca.tenantID); err != nil { + if err := uotm.AzCliLogin(lca.tenantID, lca.persistToken); err != nil { return err } glcm.Info("Login with AzCliCreds succeeded") case common.EAutoLoginType.PsCred().String(): - if err := uotm.PSContextToken(lca.tenantID); err != nil { + if err := uotm.PSContextToken(lca.tenantID, lca.persistToken); err != nil { return err } glcm.Info("Login with Powershell context succeeded") diff --git a/cmd/loginStatus.go b/cmd/loginStatus.go index c1bd597b4..6ac4ebaea 100644 --- a/cmd/loginStatus.go +++ b/cmd/loginStatus.go @@ -22,16 +22,25 @@ package cmd import ( "context" + "encoding/json" "fmt" "github.com/Azure/azure-storage-azcopy/v10/common" "github.com/Azure/azure-storage-azcopy/v10/ste" "github.com/spf13/cobra" ) +type LoginStatusOutput struct { + Valid bool `json:"valid"` + TenantID *string `json:"tenantID,omitempty"` + AADEndpoint *string `json:"AADEndpoint,omitempty"` + AuthMethod *string `json:"authMethod,omitempty"` +} + func init() { type loginStatus struct { tenantID bool endpoint bool + method bool } commandLineInput := loginStatus{} @@ -51,26 +60,56 @@ func init() { uotm := GetUserOAuthTokenManagerInstance() tokenInfo, err := uotm.GetTokenInfo(ctx) - if err == nil && !tokenInfo.IsExpired() { - glcm.Info("You have successfully refreshed your token. Your login session is still active") + var Info = LoginStatusOutput{ + Valid: err == nil && !tokenInfo.IsExpired(), + } + + logText := func(format string, a ...any) { + if azcopyOutputFormat == common.EOutputFormat.None() || azcopyOutputFormat == common.EOutputFormat.Text() { + glcm.Info(fmt.Sprintf(format, a...)) + } + } + + if Info.Valid { + logText("You have successfully refreshed your token. Your login session is still active") if commandLineInput.tenantID { - glcm.Info(fmt.Sprintf("Tenant ID: %v", tokenInfo.Tenant)) + logText("Tenant ID: %v", tokenInfo.Tenant) + Info.TenantID = &tokenInfo.Tenant } if commandLineInput.endpoint { - glcm.Info(fmt.Sprintf("Active directory endpoint: %v", tokenInfo.ActiveDirectoryEndpoint)) + logText(fmt.Sprintf("Active directory endpoint: %v", tokenInfo.ActiveDirectoryEndpoint)) + Info.AADEndpoint = &tokenInfo.ActiveDirectoryEndpoint + } + + if commandLineInput.method { + logText(fmt.Sprintf("Authorized using %s", tokenInfo.LoginType)) + method := tokenInfo.LoginType.String() + Info.AuthMethod = &method } + } else { + logText("You are currently not logged in. Please login using 'azcopy login'") + } + + if azcopyOutputFormat == common.EOutputFormat.Json() { + glcm.Output( + func(_ common.OutputFormat) string { + buf, err := json.Marshal(Info) + if err != nil { + panic(err) + } - glcm.Exit(nil, common.EExitCode.Success()) + return string(buf) + }, common.EOutputMessageType.LoginStatusInfo()) } - glcm.Info("You are currently not logged in. Please login using 'azcopy login'") - glcm.Exit(nil, common.EExitCode.Error()) + glcm.Exit(nil, common.Iff(Info.Valid, common.EExitCode.Success(), common.EExitCode.Error())) }, } lgCmd.AddCommand(lgStatus) lgStatus.PersistentFlags().BoolVar(&commandLineInput.tenantID, "tenant", false, "Prints the Microsoft Entra tenant ID that is currently being used in session.") lgStatus.PersistentFlags().BoolVar(&commandLineInput.endpoint, "endpoint", false, "Prints the Microsoft Entra endpoint that is being used in the current session.") + lgStatus.PersistentFlags().BoolVar(&commandLineInput.method, "method", false, "Prints the authorization method used in the current session.") } diff --git a/common/azure_ps_context_credential.go b/common/azure_ps_context_credential.go index 2b52ffa9f..39c820427 100644 --- a/common/azure_ps_context_credential.go +++ b/common/azure_ps_context_credential.go @@ -15,6 +15,7 @@ import ( "os" "os/exec" "regexp" + "strings" "sync" "time" @@ -24,7 +25,8 @@ import ( const credNamePSContext = "PSContextCredential" -type PSTokenProvider func(ctx context.Context, resource string, tenant string) ([]byte, error) +type PSTokenProvider func(ctx context.Context, options policy.TokenRequestOptions) ([]byte, error) + func validTenantID(tenantID string) bool { match, err := regexp.MatchString("^[0-9a-zA-Z-.]+$", tenantID) if err != nil { @@ -50,6 +52,7 @@ func resolveTenant(defaultTenant, specified, credName string, additionalTenants } return "", fmt.Errorf(`%s isn't configured to acquire tokens for tenant %q. To enable acquiring tokens for this tenant add it to the AdditionallyAllowedTenants on the credential options, or add "*" to allow acquiring tokens for any tenant`, credName, specified) } + // PowershellContextCredentialOptions contains optional parameters for AzureDeveloperCLICredential. type PowershellContextCredentialOptions struct { // TenantID identifies the tenant the credential should authenticate in. Defaults to the azd environment, @@ -96,7 +99,10 @@ func (c *PowershellContextCredential) GetToken(ctx context.Context, opts policy. } c.mu.Lock() defer c.mu.Unlock() - b, err := c.opts.tokenProvider(ctx, opts.Scopes[0], tenant) + + opts.TenantID = tenant + + b, err := c.opts.tokenProvider(ctx, opts) if err == nil { at, err = c.createAccessToken(b) } @@ -109,21 +115,30 @@ func (c *PowershellContextCredential) GetToken(ctx context.Context, opts policy. // We ignore resource because PS does not support all Resources. Disk scope is not supported // and we are here only with Storage scope -var defaultAzdTokenProvider PSTokenProvider = func(ctx context.Context, _ string, tenantID string) ([]byte, error) { +var defaultAzdTokenProvider PSTokenProvider = func(ctx context.Context, opts policy.TokenRequestOptions) ([]byte, error) { // set a default timeout for this authentication iff the application hasn't done so already var cancel context.CancelFunc if _, hasDeadline := ctx.Deadline(); !hasDeadline { - ctx, cancel = context.WithTimeout(ctx, 10 * time.Minute) + ctx, cancel = context.WithTimeout(ctx, 10*time.Minute) defer cancel() } r := regexp.MustCompile("(?s){.*Token.*ExpiresOn.*}") - if tenantID != "" { - tenantID += " -TenantId" + tenantID + cmd := "Get-AzAccessToken" + // set options + if len(opts.Scopes) != 1 { + return nil, errors.New("exactly one scope must be specified") + } else { + cmd += fmt.Sprintf(" -ResourceUrl \"%s\"", strings.TrimSuffix(opts.Scopes[0], "/.default")) } - cmd := "Get-AzAccessToken -ResourceUrl https://storage.azure.com" + tenantID + " | ConvertTo-Json" - + + if opts.TenantID != "" { + cmd += fmt.Sprintf(" -TenantId \"%s\"", opts.TenantID) + } + + // We're going to get broken on this in Az 14.0 and Az.Accounts 5.0, so we may as well fix it now. + cmd += " -AsSecureString | Foreach-Object {[PSCustomObject]@{Token= $($_.Token | ConvertFrom-SecureString -AsPlainText); ExpiresOn = $_.ExpiresOn}} | ConvertTo-Json" cliCmd := exec.CommandContext(ctx, "pwsh", "-Command", cmd) cliCmd.Env = os.Environ() @@ -142,7 +157,7 @@ var defaultAzdTokenProvider PSTokenProvider = func(ctx context.Context, _ string output = []byte(r.FindString(string(output))) if string(output) == "" { invalidTokenMsg := " Invalid output received while retrieving token with Powershell. Run command \"" + cmd + "\"" + - " on powershell and verify that the output is indeed a valid token." + " on powershell and verify that the output is indeed a valid token." return nil, errors.New(credNamePSContext + invalidTokenMsg) } return output, nil @@ -158,7 +173,7 @@ func (c *PowershellContextCredential) createAccessToken(tk []byte) (azcore.Acces if err != nil { return azcore.AccessToken{}, errors.New(err.Error()) } - + parseErr := "error parsing token expiration time %q: %v" exp, err := time.Parse(time.RFC3339, t.ExpiresOn) if err != nil { @@ -170,4 +185,4 @@ func (c *PowershellContextCredential) createAccessToken(tk []byte) (azcore.Acces }, nil } -var _ azcore.TokenCredential = (*PowershellContextCredential)(nil) \ No newline at end of file +var _ azcore.TokenCredential = (*PowershellContextCredential)(nil) diff --git a/common/credCache_windows.go b/common/credCache_windows.go index 8df60cf81..3c4ced10e 100644 --- a/common/credCache_windows.go +++ b/common/credCache_windows.go @@ -197,6 +197,10 @@ func (c *CredCache) saveTokenInternal(token OAuthTokenInfo) error { } func (c *CredCache) tokenFilePath() string { + if cacheFile := GetEnvironmentVariable(EEnvironmentVariable.LoginCacheName()); cacheFile != "" { + return path.Join(c.dpapiFilePath, "/", cacheFile) + } + return path.Join(c.dpapiFilePath, "/", defaultTokenFileName) } diff --git a/common/environment.go b/common/environment.go index 088088890..acf12348e 100644 --- a/common/environment.go +++ b/common/environment.go @@ -306,6 +306,13 @@ func (EnvironmentVariable) CacheProxyLookup() EnvironmentVariable { } } +func (EnvironmentVariable) LoginCacheName() EnvironmentVariable { + return EnvironmentVariable{ + Name: "AZCOPY_LOGIN_CACHE_NAME", + Description: "Do not use in production. Overrides the file name or key name used to cache azcopy's token. Do not use in production. This feature is not documented, intended for testing, and may break. Do not use in production.", + } +} + func (EnvironmentVariable) LogLocation() EnvironmentVariable { return EnvironmentVariable{ Name: "AZCOPY_LOG_LOCATION", diff --git a/common/oauthTokenManager.go b/common/oauthTokenManager.go index 9b7b881df..5283d04ff 100644 --- a/common/oauthTokenManager.go +++ b/common/oauthTokenManager.go @@ -175,21 +175,21 @@ func (uotm *UserOAuthTokenManager) WorkloadIdentityLogin(persist bool) error { return uotm.validateAndPersistLogin(oAuthTokenInfo) } -func (uotm *UserOAuthTokenManager) AzCliLogin(tenantID string) error { +func (uotm *UserOAuthTokenManager) AzCliLogin(tenantID string, persist bool) error { oAuthTokenInfo := &OAuthTokenInfo{ LoginType: EAutoLoginType.AzCLI(), Tenant: tenantID, - Persist: false, // AzCLI creds do not need to be persisted, AzCLI handles persistence. + Persist: persist, // AzCLI creds do not need to be persisted, AzCLI handles persistence. } return uotm.validateAndPersistLogin(oAuthTokenInfo) } -func (uotm *UserOAuthTokenManager) PSContextToken(tenantID string) error { +func (uotm *UserOAuthTokenManager) PSContextToken(tenantID string, persist bool) error { oAuthTokenInfo := &OAuthTokenInfo{ LoginType: EAutoLoginType.PsCred(), Tenant: tenantID, - Persist: false, // Powershell creds do not need to be persisted, Powershell handles persistence. + Persist: persist, // Powershell creds do not need to be persisted, Powershell handles persistence. } return uotm.validateAndPersistLogin(oAuthTokenInfo) @@ -645,6 +645,10 @@ func (credInfo *OAuthTokenInfo) GetClientSecretCredential() (azcore.TokenCredent } func (credInfo *OAuthTokenInfo) GetAzCliCredential() (azcore.TokenCredential, error) { + if credInfo.Tenant == DefaultTenantID { + credInfo.Tenant = "" + } + tc, err := azidentity.NewAzureCLICredential(&azidentity.AzureCLICredentialOptions{TenantID: credInfo.Tenant}) if err != nil { return nil, err @@ -654,7 +658,11 @@ func (credInfo *OAuthTokenInfo) GetAzCliCredential() (azcore.TokenCredential, er } func (credInfo *OAuthTokenInfo) GetPSContextCredential() (azcore.TokenCredential, error) { - tc, err := NewPowershellContextCredential(nil) + if credInfo.Tenant == DefaultTenantID { + credInfo.Tenant = "" + } + + tc, err := NewPowershellContextCredential(&PowershellContextCredentialOptions{TenantID: credInfo.Tenant}) if err != nil { return nil, err } diff --git a/common/output.go b/common/output.go index be5ad937c..db6b21041 100644 --- a/common/output.go +++ b/common/output.go @@ -36,6 +36,8 @@ func (OutputMessageType) Response() OutputMessageType { return OutputMessageType func (OutputMessageType) ListObject() OutputMessageType { return OutputMessageType(8) } func (OutputMessageType) ListSummary() OutputMessageType { return OutputMessageType(9) } +func (OutputMessageType) LoginStatusInfo() OutputMessageType { return OutputMessageType(10) } + func (o OutputMessageType) String() string { return enum.StringInt(o, reflect.TypeOf(o)) } diff --git a/e2etest/newe2e_config.go b/e2etest/newe2e_config.go index f1a0615cb..132904d09 100644 --- a/e2etest/newe2e_config.go +++ b/e2etest/newe2e_config.go @@ -28,7 +28,7 @@ All immediate fields of a mutually exclusive struct will be treated as required, Structs that are not marked "required" will present Environment errors from "required" fields when one or more options are successfully set */ -const AzurePipeline = "AzurePipeline" +const TestEnvironmentAzurePipelines = "TestEnvironmentAzurePipelines" type NewE2EConfig struct { E2EAuthConfig struct { // mutually exclusive @@ -51,9 +51,18 @@ type NewE2EConfig struct { StaticStgAcctInfo struct { StaticOAuth struct { - TenantID string `env:"NEW_E2E_STATIC_TENANT_ID"` - ApplicationID string `env:"NEW_E2E_STATIC_APPLICATION_ID,required"` - ClientSecret string `env:"NEW_E2E_STATIC_CLIENT_SECRET,required"` + TenantID string `env:"NEW_E2E_STATIC_TENANT_ID"` + + OAuthSource struct { // mutually exclusive + SPNSecret struct { + ApplicationID string `env:"NEW_E2E_STATIC_APPLICATION_ID,required"` + ClientSecret string `env:"NEW_E2E_STATIC_CLIENT_SECRET,required"` + } `env:",required"` + + PSInherit bool `env:"NEW_E2E_STATIC_PS_INHERIT,required"` + + CLIInherit bool `env:"NEW_E2E_STATIC_CLI_INHERIT,required"` + } `env:",required,mutually_exclusive"` } // todo: should we automate this somehow? Currently each of these accounts needs some marginal boilerplate. @@ -84,6 +93,36 @@ func (e NewE2EConfig) StaticResources() bool { return e.E2EAuthConfig.SubscriptionLoginInfo.SubscriptionID == "" // all subscriptionlogininfo options would have to be filled due to required } +func (e NewE2EConfig) GetSPNOptions() (present bool, tenant, applicationId, secret string) { + staticInfo := e.E2EAuthConfig.StaticStgAcctInfo.StaticOAuth + dynamicInfo := e.E2EAuthConfig.SubscriptionLoginInfo.DynamicOAuth.SPNSecret + + if e.StaticResources() { + return staticInfo.OAuthSource.SPNSecret.ApplicationID != "", + staticInfo.TenantID, + staticInfo.OAuthSource.SPNSecret.ApplicationID, + staticInfo.OAuthSource.SPNSecret.ClientSecret + } else { + return dynamicInfo.ApplicationID != "", + dynamicInfo.TenantID, + dynamicInfo.ApplicationID, + dynamicInfo.ApplicationID + } +} + +func (e NewE2EConfig) GetTenantID() string { + if e.StaticResources() { + return e.E2EAuthConfig.StaticStgAcctInfo.StaticOAuth.TenantID + } else { + dynamicInfo := e.E2EAuthConfig.SubscriptionLoginInfo.DynamicOAuth + if tid := dynamicInfo.SPNSecret.TenantID; tid != "" { + return tid + } else { + return dynamicInfo.Workload.TenantId // worst case if it bubbles down and it's all zero, that's OK. + } + } +} + // ========= Tag Definition ========== type EnvTag struct { diff --git a/e2etest/newe2e_oauth_cache.go b/e2etest/newe2e_oauth_cache.go index 53b1f7c1e..43b9fdbfd 100644 --- a/e2etest/newe2e_oauth_cache.go +++ b/e2etest/newe2e_oauth_cache.go @@ -20,30 +20,62 @@ const ( var PrimaryOAuthCache *OAuthCache func SetupOAuthCache(a Asserter) { - if GlobalConfig.StaticResources() { - return // no-op, because there's no OAuth configured - } - - dynamicLoginInfo := GlobalConfig.E2EAuthConfig.SubscriptionLoginInfo - staticLoginInfo := GlobalConfig.E2EAuthConfig.StaticStgAcctInfo.StaticOAuth - useStatic := GlobalConfig.StaticResources() - - var cred azcore.TokenCredential - var err error - var tenantId string - if dynamicLoginInfo.Environment == AzurePipeline { - tenantId = dynamicLoginInfo.DynamicOAuth.Workload.TenantId + //if GlobalConfig.StaticResources() { + // return // no-op, because there's no OAuth configured + //} + // + //dynamicLoginInfo := GlobalConfig.E2EAuthConfig.SubscriptionLoginInfo + //staticLoginInfo := GlobalConfig.E2EAuthConfig.StaticStgAcctInfo.StaticOAuth + //useStatic := GlobalConfig.StaticResources() + // + //var cred azcore.TokenCredential + //var err error + //var tenantId string + //if dynamicLoginInfo.Environment == TestEnvironmentAzurePipelines { + // tenantId = dynamicLoginInfo.DynamicOAuth.Workload.TenantId + // cred, err = azidentity.NewDefaultAzureCredential(&azidentity.DefaultAzureCredentialOptions{ + // TenantID: tenantId, + // }) + //} else { + // tenantId = common.Iff(useStatic, staticLoginInfo.TenantID, dynamicLoginInfo.DynamicOAuth.SPNSecret.TenantID) + // cred, err = azidentity.NewClientSecretCredential( + // tenantId, + // common.Iff(useStatic, staticLoginInfo.ApplicationID, dynamicLoginInfo.DynamicOAuth.SPNSecret.ApplicationID), + // common.Iff(useStatic, staticLoginInfo.ClientSecret, dynamicLoginInfo.DynamicOAuth.SPNSecret.ClientSecret), + // nil, // Hopefully the defaults should be OK? + // ) + //} + //a.NoError("create credentials", err) + // + //PrimaryOAuthCache = NewOAuthCache(cred, tenantId) + var ( + cred azcore.TokenCredential + err error + tenantId string + + staticOAuth = GlobalConfig.E2EAuthConfig.StaticStgAcctInfo.StaticOAuth + ) + + // We don't consider workload identity in here because it's only used in a few tests + + if GlobalConfig.E2EAuthConfig.SubscriptionLoginInfo.Environment == TestEnvironmentAzurePipelines { + tenantId = GlobalConfig.E2EAuthConfig.SubscriptionLoginInfo.DynamicOAuth.Workload.TenantId cred, err = azidentity.NewDefaultAzureCredential(&azidentity.DefaultAzureCredentialOptions{ TenantID: tenantId, }) - } else { - tenantId = common.Iff(useStatic, staticLoginInfo.TenantID, dynamicLoginInfo.DynamicOAuth.SPNSecret.TenantID) - cred, err = azidentity.NewClientSecretCredential( - tenantId, - common.Iff(useStatic, staticLoginInfo.ApplicationID, dynamicLoginInfo.DynamicOAuth.SPNSecret.ApplicationID), - common.Iff(useStatic, staticLoginInfo.ClientSecret, dynamicLoginInfo.DynamicOAuth.SPNSecret.ClientSecret), - nil, // Hopefully the defaults should be OK? - ) + } else if useSpn, tenant, appId, secret := GlobalConfig.GetSPNOptions(); useSpn { + tenantId = tenant + cred, err = azidentity.NewClientSecretCredential(tenant, appId, secret, nil) + } else if staticOAuth.OAuthSource.CLIInherit { + tenantId = staticOAuth.TenantID + cred, err = azidentity.NewAzureCLICredential(&azidentity.AzureCLICredentialOptions{ + TenantID: tenantId, + }) + } else if staticOAuth.OAuthSource.PSInherit { + tenantId = staticOAuth.TenantID + cred, err = common.NewPowershellContextCredential(&common.PowershellContextCredentialOptions{ + TenantID: tenantId, + }) } a.NoError("create credentials", err) diff --git a/e2etest/newe2e_runazcopy_stdout.go b/e2etest/newe2e_runazcopy_stdout.go index 5c1585141..18b45ae2d 100644 --- a/e2etest/newe2e_runazcopy_stdout.go +++ b/e2etest/newe2e_runazcopy_stdout.go @@ -239,3 +239,26 @@ func (a *AzCopyParsedJobsListStdout) Write(p []byte) (n int, err error) { } return a.AzCopyParsedStdout.Write(p) } + +type AzCopyParsedLoginStatusStdout struct { + AzCopyParsedStdout + listenChan chan<- common.JsonOutputTemplate + status cmd.LoginStatusOutput +} + +func (a *AzCopyParsedLoginStatusStdout) Write(p []byte) (n int, err error) { + if a.listenChan == nil { + a.listenChan = a.OnParsedLine.SubscribeFunc(func(line common.JsonOutputTemplate) { + if line.MessageType == common.EOutputMessageType.LoginStatusInfo().String() { + out := &cmd.LoginStatusOutput{} + err = json.Unmarshal([]byte(line.MessageContent), out) + if err != nil { + return + } + + a.status = *out + } + }) + } + return a.AzCopyParsedStdout.Write(p) +} diff --git a/e2etest/newe2e_task_runazcopy.go b/e2etest/newe2e_task_runazcopy.go index b6aaec669..0a01ce0a4 100644 --- a/e2etest/newe2e_task_runazcopy.go +++ b/e2etest/newe2e_task_runazcopy.go @@ -54,13 +54,16 @@ var _ AzCopyStdout = &AzCopyRawStdout{} type AzCopyVerb string const ( // initially supporting a limited set of verbs - AzCopyVerbCopy AzCopyVerb = "copy" - AzCopyVerbSync AzCopyVerb = "sync" - AzCopyVerbRemove AzCopyVerb = "remove" - AzCopyVerbList AzCopyVerb = "list" - AzCopyVerbLogin AzCopyVerb = "login" - AzCopyVerbLogout AzCopyVerb = "logout" - AzCopyVerbJobs AzCopyVerb = "jobs" + AzCopyVerbCopy AzCopyVerb = "copy" + AzCopyVerbSync AzCopyVerb = "sync" + AzCopyVerbRemove AzCopyVerb = "remove" + AzCopyVerbList AzCopyVerb = "list" + AzCopyVerbLogin AzCopyVerb = "login" + AzCopyVerbLoginStatus AzCopyVerb = "login status" + AzCopyVerbLogout AzCopyVerb = "logout" + AzCopyVerbJobsList AzCopyVerb = "jobs list" + AzCopyVerbJobsResume AzCopyVerb = "jobs resume" + AzCopyVerbJobsClean AzCopyVerb = "jobs clean" ) type AzCopyTarget struct { @@ -126,6 +129,8 @@ type AzCopyEnvironment struct { AzureTenantId *string `env:"AZURE_TENANT_ID"` AzureClientId *string `env:"AZURE_CLIENT_ID"` + LoginCacheName *string `env:"AZCOPY_LOGIN_CACHE_NAME"` + InheritEnvironment bool ManualLogin bool @@ -200,17 +205,28 @@ func (c *AzCopyCommand) applyTargetAuth(a Asserter, target ResourceManager) stri if c.Environment.AutoLoginMode == nil && c.Environment.ServicePrincipalAppID == nil && c.Environment.ServicePrincipalClientSecret == nil && c.Environment.AutoLoginTenantID == nil { if GlobalConfig.StaticResources() { - c.Environment.AutoLoginMode = pointerTo("SPN") - oAuthInfo := GlobalConfig.E2EAuthConfig.StaticStgAcctInfo.StaticOAuth - a.AssertNow("At least NEW_E2E_STATIC_APPLICATION_ID and NEW_E2E_STATIC_CLIENT_SECRET must be specified to use OAuth.", Empty{true}, oAuthInfo.ApplicationID, oAuthInfo.ClientSecret) - - c.Environment.ServicePrincipalAppID = &oAuthInfo.ApplicationID - c.Environment.ServicePrincipalClientSecret = &oAuthInfo.ClientSecret - c.Environment.AutoLoginTenantID = common.Iff(oAuthInfo.TenantID != "", &oAuthInfo.TenantID, nil) + staticOauth := GlobalConfig.E2EAuthConfig.StaticStgAcctInfo.StaticOAuth + tenant := staticOauth.TenantID + if useSPN, _, appId, secret := GlobalConfig.GetSPNOptions(); useSPN { + c.Environment.AutoLoginMode = pointerTo("SPN") + a.AssertNow("At least NEW_E2E_STATIC_APPLICATION_ID and NEW_E2E_STATIC_CLIENT_SECRET must be specified to use OAuth.", Empty{true}, appId, secret) + + c.Environment.ServicePrincipalAppID = &appId + c.Environment.ServicePrincipalClientSecret = &secret + c.Environment.AutoLoginTenantID = common.Iff(tenant != "", &tenant, nil) + } else if staticOauth.OAuthSource.PSInherit { + c.Environment.AutoLoginMode = pointerTo("pscred") + c.Environment.AutoLoginTenantID = common.Iff(tenant != "", &tenant, nil) + c.Environment.InheritEnvironment = true + } else if staticOauth.OAuthSource.CLIInherit { + c.Environment.AutoLoginMode = pointerTo("azcli") + c.Environment.AutoLoginTenantID = common.Iff(tenant != "", &tenant, nil) + c.Environment.InheritEnvironment = true + } } else { // oauth should reliably work oAuthInfo := GlobalConfig.E2EAuthConfig.SubscriptionLoginInfo - if oAuthInfo.Environment == AzurePipeline { + if oAuthInfo.Environment == TestEnvironmentAzurePipelines { c.Environment.InheritEnvironment = true c.Environment.AutoLoginTenantID = common.Iff(oAuthInfo.DynamicOAuth.Workload.TenantId != "", &oAuthInfo.DynamicOAuth.Workload.TenantId, nil) c.Environment.AutoLoginMode = pointerTo(common.EAutoLoginType.AzCLI().String()) @@ -223,7 +239,7 @@ func (c *AzCopyCommand) applyTargetAuth(a Asserter, target ResourceManager) stri } } else if c.Environment.AutoLoginMode != nil { oAuthInfo := GlobalConfig.E2EAuthConfig.SubscriptionLoginInfo - if strings.ToLower(*c.Environment.AutoLoginMode) == common.EAutoLoginType.Workload().String() { + if mode := strings.ToLower(*c.Environment.AutoLoginMode); mode == common.EAutoLoginType.Workload().String() { c.Environment.InheritEnvironment = true // Get the value of the AZURE_FEDERATED_TOKEN environment variable token := oAuthInfo.DynamicOAuth.Workload.FederatedToken @@ -242,7 +258,10 @@ func (c *AzCopyCommand) applyTargetAuth(a Asserter, target ResourceManager) stri c.Environment.AzureFederatedTokenFile = pointerTo(file.Name()) c.Environment.AzureTenantId = pointerTo(oAuthInfo.DynamicOAuth.Workload.TenantId) c.Environment.AzureClientId = pointerTo(oAuthInfo.DynamicOAuth.Workload.ClientId) + } else if mode == common.EAutoLoginType.SPN().String() || mode == common.EAutoLoginType.MSI().String() { + c.Environment.InheritEnvironment = true } + } } return target.URI(opts) // Generate like public @@ -267,7 +286,8 @@ func RunAzCopy(a ScenarioAsserter, commandSpec AzCopyCommand) (AzCopyStdout, *Az commandSpec.Environment = &AzCopyEnvironment{} } - out := []string{GlobalConfig.AzCopyExecutableConfig.ExecutablePath, string(commandSpec.Verb)} + out := []string{GlobalConfig.AzCopyExecutableConfig.ExecutablePath} + out = append(out, strings.Split(string(commandSpec.Verb), " ")...) for _, v := range commandSpec.PositionalArgs { out = append(out, v) @@ -326,13 +346,15 @@ func RunAzCopy(a ScenarioAsserter, commandSpec AzCopyCommand) (AzCopyStdout, *Az } case commandSpec.Verb == AzCopyVerbList: out = &AzCopyParsedListStdout{} - case commandSpec.Verb == AzCopyVerbJobs && len(commandSpec.PositionalArgs) != 0 && commandSpec.PositionalArgs[0] == "list": + case commandSpec.Verb == AzCopyVerbJobsList: out = &AzCopyParsedJobsListStdout{} - case commandSpec.Verb == AzCopyVerbJobs && len(commandSpec.PositionalArgs) != 0 && commandSpec.PositionalArgs[0] == "resume": + case commandSpec.Verb == AzCopyVerbJobsResume: out = &AzCopyParsedCopySyncRemoveStdout{ // Resume command treated the same as copy/sync/remove JobPlanFolder: *commandSpec.Environment.JobPlanLocation, LogFolder: *commandSpec.Environment.LogLocation, } + case commandSpec.Verb == AzCopyVerbLoginStatus: + out = &AzCopyParsedLoginStatusStdout{} default: // We don't know how to parse this. out = &AzCopyRawStdout{} diff --git a/e2etest/newe2e_task_runazcopy_parameters.go b/e2etest/newe2e_task_runazcopy_parameters.go index e9dc533aa..70128c926 100644 --- a/e2etest/newe2e_task_runazcopy_parameters.go +++ b/e2etest/newe2e_task_runazcopy_parameters.go @@ -417,6 +417,31 @@ type ListFlags struct { TrailingDot *common.TrailingDotOption `flag:"trailing-dot"` } +type LoginFlags struct { + GlobalFlags + + // Generic flags + TenantID *string `flag:"tenant-id"` + AADEndpoint *string `flag:"aad-endpoint"` + LoginType *common.AutoLoginType `flag:"login-type"` + + // Managed identity + IdentityClientID *string `flag:"identity-client-id"` + IdentityResourceID *string `flag:"identity-resource-id"` + + // SPN + ApplicationID *string `flag:"application-id"` + CertPath *string `flag:"certificate-path"` +} + +type LoginStatusFlags struct { + GlobalFlags + + Tenant *bool `flag:"tenant"` + Endpoint *bool `flag:"endpoint"` + Method *bool `flag:"method"` +} + type WindowsAttribute uint32 const ( diff --git a/e2etest/newe2e_workload_hook.go b/e2etest/newe2e_workload_hook.go index 6fa4bf24b..8d7345d87 100644 --- a/e2etest/newe2e_workload_hook.go +++ b/e2etest/newe2e_workload_hook.go @@ -9,7 +9,7 @@ import ( func WorkloadIdentitySetup(a Asserter) { // Run only in environments that support and are set up for Workload Identity (ex: Azure Pipeline, Azure Kubernetes Service) - if os.Getenv("NEW_E2E_ENVIRONMENT") != "AzurePipeline" { + if os.Getenv("NEW_E2E_ENVIRONMENT") != "TestEnvironmentAzurePipelines" { return // This is OK to skip, because other tests also skip if it isn't present. } diff --git a/e2etest/runner.go b/e2etest/runner.go index d0c7d5bcd..4b07b5384 100644 --- a/e2etest/runner.go +++ b/e2etest/runner.go @@ -312,7 +312,7 @@ func (t *TestRunner) ExecuteAzCopyCommand(operation Operation, src, dst string, env = append(env, "AZCOPY_TENANT_ID="+tenId) } case "", common.EAutoLoginType.AzCLI().String(): - if os.Getenv("NEW_E2E_ENVIRONMENT") == AzurePipeline { + if os.Getenv("NEW_E2E_ENVIRONMENT") == TestEnvironmentAzurePipelines { // We are already logged in with AzCLI in Azure Pipeline } else { tenId, appId, clientSecret := GlobalInputManager{}.GetServicePrincipalAuth() @@ -341,7 +341,7 @@ func (t *TestRunner) ExecuteAzCopyCommand(operation Operation, src, dst string, env = append(env, "AZCOPY_AUTO_LOGIN_TYPE=AzCLI") case "pscred": var script string - if os.Getenv("NEW_E2E_ENVIRONMENT") == AzurePipeline { + if os.Getenv("NEW_E2E_ENVIRONMENT") == TestEnvironmentAzurePipelines { tenId, clientId, token := GlobalInputManager{}.GetWorkloadIdentity() cmd := `Connect-AzAccount -ApplicationId %s -Tenant %s -FederatedToken %s` script = fmt.Sprintf(cmd, clientId, tenId, token) diff --git a/e2etest/zt_newe2e_cli_ps_persist_test.go b/e2etest/zt_newe2e_cli_ps_persist_test.go new file mode 100644 index 000000000..254847d90 --- /dev/null +++ b/e2etest/zt_newe2e_cli_ps_persist_test.go @@ -0,0 +1,72 @@ +package e2etest + +import ( + "fmt" + "github.com/Azure/azure-storage-azcopy/v10/common" +) + +func init() { + suiteManager.RegisterSuite(&TokenPersistenceSuite{}) +} + +type TokenPersistenceSuite struct{} + +func (*TokenPersistenceSuite) Scenario_InheritCred_Persist(a *ScenarioVariationManager) { + credSource := ResolveVariation(a, []common.AutoLoginType{common.EAutoLoginType.PsCred(), common.EAutoLoginType.AzCLI()}) + withSpecifiedTenantID := NamedResolveVariation(a, map[string]bool{ + "-withTenantID": true, + "": false, + }) + cfgTenantID := GlobalConfig.GetTenantID() + + azcopyEnv := &AzCopyEnvironment{ + LoginCacheName: pointerTo(fmt.Sprintf("AzCopyPersist%sTest", credSource.String())), + InheritEnvironment: true, // we want the executables in PATH + ManualLogin: true, + } + + _, _ = RunAzCopy(a, + AzCopyCommand{ + Verb: AzCopyVerbLogin, + Flags: LoginFlags{ + LoginType: &credSource, + TenantID: common.Iff(withSpecifiedTenantID && cfgTenantID != "", &cfgTenantID, nil), + }, + Environment: azcopyEnv, + }) + + loginStatus, _ := RunAzCopy(a, + AzCopyCommand{ + Verb: AzCopyVerbLoginStatus, + Flags: LoginStatusFlags{ + Method: pointerTo(true), + Endpoint: pointerTo(true), + Tenant: pointerTo(true), + }, + Environment: azcopyEnv, + }) + + parsedStdout, ok := loginStatus.(*AzCopyParsedLoginStatusStdout) + a.AssertNow("must be AzCopyParsedLoginStatusStdout", Equal{}, ok, true) + + if !a.Dryrun() { + status := parsedStdout.status + a.Assert("Login check failed", Equal{}, true, status.Valid) + + a.Assert("Tenant not returned", Not{IsNil{}}, status.TenantID) // Let's just do a little extra testing while we're at it, kill two birds with one stone. + if withSpecifiedTenantID && status.TenantID != nil && cfgTenantID != "" { + a.Assert("Tenant does not match", Equal{}, cfgTenantID, *status.TenantID) + } + a.Assert("Endpoint not returned", Not{IsNil{}}, status.AADEndpoint) + a.Assert("Auth mechanism not returned", Not{IsNil{}}, status.AuthMethod) + if status.AuthMethod != nil { + a.Assert("Incorrect auth mechanism", Equal{}, *status.AuthMethod, credSource.String()) + } + } + + _, _ = RunAzCopy(a, + AzCopyCommand{ + Verb: AzCopyVerbLogout, + Environment: azcopyEnv, + }) +} diff --git a/e2etest/zt_newe2e_jobs_clean_test.go b/e2etest/zt_newe2e_jobs_clean_test.go index a167f8159..a6f014cee 100644 --- a/e2etest/zt_newe2e_jobs_clean_test.go +++ b/e2etest/zt_newe2e_jobs_clean_test.go @@ -34,9 +34,8 @@ func (s *JobsCleanSuite) Scenario_JobsCleanBasic(svm *ScenarioVariationManager) jobsCleanOutput, _ := RunAzCopy( svm, AzCopyCommand{ - Verb: AzCopyVerbJobs, - PositionalArgs: []string{"clean"}, - ShouldFail: false, + Verb: AzCopyVerbJobsClean, + ShouldFail: false, Environment: &AzCopyEnvironment{ LogLocation: &logsDir, JobPlanLocation: &jobPlanDir, diff --git a/e2etest/zt_newe2e_jobs_list_test.go b/e2etest/zt_newe2e_jobs_list_test.go index d2fe65e41..31820870d 100644 --- a/e2etest/zt_newe2e_jobs_list_test.go +++ b/e2etest/zt_newe2e_jobs_list_test.go @@ -11,10 +11,9 @@ func (s *JobsListSuite) Scenario_JobsListBasic(svm *ScenarioVariationManager) { jobsListOutput, _ := RunAzCopy( svm, AzCopyCommand{ - Verb: AzCopyVerbJobs, - PositionalArgs: []string{"list"}, - Stdout: &AzCopyParsedJobsListStdout{}, - Flags: ListFlags{}, + Verb: AzCopyVerbJobsList, + Stdout: &AzCopyParsedJobsListStdout{}, + Flags: ListFlags{}, }) ValidateJobsListOutput(svm, jobsListOutput, 0) } diff --git a/e2etest/zt_newe2e_workload_test.go b/e2etest/zt_newe2e_workload_test.go index 6d799fae7..96f76e0e6 100644 --- a/e2etest/zt_newe2e_workload_test.go +++ b/e2etest/zt_newe2e_workload_test.go @@ -15,7 +15,7 @@ type WorkloadIdentitySuite struct{} // Run only in environments that support and are set up for Workload Identity (ex: Azure Pipeline, Azure Kubernetes Service) func (s *WorkloadIdentitySuite) Scenario_SingleFileUploadDownloadWorkloadIdentity(svm *ScenarioVariationManager) { // Run only in environments that support and are set up for Workload Identity (ex: Azure Pipeline, Azure Kubernetes Service) - if os.Getenv("NEW_E2E_ENVIRONMENT") != "AzurePipeline" { + if os.Getenv("NEW_E2E_ENVIRONMENT") != "TestEnvironmentAzurePipelines" { svm.Skip("Workload Identity is only supported in environments specifically set up for it.") } azCopyVerb := ResolveVariation(svm, []AzCopyVerb{AzCopyVerbCopy, AzCopyVerbSync}) // Calculate verb early to create the destination object early