From 8c4413e89ff27f0cfd1f426be65e92c5c49f0b02 Mon Sep 17 00:00:00 2001 From: Manika Joshi Date: Mon, 3 Mar 2025 21:50:01 -0800 Subject: [PATCH 1/8] fixed SAS at container level --- .../hadoop/fs/azurebfs/AbfsConfiguration.java | 25 +++++++-- .../fs/azurebfs/AzureBlobFileSystemStore.java | 2 +- .../azurebfs/constants/ConfigurationKeys.java | 4 ++ .../azurebfs/AbstractAbfsIntegrationTest.java | 2 +- .../ITestAzureBlobFileSystemChooseSAS.java | 53 +++++++++++++++++++ .../azurebfs/utils/ServiceSASGenerator.java | 12 ++++- 6 files changed, 91 insertions(+), 7 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java index 6a51f8d902038..a326f3061af80 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java @@ -88,6 +88,7 @@ public class AbfsConfiguration{ private final Configuration rawConfig; private final String accountName; + private final String fsName; // Service type identified from URL used to initialize FileSystem. private final AbfsServiceType fsConfiguredServiceType; private final boolean isSecure; @@ -457,11 +458,13 @@ public class AbfsConfiguration{ */ public AbfsConfiguration(final Configuration rawConfig, String accountName, + String fsName, AbfsServiceType fsConfiguredServiceType) throws IllegalAccessException, IOException { this.rawConfig = ProviderUtils.excludeIncompatibleCredentialProviders( rawConfig, AzureBlobFileSystem.class); this.accountName = accountName; + this.fsName = fsName; this.fsConfiguredServiceType = fsConfiguredServiceType; this.isSecure = getBoolean(FS_AZURE_SECURE_MODE, false); @@ -493,7 +496,7 @@ public AbfsConfiguration(final Configuration rawConfig, */ public AbfsConfiguration(final Configuration rawConfig, String accountName) throws IllegalAccessException, IOException { - this(rawConfig, accountName, AbfsServiceType.DFS); + this(rawConfig, accountName, EMPTY_STRING, AbfsServiceType.DFS); } /** @@ -589,6 +592,16 @@ public String accountConf(String key) { return key + "." + accountName; } + /** + * Appends the container, account name to a configuration key yielding the + * container-specific form. + * @param key Account-agnostic configuration key + * @return Container-specific configuration key + */ + public String containerConf(String key) { + return key + "." + fsName + "." + accountName; + } + /** * Returns the account-specific value if it exists, then looks for an * account-agnostic value. @@ -644,16 +657,20 @@ public int getInt(String key, int defaultValue) { } /** - * Returns the account-specific password in string form if it exists, then + * Returns the container-specific password if it exists, + * else searches for the account-specific password, else finally * looks for an account-agnostic value. * @param key Account-agnostic configuration key * @return value in String form if one exists, else null * @throws IOException if parsing fails. */ public String getPasswordString(String key) throws IOException { - char[] passchars = rawConfig.getPassword(accountConf(key)); + char[] passchars = rawConfig.getPassword(containerConf(key)); if (passchars == null) { - passchars = rawConfig.getPassword(key); + passchars = rawConfig.getPassword(accountConf(key)); + if(passchars == null){ + passchars = rawConfig.getPassword(key); + } } if (passchars != null) { return new String(passchars); diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java index 867edfd1438fe..f7427d3091998 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java @@ -229,7 +229,7 @@ public AzureBlobFileSystemStore( try { this.abfsConfiguration = new AbfsConfiguration(abfsStoreBuilder.configuration, - accountName, getAbfsServiceTypeFromUrl()); + accountName, fileSystemName, getAbfsServiceTypeFromUrl()); } catch (IllegalAccessException exception) { throw new FileSystemOperationUnhandledException(exception); } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java index 3742361b48445..b4e0fa6c2eca1 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java @@ -322,6 +322,10 @@ public static String accountProperty(String property, String account) { return property + "." + account; } + public static String containerProperty(String property, String FsName, String account) { + return property + "." + FsName + "." + account; + } + public static final String FS_AZURE_ENABLE_DELEGATION_TOKEN = "fs.azure.enable.delegation.token"; public static final String FS_AZURE_DELEGATION_TOKEN_PROVIDER_TYPE = "fs.azure.delegation.token.provider.type"; diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java index e31df5eec65bb..450dfaa3a52c7 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java @@ -117,7 +117,7 @@ protected AbstractAbfsIntegrationTest() throws Exception { final String abfsUrl = this.getFileSystemName() + "@" + this.getAccountName(); URI defaultUri = null; - abfsConfig = new AbfsConfiguration(rawConfig, accountName, identifyAbfsServiceTypeFromUrl(abfsUrl)); + abfsConfig = new AbfsConfiguration(rawConfig, accountName, getFileSystemName(), identifyAbfsServiceTypeFromUrl(abfsUrl)); authType = abfsConfig.getEnum(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, AuthType.SharedKey); assumeValidAuthConfigsPresent(); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java index dc40241a87576..8e8c522e4f254 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java @@ -32,11 +32,13 @@ import org.apache.hadoop.fs.azurebfs.services.AuthType; import org.apache.hadoop.fs.azurebfs.services.FixedSASTokenProvider; import org.apache.hadoop.fs.azurebfs.utils.AccountSASGenerator; +import org.apache.hadoop.fs.azurebfs.utils.ServiceSASGenerator; import org.apache.hadoop.fs.azurebfs.utils.Base64; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_SAS_FIXED_TOKEN; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_SAS_TOKEN_PROVIDER_TYPE; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.accountProperty; +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.containerProperty; import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_TEST_APP_ID; import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_TEST_APP_SECRET; import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_TEST_APP_SERVICE_PRINCIPAL_OBJECT_ID; @@ -50,6 +52,7 @@ public class ITestAzureBlobFileSystemChooseSAS extends AbstractAbfsIntegrationTest{ private String accountSAS = null; + private String containerSAS = null; private static final String TEST_PATH = "testPath"; /** @@ -69,6 +72,7 @@ public void setup() throws Exception { super.setup(); createFilesystemWithTestFileForSASTests(new Path(TEST_PATH)); generateAccountSAS(); + generateContainerSAS(); } /** @@ -85,6 +89,22 @@ private void generateAccountSAS() throws AzureBlobFileSystemException { accountSAS = configAccountSASGenerator.getAccountSAS(getAccountName()); } + /** + * Generates a Container SAS Token using the Account Shared Key to be used as a fixed SAS Token. + * Container SAS used here will have only read permissions to resources. + * This will be used by individual tests to set in the configurations. + * @throws AzureBlobFileSystemException + */ + private void generateContainerSAS() throws AzureBlobFileSystemException { + final byte[] accountKey = Base64.decode( + getConfiguration().getStorageAccountKey()); + ServiceSASGenerator configServiceSASGenerator = new ServiceSASGenerator( + accountKey); + // Setting only read permissions. + configServiceSASGenerator.setPermissions("r"); + containerSAS = configServiceSASGenerator.getContainerSASWithFullControl( + getAccountName(), getFileSystemName()); + } /** * Tests the scenario where both the custom SASTokenProvider and a fixed SAS token are configured. * Custom implementation of SASTokenProvider class should be chosen and User Delegation SAS should be used. @@ -126,6 +146,38 @@ public void testBothProviderFixedTokenConfigured() throws Exception { } } + /** + * Tests the implementation sequence if all fixed SAS configs are set. + * The expected sequence is Container Specific Fixed SAS, Account Specific Fixed SAS, Account Agnostic Fixed SAS. + * @throws IOException + */ + @Test + public void testFixedTokenPreference() throws Exception { + AbfsConfiguration testAbfsConfig = new AbfsConfiguration( + getRawConfiguration(), this.getAccountName(), this.getFileSystemName(), getAbfsServiceType()); + + // setting all types of Fixed SAS configs (container-specific, account-specific, account-agnostic) + removeAnyPresetConfiguration(testAbfsConfig); + testAbfsConfig.set(containerProperty(FS_AZURE_SAS_FIXED_TOKEN, this.getFileSystemName(), this.getAccountName()), containerSAS); + testAbfsConfig.set(accountProperty(FS_AZURE_SAS_FIXED_TOKEN, this.getAccountName()), accountSAS); + testAbfsConfig.set(FS_AZURE_SAS_FIXED_TOKEN, accountSAS); + String ContainerSASExpected = testAbfsConfig.getSASTokenProvider().getSASToken(this.getAccountName(), this.getFileSystemName(), getMethodName(), "read"); + + // Assert that Container Specific Fixed SAS is used + Assertions.assertThat(ContainerSASExpected).contains("sr=c"); + + // Assert that Account Specific Fixed SAS is used if container SAS isn't set + testAbfsConfig.unset(containerProperty(FS_AZURE_SAS_FIXED_TOKEN, this.getFileSystemName(), this.getAccountName())); + String AccountSASExpected = testAbfsConfig.getSASTokenProvider().getSASToken(this.getAccountName(), this.getFileSystemName(), getMethodName(), "read"); + Assertions.assertThat(AccountSASExpected).contains("ss=bf"); + + //Assert that Account-Agnostic fixed SAS is used if no other fixed SAS configs are set. + // The token is the same as the Account Specific Fixed SAS. + testAbfsConfig.unset(FS_AZURE_SAS_FIXED_TOKEN); + String AccountAgnosticSASExpected = testAbfsConfig.getSASTokenProvider().getSASToken(this.getAccountName(), this.getFileSystemName(), getMethodName(), "read"); + Assertions.assertThat(AccountAgnosticSASExpected).contains("ss=bf"); + } + /** * Tests the scenario where only the fixed token is configured, and no token provider class is set. * Account SAS Token configured as fixed SAS should be used. @@ -189,5 +241,6 @@ private void removeAnyPresetConfiguration(AbfsConfiguration testAbfsConfig) { testAbfsConfig.unset(FS_AZURE_SAS_FIXED_TOKEN); testAbfsConfig.unset(accountProperty(FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, this.getAccountName())); testAbfsConfig.unset(accountProperty(FS_AZURE_SAS_FIXED_TOKEN, this.getAccountName())); + testAbfsConfig.unset(containerProperty(FS_AZURE_SAS_FIXED_TOKEN, this.getFileSystemName(), this.getAccountName())); } } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/ServiceSASGenerator.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/ServiceSASGenerator.java index 0ae5239e8f2a5..2f2600ef325a7 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/ServiceSASGenerator.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/ServiceSASGenerator.java @@ -37,10 +37,11 @@ public ServiceSASGenerator(byte[] accountKey) { super(accountKey); } + private String permissions = "racwdl"; public String getContainerSASWithFullControl(String accountName, String containerName) throws InvalidConfigurationValueException { accountName = getCanonicalAccountName(accountName); - String sp = "rcwdl"; + String sp = permissions; String sv = AuthenticationVersion.Feb20.toString(); String sr = "c"; String st = ISO_8601_FORMATTER.format(Instant.now().minus(FIVE_MINUTES)); @@ -96,4 +97,13 @@ private String computeSignatureForSAS(String sp, String st, String se, String sv LOG.debug("Service SAS stringToSign: " + stringToSign.replace("\n", ".")); return computeHmac256(stringToSign); } + + /** + * By default, Container SAS has all the available permissions. Use this to + * override the default permissions and set as per the requirements. + * @param permissions + */ + public void setPermissions(final String permissions) { + this.permissions = permissions; + } } From 4a182acf271e2c390663440d7e8e02d43b09d412 Mon Sep 17 00:00:00 2001 From: Manika Joshi Date: Tue, 4 Mar 2025 01:46:54 -0800 Subject: [PATCH 2/8] passing yetus checks --- .../hadoop/fs/azurebfs/AbfsConfiguration.java | 3 ++- .../azurebfs/constants/ConfigurationKeys.java | 4 ++-- .../ITestAzureBlobFileSystemChooseSAS.java | 18 +++++++++++------- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java index a326f3061af80..53783175cea4a 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java @@ -452,6 +452,7 @@ public class AbfsConfiguration{ * Constructor for AbfsConfiguration for specified service type. * @param rawConfig used to initialize the configuration. * @param accountName the name of the azure storage account. + * @param fsName the name of the file system (container name). * @param fsConfiguredServiceType service type configured for the file system. * @throws IllegalAccessException if the field is not accessible. * @throws IOException if an I/O error occurs. @@ -666,7 +667,7 @@ public int getInt(String key, int defaultValue) { */ public String getPasswordString(String key) throws IOException { char[] passchars = rawConfig.getPassword(containerConf(key)); - if (passchars == null) { + if(passchars == null) { passchars = rawConfig.getPassword(accountConf(key)); if(passchars == null){ passchars = rawConfig.getPassword(key); diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java index b4e0fa6c2eca1..fa91e2e6891e3 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java @@ -322,8 +322,8 @@ public static String accountProperty(String property, String account) { return property + "." + account; } - public static String containerProperty(String property, String FsName, String account) { - return property + "." + FsName + "." + account; + public static String containerProperty(String property, String fsName, String account) { + return property + "." + fsName + "." + account; } public static final String FS_AZURE_ENABLE_DELEGATION_TOKEN = "fs.azure.enable.delegation.token"; diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java index 8e8c522e4f254..cdf3d1a689e58 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java @@ -146,6 +146,13 @@ public void testBothProviderFixedTokenConfigured() throws Exception { } } + /** + * Helper method to get the Fixed SAS token value + */ + private String getFixedSASToken(AbfsConfiguration config) throws Exception { + return config.getSASTokenProvider().getSASToken(this.getAccountName(), this.getFileSystemName(), getMethodName(), "read"); + } + /** * Tests the implementation sequence if all fixed SAS configs are set. * The expected sequence is Container Specific Fixed SAS, Account Specific Fixed SAS, Account Agnostic Fixed SAS. @@ -161,21 +168,18 @@ public void testFixedTokenPreference() throws Exception { testAbfsConfig.set(containerProperty(FS_AZURE_SAS_FIXED_TOKEN, this.getFileSystemName(), this.getAccountName()), containerSAS); testAbfsConfig.set(accountProperty(FS_AZURE_SAS_FIXED_TOKEN, this.getAccountName()), accountSAS); testAbfsConfig.set(FS_AZURE_SAS_FIXED_TOKEN, accountSAS); - String ContainerSASExpected = testAbfsConfig.getSASTokenProvider().getSASToken(this.getAccountName(), this.getFileSystemName(), getMethodName(), "read"); // Assert that Container Specific Fixed SAS is used - Assertions.assertThat(ContainerSASExpected).contains("sr=c"); + Assertions.assertThat(getFixedSASToken(testAbfsConfig)).contains("sr=c"); // Assert that Account Specific Fixed SAS is used if container SAS isn't set testAbfsConfig.unset(containerProperty(FS_AZURE_SAS_FIXED_TOKEN, this.getFileSystemName(), this.getAccountName())); - String AccountSASExpected = testAbfsConfig.getSASTokenProvider().getSASToken(this.getAccountName(), this.getFileSystemName(), getMethodName(), "read"); - Assertions.assertThat(AccountSASExpected).contains("ss=bf"); + Assertions.assertThat(getFixedSASToken(testAbfsConfig)).contains("ss=bf"); //Assert that Account-Agnostic fixed SAS is used if no other fixed SAS configs are set. // The token is the same as the Account Specific Fixed SAS. - testAbfsConfig.unset(FS_AZURE_SAS_FIXED_TOKEN); - String AccountAgnosticSASExpected = testAbfsConfig.getSASTokenProvider().getSASToken(this.getAccountName(), this.getFileSystemName(), getMethodName(), "read"); - Assertions.assertThat(AccountAgnosticSASExpected).contains("ss=bf"); + testAbfsConfig.unset(accountProperty(FS_AZURE_SAS_FIXED_TOKEN, this.getAccountName())); + Assertions.assertThat(getFixedSASToken(testAbfsConfig)).contains("ss=bf"); } /** From 6915d0e5cf1f4dc21cbd47b5fcdd7f3f173b25e8 Mon Sep 17 00:00:00 2001 From: Manika Joshi Date: Sun, 9 Mar 2025 21:06:38 -0700 Subject: [PATCH 3/8] comment suggestions --- .../hadoop/fs/azurebfs/AbfsConfiguration.java | 37 +++++++++++++------ .../azurebfs/AbstractAbfsIntegrationTest.java | 2 +- .../ITestAzureBlobFileSystemChooseSAS.java | 4 +- 3 files changed, 29 insertions(+), 14 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java index 53783175cea4a..0810ef67caf1f 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java @@ -88,7 +88,7 @@ public class AbfsConfiguration{ private final Configuration rawConfig; private final String accountName; - private final String fsName; + private String fsName; // Service type identified from URL used to initialize FileSystem. private final AbfsServiceType fsConfiguredServiceType; private final boolean isSecure; @@ -452,20 +452,17 @@ public class AbfsConfiguration{ * Constructor for AbfsConfiguration for specified service type. * @param rawConfig used to initialize the configuration. * @param accountName the name of the azure storage account. - * @param fsName the name of the file system (container name). * @param fsConfiguredServiceType service type configured for the file system. * @throws IllegalAccessException if the field is not accessible. * @throws IOException if an I/O error occurs. */ public AbfsConfiguration(final Configuration rawConfig, String accountName, - String fsName, AbfsServiceType fsConfiguredServiceType) throws IllegalAccessException, IOException { this.rawConfig = ProviderUtils.excludeIncompatibleCredentialProviders( rawConfig, AzureBlobFileSystem.class); this.accountName = accountName; - this.fsName = fsName; this.fsConfiguredServiceType = fsConfiguredServiceType; this.isSecure = getBoolean(FS_AZURE_SECURE_MODE, false); @@ -488,6 +485,24 @@ public AbfsConfiguration(final Configuration rawConfig, } } + /** + * Constructor for AbfsConfiguration for retrieve the FsName. + * @param rawConfig used to initialize the configuration. + * @param accountName the name of the azure storage account. + * @param fsName the name of the file system (container name). + * @param fsConfiguredServiceType service type configured for the file system. + * @throws IllegalAccessException if the field is not accessible. + * @throws IOException if an I/O error occurs. + */ + public AbfsConfiguration(final Configuration rawConfig, + String accountName, + String fsName, + AbfsServiceType fsConfiguredServiceType) + throws IllegalAccessException, IOException { + this(rawConfig, accountName, fsConfiguredServiceType); + this.fsName = fsName; + } + /** * Constructor for AbfsConfiguration for default service type i.e. DFS. * @param rawConfig used to initialize the configuration. @@ -497,7 +512,7 @@ public AbfsConfiguration(final Configuration rawConfig, */ public AbfsConfiguration(final Configuration rawConfig, String accountName) throws IllegalAccessException, IOException { - this(rawConfig, accountName, EMPTY_STRING, AbfsServiceType.DFS); + this(rawConfig, accountName, AbfsServiceType.DFS); } /** @@ -666,13 +681,11 @@ public int getInt(String key, int defaultValue) { * @throws IOException if parsing fails. */ public String getPasswordString(String key) throws IOException { - char[] passchars = rawConfig.getPassword(containerConf(key)); - if(passchars == null) { - passchars = rawConfig.getPassword(accountConf(key)); - if(passchars == null){ - passchars = rawConfig.getPassword(key); - } - } + char[] passchars = rawConfig.getPassword(containerConf(key)) != null ? + rawConfig.getPassword(containerConf(key)) : + rawConfig.getPassword(accountConf(key)) != null ? + rawConfig.getPassword(accountConf(key)) : + rawConfig.getPassword(key); if (passchars != null) { return new String(passchars); } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java index 450dfaa3a52c7..e31df5eec65bb 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java @@ -117,7 +117,7 @@ protected AbstractAbfsIntegrationTest() throws Exception { final String abfsUrl = this.getFileSystemName() + "@" + this.getAccountName(); URI defaultUri = null; - abfsConfig = new AbfsConfiguration(rawConfig, accountName, getFileSystemName(), identifyAbfsServiceTypeFromUrl(abfsUrl)); + abfsConfig = new AbfsConfiguration(rawConfig, accountName, identifyAbfsServiceTypeFromUrl(abfsUrl)); authType = abfsConfig.getEnum(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, AuthType.SharedKey); assumeValidAuthConfigsPresent(); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java index cdf3d1a689e58..f27479e6cbfa7 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java @@ -150,7 +150,9 @@ public void testBothProviderFixedTokenConfigured() throws Exception { * Helper method to get the Fixed SAS token value */ private String getFixedSASToken(AbfsConfiguration config) throws Exception { - return config.getSASTokenProvider().getSASToken(this.getAccountName(), this.getFileSystemName(), getMethodName(), "read"); + String readPermission = "read"; + return config.getSASTokenProvider().getSASToken(this.getAccountName(), this.getFileSystemName(), getMethodName(), + readPermission); } /** From f4c64584db0c4f12ba7e580a984fd238ecf47857 Mon Sep 17 00:00:00 2001 From: Manika Joshi Date: Tue, 11 Mar 2025 00:29:33 -0700 Subject: [PATCH 4/8] changes --- .../fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java index f27479e6cbfa7..97e989e94cddb 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java @@ -54,6 +54,7 @@ public class ITestAzureBlobFileSystemChooseSAS extends AbstractAbfsIntegrationTe private String accountSAS = null; private String containerSAS = null; private static final String TEST_PATH = "testPath"; + private static final String readPermission = "read"; /** * To differentiate which SASTokenProvider was used we will use different type of SAS Tokens. @@ -150,9 +151,10 @@ public void testBothProviderFixedTokenConfigured() throws Exception { * Helper method to get the Fixed SAS token value */ private String getFixedSASToken(AbfsConfiguration config) throws Exception { - String readPermission = "read"; - return config.getSASTokenProvider().getSASToken(this.getAccountName(), this.getFileSystemName(), getMethodName(), - readPermission); + return config.getSASTokenProvider() + .getSASToken(this.getAccountName(), this.getFileSystemName(), + getMethodName(), + readPermission); } /** From c51e38226ae3a2d1f53389c1bdd2e5d605cbc6e6 Mon Sep 17 00:00:00 2001 From: Manika Joshi Date: Tue, 11 Mar 2025 07:41:04 -0700 Subject: [PATCH 5/8] minor changes --- .../hadoop/fs/azurebfs/AbfsConfiguration.java | 5 ++-- .../azurebfs/constants/ConfigurationKeys.java | 6 ++-- .../ITestAzureBlobFileSystemChooseSAS.java | 30 ++++++++++++++----- 3 files changed, 29 insertions(+), 12 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java index 0810ef67caf1f..b2a2396348d15 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java @@ -75,6 +75,7 @@ import org.apache.hadoop.util.ReflectionUtils; import static org.apache.hadoop.fs.FileSystem.FS_DEFAULT_NAME_KEY; +import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.DOT; import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.EMPTY_STRING; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.*; import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.*; @@ -605,7 +606,7 @@ public String getClientCorrelationId() { * @return Account-specific configuration key */ public String accountConf(String key) { - return key + "." + accountName; + return key + DOT + accountName; } /** @@ -615,7 +616,7 @@ public String accountConf(String key) { * @return Container-specific configuration key */ public String containerConf(String key) { - return key + "." + fsName + "." + accountName; + return key + DOT + fsName + DOT + accountName; } /** diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java index fa91e2e6891e3..40095c7a802e8 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java @@ -22,6 +22,8 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.FileSystem; +import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.DOT; + /** * Responsible to keep all the Azure Blob File System configurations keys in Hadoop configuration file. */ @@ -319,11 +321,11 @@ public final class ConfigurationKeys { public static final String FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION = "fs.azure.enable.checksum.validation"; public static String accountProperty(String property, String account) { - return property + "." + account; + return property + DOT + account; } public static String containerProperty(String property, String fsName, String account) { - return property + "." + fsName + "." + account; + return property + DOT + fsName + DOT + account; } public static final String FS_AZURE_ENABLE_DELEGATION_TOKEN = "fs.azure.enable.delegation.token"; diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java index 97e989e94cddb..1d26d1a91e934 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java @@ -165,25 +165,39 @@ private String getFixedSASToken(AbfsConfiguration config) throws Exception { @Test public void testFixedTokenPreference() throws Exception { AbfsConfiguration testAbfsConfig = new AbfsConfiguration( - getRawConfiguration(), this.getAccountName(), this.getFileSystemName(), getAbfsServiceType()); + getRawConfiguration(), this.getAccountName(), this.getFileSystemName(), + getAbfsServiceType()); // setting all types of Fixed SAS configs (container-specific, account-specific, account-agnostic) removeAnyPresetConfiguration(testAbfsConfig); - testAbfsConfig.set(containerProperty(FS_AZURE_SAS_FIXED_TOKEN, this.getFileSystemName(), this.getAccountName()), containerSAS); - testAbfsConfig.set(accountProperty(FS_AZURE_SAS_FIXED_TOKEN, this.getAccountName()), accountSAS); + testAbfsConfig.set( + containerProperty(FS_AZURE_SAS_FIXED_TOKEN, this.getFileSystemName(), + this.getAccountName()), containerSAS); + testAbfsConfig.set( + accountProperty(FS_AZURE_SAS_FIXED_TOKEN, this.getAccountName()), + accountSAS); testAbfsConfig.set(FS_AZURE_SAS_FIXED_TOKEN, accountSAS); // Assert that Container Specific Fixed SAS is used - Assertions.assertThat(getFixedSASToken(testAbfsConfig)).contains("sr=c"); + Assertions.assertThat(getFixedSASToken(testAbfsConfig)) + .describedAs("Container-specific fixed SAS should've been used.") + .contains("sr=c"); // Assert that Account Specific Fixed SAS is used if container SAS isn't set - testAbfsConfig.unset(containerProperty(FS_AZURE_SAS_FIXED_TOKEN, this.getFileSystemName(), this.getAccountName())); - Assertions.assertThat(getFixedSASToken(testAbfsConfig)).contains("ss=bf"); + testAbfsConfig.unset( + containerProperty(FS_AZURE_SAS_FIXED_TOKEN, this.getFileSystemName(), + this.getAccountName())); + Assertions.assertThat(getFixedSASToken(testAbfsConfig)) + .describedAs("Account-specific fixed SAS should've been used.") + .contains("ss=bf"); //Assert that Account-Agnostic fixed SAS is used if no other fixed SAS configs are set. // The token is the same as the Account Specific Fixed SAS. - testAbfsConfig.unset(accountProperty(FS_AZURE_SAS_FIXED_TOKEN, this.getAccountName())); - Assertions.assertThat(getFixedSASToken(testAbfsConfig)).contains("ss=bf"); + testAbfsConfig.unset( + accountProperty(FS_AZURE_SAS_FIXED_TOKEN, this.getAccountName())); + Assertions.assertThat(getFixedSASToken(testAbfsConfig)) + .describedAs("Account-agnostic fixed SAS should've been used.") + .contains("ss=bf"); } /** From a800cda005673d72a4e978bb9b733bf615887909 Mon Sep 17 00:00:00 2001 From: Manika Joshi Date: Wed, 12 Mar 2025 00:15:03 -0700 Subject: [PATCH 6/8] separate account-agnostic sas token --- .../ITestAzureBlobFileSystemChooseSAS.java | 32 ++++++++++++++----- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java index 1d26d1a91e934..e9b1a27278f24 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java @@ -35,6 +35,7 @@ import org.apache.hadoop.fs.azurebfs.utils.ServiceSASGenerator; import org.apache.hadoop.fs.azurebfs.utils.Base64; +import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.EMPTY_STRING; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_SAS_FIXED_TOKEN; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_SAS_TOKEN_PROVIDER_TYPE; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.accountProperty; @@ -53,8 +54,8 @@ public class ITestAzureBlobFileSystemChooseSAS extends AbstractAbfsIntegrationTe private String accountSAS = null; private String containerSAS = null; + private String accountAgnosticSAS = null; private static final String TEST_PATH = "testPath"; - private static final String readPermission = "read"; /** * To differentiate which SASTokenProvider was used we will use different type of SAS Tokens. @@ -73,6 +74,7 @@ public void setup() throws Exception { super.setup(); createFilesystemWithTestFileForSASTests(new Path(TEST_PATH)); generateAccountSAS(); + generateAccountAgnosticSAS(); generateContainerSAS(); } @@ -90,6 +92,21 @@ private void generateAccountSAS() throws AzureBlobFileSystemException { accountSAS = configAccountSASGenerator.getAccountSAS(getAccountName()); } + /** + * Generates an Account SAS Token (for account-agnostic config) using the Account Shared Key to + * be used as a fixed SAS Token. + * Account SAS used here will only have write permissions to resources. + * This will be used by individual tests to set in the configurations. + * @throws AzureBlobFileSystemException + */ + private void generateAccountAgnosticSAS() throws AzureBlobFileSystemException { + final String accountKey = getConfiguration().getStorageAccountKey(); + AccountSASGenerator configAccountSASGenerator = new AccountSASGenerator(Base64.decode(accountKey)); + // Setting only write permissions. + configAccountSASGenerator.setPermissions("w"); + accountAgnosticSAS = configAccountSASGenerator.getAccountSAS(getAccountName()); + } + /** * Generates a Container SAS Token using the Account Shared Key to be used as a fixed SAS Token. * Container SAS used here will have only read permissions to resources. @@ -154,7 +171,7 @@ private String getFixedSASToken(AbfsConfiguration config) throws Exception { return config.getSASTokenProvider() .getSASToken(this.getAccountName(), this.getFileSystemName(), getMethodName(), - readPermission); + EMPTY_STRING); } /** @@ -163,7 +180,7 @@ private String getFixedSASToken(AbfsConfiguration config) throws Exception { * @throws IOException */ @Test - public void testFixedTokenPreference() throws Exception { + public void testFixedSASTokenProviderPreference() throws Exception { AbfsConfiguration testAbfsConfig = new AbfsConfiguration( getRawConfiguration(), this.getAccountName(), this.getFileSystemName(), getAbfsServiceType()); @@ -176,12 +193,12 @@ public void testFixedTokenPreference() throws Exception { testAbfsConfig.set( accountProperty(FS_AZURE_SAS_FIXED_TOKEN, this.getAccountName()), accountSAS); - testAbfsConfig.set(FS_AZURE_SAS_FIXED_TOKEN, accountSAS); + testAbfsConfig.set(FS_AZURE_SAS_FIXED_TOKEN, accountAgnosticSAS); // Assert that Container Specific Fixed SAS is used Assertions.assertThat(getFixedSASToken(testAbfsConfig)) .describedAs("Container-specific fixed SAS should've been used.") - .contains("sr=c"); + .isEqualTo(containerSAS); // Assert that Account Specific Fixed SAS is used if container SAS isn't set testAbfsConfig.unset( @@ -189,15 +206,14 @@ public void testFixedTokenPreference() throws Exception { this.getAccountName())); Assertions.assertThat(getFixedSASToken(testAbfsConfig)) .describedAs("Account-specific fixed SAS should've been used.") - .contains("ss=bf"); + .isEqualTo(accountSAS); //Assert that Account-Agnostic fixed SAS is used if no other fixed SAS configs are set. - // The token is the same as the Account Specific Fixed SAS. testAbfsConfig.unset( accountProperty(FS_AZURE_SAS_FIXED_TOKEN, this.getAccountName())); Assertions.assertThat(getFixedSASToken(testAbfsConfig)) .describedAs("Account-agnostic fixed SAS should've been used.") - .contains("ss=bf"); + .isEqualTo(accountAgnosticSAS); } /** From dfa397bd59e47f820eddeb597ba4fe52cce7ec14 Mon Sep 17 00:00:00 2001 From: Manika Joshi Date: Thu, 20 Mar 2025 21:51:53 -0700 Subject: [PATCH 7/8] changing md files --- .../hadoop-azure/src/site/markdown/abfs.md | 39 +++++++++++++------ 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md b/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md index fdf366f95d34b..0bb9518de289b 100644 --- a/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md +++ b/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md @@ -742,7 +742,7 @@ ADLS Gen 2 storage accounts. #### Using Account/Service SAS with ABFS - **Description**: ABFS allows user to use Account/Service SAS for authenticating -requests. User can specify them as fixed SAS Token to be used across all the requests. + requests. User can specify them as fixed SAS Token to be used across all the requests. - **Configuration**: To use this method with ABFS Driver, specify the following properties in your `core-site.xml` file: @@ -754,22 +754,39 @@ requests. User can specify them as fixed SAS Token to be used across all the req ``` - 1. Fixed SAS Token: + 2. Account SAS (Fixed SAS Token at Account Level): + ```xml + + fs.azure.sas.fixed.token.ACCOUNT_NAME + FIXED_ACCOUNT_SAS_TOKEN + + ``` + + - Replace `FIXED_ACCOUNT_SAS_TOKEN` with fixed Account/Service SAS. You can also + generate SAS from Azure portal. Account -> Security + Networking -> Shared Access Signature + + 3. Service SAS (Fixed SAS Token at Container Level): ```xml - - fs.azure.sas.fixed.token - FIXED_SAS_TOKEN - - ``` + + fs.azure.sas.fixed.token.CONTAINER_NAME.ACCOUNT_NAME + FIXED_SAS_TOKEN + + ``` + + - Replace `FIXED_SERVICE_SAS_TOKEN` with fixed Service SAS. You can also + generate SAS from Azure portal. Account -> Data storage -> Containers -> + right click on your container and select generate SAS -> + Give valid permissions and expiry time -> Click on generate SAS and copy + the SAS token. - Replace `FIXED_SAS_TOKEN` with fixed Account/Service SAS. You can also -generate SAS from Azure portal. Account -> Security + Networking -> Shared Access Signature - **Security**: Account/Service SAS requires account keys to be used which makes them less secure. There is no scope of having delegated access to different users. -*Note:* When `fs.azure.sas.token.provider.type` and `fs.azure.fixed.sas.token` -are both configured, precedence will be given to the custom token provider implementation. +*Note:* +- When both account SAS and service SAS are configured, precedence will be given to the service SAS. +- When `fs.azure.sas.token.provider.type` and `fs.azure.fixed.sas.token` + are both configured, precedence will be given to the custom token provider implementation. ## Technical notes From 0e1ce6b4f0dd516acc6176754f9a00e16d13454a Mon Sep 17 00:00:00 2001 From: Manika Joshi Date: Fri, 28 Mar 2025 03:14:54 -0700 Subject: [PATCH 8/8] doc change --- .../apache/hadoop/fs/azurebfs/AbfsConfiguration.java | 10 +++++----- hadoop-tools/hadoop-azure/src/site/markdown/abfs.md | 8 +++++--- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java index b2a2396348d15..7f78532f1c20c 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java @@ -682,11 +682,11 @@ public int getInt(String key, int defaultValue) { * @throws IOException if parsing fails. */ public String getPasswordString(String key) throws IOException { - char[] passchars = rawConfig.getPassword(containerConf(key)) != null ? - rawConfig.getPassword(containerConf(key)) : - rawConfig.getPassword(accountConf(key)) != null ? - rawConfig.getPassword(accountConf(key)) : - rawConfig.getPassword(key); + char[] passchars = rawConfig.getPassword(containerConf(key)) != null + ? rawConfig.getPassword(containerConf(key)) + : rawConfig.getPassword(accountConf(key)) != null + ? rawConfig.getPassword(accountConf(key)) + : rawConfig.getPassword(key); if (passchars != null) { return new String(passchars); } diff --git a/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md b/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md index 0bb9518de289b..680c1293ea069 100644 --- a/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md +++ b/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md @@ -784,9 +784,11 @@ ADLS Gen 2 storage accounts. them less secure. There is no scope of having delegated access to different users. *Note:* -- When both account SAS and service SAS are configured, precedence will be given to the service SAS. -- When `fs.azure.sas.token.provider.type` and `fs.azure.fixed.sas.token` - are both configured, precedence will be given to the custom token provider implementation. +- Preference order for SAS will be: + - fs.azure.sas.token.provider.type + - fs.azure.sas.fixed.token.CONTAINER_NAME.ACCOUNT_NAME + - fs.azure.sas.fixed.token.ACCOUNT_NAME + - fs.azure.sas.fixed.token ## Technical notes