From 8c4413e89ff27f0cfd1f426be65e92c5c49f0b02 Mon Sep 17 00:00:00 2001 From: Manika Joshi Date: Mon, 3 Mar 2025 21:50:01 -0800 Subject: [PATCH 1/2] fixed SAS at container level --- .../hadoop/fs/azurebfs/AbfsConfiguration.java | 25 +++++++-- .../fs/azurebfs/AzureBlobFileSystemStore.java | 2 +- .../azurebfs/constants/ConfigurationKeys.java | 4 ++ .../azurebfs/AbstractAbfsIntegrationTest.java | 2 +- .../ITestAzureBlobFileSystemChooseSAS.java | 53 +++++++++++++++++++ .../azurebfs/utils/ServiceSASGenerator.java | 12 ++++- 6 files changed, 91 insertions(+), 7 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java index 6a51f8d902038..a326f3061af80 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java @@ -88,6 +88,7 @@ public class AbfsConfiguration{ private final Configuration rawConfig; private final String accountName; + private final String fsName; // Service type identified from URL used to initialize FileSystem. private final AbfsServiceType fsConfiguredServiceType; private final boolean isSecure; @@ -457,11 +458,13 @@ public class AbfsConfiguration{ */ public AbfsConfiguration(final Configuration rawConfig, String accountName, + String fsName, AbfsServiceType fsConfiguredServiceType) throws IllegalAccessException, IOException { this.rawConfig = ProviderUtils.excludeIncompatibleCredentialProviders( rawConfig, AzureBlobFileSystem.class); this.accountName = accountName; + this.fsName = fsName; this.fsConfiguredServiceType = fsConfiguredServiceType; this.isSecure = getBoolean(FS_AZURE_SECURE_MODE, false); @@ -493,7 +496,7 @@ public AbfsConfiguration(final Configuration rawConfig, */ public AbfsConfiguration(final Configuration rawConfig, String accountName) throws IllegalAccessException, IOException { - this(rawConfig, accountName, AbfsServiceType.DFS); + this(rawConfig, accountName, EMPTY_STRING, AbfsServiceType.DFS); } /** @@ -589,6 +592,16 @@ public String accountConf(String key) { return key + "." + accountName; } + /** + * Appends the container, account name to a configuration key yielding the + * container-specific form. + * @param key Account-agnostic configuration key + * @return Container-specific configuration key + */ + public String containerConf(String key) { + return key + "." + fsName + "." + accountName; + } + /** * Returns the account-specific value if it exists, then looks for an * account-agnostic value. @@ -644,16 +657,20 @@ public int getInt(String key, int defaultValue) { } /** - * Returns the account-specific password in string form if it exists, then + * Returns the container-specific password if it exists, + * else searches for the account-specific password, else finally * looks for an account-agnostic value. * @param key Account-agnostic configuration key * @return value in String form if one exists, else null * @throws IOException if parsing fails. */ public String getPasswordString(String key) throws IOException { - char[] passchars = rawConfig.getPassword(accountConf(key)); + char[] passchars = rawConfig.getPassword(containerConf(key)); if (passchars == null) { - passchars = rawConfig.getPassword(key); + passchars = rawConfig.getPassword(accountConf(key)); + if(passchars == null){ + passchars = rawConfig.getPassword(key); + } } if (passchars != null) { return new String(passchars); diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java index 867edfd1438fe..f7427d3091998 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java @@ -229,7 +229,7 @@ public AzureBlobFileSystemStore( try { this.abfsConfiguration = new AbfsConfiguration(abfsStoreBuilder.configuration, - accountName, getAbfsServiceTypeFromUrl()); + accountName, fileSystemName, getAbfsServiceTypeFromUrl()); } catch (IllegalAccessException exception) { throw new FileSystemOperationUnhandledException(exception); } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java index 3742361b48445..b4e0fa6c2eca1 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java @@ -322,6 +322,10 @@ public static String accountProperty(String property, String account) { return property + "." + account; } + public static String containerProperty(String property, String FsName, String account) { + return property + "." + FsName + "." + account; + } + public static final String FS_AZURE_ENABLE_DELEGATION_TOKEN = "fs.azure.enable.delegation.token"; public static final String FS_AZURE_DELEGATION_TOKEN_PROVIDER_TYPE = "fs.azure.delegation.token.provider.type"; diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java index e31df5eec65bb..450dfaa3a52c7 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java @@ -117,7 +117,7 @@ protected AbstractAbfsIntegrationTest() throws Exception { final String abfsUrl = this.getFileSystemName() + "@" + this.getAccountName(); URI defaultUri = null; - abfsConfig = new AbfsConfiguration(rawConfig, accountName, identifyAbfsServiceTypeFromUrl(abfsUrl)); + abfsConfig = new AbfsConfiguration(rawConfig, accountName, getFileSystemName(), identifyAbfsServiceTypeFromUrl(abfsUrl)); authType = abfsConfig.getEnum(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, AuthType.SharedKey); assumeValidAuthConfigsPresent(); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java index dc40241a87576..8e8c522e4f254 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java @@ -32,11 +32,13 @@ import org.apache.hadoop.fs.azurebfs.services.AuthType; import org.apache.hadoop.fs.azurebfs.services.FixedSASTokenProvider; import org.apache.hadoop.fs.azurebfs.utils.AccountSASGenerator; +import org.apache.hadoop.fs.azurebfs.utils.ServiceSASGenerator; import org.apache.hadoop.fs.azurebfs.utils.Base64; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_SAS_FIXED_TOKEN; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_SAS_TOKEN_PROVIDER_TYPE; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.accountProperty; +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.containerProperty; import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_TEST_APP_ID; import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_TEST_APP_SECRET; import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_TEST_APP_SERVICE_PRINCIPAL_OBJECT_ID; @@ -50,6 +52,7 @@ public class ITestAzureBlobFileSystemChooseSAS extends AbstractAbfsIntegrationTest{ private String accountSAS = null; + private String containerSAS = null; private static final String TEST_PATH = "testPath"; /** @@ -69,6 +72,7 @@ public void setup() throws Exception { super.setup(); createFilesystemWithTestFileForSASTests(new Path(TEST_PATH)); generateAccountSAS(); + generateContainerSAS(); } /** @@ -85,6 +89,22 @@ private void generateAccountSAS() throws AzureBlobFileSystemException { accountSAS = configAccountSASGenerator.getAccountSAS(getAccountName()); } + /** + * Generates a Container SAS Token using the Account Shared Key to be used as a fixed SAS Token. + * Container SAS used here will have only read permissions to resources. + * This will be used by individual tests to set in the configurations. + * @throws AzureBlobFileSystemException + */ + private void generateContainerSAS() throws AzureBlobFileSystemException { + final byte[] accountKey = Base64.decode( + getConfiguration().getStorageAccountKey()); + ServiceSASGenerator configServiceSASGenerator = new ServiceSASGenerator( + accountKey); + // Setting only read permissions. + configServiceSASGenerator.setPermissions("r"); + containerSAS = configServiceSASGenerator.getContainerSASWithFullControl( + getAccountName(), getFileSystemName()); + } /** * Tests the scenario where both the custom SASTokenProvider and a fixed SAS token are configured. * Custom implementation of SASTokenProvider class should be chosen and User Delegation SAS should be used. @@ -126,6 +146,38 @@ public void testBothProviderFixedTokenConfigured() throws Exception { } } + /** + * Tests the implementation sequence if all fixed SAS configs are set. + * The expected sequence is Container Specific Fixed SAS, Account Specific Fixed SAS, Account Agnostic Fixed SAS. + * @throws IOException + */ + @Test + public void testFixedTokenPreference() throws Exception { + AbfsConfiguration testAbfsConfig = new AbfsConfiguration( + getRawConfiguration(), this.getAccountName(), this.getFileSystemName(), getAbfsServiceType()); + + // setting all types of Fixed SAS configs (container-specific, account-specific, account-agnostic) + removeAnyPresetConfiguration(testAbfsConfig); + testAbfsConfig.set(containerProperty(FS_AZURE_SAS_FIXED_TOKEN, this.getFileSystemName(), this.getAccountName()), containerSAS); + testAbfsConfig.set(accountProperty(FS_AZURE_SAS_FIXED_TOKEN, this.getAccountName()), accountSAS); + testAbfsConfig.set(FS_AZURE_SAS_FIXED_TOKEN, accountSAS); + String ContainerSASExpected = testAbfsConfig.getSASTokenProvider().getSASToken(this.getAccountName(), this.getFileSystemName(), getMethodName(), "read"); + + // Assert that Container Specific Fixed SAS is used + Assertions.assertThat(ContainerSASExpected).contains("sr=c"); + + // Assert that Account Specific Fixed SAS is used if container SAS isn't set + testAbfsConfig.unset(containerProperty(FS_AZURE_SAS_FIXED_TOKEN, this.getFileSystemName(), this.getAccountName())); + String AccountSASExpected = testAbfsConfig.getSASTokenProvider().getSASToken(this.getAccountName(), this.getFileSystemName(), getMethodName(), "read"); + Assertions.assertThat(AccountSASExpected).contains("ss=bf"); + + //Assert that Account-Agnostic fixed SAS is used if no other fixed SAS configs are set. + // The token is the same as the Account Specific Fixed SAS. + testAbfsConfig.unset(FS_AZURE_SAS_FIXED_TOKEN); + String AccountAgnosticSASExpected = testAbfsConfig.getSASTokenProvider().getSASToken(this.getAccountName(), this.getFileSystemName(), getMethodName(), "read"); + Assertions.assertThat(AccountAgnosticSASExpected).contains("ss=bf"); + } + /** * Tests the scenario where only the fixed token is configured, and no token provider class is set. * Account SAS Token configured as fixed SAS should be used. @@ -189,5 +241,6 @@ private void removeAnyPresetConfiguration(AbfsConfiguration testAbfsConfig) { testAbfsConfig.unset(FS_AZURE_SAS_FIXED_TOKEN); testAbfsConfig.unset(accountProperty(FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, this.getAccountName())); testAbfsConfig.unset(accountProperty(FS_AZURE_SAS_FIXED_TOKEN, this.getAccountName())); + testAbfsConfig.unset(containerProperty(FS_AZURE_SAS_FIXED_TOKEN, this.getFileSystemName(), this.getAccountName())); } } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/ServiceSASGenerator.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/ServiceSASGenerator.java index 0ae5239e8f2a5..2f2600ef325a7 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/ServiceSASGenerator.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/ServiceSASGenerator.java @@ -37,10 +37,11 @@ public ServiceSASGenerator(byte[] accountKey) { super(accountKey); } + private String permissions = "racwdl"; public String getContainerSASWithFullControl(String accountName, String containerName) throws InvalidConfigurationValueException { accountName = getCanonicalAccountName(accountName); - String sp = "rcwdl"; + String sp = permissions; String sv = AuthenticationVersion.Feb20.toString(); String sr = "c"; String st = ISO_8601_FORMATTER.format(Instant.now().minus(FIVE_MINUTES)); @@ -96,4 +97,13 @@ private String computeSignatureForSAS(String sp, String st, String se, String sv LOG.debug("Service SAS stringToSign: " + stringToSign.replace("\n", ".")); return computeHmac256(stringToSign); } + + /** + * By default, Container SAS has all the available permissions. Use this to + * override the default permissions and set as per the requirements. + * @param permissions + */ + public void setPermissions(final String permissions) { + this.permissions = permissions; + } } From 4a182acf271e2c390663440d7e8e02d43b09d412 Mon Sep 17 00:00:00 2001 From: Manika Joshi Date: Tue, 4 Mar 2025 01:46:54 -0800 Subject: [PATCH 2/2] passing yetus checks --- .../hadoop/fs/azurebfs/AbfsConfiguration.java | 3 ++- .../azurebfs/constants/ConfigurationKeys.java | 4 ++-- .../ITestAzureBlobFileSystemChooseSAS.java | 18 +++++++++++------- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java index a326f3061af80..53783175cea4a 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java @@ -452,6 +452,7 @@ public class AbfsConfiguration{ * Constructor for AbfsConfiguration for specified service type. * @param rawConfig used to initialize the configuration. * @param accountName the name of the azure storage account. + * @param fsName the name of the file system (container name). * @param fsConfiguredServiceType service type configured for the file system. * @throws IllegalAccessException if the field is not accessible. * @throws IOException if an I/O error occurs. @@ -666,7 +667,7 @@ public int getInt(String key, int defaultValue) { */ public String getPasswordString(String key) throws IOException { char[] passchars = rawConfig.getPassword(containerConf(key)); - if (passchars == null) { + if(passchars == null) { passchars = rawConfig.getPassword(accountConf(key)); if(passchars == null){ passchars = rawConfig.getPassword(key); diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java index b4e0fa6c2eca1..fa91e2e6891e3 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java @@ -322,8 +322,8 @@ public static String accountProperty(String property, String account) { return property + "." + account; } - public static String containerProperty(String property, String FsName, String account) { - return property + "." + FsName + "." + account; + public static String containerProperty(String property, String fsName, String account) { + return property + "." + fsName + "." + account; } public static final String FS_AZURE_ENABLE_DELEGATION_TOKEN = "fs.azure.enable.delegation.token"; diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java index 8e8c522e4f254..cdf3d1a689e58 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java @@ -146,6 +146,13 @@ public void testBothProviderFixedTokenConfigured() throws Exception { } } + /** + * Helper method to get the Fixed SAS token value + */ + private String getFixedSASToken(AbfsConfiguration config) throws Exception { + return config.getSASTokenProvider().getSASToken(this.getAccountName(), this.getFileSystemName(), getMethodName(), "read"); + } + /** * Tests the implementation sequence if all fixed SAS configs are set. * The expected sequence is Container Specific Fixed SAS, Account Specific Fixed SAS, Account Agnostic Fixed SAS. @@ -161,21 +168,18 @@ public void testFixedTokenPreference() throws Exception { testAbfsConfig.set(containerProperty(FS_AZURE_SAS_FIXED_TOKEN, this.getFileSystemName(), this.getAccountName()), containerSAS); testAbfsConfig.set(accountProperty(FS_AZURE_SAS_FIXED_TOKEN, this.getAccountName()), accountSAS); testAbfsConfig.set(FS_AZURE_SAS_FIXED_TOKEN, accountSAS); - String ContainerSASExpected = testAbfsConfig.getSASTokenProvider().getSASToken(this.getAccountName(), this.getFileSystemName(), getMethodName(), "read"); // Assert that Container Specific Fixed SAS is used - Assertions.assertThat(ContainerSASExpected).contains("sr=c"); + Assertions.assertThat(getFixedSASToken(testAbfsConfig)).contains("sr=c"); // Assert that Account Specific Fixed SAS is used if container SAS isn't set testAbfsConfig.unset(containerProperty(FS_AZURE_SAS_FIXED_TOKEN, this.getFileSystemName(), this.getAccountName())); - String AccountSASExpected = testAbfsConfig.getSASTokenProvider().getSASToken(this.getAccountName(), this.getFileSystemName(), getMethodName(), "read"); - Assertions.assertThat(AccountSASExpected).contains("ss=bf"); + Assertions.assertThat(getFixedSASToken(testAbfsConfig)).contains("ss=bf"); //Assert that Account-Agnostic fixed SAS is used if no other fixed SAS configs are set. // The token is the same as the Account Specific Fixed SAS. - testAbfsConfig.unset(FS_AZURE_SAS_FIXED_TOKEN); - String AccountAgnosticSASExpected = testAbfsConfig.getSASTokenProvider().getSASToken(this.getAccountName(), this.getFileSystemName(), getMethodName(), "read"); - Assertions.assertThat(AccountAgnosticSASExpected).contains("ss=bf"); + testAbfsConfig.unset(accountProperty(FS_AZURE_SAS_FIXED_TOKEN, this.getAccountName())); + Assertions.assertThat(getFixedSASToken(testAbfsConfig)).contains("ss=bf"); } /**