Release of 2024-07-30

AbfsDiagnostics prints more options
* latest manifest committer settings
* azure http client options (trunk/branch-3.4 only)

Author: steveloughran

BUILDING.md

@@ -35,8 +35,7 @@ mvn clean install -Psdk2
 
 Joint build
 ```bash
-mvn clean install -Pextra
-mvn install -Psdk2
+mvn clean install -Pextra && mvn install -Psdk2
 ```
 
 ## Releasing

src/main/java/org/apache/hadoop/fs/store/diag/AbfsDiagnosticsInfo.java

@@ -172,15 +172,23 @@ public Object[][] getEnvVars() {
       {"fs.azure.account.oauth2.user.name", false, false},
       {"fs.azure.account.oauth2.user.name", false, false},
       {"fs.azure.account.oauth2.user.password", true, true},
-      {"fs.azure.shellkeyprovider.script", true, true},
+      {"fs.azure.account.operation.idle.timeout", false, false},
       {"fs.azure.account.throttling.enabled", false, false},
+      {"fs.azure.analysis.period", false, false},
+      {"fs.azure.shellkeyprovider.script", true, true},
       {FS_AZURE_ALWAYS_USE_HTTPS, false, false},
+      {"fs.azure.apache.http.client.idle.connection.ttl", false, false},
+      {"fs.azure.apache.http.client.max.cache.connection.size", false, false},
+      {"fs.azure.apache.http.client.max.io.exception.retries", false, false},
       {"fs.azure.appendblob.directories", false, false},
       {FS_AZURE_ATOMIC_RENAME_KEY, false, false},
       {"fs.azure.block.location.impersonatedhost", false, false},
       {"fs.azure.block.size", false, false},
       {"fs.azure.buffered.pread.disable", false, false},
       {"fs.azure.client-provided-encryption-key", true, true},
+      {"fs.azure.client.correlationid", false, false},
+      {"fs.azure.cluster.name", false, false},
+      {"fs.azure.cluster.type", false, false},
       {"fs.azure.concurrentRequestCount.in", false, false},
       {"fs.azure.concurrentRequestCount.out", false, false},
       {"fs.azure.createRemoteFileSystemDuringInitialization", false, false},
@@ -201,6 +209,9 @@ public Object[][] getEnvVars() {
       {FS_AZURE_ENABLE_READAHEAD, false, false},
       {FS_AZURE_ENABLE_READAHEAD_V2, false, false},
       {"fs.azure.enable.rename.resilience", false, false},
+      {"fs.azure.encryption.context.provider.type", false, false},
+      {"fs.azure.encryption.encoded.client-provided-key", false, true},
+      {"fs.azure.encryption.encoded.client-provided-key-sha", true, true},
       {FS_AZURE_IDENTITY_TRANSFORMER_CLASS, false, false},
       {"fs.azure.identity.transformer.domain.name", false, false},
       {"fs.azure.identity.transformer.enable.short.name", false, false},
@@ -218,6 +229,8 @@ public Object[][] getEnvVars() {
       {"fs.azure.io.retry.min.backoff.interval", false, false},
       {FS_AZURE_LEASE_THREADS, false, false},
       {"fs.azure.list.max.results", false, false},
+      {"fs.azure.metric.analysis.timeout", false, false},
+      {"fs.azure.networking.library", false, false},
       {"fs.azure.oauth.token.fetch.retry.max.backoff.interval", false, false},
       {"fs.azure.oauth.token.fetch.retry.max.retries", false, false},
       {"fs.azure.oauth.token.fetch.retry.min.backoff.interval", false, false},
@@ -231,11 +244,13 @@ public Object[][] getEnvVars() {
       {FS_AZURE_READAHEADQUEUE_DEPTH, false, false},
       {"fs.azure.rename.raises.exceptions", false, false},
       {FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, false, false},
+      {"fs.azure.sas.fixed.token", true, true},
       {"fs.azure.sas.token.renew.period.for.streams", false, false},
       {"fs.azure.secure.mode", false, false},
       {"fs.azure.shellkeyprovider.script", false, false},
       {"fs.azure.skipUserGroupMetadataDuringInitialization", false, false},
       {"fs.azure.ssl.channel.mode", false, false},
+      {"fs.azure.tracingheader.format", false, false},
       {"fs.azure.use.upn", false, false},
       {"fs.azure.user.agent.prefix", false, false},
       {"fs.azure.write.enableappendwithflush", false, false},
@@ -245,16 +260,19 @@ public Object[][] getEnvVars() {
 
       /* committer */
       {"mapreduce.outputcommitter.factory.scheme.abfs", false, false},
-      {"mapreduce.fileoutputcommitter.marksuccessfuljobs", false, false},
 
       {"mapreduce.manifest.committer.cleanup.parallel.delete", false, false},
+      {"mapreduce.manifest.committer.cleanup.parallel.delete.base.first", false, false},
       {"mapreduce.manifest.committer.delete.target.files", false, false},
       {"mapreduce.manifest.committer.diagnostics.manifest.directory", false, false},
       {"mapreduce.manifest.committer.io.thread.count", false, false},
+      {"mapreduce.manifest.committer.manifest.save.attempts", false, false},
       {"mapreduce.manifest.committer.store.operations.classname", false, false},
       {"mapreduce.manifest.committer.summary.report.directory", false, false},
       {"mapreduce.manifest.committer.validate.output", false, false},
-
+      {"mapreduce.manifest.committer.writer.queue.capacity", false, false},
+      {"mapreduce.fileoutputcommitter.cleanup.skipped", false, false},
+      {"mapreduce.fileoutputcommitter.marksuccessfuljobs", false, false},
       {"mapreduce.fileoutputcommitter.algorithm.version.v1.experimental.mv.threads", false, false},
       {"mapreduce.fileoutputcommitter.algorithm.version.v1.experimental.parallel.task.commit", false, false},
       {"mapreduce.fileoutputcommitter.algorithm.version.v1.experimental.parallel.rename.recovery", false, false},