|
| 1 | +#!/usr/bin/env bash |
| 2 | + |
| 3 | +set -eo pipefail |
| 4 | + |
| 5 | +# Licensed to the Apache Software Foundation (ASF) under one or more |
| 6 | +# contributor license agreements. See the NOTICE file distributed with |
| 7 | +# this work for additional information regarding copyright ownership. |
| 8 | +# The ASF licenses this file to You under the Apache License, Version 2.0 |
| 9 | +# (the "License"); you may not use this file except in compliance with |
| 10 | +# the License. You may obtain a copy of the License at |
| 11 | +# |
| 12 | +# http://www.apache.org/licenses/LICENSE-2.0 |
| 13 | +# |
| 14 | +# Unless required by applicable law or agreed to in writing, software |
| 15 | +# distributed under the License is distributed on an "AS IS" BASIS, |
| 16 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 17 | +# See the License for the specific language governing permissions and |
| 18 | +# limitations under the License. |
| 19 | + |
| 20 | +FILE=$1 |
| 21 | +OUTPUT_FILE="" |
| 22 | + |
| 23 | +# ------------------------------------------------------------------------------ |
| 24 | +# Initialization and Dependency Checks |
| 25 | +# ------------------------------------------------------------------------------ |
| 26 | + |
| 27 | +checkDependency() { |
| 28 | + if ! command -v xmlstarlet &> /dev/null; then |
| 29 | + echo "ERROR: 'xmlstarlet' is not installed. Please install it to run this script." |
| 30 | + echo "Exiting..." |
| 31 | + exit 1 |
| 32 | + fi |
| 33 | +} |
| 34 | + |
| 35 | +validateInputFile() { |
| 36 | + if [ ! -f "$FILE" ]; then |
| 37 | + echo "Error: File '$FILE' not found. Exiting...." |
| 38 | + exit 1 |
| 39 | + fi |
| 40 | + |
| 41 | + if [[ "$FILE" != *.xml ]]; then |
| 42 | + echo "The file provided is not an XML file. Exiting...." |
| 43 | + exit 1 |
| 44 | + fi |
| 45 | +} |
| 46 | + |
| 47 | +init() { |
| 48 | + checkDependency |
| 49 | + validateInputFile |
| 50 | + OUTPUT_FILE="abfs-converted-config.xml" |
| 51 | + cp "$FILE" "$OUTPUT_FILE" |
| 52 | +} |
| 53 | + |
| 54 | +# ------------------------------------------------------------------------------ |
| 55 | +# Global Variables & Config Mappings |
| 56 | +# ------------------------------------------------------------------------------ |
| 57 | + |
| 58 | +contactTeamMsg= "For any queries or support, kindly reach out to us at '[email protected]'." |
| 59 | +endpoint=".dfs." |
| 60 | + |
| 61 | +# Mapping for renaming configurations |
| 62 | +declare -A renameConfigsMap=( |
| 63 | + ["autothrottling.enable"]="enable.autothrottling" #fs.azure.autothrottling.enable to fs.azure.enable.autothrottling |
| 64 | + ["rename.dir"]="rename.key" # fs.azure.atomic.rename.dir to fs.azure.atomic.rename.key |
| 65 | + ["block.blob.buffered.pread.disable"]="buffered.pread.disable" #fs.azure.block.blob.buffered.pread.disable to fs.azure.buffered.pread.disable |
| 66 | + ["fs.azure.sas"]="fs.azure.sas.fixed.token." #fs.azure.sas.CONTAINER_NAME.ACCOUNT_NAME to fs.azure.sas.fixed.token.CONTAINER_NAME.ACCOUNT_NAME |
| 67 | + ["check.block.md5"]="enable.checksum.validation" #fs.azure.check.block.md5 to fs.azure.enable.checksum.validation |
| 68 | +) |
| 69 | + |
| 70 | +# Configs not supported in ABFS |
| 71 | +unsupportedConfigsList=( |
| 72 | + "fs.azure.page.blob.dir" |
| 73 | + "fs.azure.block.blob.with.compaction.dir" |
| 74 | + "fs.azure.store.blob.md5" |
| 75 | +) |
| 76 | + |
| 77 | +# Configurations not required in ABFS Driver and can be removed |
| 78 | +obsoleteConfigsList=( |
| 79 | + "azure.authorization" #fs.azure.authorization, fs.azure.authorization.caching.enable , fs.azure.authorization.caching.maxentries, fs.azure.authorization.cacheentry.expiry.period, fs.azure.authorization.remote.service.urls |
| 80 | + "azure.selfthrottling" #fs.azure.selfthrottling.enable, fs.azure.selfthrottling.read.factor, fs.azure.selfthrottling.write.factor |
| 81 | + "azure.saskey" #fs.azure.saskey.cacheentry.expiry.period , fs.azure.saskey.usecontainersaskeyforallaccess |
| 82 | + "copyblob.retry" #fs.azure.io.copyblob.retry.min.backoff.interval, fs.azure.io.copyblob.retry.max.backoff.interval, fs.azure.io.copyblob.retry.backoff.interval, fs.azure.io.copyblob.retry.max.retries |
| 83 | + "service.urls" #fs.azure.cred.service.urls , fs.azure.delegation.token.service.urls, fs.azure.authorization.remote.service.urls |
| 84 | + "blob.metadata.key.case.sensitive" #fs.azure.blob.metadata.key.case.sensitive |
| 85 | + "cacheentry.expiry.period" #fs.azure.cacheentry.expiry.period |
| 86 | + "chmod.allowed.userlist" #fs.azure.chmod.allowed.userlist |
| 87 | + "chown.allowed.userlist" #fs.azure.chown.allowed.userlist |
| 88 | + "daemon.userlist" #fs.azure.daemon.userlist |
| 89 | + "delete.threads" #fs.azure.delete.threads |
| 90 | + "enable.kerberos.support" #fs.azure.enable.kerberos.support |
| 91 | + "flatlist.enable" #fs.azure.flatlist.enable |
| 92 | + "fsck.temp.expiry.seconds" #fs.azure.fsck.temp.expiry.seconds |
| 93 | + "local.sas.key.mode" #fs.azure.local.sas.key.mode |
| 94 | + "override.canonical.service.name" #fs.azure.override.canonical.service.name |
| 95 | + "permissions.supergroup" #fs.azure.permissions.supergroup |
| 96 | + "rename.threads" #fs.azure.rename.threads |
| 97 | + "secure.mode" #fs.azure.secure.mode |
| 98 | + "skip.metrics" #fs.azure.skip.metrics |
| 99 | + "storage.client.logging" #fs.azure.storage.client.logging |
| 100 | + "storage.emulator.account.name" #fs.azure.storage.emulator.account.name |
| 101 | + "storage.timeout" #fs.azure.storage.timeout |
| 102 | + "enable.append.support" #fs.azure.enable.append.support |
| 103 | +) |
| 104 | + |
| 105 | +# ------------------------------------------------------------------------------ |
| 106 | +# User Interaction |
| 107 | +# ------------------------------------------------------------------------------ |
| 108 | + |
| 109 | +promptNamespaceType() { |
| 110 | + printf "Select 'HNS' if you're migrating to ABFS driver for Hierarchical Namespace enabled account, |
| 111 | + or 'Non-HNS' if you're migrating to ABFS driver for Non-Hierarchical Namespace (FNS) account. \n" |
| 112 | + printf "WARNING: Please ensure the correct option is chosen as it will affect the configuration changes made to the file. \n" |
| 113 | + printf "If you are unsure, follow the instructions below to check from Azure Portal: \n" |
| 114 | + printf "* Go to the Azure Portal and navigate to your storage account. \n" |
| 115 | + printf "* In the left-hand menu, select 'Overview' section and look for 'Properties'. \n" |
| 116 | + printf "* Under 'Blob service', check if 'Hierarchical namespace' is enabled or disabled. \n" |
| 117 | + echo "$contactTeamMsg" |
| 118 | + select namespaceType in "HNS" "NonHNS" |
| 119 | + do |
| 120 | + case $namespaceType in |
| 121 | + HNS) |
| 122 | + xmlstarlet ed -L -i '//configuration/property[1]' -t elem -n property -v '' \ |
| 123 | + -s '//configuration/property[1]' -t elem -n name -v 'fs.azure.account.hns.enabled' \ |
| 124 | + -s '//configuration/property[1]' -t elem -n value -v 'true' "$OUTPUT_FILE" |
| 125 | + break |
| 126 | + ;; |
| 127 | + NonHNS) |
| 128 | + endpoint=".blob." |
| 129 | + break |
| 130 | + ;; |
| 131 | + *) |
| 132 | + echo "Invalid selection. Please try again. Exiting..." |
| 133 | + exit 1 |
| 134 | + ;; |
| 135 | + esac |
| 136 | + done |
| 137 | +} |
| 138 | + |
| 139 | +# ------------------------------------------------------------------------------ |
| 140 | +# Config File Transformations |
| 141 | +# ------------------------------------------------------------------------------ |
| 142 | + |
| 143 | +# Stop the script if any unsupported config is found |
| 144 | +unsupportedConfigCheck() { |
| 145 | + for key in "${unsupportedConfigsList[@]}"; do |
| 146 | + if grep -q "$key" "$OUTPUT_FILE"; then |
| 147 | + echo "Remove the following configuration from file and rerun: '$key'" |
| 148 | + failure=true |
| 149 | + fi |
| 150 | + done |
| 151 | + |
| 152 | + if [ "$failure" = true ]; then |
| 153 | + echo "FAILURE: Unsupported Config Found" |
| 154 | + echo "$contactTeamMsg" |
| 155 | + echo "Exiting..." |
| 156 | + exit 1 |
| 157 | + fi |
| 158 | +} |
| 159 | + |
| 160 | +# Renaming the configs |
| 161 | +renameConfigs() { |
| 162 | + for old in "${!renameConfigsMap[@]}"; do |
| 163 | + new="${renameConfigsMap[$old]}" |
| 164 | + xmlstarlet ed -L -u "//property/name[contains(., '$old')]" -x "concat(substring-before(., '$old'), |
| 165 | + '$new', substring-after(., '$old'))" "$OUTPUT_FILE" |
| 166 | + done |
| 167 | +} |
| 168 | + |
| 169 | +# Remove the obsolete configs |
| 170 | +removeObsoleteConfigs() { |
| 171 | + for key in "${obsoleteConfigsList[@]}"; do |
| 172 | + xmlstarlet ed -L -d "//property[name[contains(text(), '$key')]]" "$OUTPUT_FILE" |
| 173 | + done |
| 174 | +} |
| 175 | + |
| 176 | +# Change the endpoints to DFS if migrating to HNS |
| 177 | +changeEndpointForHNS() { |
| 178 | + if [ "$endpoint" = ".dfs." ]; then |
| 179 | + xmlstarlet ed -L -u "//property/name[contains(., '.blob.')]" -x "concat(substring-before(., '.blob.'), |
| 180 | + '$endpoint', substring-after(., '.blob.'))" "$OUTPUT_FILE" |
| 181 | + fi |
| 182 | +} |
| 183 | + |
| 184 | +# Change the value of fs.defaultFS |
| 185 | +handleDefaultFSValue() { |
| 186 | + if xmlstarlet sel -t -v "//property[name='fs.defaultFS']/value" "$OUTPUT_FILE" | grep -q "."; then |
| 187 | + if xmlstarlet sel -t -v "//property[name='fs.defaultFS']/value" "$OUTPUT_FILE" | grep -q ".blob."; then |
| 188 | + xmlstarlet ed -L -u "//property[name='fs.defaultFS']/value" -x "concat('abfs', substring-before(substring-after(., 'wasb'), '@'), |
| 189 | + '@', substring-before(substring-after(., '@'), '.blob.'), '$endpoint', 'core.windows.net')" "$OUTPUT_FILE" |
| 190 | + else |
| 191 | + echo "ERROR: 'fs.defaultFS' does not have 'Blob' as endpoint. Exiting..." |
| 192 | + echo "$contactTeamMsg" |
| 193 | + exit 1 |
| 194 | + fi |
| 195 | + fi |
| 196 | +} |
| 197 | + |
| 198 | +# ------------------------------------------------------------------------------ |
| 199 | +# Script Execution |
| 200 | +# ------------------------------------------------------------------------------ |
| 201 | + |
| 202 | +init |
| 203 | +promptNamespaceType |
| 204 | +unsupportedConfigCheck |
| 205 | +removeObsoleteConfigs |
| 206 | +renameConfigs |
| 207 | +changeEndpointForHNS |
| 208 | +handleDefaultFSValue |
| 209 | + |
| 210 | +# Clean up any <property> blocks with empty <name> tags |
| 211 | +xmlstarlet ed -L -d "//property[not(name) or name='']" "$OUTPUT_FILE" |
| 212 | + |
| 213 | +echo "Updated file: $OUTPUT_FILE" |
0 commit comments