From e49be3eefa00a9028815d8487225dbb138ebc0e5 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 13 Dec 2024 13:14:00 +0530 Subject: [PATCH 01/18] Add support for mariner arm64 --- docker/DockerfileMarinerArm64 | 15 +++++++++++++++ release-pipeline.yml | 3 +++ 2 files changed, 18 insertions(+) create mode 100644 docker/DockerfileMarinerArm64 diff --git a/docker/DockerfileMarinerArm64 b/docker/DockerfileMarinerArm64 new file mode 100644 index 000000000..d42dcf205 --- /dev/null +++ b/docker/DockerfileMarinerArm64 @@ -0,0 +1,15 @@ +# Create container based on Ubuntu-22.04 Jammy Jellyfish image +FROM --platform=linux/arm64 mcr.microsoft.com/cbl-mariner/base/core:2.0 + +# Install dependencies +RUN tdnf update -y +RUN tdnf install -y ca-certificates + +# Copy azcopy binary to executable path +COPY ./azcopy /usr/local/bin/ + +# Make azcopy executable +RUN chmod +x /usr/local/bin/azcopy + +WORKDIR /azcopy +CMD [ "azcopy" ] \ No newline at end of file diff --git a/release-pipeline.yml b/release-pipeline.yml index 5f4aedf5a..1aa4062cd 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -1927,6 +1927,9 @@ stages: ./dockerinstall.sh ./buildcontainer.sh DockerfileArm64 ubuntu-arm64 ./publishcontainer.sh $(AZCOPY_DOCKER_REG_USER) $(AZCOPY_DOCKER_REG_PWD) ubuntu-arm64 + + ./buildcontainer.sh DockerfileMarinerArm64 mariner-arm64 + ./publishcontainer.sh $(AZCOPY_DOCKER_REG_USER) $(AZCOPY_DOCKER_REG_PWD) mariner-arm64 displayName: "Create docker image for arm64 and push to the containers registry" workingDirectory: $(work_dir)/docker From bdd6665bec0b9d7bf7050d92540b6fc8081d7638 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 2 Jan 2025 15:51:43 +0530 Subject: [PATCH 02/18] Added template file for docker build and publish steps --- release-pipeline.yml | 94 ++++------------------------------ setup/build_publish_docker.yml | 68 ++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 84 deletions(-) create mode 100644 setup/build_publish_docker.yml diff --git a/release-pipeline.yml b/release-pipeline.yml index 1aa4062cd..1a4afca43 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -1834,105 +1834,31 @@ stages: - ${{ if eq(parameters.publish_docker_image, true) }}: - stage: BuildAndPublishDockerImage - dependsOn: TestArtifacts - condition: succeeded('TestArtifacts') + # dependsOn: TestArtifacts + # condition: succeeded('TestArtifacts') jobs: - - job: Set_1_Ubuntu_Mariner_AMD64 - strategy: + - template: build_publish_docker.yml + parameters: + jobName: Set_1_Ubuntu_Mariner_AMD64 matrix: Ubuntu_amd64: agentName: "blobfuse-ubuntu22" vmImage: 'Ubuntu-22.04' container: 'test-cnt-ubn-22' - pool: - vmImage: $(vmImage) - - variables: - - group: AZCOPY_SECRET_VAULT - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - steps: - - checkout: none - - script: | - git clone https://github.com/Azure/azure-storage-azcopy - displayName: 'Checkout Code' - workingDirectory: $(root_dir) - - - script: | - git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - displayName: 'Checkout branch' - workingDirectory: $(work_dir) - - - script: | - chmod 777 *.sh - ./dockerinstall.sh - - ./buildcontainer.sh Dockerfile ubuntu-x86_64 - ./publishcontainer.sh $(AZCOPY_DOCKER_REG_USER) $(AZCOPY_DOCKER_REG_PWD) ubuntu-x86_64 - - ./buildcontainer.sh DockerfileMariner mariner-x86_64 - ./publishcontainer.sh $(AZCOPY_DOCKER_REG_USER) $(AZCOPY_DOCKER_REG_PWD) mariner-x86_64 - - displayName: "Create docker image and push to the containers registry" - workingDirectory: $(work_dir)/docker + poolName: $(vmImage) - - job: Set_2_Ubuntu_ARM64 - timeoutInMinutes: 120 - strategy: + - template: build_publish_docker.yml + parameters: + jobName: Set_2_Ubuntu_Mariner_ARM64 matrix: Ubuntu-22-ARM64: vmImage: 'Ubuntu-22.04' container: 'test-cnt-ubn-22-arm64' AgentName: "blobfuse-ubn22-arm64" - pool: - name: "blobfuse-ubn-arm64-pool" + poolName: "blobfuse-ubn-arm64-pool" demands: - ImageOverride -equals $(AgentName) - variables: - - group: AZCOPY_SECRET_VAULT - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - steps: - - checkout: none - - script: | - git clone https://github.com/Azure/azure-storage-azcopy - displayName: 'Checkout Code' - workingDirectory: $(root_dir) - - - script: | - git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - displayName: 'Checkout branch' - workingDirectory: $(work_dir) - - - task: ShellScript@2 - inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/ $(AZCOPY_GOLANG_VERSION)" - displayName: "Installing Go tools" - - - script: | - sudo apt update - sudo apt --fix-broken install - displayName: "Install dependencies" - - - script: | - chmod 777 *.sh - ./dockerinstall.sh - ./buildcontainer.sh DockerfileArm64 ubuntu-arm64 - ./publishcontainer.sh $(AZCOPY_DOCKER_REG_USER) $(AZCOPY_DOCKER_REG_PWD) ubuntu-arm64 - - ./buildcontainer.sh DockerfileMarinerArm64 mariner-arm64 - ./publishcontainer.sh $(AZCOPY_DOCKER_REG_USER) $(AZCOPY_DOCKER_REG_PWD) mariner-arm64 - displayName: "Create docker image for arm64 and push to the containers registry" - workingDirectory: $(work_dir)/docker - - ${{ if eq(parameters.post_release, true) }}: - stage: ReleaseToGithub dependsOn: TestArtifacts diff --git a/setup/build_publish_docker.yml b/setup/build_publish_docker.yml new file mode 100644 index 000000000..3d96a5f0a --- /dev/null +++ b/setup/build_publish_docker.yml @@ -0,0 +1,68 @@ +parameters: + - name: jobName + type: string + - name: matrix + type: object + - name: poolName + type: string + - name: demands + type: object + default: [] + - name: vmImage + type: string + - name: container + type: string + +jobs: + - job: ${{ parameters.jobName }} + timeoutInMinutes: 120 + strategy: + matrix: ${{ parameters.matrix }} + pool: + name: ${{ parameters.poolName }} + vmImage: ${{ parameters.vmImage }} + demands: ${{ parameters.demands }} + + variables: + - group: AZCOPY_SECRET_VAULT + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + + steps: + # TODO remove when merging to main + - checkout: none + - script: | + git clone https://github.com/Azure/azure-storage-azcopy + displayName: 'Checkout Code' + workingDirectory: $(root_dir) + + - script: | + git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + displayName: 'Checkout branch' + workingDirectory: $(work_dir) + + - ${{ if eq(parameters.jobName, 'Set_2_Ubuntu_Mariner_ARM64') }}: + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/ $(AZCOPY_GOLANG_VERSION)" + displayName: "Installing Go tools" + + - ${{ if eq(parameters.jobName, 'Set_2_Ubuntu_Mariner_ARM64') }}: + - script: | + sudo apt update + sudo apt --fix-broken install + displayName: "Install dependencies" + + - script: | + chmod 777 *.sh + ./dockerinstall.sh + ./buildcontainer.sh Dockerfile ${{ if eq(parameters.jobName, 'Set_1_Ubuntu_Mariner_AMD64') }}ubuntu-x86_64${{ else }}ubuntu-arm64 + ./publishcontainer.sh $(AZCOPY_DOCKER_REG_USER) $(AZCOPY_DOCKER_REG_PWD) ${{ if eq(parameters.jobName, 'Set_1_Ubuntu_Mariner_AMD64') }}ubuntu-x86_64${{ else }}ubuntu-arm64 + + ./buildcontainer.sh DockerfileMariner${{ if eq(parameters.jobName, 'Set_1_Ubuntu_Mariner_AMD64') }}${{ else }}Arm64 + ./publishcontainer.sh $(AZCOPY_DOCKER_REG_USER) $(AZCOPY_DOCKER_REG_PWD) mariner-${{ if eq(parameters.jobName, 'Set_1_Ubuntu_Mariner_AMD64') }}x86_64${{ else }}arm64 + displayName: "Create docker image and push to the containers registry" + workingDirectory: $(work_dir)/docker From 6853063c072041baa84a401bb52fa87e2e663147 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 2 Jan 2025 15:55:20 +0530 Subject: [PATCH 03/18] Updated the comment --- docker/DockerfileMariner | 2 +- docker/DockerfileMarinerArm64 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/DockerfileMariner b/docker/DockerfileMariner index 33480e383..9aa96bf05 100644 --- a/docker/DockerfileMariner +++ b/docker/DockerfileMariner @@ -1,4 +1,4 @@ -# Create container based on Ubuntu-22.04 Jammy Jellyfish image +# Create container based on mariner 2.0 image FROM mcr.microsoft.com/cbl-mariner/base/core:2.0 # Install dependencies diff --git a/docker/DockerfileMarinerArm64 b/docker/DockerfileMarinerArm64 index d42dcf205..715b2438b 100644 --- a/docker/DockerfileMarinerArm64 +++ b/docker/DockerfileMarinerArm64 @@ -1,4 +1,4 @@ -# Create container based on Ubuntu-22.04 Jammy Jellyfish image +# Create container based on mariner 2.0 image FROM --platform=linux/arm64 mcr.microsoft.com/cbl-mariner/base/core:2.0 # Install dependencies From 4b3616583301d5e68afe995b77bc6920b1291e43 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 2 Jan 2025 16:02:12 +0530 Subject: [PATCH 04/18] corrcted the template file path --- release-pipeline.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release-pipeline.yml b/release-pipeline.yml index 1a4afca43..bdd900645 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -1837,7 +1837,7 @@ stages: # dependsOn: TestArtifacts # condition: succeeded('TestArtifacts') jobs: - - template: build_publish_docker.yml + - template: setup/build_publish_docker.yml parameters: jobName: Set_1_Ubuntu_Mariner_AMD64 matrix: @@ -1847,7 +1847,7 @@ stages: container: 'test-cnt-ubn-22' poolName: $(vmImage) - - template: build_publish_docker.yml + - template: setup/build_publish_docker.yml parameters: jobName: Set_2_Ubuntu_Mariner_ARM64 matrix: From 200cf9b639612b9d1daa53470eb7de9da35e90ae Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 2 Jan 2025 16:05:50 +0530 Subject: [PATCH 05/18] updated version for testing docker publish --- common/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/version.go b/common/version.go index cd360150f..8ec4a6b8c 100644 --- a/common/version.go +++ b/common/version.go @@ -1,6 +1,6 @@ package common -const AzcopyVersion = "10.28.0-Preview" +const AzcopyVersion = "10.28.0" const UserAgent = "AzCopy/" + AzcopyVersion const S3ImportUserAgent = "S3Import " + UserAgent const GCPImportUserAgent = "GCPImport " + UserAgent From 9f453d63015860083b1de3132c8b7ece1219fff6 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 2 Jan 2025 16:08:16 +0530 Subject: [PATCH 06/18] updated template file --- setup/build_publish_docker.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/setup/build_publish_docker.yml b/setup/build_publish_docker.yml index 3d96a5f0a..90e5d13de 100644 --- a/setup/build_publish_docker.yml +++ b/setup/build_publish_docker.yml @@ -31,7 +31,6 @@ jobs: value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' steps: - # TODO remove when merging to main - checkout: none - script: | git clone https://github.com/Azure/azure-storage-azcopy @@ -59,10 +58,15 @@ jobs: - script: | chmod 777 *.sh ./dockerinstall.sh - ./buildcontainer.sh Dockerfile ${{ if eq(parameters.jobName, 'Set_1_Ubuntu_Mariner_AMD64') }}ubuntu-x86_64${{ else }}ubuntu-arm64 - ./publishcontainer.sh $(AZCOPY_DOCKER_REG_USER) $(AZCOPY_DOCKER_REG_PWD) ${{ if eq(parameters.jobName, 'Set_1_Ubuntu_Mariner_AMD64') }}ubuntu-x86_64${{ else }}ubuntu-arm64 - ./buildcontainer.sh DockerfileMariner${{ if eq(parameters.jobName, 'Set_1_Ubuntu_Mariner_AMD64') }}${{ else }}Arm64 - ./publishcontainer.sh $(AZCOPY_DOCKER_REG_USER) $(AZCOPY_DOCKER_REG_PWD) mariner-${{ if eq(parameters.jobName, 'Set_1_Ubuntu_Mariner_AMD64') }}x86_64${{ else }}arm64 + # Build and publish Docker containers + ./buildcontainer.sh Dockerfile $DOCKER_TAG + ./publishcontainer.sh $(AZCOPY_DOCKER_REG_USER) $(AZCOPY_DOCKER_REG_PWD) $DOCKER_TAG + + ./buildcontainer.sh DockerfileMariner $MARINER_TAG + ./publishcontainer.sh $(AZCOPY_DOCKER_REG_USER) $(AZCOPY_DOCKER_REG_PWD) $MARINER_TAG displayName: "Create docker image and push to the containers registry" workingDirectory: $(work_dir)/docker + env: + DOCKER_TAG: ${{ if eq(parameters.jobName, 'Set_1_Ubuntu_Mariner_AMD64') }}ubuntu-x86_64${{ else }}ubuntu-arm64 + MARINER_TAG: ${{ if eq(parameters.jobName, 'Set_1_Ubuntu_Mariner_AMD64') }}mariner-x86_64${{ else }}mariner-arm64 From 1eb7cb16b00065f4d86aa5215aa114d0764bb7f7 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 2 Jan 2025 16:15:13 +0530 Subject: [PATCH 07/18] updated template file --- release-pipeline.yml | 4 ++++ setup/build_publish_docker.yml | 15 ++++++++------- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/release-pipeline.yml b/release-pipeline.yml index bdd900645..2d1a9fd4c 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -1846,6 +1846,8 @@ stages: vmImage: 'Ubuntu-22.04' container: 'test-cnt-ubn-22' poolName: $(vmImage) + ubuntu_tag: 'ubuntu-x86_64' + mari_tag: 'mariner-x86_64' - template: setup/build_publish_docker.yml parameters: @@ -1858,6 +1860,8 @@ stages: poolName: "blobfuse-ubn-arm64-pool" demands: - ImageOverride -equals $(AgentName) + ubuntu_tag: 'ubuntu-arm64' + mari_tag: 'mariner-arm64' - ${{ if eq(parameters.post_release, true) }}: - stage: ReleaseToGithub diff --git a/setup/build_publish_docker.yml b/setup/build_publish_docker.yml index 90e5d13de..1e0e12342 100644 --- a/setup/build_publish_docker.yml +++ b/setup/build_publish_docker.yml @@ -12,6 +12,10 @@ parameters: type: string - name: container type: string + - name: ubuntu_tag + type: string + - name: mariner_tag + type: string jobs: - job: ${{ parameters.jobName }} @@ -60,13 +64,10 @@ jobs: ./dockerinstall.sh # Build and publish Docker containers - ./buildcontainer.sh Dockerfile $DOCKER_TAG - ./publishcontainer.sh $(AZCOPY_DOCKER_REG_USER) $(AZCOPY_DOCKER_REG_PWD) $DOCKER_TAG + ./buildcontainer.sh Dockerfile ${{ parameters.ubuntu_tag }} + ./publishcontainer.sh $(AZCOPY_DOCKER_REG_USER) $(AZCOPY_DOCKER_REG_PWD) ${{ parameters.ubuntu_tag }} - ./buildcontainer.sh DockerfileMariner $MARINER_TAG - ./publishcontainer.sh $(AZCOPY_DOCKER_REG_USER) $(AZCOPY_DOCKER_REG_PWD) $MARINER_TAG + ./buildcontainer.sh DockerfileMariner ${{ parameters.mariner_tag }} + ./publishcontainer.sh $(AZCOPY_DOCKER_REG_USER) $(AZCOPY_DOCKER_REG_PWD) ${{ parameters.mariner_tag }} displayName: "Create docker image and push to the containers registry" workingDirectory: $(work_dir)/docker - env: - DOCKER_TAG: ${{ if eq(parameters.jobName, 'Set_1_Ubuntu_Mariner_AMD64') }}ubuntu-x86_64${{ else }}ubuntu-arm64 - MARINER_TAG: ${{ if eq(parameters.jobName, 'Set_1_Ubuntu_Mariner_AMD64') }}mariner-x86_64${{ else }}mariner-arm64 From f40595eedeae2f8801675a671dbcc2cc807d6882 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 2 Jan 2025 16:36:35 +0530 Subject: [PATCH 08/18] updated pipeline file --- release-pipeline.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release-pipeline.yml b/release-pipeline.yml index 2d1a9fd4c..2a23bd261 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -1847,7 +1847,7 @@ stages: container: 'test-cnt-ubn-22' poolName: $(vmImage) ubuntu_tag: 'ubuntu-x86_64' - mari_tag: 'mariner-x86_64' + mariner_tag: 'mariner-x86_64' - template: setup/build_publish_docker.yml parameters: @@ -1861,7 +1861,7 @@ stages: demands: - ImageOverride -equals $(AgentName) ubuntu_tag: 'ubuntu-arm64' - mari_tag: 'mariner-arm64' + mariner_tag: 'mariner-arm64' - ${{ if eq(parameters.post_release, true) }}: - stage: ReleaseToGithub From d48a36bb096bb09d9151454b060abfe9c2e7b433 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 3 Jan 2025 11:27:16 +0530 Subject: [PATCH 09/18] Updated template code --- release-pipeline.yml | 36 ++++++++++++++++++++-------------- setup/build_publish_docker.yml | 28 +++++++++----------------- 2 files changed, 30 insertions(+), 34 deletions(-) diff --git a/release-pipeline.yml b/release-pipeline.yml index 2a23bd261..8465a1d0d 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -1838,28 +1838,34 @@ stages: # condition: succeeded('TestArtifacts') jobs: - template: setup/build_publish_docker.yml - parameters: - jobName: Set_1_Ubuntu_Mariner_AMD64 - matrix: - Ubuntu_amd64: - agentName: "blobfuse-ubuntu22" - vmImage: 'Ubuntu-22.04' - container: 'test-cnt-ubn-22' - poolName: $(vmImage) - ubuntu_tag: 'ubuntu-x86_64' - mariner_tag: 'mariner-x86_64' + parameters: + jobName: Set_1_Ubuntu_Mariner_AMD64 + useDemands: false + matrix: + Ubuntu_amd64: + agentName: "blobfuse-ubuntu22" + vmImage: 'Ubuntu-22.04' + container: 'test-cnt-ubn-22' + pool: + vmImage: $(vmImage) + ubuntu_tag: 'ubuntu-x86_64' + mariner_tag: 'mariner-x86_64' + + - template: setup/build_publish_docker.yml parameters: jobName: Set_2_Ubuntu_Mariner_ARM64 + useDemands: true matrix: - Ubuntu-22-ARM64: + Ubuntu_arm64: + agentName: "blobfuse-ubn22-arm64" vmImage: 'Ubuntu-22.04' container: 'test-cnt-ubn-22-arm64' - AgentName: "blobfuse-ubn22-arm64" - poolName: "blobfuse-ubn-arm64-pool" - demands: - - ImageOverride -equals $(AgentName) + pool: + name: "blobfuse-ubn-arm64-pool" + demands: + - ImageOverride -equals $(agentName) ubuntu_tag: 'ubuntu-arm64' mariner_tag: 'mariner-arm64' diff --git a/setup/build_publish_docker.yml b/setup/build_publish_docker.yml index 1e0e12342..03732d74f 100644 --- a/setup/build_publish_docker.yml +++ b/setup/build_publish_docker.yml @@ -1,17 +1,10 @@ parameters: - name: jobName type: string + - name: useDemands: + type: boolean - name: matrix type: object - - name: poolName - type: string - - name: demands - type: object - default: [] - - name: vmImage - type: string - - name: container - type: string - name: ubuntu_tag type: string - name: mariner_tag @@ -23,16 +16,13 @@ jobs: strategy: matrix: ${{ parameters.matrix }} pool: - name: ${{ parameters.poolName }} - vmImage: ${{ parameters.vmImage }} - demands: ${{ parameters.demands }} - - variables: - - group: AZCOPY_SECRET_VAULT - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + ${{ if eq(parameters.useDemands, true) }}: + name: "blobfuse-ubn-arm64-pool" + demands: + - ImageOverride -equals ${{ parameters.matrix.agentName }} + ${{ if eq(parameters.useDemands, false) }}: + vmImage: ${{ parameters.matrix.vmImage }} + steps: - checkout: none From ad4e042a8a01792eaffdeaa5d1af709956244ee7 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 3 Jan 2025 11:29:19 +0530 Subject: [PATCH 10/18] Updated template code --- release-pipeline.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/release-pipeline.yml b/release-pipeline.yml index 8465a1d0d..faaaddfe9 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -1836,7 +1836,6 @@ stages: - stage: BuildAndPublishDockerImage # dependsOn: TestArtifacts # condition: succeeded('TestArtifacts') - jobs: - template: setup/build_publish_docker.yml parameters: jobName: Set_1_Ubuntu_Mariner_AMD64 @@ -1851,8 +1850,6 @@ stages: ubuntu_tag: 'ubuntu-x86_64' mariner_tag: 'mariner-x86_64' - - - template: setup/build_publish_docker.yml parameters: jobName: Set_2_Ubuntu_Mariner_ARM64 From ced196d5d01ba0bcc97829855ad844b10cef7492 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 3 Jan 2025 11:31:14 +0530 Subject: [PATCH 11/18] Updated template code --- release-pipeline.yml | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/release-pipeline.yml b/release-pipeline.yml index faaaddfe9..cdbff7c83 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -1836,19 +1836,20 @@ stages: - stage: BuildAndPublishDockerImage # dependsOn: TestArtifacts # condition: succeeded('TestArtifacts') + jobs: - template: setup/build_publish_docker.yml - parameters: - jobName: Set_1_Ubuntu_Mariner_AMD64 - useDemands: false - matrix: - Ubuntu_amd64: - agentName: "blobfuse-ubuntu22" - vmImage: 'Ubuntu-22.04' - container: 'test-cnt-ubn-22' - pool: - vmImage: $(vmImage) - ubuntu_tag: 'ubuntu-x86_64' - mariner_tag: 'mariner-x86_64' + parameters: + jobName: Set_1_Ubuntu_Mariner_AMD64 + useDemands: false + matrix: + Ubuntu_amd64: + agentName: "blobfuse-ubuntu22" + vmImage: 'Ubuntu-22.04' + container: 'test-cnt-ubn-22' + pool: + vmImage: $(vmImage) + ubuntu_tag: 'ubuntu-x86_64' + mariner_tag: 'mariner-x86_64' - template: setup/build_publish_docker.yml parameters: From e11cc52b1173d4fdef32f5fd504681fc7be4a657 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 3 Jan 2025 11:32:34 +0530 Subject: [PATCH 12/18] Updated template code --- setup/build_publish_docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/build_publish_docker.yml b/setup/build_publish_docker.yml index 03732d74f..86eac7331 100644 --- a/setup/build_publish_docker.yml +++ b/setup/build_publish_docker.yml @@ -1,7 +1,7 @@ parameters: - name: jobName type: string - - name: useDemands: + - name: useDemands type: boolean - name: matrix type: object From 89f412557b91db60ea169a2309b22fddd18d9673 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 3 Jan 2025 11:36:27 +0530 Subject: [PATCH 13/18] Updated template code --- release-pipeline.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/release-pipeline.yml b/release-pipeline.yml index cdbff7c83..27a5bf21f 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -1846,8 +1846,6 @@ stages: agentName: "blobfuse-ubuntu22" vmImage: 'Ubuntu-22.04' container: 'test-cnt-ubn-22' - pool: - vmImage: $(vmImage) ubuntu_tag: 'ubuntu-x86_64' mariner_tag: 'mariner-x86_64' @@ -1860,10 +1858,6 @@ stages: agentName: "blobfuse-ubn22-arm64" vmImage: 'Ubuntu-22.04' container: 'test-cnt-ubn-22-arm64' - pool: - name: "blobfuse-ubn-arm64-pool" - demands: - - ImageOverride -equals $(agentName) ubuntu_tag: 'ubuntu-arm64' mariner_tag: 'mariner-arm64' From 5ec5b12724e4b8d04d86879a22afaec6eb480d2c Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 3 Jan 2025 11:46:32 +0530 Subject: [PATCH 14/18] Updated template code --- release-pipeline.yml | 16 ++++++++-------- setup/build_publish_docker.yml | 9 ++++----- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/release-pipeline.yml b/release-pipeline.yml index 27a5bf21f..c3cff1e81 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -1844,10 +1844,10 @@ stages: matrix: Ubuntu_amd64: agentName: "blobfuse-ubuntu22" - vmImage: 'Ubuntu-22.04' - container: 'test-cnt-ubn-22' - ubuntu_tag: 'ubuntu-x86_64' - mariner_tag: 'mariner-x86_64' + vmImage: "Ubuntu-22.04" + container: "test-cnt-ubn-22" + ubuntu_tag: "ubuntu-x86_64" + mariner_tag: "mariner-x86_64" - template: setup/build_publish_docker.yml parameters: @@ -1856,10 +1856,10 @@ stages: matrix: Ubuntu_arm64: agentName: "blobfuse-ubn22-arm64" - vmImage: 'Ubuntu-22.04' - container: 'test-cnt-ubn-22-arm64' - ubuntu_tag: 'ubuntu-arm64' - mariner_tag: 'mariner-arm64' + vmImage: "Ubuntu-22.04" + container: "test-cnt-ubn-22-arm64" + ubuntu_tag: "ubuntu-arm64" + mariner_tag: "mariner-arm64" - ${{ if eq(parameters.post_release, true) }}: - stage: ReleaseToGithub diff --git a/setup/build_publish_docker.yml b/setup/build_publish_docker.yml index 86eac7331..493eec316 100644 --- a/setup/build_publish_docker.yml +++ b/setup/build_publish_docker.yml @@ -19,21 +19,20 @@ jobs: ${{ if eq(parameters.useDemands, true) }}: name: "blobfuse-ubn-arm64-pool" demands: - - ImageOverride -equals ${{ parameters.matrix.agentName }} + - ImageOverride -equals ${{ parameters.matrix['Ubuntu_arm64'].agentName }} ${{ if eq(parameters.useDemands, false) }}: - vmImage: ${{ parameters.matrix.vmImage }} - + vmImage: ${{ parameters.matrix['Ubuntu_amd64'].vmImage }} steps: - checkout: none - script: | git clone https://github.com/Azure/azure-storage-azcopy - displayName: 'Checkout Code' + displayName: "Checkout Code" workingDirectory: $(root_dir) - script: | git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - displayName: 'Checkout branch' + displayName: "Checkout branch" workingDirectory: $(work_dir) - ${{ if eq(parameters.jobName, 'Set_2_Ubuntu_Mariner_ARM64') }}: From 41358a34286dd38354fd80c8a881fad6fd209454 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 3 Jan 2025 11:55:11 +0530 Subject: [PATCH 15/18] Updated template code --- setup/build_publish_docker.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/setup/build_publish_docker.yml b/setup/build_publish_docker.yml index 493eec316..55969f374 100644 --- a/setup/build_publish_docker.yml +++ b/setup/build_publish_docker.yml @@ -22,6 +22,13 @@ jobs: - ImageOverride -equals ${{ parameters.matrix['Ubuntu_arm64'].agentName }} ${{ if eq(parameters.useDemands, false) }}: vmImage: ${{ parameters.matrix['Ubuntu_amd64'].vmImage }} + + variables: + - group: AZCOPY_SECRET_VAULT + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' steps: - checkout: none From 90f48e04ea84daf8a669755102df53608ed5c5cd Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 3 Jan 2025 13:01:32 +0530 Subject: [PATCH 16/18] Updated template code --- setup/build_publish_docker.yml | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/setup/build_publish_docker.yml b/setup/build_publish_docker.yml index 55969f374..5af02d1a2 100644 --- a/setup/build_publish_docker.yml +++ b/setup/build_publish_docker.yml @@ -31,16 +31,7 @@ jobs: value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' steps: - - checkout: none - - script: | - git clone https://github.com/Azure/azure-storage-azcopy - displayName: "Checkout Code" - workingDirectory: $(root_dir) - - - script: | - git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - displayName: "Checkout branch" - workingDirectory: $(work_dir) + - checkout: self - ${{ if eq(parameters.jobName, 'Set_2_Ubuntu_Mariner_ARM64') }}: - task: ShellScript@2 From 25b5d1521ca88d25bbe32c178377325929540433 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 3 Jan 2025 13:02:13 +0530 Subject: [PATCH 17/18] Updated template code --- release-pipeline.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release-pipeline.yml b/release-pipeline.yml index c3cff1e81..0ab644a64 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -1834,8 +1834,8 @@ stages: - ${{ if eq(parameters.publish_docker_image, true) }}: - stage: BuildAndPublishDockerImage - # dependsOn: TestArtifacts - # condition: succeeded('TestArtifacts') + dependsOn: TestArtifacts + condition: succeeded('TestArtifacts') jobs: - template: setup/build_publish_docker.yml parameters: From 32638939a673a65afc3eaf43c6692ed77445aa12 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 3 Jan 2025 13:05:02 +0530 Subject: [PATCH 18/18] reverted not required changes --- common/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/version.go b/common/version.go index 8ec4a6b8c..cd360150f 100644 --- a/common/version.go +++ b/common/version.go @@ -1,6 +1,6 @@ package common -const AzcopyVersion = "10.28.0" +const AzcopyVersion = "10.28.0-Preview" const UserAgent = "AzCopy/" + AzcopyVersion const S3ImportUserAgent = "S3Import " + UserAgent const GCPImportUserAgent = "GCPImport " + UserAgent