Feat/awscli v2 migration (#77)

* upgrade to AWS CLI v2

* upgrade dev script

* use explicite binary path for aws install

* improved local dev script, use buildx for container build

* update binaries check doc, fmt dependencies upgrade doc

---------

Co-authored-by: bgauduch <[email protected]>

Author: bgauduch

Dockerfile

@@ -10,8 +10,8 @@ FROM debian:${DEBIAN_VERSION} as terraform
 ARG TARGETARCH
 ARG TERRAFORM_VERSION
 RUN apt-get update
-RUN apt-get install --no-install-recommends -y libcurl4=7.74.0-1.3+deb11u3
-RUN apt-get install --no-install-recommends -y curl=7.74.0-1.3+deb11u3
+# RUN apt-get install --no-install-recommends -y libcurl4=7.74.0-1.3+deb11u7
+RUN apt-get install --no-install-recommends -y curl=7.74.0-1.3+deb11u7
 RUN apt-get install --no-install-recommends -y ca-certificates=20210119
 RUN apt-get install --no-install-recommends -y unzip=6.0-26+deb11u1
 RUN apt-get install --no-install-recommends -y gnupg=2.2.27-2+deb11u2
@@ -24,15 +24,25 @@ RUN gpg --verify terraform_${TERRAFORM_VERSION}_SHA256SUMS.sig terraform_${TERRA
 RUN sha256sum --check --strict --ignore-missing terraform_${TERRAFORM_VERSION}_SHA256SUMS
 RUN unzip -j terraform_${TERRAFORM_VERSION}_linux_${TARGETARCH}.zip
 
-# Install AWS CLI using PIP
+# Install AWS CLI version 2
 FROM debian:${DEBIAN_VERSION} as aws-cli
 ARG AWS_CLI_VERSION
-ARG PYTHON_MAJOR_VERSION
 RUN apt-get update
-RUN apt-get install -y --no-install-recommends python3=${PYTHON_MAJOR_VERSION}.2-3
-RUN apt-get install -y --no-install-recommends python3-pip=20.3.4-4+deb11u1
-RUN pip3 install --no-cache-dir setuptools==67.1.0
-RUN pip3 install --no-cache-dir awscli==${AWS_CLI_VERSION}
+RUN apt-get install -y --no-install-recommends curl=7.74.0-1.3+deb11u7
+RUN apt-get install -y --no-install-recommends ca-certificates=20210119
+RUN apt-get install -y --no-install-recommends unzip=6.0-26+deb11u1
+RUN apt-get install -y --no-install-recommends groff=1.22.4-6
+RUN apt-get install -y --no-install-recommends gnupg=2.2.27-2+deb11u2
+RUN apt-get install -y --no-install-recommends git=1:2.30.2-1+deb11u2
+RUN apt-get install -y --no-install-recommends jq=1.6-2.1
+WORKDIR /workspace
+RUN curl --show-error --fail --output "awscliv2.zip" --remote-name "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-${AWS_CLI_VERSION}.zip"
+COPY security/awscliv2.asc ./
+COPY security/awscli-exe-linux-x86_64-${AWS_CLI_VERSION}.zip.sig ./awscliv2.sig
+RUN gpg --import awscliv2.asc
+RUN gpg --verify awscliv2.sig awscliv2.zip
+RUN unzip -u awscliv2.zip
+RUN ./aws/install --install-dir /usr/local/aws-cli --bin-dir /usr/local/bin
 
 # Build final image
 FROM debian:${DEBIAN_VERSION} as build
@@ -41,7 +51,7 @@ ARG PYTHON_MAJOR_VERSION
 RUN apt-get update \
   && apt-get install -y --no-install-recommends \
     ca-certificates=20210119\
-    git=1:2.30.2-1 \
+    git=1:2.30.2-1+deb11u2 \
     jq=1.6-2.1 \
     python3=${PYTHON_MAJOR_VERSION}.2-3 \
     openssh-client=1:8.4p1-5+deb11u1 \
@@ -50,9 +60,8 @@ RUN apt-get update \
   && update-alternatives --install /usr/bin/python python /usr/bin/python${PYTHON_MAJOR_VERSION} 1
 WORKDIR /workspace
 COPY --from=terraform /workspace/terraform /usr/local/bin/terraform
-COPY --from=aws-cli /usr/local/bin/aws* /usr/local/bin/
-COPY --from=aws-cli /usr/local/lib/python${PYTHON_MAJOR_VERSION}/dist-packages /usr/local/lib/python${PYTHON_MAJOR_VERSION}/dist-packages
-COPY --from=aws-cli /usr/lib/python3/dist-packages /usr/lib/python3/dist-packages
+COPY --from=aws-cli /usr/local/bin/ /usr/local/bin/
+COPY --from=aws-cli /usr/local/aws-cli /usr/local/aws-cli
 
 RUN groupadd --gid 1001 nonroot \
   # user needs a home folder to store aws credentials

dev.sh

@@ -23,7 +23,7 @@ echo "Lint Successful!"
 
 # Build image
 echo "Building images with AWS_CLI_VERSION=${AWS_VERSION} and TERRAFORM_VERSION=${TF_VERSION}..."
-docker image build --build-arg AWS_CLI_VERSION="$AWS_VERSION" --build-arg TERRAFORM_VERSION="$TF_VERSION" -t $IMAGE_NAME:$IMAGE_TAG .
+docker buildx build --platform "${PLATEFORM}" --build-arg AWS_CLI_VERSION="${AWS_VERSION}" --build-arg TERRAFORM_VERSION="${TF_VERSION}" --tag ${IMAGE_NAME}:${IMAGE_TAG} .
 echo "Image successfully builded!"
 
 # Test image
@@ -32,7 +32,7 @@ export AWS_VERSION=${AWS_VERSION} && export TF_VERSION=${TF_VERSION}
 envsubst '${AWS_VERSION},${TF_VERSION}' < tests/container-structure-tests.yml.template > tests/container-structure-tests.yml
 echo "Test config successfully generated!"
 echo "Executing container structure test..."
-docker container run --rm --interactive --volume "${PWD}"/tests/container-structure-tests.yml:/tests.yml:ro -v /var/run/docker.sock:/var/run/docker.sock:ro gcr.io/gcp-runtimes/container-structure-test:v1.14.0 test --image $IMAGE_NAME:$IMAGE_TAG --config /tests.yml
+docker container run --rm --interactive --volume "${PWD}"/tests/container-structure-tests.yml:/tests.yml:ro --volume /var/run/docker.sock:/var/run/docker.sock:ro gcr.io/gcp-runtimes/container-structure-test:v1.15.0 test --image ${IMAGE_NAME}:${IMAGE_TAG} --config /tests.yml
 
 # cleanup
 unset AWS_VERSION

docs/dependencies-upgrades.md

@@ -1,6 +1,6 @@
 # ⬆️ Dependencies upgrades checklist
 
- Supported versions:
+* Supported versions:
   * check available **AWS CLI** version on the [PyPip release page](https://pypi.org/project/awscli/)
   * check available **Terraform CLI** version (keep all minor versions from 0.11) available on the [project release page](https://github.com/hashicorp/terraform/releases)
     * [Report to the doc](https://github.com/zenika-open-source/terraform-aws-cli/tree/master/docs/terraform-binaries-verifications.md) to add required security files when adding a new supported Terraform version

docs/terraform-binaries-verifications.md

@@ -1,13 +1,27 @@
-# Terraform binary verifications
+# Binary verifications
 
 ## Terraform signature and PGP verification
 
-Terraform binaries are verified against both there SHA256SUMS and signatures after donwload.
+Both Terraform SHA256SUM and signature files are verified against [Hashicorp public GPG key](https://www.hashicorp.com/security).
+
+Terraform archives are verified against there SHA256SUMS after donwload.
 
 Theses files need to be added to the [/security](https://github.com/zenika-open-source/terraform-aws-cli/tree/master/security) folder.
 
 They can be downloaded from the [official Terraform releases](https://releases.hashicorp.com/terraform).
 
-## Hashicorp signature verification
+## AWS CLI signature and PGP verification
 
-Both Terraform SHA256SUM and signature files are verified against [Hashicorp public GPG key](https://www.hashicorp.com/security).
+Both AWS CLI archives and signatures files are verified against AWS public GPG key.
+
+Theses files need to be added to the [/security](https://github.com/zenika-open-source/terraform-aws-cli/tree/master/security) folder.
+
+They can be downloaded locally using this command:
+
+```shell
+# Export target aws cli version
+export AWS_CLI_VERSION=2.12.5
+
+# Download signature file
+curl -o security/awscli-exe-linux-x86_64-${AWS_CLI_VERSION}.zip.sig https://awscli.amazonaws.com/awscli-exe-linux-x86_64-${AWS_CLI_VERSION}.zip.sig
+```

security/awscli-exe-linux-x86_64-2.12.5.zip.sig

@@ -0,0 +1,29 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+
+mQINBF2Cr7UBEADJZHcgusOJl7ENSyumXh85z0TRV0xJorM2B/JL0kHOyigQluUG
+ZMLhENaG0bYatdrKP+3H91lvK050pXwnO/R7fB/FSTouki4ciIx5OuLlnJZIxSzx
+PqGl0mkxImLNbGWoi6Lto0LYxqHN2iQtzlwTVmq9733zd3XfcXrZ3+LblHAgEt5G
+TfNxEKJ8soPLyWmwDH6HWCnjZ/aIQRBTIQ05uVeEoYxSh6wOai7ss/KveoSNBbYz
+gbdzoqI2Y8cgH2nbfgp3DSasaLZEdCSsIsK1u05CinE7k2qZ7KgKAUIcT/cR/grk
+C6VwsnDU0OUCideXcQ8WeHutqvgZH1JgKDbznoIzeQHJD238GEu+eKhRHcz8/jeG
+94zkcgJOz3KbZGYMiTh277Fvj9zzvZsbMBCedV1BTg3TqgvdX4bdkhf5cH+7NtWO
+lrFj6UwAsGukBTAOxC0l/dnSmZhJ7Z1KmEWilro/gOrjtOxqRQutlIqG22TaqoPG
+fYVN+en3Zwbt97kcgZDwqbuykNt64oZWc4XKCa3mprEGC3IbJTBFqglXmZ7l9ywG
+EEUJYOlb2XrSuPWml39beWdKM8kzr1OjnlOm6+lpTRCBfo0wa9F8YZRhHPAkwKkX
+XDeOGpWRj4ohOx0d2GWkyV5xyN14p2tQOCdOODmz80yUTgRpPVQUtOEhXQARAQAB
+tCFBV1MgQ0xJIFRlYW0gPGF3cy1jbGlAYW1hem9uLmNvbT6JAlQEEwEIAD4WIQT7
+Xbd/1cEYuAURraimMQrMRnJHXAUCXYKvtQIbAwUJB4TOAAULCQgHAgYVCgkICwIE
+FgIDAQIeAQIXgAAKCRCmMQrMRnJHXJIXEAChLUIkg80uPUkGjE3jejvQSA1aWuAM
+yzy6fdpdlRUz6M6nmsUhOExjVIvibEJpzK5mhuSZ4lb0vJ2ZUPgCv4zs2nBd7BGJ
+MxKiWgBReGvTdqZ0SzyYH4PYCJSE732x/Fw9hfnh1dMTXNcrQXzwOmmFNNegG0Ox
+au+VnpcR5Kz3smiTrIwZbRudo1ijhCYPQ7t5CMp9kjC6bObvy1hSIg2xNbMAN/Do
+ikebAl36uA6Y/Uczjj3GxZW4ZWeFirMidKbtqvUz2y0UFszobjiBSqZZHCreC34B
+hw9bFNpuWC/0SrXgohdsc6vK50pDGdV5kM2qo9tMQ/izsAwTh/d/GzZv8H4lV9eO
+tEis+EpR497PaxKKh9tJf0N6Q1YLRHof5xePZtOIlS3gfvsH5hXA3HJ9yIxb8T0H
+QYmVr3aIUes20i6meI3fuV36VFupwfrTKaL7VXnsrK2fq5cRvyJLNzXucg0WAjPF
+RrAGLzY7nP1xeg1a0aeP+pdsqjqlPJom8OCWc1+6DWbg0jsC74WoesAqgBItODMB
+rsal1y/q+bPzpsnWjzHV8+1/EtZmSc8ZUGSJOPkfC7hObnfkl18h+1QtKTjZme4d
+H17gsBJr+opwJw/Zio2LMjQBOqlm3K1A4zFTh7wBC7He6KPQea1p2XAMgtvATtNe
+YLZATHZKTJyiqA==
+=vYOk
+-----END PGP PUBLIC KEY BLOCK-----

security/awscliv2.asc

@@ -0,0 +1,14 @@
+a73326ea8fb06f6976597e005f8047cbd55ac76ed1e517303d8f6395db6c7805  terraform_1.3.9_darwin_amd64.zip
+d8a59a794a7f99b484a07a0ed2aa6520921d146ac5a7f4b1b806dcf5c4af0525  terraform_1.3.9_darwin_arm64.zip
+d786d97b4cf21b251c152bae76df19443408fd4c2cc1a4b2c5d015a6691fdcb1  terraform_1.3.9_freebsd_386.zip
+6b21e450538227528b65f4f511ad52789e182eeec666ca6e8a08223fc6395497  terraform_1.3.9_freebsd_amd64.zip
+5740901b1c86bb8898d63521710a27d54d65a59896295855bd43797ec248ddb8  terraform_1.3.9_freebsd_arm.zip
+d321b0ef810a3f972b031176e329006644c8915b83ecfcf506099952c5fbaccc  terraform_1.3.9_linux_386.zip
+53048fa573effdd8f2a59b726234c6f450491fe0ded6931e9f4c6e3df6eece56  terraform_1.3.9_linux_amd64.zip
+da571087268c5faf884912c4239c6b9c8e1ed8e8401ab1dcb45712df70f42f1b  terraform_1.3.9_linux_arm64.zip
+58203da8e5468ab5c22d26d242bf64e305c038d99717199f6783a3686130b60e  terraform_1.3.9_linux_arm.zip
+93e587c321e4ea8823741d67be985eccfb842afb1c21d32a44c2a9a2322e7c9b  terraform_1.3.9_openbsd_386.zip
+d03e4f761dd0b2ff77d68da8f920a71322a9f9dda4a9c9b2534c2f93578d3fca  terraform_1.3.9_openbsd_amd64.zip
+0076a092051fad5bef394fa0b727433957d4897fd5ef0d1bd425020d0dc08f78  terraform_1.3.9_solaris_amd64.zip
+620c9d08b3b3d0f43c8035a353d08867abbde3ed5e0cabd368ba85fcc14b850f  terraform_1.3.9_windows_386.zip
+d3de5ad925cb01d194d5c944c87afc48cf4a34512b67bac1f277ce043346da69  terraform_1.3.9_windows_amd64.zip

security/terraform_1.3.7_SHA256SUMS

@@ -0,0 +1,14 @@
+5d8332994b86411b049391d31ad1a0785dfb470db8b9c50617de28ddb5d1f25d  terraform_1.4.6_darwin_amd64.zip
+30a2f87298ff9f299452119bd14afaa8d5b000c572f62fa64baf432e35d9dec1  terraform_1.4.6_darwin_arm64.zip
+546b23de458f22f40ed80b38ce6263f07fdb2e90b54e869c8797a4df00704dad  terraform_1.4.6_freebsd_386.zip
+b6be45b3b543421b1c5ab5513bbccb08b205159a818a17c4ef05606c66ec1204  terraform_1.4.6_freebsd_amd64.zip
+3bbfe96a215869fee5b809e5b1ed8bfa5dd76ea6bc7933cafb7d36ee3e60f736  terraform_1.4.6_freebsd_arm.zip
+f802ead8d46b90e5b5ec2ef5aaf5a0438bd9a7621fcc80f192b3a93ba25d679c  terraform_1.4.6_linux_386.zip
+e079db1a8945e39b1f8ba4e513946b3ab9f32bd5a2bdf19b9b186d22c5a3d53b  terraform_1.4.6_linux_amd64.zip
+e03fa047d5cd093929c89e261ee0852968c92097e3d575f7b56b1cb3720c82fd  terraform_1.4.6_linux_arm.zip
+b38f5db944ac4942f11ceea465a91e365b0636febd9998c110fbbe95d61c3b26  terraform_1.4.6_linux_arm64.zip
+eaf543d89e35e8356927cc615271c99bbf0a636b7e87c015d3a433d64baa071a  terraform_1.4.6_openbsd_386.zip
+b5b984e57c32e37dbc3b38adb350b4412c43b2676f90408291074c2a50d7b8f2  terraform_1.4.6_openbsd_amd64.zip
+5f0ce9c088ec184e27c4b3b9ef0fb8919747656099b07e61339176d983c42f52  terraform_1.4.6_solaris_amd64.zip
+6baf3a5071043da04d4b7880f2cc30dd85dd05fe01f4aad5e58e27d2a530f96e  terraform_1.4.6_windows_386.zip
+f666aa1388f94c9b86ea01cb884ba53b9132d2cec3d9cac976ad93a2aba901d5  terraform_1.4.6_windows_amd64.zip

security/terraform_1.3.7_SHA256SUMS.sig

@@ -0,0 +1,14 @@
+0484b5c7d5daa17cfff476f29b027398d805c00a8c276f884734b4c6fadd15ec  terraform_1.5.2_darwin_amd64.zip
+75c5632f221adbba38d569bdaeb6c3cb90b7f82e26b01e39b3b7e1c16bb0e4d4  terraform_1.5.2_darwin_arm64.zip
+3729c2bc8f5950c6312127e9a5d5ce45d44f656cf1a596cf3de5dfa56604737e  terraform_1.5.2_freebsd_386.zip
+f5d208cf09e2279e1113759dac340cbc8b13a1af2baed0316a7ee20af4a9595e  terraform_1.5.2_freebsd_amd64.zip
+21eaf6bdedff3342a3dc73285901f7510e755571851636f7c82c0ecc786f0393  terraform_1.5.2_freebsd_arm.zip
+41bd04b8603c8fb5b04ee899a09e78229c6f774d00c7641b2abb5dc271c70a47  terraform_1.5.2_linux_386.zip
+781ffe0c8888d35b3f5bd0481e951cebe9964b9cfcb27e352f22687975401bcd  terraform_1.5.2_linux_amd64.zip
+6a3229d9ebe8bfe219568ca7e060ced26e6e568c419a97601f2d9a756b998dbb  terraform_1.5.2_linux_arm.zip
+c39a081830f708fa9e50e5fe1462525ded4de1b4308dcf91f64914d6f4e988b3  terraform_1.5.2_linux_arm64.zip
+2067ef837c4529af72cfd39199e1eb3b6f964a38996f00a9a30ac3681e34b4ec  terraform_1.5.2_openbsd_386.zip
+681a58bcf8255ca90cb18a7c7f30beca284bc6490b592d0eb5ed3f9563d09d62  terraform_1.5.2_openbsd_amd64.zip
+fc5511eb8ca9c3b36cd505041e0037aacddc3c7e5e14647c2c99ae4faa4f7c86  terraform_1.5.2_solaris_amd64.zip
+07732bf900066f78648c9bedafbae22b10376a43e93b2eb68b2e1cb3897b87fc  terraform_1.5.2_windows_386.zip
+b27e63bb1007b27282444680b724c6ce2b5cea00eb3139ca953fd5c578bc4fc1  terraform_1.5.2_windows_amd64.zip

security/terraform_1.3.9_SHA256SUMS

@@ -8,9 +8,11 @@
     "1.0.11",
     "1.1.9",
     "1.2.9",
-    "1.3.7"
+    "1.3.9",
+    "1.4.6",
+    "1.5.2"
   ],
   "awscli_versions": [
-    "1.27.60"
+    "2.12.5"
   ]
 }