allow s3 commands to work from non commercial environments

when running under something like us-gov-east-1, the latest-binaries.sh
script fails b/c it tries to access the us-west-2 bucket from the wrong
endpoints.

this can be avoided by setting AWS_ENDPOINT_URL_S3 to point to
us-west-2, but you still end up trying to use the gov-cloud creds in the
request which would fail with:

$ ./hack/latest-binaries.sh 1.29

An error occurred (InvalidToken) when calling the ListObjectsV2 operation:
The provided token is malformed or otherwise invalid.

so, specify to perform an unauthenticated s3 api request b/c the
govcloud creds wouldn't work against the commercial cloud endpoints.

in other places in the install-worker.sh script, there are 'aws s3'
commands that would fail if running under something like the
us-gov-east-1 environment.

similar to the changes to the latest-binaries.sh script, update the
'aws' cli calls to ensure the requests are unsinged (to avoid trying
to use us-gov creds against a non-gov endpoint).

and plumb through using the user-specified AWS_ENDPOINT_URL_S3 env var
into the install-worker.sh script so that the alternative endpoints can
be used instead of the us-govcloud ones when running in a govcloud
environment.

Author: joelddiaz

hack/latest-binaries.sh

@@ -13,7 +13,7 @@ MINOR_VERSION="${1}"
 
 # retrieve the available "VERSION/BUILD_DATE" prefixes (e.g. "1.28.1/2023-09-14")
 # from the binary object keys, sorted in descending semver order, and pick the first one
-LATEST_BINARIES=$(aws s3api list-objects-v2 --bucket amazon-eks --prefix "${MINOR_VERSION}" --query 'Contents[*].[Key]' --output text | cut -d'/' -f-2 | sort -Vru | head -n1)
+LATEST_BINARIES=$(aws s3api list-objects-v2 --bucket amazon-eks --prefix "${MINOR_VERSION}" --query 'Contents[*].[Key]' --output text --no-sign-request | cut -d'/' -f-2 | sort -Vru | head -n1)
 
 if [ "${LATEST_BINARIES}" == "None" ]; then
   echo >&2 "No binaries available for minor version: ${MINOR_VERSION}"

templates/al2/provisioners/install-worker.sh

@@ -274,8 +274,8 @@ BINARIES=(
 for binary in ${BINARIES[*]}; do
   if [[ -n "$AWS_ACCESS_KEY_ID" ]]; then
     echo "AWS cli present - using it to copy binaries from s3."
-    aws s3 cp --region $BINARY_BUCKET_REGION $S3_PATH/$binary .
-    aws s3 cp --region $BINARY_BUCKET_REGION $S3_PATH/$binary.sha256 .
+    aws s3 cp --no-sign-request --region $BINARY_BUCKET_REGION $S3_PATH/$binary .
+    aws s3 cp --no-sign-request --region $BINARY_BUCKET_REGION $S3_PATH/$binary.sha256 .
   else
     echo "AWS cli missing - using wget to fetch binaries from s3. Note: This won't work for private bucket."
     sudo wget $S3_URL_BASE/$binary
@@ -308,8 +308,8 @@ if [ "$PULL_CNI_FROM_GITHUB" = "true" ]; then
 else
   if [[ -n "$AWS_ACCESS_KEY_ID" ]]; then
     echo "AWS cli present - using it to copy binaries from s3."
-    aws s3 cp --region $BINARY_BUCKET_REGION $S3_PATH/${CNI_PLUGIN_FILENAME}.tgz .
-    aws s3 cp --region $BINARY_BUCKET_REGION $S3_PATH/${CNI_PLUGIN_FILENAME}.tgz.sha256 .
+    aws s3 cp --no-sign-request --region $BINARY_BUCKET_REGION $S3_PATH/${CNI_PLUGIN_FILENAME}.tgz .
+    aws s3 cp --no-sign-request --region $BINARY_BUCKET_REGION $S3_PATH/${CNI_PLUGIN_FILENAME}.tgz.sha256 .
   else
     echo "AWS cli missing - using wget to fetch cni binaries from s3. Note: This won't work for private bucket."
     sudo wget "$S3_URL_BASE/${CNI_PLUGIN_FILENAME}.tgz"
@@ -369,7 +369,7 @@ sudo chmod +x /etc/eks/max-pods-calculator.sh
 ECR_CREDENTIAL_PROVIDER_BINARY="ecr-credential-provider"
 if [[ -n "$AWS_ACCESS_KEY_ID" ]]; then
   echo "AWS cli present - using it to copy ${ECR_CREDENTIAL_PROVIDER_BINARY} from s3."
-  aws s3 cp --region $BINARY_BUCKET_REGION $S3_PATH/$ECR_CREDENTIAL_PROVIDER_BINARY .
+  aws s3 cp --no-sign-request --region $BINARY_BUCKET_REGION $S3_PATH/$ECR_CREDENTIAL_PROVIDER_BINARY .
 else
   echo "AWS cli missing - using wget to fetch ${ECR_CREDENTIAL_PROVIDER_BINARY} from s3. Note: This won't work for private bucket."
   sudo wget "$S3_URL_BASE/$ECR_CREDENTIAL_PROVIDER_BINARY"

templates/al2/template.json

@@ -33,6 +33,7 @@
     "pull_cni_from_github": null,
     "remote_folder": null,
     "runc_version": null,
+    "aws_endpoint_url_s3": null,
     "security_group_id": null,
     "source_ami_filter_name": null,
     "source_ami_id": null,
@@ -191,6 +192,7 @@
       "script": "{{template_dir}}/provisioners/install-worker.sh",
       "environment_vars": [
         "AWS_ACCESS_KEY_ID={{user `aws_access_key_id`}}",
+	"AWS_ENDPOINT_URL_S3={{ user `aws_endpoint_url_s3`}}",
         "AWS_SECRET_ACCESS_KEY={{user `aws_secret_access_key`}}",
         "AWS_SESSION_TOKEN={{user `aws_session_token`}}",
         "BINARY_BUCKET_NAME={{user `binary_bucket_name`}}",

templates/al2/variables-default.json

@@ -26,6 +26,7 @@
     "pull_cni_from_github": "true",
     "remote_folder": "/tmp",
     "runc_version": "1.1.*",
+    "aws_endpoint_url_s3": "{{env `AWS_ENDPOINT_URL_S3`}}",
     "security_group_id": "",
     "source_ami_filter_name": "amzn2-ami-minimal-hvm-*",
     "source_ami_id": "",