From fc561e33de90a3806a8b306398fd31aabd297b63 Mon Sep 17 00:00:00 2001 From: Jarno Henneman Date: Fri, 9 Mar 2018 20:09:39 +0800 Subject: [PATCH 1/5] Make ENV variables optional, so Role on container can be used --- start.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/start.sh b/start.sh index 115733f..5da9e0b 100644 --- a/start.sh +++ b/start.sh @@ -2,9 +2,13 @@ set -e -export AWS_ACCESS_KEY_ID=$KEY -export AWS_SECRET_ACCESS_KEY=$SECRET -export AWS_DEFAULT_REGION=$REGION +if [[ $KEY ]]; then + echo "Set AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_DEFAULT_REGION" + + export AWS_ACCESS_KEY_ID=$KEY + export AWS_SECRET_ACCESS_KEY=$SECRET + export AWS_DEFAULT_REGION=$REGION +fi if [[ "$1" == 'now' ]]; then exec /sync.sh From b51e286eca41bf9d6f2c8b8028ee548c7bc2dbb9 Mon Sep 17 00:00:00 2001 From: Jarno Henneman Date: Fri, 9 Mar 2018 22:28:21 +0800 Subject: [PATCH 2/5] . --- start.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/start.sh b/start.sh index 5da9e0b..b3ce29c 100644 --- a/start.sh +++ b/start.sh @@ -2,12 +2,14 @@ set -e -if [[ $KEY ]]; then +if [[ -z "$KEY" ]] || [[ -z "$SECRET" ]] || [[ -z "$REGION" ]]; then echo "Set AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_DEFAULT_REGION" export AWS_ACCESS_KEY_ID=$KEY export AWS_SECRET_ACCESS_KEY=$SECRET export AWS_DEFAULT_REGION=$REGION +else + echo "Missing one or more ENV variables, using container role" fi if [[ "$1" == 'now' ]]; then From 9900625ee9a9ab24b8683a239546aee8edc75033 Mon Sep 17 00:00:00 2001 From: Jarno Henneman Date: Fri, 16 Mar 2018 12:23:36 +0800 Subject: [PATCH 3/5] Updated Readme.md with AWS Role. --- README.md | 91 ++++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 66 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 7f5980a..f70c019 100644 --- a/README.md +++ b/README.md @@ -1,26 +1,24 @@ -[![](https://images.microbadger.com/badges/image/futurevision/aws-s3-sync.svg)](https://microbadger.com/images/futurevision/aws-s3-sync "Get your own image badge on microbadger.com") -[![](https://images.microbadger.com/badges/version/futurevision/aws-s3-sync.svg)](https://microbadger.com/images/futurevision/aws-s3-sync "Get your own version badge on microbadger.com") - -# futurevision/aws-s3-sync +# docker-aws-s3-sync Docker container that periodically syncs a folder to Amazon S3 using the [AWS Command Line Interface tool](https://aws.amazon.com/cli/) and cron. +Can be used with AWS Credentials and ECS Container Role + ## Usage docker run -d [OPTIONS] futurevision/aws-s3-sync - ### Required Parameters: -* `-e KEY=`: User Access Key -* `-e SECRET=`: User Access Secret -* `-e REGION=`: Region of your bucket -* `-e BUCKET=`: The name of your bucket +* `-e BUCKET=`: The name of your bucket, ex. dev-efs-8xabdop9fqb1 * `-v /path/to/backup:/data:ro`: mount target local folder to container's data folder. Content of this folder will be synced with S3 bucket. ### Optional parameters: +* `-e KEY=`: User Access Key, if not using a container role +* `-e SECRET=`: User Access Secret,if not using a container role +* `-e REGION=`: Region of your bucket, if not using a container role * `-e PARAMS=`: parameters to pass to the sync command ([full list here](http://docs.aws.amazon.com/cli/latest/reference/s3/sync.html)). * `-e BUCKET_PATH=`: The path of your s3 bucket where the files should be synced to (must start with a slash), defaults to "/" to sync to bucket root * `-e CRON_SCHEDULE="0 1 * * *"`: specifies when cron job starts ([details](http://en.wikipedia.org/wiki/Cron)), defaults to `0 1 * * *` (runs every night at 1:00). @@ -28,30 +26,73 @@ Docker container that periodically syncs a folder to Amazon S3 using the [AWS Co ## Examples: +### Once a hour Sync every hour with cron schedule (container keeps running): docker run -d \ - -e KEY=mykey \ - -e SECRET=mysecret \ - -e REGION=region \ - -e BUCKET=mybucket \ - -e CRON_SCHEDULE="0 * * * *" \ - -e BUCKET_PATH=/path \ - -v /home/user/data:/data:ro \ - futurevision/aws-s3-sync - + -e KEY=mykey \ + -e SECRET=mysecret \ + -e REGION=eu-central-1 \ + -e BUCKET=dev-efs-8xabdop9fqb1 \ + -e CRON_SCHEDULE="0 * * * *" \ + -e BUCKET_PATH=/path \ + -v /home/user/data:/data:ro \ + futurevision/aws-s3-sync + +### Only once Sync just once (container is deleted afterwards): docker run --rm \ - -e KEY=mykey \ - -e SECRET=mysecret \ - -e REGION=region \ - -e BUCKET=mybucket \ - -v /home/user/data:/data:ro \ - futurevision/aws-s3-sync no-cron + -e KEY=mykey \ + -e SECRET=mysecret \ + -e REGION=eu-central-1 \ + -e BUCKET=dev-efs-8xabdop9fqb1 \ + -v /home/user/data:/data:ro \ + futurevision/aws-s3-sync no-cron + +### AWS Role + + docker run -d \ + -e BUCKET=dev-efs-8xabdop9fqb1 \ + -e CRON_SCHEDULE="0 * * * *" \ + -v /home/user/data:/data:ro \ + futurevision/aws-s3-sync + +#### AWS Policy role + + { + "Version": "2012-10-17", + "Statement": [ + { + "Action": [ + "s3:ListBucket", + "s3:GetBucketLocation" + ], + "Resource": "arn:aws:s3:::dev-efs-8xabdop9fqb1", + "Effect": "Allow" + }, + { + "Action": [ + "s3:PutObject", + "s3:PutObjectAcl", + "s3:PutObjectTagging", + "s3:Get*", + "s3:DeleteObject", + "s3:DeleteObjectTagging" + ], + "Resource": "arn:aws:s3:::dev-efs-8xabdop9fqb1/*", + "Effect": "Allow" + } + ] + } + +More information about TaskDefinition roles can be seen in the [AWS Developer guide](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/create-task-definition.html) under Using the EC2 launch type compatibility template. ## Credits This container is heavily inspired by [istepanov/backup-to-s3](https://github.com/istepanov/docker-backup-to-s3/blob/master/README.md). -The main difference is that this container is using Alpine Linux instead of Debian to be more light weight. It also uses a different method of using the AWS CLI tool. +Key differences are; +- The container is using Alpine Linux instead of Debian to be more light weight. +- It uses different methods of the AWS CLI tool. +- Supports AWS Role rather then AWS Credentials. From e59a6207a9eba57a387ad9b8383be415972cf0c4 Mon Sep 17 00:00:00 2001 From: Jarno Henneman Date: Wed, 18 Apr 2018 11:51:08 +0800 Subject: [PATCH 4/5] Updated to alpine:3.7 --- Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 40a94f4..bc085b1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,9 @@ -FROM alpine:3.3 +FROM alpine:3.7 RUN apk --no-cache add \ py-pip \ - python &&\ + python \ + curl &&\ pip install --upgrade \ pip \ awscli From e4e4a170d9ec40c64871d6dccc7c1d4d071a7b74 Mon Sep 17 00:00:00 2001 From: Jarno Henneman Date: Wed, 18 Apr 2018 16:34:42 +0800 Subject: [PATCH 5/5] Update README.md with mentioned of Network Mode to Host --- README.md | 2 ++ sync.sh | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f70c019..8213699 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,8 @@ Sync just once (container is deleted afterwards): -v /home/user/data:/data:ro \ futurevision/aws-s3-sync +If using AWS Role based, make sure to set Network Mode to host. + #### AWS Policy role { diff --git a/sync.sh b/sync.sh index 12bee6d..3a5290f 100644 --- a/sync.sh +++ b/sync.sh @@ -6,4 +6,4 @@ echo "$(date) - Start" aws s3 sync /data s3://$BUCKET$BUCKET_PATH $PARAMS -echo "$(date) End" +echo "$(date) - End"