Merge branch 'develop'

Author: d47853

.gitignore

@@ -14,6 +14,7 @@
 
 */*.iml
 *.iml
+.idea
 */.settings
 */.project
 .DS_Store

app_version

@@ -1 +1 @@
-0.5.0
+0.6.0

cd/deploy.sh

@@ -35,6 +35,18 @@ pushTags() {
     done
 }
 
+# Adds an Chart to a github release 
+uploadChart() {
+    chart=$1
+    version=$2
+    token=$3
+
+    helm dependency update "kubernetes/${chart}"
+    helm package "kubernetes/${chart}"
+    curl -v -H "Authorization: token $token" -H "Content-Type: application/zip" --data-binary @${chart}-${version}.tgz "https://api.github.com/repos/gchq/gaffer-docker/releases/tag/v${version}/assets"
+    rm ${chart}-${version}.tgz
+}
+
 # If branch is not master or is pull request, exit
 if [ "${TRAVIS_PULL_REQUEST}" != "false" ] || [ "${TRAVIS_BRANCH}" != "master" ]; then
     exit 0
@@ -66,7 +78,7 @@ git config --global credential.helper "store --file=.git/credentials"
 echo "https://${GITHUB_TOKEN}:@github.com" > .git/credentials
 
 # Add Develop branch
-git remote set-branches --add origin develop
+git remote set-branches --add origin develop gh-pages
 git pull
 
 # Tag release in Git
@@ -79,22 +91,26 @@ REPO_NAME="Gaffer-Docker"
 JSON_DATA="{
                 \"tag_name\": \"${TAG_NAME}\",
                 \"name\": \"${REPO_NAME} ${APP_VERSION}\",
-                \"body\": \"[${APP_VERSION} headliners](https://github.com/gchq/${REPO_NAME}/issues?q=milestone%3A${TAG_NAME}+label%3Aheadliner)
-
-[${APP_VERSION} enhancements](https://github.com/gchq/${REPO_NAME}/issues?q=milestone%3A${TAG_NAME}+label%3Aenhancement)
-
-[${APP_VERSION} bugs fixed](https://github.com/gchq/${REPO_NAME}/issues?q=milestone%3A${TAG_NAME}+label%3Abug)
-
-[${APP_VERSION} migration notes](https://github.com/gchq/${REPO_NAME}/issues?q=milestone%3A${TAG_NAME}+label%3Amigration-required)
-
-[${APP_VERSION} all issues resolved](https://github.com/gchq/${REPO_NAME}/issues?q=milestone%3A${TAG_NAME})\",
+                \"body\": \"[${APP_VERSION} headliners](https://github.com/gchq/${REPO_NAME}/issues?q=milestone%3A${TAG_NAME}+label%3Aheadliner)\n\n[${APP_VERSION} enhancements](https://github.com/gchq/${REPO_NAME}/issues?q=milestone%3A${TAG_NAME}+label%3Aenhancement)\n\n[${APP_VERSION} bugs fixed](https://github.com/gchq/${REPO_NAME}/issues?q=milestone%3A${TAG_NAME}+label%3Abug)\n\n[${APP_VERSION} migration notes](https://github.com/gchq/${REPO_NAME}/issues?q=milestone%3A${TAG_NAME}+label%3Amigration-required)\n\n[${APP_VERSION} all issues resolved](https://github.com/gchq/${REPO_NAME}/issues?q=milestone%3A${TAG_NAME})\",
                 \"draft\": false
             }"
 echo "${JSON_DATA}"
-curl -v --data "${JSON_DATA}" https://api.github.com/repos/gchq/"${REPO_NAME}"/releases?access_token="${GITHUB_TOKEN}"
+curl -v --data "${JSON_DATA}" -H "Authorization: token ${GITHUB_TOKEN}" "https://api.github.com/repos/gchq/${REPO_NAME}/releases"
 
 # Update version on develop
 git checkout develop
 ./cd/update_app_version.sh
 git commit -a -m "Updated App version"
 git push
+
+# Upload Charts to Github releases
+uploadChart hdfs "${APP_VERSION}" "${GITHUB_TOKEN}"
+uploadChart gaffer "${APP_VERSION}" "${GITHUB_TOKEN}"
+uploadChart gaffer-road-traffic "${APP_VERSION}" "${GITHUB_TOKEN}"
+
+# Update gh-pages
+git checkout gh-pages
+git merge master -m "Updated docs to latest version"
+helm repo index . --url "https://github.com/gchq/gaffer-docker/releases/tag/${TAG_NAME}" --merge index.yaml
+git commit -am "Updated index.yaml"
+git push

cd/install.sh

@@ -50,10 +50,10 @@ buildImages
 
 # Deploy Images to Kind
 kind load docker-image gchq/hdfs:3.2.1
-kind load docker-image gchq/gaffer:1.11.0
-kind load docker-image gchq/gaffer-rest:1.11.0
-kind load docker-image gchq/gaffer-road-traffic-loader:1.11.0
-kind load docker-image gchq/gaffer-operation-runner:1.11.0
+kind load docker-image gchq/gaffer:1.12.0
+kind load docker-image gchq/gaffer-rest:1.12.0
+kind load docker-image gchq/gaffer-road-traffic-loader:1.12.0
+kind load docker-image gchq/gaffer-operation-runner:1.12.0
 
 # Deploy containers onto Kind
 # Travis needs this setting to avoid reverse dns lookup errors

docker/accumulo/docker-compose.yaml

@@ -50,10 +50,6 @@ services:
     depends_on:
     - hdfs-namenode
     image: gchq/hdfs:${HADOOP_VERSION}
-    build:
-      context: ../hdfs/
-      args:
-        HADOOP_VERSION: ${HADOOP_VERSION}
     command: datanode
     container_name: hdfs-datanode
     hostname: hdfs-datanode
@@ -96,12 +92,6 @@ services:
     depends_on:
     - accumulo-master
     image: gchq/accumulo:${ACCUMULO_VERSION}
-    build:
-      context: .
-      args:
-        ACCUMULO_VERSION: ${ACCUMULO_VERSION}
-        HADOOP_VERSION: ${HADOOP_VERSION}
-        ZOOKEEPER_VERSION: ${ZOOKEEPER_VERSION}
     command: tserver
     restart: on-failure
     container_name: accumulo-tserver
@@ -121,12 +111,6 @@ services:
     depends_on:
     - accumulo-master
     image: gchq/accumulo:${ACCUMULO_VERSION}
-    build:
-      context: .
-      args:
-        ACCUMULO_VERSION: ${ACCUMULO_VERSION}
-        HADOOP_VERSION: ${HADOOP_VERSION}
-        ZOOKEEPER_VERSION: ${ZOOKEEPER_VERSION}
     command: monitor
     restart: on-failure
     container_name: accumulo-monitor
@@ -148,12 +132,6 @@ services:
     depends_on:
     - accumulo-master
     image: gchq/accumulo:${ACCUMULO_VERSION}
-    build:
-      context: .
-      args:
-        ACCUMULO_VERSION: ${ACCUMULO_VERSION}
-        HADOOP_VERSION: ${HADOOP_VERSION}
-        ZOOKEEPER_VERSION: ${ZOOKEEPER_VERSION}
     command: gc
     restart: on-failure
     container_name: accumulo-gc

docker/gaffer-operation-runner/.env

@@ -1,5 +1,5 @@
-GAFFER_VERSION=1.11.0
-GAFFER_TOOLS_VERSION=1.11.0
+GAFFER_VERSION=1.12.0
+GAFFER_TOOLS_VERSION=1.12.0
 ACCUMULO_VERSION=1.9.3
 HADOOP_VERSION=3.2.1
 ACCUMULO_CONF_DIR=/etc/accumulo/conf

docker/gaffer-operation-runner/Dockerfile

@@ -16,7 +16,7 @@ ARG BUILDER_IMAGE_NAME=maven
 ARG BUILDER_IMAGE_TAG=3.6.3-jdk-8
 
 ARG BASE_IMAGE_NAME=gchq/gaffer
-ARG BASE_IMAGE_TAG=1.11.0
+ARG BASE_IMAGE_TAG=1.12.0
 
 FROM ${BUILDER_IMAGE_NAME}:${BUILDER_IMAGE_TAG} as builder
 

docker/gaffer-operation-runner/docker-compose.yaml

@@ -50,10 +50,6 @@ services:
     depends_on:
     - hdfs-namenode
     image: gchq/hdfs:${HADOOP_VERSION}
-    build:
-      context: ../hdfs/
-      args:
-        HADOOP_VERSION: ${HADOOP_VERSION}
     command: datanode
     container_name: hdfs-datanode
     hostname: hdfs-datanode
@@ -96,12 +92,6 @@ services:
     depends_on:
     - accumulo-master
     image: gchq/gaffer:${GAFFER_VERSION}
-    build:
-      context: ../gaffer/
-      args:
-        GAFFER_VERSION: ${GAFFER_VERSION}
-        BASE_IMAGE_NAME: gchq/accumulo
-        BASE_IMAGE_TAG: ${ACCUMULO_VERSION}
     command: tserver
     restart: on-failure
     container_name: accumulo-tserver
@@ -121,12 +111,6 @@ services:
     depends_on:
     - accumulo-master
     image: gchq/gaffer:${GAFFER_VERSION}
-    build:
-      context: ../gaffer/
-      args:
-        GAFFER_VERSION: ${GAFFER_VERSION}
-        BASE_IMAGE_NAME: gchq/accumulo
-        BASE_IMAGE_TAG: ${ACCUMULO_VERSION}
     command: monitor
     restart: on-failure
     container_name: accumulo-monitor
@@ -148,12 +132,6 @@ services:
     depends_on:
     - accumulo-master
     image: gchq/gaffer:${GAFFER_VERSION}
-    build:
-      context: ../gaffer/
-      args:
-        GAFFER_VERSION: ${GAFFER_VERSION}
-        BASE_IMAGE_NAME: gchq/accumulo
-        BASE_IMAGE_TAG: ${ACCUMULO_VERSION}
     command: gc
     restart: on-failure
     container_name: accumulo-gc

docker/gaffer-operation-runner/operation-runner/pom.xml

@@ -35,7 +35,7 @@
     <dependency>
       <groupId>uk.gov.gchq.gaffer</groupId>
       <artifactId>graph</artifactId>
-      <version>1.11.0</version>
+      <version>1.12.0</version>
     </dependency>
   </dependencies>
 

docker/gaffer-rest/.env

@@ -1,2 +1,2 @@
-GAFFER_VERSION=1.11.0
-GAFFER_TOOLS_VERSION=1.11.0
+GAFFER_VERSION=1.12.0
+GAFFER_TOOLS_VERSION=1.12.0

docker/gaffer-rest/Dockerfile

@@ -20,7 +20,7 @@ ARG BASE_IMAGE_TAG=10.1.0.Final
 
 FROM ${BUILDER_IMAGE_NAME}:${BUILDER_IMAGE_TAG} as builder
 
-ARG GAFFER_VERSION=1.11.0
+ARG GAFFER_VERSION=1.12.0
 ARG GAFFER_TOOLS_VERSION=${GAFFER_VERSION}
 ARG GAFFER_GIT_REPO=https://github.com/gchq/Gaffer.git
 ARG GAFFER_TOOLS_GIT_REPO=https://github.com/gchq/gaffer-tools.git

docker/gaffer-road-traffic-loader/.env

@@ -1,5 +1,5 @@
-GAFFER_VERSION=1.11.0
-GAFFER_TOOLS_VERSION=1.11.0
+GAFFER_VERSION=1.12.0
+GAFFER_TOOLS_VERSION=1.12.0
 ACCUMULO_VERSION=1.9.3
 HADOOP_VERSION=3.2.1
 ACCUMULO_CONF_DIR=/etc/accumulo/conf

docker/gaffer-road-traffic-loader/Dockerfile

@@ -20,7 +20,7 @@ ARG BASE_IMAGE_TAG=8u242-jre-slim-buster
 
 FROM ${BUILDER_IMAGE_NAME}:${BUILDER_IMAGE_TAG} as builder
 
-ARG GAFFER_VERSION=1.11.0
+ARG GAFFER_VERSION=1.12.0
 ARG GAFFER_GIT_REPO=https://github.com/gchq/Gaffer.git
 ARG GAFFER_DOWNLOAD_URL=https://repo1.maven.org/maven2
 

docker/gaffer-road-traffic-loader/docker-compose.yaml

@@ -50,10 +50,6 @@ services:
     depends_on:
     - hdfs-namenode
     image: gchq/hdfs:${HADOOP_VERSION}
-    build:
-      context: ../hdfs/
-      args:
-        HADOOP_VERSION: ${HADOOP_VERSION}
     command: datanode
     container_name: hdfs-datanode
     hostname: hdfs-datanode
@@ -96,12 +92,6 @@ services:
     depends_on:
     - accumulo-master
     image: gchq/gaffer:${GAFFER_VERSION}
-    build:
-      context: ../gaffer/
-      args:
-        GAFFER_VERSION: ${GAFFER_VERSION}
-        BASE_IMAGE_NAME: gchq/accumulo
-        BASE_IMAGE_TAG: ${ACCUMULO_VERSION}
     command: tserver
     restart: on-failure
     container_name: accumulo-tserver
@@ -121,12 +111,6 @@ services:
     depends_on:
     - accumulo-master
     image: gchq/gaffer:${GAFFER_VERSION}
-    build:
-      context: ../gaffer/
-      args:
-        GAFFER_VERSION: ${GAFFER_VERSION}
-        BASE_IMAGE_NAME: gchq/accumulo
-        BASE_IMAGE_TAG: ${ACCUMULO_VERSION}
     command: monitor
     restart: on-failure
     container_name: accumulo-monitor
@@ -148,12 +132,6 @@ services:
     depends_on:
     - accumulo-master
     image: gchq/gaffer:${GAFFER_VERSION}
-    build:
-      context: ../gaffer/
-      args:
-        GAFFER_VERSION: ${GAFFER_VERSION}
-        BASE_IMAGE_NAME: gchq/accumulo
-        BASE_IMAGE_TAG: ${ACCUMULO_VERSION}
     command: gc
     restart: on-failure
     container_name: accumulo-gc

docker/gaffer/.env

@@ -1,5 +1,5 @@
-GAFFER_VERSION=1.11.0
-GAFFER_TOOLS_VERSION=1.11.0
+GAFFER_VERSION=1.12.0
+GAFFER_TOOLS_VERSION=1.12.0
 ACCUMULO_VERSION=1.9.3
 HADOOP_VERSION=3.2.1
 ACCUMULO_CONF_DIR=/etc/accumulo/conf

docker/gaffer/Dockerfile

@@ -20,7 +20,7 @@ ARG BASE_IMAGE_TAG=1.9.3
 
 FROM ${BUILDER_IMAGE_NAME}:${BUILDER_IMAGE_TAG} as builder
 
-ARG GAFFER_VERSION=1.11.0
+ARG GAFFER_VERSION=1.12.0
 ARG GAFFER_LIBS=bitmap-library,sketches-library,time-library
 ARG GAFFER_DOWNLOAD_URL=https://repo1.maven.org/maven2
 ARG GAFFER_GIT_REPO=https://github.com/gchq/Gaffer.git

docker/gaffer/docker-compose.yaml

@@ -50,10 +50,6 @@ services:
     depends_on:
     - hdfs-namenode
     image: gchq/hdfs:${HADOOP_VERSION}
-    build:
-      context: ../hdfs/
-      args:
-        HADOOP_VERSION: ${HADOOP_VERSION}
     command: datanode
     container_name: hdfs-datanode
     hostname: hdfs-datanode
@@ -96,12 +92,6 @@ services:
     depends_on:
     - accumulo-master
     image: gchq/gaffer:${GAFFER_VERSION}
-    build:
-      context: .
-      args:
-        GAFFER_VERSION: ${GAFFER_VERSION}
-        BASE_IMAGE_NAME: gchq/accumulo
-        BASE_IMAGE_TAG: ${ACCUMULO_VERSION}
     command: tserver
     restart: on-failure
     container_name: accumulo-tserver
@@ -121,12 +111,6 @@ services:
     depends_on:
     - accumulo-master
     image: gchq/gaffer:${GAFFER_VERSION}
-    build:
-      context: .
-      args:
-        GAFFER_VERSION: ${GAFFER_VERSION}
-        BASE_IMAGE_NAME: gchq/accumulo
-        BASE_IMAGE_TAG: ${ACCUMULO_VERSION}
     command: monitor
     restart: on-failure
     container_name: accumulo-monitor
@@ -148,12 +132,6 @@ services:
     depends_on:
     - accumulo-master
     image: gchq/gaffer:${GAFFER_VERSION}
-    build:
-      context: .
-      args:
-        GAFFER_VERSION: ${GAFFER_VERSION}
-        BASE_IMAGE_NAME: gchq/accumulo
-        BASE_IMAGE_TAG: ${ACCUMULO_VERSION}
     command: gc
     restart: on-failure
     container_name: accumulo-gc

docker/hdfs/docker-compose.yaml

@@ -39,10 +39,6 @@ services:
     depends_on:
     - hdfs-namenode
     image: gchq/hdfs:${HADOOP_VERSION}
-    build:
-      context: .
-      args:
-        HADOOP_VERSION: ${HADOOP_VERSION}
     command: datanode
     container_name: hdfs-datanode
     hostname: hdfs-datanode

kubernetes/aws-eks-deployment.md

@@ -20,7 +20,7 @@ The following instructions build all the container images and host them in AWS E
 
 ```bash
 export HADOOP_VERSION=${HADOOP_VERSION:-3.2.1}
-export GAFFER_VERSION=${GAFFER_VERSION:-1.11.0}
+export GAFFER_VERSION=${GAFFER_VERSION:-1.12.0}
 export GAFFER_TOOLS_VERSION=${GAFFER_TOOLS_VERSION:-$GAFFER_VERSION}
 
 docker-compose --project-directory ../docker/accumulo/ -f ../docker/accumulo/docker-compose.yaml build
@@ -38,7 +38,7 @@ for repo in ${HADOOP_IMAGES} ${GAFFER_IMAGES}; do
   aws ecr create-repository --repository-name gchq/${repo}
 done
 
-$(aws ecr get-login --no-include-email)
+echo $(aws ecr get-login-password) | docker login -u AWS --password-stdin https://${ACCOUNT}.dkr.ecr.${REGION}.amazonaws.com
 
 for repo in ${HADOOP_IMAGES}; do
   docker image tag gchq/${repo}:${HADOOP_VERSION} ${REPO_PREFIX}/${repo}:${HADOOP_VERSION}