Merge branch 'develop'

d47853 · d47853 · commit 40b927238e8d · 2020-08-27T16:49:23.000+01:00
diff --git a/README.md b/README.md
@@ -14,10 +14,22 @@ It also contains Helm Charts so that the following applications can be deployed
 * [Gaffer](kubernetes/gaffer/)
 * [Example Gaffer Graph containing Road Traffic Dataset](kubernetes/gaffer-road-traffic/)
 
+These charts can be accessed by cloning our repository or by using our Helm repo hosted on our [Github Pages Site](https://gchq.github.io/gaffer-docker)
+
+To use this repo, run:
+```bash
+helm repo add gaffer https://gchq.github.io/gaffer-docker
+```
+
 There are guides on how to deploy the charts on:
 * a local Kubernetes cluster, [using kind (Kubernetes IN Docker)](kubernetes/kind-deployment.md)
 * an [AWS EKS cluster](kubernetes/aws-eks-deployment.md)
 
+These guides assume you've cloned the repository and are using the Helm charts contained within. We do also host our Helm Charts index on our Github pages site. You can get access to them by running:
+```bash
+helm repo add gaffer https://gchq.github.io/gaffer-docker
+```
+
 ## Versioning
 Each of our images will be tagged in DockerHub with the version of the software they represent. Every release,
 we update the image for that tag and add a new release which has the corresponding git tag.
diff --git a/app_version b/app_version
@@ -1 +1 @@
-0.6.1
+0.6.2
diff --git a/cd/deploy.sh b/cd/deploy.sh
@@ -40,11 +40,11 @@ uploadChart() {
     chart=$1
     version=$2
     token=$3
+    id=$4
 
     helm dependency update "kubernetes/${chart}"
     helm package "kubernetes/${chart}"
-    curl -v -H "Authorization: token $token" -H "Content-Type: application/zip" --data-binary @${chart}-${version}.tgz "https://api.github.com/repos/gchq/gaffer-docker/releases/tag/v${version}/assets"
-    rm ${chart}-${version}.tgz
+    curl -v -H "Authorization: token $token" -H "Content-Type: application/zip" --data-binary @${chart}-${version}.tgz "https://uploads.github.com/repos/gchq/gaffer-docker/releases/${id}/assets?name=${chart}-${version}.tgz"
 }
 
 # If branch is not master or is pull request, exit
@@ -95,18 +95,22 @@ JSON_DATA="{
                 \"draft\": false
             }"
 echo "${JSON_DATA}"
-curl -v --data "${JSON_DATA}" -H "Authorization: token ${GITHUB_TOKEN}" "https://api.github.com/repos/gchq/${REPO_NAME}/releases"
+curl -v -o output.json --data "${JSON_DATA}" -H "Authorization: token ${GITHUB_TOKEN}" "https://api.github.com/repos/gchq/${REPO_NAME}/releases"
+idLine="$(grep '^  "id"' output.json)"
+id="$(echo ${idLine} | sed -e 's|^"id": \([0-9]*\).*|\1|')"
+rm output.json
 
 # Upload Charts to Github releases
-uploadChart hdfs "${APP_VERSION}" "${GITHUB_TOKEN}"
-uploadChart gaffer "${APP_VERSION}" "${GITHUB_TOKEN}"
-uploadChart gaffer-road-traffic "${APP_VERSION}" "${GITHUB_TOKEN}"
+uploadChart hdfs "${APP_VERSION}" "${GITHUB_TOKEN}" "${id}"
+uploadChart gaffer "${APP_VERSION}" "${GITHUB_TOKEN}" "${id}"
+uploadChart gaffer-road-traffic "${APP_VERSION}" "${GITHUB_TOKEN}" "${id}"
 
 # Update gh-pages
 git checkout gh-pages
-git merge master -m "Updated docs to latest version"
-helm repo index . --url "https://github.com/gchq/gaffer-docker/releases/tag/${TAG_NAME}" --merge index.yaml
+helm repo index . --url "https://github.com/gchq/gaffer-docker/releases/download/${TAG_NAME}" --merge index.yaml
+rm *.tgz
 git commit -am "Updated index.yaml"
+git merge master -m "Updated docs to latest version"
 git push
 
 # Update version on develop
diff --git a/cd/update_app_version.sh b/cd/update_app_version.sh
@@ -43,7 +43,12 @@ createNewVersion() {
 
 # Performs a Find and replace on the Helm charts and the app_version file
 findAndReplace() {
-    find "$(getRootDirectory)" \( -iname Chart.y*ml -o -name app_version \) -exec sed -i'' -e "s:$1:$2:g" {} +
+    # Replace the app version file
+    echo $2 > "$(getRootDirectory)/app_version"
+    # Replace version marked with # managed version in the Chart.yaml files
+    managed_version_tag='# managed version'
+    find "$(getRootDirectory)" -iname Chart.y*ml -exec sed -i'' -e "s:$1 ${managed_version_tag}:$2 ${managed_version_tag}:g" {} +
+
 }
 
 if [ $# -gt 1 ]; then
diff --git a/kubernetes/aws-eks-deployment.md b/kubernetes/aws-eks-deployment.md
@@ -16,7 +16,8 @@ The following instructions will guide you through provisioning and configuring a
 
 If the versions of the container images you would like to deploy are not available in [Docker Hub](https://hub.docker.com/u/gchq) then you will need to host them in a registry yourself.
 
-The following instructions build all the container images and host them in AWS ECR:
+The following instructions build all the container images and host them in AWS ECR when run from the ./kubernetes folder:
+
 
 ```bash
 export HADOOP_VERSION=${HADOOP_VERSION:-3.2.1}
diff --git a/kubernetes/gaffer-road-traffic/Chart.yaml b/kubernetes/gaffer-road-traffic/Chart.yaml
@@ -16,13 +16,13 @@ apiVersion: v2
 name: gaffer-road-traffic
 description: A Gaffer instance containing sample GB road traffic data from the Department of Transport
 type: application
-version: 0.6.1
-appVersion: 1.12.0
+version: 0.6.2 # managed version
+appVersion: 1.12.0 
 home: https://github.com/gchq/Gaffer
 sources:
 - https://github.com/gchq/gaffer-docker
 - https://data.gov.uk/dataset/208c0e7b-353f-4e2d-8b7a-1a7118467acc/gb-road-traffic-counts
 dependencies:
 - name: gaffer
-  version: ^0.6.1
+  version: ^0.6.2 # managed version
   repository: file://../gaffer/
diff --git a/kubernetes/gaffer-road-traffic/docs/aws-eks-deployment.md b/kubernetes/gaffer-road-traffic/docs/aws-eks-deployment.md
@@ -1,7 +1,9 @@
 # Deploying Road Traffic Gaffer Graph on AWS EKS
+All scripts listed here are intended to be run from the kubernetes/gaffer-road-traffic folder
 
 First follow the [instructions here](../../aws-eks-deployment.md) to provision and configure an [AWS EKS](https://aws.amazon.com/eks/) cluster that the Gaffer Road Traffic Helm Chart can be deployed on.
 
+## Using ECR
 If you are hosting the container images in your AWS account, using ECR, then run the following commands to configure the Helm Chart to use them:
 
 ```bash
@@ -23,7 +25,11 @@ else
 fi
 ```
 
-Deploy the Helm Chart with:
+## Deploy Helm Chart
+
+The last thing before deploying is to set the passwords for the various accumulo users in the values.yaml file. These are found under `accumulo.config.userManagement`.
+
+Finally, deploy the Helm Chart by running this from the kubernetes/gaffer-road-traffic folder:
 
 ```
 export HADOOP_VERSION=${HADOOP_VERSION:-3.2.1}
diff --git a/kubernetes/gaffer-road-traffic/docs/kind-deployment.md b/kubernetes/gaffer-road-traffic/docs/kind-deployment.md
@@ -1,7 +1,11 @@
 # Deploying Road Traffic Gaffer Graph using kind
+All scripts listed here are intended to be run from the kubernetes/gaffer-road-traffic folder
 
 First follow the [instructions here](../../kind-deployment.md) to provision and configure a local Kubernetes cluster, using [kind](https://kind.sigs.k8s.io/) (Kubernetes IN Docker), that the Gaffer Road Traffic Helm Chart can be deployed on.
 
+After the cluster is provisioned, update the values.yaml with the passwords for the various accumulo users. These are found under `accumulo.config.userManagement`.
+
+Once that's done, run this to deploy and test the Road Traffic Graph. 
 ```bash
 export HADOOP_VERSION=${HADOOP_VERSION:-3.2.1}
 export GAFFER_VERSION=${GAFFER_VERSION:-1.12.0}
diff --git a/kubernetes/gaffer-road-traffic/templates/check-data-loaded-hook.yaml b/kubernetes/gaffer-road-traffic/templates/check-data-loaded-hook.yaml
@@ -25,6 +25,7 @@ metadata:
     app.kubernetes.io/component: hook
   annotations:
     helm.sh/hook: post-install,post-upgrade
+    helm.sh/hook-delete-policy: hook-succeeded
     helm.sh/hook-weight: "40"
 spec:
   containers:
diff --git a/kubernetes/gaffer-road-traffic/templates/data-loader-job.yaml b/kubernetes/gaffer-road-traffic/templates/data-loader-job.yaml
@@ -36,12 +36,9 @@ spec:
         - name: graph-config
           mountPath: /gaffer/graph/
           readOnly: true
-        # Schema isn't mounted because of GAFFER-2262
-        # Can be un-commented once it has been resolved and released
-        # In the mean time use the schema built into the container
-        # - name: schema
-        #   mountPath: /gaffer/schema/
-        #   readOnly: true
+        - name: schema
+          mountPath: /gaffer/schema/
+          readOnly: true
         - name: store-properties
           mountPath: /gaffer/store/
           readOnly: true
diff --git a/kubernetes/gaffer-road-traffic/templates/tests/integration/hdfs/hdfs-test.yaml b/kubernetes/gaffer-road-traffic/templates/tests/integration/hdfs/hdfs-test.yaml
@@ -25,6 +25,7 @@ metadata:
     app.kubernetes.io/component: test
   annotations:
     helm.sh/hook: test-success
+    helm.sh/hook-delete-policy: hook-succeeded
 spec:
   restartPolicy: Never
   initContainers:
@@ -87,12 +88,9 @@ spec:
     - name: store-properties
       mountPath: /gaffer/store/
       readOnly: true
-    # Schema isn't mounted because of GAFFER-2262
-    # Can be un-commented once it has been resolved and released
-    # In the meantime use the schema built into the container
-    # - name: schema
-    #   mountPath: /gaffer/schema/
-    #   readOnly: true
+    - name: schema
+      mountPath: /gaffer/schema/
+      readOnly: true
   containers:
   - name: hdfs-test
     image: {{ .Values.gaffer.testImages.curl.repository }}:{{ .Values.gaffer.testImages.curl.tag }}
diff --git a/kubernetes/gaffer-road-traffic/templates/tests/test-queries.yaml b/kubernetes/gaffer-road-traffic/templates/tests/test-queries.yaml
@@ -21,6 +21,7 @@ metadata:
     app.kubernetes.io/component: test
   annotations:
     helm.sh/hook: test-success
+    helm.sh/hook-delete-policy: hook-succeeded
 spec:
   restartPolicy: Never
   containers:
diff --git a/kubernetes/gaffer/Chart.yaml b/kubernetes/gaffer/Chart.yaml
@@ -16,7 +16,7 @@ apiVersion: v2
 name: gaffer
 description: A large-scale entity and relation database supporting aggregation of properties
 type: application
-version: 0.6.1
+version: 0.6.2 # managed version
 appVersion: 1.12.0
 home: https://github.com/gchq/Gaffer
 sources:
@@ -27,6 +27,6 @@ dependencies:
   repository: https://kubernetes-charts-incubator.storage.googleapis.com
   condition: zookeeper.enabled
 - name: hdfs
-  version: ^0.6.1
+  version: ^0.6.2 # managed version
   repository: file://../hdfs/
   condition: hdfs.enabled
diff --git a/kubernetes/gaffer/docs/aws-eks-deployment.md b/kubernetes/gaffer/docs/aws-eks-deployment.md
@@ -1,7 +1,9 @@
 # Deploying Gaffer on AWS EKS
+All the scripts found here are designed to be run from the kubernetes/gaffer folder.
 
 First follow the [instructions here](../../aws-eks-deployment.md) to provision and configure an [AWS EKS](https://aws.amazon.com/eks/) cluster that the Gaffer Helm Chart can be deployed on.
 
+## Using ECR
 If you are hosting the container images in your AWS account, using ECR, then run the following commands to configure the Helm Chart to use them:
 
 ```bash
@@ -22,8 +24,10 @@ else
 fi
 ```
 
-Deploy the Helm Chart with:
+## Deploy Helm Chart
+Next you'll need to setup the passwords for the accumulo users in the values.yaml file. These are found under `accumulo.config.userManagement`.
 
+Finally, deploy the Helm Chart:
 ```bash
 export HADOOP_VERSION=${HADOOP_VERSION:-3.2.1}
 export GAFFER_VERSION=${GAFFER_VERSION:-1.12.0}
diff --git a/kubernetes/gaffer/docs/kind-deployment.md b/kubernetes/gaffer/docs/kind-deployment.md
@@ -1,7 +1,12 @@
 # Deploying Gaffer using kind
+All the scripts found here are designed to be run from the kubernetes/gaffer folder.
 
 First follow the [instructions here](../../kind-deployment.md) to provision and configure a local Kubernetes cluster, using [kind](https://kind.sigs.k8s.io/) (Kubernetes IN Docker), that the Gaffer Helm Chart can be deployed on.
 
+Then update the Accumulo users' passwords in the values.yaml file. These are found under `accumulo.config.userManagement`.
+
+Gaffer is then ready be deployed:
+
 ```bash
 export HADOOP_VERSION=${HADOOP_VERSION:-3.2.1}
 export GAFFER_VERSION=${GAFFER_VERSION:-1.12.0}
diff --git a/kubernetes/gaffer/templates/accumulo/post-install-cmds/hook.yaml b/kubernetes/gaffer/templates/accumulo/post-install-cmds/hook.yaml
@@ -22,6 +22,7 @@ metadata:
   name: {{ include "gaffer.fullname" . }}-accumulo-post-install-cmds
   annotations:
     helm.sh/hook: post-install,post-upgrade
+    helm.sh/hook-delete-policy: hook-succeeded
     helm.sh/hook-weight: "20"
   labels:
     {{- include "gaffer.labels" . | nindent 4 }}
diff --git a/kubernetes/gaffer/templates/graph-status-check-hook.yaml b/kubernetes/gaffer/templates/graph-status-check-hook.yaml
@@ -25,6 +25,7 @@ metadata:
     app.kubernetes.io/component: hook
   annotations:
     helm.sh/hook: post-install,post-upgrade
+    helm.sh/hook-delete-policy: hook-succeeded
     helm.sh/hook-weight: "30"
 spec:
   containers:
diff --git a/kubernetes/gaffer/templates/graph-upgrade-hook.yaml b/kubernetes/gaffer/templates/graph-upgrade-hook.yaml
@@ -21,6 +21,7 @@ metadata:
     app.kubernetes.io/component: hook
   annotations:
     helm.sh/hook: post-upgrade
+    helm.sh/hook-delete-policy: hook-succeeded
     helm.sh/hook-weight: "35"
 spec:
   restartPolicy: Never
@@ -31,8 +32,7 @@ spec:
     args:
     - uk.gov.gchq.gaffer.accumulostore.utils.AddUpdateTableIterator
     - {{ .Values.graph.config.graphId }}
-    # Argument for schema directory should be /gaffer/schema/, but this is a workaround for GAFFER-2262
-    - /gaffer/schema/..data/
+    - /gaffer/schema/
     - /gaffer/store/store.properties
     - update
     env:
diff --git a/kubernetes/gaffer/templates/tests/test-graph-status.yaml b/kubernetes/gaffer/templates/tests/test-graph-status.yaml
@@ -20,7 +20,8 @@ metadata:
     {{- include "gaffer.labels" . | nindent 4 }}
     app.kubernetes.io/component: test
   annotations:
-    "helm.sh/hook": test-success
+    helm.sh/hook: test-success
+    helm.sh/hook-delete-policy: hook-succeeded
 spec:
   containers:
   - name: curl
diff --git a/kubernetes/hdfs/Chart.yaml b/kubernetes/hdfs/Chart.yaml
@@ -16,5 +16,5 @@ apiVersion: v2
 name: hdfs
 description: Deploys the Hadoop Distributed File System (HDFS)
 type: application
-version: 0.6.1
+version: 0.6.2 # managed version
 appVersion: 3.2.1
diff --git a/kubernetes/hdfs/docs/aws-eks-deployment.md b/kubernetes/hdfs/docs/aws-eks-deployment.md
@@ -1,7 +1,9 @@
 # Deploying HDFS on AWS EKS
+All scripts listed here are intended to be run from the kubernetes/hdfs folder
 
 First follow the [instructions here](../../aws-eks-deployment.md) to provision and configure an [AWS EKS](https://aws.amazon.com/eks/) cluster that the HDFS Helm Chart can be deployed on.
 
+## Using ECR
 If you are hosting the container images in your AWS account, using ECR, then run the following commands to configure the Helm Chart to use them:
 
 ```bash
@@ -20,7 +22,7 @@ else
 fi
 ```
 
-Deploy the Helm Chart with:
+## Deploy Helm chart
 
 ```bash
 export HADOOP_VERSION=${HADOOP_VERSION:-3.2.1}
diff --git a/kubernetes/hdfs/docs/kind-deployment.md b/kubernetes/hdfs/docs/kind-deployment.md
@@ -1,7 +1,10 @@
 # Deploying HDFS using kind
+All scripts listed here are intended to be run from the kubernetes/hdfs folder
 
 First follow the [instructions here](../../kind-deployment.md) to provision and configure a local Kubernetes cluster, using [kind](https://kind.sigs.k8s.io/) (Kubernetes IN Docker), that the HDFS Helm Chart can be deployed on.
 
+## Deploying Helm charts
+
 ```bash
 export HADOOP_VERSION=${HADOOP_VERSION:-3.2.1}
 
@@ -13,7 +16,6 @@ helm install hdfs . \
 helm test hdfs
 ```
 
-
 ## Accessing Web UI (via `kubectl port-forward`)
 
 ```
@@ -26,17 +28,17 @@ Then browse to: http://localhost:9870
 ## Accessing Web UI (via [Nginx Ingress Controller](https://github.com/kubernetes/ingress-nginx))
 
 Register the FQDNs for each component in DNS e.g.
-```
+```bash
 echo "127.0.0.1 hdfs.k8s.local" | sudo tee -a /etc/hosts
 ```
 
 Update the HDFS deployment to route ingress based on FQDNs:
-```
+```bash
 helm upgrade hdfs . -f ./values-host-based-ingress.yaml
 ```
 
 Set up port forwarding to the nginx ingress controller:
-```
+```bash
 sudo KUBECONFIG=$HOME/.kube/config kubectl port-forward -n ingress-nginx svc/ingress-nginx 80:80
 ```
 
diff --git a/kubernetes/hdfs/templates/post-install-cmds.yaml b/kubernetes/hdfs/templates/post-install-cmds.yaml
@@ -22,6 +22,7 @@ metadata:
   name: {{ template "hdfs.fullname" . }}-post-install-cmds
   annotations:
     helm.sh/hook: post-install,post-upgrade
+    helm.sh/hook-delete-policy: hook-succeeded
     helm.sh/hook-weight: "10"
   labels:
     {{- include "hdfs.labels" . | nindent 4 }}
diff --git a/kubernetes/hdfs/templates/tests/auth-test.yaml b/kubernetes/hdfs/templates/tests/auth-test.yaml
@@ -18,6 +18,7 @@ metadata:
   name: {{ template "hdfs.fullname" . }}-auth-test
   annotations:
     helm.sh/hook: test
+    helm.sh/hook-delete-policy: hook-succeeded
   labels:
     {{- include "hdfs.labels" . | nindent 4 }}
     app.kubernetes.io/component: test
diff --git a/kubernetes/hdfs/templates/tests/webhdfs-test.yaml b/kubernetes/hdfs/templates/tests/webhdfs-test.yaml
@@ -18,6 +18,7 @@ metadata:
   name: {{ template "hdfs.fullname" . }}-webhdfs-test
   annotations:
     helm.sh/hook: test
+    helm.sh/hook-delete-policy: hook-succeeded
   labels:
     {{- include "hdfs.labels" . | nindent 4 }}
     app.kubernetes.io/component: test
diff --git a/kubernetes/kind-deployment.md b/kubernetes/kind-deployment.md
