Skip to content

Commit c20dee1

Browse files
tb06904GCHQDeveloper314
tb06904
and
GCHQDeveloper314
authored
Gh-334: Smoother gaffer gremlin deployment (#337)
* add basic proof of concept using proxy store * tidy and hook into existing CI * readme updates * add to publish ci * merge existing demo image with new image * move build to docker compose and tidy --------- Co-authored-by: GCHQDeveloper314 <[email protected]>
1 parent a893cce commit c20dee1

26 files changed

+314
-181
lines changed

Diff for: ‎.github/workflows/continuous-integration.yaml

+2-2
Original file line numberDiff line numberDiff line change
@@ -58,8 +58,8 @@ jobs:
5858
run: ./cd/lint_charts.sh
5959

6060
- name: Deploy to Kubernetes
61-
run: ./cd/deploy_to_kind.sh
62-
61+
run: ./cd/deploy_to_kind.sh ./docker/accumulo2.env
62+
6363
- name: Run gaffer-road-traffic Tests
6464
run: helm test gaffer || (kubectl get po && kubectl describe po && kubectl logs -l app.kubernetes.io/component=test --tail=-1 && df -h && false)
6565

Diff for: ‎.gitignore

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright 2020-2023 Crown Copyright
1+
# Copyright 2020-2024 Crown Copyright
22
#
33
# Licensed under the Apache License, Version 2.0 (the "License");
44
# you may not use this file except in compliance with the License.
@@ -21,3 +21,4 @@
2121
**/.DS_Store
2222
kubernetes/*/Chart.lock
2323
kubernetes/*/charts/
24+
target/

Diff for: ‎cd/build_images.sh

+15-10
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#!/bin/bash
22

3-
# Copyright 2020-2023 Crown Copyright
3+
# Copyright 2020-2024 Crown Copyright
44
#
55
# Licensed under the Apache License, Version 2.0 (the "License");
66
# you may not use this file except in compliance with the License.
@@ -18,31 +18,36 @@
1818
# Required for variables from sourced env file to automatically be visible to docker compose.
1919
set -e -a
2020

21-
root_directory="$( cd $(dirname $(dirname $0)) > /dev/null 2>&1 && pwd )"
22-
cd $root_directory
21+
ROOT_DIR="$(readlink -f "$(dirname "$(dirname "${0}")")")"
2322

24-
if [ ! -z "$1" ]; then
25-
ENV_FILE=$1
26-
else
27-
echo "Error - Environment file not set"; exit 1;
28-
fi
23+
pushd "${ROOT_DIR}" || exit 1
2924

30-
# The following command sets:
25+
# The following env file will be sourced to set:
3126
# HADOOP_VERSION
3227
# GAFFER_VERSION
3328
# GAFFERPY_VERSION
3429
# ACCUMULO_VERSION
3530
# SPARK_VERSION
36-
source "${ENV_FILE}"
31+
# TINKERPOP_VERSION
32+
if [[ -f "${1}" ]]; then
33+
source "${1}"
34+
else
35+
echo "Error - Environment file not set"
36+
exit 1
37+
fi
3738

3839
# Builds all of the Gaffer and Accumulo related images:
3940
docker compose --project-directory ./docker/accumulo/ -f ./docker/accumulo/docker-compose.yaml build
4041
docker compose --project-directory ./docker/gaffer-road-traffic-loader/ -f ./docker/gaffer-road-traffic-loader/docker-compose.yaml build
4142
# Builds all of the notebook related images:
4243
docker compose --project-directory ./docker/gaffer-pyspark-notebook/ -f ./docker/gaffer-pyspark-notebook/docker-compose.yaml build notebook
4344
docker compose --project-directory ./docker/spark-py/ -f ./docker/spark-py/docker-compose.yaml build
45+
# Builds the Gaffer Gremlin server
46+
./docker/gaffer-gremlin/build.sh
4447

4548
# Set $JHUB_OPTIONS_SERVER_VERSION
4649
source ./docker/gaffer-jhub-options-server/get-version.sh
4750
# Builds the jhub options server:
4851
docker compose --project-directory ./docker/gaffer-jhub-options-server/ -f ./docker/gaffer-jhub-options-server/docker-compose.yaml build
52+
53+
popd || exit 1

Diff for: ‎cd/deploy_to_kind.sh

+8-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#!/bin/bash
22

3-
# Copyright 2020-2023 Crown Copyright
3+
# Copyright 2020-2024 Crown Copyright
44
#
55
# Licensed under the Apache License, Version 2.0 (the "License");
66
# you may not use this file except in compliance with the License.
@@ -22,7 +22,13 @@ kind create cluster --quiet --config ./cd/kind.yaml --image kindest/node:v1.24.4
2222
# HADOOP_VERSION
2323
# GAFFER_VERSION
2424
# SPARK_VERSION
25-
source ./docker/gaffer-pyspark-notebook/.env
25+
# KUBECTL_VERSION
26+
if [[ -f "${1}" ]]; then
27+
source "${1}"
28+
else
29+
echo "Error - Environment file not set"
30+
exit 1
31+
fi
2632
# JHUB_OPTIONS_SERVER_VERSION
2733
source ./docker/gaffer-jhub-options-server/get-version.sh
2834

Diff for: ‎cd/publish_images.sh

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#!/bin/bash
22

3-
# Copyright 2020-2023 Crown Copyright
3+
# Copyright 2020-2024 Crown Copyright
44
#
55
# Licensed under the Apache License, Version 2.0 (the "License");
66
# you may not use this file except in compliance with the License.
@@ -101,6 +101,7 @@ pushContainer gchq/accumulo "${ACCUMULO_VERSION}"
101101
pushContainer gchq/gaffer "${GAFFER_VERSION}-accumulo-${ACCUMULO_VERSION}"
102102
pushContainer gchq/gaffer-rest "${GAFFER_VERSION}-accumulo-${ACCUMULO_VERSION}"
103103
pushContainer gchq/gaffer-road-traffic-loader "${GAFFER_VERSION}"
104+
pushContainer gchq/gaffer-gremlin "${GAFFER_VERSION}"
104105
pushContainer gchq/gaffer-pyspark-notebook "${GAFFER_VERSION}"
105106
pushContainer gchq/gaffer-jhub-options-server "${JHUB_OPTIONS_SERVER_VERSION}"
106107
pushContainer gchq/spark-py "${SPARK_VERSION}"

Diff for: ‎cd/update_versions.sh

+3-3
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#!/bin/bash
22

3-
# Copyright 2020-2023 Crown Copyright
3+
# Copyright 2020-2024 Crown Copyright
44
#
55
# Licensed under the Apache License, Version 2.0 (the "License");
66
# you may not use this file except in compliance with the License.
@@ -23,9 +23,9 @@ fi
2323
# GAFFER_VERSION
2424
# GAFFERPY_VERSION
2525
# SPARK_VERSION
26-
source ./docker/gaffer-pyspark-notebook/.env
26+
source ./docker/accumulo2.env
2727
# JHUB_OPTIONS_SERVER_VERSION
28-
source docker/gaffer-jhub-options-server/get-version.sh
28+
source ./docker/gaffer-jhub-options-server/get-version.sh
2929

3030
# hdfs
3131
[ ! -z "${APP_VERSION}" ] && yq eval ".version = \"${APP_VERSION}\"" -i ./kubernetes/hdfs/Chart.yaml

Diff for: ‎docker/accumulo2.env

+2
Original file line numberDiff line numberDiff line change
@@ -7,3 +7,5 @@ GAFFER_VERSION=2.1.0
77
GAFFER_TESTER_VERSION=2.1.0
88
GAFFERPY_VERSION=2.1.0
99
SPARK_VERSION=3.1.2
10+
TINKERPOP_VERSION=3.7.1
11+
KUBECTL_VERSION=1.23.0

Diff for: ‎docker/gaffer-gremlin/Dockerfile

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
# Copyright 2023-2024 Crown Copyright
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
ARG BASE_IMAGE_NAME=tinkerpop/gremlin-server
16+
ARG BASE_IMAGE_TAG=3.7.1
17+
18+
# Base image provides WORKDIR, USER and ENTRYPOINT
19+
FROM ${BASE_IMAGE_NAME}:${BASE_IMAGE_TAG}
20+
21+
# Copy JARS
22+
COPY ./target/dependency/*.jar ext/gafferpop/plugin/
23+
24+
# Copy configs
25+
COPY ./conf/ conf/
26+
27+
# Set server to run
28+
CMD ["conf/gaffer-gremlin-server.yaml"]

Diff for: ‎docker/gaffer-gremlin/README.md

+56
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
# gaffer-gremlin
2+
3+
## Build
4+
5+
Run the supplied `build.sh` script with the required environment vars (can
6+
source `accumolo2.env` file) to pull all dependencies and build the container.
7+
8+
_Note requires Maven and Docker_
9+
10+
Optionally run each command separately to configure the containers tags etc.
11+
12+
## Configure
13+
14+
The container will use the Gaffer Proxy store by default to connect to an
15+
existing graph and provide a Gremlin endpoint to connect to (see the [Tinkerpop docs](https://tinkerpop.apache.org/docs/current/reference/#connecting-gremlin-server)).
16+
17+
The Gaffer graph the container will connect to can be configured as usual by
18+
editing the `store.properties` file, which you can also bind mount over on an
19+
existing image. The config file locations are under the predefined workdir
20+
set by the parent gremlin server image the key locations in the image are:
21+
22+
- `/opt/gremlin-server/conf/gaffer/store.properties` - Override for custom store properties.
23+
- `/opt/gremlin-server/conf/gafferpop/gafferpop.properties` - Override to configure the graph.
24+
25+
The configuration for the Gremlin server is provided by the `gaffer-gremlin-server.yaml`
26+
this again can be modified as needed or bind mounted over. Please see the
27+
[official Gaffer docs](https://gchq.github.io/gaffer-doc/latest/administration-guide/gaffer-deployment/gremlin/)
28+
for more information on configuring this image.
29+
30+
## Run
31+
32+
Simply run the container to publish the Gremlin server making sure to make the
33+
configured port available (port 8182 by default) this can then be connected to
34+
via the address specified in the server's yaml config to use Gremlin traversal.
35+
The server can then be connected via the Gremlin console e.g. if using
36+
`gremlinpython`:
37+
38+
```python
39+
from gremlin_python.process.anonymous_traversal_source import traversal
40+
41+
g = traversal().withRemote(
42+
DriverRemoteConnection('ws://localhost:8182/gremlin', 'g'))
43+
```
44+
45+
### Demo Deployment
46+
47+
A demo/example using the tinkerpop 'modern' dataset and accumulo backed Gaffer
48+
is available under the `example` directory. This can be ran using docker compose
49+
to deploy the containers then, the provided jupiter notebook demonstrates how to
50+
connect and some basic queries on the data using `gremlinpython`.
51+
52+
Run the example with:
53+
54+
```bash
55+
docker compose up
56+
```

Diff for: ‎docker/gaffer-gremlin/build.sh

+38
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
#!/bin/bash
2+
# Copyright 2023-2024 Crown Copyright
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
declare SCRIPT_DIR="$(readlink -f "$(dirname "${0}")")"
17+
18+
# Check if already have env var for version
19+
[[ -z "${GAFFER_VERSION}" ]] && \
20+
echo "Missing GAFFER_VERSION env var" && \
21+
exit 1
22+
[[ -z "${TINKERPOP_VERSION}" ]] && \
23+
echo "Missing TINKERPOP_VERSION env var" && \
24+
exit 1
25+
26+
# Build from relevant directory
27+
pushd "${SCRIPT_DIR}" || exit 1
28+
# Download JARs
29+
mvn clean dependency:copy-dependencies --define gaffer.version="${GAFFER_VERSION}"
30+
31+
# Build container
32+
docker compose build
33+
34+
# Clean
35+
mvn clean
36+
popd || exit 1
37+
38+
echo "Build Successful"

Diff for: ‎docker/gaffer-gremlin/compose.yaml

+25
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
# Copyright 2024 Crown Copyright
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
version: "3.7"
16+
17+
services:
18+
gaffer-gremlin:
19+
image: gchq/gaffer-gremlin:${GAFFER_VERSION}-gremlin-${TINKERPOP_VERSION}
20+
build:
21+
context: .
22+
args:
23+
BASE_IMAGE_TAG: ${TINKERPOP_VERSION}
24+
ports:
25+
- 8182:8182

Diff for: ‎docker/gremlin-gaffer/conf/gremlin-server-empty-gaffer.yaml renamed to ‎docker/gaffer-gremlin/conf/gaffer-gremlin-server.yaml

+11-5
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright 2023 Crown Copyright
1+
# Copyright 2023-2024 Crown Copyright
22
#
33
# Licensed under the Apache License, Version 2.0 (the "License");
44
# you may not use this file except in compliance with the License.
@@ -16,21 +16,27 @@ host: localhost
1616
port: 8182
1717
evaluationTimeout: 30000
1818
graphs: {
19-
graph: conf/gafferpop/gafferpop-tinkerpop-modern.properties}
19+
graph: conf/gafferpop/gafferpop.properties
20+
}
2021
scriptEngines: {
2122
gremlin-groovy: {
2223
plugins: { org.apache.tinkerpop.gremlin.server.jsr223.GremlinServerGremlinPlugin: {},
2324
uk.gov.gchq.gaffer.tinkerpop.gremlinplugin.GafferPopGremlinPlugin: {},
2425
org.apache.tinkerpop.gremlin.groovy.jsr223.GroovyCompilerGremlinPlugin: {enableThreadInterrupt: true},
2526
org.apache.tinkerpop.gremlin.jsr223.ImportGremlinPlugin: {classImports: [java.lang.Math], methodImports: [java.lang.Math#*]},
26-
org.apache.tinkerpop.gremlin.jsr223.ScriptFileGremlinPlugin: {files: [scripts/empty-sample.groovy]}}
27+
org.apache.tinkerpop.gremlin.jsr223.ScriptFileGremlinPlugin: {files: [scripts/empty-sample.groovy]}
28+
}
2729
}
2830
}
2931
serializers:
30-
- { className: org.apache.tinkerpop.gremlin.driver.ser.GraphBinaryMessageSerializerV1, config: { serializeResultToString: true }} # application/vnd.graphbinary-v1.0-stringd
32+
# application/vnd.graphbinary-v1.0
33+
- { className: org.apache.tinkerpop.gremlin.util.ser.GraphBinaryMessageSerializerV1 }
34+
# application/vnd.graphbinary-v1.0-stringd
35+
- { className: org.apache.tinkerpop.gremlin.util.ser.GraphBinaryMessageSerializerV1, config: { serializeResultToString: true }}
3136

3237
metrics: {
33-
slf4jReporter: {enabled: true, interval: 180000}}
38+
slf4jReporter: {enabled: true, interval: 180000}
39+
}
3440
strictTransactionManagement: false
3541
idleConnectionTimeout: 0
3642
keepAliveInterval: 0

Diff for: ‎docker/gaffer-gremlin/conf/gaffer/store.properties

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
#
2+
# Copyright 2023-2024 Crown Copyright
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
#
16+
gaffer.store.class=uk.gov.gchq.gaffer.proxystore.ProxyStore
17+
# These can be configured to an existing graph deployment
18+
gaffer.host=localhost
19+
gaffer.port=8080
20+
gaffer.context-root=/rest/latest

Diff for: ‎docker/gremlin-gaffer/conf/gafferpop/gafferpop-tinkerpop-modern.properties renamed to ‎docker/gaffer-gremlin/conf/gafferpop/gafferpop.properties

+3-3
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#
2-
# Copyright 2016-2023 Crown Copyright
2+
# Copyright 2023-2024 Crown Copyright
33
#
44
# Licensed under the Apache License, Version 2.0 (the "License");
55
# you may not use this file except in compliance with the License.
@@ -15,6 +15,6 @@
1515
#
1616
gremlin.graph=uk.gov.gchq.gaffer.tinkerpop.GafferPopGraph
1717
gaffer.graphId=graph1
18-
gaffer.storeproperties=conf/gafferpop/gaffer/store.properties
19-
gaffer.schemas=conf/gafferpop/gaffer/schema/
18+
gaffer.storeproperties=conf/gaffer/store.properties
19+
gaffer.schemas=conf/gaffer/schema/
2020
gaffer.userId=user01

0 commit comments

Comments
 (0)