From 6d3b2a0a5ac14a0099e19931e09e7b0f7f27df30 Mon Sep 17 00:00:00 2001 From: geoHeil Date: Sat, 21 Jan 2017 19:13:23 +0100 Subject: [PATCH 01/11] add instructions to get started with the tutorial locally via docker --- README.md | 23 +++++++++++++++++++++++ build.sh | 4 ++-- 2 files changed, 25 insertions(+), 2 deletions(-) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..564ffa6 --- /dev/null +++ b/README.md @@ -0,0 +1,23 @@ +# building a current image + +- update accumulo and geomesa version number +git clone https://github.com/locationtech/geomesa.git +cd geomesa +git checkout geomesa_2.11-1.3.0 +update ./build.sh to desired version numbers then +build/mvn clean install -DskipTests=true #test failures otherwise (timeouts on macbook) + +## to start the image + +cd geodocker-accumulo-geomesa +docker-compose up + +## geomesa tutorial quickstart + +git clone the tutorials + +make sure versions are matching (you can get git clone https://github.com/geoHeil/geomesa-tutorials.git) +Change all pom xml files + +build the quickstart via +mvn clean install -pl geomesa-quickstart-accumulo \ No newline at end of file diff --git a/build.sh b/build.sh index 164f09e..7e03794 100755 --- a/build.sh +++ b/build.sh @@ -2,8 +2,8 @@ usage() { echo "Usage: $0 [-t ] [-g ] [-a ]" 1>&2; exit 1; } -ACCUMULO_VERSION=1.7.2 -GEOMESA_VERSION=1.3.0-m3-SNAPSHOT +ACCUMULO_VERSION=1.8.0 +GEOMESA_VERSION=1.3.0 while getopts ":t:g:a:" o; do case "${o}" in From 6c20d767c3009f3ffe4a2c9e4dc13aa85bb53de9 Mon Sep 17 00:00:00 2001 From: geoHeil Date: Sat, 21 Jan 2017 19:50:36 +0100 Subject: [PATCH 02/11] start on running the tutorial, unclear how to connect / what are the credentials --- README.md | 16 +++++++++++++++- geodocker-accumulo-geomesa/docker-compose.yml | 5 +++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 564ffa6..a3855d4 100644 --- a/README.md +++ b/README.md @@ -20,4 +20,18 @@ make sure versions are matching (you can get git clone https://github.com/geoHei Change all pom xml files build the quickstart via -mvn clean install -pl geomesa-quickstart-accumulo \ No newline at end of file +mvn clean install -pl geomesa-quickstart-accumulo + +copy jar into java container which is connected to geomesa +docker cp geomesa-quickstart-accumulo/target/geomesa-quickstart-accumulo-1.3.0.jar java8JDK:/geomesaQuickstart.jar +docker exec -ti bash java8JDK bash +java -cp geomesaQuickstart.jar \ + com.example.geomesa.accumulo.AccumuloQuickStart \ + -instanceId accumulo \ + -zookeepers zookeeper \ + -user accumulo \ + -password secret \ + -tableName quickstart + + **todo** + what is the user /password of the default dockerized geDocker version? \ No newline at end of file diff --git a/geodocker-accumulo-geomesa/docker-compose.yml b/geodocker-accumulo-geomesa/docker-compose.yml index bd69961..af148c5 100644 --- a/geodocker-accumulo-geomesa/docker-compose.yml +++ b/geodocker-accumulo-geomesa/docker-compose.yml @@ -1,5 +1,10 @@ version: '2' services: + tutorial: + image: java:8-jdk + links: + - zookeeper:zookeeper + - accumulo-master:accumulo geoserver: image: quay.io/geomesa/geoserver:latest ports: From 582673e22830e6a3a7fd07cd0c02d439e83437be Mon Sep 17 00:00:00 2001 From: geoHeil Date: Sat, 21 Jan 2017 19:52:32 +0100 Subject: [PATCH 03/11] reformat --- README.md | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index a3855d4..6132f70 100644 --- a/README.md +++ b/README.md @@ -1,30 +1,41 @@ # building a current image - update accumulo and geomesa version number +``` git clone https://github.com/locationtech/geomesa.git cd geomesa git checkout geomesa_2.11-1.3.0 update ./build.sh to desired version numbers then build/mvn clean install -DskipTests=true #test failures otherwise (timeouts on macbook) +``` ## to start the image +``` cd geodocker-accumulo-geomesa docker-compose up +``` ## geomesa tutorial quickstart +``` git clone the tutorials - +``` make sure versions are matching (you can get git clone https://github.com/geoHeil/geomesa-tutorials.git) Change all pom xml files build the quickstart via +``` mvn clean install -pl geomesa-quickstart-accumulo - -copy jar into java container which is connected to geomesa +``` +copy jar into java container which is connected to geomesa and connect into the java container +``` docker cp geomesa-quickstart-accumulo/target/geomesa-quickstart-accumulo-1.3.0.jar java8JDK:/geomesaQuickstart.jar docker exec -ti bash java8JDK bash +``` + +run the quickstart. This requires connection credentials +``` java -cp geomesaQuickstart.jar \ com.example.geomesa.accumulo.AccumuloQuickStart \ -instanceId accumulo \ @@ -32,6 +43,6 @@ java -cp geomesaQuickstart.jar \ -user accumulo \ -password secret \ -tableName quickstart - +``` **todo** what is the user /password of the default dockerized geDocker version? \ No newline at end of file From 74f1f405e74553bc977a4542f671f110828a9a7c Mon Sep 17 00:00:00 2001 From: geoHeil Date: Sun, 22 Jan 2017 13:05:00 +0100 Subject: [PATCH 04/11] use own images --- geodocker-accumulo-geomesa/docker-compose.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/geodocker-accumulo-geomesa/docker-compose.yml b/geodocker-accumulo-geomesa/docker-compose.yml index af148c5..85d1b2e 100644 --- a/geodocker-accumulo-geomesa/docker-compose.yml +++ b/geodocker-accumulo-geomesa/docker-compose.yml @@ -10,14 +10,14 @@ services: ports: - 9090:9090 hdfs-name: - image: quay.io/geomesa/hdfs:latest + image: quay.io/geomesa/hdfs:geomesa-1.3.0-accumulo-1.8.0 command: name environment: HADOOP_MASTER_ADDRESS: hdfs-name ports: - 50070:50070 hdfs-data: - image: quay.io/geomesa/hdfs:latest + image: quay.io/geomesa/hdfs:geomesa-1.3.0-accumulo-1.8.0 command: data environment: HADOOP_MASTER_ADDRESS: hdfs-name @@ -28,7 +28,7 @@ services: ports: - 2181:2181 accumulo-master: - image: quay.io/geomesa/accumulo-geomesa:latest + image: quay.io/geomesa/accumulo-geomesa:geomesa-1.3.0-accumulo-1.8.0 command: master --auto-init environment: HADOOP_MASTER_ADDRESS: hdfs-name @@ -37,7 +37,7 @@ services: depends_on: - zookeeper accumulo-monitor: - image: quay.io/geomesa/accumulo-geomesa:latest + image: quay.io/geomesa/accumulo-geomesa:geomesa-1.3.0-accumulo-1.8.0 command: monitor environment: HADOOP_MASTER_ADDRESS: hdfs-name @@ -49,7 +49,7 @@ services: - zookeeper - accumulo-master accumulo-tserver: - image: quay.io/geomesa/accumulo-geomesa:latest + image: quay.io/geomesa/accumulo-geomesa:geomesa-1.3.0-accumulo-1.8.0 command: tserver environment: HADOOP_MASTER_ADDRESS: hdfs-name From 39ac58acaf2d3a1b83aa1102688ed3c7e23e7f25 Mon Sep 17 00:00:00 2001 From: geoHeil Date: Sun, 22 Jan 2017 13:49:55 +0100 Subject: [PATCH 05/11] Create geomesa-entrypoint.sh --- .../fs/sbin/geomesa-entrypoint.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 geodocker-accumulo-geomesa/fs/sbin/geomesa-entrypoint.sh diff --git a/geodocker-accumulo-geomesa/fs/sbin/geomesa-entrypoint.sh b/geodocker-accumulo-geomesa/fs/sbin/geomesa-entrypoint.sh new file mode 100644 index 0000000..d463caa --- /dev/null +++ b/geodocker-accumulo-geomesa/fs/sbin/geomesa-entrypoint.sh @@ -0,0 +1,13 @@ +#! /usr/bin/env bash +set -euo pipefail +IFS=$'\n\t' +source /sbin/accumulo-lib.sh + +# The first argument determines this container's role in the accumulo cluster +ROLE=${1:-} + +if [ $ROLE = "master" ]; then + POSTINIT="/sbin/register.sh" /sbin/entrypoint.sh "$@" +else + /sbin/entrypoint.sh "$@" +fi From 127a554661fe178408ad3c6947b0f7954815962e Mon Sep 17 00:00:00 2001 From: geoHeil Date: Fri, 27 Jan 2017 20:07:50 +0100 Subject: [PATCH 06/11] update readme --- README.md | 23 ++++++++----------- geodocker-accumulo-geomesa/docker-compose.yml | 2 +- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 6132f70..52627a1 100644 --- a/README.md +++ b/README.md @@ -19,30 +19,27 @@ docker-compose up ## geomesa tutorial quickstart ``` -git clone the tutorials +git clone https://github.com/geoHeil/geomesa-tutorials ``` +- follow the readme there to build the container + make sure versions are matching (you can get git clone https://github.com/geoHeil/geomesa-tutorials.git) Change all pom xml files -build the quickstart via -``` -mvn clean install -pl geomesa-quickstart-accumulo +connect into the java container which is already running ``` -copy jar into java container which is connected to geomesa and connect into the java container -``` -docker cp geomesa-quickstart-accumulo/target/geomesa-quickstart-accumulo-1.3.0.jar java8JDK:/geomesaQuickstart.jar -docker exec -ti bash java8JDK bash +docker ps # shows all the containers +e5b2bb33c0b0 geomesa-tutorial:1.3.0 # I see this one +docker exec -ti bash # id is here e5b2bb33c0b0 ``` -run the quickstart. This requires connection credentials +Now run the quickstart. This requires connection credentials as defined in the `docker-compose.yml` file: ``` -java -cp geomesaQuickstart.jar \ +java -cp geomesa-quickstart-accumulo/target/geomesa-quickstart-accumulo-1.3.0.jar \ com.example.geomesa.accumulo.AccumuloQuickStart \ -instanceId accumulo \ -zookeepers zookeeper \ -user accumulo \ -password secret \ -tableName quickstart -``` - **todo** - what is the user /password of the default dockerized geDocker version? \ No newline at end of file +``` \ No newline at end of file diff --git a/geodocker-accumulo-geomesa/docker-compose.yml b/geodocker-accumulo-geomesa/docker-compose.yml index 85d1b2e..c6b6881 100644 --- a/geodocker-accumulo-geomesa/docker-compose.yml +++ b/geodocker-accumulo-geomesa/docker-compose.yml @@ -1,7 +1,7 @@ version: '2' services: tutorial: - image: java:8-jdk + image: geomesa-tutorial:1.3.0 links: - zookeeper:zookeeper - accumulo-master:accumulo From 242c6997d6b8b320aa2568b5d7f1be555608eaa0 Mon Sep 17 00:00:00 2001 From: geoHeil Date: Fri, 27 Jan 2017 21:47:40 +0100 Subject: [PATCH 07/11] fix user --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 52627a1..e7a1097 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ java -cp geomesa-quickstart-accumulo/target/geomesa-quickstart-accumulo-1.3.0.ja com.example.geomesa.accumulo.AccumuloQuickStart \ -instanceId accumulo \ -zookeepers zookeeper \ - -user accumulo \ - -password secret \ + -user root \ + -password GisPwd \ -tableName quickstart ``` \ No newline at end of file From 4577ab73a2356ec87df7d1e607f0c86b91a68263 Mon Sep 17 00:00:00 2001 From: geoHeil Date: Fri, 27 Jan 2017 21:55:35 +0100 Subject: [PATCH 08/11] update readme --- README.md | 54 ++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 36 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index e7a1097..e9cee62 100644 --- a/README.md +++ b/README.md @@ -1,32 +1,37 @@ -# building a current image +# building the current images -- update accumulo and geomesa version number -``` -git clone https://github.com/locationtech/geomesa.git -cd geomesa -git checkout geomesa_2.11-1.3.0 -update ./build.sh to desired version numbers then -build/mvn clean install -DskipTests=true #test failures otherwise (timeouts on macbook) -``` +**step 0: clone the repos** + +`git clone https://github.com/geoHeil/geodocker-geomesa.git` -## to start the image +**step 1: choose correct settings** +- choose settings for accumulo / geomesa / ... version which fits your environment +- if there are major changes you might need to recompile geomsa as well + ``` + git clone https://github.com/locationtech/geomesa.git + cd geomesa + git checkout geomesa_2.11-1.3.0 + update ./build.sh to desired version numbers then + build/mvn clean install -DskipTests=true #test failures otherwise (timeouts on macbook) + ``` + +**step 2: build the image for the tutorial** ``` -cd geodocker-accumulo-geomesa -docker-compose up +git clone https://github.com/geoHeil/geomesa-tutorials.git +cd geomesa-tutorials/docker-tutorial/ +docker build -t geomesa-tutorial:1.3.0 . ``` -## geomesa tutorial quickstart +**step 3: start all the images** ``` -git clone https://github.com/geoHeil/geomesa-tutorials +docker-compose up ``` -- follow the readme there to build the container -make sure versions are matching (you can get git clone https://github.com/geoHeil/geomesa-tutorials.git) -Change all pom xml files +**step 4: run the quickstart** -connect into the java container which is already running +- connect into the java container which is already running ``` docker ps # shows all the containers e5b2bb33c0b0 geomesa-tutorial:1.3.0 # I see this one @@ -34,6 +39,7 @@ docker exec -ti bash # id is here e5b2bb33c0b0 ``` Now run the quickstart. This requires connection credentials as defined in the `docker-compose.yml` file: + ``` java -cp geomesa-quickstart-accumulo/target/geomesa-quickstart-accumulo-1.3.0.jar \ com.example.geomesa.accumulo.AccumuloQuickStart \ @@ -42,4 +48,16 @@ java -cp geomesa-quickstart-accumulo/target/geomesa-quickstart-accumulo-1.3.0.ja -user root \ -password GisPwd \ -tableName quickstart +``` + +# TODO + + - currently it is crashing +``` +The version of ZooKeeper being used doesn't support Container nodes. CreateMode.PERSISTENT will be used instead. +Configured server-side iterators do not match client version - client version: 1.3.0, server version: unavailable +Creating new features +Inserting new features +Submitting query +Exception in thread "main" java.lang.RuntimeException: org.apache.accumulo.core.client.impl.AccumuloServerException: Error on server 845c80b29b3a:9997 ``` \ No newline at end of file From de64f4152338449509651f70156441265abd599c Mon Sep 17 00:00:00 2001 From: geoHeil Date: Fri, 27 Jan 2017 23:15:06 +0100 Subject: [PATCH 09/11] adding lib for iterators --- README.md | 4 ++++ geodocker-accumulo-geomesa/Dockerfile.template | 2 ++ 2 files changed, 6 insertions(+) diff --git a/README.md b/README.md index e9cee62..b7752b7 100644 --- a/README.md +++ b/README.md @@ -49,10 +49,14 @@ java -cp geomesa-quickstart-accumulo/target/geomesa-quickstart-accumulo-1.3.0.ja -password GisPwd \ -tableName quickstart ``` +- you can monitor accumulo here http://localhost:9995/accumulo # TODO - currently it is crashing + - checkout geomesas gitter, there are some really interesting notes + - apparently the iterators ore (not yet out of the box) available in the geodocker implementation + - you need to add the jar into `ACCUMULO_HOME/lib` to *globally* install it. Otherwise namespaces are possible as well ``` The version of ZooKeeper being used doesn't support Container nodes. CreateMode.PERSISTENT will be used instead. Configured server-side iterators do not match client version - client version: 1.3.0, server version: unavailable diff --git a/geodocker-accumulo-geomesa/Dockerfile.template b/geodocker-accumulo-geomesa/Dockerfile.template index f40bcfa..d59cf8c 100644 --- a/geodocker-accumulo-geomesa/Dockerfile.template +++ b/geodocker-accumulo-geomesa/Dockerfile.template @@ -15,6 +15,8 @@ ENV PATH="${PATH}:${GEOMESA_HOME}/bin" ADD geomesa-accumulo_2.11-${GEOMESA_VERSION} ${GEOMESA_HOME} ADD geomesa-accumulo-distributed-runtime_2.11-${GEOMESA_VERSION}.jar ${GEOMESA_RUNTIME}/ +ADD geomesa-accumulo-distributed-runtime_2.11-${GEOMESA_VERSION}.jar ${ACCUMULO_HOME}/lib/ + RUN set -x \ && (echo yes | ${GEOMESA_DIST}/bin/install-jai.sh) \ && (echo yes | ${GEOMESA_DIST}/bin/install-jline.sh) From 6107d2c6de25661ccc7148fa91feeca25e0265ff Mon Sep 17 00:00:00 2001 From: geoHeil Date: Fri, 27 Jan 2017 23:51:56 +0100 Subject: [PATCH 10/11] make executable --- geodocker-accumulo-geomesa/fs/sbin/geomesa-entrypoint.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 geodocker-accumulo-geomesa/fs/sbin/geomesa-entrypoint.sh diff --git a/geodocker-accumulo-geomesa/fs/sbin/geomesa-entrypoint.sh b/geodocker-accumulo-geomesa/fs/sbin/geomesa-entrypoint.sh old mode 100644 new mode 100755 From 3fd8faab9121af710ee4ab29f597ab235945b039 Mon Sep 17 00:00:00 2001 From: geoHeil Date: Sat, 28 Jan 2017 00:22:22 +0100 Subject: [PATCH 11/11] fixing typo in original docker file --- geodocker-accumulo-geomesa/Dockerfile.template | 2 +- .../fs/sbin/geomesa-entrypoint.sh | 13 ------------- geodocker-accumulo-geomesa/fs/sbin/register.sh | 2 +- 3 files changed, 2 insertions(+), 15 deletions(-) delete mode 100755 geodocker-accumulo-geomesa/fs/sbin/geomesa-entrypoint.sh diff --git a/geodocker-accumulo-geomesa/Dockerfile.template b/geodocker-accumulo-geomesa/Dockerfile.template index d59cf8c..4f4caee 100644 --- a/geodocker-accumulo-geomesa/Dockerfile.template +++ b/geodocker-accumulo-geomesa/Dockerfile.template @@ -22,4 +22,4 @@ RUN set -x \ && (echo yes | ${GEOMESA_DIST}/bin/install-jline.sh) COPY ./fs / -ENTRYPOINT [ "/sbin/geomesa-entrypoint.sh" ] +ENTRYPOINT [ "/sbin/entrypoint.sh" ] diff --git a/geodocker-accumulo-geomesa/fs/sbin/geomesa-entrypoint.sh b/geodocker-accumulo-geomesa/fs/sbin/geomesa-entrypoint.sh deleted file mode 100755 index d463caa..0000000 --- a/geodocker-accumulo-geomesa/fs/sbin/geomesa-entrypoint.sh +++ /dev/null @@ -1,13 +0,0 @@ -#! /usr/bin/env bash -set -euo pipefail -IFS=$'\n\t' -source /sbin/accumulo-lib.sh - -# The first argument determines this container's role in the accumulo cluster -ROLE=${1:-} - -if [ $ROLE = "master" ]; then - POSTINIT="/sbin/register.sh" /sbin/entrypoint.sh "$@" -else - /sbin/entrypoint.sh "$@" -fi diff --git a/geodocker-accumulo-geomesa/fs/sbin/register.sh b/geodocker-accumulo-geomesa/fs/sbin/register.sh index 29ab496..9920b09 100755 --- a/geodocker-accumulo-geomesa/fs/sbin/register.sh +++ b/geodocker-accumulo-geomesa/fs/sbin/register.sh @@ -12,4 +12,4 @@ accumulo shell -u ${ACCUMULO_USER} -p ${ACCUMULO_PASSWORD} -e \ "config -s general.vfs.context.classpath.geomesa=file:///opt/geomesa/accumulo/[^.].*.jar" accumulo shell -u ${ACCUMULO_USER} -p ${ACCUMULO_PASSWORD} -e \ "config -ns geomesa -s table.classpath.context=geomesa" -echo "Accumulo namespace configured: geomesa" +echo "Accumulo namespace configured: geomesa" \ No newline at end of file