Upgrade to ES 5

Dockerfile

@@ -1,54 +1,65 @@
-FROM debian:jessie
+FROM docker.elastic.co/elasticsearch/elasticsearch:5.3.0
 
-RUN apt-get update && \
-    apt-get install -y \
-    openjdk-7-jre-headless \
-    curl \
-    jq \
-    && rm -rf /var/lib/apt/lists/*
+# need to drop back into root!
+USER root
 
-ENV JAVA_HOME /usr/lib/jvm/java-7-openjdk-amd64
-
-# If we wanted the development version we could pull that instead but we want to
-# run a production environment here.
-RUN export ES_PKG=elasticsearch-2.2.0.deb && \
-    curl -LO https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/distribution/deb/elasticsearch/2.2.0/${ES_PKG} && \
-    dpkg -i ${ES_PKG} && \
-    rm ${ES_PKG} && \
-    rm /etc/elasticsearch/elasticsearch.yml
+RUN apk update && \
+    apk add jq curl unzip tar && \
+    rm -rf /var/cache/apk/*
 
 # Add Containerpilot and set its configuration
-ENV CONTAINERPILOT_VER 2.1.0
-ENV CONTAINERPILOT file:///etc/containerpilot.json
+ENV CONSUL_VERSION=0.7.5 \
+    CONSUL_CLI_VER=0.3.1 \
+    CONTAINERPILOT_VER=2.7.2 \
+    CONTAINERPILOT=file:///etc/containerpilot.json
+
+# Add consul agent
+RUN export CONSUL_CHECKSUM=40ce7175535551882ecdff21fdd276cef6eaab96be8a8260e0599fadb6f1f5b8 \
+    && curl --retry 7 --fail -vo /tmp/consul.zip "https://releases.hashicorp.com/consul/${CONSUL_VERSION}/consul_${CONSUL_VERSION}_linux_amd64.zip" \
+    && echo "${CONSUL_CHECKSUM}  /tmp/consul.zip" | sha256sum -c \
+    && unzip /tmp/consul -d /usr/local/bin \
+    && rm /tmp/consul.zip
+
+# Consul client
+RUN export CONSUL_CLIENT_CHECKSUM=037150d3d689a0babf4ba64c898b4497546e2fffeb16354e25cef19867e763f1 \
+    && curl -Lso /tmp/consul-cli.tgz "https://github.com/CiscoCloud/consul-cli/releases/download/v${CONSUL_CLI_VER}/consul-cli_${CONSUL_CLI_VER}_linux_amd64.tar.gz" \
+    && echo "${CONSUL_CLIENT_CHECKSUM}  /tmp/consul-cli.tgz" | sha256sum -c \
+    && tar zxf /tmp/consul-cli.tgz -C /usr/local/bin --strip-components 1 \
+    && rm /tmp/consul-cli.tgz
 
-RUN export CONTAINERPILOT_CHECKSUM=e7973bf036690b520b450c3a3e121fc7cd26f1a2 \
+# Add ContainerPilot and set its configuration file path
+RUN export CONTAINERPILOT_CHECKSUM=e886899467ced6d7c76027d58c7f7554c2fb2bcc \
     && curl -Lso /tmp/containerpilot.tar.gz \
-         "https://github.com/joyent/containerpilot/releases/download/${CONTAINERPILOT_VER}/containerpilot-${CONTAINERPILOT_VER}.tar.gz" \
+        "https://github.com/joyent/containerpilot/releases/download/${CONTAINERPILOT_VER}/containerpilot-${CONTAINERPILOT_VER}.tar.gz" \
     && echo "${CONTAINERPILOT_CHECKSUM}  /tmp/containerpilot.tar.gz" | sha1sum -c \
     && tar zxf /tmp/containerpilot.tar.gz -C /usr/local/bin \
     && rm /tmp/containerpilot.tar.gz
 
+# Add our configuration files and scripts
+COPY /etc/containerpilot.json /etc/containerpilot.json
+COPY /etc/elasticsearch.yml /usr/share/elasticsearch/config/elasticsearch.yml
+COPY /bin/* /usr/local/bin/
+
+# Should we remove unzip?
+# RUN apk del unzip tar
+
 # Create and take ownership over required directories
-RUN mkdir -p /var/lib/elasticsearch/data && \
-    chown -R elasticsearch:elasticsearch /var/lib/elasticsearch/data && \
-    chown -R root:elasticsearch /etc/elasticsearch && \
-    chmod g+w /etc/elasticsearch
+RUN mkdir -p /opt/consul/config \
+    && mkdir -p /opt/consul/data \
+    && chmod 770 /opt/consul/data \
+    && chown -R elasticsearch:elasticsearch /opt/consul \
+    && mkdir -p /etc/containerpilot \
+    && chmod -R g+w /etc/containerpilot \
+    && chmod +x /usr/local/bin/elastic-server.sh \
+    && chown -R elasticsearch:elasticsearch /etc/containerpilot
 
+# back to elastic USER
 USER elasticsearch
 
-# Add our configuration files and scripts
-COPY /etc/containerpilot.json /etc
-COPY /etc/elasticsearch.yml /etc/elasticsearch/elasticsearch.yml
-COPY /bin/manage.sh /usr/local/bin
-
 # Expose the data directory as a volume in case we want to mount these
 # as a --volumes-from target; it's important that this VOLUME comes
 # after the creation of the directory so that we preserve ownership.
-VOLUME /var/lib/elasticsearch/data
-
-# We don't need to expose these ports in order for other containers on Triton
-# to reach this container in the default networking environment, but if we
-# leave this here then we get the ports as well-known environment variables
-# for purposes of linking.
-EXPOSE 9200
-EXPOSE 9300
+VOLUME ["/usr/share/elasticsearch/data"]
+
+# Start with containerpilot then to our wrapper
+CMD ["containerpilot", "/usr/local/bin/elastic-server.sh"]

bin/elastic-server.sh

@@ -0,0 +1,6 @@
+#!/bin/sh -xe
+manage.sh onStart #|| exit $?
+if [[ $? != 0 ]]; then
+    exit $?
+fi
+exec /usr/share/elasticsearch/bin/es-docker $*

bin/manage.sh

@@ -1,67 +1,130 @@
 #!/bin/bash
 
 MASTER=null
+CONSUL_HOST=${CONSUL}
+CONSUL_AGENT=${CONSUL_AGENT:=false}
 
-if [[ -z ${CONSUL} ]]; then
+readonly lockPath=service/elasticsearch-master/locks/master
+
+if [ $CONSUL_AGENT != false ]; then
+    CONSUL_HOST='localhost'
+fi
+
+if [[ -z $CONSUL_HOST ]]; then
     echo "Missing CONSUL environment variable"
     exit 1
 fi
 
+consulCommand() {
+    consul-cli --quiet --consul="${CONSUL_HOST}:8500" $*
+}
+
 preStart() {
-    # happy path is that there's a master available and we can cluster
-    configureMaster
-
-    # data-only nodes can only loop until there's a master available
-    if [ ${ES_NODE_MASTER} == false ]; then
-        while true
-        do
-            sleep 1.7
-            configureMaster
+    logDebug "preStart"
+}
+
+onStart() {
+    logDebug "onStart"
+
+    waitForLeader
+
+    getRegisteredServiceName
+    if [[ "${registeredServiceName}" == "elasticsearch-data" ]]; then
+
+        # wait for a healthy master
+        local i
+        for (( i = 0; i < ${MASTER_WAIT_TIMEOUT-60}; i++ )); do
+            getServiceAddresses "elasticsearch-master"
+            if [[ ${serviceAddresses} ]]; then
+                MASTER=$serviceAddresses
+                break
+            fi
+            sleep 1
         done
-        exit 0
-    fi
 
-    # for a master+data node, we'll retry to see if there's another
-    # master in the cluster in the process of starting up. But we
-    # bail out if we exceed the retries and just bootstrap the cluster
-    if [ ${ES_NODE_DATA} == true ]; then
-        local n=0
-        until [ $n -ge 2 ]
-        do
-            sleep 1.7
-            configureMaster
-            n=$((n+1))
+    else
+
+        # wait for a healthy master
+        local i
+        for (( i = 0; i < ${MASTER_WAIT_TIMEOUT-60}; i++ )); do
+            getServiceAddresses "elasticsearch-master"
+            if [[ ${serviceAddresses} ]]; then
+                MASTER=$serviceAddresses
+                break
+            fi
+            sleep 1
         done
+
     fi
 
-    # for a master-only node or master+data node that's exceeded the
-    # retry attempts, we'll assume this is the first master and bootstrap
-    # the cluster
-    MASTER=127.0.0.1
-    replace
+    # replace zen hosts
+    replaceZenHosts
 }
 
-# get the list of ES master nodes from Consul
-configureMaster() {
-    MASTER=$(curl -Ls --fail http://${CONSUL}:8500/v1/catalog/service/elasticsearch-master | jq -r '.[0].ServiceAddress')
-    if [[ $MASTER != "null" ]] && [[ -n $MASTER ]]; then
-        replace
-        exit 0
-    fi
-    # if there's no master we fall thru and let the caller figure
-    # out what to do next
+health() {
+    local privateIp=$(ip addr show eth0 | grep -o '[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}')
+    /usr/bin/curl --fail -s -o /dev/null http://${privateIp}:9200
+}
+
+waitForLeader() {
+    logDebug "Waiting for consul leader"
+    local tries=0
+    while true
+    do
+        logDebug "Waiting for consul leader"
+        tries=$((tries + 1))
+        local leader=$(consulCommand --template="{{.}}" status leader)
+        if [[ -n "$leader" ]]; then
+            break
+        elif [[ $tries -eq 60 ]]; then
+            echo "No consul leader"
+            exit 1
+        fi
+        sleep 1
+    done
+}
+
+getServiceAddresses() {
+    local serviceInfo=$(consulCommand health service --passing "$1")
+    serviceAddresses=($(echo $serviceInfo | jq -r '.[].Service.Address'))
+    logDebug "serviceAddresses $1 ${serviceAddresses[*]}"
+}
+
+getRegisteredServiceName() {
+    registeredServiceName=$(jq -r '.services[0].name' /etc/containerpilot.json)
+}
+
+getNodeAddress() {
+    nodeAddress=$(ifconfig eth0 | awk '/inet addr/ {gsub("addr:", "", $2); print $2}')
 }
 
-# update discovery.zen.ping.unicast.hosts
-replace() {
+replaceZenHosts() {
     REPLACEMENT=$(printf 's/^discovery\.zen\.ping\.unicast\.hosts.*$/discovery.zen.ping.unicast.hosts: ["%s"]/' ${MASTER})
-    sed -i "${REPLACEMENT}" /etc/elasticsearch/elasticsearch.yml
+    sed -i "${REPLACEMENT}" /usr/share/elasticsearch/config/elasticsearch.yml
 }
 
-health() {
-    local privateIp=$(ip addr show eth0 | grep -o '[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}')
-    /usr/bin/curl --fail -s -o /dev/null http://${privateIp}:9200
+logDebug() {
+    if [[ "${LOG_LEVEL}" == "DEBUG" ]]; then
+        echo "manage: $*"
+    fi
+}
+
+help() {
+    echo "Usage: ./manage.sh preStart       => configure Consul agent"
+    echo "       ./manage.sh onStart        => first-run configuration"
+    echo "       ./manage.sh health         => health check Elastic"
+    echo "       ./manage.sh preStop        => prepare for stop"
 }
 
-# do whatever the arg is
-$1
+until
+    cmd=$1
+    if [[ -z "$cmd" ]]; then
+        help
+    fi
+    shift 1
+    $cmd "$@"
+    [ "$?" -ne 127 ]
+do
+    help
+    exit
+done

etc/containerpilot.json

@@ -1,13 +1,29 @@
 {
-  "consul": "{{ .CONSUL }}:8500",
-  "preStart": "/usr/local/bin/manage.sh preStart",
-  "services": [
-    {
-      "name": "{{ .ES_SERVICE_NAME }}",
-      "port": 9300,
-      "health": "/usr/local/bin/manage.sh health",
-      "poll": 10,
-      "ttl": 25
-    }
-  ]
+    "consul": "{{ if .CONSUL_AGENT }}localhost{{ else }}{{ if .CONSUL }}{{ .CONSUL }}{{ else }}consul{{ end }}{{ end }}:8500",
+    "logging": {
+        "level": "{{ if .LOG_LEVEL }}{{ .LOG_LEVEL }}{{ else }}INFO{{ end }}",
+        "format": "text",
+        "output": "stdout"
+    },
+    "preStart": "/usr/local/bin/manage.sh preStart",
+    "services": [{
+        "name": "{{ .ES_SERVICE_NAME }}",
+        "port": 9300,
+        "health": "/usr/local/bin/manage.sh health",
+        "poll": 10,
+        "ttl": 25
+    }],
+    "coprocesses": [{{ if .CONSUL_AGENT }}
+        {
+            "command": ["/usr/local/bin/consul", "agent",
+                "-data-dir=/opt/consul/data",
+                "-config-dir=/opt/consul/config",
+                "-rejoin",
+                "-retry-join", "{{ if .CONSUL }}{{ .CONSUL }}{{ else }}consul{{ end }}",
+                "-retry-max", "10",
+                "-retry-interval", "10s"
+            ],
+            "restarts": "unlimited"
+        }
+    {{ end }}]
 }