Skip to content
Open
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -1,11 +1,12 @@
FROM mongo:3.2
FROM mongo:3.4

RUN apt-get update \
&& apt-get install -y \
python \
python-dev \
gcc \
curl \
netcat \
libffi-dev \
libssl-dev \
&& rm -rf /var/lib/apt/lists/*
Expand Down Expand Up @@ -44,6 +45,6 @@ ENTRYPOINT []
CMD [ \
"containerpilot", \
"mongod", \
"--replSet=joyent" \
"--config", \
"/etc/mongod.conf" \
]

6 changes: 5 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,10 @@

A running cluster includes the following components:
- [ContainerPilot](https://www.joyent.com/containerpilot): included in our MongoDB containers to orchestrate bootstrap behavior and coordinate replica joining using keys and checks stored in Consul in the `health`, and `onChange` handlers
- [MongoDB](https://www.mongodb.com/community): we're using MongoDB 3.2 and setting up a [replica set](https://docs.mongodb.com/manual/replication/)
- [MongoDB](https://www.mongodb.com/community): we're using MongoDB 3.4 and setting up a [replica set](https://docs.mongodb.com/manual/replication/)
- [Consul](https://www.consul.io/): used to coordinate replication and failover


## Running the cluster

Starting a new cluster is easy once you have [your `_env` file set with the configuration details](#configuration)
Expand All @@ -25,6 +26,9 @@ In a few moments you'll have a running MongoDB ready for a replica set. Both the
Pass these variables via an `_env` file.

- `LOG_LEVEL`: control the amount of logging from ContainerPilot
- `MONGO_USER`: the user that will be created as the default admin user
- `MONGO_PASSWORD`: password to use for the admin user
- `MONGO_KEY`: secret key contents to use for replica member authentication
- when the primary node is sent a `SIGTERM` it will [step down](https://docs.mongodb.com/manual/reference/command/replSetStepDown/) as primary; the following control those timeouts
- `MONGO_SECONDARY_CATCHUP_PERIOD`: the number of seconds that the mongod will wait for an electable secondary to catch up to the primary
- `MONGO_STEPDOWN_TIME`: the number of seconds to step down the primary, during which time the stepdown member is ineligible for becoming primary
Expand Down
28 changes: 22 additions & 6 deletions bin/manage.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import time

from functools import wraps
from urllib import quote_plus

import consul as pyconsul
#import manta
Expand Down Expand Up @@ -65,6 +66,9 @@ def get_environ(key, default):
SESSION_NAME = get_environ('SESSION_NAME', 'mongodb-replica-set-lock')
SESSION_TTL = int(get_environ('SESSION_TTL', 60))

MONGO_USER = get_environ('MONGO_USER', 'admin')
MONGO_PASSWORD = get_environ('MONGO_PASSWORD', 'admin')

# consts for node state
PRIMARY = 'mongodb-replicaset'
#SECONDARY = 'mongodb-secondary'
Expand Down Expand Up @@ -99,8 +103,7 @@ def pre_stop():
because we are about to be shut down
"""

ip = get_ip()
local_mongo = MongoClient(ip, connect=False)
local_mongo = get_local_mongo()

# since we are shutting down, it is ok to stop if mongo is already non-responsive
if not is_mongo_up(local_mongo):
Expand Down Expand Up @@ -142,7 +145,7 @@ def pre_stop():
return False
timeout += 1
# use a replica client so that we get "primary" data
mongo_client = MongoClient(ip, connect=False, replicaset=repl_status['set'], serverSelectionTimeoutMS=500)
mongo_client = MongoClient(get_local_mongo_uri(), connect=False, replicaset=repl_status['set'], serverSelectionTimeoutMS=500)
# is_mongo_up will sleep on failure, so we don't need a "time.sleep(1)"
if is_mongo_up(mongo_client, 1):
primary = mongo_client.primary
Expand All @@ -162,8 +165,7 @@ def health():
# TODO periodic mongodumps to Manta

hostname = socket.gethostname()
ip = get_ip()
local_mongo = MongoClient(ip, connect=False)
local_mongo = get_local_mongo()

# check that mongo is responsive
if not is_mongo_up(local_mongo):
Expand Down Expand Up @@ -212,7 +214,7 @@ def on_change():
'''
hostname = socket.gethostname()
ip = get_ip()
local_mongo = MongoClient(ip, connect=False)
local_mongo = get_local_mongo()

try:
repl_status = local_mongo.admin.command('replSetGetStatus')
Expand Down Expand Up @@ -313,6 +315,7 @@ def mongo_update_replset_config(local_mongo, hostname):
for new_mongo in new_mongos:
new_id = max(ids) + 1
ids.append(new_id)

members.append({'_id': new_id, 'host': new_mongo})

# TODO voting membership
Expand All @@ -332,6 +335,19 @@ def mongo_update_replset_config(local_mongo, hostname):
log.exception(e)
sys.exit(1)


def get_local_mongo_uri():
ip = get_ip()
uri = "mongodb://%s:%s@%s" % (quote_plus(MONGO_USER), quote_plus(MONGO_PASSWORD), ip)

return uri

def get_local_mongo():
local_mongo = MongoClient(get_local_mongo_uri(), connect=False)

return local_mongo


def consul_to_mongo_hostname(service):
# if name.startswith(SECONDARY + '-'):
# prefix = SECONDARY + '-'
Expand Down
21 changes: 21 additions & 0 deletions bin/setup_mongo.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
#!/bin/bash

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Might want to have a set -e here so that failures of individual lines will cause the script to stop.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Updated

echo "Start MongoDB without access control."
mongod --port 27017 &

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This should probably be mongod --fork --bind_ip 127.0.0.1 --logpath /dev/stdout --config /etc/mongod.conf to ensure that it runs with only local access, has the same config as the real instance, and when it returns will be ready for access and so we won't need the while ! nc .... The reason for the bind_ip is to prevent external services from getting a connection followed by a rejection (since auth is on and no user created), and instead just prevent connections altogether.

Not sure if a --port is necessary, since the default is 27017.

(Similar discussion on the official image docker-library/mongo#53)

while ! nc -z 127.0.0.1 27017; do sleep 1; done


echo "Create the user administrator."
mongo admin --eval "db.createUser({ user: '${MONGO_USER}', pwd: '${MONGO_PASSWORD}', roles: [ { role: 'dbAdminAnyDatabase', db: 'admin' }, { role: 'clusterAdmin', db: 'admin' } ] });"


echo "Shutdown the MongoDB service."
mongod --shutdown


echo "Creating keyFile for replication."
echo -e ${MONGO_KEY} > /etc/mongod.key
chmod 400 /etc/mongod.key

echo "Overwrite setup_mongo.sh so that this is a one-time setup"
echo "#!/bin/bash" > ./setup_user.sh

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Assuming this overwrote itself (setup_mongo.sh), this would only work on restarts of the current container, but would not prevent a second user creation when a user reuses an old data directory (like docker-compose when run on non-triton). In looking to add similar functionality to the mongo official image (docker-library/mongo#53 (comment)), we didn't yet have a way to detect that this was an already initialized database directory to prevent trying to create the user a second time.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I am not sure how to solve this either.

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

mkdir an empty dir into a well-known location in the data directory. If the mkdir fails (which it will if it already exists), we can just bail out there.

2 changes: 1 addition & 1 deletion etc/containerpilot.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"consul": "{{ if .CONSUL }}{{ .CONSUL }}{{ else }}consul{{ end }}:8500",
"preStart": "python /usr/local/bin/manage.py",
"preStart": "./usr/local/bin/setup_mongo.sh && python /usr/local/bin/manage.py",
"preStop": "python /usr/local/bin/manage.py pre_stop",
"services": [
{
Expand Down
6 changes: 6 additions & 0 deletions etc/mongod.conf
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
replication:
replSetName: "joyent"

security:
keyFile: "/etc/mongod.key"
authorization: "enabled"
3 changes: 2 additions & 1 deletion local-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ services:
mem_limit: 512m
build: .
env_file: _env
environment:
- CONSUL=consul
ports:
- 27017

Expand All @@ -24,4 +26,3 @@ services:
- 8302
- 8400
- 8500

217 changes: 217 additions & 0 deletions setup.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,217 @@
#!/bin/bash
set -e -o pipefail

help() {
echo
echo 'Usage ./setup.sh ~/path/to/MANTA_PRIVATE_KEY ~/path/to/MONGO_KEYFILE'
echo
echo 'Checks that your Triton and Docker environment is sane and configures'
echo 'an environment file to use.'
echo
echo 'MANTA_PRIVATE_KEY is the filesystem path to an SSH private key'
echo 'used to connect to Manta for the database backups.'
echo
echo 'MONGO_KEYFILE is the filesystem path to a file that contains a secret'
echo 'value between 6 and 1024 characters for authenticating replica members'
echo
echo 'Additional details must be configured in the _env file, but this script will properly'
echo 'encode the SSH key details for use with this MongoDB image.'
echo
}


# populated by `check` function whenever we're using Triton
TRITON_USER=
TRITON_DC=
TRITON_ACCOUNT=

# ---------------------------------------------------
# Top-level commands

# Check for correct configuration and setup _env file
envcheck() {

if [ -z "$1" ]; then
tput rev # reverse
tput bold # bold
echo 'Please provide a path to a SSH private key to access Manta.'
tput sgr0 # clear

help
exit 1
fi

if [ ! -f "$1" ]; then
tput rev # reverse
tput bold # bold
echo 'SSH private key for Manta is unreadable.'
tput sgr0 # clear

help
exit 1
fi

if [ -z "$2" ]; then
tput rev # reverse
tput bold # bold
echo 'Please provide a path to a key file for MongoDB replica members.'
tput sgr0 # clear

help
exit 1
fi

if [ ! -f "$2" ]; then
tput rev # reverse
tput bold # bold
echo 'MongoDB replica key file is unreadable.'
tput sgr0 # clear

help
exit 1
fi

# Assign args to named vars
MANTA_PRIVATE_KEY_PATH=$1
MONGO_KEYFILE_PATH=$2

command -v docker >/dev/null 2>&1 || {
echo
tput rev # reverse
tput bold # bold
echo 'Docker is required, but does not appear to be installed.'
tput sgr0 # clear
echo 'See https://docs.joyent.com/public-cloud/api-access/docker'
exit 1
}
command -v json >/dev/null 2>&1 || {
echo
tput rev # reverse
tput bold # bold
echo 'Error! JSON CLI tool is required, but does not appear to be installed.'
tput sgr0 # clear
echo 'See https://apidocs.joyent.com/cloudapi/#getting-started'
exit 1
}

command -v triton >/dev/null 2>&1 || {
echo
tput rev # reverse
tput bold # bold
echo 'Error! Joyent Triton CLI is required, but does not appear to be installed.'
tput sgr0 # clear
echo 'See https://www.joyent.com/blog/introducing-the-triton-command-line-tool'
exit 1
}

# make sure Docker client is pointed to the same place as the Triton client
local docker_user=$(docker info 2>&1 | awk -F": " '/SDCAccount:/{print $2}')
local docker_dc=$(echo $DOCKER_HOST | awk -F"/" '{print $3}' | awk -F'.' '{print $1}')
TRITON_USER=$(triton profile get | awk -F": " '/account:/{print $2}')
TRITON_DC=$(triton profile get | awk -F"/" '/url:/{print $3}' | awk -F'.' '{print $1}')
TRITON_ACCOUNT=$(triton account get | awk -F": " '/id:/{print $2}')
if [ ! "$docker_user" = "$TRITON_USER" ] || [ ! "$docker_dc" = "$TRITON_DC" ]; then
echo
tput rev # reverse
tput bold # bold
echo 'Error! The Triton CLI configuration does not match the Docker CLI configuration.'
tput sgr0 # clear
echo
echo "Docker user: ${docker_user}"
echo "Triton user: ${TRITON_USER}"
echo "Docker data center: ${docker_dc}"
echo "Triton data center: ${TRITON_DC}"
exit 1
fi

local triton_cns_enabled=$(triton account get | awk -F": " '/cns/{print $2}')
if [ ! "true" == "$triton_cns_enabled" ]; then
echo
tput rev # reverse
tput bold # bold
echo 'Error! Triton CNS is required and not enabled.'
tput sgr0 # clear
echo
exit 1
fi

# setup environment file
if [ ! -f "_env" ]; then
echo '# Environment variables for MongoDB service' > _env
echo 'MONGO_USER=dbuser' >> _env
echo 'MONGO_PASSWORD='$(cat /dev/urandom | LC_ALL=C tr -dc 'A-Za-z0-9' | head -c 7) >> _env
echo MONGO_KEY=$(cat ${MONGO_KEYFILE_PATH} | tr '\n' '#') >> _env
echo >> _env

echo '# Environment variables for backups to Manta' >> _env
echo 'MANTA_URL=https://us-east.manta.joyent.com' >> _env
echo 'MANTA_BUCKET= # an existing Manta bucket' >> _env
echo 'MANTA_USER= # a user with access to that bucket' >> _env
echo 'MANTA_SUBUSER=' >> _env
echo 'MANTA_ROLE=' >> _env

# MANTA_KEY_ID must be the md5 formatted key fingerprint. A SHA256 will result in errors.
set +o pipefail
# The -E option was added to ssh-keygen recently; if it doesn't work, then
# assume we're using an older version of ssh-keygen that only outputs MD5 fingerprints
ssh-keygen -yl -E md5 -f ${MANTA_PRIVATE_KEY_PATH} > /dev/null 2>&1
if [ $? -eq 0 ]; then
echo MANTA_KEY_ID=$(ssh-keygen -yl -E md5 -f ${MANTA_PRIVATE_KEY_PATH} | awk '{print substr($2,5)}') >> _env
else
echo MANTA_KEY_ID=$(ssh-keygen -yl -f ${MANTA_PRIVATE_KEY_PATH} | awk '{print $2}') >> _env
fi
set -o pipefail

# munge the private key so that we can pass it into an env var sanely
# and then unmunge it in our startup script
echo MANTA_PRIVATE_KEY=$(cat ${MANTA_PRIVATE_KEY_PATH} | tr '\n' '#') >> _env
echo >> _env

echo '# Consul discovery via Triton CNS' >> _env
echo CONSUL=mongodb-consul.svc.${TRITON_ACCOUNT}.${TRITON_DC}.cns.joyent.com >> _env
echo >> _env

echo 'Edit the _env file with your desired MONGO_* and MANTA_* config'
else
echo 'Existing _env file found, exiting'
exit
fi
}

get_root_password() {
echo $(docker logs ${COMPOSE_PROJECT_NAME:-mongodb}_mongodb_1 2>&1 | \
awk '/Generated root password/{print $NF}' | \
awk '{$1=$1};1'
) | pbcopy
}



# ---------------------------------------------------
# parse arguments

# Get function list
funcs=($(declare -F -p | cut -d " " -f 3))

until
if [ ! -z "$1" ]; then
# check if the first arg is a function in this file, or use a default
if [[ " ${funcs[@]} " =~ " $1 " ]]; then
cmd=$1
shift 1
else
cmd="envcheck"
fi

$cmd "$@"
if [ $? == 127 ]; then
help
fi

exit
else
help
fi
do
echo
done