diff --git a/riak_test/docker/Dockerfile.tmpl b/riak_test/docker/Dockerfile.tmpl new file mode 100644 index 000000000..6af2976df --- /dev/null +++ b/riak_test/docker/Dockerfile.tmpl @@ -0,0 +1,31 @@ +FROM centos:centos6 +MAINTAINER Shunichi Shinohara + +RUN yum install -y tar +RUN yum install -y epel-release +RUN yum install -y s3cmd +# RUN apt-get install -qq logrotate sudo python-pip +# RUN pip install s3cmd + +ADD @@RIAK_PKG@@ / +ADD @@ST_PKG@@ / +ADD @@CS_PKG@@ / + +RUN rpm -ivh @@RIAK_PKG@@ +RUN rpm -ivh @@ST_PKG@@ +RUN rpm -ivh @@CS_PKG@@ + +COPY riak.conf /etc/riak/riak.conf +COPY riak.advanced.config /etc/riak/advanced.config +COPY stanchion.conf /etc/stanchion/stanchion.conf +COPY stanchion.advanced.config /etc/stanchion/advanced.config +COPY riak-cs.conf /etc/riak-cs/riak-cs.conf +COPY riak-cs.advanced.config /etc/riak-cs/advanced.config + +COPY local.s3cfg /.s3cfg +COPY confirm.sh /root/confirm.sh +RUN chmod 755 /root/confirm.sh + +RUN riak version +RUN stanchion version +RUN riak-cs version diff --git a/riak_test/docker/README.md b/riak_test/docker/README.md new file mode 100644 index 000000000..9e4c11d35 --- /dev/null +++ b/riak_test/docker/README.md @@ -0,0 +1,25 @@ +# Test release packages + +This directory contains some configuration file templates and a Dockerfile template which +will enable you to construct a Docker container to test riak_cs release packages on various +operating system flavors. + +Currently the Dockerfile builds an image based on centos6 + +## Prequisites + +- docker properly installed and ready +- Riak EE, Riak CS EE and Stanchion RPM packages + +## Usage + +Make sure the rpm files are in this directory. Docker assumes you have root, which +is why the test script uses sudo. Make sure your user id has sufficient permissions +in the sudoers file to execute the commands in the test shell script. + +The versions following the test script are for the riak_ee, riak_cs and stanchion +RPMs. Please make sure they match the version strings encoded in the RPM filenames. + +``` +./test.sh 2.0.5 2.0.0 2.0.0 +``` diff --git a/riak_test/docker/confirm.sh b/riak_test/docker/confirm.sh new file mode 100755 index 000000000..c5d692285 --- /dev/null +++ b/riak_test/docker/confirm.sh @@ -0,0 +1,83 @@ +#!/bin/bash + +trap 'echo "[ERROR] $0:$LINENO \"$BASH_COMMAND\"" ; exit 1' ERR + +echo "starting all nodes" +riak start +riak-admin wait-for-service riak_kv riak@127.0.0.1 +stanchion start +riak-cs start +echo "done." + +riak ping +stanchion ping +riak-cs ping + +riak version +stanchion version +riak-cs version + +# which riak-cs-multibag +# echo `riak-cs-multibag list-bags` +# echo `riak-cs-multibag weight master 10` +# echo `riak-cs-multibag weight bag-A 10` +# echo `riak-cs-multibag weight` + +echo "creating init user" + +/usr/lib64/riak-cs/erts-5.10.3/bin/erl \ + -name n@127.0.0.1 -setcookie riak \ + -eval "io:format(\"~p\", [rpc:call('riak-cs@127.0.0.1', riak_cs_user, create_user, [\"name\", \"me@admin.com\", \"admin-key\", \"admin-secret\"])])." \ + -s init stop + +echo "\n... probably user created" + +echo "sleeping ..." + +riak-cs-admin gc set-leeway 2 +riak-cs-admin gc batch +riak-cs-admin gc status +riak-cs-admin gc set-interval 5 + +S3CMD=s3cmd + +${S3CMD} mb s3://test-bucket +${S3CMD} ls +${S3CMD} put *.rpm s3://test-bucket +${S3CMD} put -r /bin s3://test-bucket +${S3CMD} ls -r s3://test-bucket + +riak-cs-admin access flush +riak-cs-admin storage batch +riak-cs-admin storage status + +riak-cs-admin stanchion show +riak-cs-admin cleanup-orphan-multipart + +CS_LOG_DIR=/var/log/riak-cs +touch -d "2016-03-01" ${CS_LOG_DIR}/console.log.old1 +touch -d "2016-03-01" ${CS_LOG_DIR}/access.log.old1 + +riak-cs-debug -h +riak-cs-debug /tmp/rcs-debug-default.tar.gz +riak-cs-debug --log-mtime 2 /tmp/rcs-debug-log-mtime.tar.gz +riak-cs-debug --skip-accesslog /tmp/rcs-debug-log-skip-alog.tar.gz + +echo "teardown" +${S3CMD} del -r -f s3://test-bucket +${S3CMD} -d rb s3://test-bucket +${S3CMD} -d ls + +# /usr/lib/riak-cs/erts-5.9.1/bin/erl -name n@127.0.0.1 -setcookie riak -eval "io:format(\"~p\", [rpc:call('riak-cs@127.0.0.1', code, ensure_loaded, [riak_repl_pb_api])])." -s init stop + +riak-cs stop +riak-cs-debug /tmp/rcs-debug-stopped.tar.gz + +for f in /tmp/rcs-debug-*.tar.gz; do + echo "Extract "$f" to logs/riak-cs/..." + b=$(basename $f .tar.gz) + mkdir -p ${CS_LOG_DIR}/$b + tar zxf $f -C ${CS_LOG_DIR}/$b +done + +echo "looks like package test has passed." diff --git a/riak_test/docker/local.s3cfg b/riak_test/docker/local.s3cfg new file mode 100644 index 000000000..8bc1f427d --- /dev/null +++ b/riak_test/docker/local.s3cfg @@ -0,0 +1,44 @@ +[default] + +access_key = admin-key +secret_key = admin-secret +signature_v2 = True + +bucket_location = US +default_mime_type = binary/octet-stream +delete_removed = False +dry_run = False +enable_multipart = True +encoding = UTF-8 +encrypt = False +follow_symlinks = False +force = False +get_continue = False +gpg_command = /usr/bin/gpg +gpg_decrypt = %(gpg_command)s -d --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s +gpg_encrypt = %(gpg_command)s -c --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s +gpg_passphrase = +guess_mime_type = True +host_base = s3.amazonaws.com +host_bucket = %(bucket)s.s3.amazonaws.com +human_readable_sizes = False +invalidate_on_cf = False +list_md5 = False +log_target_prefix = +mime_type = +multipart_chunk_size_mb = 15 +preserve_attrs = True +progress_meter = True +proxy_host = localhost +proxy_port = 8080 +#proxy_port = 443 +##proxy_port = 8000 +recursive = False +recv_chunk = 4096 +reduced_redundancy = False +send_chunk = 4096 +skip_existing = False +socket_timeout = 300 +urlencoding_mode = normal +use_https = False +verbosity = WARNING diff --git a/riak_test/docker/riak-cs.advanced.config b/riak_test/docker/riak-cs.advanced.config new file mode 100644 index 000000000..9f23a5e7e --- /dev/null +++ b/riak_test/docker/riak-cs.advanced.config @@ -0,0 +1,5 @@ +[ + {riak_cs, + [ + ]} +]. diff --git a/riak_test/docker/riak-cs.conf b/riak_test/docker/riak-cs.conf new file mode 100644 index 000000000..2cb9e1a45 --- /dev/null +++ b/riak_test/docker/riak-cs.conf @@ -0,0 +1,469 @@ +## Riak CS http/https port and IP address to listen at for object +## storage activity +## +## Default: 127.0.0.1:8080 +## +## Acceptable values: +## - an IP/port pair, e.g. 127.0.0.1:10011 +listener = 127.0.0.1:8080 + +## Riak node to which Riak CS accesses +## +## Default: 127.0.0.1:8087 +## +## Acceptable values: +## - an IP/port pair, e.g. 127.0.0.1:10011 +riak_host = 127.0.0.1:8087 + +## Configuration for access to request +## serialization service +## +## Default: 127.0.0.1:8085 +## +## Acceptable values: +## - an IP/port pair, e.g. 127.0.0.1:10011 +stanchion_host = 127.0.0.1:8085 + +## SSL configuration for access to request serialization +## service. With `on`, Riak CS connects to Stanchion with SSL. +## +## Default: off +## +## Acceptable values: +## - on or off +stanchion.ssl = off + +## Default cert location for https can be overridden +## with the ssl config variable, for example: +## +## Acceptable values: +## - the path to a file +## ssl.certfile = $(platform_etc_dir)/cert.pem + +## Default key location for https can be overridden with the ssl +## config variable, for example: +## +## Acceptable values: +## - the path to a file +## ssl.keyfile = $(platform_etc_dir)/key.pem + +## Enable this to allow the creation of an admin user when +## setting up a system. It is recommended to only enable this +## temporarily unless your use-case specifically dictates letting +## anonymous users to create accounts. +## +## Default: off +## +## Acceptable values: +## - on or off +anonymous_user_creation = on + +## IP address to listen on for system administration tasks. +## +## Acceptable values: +## - text +## admin.ip = 127.0.0.1 + +## Port to listen on for system administration tasks. +## +## Acceptable values: +## - an integer +## admin.port = 8000 + +## Admin user credentials. Admin access like /riak-cs/stats +## requires this entry to be set properly. The credentials specified +## here must match the admin credentials specified in the stanchion +## app.config for the system to function properly. +## +## Default: admin-key +## +## Acceptable values: +## - text +admin.key = admin-key + +## +## Default: admin-secret +## +## Acceptable values: +## - text +admin.secret = admin-secret + +## Root host name which Riak CS accepts. +## Your CS bucket at s3.example.com will be accessible +## via URL like http://bucket.s3.example.com/object/name +## +## Default: s3.amazonaws.com +## +## Acceptable values: +## - text +root_host = s3.amazonaws.com + +## Fixed pool size of primary connection pool which is used +## to service the majority of API requests related to the upload +## or retrieval of objects. +## +## Default: 128 +## +## Acceptable values: +## - an integer +pool.request.size = 128 + +## Fixed pool size of secondary connection pool which is used +## strictly for requests to list the contents. +## +## Default: 5 +## +## Acceptable values: +## - an integer +pool.list.size = 5 + +## Switch whether Riak CS trusts 'X-Forwarded-For' header. +## If your load balancer adds 'X-Forwarded-For' header +## and it is reliable (able to gaurantee it is not added +## by malicious user), turn this true. Otherwise, by +## default, Riak CS takes source IP address as an input. +## +## Default: off +## +## Acceptable values: +## - on or off +trust_x_forwarded_for = off + +## The time to retain the block for an object after it has been +## deleted. This leeway time is set to give the delete indication +## time to propogate to all replicas. +## +## Default: 24h +## +## Acceptable values: +## - a time duration with units, e.g. '10s' for 10 seconds +gc.leeway_period = 24h + +## How often the garbage collection daemon +## waits in-between gc batches. +## +## Default: 15m +## +## Acceptable values: +## - a time duration with units, e.g. '10s' for 10 seconds +gc.interval = 15m + +## How long a move to the garbage +## collection to do list can remain +## failed, before we retry it. +## +## Default: 6h +## +## Acceptable values: +## - a time duration with units, e.g. '10s' for 10 seconds +gc.retry_interval = 6h + +## How often to flush the access log; integer factor of +## access_archive_period (1 == once per period; 2 == twice per period, +## etc.) +## +## Default: 1 +## +## Acceptable values: +## - an integer +stats.access.log.flush_factor = 1 + +## Additional access log flush trigger - flush after +## this many accesses are recorded, even if the flush +## interval has not expired; integer number of accesses +## +## Default: 1000000 +## +## Acceptable values: +## - an integer +stats.access.log.flush_size = 1000000 + +## How large each access archive object is. Should be a +## multiple of access_log_flush_interval. +## +## Default: 1h +## +## Acceptable values: +## - a time duration with units, e.g. '10s' for 10 seconds +stats.access.archive_period = 1h + +## How many access logs are allowed to pile up in the +## archiver's queue before it starts skipping to catch +## up; integer number of logs +## +## Default: 2 +## +## Acceptable values: +## - an integer +stats.access.archiver.max_backlog = 2 + +## How many workers to put access log data to Riak are +## allowed to run concurrently +## +## Default: 2 +## +## Acceptable values: +## - an integer +stats.access.archiver.max_workers = 2 + +## How large each storage archive object is. Should be +## chosen such that each storage_schedule entry falls in +## a different period. +## +## Default: 1d +## +## Acceptable values: +## - a time duration with units, e.g. '10s' for 10 seconds +stats.storage.archive_period = 1d + +## How many archive periods a user can request in one usage read, +## applied independently to access and storage. If archive_periods +## are defined as 1 hour, then 744 * 1 hour = 31 days will be the +## longest limit. +## +## Default: 744 +## +## Acceptable values: +## - an integer +stats.usage_request_limit = 744 + +## custom server name at http response header "Server: Riak CS" +## +## Default: Riak CS +## +## Acceptable values: +## - text +server.name = Riak CS + +## Access log directory. +## +## Default: $(platform_log_dir) +## +## Acceptable values: +## - text +log.access.dir = $(platform_log_dir) + +## URL rewrite module. +## +## Acceptable values: +## - text +## rewrite_module = riak_cs_s3_rewrite + +## Authentication module. +## +## Acceptable values: +## - text +## auth_module = riak_cs_s3_auth + +## Riak CS version number. This is used to selectively +## enable new features for the current version to better +## support rolling upgrades. New installs should not +## need to modify this. If peforming a rolling upgrade +## then keep the original value (if not defined, Riak CS +## uses 0 instead) of old app.config until all nodes +## have been upgraded and then set to the new value. +## +## Default: 10300 +## +## Acceptable values: +## - an integer +cs_version = 10300 + +## If your Erlang virtual machine supports DTrace (or +## user-space SystemTap), set dtrace_support to true. +## +## Default: off +## +## Acceptable values: +## - on or off +dtrace = off + +## Where to emit the default log messages (typically at 'info' +## severity): +## off: disabled +## file: the file specified by log.console.file +## console: to standard output (seen when using `riak attach-direct`) +## both: log.console.file and standard out. +## +## Default: file +## +## Acceptable values: +## - one of: off, file, console, both +log.console = file + +## The severity level of the console log, default is 'info'. +## +## Default: info +## +## Acceptable values: +## - one of: debug, info, notice, warning, error, critical, alert, emergency, none +log.console.level = info + +## When 'log.console' is set to 'file' or 'both', the file where +## console messages will be logged. +## +## Default: $(platform_log_dir)/console.log +## +## Acceptable values: +## - the path to a file +log.console.file = $(platform_log_dir)/console.log + +## The file where error messages will be logged. +## +## Default: $(platform_log_dir)/error.log +## +## Acceptable values: +## - the path to a file +log.error.file = $(platform_log_dir)/error.log + +## When set to 'on', enables log output to syslog. +## +## Default: off +## +## Acceptable values: +## - on or off +log.syslog = off + +## Whether to enable the crash log. +## +## Default: on +## +## Acceptable values: +## - on or off +log.crash = on + +## If the crash log is enabled, the file where its messages will +## be written. +## +## Default: $(platform_log_dir)/crash.log +## +## Acceptable values: +## - the path to a file +log.crash.file = $(platform_log_dir)/crash.log + +## Maximum size in bytes of individual messages in the crash log +## +## Default: 64KB +## +## Acceptable values: +## - a byte size with units, e.g. 10GB +log.crash.maximum_message_size = 64KB + +## Maximum size of the crash log in bytes, before it is rotated +## +## Default: 10MB +## +## Acceptable values: +## - a byte size with units, e.g. 10GB +log.crash.size = 10MB + +## The schedule on which to rotate the crash log. For more +## information see: +## https://github.com/basho/lager/blob/master/README.md#internal-log-rotation +## +## Default: $D0 +## +## Acceptable values: +## - text +log.crash.rotation = $D0 + +## The number of rotated crash logs to keep. When set to +## 'current', only the current open log file is kept. +## +## Default: 5 +## +## Acceptable values: +## - an integer +## - the text "current" +log.crash.rotation.keep = 5 + +## +## Default: /var/log/riak-cs +## +## Acceptable values: +## - the path to a directory +platform_log_dir = /var/log/riak-cs + +## Name of the Erlang node +## +## Default: riak-cs@127.0.0.1 +## +## Acceptable values: +## - text +nodename = riak-cs@127.0.0.1 + +## Cookie for distributed node communication. All nodes in the +## same cluster should use the same cookie or they will not be able to +## communicate. +## +## Default: riak +## +## Acceptable values: +## - text +distributed_cookie = riak + +## Sets the number of threads in async thread pool, valid range +## is 0-1024. If thread support is available, the default is 64. +## More information at: http://erlang.org/doc/man/erl.html +## +## Default: 64 +## +## Acceptable values: +## - an integer +erlang.async_threads = 64 + +## The number of concurrent ports/sockets +## Valid range is 1024-134217727 +## +## Default: 65536 +## +## Acceptable values: +## - an integer +erlang.max_ports = 65536 + +## Set scheduler forced wakeup interval. All run queues will be +## scanned each Interval milliseconds. While there are sleeping +## schedulers in the system, one scheduler will be woken for each +## non-empty run queue found. An Interval of zero disables this +## feature, which also is the default. +## This feature is a workaround for lengthy executing native code, and +## native code that do not bump reductions properly. +## More information: http://www.erlang.org/doc/man/erl.html#+sfwi +## +## Default: 500 +## +## Acceptable values: +## - an integer +## erlang.schedulers.force_wakeup_interval = 500 + +## Enable or disable scheduler compaction of load. By default +## scheduler compaction of load is enabled. When enabled, load +## balancing will strive for a load distribution which causes as many +## scheduler threads as possible to be fully loaded (i.e., not run out +## of work). This is accomplished by migrating load (e.g. runnable +## processes) into a smaller set of schedulers when schedulers +## frequently run out of work. When disabled, the frequency with which +## schedulers run out of work will not be taken into account by the +## load balancing logic. +## More information: http://www.erlang.org/doc/man/erl.html#+scl +## +## Default: false +## +## Acceptable values: +## - one of: true, false +## erlang.schedulers.compaction_of_load = false + +## Enable or disable scheduler utilization balancing of load. By +## default scheduler utilization balancing is disabled and instead +## scheduler compaction of load is enabled which will strive for a +## load distribution which causes as many scheduler threads as +## possible to be fully loaded (i.e., not run out of work). When +## scheduler utilization balancing is enabled the system will instead +## try to balance scheduler utilization between schedulers. That is, +## strive for equal scheduler utilization on all schedulers. +## More information: http://www.erlang.org/doc/man/erl.html#+sub +## +## Acceptable values: +## - one of: true, false +## erlang.schedulers.utilization_balancing = true + diff --git a/riak_test/docker/riak.advanced.config b/riak_test/docker/riak.advanced.config new file mode 100644 index 000000000..2734945ab --- /dev/null +++ b/riak_test/docker/riak.advanced.config @@ -0,0 +1,20 @@ +[ + {riak_kv, + [ + {add_paths, ["/usr/lib64/riak-cs/lib/riak_cs-2.1.2rc1/ebin"]}, + {storage_backend, riak_cs_kv_multi_backend}, + {multi_backend_prefix_list, [{<<"0b:">>, be_blocks}]}, + {multi_backend_default, be_default}, + {multi_backend, + [{be_default, riak_kv_eleveldb_backend, + [ + {max_open_files, 50}, + {data_root, "/var/lib/riak/leveldb"} + ]}, + {be_blocks, riak_kv_bitcask_backend, + [ + {data_root, "/var/lib/riak/bitcask"} + ]} + ]} + ]} +]. diff --git a/riak_test/docker/riak.advanced.config.tmpl b/riak_test/docker/riak.advanced.config.tmpl new file mode 100644 index 000000000..1dcdf5aa6 --- /dev/null +++ b/riak_test/docker/riak.advanced.config.tmpl @@ -0,0 +1,20 @@ +[ + {riak_kv, + [ + {add_paths, ["/usr/lib64/riak-cs/lib/riak_cs-@@CS_LIB_VSN@@/ebin"]}, + {storage_backend, riak_cs_kv_multi_backend}, + {multi_backend_prefix_list, [{<<"0b:">>, be_blocks}]}, + {multi_backend_default, be_default}, + {multi_backend, + [{be_default, riak_kv_eleveldb_backend, + [ + {max_open_files, 50}, + {data_root, "/var/lib/riak/leveldb"} + ]}, + {be_blocks, riak_kv_bitcask_backend, + [ + {data_root, "/var/lib/riak/bitcask"} + ]} + ]} + ]} +]. diff --git a/riak_test/docker/riak.conf b/riak_test/docker/riak.conf new file mode 100644 index 000000000..7e904426b --- /dev/null +++ b/riak_test/docker/riak.conf @@ -0,0 +1,495 @@ +buckets.default.allow_mult = true + +## Where to emit the default log messages (typically at 'info' +## severity): +## off: disabled +## file: the file specified by log.console.file +## console: to standard output (seen when using `riak attach-direct`) +## both: log.console.file and standard out. +## +## Default: file +## +## Acceptable values: +## - one of: off, file, console, both +log.console = file + +## The severity level of the console log, default is 'info'. +## +## Default: info +## +## Acceptable values: +## - one of: debug, info, notice, warning, error, critical, alert, emergency, none +log.console.level = info + +## When 'log.console' is set to 'file' or 'both', the file where +## console messages will be logged. +## +## Default: $(platform_log_dir)/console.log +## +## Acceptable values: +## - the path to a file +log.console.file = $(platform_log_dir)/console.log + +## The file where error messages will be logged. +## +## Default: $(platform_log_dir)/error.log +## +## Acceptable values: +## - the path to a file +log.error.file = $(platform_log_dir)/error.log + +## When set to 'on', enables log output to syslog. +## +## Default: off +## +## Acceptable values: +## - on or off +log.syslog = off + +## Whether to enable the crash log. +## +## Default: on +## +## Acceptable values: +## - on or off +log.crash = on + +## If the crash log is enabled, the file where its messages will +## be written. +## +## Default: $(platform_log_dir)/crash.log +## +## Acceptable values: +## - the path to a file +log.crash.file = $(platform_log_dir)/crash.log + +## Maximum size in bytes of individual messages in the crash log +## +## Default: 64KB +## +## Acceptable values: +## - a byte size with units, e.g. 10GB +log.crash.maximum_message_size = 64KB + +## Maximum size of the crash log in bytes, before it is rotated +## +## Default: 10MB +## +## Acceptable values: +## - a byte size with units, e.g. 10GB +log.crash.size = 10MB + +## The schedule on which to rotate the crash log. For more +## information see: +## https://github.com/basho/lager/blob/master/README.md#internal-log-rotation +## +## Default: $D0 +## +## Acceptable values: +## - text +log.crash.rotation = $D0 + +## The number of rotated crash logs to keep. When set to +## 'current', only the current open log file is kept. +## +## Default: 5 +## +## Acceptable values: +## - an integer +## - the text "current" +log.crash.rotation.keep = 5 + +## Name of the Erlang node +## +## Default: riak@127.0.0.1 +## +## Acceptable values: +## - text +nodename = riak@127.0.0.1 + +## Cookie for distributed node communication. All nodes in the +## same cluster should use the same cookie or they will not be able to +## communicate. +## +## Default: riak +## +## Acceptable values: +## - text +distributed_cookie = riak + +## Sets the number of threads in async thread pool, valid range +## is 0-1024. If thread support is available, the default is 64. +## More information at: http://erlang.org/doc/man/erl.html +## +## Default: 64 +## +## Acceptable values: +## - an integer +erlang.async_threads = 64 + +## The number of concurrent ports/sockets +## Valid range is 1024-134217727 +## +## Default: 65536 +## +## Acceptable values: +## - an integer +erlang.max_ports = 65536 + +## Set scheduler forced wakeup interval. All run queues will be +## scanned each Interval milliseconds. While there are sleeping +## schedulers in the system, one scheduler will be woken for each +## non-empty run queue found. An Interval of zero disables this +## feature, which also is the default. +## This feature is a workaround for lengthy executing native code, and +## native code that do not bump reductions properly. +## More information: http://www.erlang.org/doc/man/erl.html#+sfwi +## +## Default: 500 +## +## Acceptable values: +## - an integer +## erlang.schedulers.force_wakeup_interval = 500 + +## Enable or disable scheduler compaction of load. By default +## scheduler compaction of load is enabled. When enabled, load +## balancing will strive for a load distribution which causes as many +## scheduler threads as possible to be fully loaded (i.e., not run out +## of work). This is accomplished by migrating load (e.g. runnable +## processes) into a smaller set of schedulers when schedulers +## frequently run out of work. When disabled, the frequency with which +## schedulers run out of work will not be taken into account by the +## load balancing logic. +## More information: http://www.erlang.org/doc/man/erl.html#+scl +## +## Default: false +## +## Acceptable values: +## - one of: true, false +## erlang.schedulers.compaction_of_load = false + +## Enable or disable scheduler utilization balancing of load. By +## default scheduler utilization balancing is disabled and instead +## scheduler compaction of load is enabled which will strive for a +## load distribution which causes as many scheduler threads as +## possible to be fully loaded (i.e., not run out of work). When +## scheduler utilization balancing is enabled the system will instead +## try to balance scheduler utilization between schedulers. That is, +## strive for equal scheduler utilization on all schedulers. +## More information: http://www.erlang.org/doc/man/erl.html#+sub +## +## Acceptable values: +## - one of: true, false +## erlang.schedulers.utilization_balancing = true + +## Number of partitions in the cluster (only valid when first +## creating the cluster). Must be a power of 2, minimum 8 and maximum +## 1024. +## +## Default: 64 +## +## Acceptable values: +## - an integer +ring_size = 8 + +## Number of concurrent node-to-node transfers allowed. +## +## Default: 2 +## +## Acceptable values: +## - an integer +## transfer_limit = 2 + +## Default cert location for https can be overridden +## with the ssl config variable, for example: +## +## Acceptable values: +## - the path to a file +## ssl.certfile = $(platform_etc_dir)/cert.pem + +## Default key location for https can be overridden with the ssl +## config variable, for example: +## +## Acceptable values: +## - the path to a file +## ssl.keyfile = $(platform_etc_dir)/key.pem + +## Default signing authority location for https can be overridden +## with the ssl config variable, for example: +## +## Acceptable values: +## - the path to a file +## ssl.cacertfile = $(platform_etc_dir)/cacertfile.pem + +## DTrace support Do not enable 'dtrace' unless your Erlang/OTP +## runtime is compiled to support DTrace. DTrace is available in +## R15B01 (supported by the Erlang/OTP official source package) and in +## R14B04 via a custom source repository & branch. +## +## Default: off +## +## Acceptable values: +## - on or off +dtrace = off + +## Platform-specific installation paths (substituted by rebar) +## +## Default: /usr/sbin +## +## Acceptable values: +## - the path to a directory +platform_bin_dir = /usr/sbin + +## +## Default: /var/lib/riak +## +## Acceptable values: +## - the path to a directory +platform_data_dir = /var/lib/riak + +## +## Default: /etc/riak +## +## Acceptable values: +## - the path to a directory +platform_etc_dir = /etc/riak + +## +## Default: /usr/lib64/riak/lib +## +## Acceptable values: +## - the path to a directory +platform_lib_dir = /usr/lib64/riak/lib + +## +## Default: /var/log/riak +## +## Acceptable values: +## - the path to a directory +platform_log_dir = /var/log/riak + +## Enable consensus subsystem. Set to 'on' to enable the +## consensus subsystem used for strongly consistent Riak operations. +## +## Default: off +## +## Acceptable values: +## - on or off +## strong_consistency = on + +## listener.http. is an IP address and TCP port that the Riak +## HTTP interface will bind. +## +## Default: 127.0.0.1:8098 +## +## Acceptable values: +## - an IP/port pair, e.g. 127.0.0.1:10011 +listener.http.internal = 127.0.0.1:8098 + +## listener.protobuf. is an IP address and TCP port that the Riak +## Protocol Buffers interface will bind. +## +## Default: 127.0.0.1:8087 +## +## Acceptable values: +## - an IP/port pair, e.g. 127.0.0.1:10011 +listener.protobuf.internal = 127.0.0.1:8087 + +## The maximum length to which the queue of pending connections +## may grow. If set, it must be an integer > 0. If you anticipate a +## huge number of connections being initialized *simultaneously*, set +## this number higher. +## +## Default: 128 +## +## Acceptable values: +## - an integer +protobuf.backlog = 128 + +## listener.https. is an IP address and TCP port that the Riak +## HTTPS interface will bind. +## +## Acceptable values: +## - an IP/port pair, e.g. 127.0.0.1:10011 +## listener.https.internal = 127.0.0.1:8098 + +## How Riak will repair out-of-sync keys. Some features require +## this to be set to 'active', including search. +## * active: out-of-sync keys will be repaired in the background +## * passive: out-of-sync keys are only repaired on read +## * active-debug: like active, but outputs verbose debugging +## information +## +## Default: active +## +## Acceptable values: +## - one of: active, passive, active-debug +anti_entropy = active + +## Specifies the storage engine used for Riak's key-value data +## and secondary indexes (if supported). +## +## Default: bitcask +## +## Acceptable values: +## - one of: bitcask, leveldb, memory, multi +storage_backend = bitcask + +## Controls which binary representation of a riak value is stored +## on disk. +## * 0: Original erlang:term_to_binary format. Higher space overhead. +## * 1: New format for more compact storage of small values. +## +## Default: 1 +## +## Acceptable values: +## - the integer 1 +## - the integer 0 +object.format = 1 + +## Reading or writing objects bigger than this size will write a +## warning in the logs. +## +## Default: 5MB +## +## Acceptable values: +## - a byte size with units, e.g. 10GB +object.size.warning_threshold = 5MB + +## Writing an object bigger than this will send a failure to the +## client. +## +## Default: 50MB +## +## Acceptable values: +## - a byte size with units, e.g. 10GB +object.size.maximum = 50MB + +## Writing an object with more than this number of siblings will +## generate a warning in the logs. +## +## Default: 25 +## +## Acceptable values: +## - an integer +object.siblings.warning_threshold = 25 + +## Writing an object with more than this number of siblings will +## send a failure to the client. +## +## Default: 100 +## +## Acceptable values: +## - an integer +object.siblings.maximum = 100 + +## A path under which bitcask data files will be stored. +## +## Default: $(platform_data_dir)/bitcask +## +## Acceptable values: +## - the path to a directory +bitcask.data_root = $(platform_data_dir)/bitcask + +## Configure how Bitcask writes data to disk. +## erlang: Erlang's built-in file API +## nif: Direct calls to the POSIX C API +## The NIF mode provides higher throughput for certain +## workloads, but has the potential to negatively impact +## the Erlang VM, leading to higher worst-case latencies +## and possible throughput collapse. +## +## Default: erlang +## +## Acceptable values: +## - one of: erlang, nif +bitcask.io_mode = erlang + +## Set to 'off' to disable the admin panel. +## +## Default: off +## +## Acceptable values: +## - on or off +riak_control = off + +## Authentication mode used for access to the admin panel. +## +## Default: off +## +## Acceptable values: +## - one of: off, userlist +riak_control.auth.mode = off + +## If riak control's authentication mode (riak_control.auth.mode) +## is set to 'userlist' then this is the list of usernames and +## passwords for access to the admin panel. +## To create users with given names, add entries of the format: +## riak_control.auth.user.USERNAME.password = PASSWORD +## replacing USERNAME with the desired username and PASSWORD with the +## desired password for that user. +## +## Acceptable values: +## - text +## riak_control.auth.user.admin.password = pass + +## This parameter defines the percentage of total server memory +## to assign to LevelDB. LevelDB will dynamically adjust its internal +## cache sizes to stay within this size. The memory size can +## alternately be assigned as a byte count via leveldb.maximum_memory +## instead. +## +## Default: 70 +## +## Acceptable values: +## - an integer +leveldb.maximum_memory.percent = 70 + +## To enable Search set this 'on'. +## +## Default: off +## +## Acceptable values: +## - on or off +search = off + +## How long Riak will wait for Solr to start. The start sequence +## will be tried twice. If both attempts timeout, then the Riak node +## will be shutdown. This may need to be increased as more data is +## indexed and Solr takes longer to start. Values lower than 1s will +## be rounded up to the minimum 1s. +## +## Default: 30s +## +## Acceptable values: +## - a time duration with units, e.g. '10s' for 10 seconds +search.solr.start_timeout = 30s + +## The port number which Solr binds to. +## NOTE: Binds on every interface. +## +## Default: 8093 +## +## Acceptable values: +## - an integer +search.solr.port = 8093 + +## The port number which Solr JMX binds to. +## NOTE: Binds on every interface. +## +## Default: 8985 +## +## Acceptable values: +## - an integer +search.solr.jmx_port = 8985 + +## The options to pass to the Solr JVM. Non-standard options, +## i.e. -XX, may not be portable across JVM implementations. +## E.g. -XX:+UseCompressedStrings +## +## Default: -d64 -Xms1g -Xmx1g -XX:+UseStringCache -XX:+UseCompressedOops +## +## Acceptable values: +## - text +search.solr.jvm_options = -d64 -Xms1g -Xmx1g -XX:+UseStringCache -XX:+UseCompressedOops + diff --git a/riak_test/docker/solr-log4j.properties b/riak_test/docker/solr-log4j.properties new file mode 100644 index 000000000..4448875be --- /dev/null +++ b/riak_test/docker/solr-log4j.properties @@ -0,0 +1,8 @@ +log4j.rootLogger=WARN, rotate + +log4j.appender.rotate=org.apache.log4j.RollingFileAppender +log4j.appender.rotate.File=/var/log/riak/solr.log +log4j.appender.rotate.MaxFileSize=10MB +log4j.appender.rotate.MaxBackupIndex=5 +log4j.appender.rotate.layout=org.apache.log4j.PatternLayout +log4j.appender.rotate.layout.ConversionPattern=%d [%p] <%t>@%F:%L %m%n diff --git a/riak_test/docker/stanchion.advanced.config b/riak_test/docker/stanchion.advanced.config new file mode 100644 index 000000000..a5330263a --- /dev/null +++ b/riak_test/docker/stanchion.advanced.config @@ -0,0 +1 @@ +[{stanchion, []}]. diff --git a/riak_test/docker/stanchion.conf b/riak_test/docker/stanchion.conf new file mode 100644 index 000000000..ba744ab5a --- /dev/null +++ b/riak_test/docker/stanchion.conf @@ -0,0 +1,265 @@ +## listen port and IP address +## +## Default: 127.0.0.1:8085 +## +## Acceptable values: +## - an IP/port pair, e.g. 127.0.0.1:10011 +listener = 127.0.0.1:8085 + +## Default cert location for https can be overridden +## with the ssl config variable, for example: +## +## Acceptable values: +## - the path to a file +## ssl.certfile = $(platform_etc_dir)/cert.pem + +## Default key location for https can be overridden with the ssl +## config variable, for example: +## +## Acceptable values: +## - the path to a file +## ssl.keyfile = $(platform_etc_dir)/key.pem + +## Riak IP address and port number where Stanchion connects +## +## Default: 127.0.0.1:8087 +## +## Acceptable values: +## - an IP/port pair, e.g. 127.0.0.1:10011 +riak_host = 127.0.0.1:8087 + +## Admin user credentials. The credentials specified here must +## match the admin credentials specified in the riak-cs app.config +## for the system to function properly. +## +## Default: admin-key +## +## Acceptable values: +## - text +admin.key = admin-key + +## +## Default: admin-secret +## +## Acceptable values: +## - text +admin.secret = admin-secret + +## Platform-specific installation paths +## +## Default: /usr/sbin +## +## Acceptable values: +## - the path to a directory +platform_bin_dir = /usr/sbin + +## +## Default: /var/lib/stanchion +## +## Acceptable values: +## - the path to a directory +platform_data_dir = /var/lib/stanchion + +## +## Default: /etc/stanchion +## +## Acceptable values: +## - the path to a directory +platform_etc_dir = /etc/stanchion + +## +## Default: /usr/lib64/stanchion/lib +## +## Acceptable values: +## - the path to a directory +platform_lib_dir = /usr/lib64/stanchion/lib + +## +## Default: /var/log/stanchion +## +## Acceptable values: +## - the path to a directory +platform_log_dir = /var/log/stanchion + +## Where to emit the default log messages (typically at 'info' +## severity): +## off: disabled +## file: the file specified by log.console.file +## console: to standard output (seen when using `riak attach-direct`) +## both: log.console.file and standard out. +## +## Default: file +## +## Acceptable values: +## - one of: off, file, console, both +log.console = file + +## The severity level of the console log, default is 'info'. +## +## Default: info +## +## Acceptable values: +## - one of: debug, info, notice, warning, error, critical, alert, emergency, none +log.console.level = info + +## When 'log.console' is set to 'file' or 'both', the file where +## console messages will be logged. +## +## Default: $(platform_log_dir)/console.log +## +## Acceptable values: +## - the path to a file +log.console.file = $(platform_log_dir)/console.log + +## The file where error messages will be logged. +## +## Default: $(platform_log_dir)/error.log +## +## Acceptable values: +## - the path to a file +log.error.file = $(platform_log_dir)/error.log + +## When set to 'on', enables log output to syslog. +## +## Default: off +## +## Acceptable values: +## - on or off +log.syslog = off + +## Whether to enable the crash log. +## +## Default: on +## +## Acceptable values: +## - on or off +log.crash = on + +## If the crash log is enabled, the file where its messages will +## be written. +## +## Default: $(platform_log_dir)/crash.log +## +## Acceptable values: +## - the path to a file +log.crash.file = $(platform_log_dir)/crash.log + +## Maximum size in bytes of individual messages in the crash log +## +## Default: 64KB +## +## Acceptable values: +## - a byte size with units, e.g. 10GB +log.crash.maximum_message_size = 64KB + +## Maximum size of the crash log in bytes, before it is rotated +## +## Default: 10MB +## +## Acceptable values: +## - a byte size with units, e.g. 10GB +log.crash.size = 10MB + +## The schedule on which to rotate the crash log. For more +## information see: +## https://github.com/basho/lager/blob/master/README.md#internal-log-rotation +## +## Default: $D0 +## +## Acceptable values: +## - text +log.crash.rotation = $D0 + +## The number of rotated crash logs to keep. When set to +## 'current', only the current open log file is kept. +## +## Default: 5 +## +## Acceptable values: +## - an integer +## - the text "current" +log.crash.rotation.keep = 5 + +## Name of the Erlang node +## +## Default: stanchion@127.0.0.1 +## +## Acceptable values: +## - text +nodename = stanchion@127.0.0.1 + +## Cookie for distributed node communication. All nodes in the +## same cluster should use the same cookie or they will not be able to +## communicate. +## +## Default: riak +## +## Acceptable values: +## - text +distributed_cookie = riak + +## Sets the number of threads in async thread pool, valid range +## is 0-1024. If thread support is available, the default is 64. +## More information at: http://erlang.org/doc/man/erl.html +## +## Default: 64 +## +## Acceptable values: +## - an integer +erlang.async_threads = 64 + +## The number of concurrent ports/sockets +## Valid range is 1024-134217727 +## +## Default: 65536 +## +## Acceptable values: +## - an integer +erlang.max_ports = 65536 + +## Set scheduler forced wakeup interval. All run queues will be +## scanned each Interval milliseconds. While there are sleeping +## schedulers in the system, one scheduler will be woken for each +## non-empty run queue found. An Interval of zero disables this +## feature, which also is the default. +## This feature is a workaround for lengthy executing native code, and +## native code that do not bump reductions properly. +## More information: http://www.erlang.org/doc/man/erl.html#+sfwi +## +## Default: 500 +## +## Acceptable values: +## - an integer +## erlang.schedulers.force_wakeup_interval = 500 + +## Enable or disable scheduler compaction of load. By default +## scheduler compaction of load is enabled. When enabled, load +## balancing will strive for a load distribution which causes as many +## scheduler threads as possible to be fully loaded (i.e., not run out +## of work). This is accomplished by migrating load (e.g. runnable +## processes) into a smaller set of schedulers when schedulers +## frequently run out of work. When disabled, the frequency with which +## schedulers run out of work will not be taken into account by the +## load balancing logic. +## More information: http://www.erlang.org/doc/man/erl.html#+scl +## +## Default: false +## +## Acceptable values: +## - one of: true, false +## erlang.schedulers.compaction_of_load = false + +## Enable or disable scheduler utilization balancing of load. By +## default scheduler utilization balancing is disabled and instead +## scheduler compaction of load is enabled which will strive for a +## load distribution which causes as many scheduler threads as +## possible to be fully loaded (i.e., not run out of work). When +## scheduler utilization balancing is enabled the system will instead +## try to balance scheduler utilization between schedulers. That is, +## strive for equal scheduler utilization on all schedulers. +## More information: http://www.erlang.org/doc/man/erl.html#+sub +## +## Acceptable values: +## - one of: true, false +## erlang.schedulers.utilization_balancing = true + diff --git a/riak_test/docker/test.sh b/riak_test/docker/test.sh new file mode 100755 index 000000000..836c723cb --- /dev/null +++ b/riak_test/docker/test.sh @@ -0,0 +1,79 @@ +#!/bin/bash + +trap 'echo "[ERROR] $0:$LINENO \"$BASH_COMMAND\"" ; exit 1' ERR + +VSN=$@ +RIAK_LIB_VSN=$1 +ST_LIB_VSN=$2 +CS_LIB_VSN=$3 + +RIAK_VSN=${RIAK_LIB_VSN//-/.} +ST_VSN=${ST_LIB_VSN//-/.} +CS_VSN=${CS_LIB_VSN//-/.} + +echo "Riak : " ${RIAK_VSN} +echo "Stanchion: " ${ST_VSN} +echo "Riak CS : " ${CS_VSN} + +usage() { + echo "usage: ./test.sh RIAK_LIB_VSN ST_LIB_VSN CS_LIB_VSN" + exit 1 +} + +echo "Build package tester ${VSN}..." + +RIAK_PKG_LOCAL=riak-ee-${RIAK_VSN}-1.el6.x86_64.rpm +ST_PKG_LOCAL=stanchion-${ST_VSN}-1.el6.x86_64.rpm +CS_PKG_LOCAL=riak-cs-ee-${CS_VSN}-1.el6.x86_64.rpm + +echo "Riak : " ${RIAK_LIB_VSN} +echo "Stanchion: " ${ST_LIB_VSN} +echo "Riak CS : " ${CS_LIB_VSN} + +echo "Riak : " ${RIAK_PKG_LOCAL} +echo "Stanchion: " ${ST_PKG_LOCAL} +echo "Riak CS : " ${CS_PKG_LOCAL} + + +# rpm riak-cs-1.5.4.114.g9b7ca08-1 +# /usr/lib64/riak-cs/lib/riak_cs-1.5.4-114-g9b7ca08 + +cp riak.advanced.config.tmpl riak.advanced.config +sed -i s:@@CS_LIB_VSN@@:${CS_LIB_VSN}:g riak.advanced.config + +cp Dockerfile.tmpl Dockerfile +sed -i s:@@RIAK_PKG@@:$RIAK_PKG_LOCAL:g Dockerfile +sed -i s:@@ST_PKG@@:$ST_PKG_LOCAL:g Dockerfile +sed -i s:@@CS_PKG@@:$CS_PKG_LOCAL:g Dockerfile + +TAG=package-tester-${CS_VSN} +echo "building package ${TAG}" + +sudo docker build -t ${TAG} . + +# sudo docker run \ +# --name riak-cs \ +# -p 8080:8080 -p 8098:8098 -p 8087:8087\ +# -v `pwd`/logs:/var/log/supervisor \ +# -v `pwd`/logs/riak:/var/log/riak \ +# -v `pwd`/logs/riak-cs:/var/log/riak-cs \ +# -v `pwd`/logs/stanchion:/var/log/stanchion \ +# -d ksauzz/riak-cs + +rm -rf ${PWD}/logs/ +mkdir -p ${PWD}/logs/ +mkdir -p ${PWD}/logs/riak/ +mkdir -p ${PWD}/logs/stanchion/ +mkdir -p ${PWD}/logs/riak-cs/ +chmod -R 777 ${PWD}/logs/ + +sudo docker \ + run -it \ + -v ${PWD}/logs:/var/log/supervisor \ + -v ${PWD}/logs/riak:/var/log/riak \ + -v ${PWD}/logs/riak-cs:/var/log/riak-cs \ + -v ${PWD}/logs/stanchion:/var/log/stanchion \ + ${TAG} \ + /root/confirm.sh + +echo "Package built as ${TAG}"