All views are rebuilt after restarting bigcouch

[nickbecker]

We're running an 8 node cluster on ubuntu 12.04, Erlang R14B04, with 3 different databases, each with ~15 million docs. When the bigcouch service is restarted, all of the views in each database start rebuilding from scratch, causing major downtime for our applications. This genrally happens when the server has been running for over 24 hours.

This is the only log entry we see when the server starts up.

[Thu, 12 Jul 2012 15:25:03 GMT] [error] [emulator] [--------] Error in process <0.4807.5> on node 'xx@xx.xx.xx' with exit value: {{nocatch,conflict},[{fabric,update_doc,3},{chttpd_db,'-update_doc/6-fun-0-',3}]}

[rnewson]

what version of bigcouch?

[nickbecker]

We are running the tagged version of 0.4.0.

The user's ulimit setting is set to 1000000.

vm.args
-name xxx@xxx.xxx.xxx
-setcookie XXXXXXXX
-sasl errlog_type error
+K true
+A 16
+P 262144
+Bd -noinput

config
{
httpd_design_handlers: {
_info: "{couch_httpd_db, handle_design_info_req}",
_list: "{couch_httpd_show, handle_view_list_req}",
_rewrite: "{couch_httpd_rewrite, handle_rewrite_req}",
_show: "{couch_httpd_show, handle_doc_show_req}",
_update: "{couch_httpd_show, handle_doc_update_req}",
_view: "{couch_httpd_view, handle_view_req}"
},
uuids: {
algorithm: "sequential"
},
cluster: {
n: "3",
q: "32",
r: "2",
w: "2"
},
stats: {
rate: "1000",
samples: "[0, 60, 300, 900]"
},
chttpd: {
WWW-Authenticate: "Basic realm="administrator"",
bind_address: "0.0.0.0",
changes_timeout: "1200000",
docroot: "/opt/bigcouch/share/www",
max_connections: "4096",
port: "5984",
server_options: "[{backlog, 512}, {acceptor_pool_size, 64}]",
socket_options: "[{nodelay, true}]"
},
httpd_global_handlers: {
/: "{couch_httpd_misc_handlers, handle_welcome_req, <<"Welcome">>}",
_active_tasks: "{couch_httpd_misc_handlers, handle_task_status_req}",
_all_dbs: "{couch_httpd_misc_handlers, handle_all_dbs_req}",
_config: "{couch_httpd_misc_handlers, handle_config_req}",
_log: "{couch_httpd_misc_handlers, handle_log_req}",
_oauth: "{couch_httpd_oauth, handle_oauth_req}",
_replicate: "{couch_httpd_misc_handlers, handle_replicate_req}",
_restart: "{couch_httpd_misc_handlers, handle_restart_req}",
_session: "{couch_httpd_auth, handle_session_req}",
_stats: "{couch_httpd_stats_handlers, handle_stats_req}",
_system: "{chttpd_misc, handle_system_req}",
_utils: "{couch_httpd_misc_handlers, handle_utils_dir_req, "/opt/bigcouch/share/www"}",
_uuids: "{couch_httpd_misc_handlers, handle_uuids_req}",
favicon.ico: "{couch_httpd_misc_handlers, handle_favicon_req, "/opt/bigcouch/share/www"}"
},
attachments: {
compressible_types: "text/*, application/javascript, application/json, application/xml",
compression_level: "8"
},
admins: {
admin: "-hashed-xxx"
},
query_server_config: {
os_process_soft_limit: "200",
reduce_limit: "true"
},
replicator: {
db: "_replicator",
max_http_pipeline_size: "50",
max_http_sessions: "20",
max_replication_retry_count: "10",
ssl_certificate_max_depth: "3",
verify_ssl_certificates: "false"
},
ssl: {
port: "6984"
},
log: {
file: "/storage/log/bigcouch.log",
include_sasl: "true",
level: "debug"
},
query_servers: {
javascript: "/opt/bigcouch/bin/couchjs /opt/bigcouch/share/couchjs/main.js"
},
daemons: {
auth_cache: "{couch_auth_cache, start_link, []}",
external_manager: "{couch_external_manager, start_link, []}",
httpd: "{couch_httpd, start_link, []}",
os_daemons: "{couch_os_daemons, start_link, []}",
query_servers: "{couch_proc_manager, start_link, []}",
replication_manager: "{couch_replication_manager, start_link, []}",
stats_aggregator: "{couch_stats_aggregator, start, []}",
stats_collector: "{couch_stats_collector, start, []}",
uuids: "{couch_uuids, start, []}",
vhosts: "{couch_httpd_vhost, start_link, []}",
view_manager: "{couch_view, start_link, []}"
},
httpd: {
WWW-Authenticate: "Basic realm="administrator"",
allow_jsonp: "true",
authentication_handlers: "{couch_httpd_oauth, oauth_authentication_handler}, {couch_httpd_auth, cookie_authentication_handler}, {couch_httpd_auth, default_authentication_handler}",
bind_address: "0.0.0.0",
changes_timeout: "120000",
default_handler: "{couch_httpd_db, handle_request}",
log_max_chunk_size: "1000000",
max_connections: "4096",
port: "5986",
secure_rewrites: "true",
server_options: "[{backlog, 512}, {acceptor_pool_size, 64}]",
socket_options: "[{nodelay, true}, {keepalive, true}]",
vhost_global_handlers: "_utils, _uuids, _session, _oauth, _users"
},
httpd_db_handlers: {
_changes: "{couch_httpd_db, handle_changes_req}",
_compact: "{couch_httpd_db, handle_compact_req}",
_design: "{couch_httpd_db, handle_design_req}",
_temp_view: "{couch_httpd_view, handle_temp_view_req}",
_view_cleanup: "{couch_httpd_db, handle_view_cleanup_req}"
},
couch_httpd_auth: {
auth_cache_size: "50",
authentication_db: "_users",
authentication_redirect: "/_utils/session.html",
require_valid_user: "true",
secret: "",
timeout: "43200"
},
chttps: {
port: "6984",
ssl_certificate_max_depth: "1",
verify_ssl_certificates: "false"
},
couchdb: {
database_dir: "/storage/data",
delayed_commits: "false",
max_dbs_open: "500",
max_document_size: "67108864",
os_process_timeout: "20000",
view_index_dir: "/storage2/data"
}
}

[rnewson]

Could you switch to debug logging and see if you see any view checkpoints happening?

"checkpointing view update at seq ~p for ~s ~s""

You can tweak log level with;

curl localhost:5986/_config/log/level -XPUT -d '"debug"'

I suggest you do this on one machine for a short period of time as debug level is quite verbose. Obviously be sure to add or update a document and query the view while at debug level.

[nickbecker]

The server had been running fine. I changed the log level then restarted the server. The first time this particular design doc was accessed after the restart, the following log lines appeared. I have a fairly clean snippet of the log file that I can post if you need more info.

[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.3386.0>] [--------] Spawning new group server for view group _design/userpollstate in database shards/28000000-2fffffff/geopollsurveys.1340078802.
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.3384.0>] [--------] Spawning new group server for view group _design/userpollstate in database shards/08000000-0fffffff/geopollsurveys.1340078802.
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.3389.0>] [--------] Spawning new group server for view group _design/userpollstate in database shards/48000000-4fffffff/geopollsurveys.1340078802.
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.3390.0>] [--------] Spawning new group server for view group _design/userpollstate in database shards/68000000-6fffffff/geopollsurveys.1340078802.
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.3397.0>] [--------] Spawning new group server for view group _design/userpollstate in database shards/c8000000-cfffffff/geopollsurveys.1340078802.
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.3395.0>] [--------] Spawning new group server for view group _design/userpollstate in database shards/88000000-8fffffff/geopollsurveys.1340078802.
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.3396.0>] [--------] Spawning new group server for view group _design/userpollstate in database shards/a8000000-afffffff/geopollsurveys.1340078802.
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.3401.0>] [--------] Spawning new group server for view group _design/userpollstate in database shards/e8000000-efffffff/geopollsurveys.1340078802.
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.3394.0>] [--------] Resetting group index "_design/userpollstate" in db shards/28000000-2fffffff/geopollsurveys.1340078802
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.3403.0>] [--------] Resetting group index "_design/userpollstate" in db shards/48000000-4fffffff/geopollsurveys.1340078802
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.3400.0>] [--------] Resetting group index "_design/userpollstate" in db shards/08000000-0fffffff/geopollsurveys.1340078802
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.3393.0>] [7fdcb02b] request_group {Pid, Seq} {<0.3407.0>,577750}
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.3387.0>] [7fdcb02b] request_group {Pid, Seq} {<0.3403.0>,577619}
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.3385.0>] [7fdcb02b] request_group {Pid, Seq} {<0.3394.0>,577239}
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.3410.0>] [--------] Resetting group index "_design/userpollstate" in db shards/88000000-8fffffff/geopollsurveys.1340078802
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.3415.0>] [--------] Resetting group index "_design/userpollstate" in db shards/e8000000-efffffff/geopollsurveys.1340078802
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.3383.0>] [7fdcb02b] request_group {Pid, Seq} {<0.3400.0>,581400}
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.2527.0>] [--------] checkpointing view update at seq 35 for shards/60000000-67ffffff/billing.1332864988 _design/bill
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.3391.0>] [7fdcb02b] request_group {Pid, Seq} {<0.3410.0>,558045}
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.3399.0>] [7fdcb02b] request_group {Pid, Seq} {<0.3415.0>,581678}
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.3413.0>] [--------] Resetting group index "_design/userpollstate" in db shards/a8000000-afffffff/geopollsurveys.1340078802
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.3392.0>] [7fdcb02b] request_group {Pid, Seq} {<0.3413.0>,579831}
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.2527.0>] [--------] checkpointing view update at seq 36 for shards/60000000-67ffffff/billing.1332864988 _design/bill
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.539.0>] [--------] checkpointing view update at seq 141 for shards/b4000000-b7ffffff/geopoll.1340989015 _design/StaticRecipientLists
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.2527.0>] [--------] checkpointing view update at seq 42 for shards/60000000-67ffffff/billing.1332864988 _design/bill
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.2511.0>] [--------] checkpointing view update at seq 38 for shards/20000000-27ffffff/billing.1332864988 _design/bill
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.539.0>] [--------] checkpointing view update at seq 143 for shards/b4000000-b7ffffff/geopoll.1340989015 _design/StaticRecipientLists
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.539.0>] [--------] checkpointing view update at seq 145 for shards/b4000000-b7ffffff/geopoll.1340989015 _design/StaticRecipientLists
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.539.0>] [--------] checkpointing view update at seq 147 for shards/b4000000-b7ffffff/geopoll.1340989015 _design/StaticRecipientLists
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.2527.0>] [--------] checkpointing view update at seq 43 for shards/60000000-67ffffff/billing.1332864988 _design/bill
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.3403.0>] [--------] checkpointing view update at seq 1 for shards/48000000-4fffffff/geopollsurveys.1340078802 _design/userpollstate
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.2527.0>] [--------] checkpointing view update at seq 47 for shards/60000000-67ffffff/billing.1332864988 _design/bill
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.2527.0>] [--------] checkpointing view update at seq 50 for shards/60000000-67ffffff/billing.1332864988 _design/bill
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.539.0>] [--------] checkpointing view update at seq 151 for shards/b4000000-b7ffffff/geopoll.1340989015 _design/StaticRecipientLists
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.2527.0>] [--------] checkpointing view update at seq 58 for shards/60000000-67ffffff/billing.1332864988 _design/bill
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.539.0>] [--------] checkpointing view update at seq 154 for shards/b4000000-b7ffffff/geopoll.1340989015 _design/StaticRecipientLists
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.3406.0>] [--------] Resetting group index "_design/userpollstate" in db shards/68000000-6fffffff/geopollsurveys.1340078802
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.3403.0>] [--------] checkpointing view update at seq 2 for shards/48000000-4fffffff/geopollsurveys.1340078802 _design/userpollstate
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.2527.0>] [--------] checkpointing view update at seq 59 for shards/60000000-67ffffff/billing.1332864988 _design/bill
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.2511.0>] [--------] checkpointing view update at seq 39 for shards/20000000-27ffffff/billing.1332864988 _design/bill
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.2540.0>] [--------] checkpointing view update at seq 1789 for shards/88000000-8fffffff/billing.1332864988 _design/bill
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.539.0>] [--------] checkpointing view update at seq 157 for shards/b4000000-b7ffffff/geopoll.1340989015 _design/StaticRecipientLists
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.2511.0>] [--------] checkpointing view update at seq 40 for shards/20000000-27ffffff/billing.1332864988 _design/bill
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.3388.0>] [7fdcb02b] request_group {Pid, Seq} {<0.3406.0>,581838}
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.2516.0>] [--------] checkpointing view update at seq 50 for shards/48000000-4fffffff/billing.1332864988 _design/bill
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.3403.0>] [--------] checkpointing view update at seq 3 for shards/48000000-4fffffff/geopollsurveys.1340078802 _design/userpollstate
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.2511.0>] [--------] checkpointing view update at seq 41 for shards/20000000-27ffffff/billing.1332864988 _design/bill
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.539.0>] [--------] checkpointing view update at seq 159 for shards/b4000000-b7ffffff/geopoll.1340989015 _design/StaticRecipientLists
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.3403.0>] [--------] checkpointing view update at seq 6 for shards/48000000-4fffffff/geopollsurveys.1340078802 _design/userpollstate
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.539.0>] [--------] checkpointing view update at seq 162 for shards/b4000000-b7ffffff/geopoll.1340989015 _design/StaticRecipientLists
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.2512.0>] [--------] checkpointing view update at seq 9 for shards/30000000-37ffffff/billing.1332864988 _design/bill
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.2511.0>] [--------] checkpointing view update at seq 42 for shards/20000000-27ffffff/billing.1332864988 _design/bill
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.539.0>] [--------] checkpointing view update at seq 164 for shards/b4000000-b7ffffff/geopoll.1340989015 _design/StaticRecipientLists
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.3403.0>] [--------] checkpointing view update at seq 12 for shards/48000000-4fffffff/geopollsurveys.1340078802 _design/userpollstate
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.2516.0>] [--------] checkpointing view update at seq 52 for shards/48000000-4fffffff/billing.1332864988 _design/bill
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.2511.0>] [--------] checkpointing view update at seq 43 for shards/20000000-27ffffff/billing.1332864988 _design/bill
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.539.0>] [--------] checkpointing view update at seq 166 for shards/b4000000-b7ffffff/geopoll.1340989015 _design/StaticRecipientLists
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.2516.0>] [--------] checkpointing view update at seq 54 for shards/48000000-4fffffff/billing.1332864988 _design/bill
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.2511.0>] [--------] checkpointing view update at seq 44 for shards/20000000-27ffffff/billing.1332864988 _design/bill
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.2511.0>] [--------] checkpointing view update at seq 45 for shards/20000000-27ffffff/billing.1332864988 _design/bill
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.539.0>] [--------] checkpointing view update at seq 168 for shards/b4000000-b7ffffff/geopoll.1340989015 _design/StaticRecipientLists
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.2511.0>] [--------] checkpointing view update at seq 46 for shards/20000000-27ffffff/billing.1332864988 _design/bill
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.3394.0>] [--------] checkpointing view update at seq 1 for shards/28000000-2fffffff/geopollsurveys.1340078802 _design/userpollstate
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.2511.0>] [--------] checkpointing view update at seq 51 for shards/20000000-27ffffff/billing.1332864988 _design/bill
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.3403.0>] [--------] checkpointing view update at seq 13 for shards/48000000-4fffffff/geopollsurveys.1340078802 _design/userpollstate
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.2527.0>] [--------] checkpointing view update at seq 60 for shards/60000000-67ffffff/billing.1332864988 _design/bill
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.3403.0>] [--------] checkpointing view update at seq 51 for shards/48000000-4fffffff/geopollsurveys.1340078802 _design/userpollstate
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.3403.0>] [--------] checkpointing view update at seq 52 for shards/48000000-4fffffff/geopollsurveys.1340078802 _design/userpollstate
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.539.0>] [--------] checkpointing view update at seq 170 for shards/b4000000-b7ffffff/geopoll.1340989015 _design/StaticRecipientLists
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.2527.0>] [--------] checkpointing view update at seq 69 for shards/60000000-67ffffff/billing.1332864988 _design/bill
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.3406.0>] [--------] checkpointing view update at seq 1 for shards/68000000-6fffffff/geopollsurveys.1340078802 _design/userpollstate
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.2516.0>] [--------] checkpointing view update at seq 55 for shards/48000000-4fffffff/billing.1332864988 _design/bill
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.539.0>] [--------] checkpointing view update at seq 173 for shards/b4000000-b7ffffff/geopoll.1340989015 _design/StaticRecipientLists
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.2516.0>] [--------] checkpointing view update at seq 60 for shards/48000000-4fffffff/billing.1332864988 _design/bill
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.2511.0>] [--------] checkpointing view update at seq 52 for shards/20000000-27ffffff/billing.1332864988 _design/bill
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.3394.0>] [--------] checkpointing view update at seq 2 for shards/28000000-2fffffff/geopollsurveys.1340078802 _design/userpollstate
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.3394.0>] [--------] checkpointing view update at seq 3 for shards/28000000-2fffffff/geopollsurveys.1340078802 _design/userpollstate
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.2516.0>] [--------] checkpointing view update at seq 61 for shards/48000000-4fffffff/billing.1332864988 _design/bill
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.2511.0>] [--------] checkpointing view update at seq 53 for shards/20000000-27ffffff/billing.1332864988 _design/bill
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.3394.0>] [--------] checkpointing view update at seq 4 for shards/28000000-2fffffff/geopollsurveys.1340078802 _design/userpollstate
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.2511.0>] [--------] checkpointing view update at seq 57 for shards/20000000-27ffffff/billing.1332864988 _design/bill
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.2540.0>] [--------] checkpointing view update at seq 1953 for shards/88000000-8fffffff/billing.1332864988 _design/bill
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.2516.0>] [--------] checkpointing view update at seq 66 for shards/48000000-4fffffff/billing.1332864988 _design/bill
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.2511.0>] [--------] checkpointing view update at seq 58 for shards/20000000-27ffffff/billing.1332864988 _design/bill
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.3403.0>] [--------] checkpointing view update at seq 53 for shards/48000000-4fffffff/geopollsurveys.1340078802 _design/userpollstate
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.2516.0>] [--------] checkpointing view update at seq 67 for shards/48000000-4fffffff/billing.1332864988 _design/bill
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.539.0>] [--------] checkpointing view update at seq 177 for shards/b4000000-b7ffffff/geopoll.1340989015 _design/StaticRecipientLists
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.3406.0>] [--------] checkpointing view update at seq 2 for shards/68000000-6fffffff/geopollsurveys.1340078802 _design/userpollstate
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.3394.0>] [--------] checkpointing view update at seq 5 for shards/28000000-2fffffff/geopollsurveys.1340078802 _design/userpollstate
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.2511.0>] [--------] checkpointing view update at seq 59 for shards/20000000-27ffffff/billing.1332864988 _design/bill
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.3394.0>] [--------] checkpointing view update at seq 6 for shards/28000000-2fffffff/geopollsurveys.1340078802 _design/userpollstate
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.2516.0>] [--------] checkpointing view update at seq 68 for shards/48000000-4fffffff/billing.1332864988 _design/bill
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.3394.0>] [--------] checkpointing view update at seq 7 for shards/28000000-2fffffff/geopollsurveys.1340078802 _design/userpollstate
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.2511.0>] [--------] checkpointing view update at seq 60 for shards/20000000-27ffffff/billing.1332864988 _design/bill
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.2516.0>] [--------] checkpointing view update at seq 69 for shards/48000000-4fffffff/billing.1332864988 _design/bill
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.539.0>] [--------] checkpointing view update at seq 181 for shards/b4000000-b7ffffff/geopoll.1340989015 _design/StaticRecipientLists
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.539.0>] [--------] checkpointing view update at seq 185 for shards/b4000000-b7ffffff/geopoll.1340989015 _design/StaticRecipientLists
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.3394.0>] [--------] checkpointing view update at seq 8 for shards/28000000-2fffffff/geopollsurveys.1340078802 _design/userpollstate
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.3415.0>] [--------] checkpointing view update at seq 1 for shards/e8000000-efffffff/geopollsurveys.1340078802 _design/userpollstate
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.2516.0>] [--------] checkpointing view update at seq 70 for shards/48000000-4fffffff/billing.1332864988 _design/bill
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.539.0>] [--------] checkpointing view update at seq 190 for shards/b4000000-b7ffffff/geopoll.1340989015 _design/StaticRecipientLists
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.539.0>] [--------] checkpointing view update at seq 193 for shards/b4000000-b7ffffff/geopoll.1340989015 _design/StaticRecipientLists
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.3406.0>] [--------] checkpointing view update at seq 3 for shards/68000000-6fffffff/geopollsurveys.1340078802 _design/userpollstate
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.3406.0>] [--------] checkpointing view update at seq 4 for shards/68000000-6fffffff/geopollsurveys.1340078802 _design/userpollstate
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.3415.0>] [--------] checkpointing view update at seq 2 for shards/e8000000-efffffff/geopollsurveys.1340078802 _design/userpollstate
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.3415.0>] [--------] checkpointing view update at seq 3 for shards/e8000000-efffffff/geopollsurveys.1340078802 _design/userpollstate
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.3415.0>] [--------] checkpointing view update at seq 4 for shards/e8000000-efffffff/geopollsurveys.1340078802 _design/userpollstate
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.3413.0>] [--------] checkpointing view update at seq 1 for shards/a8000000-afffffff/geopollsurveys.1340078802 _design/userpollstate
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.3415.0>] [--------] checkpointing view update at seq 6 for shards/e8000000-efffffff/geopollsurveys.1340078802 _design/userpollstate
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.3406.0>] [--------] checkpointing view update at seq 5 for shards/68000000-6fffffff/geopollsurveys.1340078802 _design/userpollstate
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.3415.0>] [--------] checkpointing view update at seq 8 for shards/e8000000-efffffff/geopollsurveys.1340078802 _design/userpollstate
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.3394.0>] [--------] checkpointing view update at seq 9 for shards/28000000-2fffffff/geopollsurveys.1340078802 _design/userpollstate
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.3394.0>] [--------] checkpointing view update at seq 10 for shards/28000000-2fffffff/geopollsurveys.1340078802 _design/userpollstate
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.2516.0>] [--------] checkpointing view update at seq 71 for shards/48000000-4fffffff/billing.1332864988 _design/bill
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.2511.0>] [--------] checkpointing view update at seq 61 for shards/20000000-27ffffff/billing.1332864988 _design/bill
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.2516.0>] [--------] checkpointing view update at seq 72 for shards/48000000-4fffffff/billing.1332864988 _design/bill
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.3406.0>] [--------] checkpointing view update at seq 6 for shards/68000000-6fffffff/geopollsurveys.1340078802 _design/userpollstate
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.3406.0>] [--------] checkpointing view update at seq 7 for shards/68000000-6fffffff/geopollsurveys.1340078802 _design/userpollstate
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.2511.0>] [--------] checkpointing view update at seq 62 for shards/20000000-27ffffff/billing.1332864988 _design/bill
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.3406.0>] [--------] checkpointing view update at seq 8 for shards/68000000-6fffffff/geopollsurveys.1340078802 _design/userpollstate
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.2516.0>] [--------] checkpointing view update at seq 73 for shards/48000000-4fffffff/billing.1332864988 _design/bill
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.2540.0>] [--------] checkpointing view update at seq 2035 for shards/88000000-8fffffff/billing.1332864988 _design/bill
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.3394.0>] [--------] checkpointing view update at seq 11 for shards/28000000-2fffffff/geopollsurveys.1340078802 _design/userpollstate
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.2516.0>] [--------] checkpointing view update at seq 74 for shards/48000000-4fffffff/billing.1332864988 _design/bill
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.2511.0>] [--------] checkpointing view update at seq 63 for shards/20000000-27ffffff/billing.1332864988 _design/bill
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.3415.0>] [--------] checkpointing view update at seq 9 for shards/e8000000-efffffff/geopollsurveys.1340078802 _design/userpollstate
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.2511.0>] [--------] checkpointing view update at seq 64 for shards/20000000-27ffffff/billing.1332864988 _design/bill
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.3406.0>] [--------] checkpointing view update at seq 9 for shards/68000000-6fffffff/geopollsurveys.1340078802 _design/userpollstate
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.3394.0>] [--------] checkpointing view update at seq 12 for shards/28000000-2fffffff/geopollsurveys.1340078802 _design/userpollstate
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.2516.0>] [--------] checkpointing view update at seq 75 for shards/48000000-4fffffff/billing.1332864988 _design/bill
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.2511.0>] [--------] checkpointing view update at seq 65 for shards/20000000-27ffffff/billing.1332864988 _design/bill
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.3406.0>] [--------] checkpointing view update at seq 10 for shards/68000000-6fffffff/geopollsurveys.1340078802 _design/userpollstate
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.539.0>] [--------] checkpointing view update at seq 194 for shards/b4000000-b7ffffff/geopoll.1340989015 _design/StaticRecipientLists
[Wed, 18 Jul 2012 12:06:37 GMT] [debug] [<0.3406.0>] [--------] checkpointing view update at seq 12 for shards/68000000-6fffffff/geopollsurveys.1340078802 _design/userpollstate