============================================================================== couchbase logs (info) cbbrowse_logs info ============================================================================== [ns_server:info,2012-11-02T18:23:11.751,nonode@nohost:<0.59.0>:ns_server:init_logging:225]Started & configured logging [ns_server:info,2012-11-02T18:23:11.752,nonode@nohost:<0.59.0>:ns_server:log_pending:30]Static config terms: [{error_logger_mf_dir,"/opt/couchbase/var/lib/couchbase/logs"}, {error_logger_mf_maxbytes,10485760}, {error_logger_mf_maxfiles,20}, {path_config_bindir,"/opt/couchbase/bin"}, {path_config_etcdir,"/opt/couchbase/etc/couchbase"}, {path_config_libdir,"/opt/couchbase/lib"}, {path_config_datadir,"/opt/couchbase/var/lib/couchbase"}, {path_config_tmpdir,"/opt/couchbase/var/lib/couchbase/tmp"}, {loglevel_default,debug}, {loglevel_couchdb,info}, {loglevel_ns_server,debug}, {loglevel_error_logger,debug}, {loglevel_user,debug}, {loglevel_menelaus,debug}, {loglevel_ns_doctor,debug}, {loglevel_stats,debug}, {loglevel_rebalance,debug}, {loglevel_cluster,debug}, {loglevel_views,debug}, {loglevel_mapreduce_errors,debug}, {loglevel_xdcr,debug}] [error_logger:info,2012-11-02T18:23:11.775,nonode@nohost:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,crypto_sup} started: [{pid,<0.173.0>}, {name,crypto_server}, {mfargs,{crypto_server,start_link,[]}}, {restart_type,permanent}, {shutdown,2000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:11.775,nonode@nohost:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= application: crypto started_at: nonode@nohost [error_logger:info,2012-11-02T18:23:11.777,nonode@nohost:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= application: public_key started_at: nonode@nohost [error_logger:info,2012-11-02T18:23:11.780,nonode@nohost:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,inets_sup} started: [{pid,<0.179.0>}, {name,ftp_sup}, {mfargs,{ftp_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:23:11.788,nonode@nohost:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,httpc_profile_sup} started: [{pid,<0.182.0>}, {name,httpc_manager}, {mfargs, {httpc_manager,start_link, [default,only_session_cookies,inets]}}, {restart_type,permanent}, {shutdown,4000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:11.788,nonode@nohost:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,httpc_sup} started: [{pid,<0.181.0>}, {name,httpc_profile_sup}, {mfargs, {httpc_profile_sup,start_link, [[{httpc,{default,only_session_cookies}}]]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:23:11.789,nonode@nohost:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,httpc_sup} started: [{pid,<0.183.0>}, {name,httpc_handler_sup}, {mfargs,{httpc_handler_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:23:11.790,nonode@nohost:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,inets_sup} started: [{pid,<0.180.0>}, {name,httpc_sup}, {mfargs, {httpc_sup,start_link, [[{httpc,{default,only_session_cookies}}]]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:23:11.791,nonode@nohost:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,inets_sup} started: [{pid,<0.184.0>}, {name,httpd_sup}, {mfargs,{httpd_sup,start_link,[[]]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:23:11.792,nonode@nohost:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,inets_sup} started: [{pid,<0.185.0>}, {name,tftp_sup}, {mfargs,{tftp_sup,start_link,[[]]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:23:11.792,nonode@nohost:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= application: inets started_at: nonode@nohost [error_logger:info,2012-11-02T18:23:11.792,nonode@nohost:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= application: oauth started_at: nonode@nohost [error_logger:info,2012-11-02T18:23:11.795,nonode@nohost:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ssl_sup} started: [{pid,<0.191.0>}, {name,ssl_broker_sup}, {mfargs,{ssl_broker_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,2000}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:23:11.798,nonode@nohost:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ssl_sup} started: [{pid,<0.192.0>}, {name,ssl_manager}, {mfargs,{ssl_manager,start_link,[[]]}}, {restart_type,permanent}, {shutdown,4000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:11.798,nonode@nohost:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ssl_sup} started: [{pid,<0.193.0>}, {name,ssl_connection}, {mfargs,{ssl_connection_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,4000}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:23:11.798,nonode@nohost:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= application: ssl started_at: nonode@nohost [error_logger:info,2012-11-02T18:23:11.820,nonode@nohost:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ssl_sup} started: [{pid,<0.200.0>}, {name,ssl_server}, {mfargs,{ssl_server,start_link,[]}}, {restart_type,permanent}, {shutdown,2000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:11.820,nonode@nohost:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,lhttpc_sup} started: [{pid,<0.198.0>}, {name,lhttpc_manager}, {mfargs, {lhttpc_manager,start_link, [[{name,lhttpc_manager}]]}}, {restart_type,permanent}, {shutdown,10000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:11.820,nonode@nohost:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= application: lhttpc started_at: nonode@nohost [error_logger:info,2012-11-02T18:23:11.821,nonode@nohost:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= application: mochiweb started_at: nonode@nohost [error_logger:info,2012-11-02T18:23:11.822,nonode@nohost:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= application: couch_view_parser started_at: nonode@nohost [error_logger:info,2012-11-02T18:23:11.823,nonode@nohost:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= application: couch_set_view started_at: nonode@nohost [error_logger:info,2012-11-02T18:23:11.823,nonode@nohost:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= application: couch_index_merger started_at: nonode@nohost [error_logger:info,2012-11-02T18:23:11.824,nonode@nohost:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= application: mapreduce started_at: nonode@nohost [error_logger:info,2012-11-02T18:23:11.844,nonode@nohost:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,couch_server_sup} started: [{pid,<0.209.0>}, {name,couch_config}, {mfargs, {couch_server_sup,couch_config_start_link_wrapper, [["/opt/couchbase/etc/couchdb/default.ini", "/opt/couchbase/etc/couchdb/default.d/capi.ini", "/opt/couchbase/etc/couchdb/default.d/geocouch.ini", "/opt/couchbase/etc/couchdb/local.ini"], <0.209.0>]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:11.852,nonode@nohost:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,couch_primary_services} started: [{pid,<0.212.0>}, {name,collation_driver}, {mfargs,{couch_drv,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:23:11.852,nonode@nohost:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,couch_primary_services} started: [{pid,<0.213.0>}, {name,couch_task_events}, {mfargs, {gen_event,start_link,[{local,couch_task_events}]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:11.853,nonode@nohost:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,couch_primary_services} started: [{pid,<0.214.0>}, {name,couch_task_status}, {mfargs,{couch_task_status,start_link,[]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:11.853,nonode@nohost:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,couch_primary_services} started: [{pid,<0.215.0>}, {name,couch_file_write_guard}, {mfargs,{couch_file_write_guard,sup_start_link,[]}}, {restart_type,permanent}, {shutdown,10000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:11.857,nonode@nohost:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,couch_primary_services} started: [{pid,<0.216.0>}, {name,couch_server}, {mfargs,{couch_server,sup_start_link,[]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:11.857,nonode@nohost:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,couch_primary_services} started: [{pid,<0.217.0>}, {name,couch_db_update_event}, {mfargs, {gen_event,start_link,[{local,couch_db_update}]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:11.857,nonode@nohost:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,couch_primary_services} started: [{pid,<0.218.0>}, {name,couch_replication_event}, {mfargs, {gen_event,start_link,[{local,couch_replication}]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:11.857,nonode@nohost:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,couch_primary_services} started: [{pid,<0.219.0>}, {name,couch_replication_supervisor}, {mfargs,{couch_rep_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:23:11.858,nonode@nohost:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,couch_primary_services} started: [{pid,<0.220.0>}, {name,couch_log}, {mfargs,{couch_log,start_link,[]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:11.859,nonode@nohost:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,couch_primary_services} started: [{pid,<0.221.0>}, {name,couch_main_index_barrier}, {mfargs, {couch_index_barrier,start_link, [couch_main_index_barrier, "max_parallel_indexers"]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:11.860,nonode@nohost:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,couch_primary_services} started: [{pid,<0.222.0>}, {name,couch_replica_index_barrier}, {mfargs, {couch_index_barrier,start_link, [couch_replica_index_barrier, "max_parallel_replica_indexers"]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:11.860,nonode@nohost:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,couch_server_sup} started: [{pid,<0.211.0>}, {name,couch_primary_services}, {mfargs,{couch_primary_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:23:11.862,nonode@nohost:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,couch_secondary_services} started: [{pid,<0.224.0>}, {name,couch_db_update_notifier_sup}, {mfargs,{couch_db_update_notifier_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:23:11.892,nonode@nohost:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,couch_secondary_services} started: [{pid,<0.225.0>}, {name,auth_cache}, {mfargs,{couch_auth_cache,start_link,[]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:11.897,nonode@nohost:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,couch_secondary_services} started: [{pid,<0.236.0>}, {name,set_view_manager}, {mfargs,{couch_set_view,start_link,[]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:11.899,nonode@nohost:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,couch_secondary_services} started: [{pid,<0.239.0>}, {name,spatial_manager}, {mfargs,{couch_spatial,start_link,[]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:11.899,nonode@nohost:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,couch_secondary_services} started: [{pid,<0.241.0>}, {name,index_merger_pool}, {mfargs, {lhttpc_manager,start_link, [[{connection_timeout,90000}, {pool_size,10000}, {name,couch_index_merger_connection_pool}]]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:11.902,nonode@nohost:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,couch_secondary_services} started: [{pid,<0.242.0>}, {name,query_servers}, {mfargs,{couch_query_servers,start_link,[]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:11.904,nonode@nohost:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,couch_secondary_services} started: [{pid,<0.244.0>}, {name,couch_set_view_ddoc_cache}, {mfargs,{couch_set_view_ddoc_cache,start_link,[]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:11.906,nonode@nohost:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,couch_secondary_services} started: [{pid,<0.246.0>}, {name,view_manager}, {mfargs,{couch_view,start_link,[]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:11.913,nonode@nohost:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,couch_secondary_services} started: [{pid,<0.248.0>}, {name,httpd}, {mfargs,{couch_httpd,start_link,[]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:11.913,nonode@nohost:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,couch_secondary_services} started: [{pid,<0.265.0>}, {name,uuids}, {mfargs,{couch_uuids,start,[]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:11.913,nonode@nohost:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,couch_server_sup} started: [{pid,<0.223.0>}, {name,couch_secondary_services}, {mfargs,{couch_secondary_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:23:11.913,nonode@nohost:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,cb_couch_sup} started: [{pid,<0.210.0>}, {name,couch_app}, {mfargs, {couch_app,start, [fake, ["/opt/couchbase/etc/couchdb/default.ini", "/opt/couchbase/etc/couchdb/local.ini"]]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:23:11.913,nonode@nohost:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_cluster_sup} started: [{pid,<0.168.0>}, {name,cb_couch_sup}, {mfargs,{cb_couch_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,supervisor}] [ns_server:info,2012-11-02T18:23:11.917,nonode@nohost:ns_server_cluster_sup<0.167.0>:log_os_info:start_link:25]OS type: {unix,linux} Version: {2,6,18} Runtime info: [{otp_release,"R14B04"}, {erl_version,"5.8.5"}, {erl_version_long, "Erlang R14B04 (erts-5.8.5) [source] [64-bit] [smp:128:128] [rq:128] [async-threads:0] [kernel-poll:true]\n"}, {system_arch_raw,"x86_64-unknown-linux-gnu"}, {system_arch,"x86_64-unknown-linux-gnu"}, {localtime,{{2012,11,2},{18,23,11}}}, {memory, [{total,32615528}, {processes,5992400}, {processes_used,5984112}, {system,26623128}, {atom,832377}, {atom_used,824386}, {binary,51968}, {code,7938149}, {ets,636376}]}, {loaded, [ns_info,log_os_info,couch_config_writer,cb_init_loggers, mochiweb_acceptor,inet_tcp,gen_tcp,mochiweb_socket, mochiweb_socket_server,mochilists,mochiweb_http,eval_bits, couch_httpd,couch_view,couch_set_view_ddoc_cache, couch_query_servers,couch_spatial,mapreduce, couch_set_view,snappy,couch_compress, couch_spatial_validation,couch_set_view_mapreduce,ejson, couch_doc,couch_db_update_notifier,couch_btree, couch_ref_counter,couch_uuids,couch_db_updater,couch_db, couch_auth_cache,couch_db_update_notifier_sup, couch_secondary_sup,couch_index_barrier,couch_event_sup, couch_log,couch_rep_sup,httpd_util,filelib,couch_file, couch_file_write_guard,couch_task_status,erl_ddll, couch_drv,couch_primary_sup,couch_server,string,re,file2, couch_util,couch_config,couch_server_sup,mochiweb_sup, mochiweb_app,ssl_server,crypto,ssl,lhttpc_manager, lhttpc_sup,lhttpc,ssl_connection_sup,ssl_session_cache, ssl_certificate_db,ssl_manager,ssl_broker_sup,ssl_sup, ssl_app,tftp_sup,httpd_sup,httpc_handler_sup,httpc_cookie, inets,httpc_manager,httpc,httpc_profile_sup,httpc_sup, ftp_sup,inets_sup,inets_app,crypto_server,crypto_sup, crypto_app,couch_app,cb_couch_sup,ns_server_cluster_sup, calendar,ale_default_formatter,otp_internal,misc, 'ale_logger-xdcr','ale_logger-mapreduce_errors', 'ale_logger-views','ale_logger-cluster', 'ale_logger-rebalance','ale_logger-stats', 'ale_logger-ns_doctor','ale_logger-menelaus', 'ale_logger-user','ale_logger-ns_server', 'ale_logger-couchdb',ns_log_sink,disk_log_sup, disk_log_server,disk_log_1,disk_log,ale_disk_sink, ns_server,timer,io_lib_fread,cpu_sup,memsup,disksup, os_mon,io,release_handler,overload,alarm_handler,log_mf_h, sasl,ale_error_logger_handler,'ale_logger-ale_logger', 'ale_logger-error_logger',beam_opcodes,beam_dict,beam_asm, beam_validator,beam_flatten,beam_trim,beam_receive, beam_bsm,beam_peep,beam_dead,beam_type,beam_bool, beam_clean,beam_utils,beam_jump,beam_block,v3_codegen, v3_life,v3_kernel,sys_core_dsetel,erl_bifs,sys_core_fold, cerl_trees,sys_core_inline,core_lib,cerl,v3_core,erl_bits, erl_expand_records,sys_pre_expand,sofs,erl_internal,sets, ordsets,erl_lint,compile,dynamic_compile,ale_utils, io_lib_pretty,io_lib_format,io_lib,ale_codegen,dict,ale, ale_dynamic_sup,ale_sup,ale_app,ns_bootstrap, file_io_server,orddict,erl_eval,file,c,queue, kernel_config,user,user_sup,supervisor_bridge, standard_error,unicode,binary,ets,gb_sets, hipe_unified_loader,packages,code_server,code,file_server, net_kernel,global_group,erl_distribution,filename, inet_gethost_native,os,inet_parse,inet,inet_udp, inet_config,inet_db,global,gb_trees,rpc,supervisor,kernel, application_master,sys,application,gen_server,erl_parse, proplists,erl_scan,lists,application_controller,proc_lib, gen,gen_event,error_logger,heart,error_handler,erlang, erl_prim_loader,prim_zip,zlib,prim_file,prim_inet,init, otp_ring0]}, {applications, [{public_key,"Public key infrastructure","0.13"}, {lhttpc,"Lightweight HTTP Client","1.3.0"}, {ale,"Another Logger for Erlang","8cffe61"}, {os_mon,"CPO CXC 138 46","2.2.7"}, {couch_set_view,"Set views","1.2.0a-47a991b-git"}, {inets,"INETS CXC 138 49","5.7.1"}, {couch,"Apache CouchDB","1.2.0a-47a991b-git"}, {mapreduce,"MapReduce using V8 JavaScript engine","1.0.0"}, {couch_index_merger,"Index merger","1.2.0a-47a991b-git"}, {kernel,"ERTS CXC 138 10","2.14.5"}, {crypto,"CRYPTO version 2","2.0.4"}, {ssl,"Erlang/OTP SSL application","4.1.6"}, {sasl,"SASL CXC 138 11","2.1.10"}, {couch_view_parser,"Couch view parser","1.0.0"}, {ns_server,"Couchbase server","2.0.0-1925-rel-enterprise"}, {mochiweb,"MochiMedia Web Server","1.4.1"}, {oauth,"Erlang OAuth implementation","7d85d3ef"}, {stdlib,"ERTS CXC 138 10","1.17.5"}]}, {pre_loaded, [erlang,erl_prim_loader,prim_zip,zlib,prim_file,prim_inet, init,otp_ring0]}, {process_count,151}, {node,nonode@nohost}, {nodes,[]}, {registered, ['sink-disk_stats',kernel_safe_sup,'sink-disk_xdcr_errors', 'sink-disk_xdcr','sink-disk_debug','sink-disk_couchdb', inets_sup,'sink-disk_mapreduce_errors',crypto_server, 'sink-disk_views',crypto_sup,'sink-disk_error', couch_secondary_services,mochiweb_sup, couch_primary_services,ale_sup,couch_db_update, ale_dynamic_sup,sasl_safe_sup,couch_query_servers, couch_httpd,couch_drv,couch_server,ale, couch_set_view_ddoc_cache, couch_index_merger_connection_pool,error_logger, couch_spatial,lhttpc_manager,standard_error, couch_replica_index_barrier,standard_error_sup, couch_main_index_barrier,tftp_sup,sasl_sup, couch_replication,couch_auth_cache,couch_task_events, timer_server,couch_task_status,erl_prim_loader, release_handler,inet_db,overload,rex,kernel_sup, alarm_handler,global_name_server,couch_view,file_server_2, couch_rep_sup,global_group,'sink-disk_default', ssl_connection_sup,ns_server_cluster_sup,couch_config, httpd_sup,ssl_manager,init,couch_server_sup, couch_file_write_guard,couch_uuids, couch_db_update_notifier_sup,disk_log_sup,disk_log_server, user,ssl_broker_sup,code_server,ssl_server,ssl_sup, couch_log,application_controller,cb_couch_sup,httpc_sup, httpc_profile_sup,os_mon_sup,httpc_manager, httpc_handler_sup,lhttpc_sup,cpu_sup,ftp_sup,memsup, 'sink-ns_log',disksup,couch_set_view]}, {cookie,nocookie}, {wordsize,8}, {wall_clock,0}] [ns_server:info,2012-11-02T18:23:11.919,nonode@nohost:ns_server_cluster_sup<0.167.0>:log_os_info:start_link:27]Manifest: ["","", " ", " ", " ", " ", " ", " ", " "," ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ",""] [error_logger:info,2012-11-02T18:23:11.919,nonode@nohost:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_cluster_sup} started: [{pid,<0.267.0>}, {name,timeout_diag_logger}, {mfargs,{timeout_diag_logger,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [ns_server:info,2012-11-02T18:23:11.920,nonode@nohost:dist_manager<0.268.0>:dist_manager:read_address_config:55]Reading ip config from "/opt/couchbase/var/lib/couchbase/ip" [ns_server:info,2012-11-02T18:23:11.920,nonode@nohost:dist_manager<0.268.0>:dist_manager:init:125]ip config not found. Looks like we're brand new node [error_logger:info,2012-11-02T18:23:11.922,nonode@nohost:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,inet_gethost_native_sup} started: [{pid,<0.270.0>},{mfa,{inet_gethost_native,init,[[]]}}] [error_logger:info,2012-11-02T18:23:11.922,nonode@nohost:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,kernel_safe_sup} started: [{pid,<0.269.0>}, {name,inet_gethost_native_sup}, {mfargs,{inet_gethost_native,start_link,[]}}, {restart_type,temporary}, {shutdown,1000}, {child_type,worker}] [ns_server:info,2012-11-02T18:23:11.924,nonode@nohost:dist_manager<0.268.0>:dist_manager:bringup:193]Attempting to bring up net_kernel with name 'ns_1@127.0.0.1' [error_logger:info,2012-11-02T18:23:11.925,nonode@nohost:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,net_sup} started: [{pid,<0.272.0>}, {name,erl_epmd}, {mfargs,{erl_epmd,start_link,[]}}, {restart_type,permanent}, {shutdown,2000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:11.925,nonode@nohost:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,net_sup} started: [{pid,<0.273.0>}, {name,auth}, {mfargs,{auth,start_link,[]}}, {restart_type,permanent}, {shutdown,2000}, {child_type,worker}] [ns_server:info,2012-11-02T18:23:11.926,ns_1@127.0.0.1:dist_manager<0.268.0>:dist_manager:save_node:112]saving node to "/opt/couchbase/var/lib/couchbase/couchbase-server.node" [error_logger:info,2012-11-02T18:23:11.926,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,net_sup} started: [{pid,<0.274.0>}, {name,net_kernel}, {mfargs, {net_kernel,start_link, [['ns_1@127.0.0.1',longnames]]}}, {restart_type,permanent}, {shutdown,2000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:11.926,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,kernel_sup} started: [{pid,<0.271.0>}, {name,net_sup_dynamic}, {mfargs, {erl_distribution,start_link, [['ns_1@127.0.0.1',longnames]]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:23:11.928,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_cluster_sup} started: [{pid,<0.268.0>}, {name,dist_manager}, {mfargs,{dist_manager,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:11.929,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_cluster_sup} started: [{pid,<0.277.0>}, {name,ns_cookie_manager}, {mfargs,{ns_cookie_manager,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:11.930,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_cluster_sup} started: [{pid,<0.278.0>}, {name,ns_cluster}, {mfargs,{ns_cluster,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:11.930,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mb_mnesia_sup} started: [{pid,<0.280.0>}, {name,mb_mnesia_events}, {mfargs, {gen_event,start_link,[{local,mb_mnesia_events}]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:11.954,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mnesia_sup} started: [{pid,<0.287.0>}, {name,mnesia_event}, {mfargs,{mnesia_sup,start_event,[]}}, {restart_type,permanent}, {shutdown,30000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:11.955,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mnesia_kernel_sup} started: [{pid,<0.289.0>}, {name,mnesia_monitor}, {mfargs,{mnesia_monitor,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:11.955,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mnesia_kernel_sup} started: [{pid,<0.290.0>}, {name,mnesia_subscr}, {mfargs,{mnesia_subscr,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:11.958,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mnesia_kernel_sup} started: [{pid,<0.291.0>}, {name,mnesia_locker}, {mfargs,{mnesia_locker,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:11.960,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mnesia_kernel_sup} started: [{pid,<0.292.0>}, {name,mnesia_recover}, {mfargs,{mnesia_recover,start,[]}}, {restart_type,permanent}, {shutdown,180000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:11.965,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mnesia_kernel_sup} started: [{pid,<0.293.0>}, {name,mnesia_tm}, {mfargs,{mnesia_tm,start,[]}}, {restart_type,permanent}, {shutdown,30000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:11.966,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mnesia_kernel_sup} started: [{pid,<0.294.0>}, {name,mnesia_checkpoint_sup}, {mfargs,{mnesia_checkpoint_sup,start,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:23:11.967,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mnesia_kernel_sup} started: [{pid,<0.295.0>}, {name,mnesia_snmp_sup}, {mfargs,{mnesia_snmp_sup,start,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:23:11.969,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mnesia_kernel_sup} started: [{pid,<0.296.0>}, {name,mnesia_controller}, {mfargs,{mnesia_controller,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:11.973,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mnesia_kernel_sup} started: [{pid,<0.297.0>}, {name,mnesia_late_loader}, {mfargs,{mnesia_late_loader,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:11.973,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mnesia_sup} started: [{pid,<0.288.0>}, {name,mnesia_kernel_sup}, {mfargs,{mnesia_kernel_sup,start,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:23:11.973,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= application: mnesia started_at: 'ns_1@127.0.0.1' [error_logger:info,2012-11-02T18:23:11.986,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,kernel_safe_sup} started: [{pid,<0.303.0>}, {name,dets_sup}, {mfargs,{dets_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:23:11.986,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,kernel_safe_sup} started: [{pid,<0.304.0>}, {name,dets}, {mfargs,{dets_server,start_link,[]}}, {restart_type,permanent}, {shutdown,2000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:12.014,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mb_mnesia_sup} started: [{pid,<0.281.0>}, {name,mb_mnesia}, {mfargs,{mb_mnesia,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:info,2012-11-02T18:23:12.014,ns_1@127.0.0.1:ns_config_sup<0.329.0>:ns_config_sup:init:32]loading static ns_config from "/opt/couchbase/etc/couchbase/config" [error_logger:info,2012-11-02T18:23:12.014,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_cluster_sup} started: [{pid,<0.279.0>}, {name,mb_mnesia_sup}, {mfargs,{mb_mnesia_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:23:12.019,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_config_sup} started: [{pid,<0.330.0>}, {name,ns_config_events}, {mfargs, {gen_event,start_link,[{local,ns_config_events}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:12.019,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_config_sup} started: [{pid,<0.331.0>}, {name,ns_config_events_local}, {mfargs, {gen_event,start_link, [{local,ns_config_events_local}]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [ns_server:info,2012-11-02T18:23:12.020,ns_1@127.0.0.1:ns_config<0.332.0>:ns_config:load_config:674]Loading static config from "/opt/couchbase/etc/couchbase/config" [ns_server:info,2012-11-02T18:23:12.021,ns_1@127.0.0.1:ns_config<0.332.0>:ns_config:load_config:688]Loading dynamic config from "/opt/couchbase/var/lib/couchbase/config/config.dat" [ns_server:info,2012-11-02T18:23:12.021,ns_1@127.0.0.1:ns_config<0.332.0>:ns_config:load_config:692]No dynamic config file found. Assuming we're brand new node [ns_server:info,2012-11-02T18:23:12.021,ns_1@127.0.0.1:ns_config<0.332.0>:ns_config:load_config:706]Here's full dynamic config we loaded + static & default config: [{auto_failover_cfg,[{enabled,false},{timeout,30},{max_nodes,1},{count,0}]}, {replication,[{enabled,true}]}, {email_alerts, [{recipients,["root@localhost"]}, {sender,"couchbase@localhost"}, {enabled,false}, {email_server, [{user,[]},{pass,[]},{host,"localhost"},{port,25},{encrypt,false}]}, {alerts, [auto_failover_node,auto_failover_maximum_reached, auto_failover_other_nodes_down,auto_failover_cluster_too_small,ip, disk,overhead,ep_oom_errors,ep_item_commit_failed]}]}, {{node,'ns_1@127.0.0.1',ns_log}, [{filename,"/opt/couchbase/var/lib/couchbase/data/ns_log"}]}, {{node,'ns_1@127.0.0.1',port_servers}, [{moxi,"/opt/couchbase/bin/moxi", ["-Z", {"port_listen=~B,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", [port]}, "-z", {"url=http://127.0.0.1:~B/pools/default/saslBucketsStreaming", [{misc,this_node_rest_port,[]}]}, "-p","0","-Y","y","-O","stderr", {"~s",[verbosity]}], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR",{"~s",[{ns_moxi_sup,rest_user,[]}]}}, {"MOXI_SASL_PLAIN_PWD",{"~s",[{ns_moxi_sup,rest_pass,[]}]}}]}, use_stdio,exit_status,port_server_send_eol,stderr_to_stdout,stream]}, {memcached,"/opt/couchbase/bin/memcached", ["-X","/opt/couchbase/lib/memcached/stdin_term_handler.so","-X", {"/opt/couchbase/lib/memcached/file_logger.so,cyclesize=~B;sleeptime=~B;filename=~s/~s", [log_cyclesize,log_sleeptime,log_path,log_prefix]}, "-l", {"0.0.0.0:~B,0.0.0.0:~B:1000",[port,dedicated_port]}, "-p", {"~B",[port]}, "-E","/opt/couchbase/lib/memcached/bucket_engine.so","-B","binary", "-r","-c","10000","-e", {"admin=~s;default_bucket_name=default;auto_create=false", [admin_user]}, {"~s",[verbosity]}], [{env, [{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE",{"~s",[{isasl,path}]}}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,exit_status,port_server_send_eol,stream]}]}, {{node,'ns_1@127.0.0.1',moxi},[{port,11211},{verbosity,[]}]}, {buckets,[{configs,[]}]}, {memory_quota,31074}, {{node,'ns_1@127.0.0.1',memcached}, [{'_vclock',[{'_',{1,0}}]}, {port,11210}, {mccouch_port,11213}, {dedicated_port,11209}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine,"/opt/couchbase/lib/memcached/bucket_engine.so"}, {engines, [{membase, [{engine,"/opt/couchbase/lib/memcached/ep.so"}, {static_config_string, "vb0=false;waitforwarmup=false;failpartialwarmup=false"}]}, {memcached, [{engine,"/opt/couchbase/lib/memcached/default_engine.so"}, {static_config_string,"vb0=true"}]}]}, {log_path,"/opt/couchbase/var/lib/couchbase/logs"}, {log_prefix,"memcached.log"}, {log_generations,10}, {log_cyclesize,104857600}, {log_sleeptime,19}, {log_rotation_period,39003}, {verbosity,[]}]}, {{node,'ns_1@127.0.0.1',isasl}, [{path,"/opt/couchbase/var/lib/couchbase/data/isasl.pw"}]}, {remote_clusters,[]}, {rest_creds,[{creds,[]}]}, {{node,'ns_1@127.0.0.1',capi_port},8092}, {{node,'ns_1@127.0.0.1',rest},[{port,8091},{port_meta,global}]}, {{couchdb,max_parallel_replica_indexers},2}, {{couchdb,max_parallel_indexers},4}, {rest,[{port,8091}]}, {{node,'ns_1@127.0.0.1',membership},active}, {nodes_wanted,['ns_1@127.0.0.1']}, {{node,'ns_1@127.0.0.1',compaction_daemon}, [{check_interval,30},{min_file_size,131072}]}, {fast_warmup, [{fast_warmup_enabled,true}, {min_memory_threshold,10}, {min_items_threshold,10}]}, {set_view_update_daemon, [{update_interval,5000}, {update_min_changes,5000}, {replica_update_min_changes,5000}]}, {autocompaction, [{database_fragmentation_threshold,{30,undefined}}, {view_fragmentation_threshold,{30,undefined}}]}, {max_bucket_count,10}, {index_aware_rebalance_disabled,false}, {xdcr_capi_checkpoint_timeout,10}, {xdcr_failure_restart_interval,30}, {xdcr_doc_batch_size_kb,512}, {xdcr_checkpoint_interval,1800}] [ns_server:info,2012-11-02T18:23:12.022,ns_1@127.0.0.1:ns_config<0.332.0>:ns_config_default:upgrade_config_from_1_7_to_1_7_1:296]Upgrading config from 1.7 to 1.7.1 [ns_server:info,2012-11-02T18:23:12.023,ns_1@127.0.0.1:ns_config<0.332.0>:ns_config_default:upgrade_config_from_1_7_1_to_1_7_2:307]Upgrading config from 1.7.1 to 1.7.2 [ns_server:info,2012-11-02T18:23:12.024,ns_1@127.0.0.1:ns_config<0.332.0>:ns_config_default:upgrade_config_from_1_7_2_to_1_8_0:361]Upgrading config from 1.7.2 to 1.8.0 [ns_server:info,2012-11-02T18:23:12.025,ns_1@127.0.0.1:ns_config<0.332.0>:ns_config_default:upgrade_config_from_1_8_0_to_1_8_1:398]Upgrading config from 1.8.0 to 1.8.1 [ns_server:info,2012-11-02T18:23:12.026,ns_1@127.0.0.1:ns_config<0.332.0>:ns_config_default:upgrade_config_from_1_8_1_to_2_0:427]Upgrading config from 1.8.1 to 2.0 [error_logger:info,2012-11-02T18:23:12.029,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_config_sup} started: [{pid,<0.332.0>}, {name,ns_config}, {mfargs, {ns_config,start_link, ["/opt/couchbase/etc/couchbase/config", ns_config_default]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:12.037,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_config_sup} started: [{pid,<0.335.0>}, {name,ns_config_remote}, {mfargs, {ns_config_replica,start_link, [{local,ns_config_remote}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [ns_server:warn,2012-11-02T18:23:12.041,ns_1@127.0.0.1:ns_config_isasl_sync<0.336.0>:ns_memcached:connect:1103]Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:info,2012-11-02T18:23:13.042,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_config_sup} started: [{pid,<0.336.0>}, {name,ns_config_isasl_sync}, {mfargs,{ns_config_isasl_sync,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:13.042,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_config_sup} started: [{pid,<0.339.0>}, {name,ns_config_log}, {mfargs,{ns_config_log,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:13.044,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_config_sup} started: [{pid,<0.341.0>}, {name,cb_config_couch_sync}, {mfargs,{cb_config_couch_sync,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:13.044,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_cluster_sup} started: [{pid,<0.329.0>}, {name,ns_config_sup}, {mfargs,{ns_config_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:23:13.044,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_cluster_sup} started: [{pid,<0.343.0>}, {name,vbucket_filter_changes_registry}, {mfargs, {ns_process_registry,start_link, [vbucket_filter_changes_registry]}}, {restart_type,permanent}, {shutdown,100}, {child_type,worker}] [ns_server:warn,2012-11-02T18:23:13.046,ns_1@127.0.0.1:ns_log<0.345.0>:ns_log:read_logs:69]Couldn't load logs from "/opt/couchbase/var/lib/couchbase/data/ns_log" (perhaps it's first startup): {error, enoent} [error_logger:info,2012-11-02T18:23:13.047,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.345.0>}, {name,ns_log}, {mfargs,{ns_log,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:13.048,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.346.0>}, {name,ns_config_ets_dup}, {mfargs,{ns_config_ets_dup,start_link,[]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:13.048,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.347.0>}, {name,ns_log_events}, {mfargs,{gen_event,start_link,[{local,ns_log_events}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:13.048,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_node_disco_sup} started: [{pid,<0.350.0>}, {name,ns_node_disco_events}, {mfargs, {gen_event,start_link, [{local,ns_node_disco_events}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [user:info,2012-11-02T18:23:13.050,ns_1@127.0.0.1:ns_cookie_manager<0.277.0>:ns_cookie_manager:do_cookie_init:91]Initial otp cookie generated: phwwpfkaopxukihc [error_logger:info,2012-11-02T18:23:13.055,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_node_disco_sup} started: [{pid,<0.351.0>}, {name,ns_node_disco}, {mfargs,{ns_node_disco,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:13.056,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_node_disco_sup} started: [{pid,<0.355.0>}, {name,ns_node_disco_log}, {mfargs,{ns_node_disco_log,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:13.056,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_node_disco_sup} started: [{pid,<0.356.0>}, {name,ns_node_disco_conf_events}, {mfargs,{ns_node_disco_conf_events,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:13.057,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_node_disco_sup} started: [{pid,<0.357.0>}, {name,ns_config_rep_merger}, {mfargs,{ns_config_rep,start_link_merger,[]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:13.059,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_node_disco_sup} started: [{pid,<0.358.0>}, {name,ns_config_rep}, {mfargs,{ns_config_rep,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:13.060,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.349.0>}, {name,ns_node_disco_sup}, {mfargs,{ns_node_disco_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:23:13.061,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.362.0>}, {name,vbucket_map_mirror}, {mfargs,{vbucket_map_mirror,start_link,[]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:13.067,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.365.0>}, {name,ns_tick_event}, {mfargs,{gen_event,start_link,[{local,ns_tick_event}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:13.070,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.366.0>}, {name,mb_master_events}, {mfargs, {gen_event,start_link,[{local,mb_master_events}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [ns_server:info,2012-11-02T18:23:13.072,ns_1@127.0.0.1:ns_config_log<0.339.0>:ns_config_log:handle_info:57]config change: rest_creds -> ******** [error_logger:info,2012-11-02T18:23:13.072,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.367.0>}, {name,buckets_events}, {mfargs, {gen_event,start_link,[{local,buckets_events}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:13.075,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_mail_sup} started: [{pid,<0.369.0>}, {name,ns_mail_log}, {mfargs,{ns_mail_log,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:13.076,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.368.0>}, {name,ns_mail_sup}, {mfargs,{ns_mail_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:23:13.076,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.370.0>}, {name,ns_stats_event}, {mfargs, {gen_event,start_link,[{local,ns_stats_event}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:13.077,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.372.0>}, {name,ns_heart}, {mfargs,{ns_heart,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:13.077,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.374.0>}, {name,ns_doctor}, {mfargs,{ns_doctor,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:13.079,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.377.0>}, {name,remote_clusters_info}, {mfargs,{remote_clusters_info,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [ns_server:error,2012-11-02T18:23:13.081,ns_1@127.0.0.1:ns_heart<0.372.0>:ns_heart:grab_samples_loading_tasks:298]Failed to grab samples loader tasks: {exit, {noproc, {gen_server,call, [samples_loader_tasks,get_tasks, 2000]}}, [{gen_server,call,3}, {ns_heart,grab_samples_loading_tasks,0}, {ns_heart,current_status,0}, {ns_heart,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} [user:info,2012-11-02T18:23:13.082,ns_1@127.0.0.1:mb_master<0.380.0>:mb_master:init:89]I'm the only node, so I'm the master. [ns_server:info,2012-11-02T18:23:13.090,ns_1@127.0.0.1:ns_config<0.332.0>:ns_online_config_upgrader:upgrade_config_on_join_from_pre_2_0_to_2_0:65]Adding some 2.0 specific keys to the config [ns_server:error,2012-11-02T18:23:13.090,ns_1@127.0.0.1:ns_heart<0.372.0>:ns_heart:grab_samples_loading_tasks:298]Failed to grab samples loader tasks: {exit, {noproc, {gen_server,call, [samples_loader_tasks,get_tasks, 2000]}}, [{gen_server,call,3}, {ns_heart,grab_samples_loading_tasks,0}, {ns_heart,current_status,0}, {ns_heart,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} [user:warn,2012-11-02T18:23:13.094,ns_1@127.0.0.1:<0.383.0>:ns_orchestrator:consider_switching_compat_mode:665]Changed cluster compat mode from undefined to [2,0] [ns_server:info,2012-11-02T18:23:13.094,ns_1@127.0.0.1:ns_config<0.332.0>:ns_online_config_upgrader:upgrade_config_from_pre_2_0_to_2_0:69]Performing online config upgrade to 2.0 version [error_logger:info,2012-11-02T18:23:13.095,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mb_master_sup} started: [{pid,<0.383.0>}, {name,ns_orchestrator}, {mfargs,{ns_orchestrator,start_link,[]}}, {restart_type,permanent}, {shutdown,20}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:13.100,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mb_master_sup} started: [{pid,<0.395.0>}, {name,ns_tick}, {mfargs,{ns_tick,start_link,[]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:13.100,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mb_master_sup} started: [{pid,<0.397.0>}, {name,auto_failover}, {mfargs,{auto_failover,start_link,[]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:13.101,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.380.0>}, {name,mb_master}, {mfargs,{mb_master,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:23:13.101,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.398.0>}, {name,master_activity_events}, {mfargs, {gen_event,start_link, [{local,master_activity_events}]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:13.101,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.399.0>}, {name,master_activity_events_ingress}, {mfargs, {gen_event,start_link, [{local,master_activity_events_ingress}]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:13.102,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.400.0>}, {name,master_activity_events_timestamper}, {mfargs, {master_activity_events,start_link_timestamper,[]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:13.102,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.401.0>}, {name,master_activity_events_pids_watcher}, {mfargs, {master_activity_events_pids_watcher,start_link, []}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:13.103,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.402.0>}, {name,master_activity_events_keeper}, {mfargs,{master_activity_events_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:13.113,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.405.0>}, {name,menelaus_web}, {mfargs,{menelaus_web,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:13.114,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.422.0>}, {name,menelaus_event}, {mfargs,{menelaus_event,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:13.115,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.423.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:13.116,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.424.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [user:info,2012-11-02T18:23:13.117,ns_1@127.0.0.1:ns_server_sup<0.344.0>:menelaus_sup:start_link:44]Couchbase Server has started on web port 8091 on node 'ns_1@127.0.0.1'. [error_logger:info,2012-11-02T18:23:13.117,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.404.0>}, {name,menelaus}, {mfargs,{menelaus_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:23:13.118,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mc_sup} started: [{pid,<0.426.0>}, {name,mc_couch_events}, {mfargs, {gen_event,start_link,[{local,mc_couch_events}]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:13.118,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mc_sup} started: [{pid,<0.427.0>}, {name,mc_conn_sup}, {mfargs,{mc_conn_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,supervisor}] [ns_server:info,2012-11-02T18:23:13.119,ns_1@127.0.0.1:<0.428.0>:mc_tcp_listener:init:24]mccouch is listening on port 11213 [error_logger:info,2012-11-02T18:23:13.119,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mc_sup} started: [{pid,<0.428.0>}, {name,mc_tcp_listener}, {mfargs,{mc_tcp_listener,start_link,[11213]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:13.120,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.425.0>}, {name,mc_sup}, {mfargs,{mc_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:23:13.123,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_port_sup} started: [{pid,<0.430.0>}, {name,ns_port_init}, {mfargs,{ns_port_init,start_link,[]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:13.126,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_port_sup} started: [{pid,<0.431.0>}, {name, {moxi,"/opt/couchbase/bin/moxi", ["-Z", "port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", "-z", "url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming", "-p","0","-Y","y","-O","stderr",[]], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR",[]}, {"MOXI_SASL_PLAIN_PWD",[]}]}, use_stdio,exit_status,port_server_send_eol, stderr_to_stdout,stream]}}, {mfargs, {supervisor_cushion,start_link, [moxi,5000,10000,ns_port_server,start_link, [moxi,"/opt/couchbase/bin/moxi", ["-Z", "port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", "-z", "url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming", "-p","0","-Y","y","-O","stderr",[]], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR",[]}, {"MOXI_SASL_PLAIN_PWD",[]}]}, use_stdio,exit_status,port_server_send_eol, stderr_to_stdout,stream]]]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:13.128,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_port_sup} started: [{pid,<0.433.0>}, {name, {memcached,"/opt/couchbase/bin/memcached", ["-X", "/opt/couchbase/lib/memcached/stdin_term_handler.so", "-X", "/opt/couchbase/lib/memcached/file_logger.so,cyclesize=104857600;sleeptime=19;filename=/opt/couchbase/var/lib/couchbase/logs/memcached.log", "-l","0.0.0.0:11210,0.0.0.0:11209:1000","-p", "11210","-E", "/opt/couchbase/lib/memcached/bucket_engine.so", "-B","binary","-r","-c","10000","-e", "admin=_admin;default_bucket_name=default;auto_create=false", []], [{env, [{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE", "/opt/couchbase/var/lib/couchbase/data/isasl.pw"}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,exit_status, port_server_send_eol,stream]}}, {mfargs, {erlang,apply, [#Fun, [memcached,"/opt/couchbase/bin/memcached", ["-X", "/opt/couchbase/lib/memcached/stdin_term_handler.so", "-X", "/opt/couchbase/lib/memcached/file_logger.so,cyclesize=104857600;sleeptime=19;filename=/opt/couchbase/var/lib/couchbase/logs/memcached.log", "-l","0.0.0.0:11210,0.0.0.0:11209:1000", "-p","11210","-E", "/opt/couchbase/lib/memcached/bucket_engine.so", "-B","binary","-r","-c","10000","-e", "admin=_admin;default_bucket_name=default;auto_create=false", []], [{env, [{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE", "/opt/couchbase/var/lib/couchbase/data/isasl.pw"}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,exit_status, port_server_send_eol,stream]]]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:13.129,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.429.0>}, {name,ns_port_sup}, {mfargs,{ns_port_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:23:13.129,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.435.0>}, {name,ns_port_memcached_killer}, {mfargs,{ns_port_sup,start_memcached_force_killer,[]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [ns_server:info,2012-11-02T18:23:13.130,ns_1@127.0.0.1:<0.436.0>:ns_memcached_log_rotator:init:28]Starting log rotator on "/opt/couchbase/var/lib/couchbase/logs"/"memcached.log"* with an initial period of 39003ms [error_logger:info,2012-11-02T18:23:13.130,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.436.0>}, {name,ns_memcached_log_rotator}, {mfargs,{ns_memcached_log_rotator,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:13.131,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.437.0>}, {name,ns_bucket_worker}, {mfargs,{work_queue,start_link,[ns_bucket_worker]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:13.157,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.439.0>}, {name,xdc_replication_sup}, {mfargs,{xdc_replication_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:23:13.176,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.441.0>}, {name,xdc_rep_manager}, {mfargs,{xdc_rep_manager,start_link,[]}}, {restart_type,permanent}, {shutdown,30000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:13.177,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_bucket_sup} started: [{pid,<0.451.0>}, {name,buckets_observing_subscription}, {mfargs,{ns_bucket_sup,subscribe_on_config_events,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:13.178,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.450.0>}, {name,ns_bucket_sup}, {mfargs,{ns_bucket_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:23:13.185,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.453.0>}, {name,system_stats_collector}, {mfargs,{system_stats_collector,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:13.186,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.455.0>}, {name,{stats_archiver,"@system"}}, {mfargs,{stats_archiver,start_link,["@system"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:13.186,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.457.0>}, {name,{stats_reader,"@system"}}, {mfargs,{stats_reader,start_link,["@system"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:13.186,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.458.0>}, {name,ns_moxi_sup_work_queue}, {mfargs, {work_queue,start_link,[ns_moxi_sup_work_queue]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:13.186,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.459.0>}, {name,ns_moxi_sup}, {mfargs,{ns_moxi_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:23:13.187,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.463.0>}, {name,compaction_daemon}, {mfargs, {supervisor_cushion,start_link, [compaction_daemon,3000,1000,compaction_daemon, start_link,[]]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:13.189,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.470.0>}, {name,xdc_rdoc_replication_srv}, {mfargs,{xdc_rdoc_replication_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [ns_server:info,2012-11-02T18:23:13.193,ns_1@127.0.0.1:set_view_update_daemon<0.472.0>:set_view_update_daemon:init:50]Set view update daemon, starting with the following settings: update interval: 5000ms minimum number of changes: 5000 [error_logger:info,2012-11-02T18:23:13.193,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.472.0>}, {name,set_view_update_daemon}, {mfargs,{set_view_update_daemon,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:13.194,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.475.0>}, {name,samples_loader_tasks}, {mfargs,{samples_loader_tasks,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:13.194,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_cluster_sup} started: [{pid,<0.344.0>}, {name,ns_server_sup}, {mfargs,{ns_server_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:23:13.194,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= application: ns_server started_at: 'ns_1@127.0.0.1' [ns_server:info,2012-11-02T18:23:18.012,ns_1@127.0.0.1:ns_config_events<0.330.0>:ns_port_sup:terminate_port:129]unsupervising port: {moxi,"/opt/couchbase/bin/moxi", ["-Z", "port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", "-z", "url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming", "-p","0","-Y","y","-O","stderr",[]], [{env,[{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR",[]}, {"MOXI_SASL_PLAIN_PWD",[]}]}, use_stdio,exit_status,port_server_send_eol, stderr_to_stdout,stream]} [ns_server:info,2012-11-02T18:23:18.013,ns_1@127.0.0.1:<0.432.0>:ns_port_server:handle_info:104]Port server moxi exited with status 0 [ns_server:info,2012-11-02T18:23:18.013,ns_1@127.0.0.1:<0.432.0>:ns_port_server:log:171]moxi<0.432.0>: EOL on stdin. Exiting [ns_server:info,2012-11-02T18:23:18.013,ns_1@127.0.0.1:ns_config_events<0.330.0>:ns_port_sup:launch_port:74]supervising port: {moxi,"/opt/couchbase/bin/moxi", ["-Z", "port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", "-z", "url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming", "-p","0","-Y","y","-O","stderr",[]], [{env,[{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR","Administrator"}, {"MOXI_SASL_PLAIN_PWD","password"}]}, use_stdio,exit_status,port_server_send_eol, stderr_to_stdout,stream]} [error_logger:info,2012-11-02T18:23:18.015,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_port_sup} started: [{pid,<0.522.0>}, {name, {moxi,"/opt/couchbase/bin/moxi", ["-Z", "port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", "-z", "url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming", "-p","0","-Y","y","-O","stderr",[]], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR","Administrator"}, {"MOXI_SASL_PLAIN_PWD","password"}]}, use_stdio,exit_status,port_server_send_eol, stderr_to_stdout,stream]}}, {mfargs, {supervisor_cushion,start_link, [moxi,5000,10000,ns_port_server,start_link, [moxi,"/opt/couchbase/bin/moxi", ["-Z", "port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", "-z", "url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming", "-p","0","-Y","y","-O","stderr",[]], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR","Administrator"}, {"MOXI_SASL_PLAIN_PWD","password"}]}, use_stdio,exit_status,port_server_send_eol, stderr_to_stdout,stream]]]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [ns_server:info,2012-11-02T18:23:18.016,ns_1@127.0.0.1:ns_config_log<0.339.0>:ns_config_log:handle_info:57]config change: rest_creds -> ******** [ns_server:info,2012-11-02T18:23:18.225,ns_1@127.0.0.1:<0.523.0>:ns_port_server:log:171]moxi<0.523.0>: 2012-11-02 18:23:18: (cproxy_config.c.317) env: MOXI_SASL_PLAIN_USR (13) moxi<0.523.0>: 2012-11-02 18:23:18: (cproxy_config.c.326) env: MOXI_SASL_PLAIN_PWD (8) [user:info,2012-11-02T18:23:22.487,ns_1@127.0.0.1:<0.418.0>:ns_storage_conf:setup_disk_storage_conf:148]Setting database directory path to /data and index directory path to /data2 [ns_server:info,2012-11-02T18:23:22.488,ns_1@127.0.0.1:<0.418.0>:ns_storage_conf:setup_disk_storage_conf:156]Removing all the buckets because database path has changed [ns_server:info,2012-11-02T18:23:22.489,ns_1@127.0.0.1:<0.418.0>:ns_storage_conf:setup_disk_storage_conf:161]Removing all unused database files [error_logger:error,2012-11-02T18:23:22.498,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,ns_bucket_sup} Context: shutdown_error Reason: normal Offender: [{pid,<0.451.0>}, {name,buckets_observing_subscription}, {mfargs,{ns_bucket_sup,subscribe_on_config_events,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [ns_server:info,2012-11-02T18:23:22.499,ns_1@127.0.0.1:<0.523.0>:ns_port_server:handle_info:104]Port server moxi exited with status 0 [ns_server:info,2012-11-02T18:23:22.499,ns_1@127.0.0.1:<0.523.0>:ns_port_server:log:171]moxi<0.523.0>: EOL on stdin. Exiting [ns_server:info,2012-11-02T18:23:22.699,ns_1@127.0.0.1:ns_port_memcached<0.434.0>:ns_port_server:log:171]memcached<0.434.0>: EOL on stdin. Initiating shutdown [ns_server:error,2012-11-02T18:23:23.067,ns_1@127.0.0.1:ns_heart<0.372.0>:ns_heart:grab_samples_loading_tasks:298]Failed to grab samples loader tasks: {exit, {noproc, {gen_server,call, [samples_loader_tasks,get_tasks, 2000]}}, [{gen_server,call,3}, {ns_heart,grab_samples_loading_tasks,0}, {ns_heart,current_status,0}, {ns_heart,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} [ns_server:info,2012-11-02T18:23:23.196,ns_1@127.0.0.1:ns_port_memcached<0.434.0>:ns_port_server:handle_info:104]Port server memcached exited with status 0 [ns_server:info,2012-11-02T18:23:23.197,ns_1@127.0.0.1:mb_master<0.380.0>:mb_master:terminate:288]Synchronously shutting down child mb_master_sup [error_logger:error,2012-11-02T18:23:23.300,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,ns_server_cluster_sup} Context: shutdown_error Reason: killed Offender: [{pid,<0.343.0>}, {name,vbucket_filter_changes_registry}, {mfargs, {ns_process_registry,start_link, [vbucket_filter_changes_registry]}}, {restart_type,permanent}, {shutdown,100}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:23.309,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================INFO REPORT========================= application: mnesia exited: stopped type: temporary [ns_server:info,2012-11-02T18:23:23.309,ns_1@127.0.0.1:mb_mnesia<0.281.0>:mb_mnesia:terminate:277]Shut Mnesia down: shutdown. Exiting. [error_logger:error,2012-11-02T18:23:23.317,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: couch_file:spawn_writer/2 pid: <0.445.0> registered_name: [] exception exit: {noproc, {gen_server,call, [couch_file_write_guard, {remove,<0.445.0>}, infinity]}} in function gen_server:call/3 in call from couch_file:writer_loop/4 ancestors: [<0.442.0>,couch_server,couch_primary_services, couch_server_sup,cb_couch_sup,ns_server_cluster_sup, <0.59.0>] messages: [] links: [] dictionary: [] trap_exit: true status: running heap_size: 377 stack_size: 24 reductions: 1837 neighbours: [error_logger:error,2012-11-02T18:23:23.318,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: couch_file:spawn_writer/2 pid: <0.229.0> registered_name: [] exception exit: {noproc, {gen_server,call, [couch_file_write_guard, {remove,<0.229.0>}, infinity]}} in function gen_server:call/3 in call from couch_file:writer_loop/4 ancestors: [<0.226.0>,couch_server,couch_primary_services, couch_server_sup,cb_couch_sup,ns_server_cluster_sup, <0.59.0>] messages: [] links: [] dictionary: [] trap_exit: true status: running heap_size: 233 stack_size: 24 reductions: 2140 neighbours: [error_logger:error,2012-11-02T18:23:23.318,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_msg:76]** Generic server <0.446.0> terminating ** Last message in was {'EXIT',<0.216.0>,killed} ** When Server state == {db,<0.446.0>,<0.447.0>,nil,<<"1351905793165775">>, <0.442.0>,<0.448.0>, {db_header,10,1, <<0,0,0,0,11,84,0,0,0,0,0,60,0,0,0,0,1,0,0,0,0, 0,0,0,0,0,11,50>>, <<0,0,0,0,11,144,0,0,0,0,0,58,0,0,0,0,1>>, nil,0,nil,nil}, 1, {btree,<0.442.0>, {2900, <<0,0,0,0,1,0,0,0,0,0,0,0,0,0,11,50>>, 60}, #Fun, #Fun, #Fun, #Fun,1279,2558, true}, {btree,<0.442.0>, {2960,<<0,0,0,0,1>>,58}, #Fun, #Fun, #Fun, #Fun,1279,2558, true}, {btree,<0.442.0>,nil, #Fun, #Fun, #Fun,nil,1279,2558, true}, 1,<<"_replicator">>, "/opt/couchbase/var/lib/couchbase/data/_replicator.couch.1", [],nil, {user_ctx,null,[],undefined}, nil, [before_header,after_header,on_file_open], [create,sys_db, {user_ctx, {user_ctx,null, [<<"_admin">>,<<"_replicator">>], undefined}}]} ** Reason for termination == ** killed [error_logger:error,2012-11-02T18:23:23.319,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: couch_db:init/1 pid: <0.446.0> registered_name: [] exception exit: killed in function gen_server:terminate/6 ancestors: [couch_server,couch_primary_services,couch_server_sup, cb_couch_sup,ns_server_cluster_sup,<0.59.0>] messages: [] links: [] dictionary: [] trap_exit: true status: running heap_size: 987 stack_size: 24 reductions: 309 neighbours: [error_logger:error,2012-11-02T18:23:23.319,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_msg:76]** Generic server <0.230.0> terminating ** Last message in was {'EXIT',<0.216.0>,killed} ** When Server state == {db,<0.230.0>,<0.231.0>,nil,<<"1351905791877517">>, <0.226.0>,<0.232.0>, {db_header,10,1, <<0,0,0,0,13,103,0,0,0,0,0,49,0,0,0,0,1,0,0,0, 0,0,0,0,0,0,13,69>>, <<0,0,0,0,13,152,0,0,0,0,0,47,0,0,0,0,1>>, nil,0,nil,nil}, 1, {btree,<0.226.0>, {3431, <<0,0,0,0,1,0,0,0,0,0,0,0,0,0,13,69>>, 49}, #Fun, #Fun, #Fun, #Fun,1279,2558, true}, {btree,<0.226.0>, {3480,<<0,0,0,0,1>>,47}, #Fun, #Fun, #Fun, #Fun,1279,2558, true}, {btree,<0.226.0>,nil, #Fun, #Fun, #Fun,nil,1279,2558, true}, 1,<<"_users">>, "/opt/couchbase/var/lib/couchbase/data/_users.couch.1", [],nil, {user_ctx,null,[],undefined}, nil, [before_header,after_header,on_file_open], [create, {user_ctx, {user_ctx,null,[<<"_admin">>],undefined}}, sys_db]} ** Reason for termination == ** killed [error_logger:error,2012-11-02T18:23:23.320,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: couch_db:init/1 pid: <0.230.0> registered_name: [] exception exit: killed in function gen_server:terminate/6 ancestors: [couch_server,couch_primary_services,couch_server_sup, cb_couch_sup,ns_server_cluster_sup,<0.59.0>] messages: [] links: [] dictionary: [] trap_exit: true status: running heap_size: 987 stack_size: 24 reductions: 284 neighbours: [error_logger:info,2012-11-02T18:23:23.321,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================INFO REPORT========================= application: mapreduce exited: stopped type: temporary [error_logger:info,2012-11-02T18:23:23.321,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================INFO REPORT========================= application: couch_view_parser exited: stopped type: temporary [error_logger:info,2012-11-02T18:23:23.321,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================INFO REPORT========================= application: couch_index_merger exited: stopped type: temporary [error_logger:info,2012-11-02T18:23:23.321,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================INFO REPORT========================= application: couch_set_view exited: stopped type: temporary [error_logger:info,2012-11-02T18:23:23.321,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= application: couch_view_parser started_at: 'ns_1@127.0.0.1' [error_logger:info,2012-11-02T18:23:23.321,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= application: couch_set_view started_at: 'ns_1@127.0.0.1' [error_logger:info,2012-11-02T18:23:23.321,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= application: couch_index_merger started_at: 'ns_1@127.0.0.1' [error_logger:info,2012-11-02T18:23:23.322,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= application: mapreduce started_at: 'ns_1@127.0.0.1' [error_logger:info,2012-11-02T18:23:23.337,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,couch_server_sup} started: [{pid,<0.557.0>}, {name,couch_config}, {mfargs, {couch_server_sup,couch_config_start_link_wrapper, [["/opt/couchbase/etc/couchdb/default.ini", "/opt/couchbase/etc/couchdb/default.d/capi.ini", "/opt/couchbase/etc/couchdb/default.d/geocouch.ini", "/opt/couchbase/etc/couchdb/local.ini"], <0.557.0>]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:23.337,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,couch_primary_services} started: [{pid,<0.560.0>}, {name,collation_driver}, {mfargs,{couch_drv,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:23:23.337,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,couch_primary_services} started: [{pid,<0.561.0>}, {name,couch_task_events}, {mfargs, {gen_event,start_link,[{local,couch_task_events}]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:23.337,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,couch_primary_services} started: [{pid,<0.562.0>}, {name,couch_task_status}, {mfargs,{couch_task_status,start_link,[]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:23.337,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,couch_primary_services} started: [{pid,<0.563.0>}, {name,couch_file_write_guard}, {mfargs,{couch_file_write_guard,sup_start_link,[]}}, {restart_type,permanent}, {shutdown,10000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:23.337,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,couch_primary_services} started: [{pid,<0.564.0>}, {name,couch_server}, {mfargs,{couch_server,sup_start_link,[]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:23.337,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,couch_primary_services} started: [{pid,<0.565.0>}, {name,couch_db_update_event}, {mfargs, {gen_event,start_link,[{local,couch_db_update}]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:23.338,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,couch_primary_services} started: [{pid,<0.566.0>}, {name,couch_replication_event}, {mfargs, {gen_event,start_link,[{local,couch_replication}]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:23.338,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,couch_primary_services} started: [{pid,<0.567.0>}, {name,couch_replication_supervisor}, {mfargs,{couch_rep_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:23:23.338,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,couch_primary_services} started: [{pid,<0.568.0>}, {name,couch_log}, {mfargs,{couch_log,start_link,[]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:23.338,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,couch_primary_services} started: [{pid,<0.569.0>}, {name,couch_main_index_barrier}, {mfargs, {couch_index_barrier,start_link, [couch_main_index_barrier, "max_parallel_indexers"]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:23.338,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,couch_primary_services} started: [{pid,<0.570.0>}, {name,couch_replica_index_barrier}, {mfargs, {couch_index_barrier,start_link, [couch_replica_index_barrier, "max_parallel_replica_indexers"]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:23.338,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,couch_server_sup} started: [{pid,<0.559.0>}, {name,couch_primary_services}, {mfargs,{couch_primary_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:23:23.338,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,couch_secondary_services} started: [{pid,<0.572.0>}, {name,couch_db_update_notifier_sup}, {mfargs,{couch_db_update_notifier_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:23:23.341,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,couch_secondary_services} started: [{pid,<0.573.0>}, {name,auth_cache}, {mfargs,{couch_auth_cache,start_link,[]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:23.342,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,couch_secondary_services} started: [{pid,<0.582.0>}, {name,set_view_manager}, {mfargs,{couch_set_view,start_link,[]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:23.344,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,couch_secondary_services} started: [{pid,<0.584.0>}, {name,spatial_manager}, {mfargs,{couch_spatial,start_link,[]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:23.344,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,couch_secondary_services} started: [{pid,<0.586.0>}, {name,index_merger_pool}, {mfargs, {lhttpc_manager,start_link, [[{connection_timeout,90000}, {pool_size,10000}, {name,couch_index_merger_connection_pool}]]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:23.344,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,couch_secondary_services} started: [{pid,<0.587.0>}, {name,query_servers}, {mfargs,{couch_query_servers,start_link,[]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:23.344,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,couch_secondary_services} started: [{pid,<0.589.0>}, {name,couch_set_view_ddoc_cache}, {mfargs,{couch_set_view_ddoc_cache,start_link,[]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:23.345,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,couch_secondary_services} started: [{pid,<0.591.0>}, {name,view_manager}, {mfargs,{couch_view,start_link,[]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:23.346,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,couch_secondary_services} started: [{pid,<0.593.0>}, {name,httpd}, {mfargs,{couch_httpd,start_link,[]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:23.346,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,couch_secondary_services} started: [{pid,<0.610.0>}, {name,uuids}, {mfargs,{couch_uuids,start,[]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:23.346,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,couch_server_sup} started: [{pid,<0.571.0>}, {name,couch_secondary_services}, {mfargs,{couch_secondary_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:23:23.346,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,cb_couch_sup} started: [{pid,<0.558.0>}, {name,couch_app}, {mfargs, {couch_app,start, [fake, ["/opt/couchbase/etc/couchdb/default.ini", "/opt/couchbase/etc/couchdb/local.ini"]]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [ns_server:info,2012-11-02T18:23:23.350,ns_1@127.0.0.1:ns_server_cluster_sup<0.167.0>:log_os_info:start_link:25]OS type: {unix,linux} Version: {2,6,18} Runtime info: [{otp_release,"R14B04"}, {erl_version,"5.8.5"}, {erl_version_long, "Erlang R14B04 (erts-5.8.5) [source] [64-bit] [smp:128:128] [rq:128] [async-threads:0] [kernel-poll:true]\n"}, {system_arch_raw,"x86_64-unknown-linux-gnu"}, {system_arch,"x86_64-unknown-linux-gnu"}, {localtime,{{2012,11,2},{18,23,23}}}, {memory, [{total,39362488}, {processes,7183832}, {processes_used,7047808}, {system,32178656}, {atom,1242281}, {atom_used,1240746}, {binary,117256}, {code,12684005}, {ets,836976}]}, {loaded, [lib,mb_grid,capi_utils,base64,set_view_update_daemon, xdc_rdoc_replication_srv,compaction_daemon,stats_archiver, couch_httpd_view,system_stats_collector,mochijson2, couch_changes,ns_bucket_sup,mochiweb_response, couch_set_view_util,menelaus_web_buckets,menelaus_auth, geocouch_duplicates,mochiweb_util,mochiweb_request, mochiweb_headers,mochiweb,xdc_rep_manager, ns_cluster_membership,ns_memcached_log_rotator, ns_port_server,supervisor_cushion,ns_port_init, ns_moxi_sup,ns_port_sup,mc_tcp_listener,mc_conn_sup, mc_sup,menelaus_web_alerts_srv,hot_keys_keeper, menelaus_event,menelaus_util,menelaus_deps,menelaus_web, menelaus_sup,ringbuffer,master_activity_events_keeper, master_activity_events_pids_watcher,auto_failover,ns_tick, ns_online_config_upgrader,ns_orchestrator, failover_safeness_level,master_activity_events, cluster_compat_mode,mb_master_sup,gen_fsm, samples_loader_tasks,mb_master,xdc_replication_sup, remote_clusters_info,ns_bucket,ns_doctor,stats_reader, ns_heart,ns_mail_log,ns_mail_sup,work_queue, vbucket_map_mirror,ns_node_disco_rep_events,ns_config_rep, ns_node_disco_conf_events,ns_node_disco_log,net_adm, ns_node_disco,ns_node_disco_sup,ns_config_ets_dup,random, ns_log,ns_server_sup,ns_process_registry, cb_config_couch_sync,ns_config_log,ns_memcached,ns_pubsub, ns_config_isasl_sync,ns_config_replica,vclock, ns_storage_conf,ns_config_default,ns_config,ns_config_sup, mnesia_index,mnesia_loader,file_sorter,dets_v9,dets_utils, dets_sup,dets_server,dets,mnesia_log,mnesia_snmp_hook, mnesia_checkpoint,mnesia_late_loader,mnesia_dumper, mnesia_snmp_sup,mnesia_checkpoint_sup,mnesia_frag, mnesia_tm,mnesia_recover,mnesia_sp,mnesia_locker, mnesia_event,mnesia_kernel_sup,mnesia_sup,mnesia_bup, mnesia_schema,mnesia_controller,mnesia_lib,mnesia_monitor, mnesia_subscr,mnesia,mb_mnesia,mb_mnesia_sup,ns_cluster, ns_cookie_manager,erl_epmd,inet_tcp_dist,gen_udp, dist_manager,timeout_diag_logger,path_config,diag_handler, auth,ns_info,log_os_info,couch_config_writer, cb_init_loggers,mochiweb_acceptor,inet_tcp,gen_tcp, mochiweb_socket,mochiweb_socket_server,mochilists, mochiweb_http,eval_bits,couch_httpd,couch_view, couch_set_view_ddoc_cache,couch_query_servers, couch_spatial,mapreduce,couch_set_view,snappy, couch_compress,couch_spatial_validation, couch_set_view_mapreduce,ejson,couch_doc, couch_db_update_notifier,couch_btree,couch_ref_counter, couch_uuids,couch_db_updater,couch_db,couch_auth_cache, couch_db_update_notifier_sup,couch_secondary_sup, couch_index_barrier,couch_event_sup,couch_log, couch_rep_sup,httpd_util,filelib,couch_file, couch_file_write_guard,couch_task_status,erl_ddll, couch_drv,couch_primary_sup,couch_server,string,re,file2, couch_util,couch_config,couch_server_sup,mochiweb_sup, mochiweb_app,ssl_server,crypto,ssl,lhttpc_manager, lhttpc_sup,lhttpc,ssl_connection_sup,ssl_session_cache, ssl_certificate_db,ssl_manager,ssl_broker_sup,ssl_sup, ssl_app,tftp_sup,httpd_sup,httpc_handler_sup,httpc_cookie, inets,httpc_manager,httpc,httpc_profile_sup,httpc_sup, ftp_sup,inets_sup,inets_app,crypto_server,crypto_sup, crypto_app,couch_app,cb_couch_sup,ns_server_cluster_sup, calendar,ale_default_formatter,otp_internal,misc, 'ale_logger-xdcr','ale_logger-mapreduce_errors', 'ale_logger-views','ale_logger-cluster', 'ale_logger-rebalance','ale_logger-stats', 'ale_logger-ns_doctor','ale_logger-menelaus', 'ale_logger-user','ale_logger-ns_server', 'ale_logger-couchdb',ns_log_sink,disk_log_sup, disk_log_server,disk_log_1,disk_log,ale_disk_sink, ns_server,timer,io_lib_fread,cpu_sup,memsup,disksup, os_mon,io,release_handler,overload,alarm_handler,log_mf_h, sasl,ale_error_logger_handler,'ale_logger-ale_logger', 'ale_logger-error_logger',beam_opcodes,beam_dict,beam_asm, beam_validator,beam_flatten,beam_trim,beam_receive, beam_bsm,beam_peep,beam_dead,beam_type,beam_bool, beam_clean,beam_utils,beam_jump,beam_block,v3_codegen, v3_life,v3_kernel,sys_core_dsetel,erl_bifs,sys_core_fold, cerl_trees,sys_core_inline,core_lib,cerl,v3_core,erl_bits, erl_expand_records,sys_pre_expand,sofs,erl_internal,sets, ordsets,erl_lint,compile,dynamic_compile,ale_utils, io_lib_pretty,io_lib_format,io_lib,ale_codegen,dict,ale, ale_dynamic_sup,ale_sup,ale_app,ns_bootstrap, file_io_server,orddict,erl_eval,file,c,queue, kernel_config,user,user_sup,supervisor_bridge, standard_error,unicode,binary,ets,gb_sets, hipe_unified_loader,packages,code_server,code,file_server, net_kernel,global_group,erl_distribution,filename, inet_gethost_native,os,inet_parse,inet,inet_udp, inet_config,inet_db,global,gb_trees,rpc,supervisor,kernel, application_master,sys,application,gen_server,erl_parse, proplists,erl_scan,lists,application_controller,proc_lib, gen,gen_event,error_logger,heart,error_handler,erlang, erl_prim_loader,prim_zip,zlib,prim_file,prim_inet,init, otp_ring0]}, {applications, [{public_key,"Public key infrastructure","0.13"}, {lhttpc,"Lightweight HTTP Client","1.3.0"}, {ale,"Another Logger for Erlang","8cffe61"}, {os_mon,"CPO CXC 138 46","2.2.7"}, {couch_set_view,"Set views","1.2.0a-47a991b-git"}, {mnesia,"MNESIA CXC 138 12","4.5"}, {inets,"INETS CXC 138 49","5.7.1"}, {couch,"Apache CouchDB","1.2.0a-47a991b-git"}, {mapreduce,"MapReduce using V8 JavaScript engine","1.0.0"}, {couch_index_merger,"Index merger","1.2.0a-47a991b-git"}, {kernel,"ERTS CXC 138 10","2.14.5"}, {crypto,"CRYPTO version 2","2.0.4"}, {ssl,"Erlang/OTP SSL application","4.1.6"}, {sasl,"SASL CXC 138 11","2.1.10"}, {couch_view_parser,"Couch view parser","1.0.0"}, {ns_server,"Couchbase server","2.0.0-1925-rel-enterprise"}, {mochiweb,"MochiMedia Web Server","1.4.1"}, {oauth,"Erlang OAuth implementation","7d85d3ef"}, {stdlib,"ERTS CXC 138 10","1.17.5"}]}, {pre_loaded, [erlang,erl_prim_loader,prim_zip,zlib,prim_file,prim_inet, init,otp_ring0]}, {process_count,161}, {node,'ns_1@127.0.0.1'}, {nodes,[]}, {registered, [couch_set_view_ddoc_cache,disk_log_sup,disk_log_server, couch_index_merger_connection_pool,code_server, couch_spatial,inet_gethost_native_sup, couch_replica_index_barrier,application_controller, 'sink-disk_default',couch_main_index_barrier, couch_replication,standard_error_sup,couch_task_events, dist_manager,couch_log,httpd_sup,ssl_connection_sup, cb_couch_sup,ssl_manager,os_mon_sup,error_logger,cpu_sup, memsup,disksup,ssl_broker_sup,ssl_server,ssl_sup, httpc_sup,httpc_profile_sup,httpc_manager, httpc_handler_sup,sasl_safe_sup,kernel_safe_sup,ftp_sup, auth,inets_sup,crypto_server,crypto_sup, couch_secondary_services,couch_primary_services, couch_db_update,timer_server,sasl_sup,couch_query_servers, couch_httpd,couch_drv,couch_server,rex,net_sup, release_handler,kernel_sup,global_name_server,net_kernel, overload,file_server_2,alarm_handler,ale_sup, couch_auth_cache,ale_dynamic_sup,couch_task_status,ale, ns_server_cluster_sup,couch_view,couch_rep_sup, couch_config,couch_server_sup,couch_file_write_guard, couch_uuids,couch_db_update_notifier_sup,'sink-ns_log', 'sink-disk_stats','sink-disk_xdcr_errors', 'sink-disk_xdcr','sink-disk_debug',dets_sup, 'sink-disk_couchdb','sink-disk_mapreduce_errors',dets, 'sink-disk_views','sink-disk_error',couch_set_view, lhttpc_sup,standard_error,erl_prim_loader,mochiweb_sup, init,inet_gethost_native,inet_db,lhttpc_manager, global_group,user,tftp_sup,erl_epmd]}, {cookie,phwwpfkaopxukihc}, {wordsize,8}, {wall_clock,12}] [ns_server:info,2012-11-02T18:23:23.353,ns_1@127.0.0.1:ns_server_cluster_sup<0.167.0>:log_os_info:start_link:27]Manifest: ["","", " ", " ", " ", " ", " ", " ", " "," ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ",""] [error_logger:info,2012-11-02T18:23:23.354,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_cluster_sup} started: [{pid,<0.612.0>}, {name,timeout_diag_logger}, {mfargs,{timeout_diag_logger,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:23.354,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_cluster_sup} started: [{pid,<0.613.0>}, {name,ns_cookie_manager}, {mfargs,{ns_cookie_manager,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:23.355,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_cluster_sup} started: [{pid,<0.614.0>}, {name,ns_cluster}, {mfargs,{ns_cluster,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:23.355,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mb_mnesia_sup} started: [{pid,<0.616.0>}, {name,mb_mnesia_events}, {mfargs, {gen_event,start_link,[{local,mb_mnesia_events}]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:23.357,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mnesia_sup} started: [{pid,<0.626.0>}, {name,mnesia_event}, {mfargs,{mnesia_sup,start_event,[]}}, {restart_type,permanent}, {shutdown,30000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:23.358,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mnesia_kernel_sup} started: [{pid,<0.628.0>}, {name,mnesia_monitor}, {mfargs,{mnesia_monitor,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:23.358,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mnesia_kernel_sup} started: [{pid,<0.629.0>}, {name,mnesia_subscr}, {mfargs,{mnesia_subscr,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:23.358,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mnesia_kernel_sup} started: [{pid,<0.630.0>}, {name,mnesia_locker}, {mfargs,{mnesia_locker,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:23.359,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mnesia_kernel_sup} started: [{pid,<0.631.0>}, {name,mnesia_recover}, {mfargs,{mnesia_recover,start,[]}}, {restart_type,permanent}, {shutdown,180000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:23.380,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mnesia_kernel_sup} started: [{pid,<0.632.0>}, {name,mnesia_tm}, {mfargs,{mnesia_tm,start,[]}}, {restart_type,permanent}, {shutdown,30000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:23.380,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mnesia_kernel_sup} started: [{pid,<0.653.0>}, {name,mnesia_checkpoint_sup}, {mfargs,{mnesia_checkpoint_sup,start,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:23:23.381,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mnesia_kernel_sup} started: [{pid,<0.654.0>}, {name,mnesia_snmp_sup}, {mfargs,{mnesia_snmp_sup,start,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:23:23.381,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mnesia_kernel_sup} started: [{pid,<0.655.0>}, {name,mnesia_controller}, {mfargs,{mnesia_controller,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:23.382,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mnesia_kernel_sup} started: [{pid,<0.656.0>}, {name,mnesia_late_loader}, {mfargs,{mnesia_late_loader,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:23.382,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mnesia_sup} started: [{pid,<0.627.0>}, {name,mnesia_kernel_sup}, {mfargs,{mnesia_kernel_sup,start,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:23:23.382,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= application: mnesia started_at: 'ns_1@127.0.0.1' [error_logger:info,2012-11-02T18:23:23.391,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mb_mnesia_sup} started: [{pid,<0.617.0>}, {name,mb_mnesia}, {mfargs,{mb_mnesia,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:23.391,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_cluster_sup} started: [{pid,<0.615.0>}, {name,mb_mnesia_sup}, {mfargs,{mb_mnesia_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [ns_server:info,2012-11-02T18:23:23.391,ns_1@127.0.0.1:ns_config_sup<0.700.0>:ns_config_sup:init:32]loading static ns_config from "/opt/couchbase/etc/couchbase/config" [error_logger:info,2012-11-02T18:23:23.392,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_config_sup} started: [{pid,<0.701.0>}, {name,ns_config_events}, {mfargs, {gen_event,start_link,[{local,ns_config_events}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:23.392,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_config_sup} started: [{pid,<0.702.0>}, {name,ns_config_events_local}, {mfargs, {gen_event,start_link, [{local,ns_config_events_local}]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [ns_server:info,2012-11-02T18:23:23.393,ns_1@127.0.0.1:ns_config<0.703.0>:ns_config:load_config:674]Loading static config from "/opt/couchbase/etc/couchbase/config" [ns_server:info,2012-11-02T18:23:23.393,ns_1@127.0.0.1:ns_config<0.703.0>:ns_config:load_config:688]Loading dynamic config from "/opt/couchbase/var/lib/couchbase/config/config.dat" [ns_server:info,2012-11-02T18:23:23.396,ns_1@127.0.0.1:ns_config<0.703.0>:ns_config:load_config:706]Here's full dynamic config we loaded + static & default config: [{{node,'ns_1@127.0.0.1',rest},[{port,8091},{port_meta,global}]}, {{node,'ns_1@127.0.0.1',port_servers}, [{'_vclock',[{'ns_1@127.0.0.1',{2,63519124992}}]}, {moxi,"/opt/couchbase/bin/moxi", ["-Z", {"port_listen=~B,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", [port]}, "-z", {"url=http://127.0.0.1:~B/pools/default/saslBucketsStreaming", [{misc,this_node_rest_port,[]}]}, "-p","0","-Y","y","-O","stderr", {"~s",[verbosity]}], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR",{"~s",[{ns_moxi_sup,rest_user,[]}]}}, {"MOXI_SASL_PLAIN_PWD",{"~s",[{ns_moxi_sup,rest_pass,[]}]}}]}, use_stdio,exit_status,port_server_send_eol,stderr_to_stdout,stream]}, {memcached,"/opt/couchbase/bin/memcached", ["-X","/opt/couchbase/lib/memcached/stdin_term_handler.so","-X", {"/opt/couchbase/lib/memcached/file_logger.so,cyclesize=~B;sleeptime=~B;filename=~s/~s", [log_cyclesize,log_sleeptime,log_path,log_prefix]}, "-l", {"0.0.0.0:~B,0.0.0.0:~B:1000",[port,dedicated_port]}, "-p", {"~B",[port]}, "-E","/opt/couchbase/lib/memcached/bucket_engine.so","-B","binary", "-r","-c","10000","-e", {"admin=~s;default_bucket_name=default;auto_create=false", [admin_user]}, {"~s",[verbosity]}], [{env, [{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE",{"~s",[{isasl,path}]}}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,exit_status,port_server_send_eol,stream]}]}, {{node,'ns_1@127.0.0.1',ns_log}, [{'_vclock',[{'ns_1@127.0.0.1',{1,63519124992}}]}, {filename,"/opt/couchbase/var/lib/couchbase/data/ns_log"}]}, {{node,'ns_1@127.0.0.1',moxi},[{port,11211},{verbosity,[]}]}, {{node,'ns_1@127.0.0.1',memcached}, [{'_vclock',[{'_',{1,0}},{'ns_1@127.0.0.1',{2,63519124992}}]}, {mccouch_port,11213}, {engines, [{membase, [{engine,"/opt/couchbase/lib/memcached/ep.so"}, {static_config_string, "vb0=false;waitforwarmup=false;failpartialwarmup=false"}]}, {memcached, [{engine,"/opt/couchbase/lib/memcached/default_engine.so"}, {static_config_string,"vb0=true"}]}]}, {log_path,"/opt/couchbase/var/lib/couchbase/logs"}, {log_prefix,"memcached.log"}, {log_generations,10}, {log_cyclesize,104857600}, {log_sleeptime,19}, {log_rotation_period,39003}, {dedicated_port,11209}, {bucket_engine,"/opt/couchbase/lib/memcached/bucket_engine.so"}, {port,11210}, {dedicated_port,11209}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {verbosity,[]}]}, {{node,'ns_1@127.0.0.1',membership},active}, {{node,'ns_1@127.0.0.1',isasl}, [{'_vclock',[{'ns_1@127.0.0.1',{1,63519124992}}]}, {path,"/opt/couchbase/var/lib/couchbase/data/isasl.pw"}]}, {{node,'ns_1@127.0.0.1',compaction_daemon}, [{check_interval,30},{min_file_size,131072}]}, {{node,'ns_1@127.0.0.1',capi_port},8092}, {{couchdb,max_parallel_replica_indexers},2}, {{couchdb,max_parallel_indexers},4}, {xdcr_failure_restart_interval,30}, {xdcr_doc_batch_size_kb,512}, {xdcr_checkpoint_interval,1800}, {xdcr_capi_checkpoint_timeout,10}, {set_view_update_daemon, [{update_interval,5000}, {update_min_changes,5000}, {replica_update_min_changes,5000}]}, {replication,[{enabled,true}]}, {remote_clusters,[]}, {nodes_wanted,['ns_1@127.0.0.1']}, {max_bucket_count,10}, {index_aware_rebalance_disabled,false}, {fast_warmup, [{fast_warmup_enabled,true}, {min_memory_threshold,10}, {min_items_threshold,10}]}, {email_alerts, [{'_vclock',[{'ns_1@127.0.0.1',{1,63519124992}}]}, {recipients,["root@localhost"]}, {sender,"couchbase@localhost"}, {enabled,false}, {email_server, [{user,[]},{pass,[]},{host,"localhost"},{port,25},{encrypt,false}]}, {alerts, [auto_failover_node,auto_failover_maximum_reached, auto_failover_other_nodes_down,auto_failover_cluster_too_small,ip, disk,overhead,ep_oom_errors,ep_item_commit_failed]}]}, {buckets,[{configs,[]}]}, {autocompaction, [{database_fragmentation_threshold,{30,undefined}}, {view_fragmentation_threshold,{30,undefined}}]}, {auto_failover_cfg, [{'_vclock',[{'ns_1@127.0.0.1',{1,63519124992}}]}, {enabled,false}, {timeout,30}, {max_nodes,1}, {count,0}]}, {{node,'ns_1@127.0.0.1',config_version}, [{'_vclock',[{'ns_1@127.0.0.1',{5,63519124992}}]}|{2,0}]}, {otp, [{'_vclock',[{'ns_1@127.0.0.1',{1,63519124993}}]}, {cookie,phwwpfkaopxukihc}]}, {vbucket_map_history,[{'_vclock',[{'ns_1@127.0.0.1',{1,63519124993}}]}]}, {cluster_compat_version, [{'_vclock',[{'ns_1@127.0.0.1',{1,63519124993}}]},2,0]}, {dynamic_config_version, [{'_vclock',[{'ns_1@127.0.0.1',{4,63519124993}}]},2,0]}, {rest,[{port,8091}]}, {rest_creds, [{'_vclock',[{'ns_1@127.0.0.1',{1,63519124998}}]}, {creds,[{"Administrator",[{password,"password"}]}]}]}, {memory_quota,[{'_vclock',[{'ns_1@127.0.0.1',{1,63519124998}}]}|25678]}] [error_logger:info,2012-11-02T18:23:23.397,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_config_sup} started: [{pid,<0.703.0>}, {name,ns_config}, {mfargs, {ns_config,start_link, ["/opt/couchbase/etc/couchbase/config", ns_config_default]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:23.398,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_config_sup} started: [{pid,<0.705.0>}, {name,ns_config_remote}, {mfargs, {ns_config_replica,start_link, [{local,ns_config_remote}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [ns_server:warn,2012-11-02T18:23:23.400,ns_1@127.0.0.1:ns_config_isasl_sync<0.706.0>:ns_memcached:connect:1103]Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:info,2012-11-02T18:23:24.401,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_config_sup} started: [{pid,<0.706.0>}, {name,ns_config_isasl_sync}, {mfargs,{ns_config_isasl_sync,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:24.401,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_config_sup} started: [{pid,<0.709.0>}, {name,ns_config_log}, {mfargs,{ns_config_log,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:24.401,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_config_sup} started: [{pid,<0.711.0>}, {name,cb_config_couch_sync}, {mfargs,{cb_config_couch_sync,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:24.402,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_cluster_sup} started: [{pid,<0.700.0>}, {name,ns_config_sup}, {mfargs,{ns_config_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:23:24.402,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_cluster_sup} started: [{pid,<0.713.0>}, {name,vbucket_filter_changes_registry}, {mfargs, {ns_process_registry,start_link, [vbucket_filter_changes_registry]}}, {restart_type,permanent}, {shutdown,100}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:24.402,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.715.0>}, {name,ns_log}, {mfargs,{ns_log,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:24.403,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.716.0>}, {name,ns_config_ets_dup}, {mfargs,{ns_config_ets_dup,start_link,[]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:24.403,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.717.0>}, {name,ns_log_events}, {mfargs,{gen_event,start_link,[{local,ns_log_events}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:24.403,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_node_disco_sup} started: [{pid,<0.720.0>}, {name,ns_node_disco_events}, {mfargs, {gen_event,start_link, [{local,ns_node_disco_events}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:24.406,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_node_disco_sup} started: [{pid,<0.721.0>}, {name,ns_node_disco}, {mfargs,{ns_node_disco,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:24.406,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_node_disco_sup} started: [{pid,<0.724.0>}, {name,ns_node_disco_log}, {mfargs,{ns_node_disco_log,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [ns_server:info,2012-11-02T18:23:24.406,ns_1@127.0.0.1:ns_node_disco_events<0.720.0>:ns_node_disco_log:handle_event:46]ns_node_disco_log: nodes changed: ['ns_1@127.0.0.1'] [error_logger:info,2012-11-02T18:23:24.406,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_node_disco_sup} started: [{pid,<0.725.0>}, {name,ns_node_disco_conf_events}, {mfargs,{ns_node_disco_conf_events,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:24.407,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_node_disco_sup} started: [{pid,<0.726.0>}, {name,ns_config_rep_merger}, {mfargs,{ns_config_rep,start_link_merger,[]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:24.408,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_node_disco_sup} started: [{pid,<0.727.0>}, {name,ns_config_rep}, {mfargs,{ns_config_rep,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:24.408,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.719.0>}, {name,ns_node_disco_sup}, {mfargs,{ns_node_disco_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:23:24.409,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.730.0>}, {name,vbucket_map_mirror}, {mfargs,{vbucket_map_mirror,start_link,[]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:24.410,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.734.0>}, {name,ns_tick_event}, {mfargs,{gen_event,start_link,[{local,ns_tick_event}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:24.410,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.736.0>}, {name,mb_master_events}, {mfargs, {gen_event,start_link,[{local,mb_master_events}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:24.410,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.737.0>}, {name,buckets_events}, {mfargs, {gen_event,start_link,[{local,buckets_events}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:24.411,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_mail_sup} started: [{pid,<0.739.0>}, {name,ns_mail_log}, {mfargs,{ns_mail_log,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:24.411,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.738.0>}, {name,ns_mail_sup}, {mfargs,{ns_mail_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [ns_server:error,2012-11-02T18:23:24.411,ns_1@127.0.0.1:ns_heart<0.741.0>:ns_heart:grab_samples_loading_tasks:298]Failed to grab samples loader tasks: {exit, {noproc, {gen_server,call, [samples_loader_tasks,get_tasks, 2000]}}, [{gen_server,call,3}, {ns_heart,grab_samples_loading_tasks,0}, {ns_heart,current_status,0}, {ns_heart,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} [user:info,2012-11-02T18:23:24.412,ns_1@127.0.0.1:mb_master<0.748.0>:mb_master:init:89]I'm the only node, so I'm the master. [error_logger:info,2012-11-02T18:23:24.412,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.740.0>}, {name,ns_stats_event}, {mfargs, {gen_event,start_link,[{local,ns_stats_event}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [ns_server:info,2012-11-02T18:23:24.412,ns_1@127.0.0.1:ns_config_log<0.709.0>:ns_config_log:handle_info:57]config change: rest_creds -> ******** [error_logger:info,2012-11-02T18:23:24.412,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.741.0>}, {name,ns_heart}, {mfargs,{ns_heart,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:24.413,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.743.0>}, {name,ns_doctor}, {mfargs,{ns_doctor,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:24.413,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.746.0>}, {name,remote_clusters_info}, {mfargs,{remote_clusters_info,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:24.413,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mb_master_sup} started: [{pid,<0.752.0>}, {name,ns_orchestrator}, {mfargs,{ns_orchestrator,start_link,[]}}, {restart_type,permanent}, {shutdown,20}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:24.413,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mb_master_sup} started: [{pid,<0.753.0>}, {name,ns_tick}, {mfargs,{ns_tick,start_link,[]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:24.414,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mb_master_sup} started: [{pid,<0.754.0>}, {name,auto_failover}, {mfargs,{auto_failover,start_link,[]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:24.415,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.748.0>}, {name,mb_master}, {mfargs,{mb_master,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:23:24.415,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.755.0>}, {name,master_activity_events}, {mfargs, {gen_event,start_link, [{local,master_activity_events}]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:24.415,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.756.0>}, {name,master_activity_events_ingress}, {mfargs, {gen_event,start_link, [{local,master_activity_events_ingress}]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:24.415,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.757.0>}, {name,master_activity_events_timestamper}, {mfargs, {master_activity_events,start_link_timestamper,[]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:24.416,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.758.0>}, {name,master_activity_events_pids_watcher}, {mfargs, {master_activity_events_pids_watcher,start_link, []}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:24.416,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.759.0>}, {name,master_activity_events_keeper}, {mfargs,{master_activity_events_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:24.416,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.762.0>}, {name,menelaus_web}, {mfargs,{menelaus_web,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [user:info,2012-11-02T18:23:24.416,ns_1@127.0.0.1:ns_server_sup<0.714.0>:menelaus_sup:start_link:44]Couchbase Server has started on web port 8091 on node 'ns_1@127.0.0.1'. [ns_server:error,2012-11-02T18:23:24.416,ns_1@127.0.0.1:ns_heart<0.741.0>:ns_heart:grab_samples_loading_tasks:298]Failed to grab samples loader tasks: {exit, {noproc, {gen_server,call, [samples_loader_tasks,get_tasks, 2000]}}, [{gen_server,call,3}, {ns_heart,grab_samples_loading_tasks,0}, {ns_heart,current_status,0}, {ns_heart,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} [error_logger:info,2012-11-02T18:23:24.416,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.779.0>}, {name,menelaus_event}, {mfargs,{menelaus_event,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:24.417,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.780.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:24.417,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.781.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:info,2012-11-02T18:23:24.417,ns_1@127.0.0.1:<0.788.0>:mc_tcp_listener:init:24]mccouch is listening on port 11213 [error_logger:info,2012-11-02T18:23:24.417,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.761.0>}, {name,menelaus}, {mfargs,{menelaus_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:23:24.417,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mc_sup} started: [{pid,<0.786.0>}, {name,mc_couch_events}, {mfargs, {gen_event,start_link,[{local,mc_couch_events}]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:24.418,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mc_sup} started: [{pid,<0.787.0>}, {name,mc_conn_sup}, {mfargs,{mc_conn_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:23:24.418,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mc_sup} started: [{pid,<0.788.0>}, {name,mc_tcp_listener}, {mfargs,{mc_tcp_listener,start_link,[11213]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:24.418,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.784.0>}, {name,mc_sup}, {mfargs,{mc_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:23:24.418,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_port_sup} started: [{pid,<0.790.0>}, {name,ns_port_init}, {mfargs,{ns_port_init,start_link,[]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:24.421,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_port_sup} started: [{pid,<0.791.0>}, {name, {moxi,"/opt/couchbase/bin/moxi", ["-Z", "port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", "-z", "url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming", "-p","0","-Y","y","-O","stderr",[]], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR","Administrator"}, {"MOXI_SASL_PLAIN_PWD","password"}]}, use_stdio,exit_status,port_server_send_eol, stderr_to_stdout,stream]}}, {mfargs, {supervisor_cushion,start_link, [moxi,5000,10000,ns_port_server,start_link, [moxi,"/opt/couchbase/bin/moxi", ["-Z", "port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", "-z", "url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming", "-p","0","-Y","y","-O","stderr",[]], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR","Administrator"}, {"MOXI_SASL_PLAIN_PWD","password"}]}, use_stdio,exit_status,port_server_send_eol, stderr_to_stdout,stream]]]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [ns_server:info,2012-11-02T18:23:24.422,ns_1@127.0.0.1:<0.796.0>:ns_memcached_log_rotator:init:28]Starting log rotator on "/opt/couchbase/var/lib/couchbase/logs"/"memcached.log"* with an initial period of 39003ms [error_logger:info,2012-11-02T18:23:24.423,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_port_sup} started: [{pid,<0.793.0>}, {name, {memcached,"/opt/couchbase/bin/memcached", ["-X", "/opt/couchbase/lib/memcached/stdin_term_handler.so", "-X", "/opt/couchbase/lib/memcached/file_logger.so,cyclesize=104857600;sleeptime=19;filename=/opt/couchbase/var/lib/couchbase/logs/memcached.log", "-l","0.0.0.0:11210,0.0.0.0:11209:1000","-p", "11210","-E", "/opt/couchbase/lib/memcached/bucket_engine.so", "-B","binary","-r","-c","10000","-e", "admin=_admin;default_bucket_name=default;auto_create=false", []], [{env, [{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE", "/opt/couchbase/var/lib/couchbase/data/isasl.pw"}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,exit_status, port_server_send_eol,stream]}}, {mfargs, {erlang,apply, [#Fun, [memcached,"/opt/couchbase/bin/memcached", ["-X", "/opt/couchbase/lib/memcached/stdin_term_handler.so", "-X", "/opt/couchbase/lib/memcached/file_logger.so,cyclesize=104857600;sleeptime=19;filename=/opt/couchbase/var/lib/couchbase/logs/memcached.log", "-l","0.0.0.0:11210,0.0.0.0:11209:1000", "-p","11210","-E", "/opt/couchbase/lib/memcached/bucket_engine.so", "-B","binary","-r","-c","10000","-e", "admin=_admin;default_bucket_name=default;auto_create=false", []], [{env, [{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE", "/opt/couchbase/var/lib/couchbase/data/isasl.pw"}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,exit_status, port_server_send_eol,stream]]]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:24.423,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.789.0>}, {name,ns_port_sup}, {mfargs,{ns_port_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:23:24.423,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.795.0>}, {name,ns_port_memcached_killer}, {mfargs,{ns_port_sup,start_memcached_force_killer,[]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:24.424,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.796.0>}, {name,ns_memcached_log_rotator}, {mfargs,{ns_memcached_log_rotator,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:24.424,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.798.0>}, {name,ns_bucket_worker}, {mfargs,{work_queue,start_link,[ns_bucket_worker]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:24.424,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.799.0>}, {name,xdc_replication_sup}, {mfargs,{xdc_replication_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:23:24.430,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.800.0>}, {name,xdc_rep_manager}, {mfargs,{xdc_rep_manager,start_link,[]}}, {restart_type,permanent}, {shutdown,30000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:24.430,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_bucket_sup} started: [{pid,<0.811.0>}, {name,buckets_observing_subscription}, {mfargs,{ns_bucket_sup,subscribe_on_config_events,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:24.430,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.810.0>}, {name,ns_bucket_sup}, {mfargs,{ns_bucket_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:23:24.432,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.812.0>}, {name,system_stats_collector}, {mfargs,{system_stats_collector,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:24.433,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.814.0>}, {name,{stats_archiver,"@system"}}, {mfargs,{stats_archiver,start_link,["@system"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:24.433,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.816.0>}, {name,{stats_reader,"@system"}}, {mfargs,{stats_reader,start_link,["@system"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:24.433,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.817.0>}, {name,ns_moxi_sup_work_queue}, {mfargs, {work_queue,start_link,[ns_moxi_sup_work_queue]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:24.433,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.818.0>}, {name,ns_moxi_sup}, {mfargs,{ns_moxi_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [ns_server:info,2012-11-02T18:23:24.433,ns_1@127.0.0.1:set_view_update_daemon<0.825.0>:set_view_update_daemon:init:50]Set view update daemon, starting with the following settings: update interval: 5000ms minimum number of changes: 5000 [error_logger:info,2012-11-02T18:23:24.433,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.820.0>}, {name,compaction_daemon}, {mfargs, {supervisor_cushion,start_link, [compaction_daemon,3000,1000,compaction_daemon, start_link,[]]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:24.434,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.823.0>}, {name,xdc_rdoc_replication_srv}, {mfargs,{xdc_rdoc_replication_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:24.434,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.825.0>}, {name,set_view_update_daemon}, {mfargs,{set_view_update_daemon,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:24.434,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.827.0>}, {name,samples_loader_tasks}, {mfargs,{samples_loader_tasks,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:23:24.434,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_cluster_sup} started: [{pid,<0.714.0>}, {name,ns_server_sup}, {mfargs,{ns_server_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [ns_server:info,2012-11-02T18:23:24.633,ns_1@127.0.0.1:<0.792.0>:ns_port_server:log:171]moxi<0.792.0>: 2012-11-02 18:23:24: (cproxy_config.c.317) env: MOXI_SASL_PLAIN_USR (13) moxi<0.792.0>: 2012-11-02 18:23:24: (cproxy_config.c.326) env: MOXI_SASL_PLAIN_PWD (8) [cluster:info,2012-11-02T18:26:10.062,ns_1@127.0.0.1:ns_cluster<0.614.0>:ns_cluster:do_change_address:281]Decided to change address to "10.2.1.67" [user:warn,2012-11-02T18:26:10.074,nonode@nohost:ns_node_disco<0.721.0>:ns_node_disco:handle_info:168]Node nonode@nohost saw that node 'ns_1@127.0.0.1' went down. [ns_server:info,2012-11-02T18:26:10.075,nonode@nohost:dist_manager<0.268.0>:dist_manager:handle_call:215]Adjusted IP to "10.2.1.67" [ns_server:info,2012-11-02T18:26:10.075,nonode@nohost:dist_manager<0.268.0>:dist_manager:bringup:193]Attempting to bring up net_kernel with name 'ns_1@10.2.1.67' [error_logger:info,2012-11-02T18:26:10.075,nonode@nohost:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,net_sup} started: [{pid,<0.1263.0>}, {name,erl_epmd}, {mfargs,{erl_epmd,start_link,[]}}, {restart_type,permanent}, {shutdown,2000}, {child_type,worker}] [error_logger:info,2012-11-02T18:26:10.076,nonode@nohost:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,net_sup} started: [{pid,<0.1264.0>}, {name,auth}, {mfargs,{auth,start_link,[]}}, {restart_type,permanent}, {shutdown,2000}, {child_type,worker}] [user:info,2012-11-02T18:26:10.076,ns_1@10.2.1.67:ns_node_disco<0.721.0>:ns_node_disco:handle_info:162]Node 'ns_1@10.2.1.67' saw that node 'ns_1@10.2.1.67' came up. [error_logger:info,2012-11-02T18:26:10.077,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,net_sup} started: [{pid,<0.1265.0>}, {name,net_kernel}, {mfargs, {net_kernel,start_link, [['ns_1@10.2.1.67',longnames]]}}, {restart_type,permanent}, {shutdown,2000}, {child_type,worker}] [ns_server:info,2012-11-02T18:26:10.077,ns_1@10.2.1.67:dist_manager<0.268.0>:dist_manager:save_node:112]saving node to "/opt/couchbase/var/lib/couchbase/couchbase-server.node" [error_logger:info,2012-11-02T18:26:10.077,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,kernel_sup} started: [{pid,<0.1262.0>}, {name,net_sup_dynamic}, {mfargs, {erl_distribution,start_link, [['ns_1@10.2.1.67',longnames]]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,supervisor}] [ns_server:info,2012-11-02T18:26:10.078,ns_1@10.2.1.67:dist_manager<0.268.0>:dist_manager:handle_call:219]Re-setting cookie {phwwpfkaopxukihc,'ns_1@10.2.1.67'} [ns_server:info,2012-11-02T18:26:10.079,ns_1@10.2.1.67:dist_manager<0.268.0>:dist_manager:save_address_config:108]saving ip config to "/opt/couchbase/var/lib/couchbase/ip" [error_logger:info,2012-11-02T18:26:10.094,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================INFO REPORT========================= application: mnesia exited: stopped type: temporary [ns_server:info,2012-11-02T18:26:10.099,ns_1@10.2.1.67:<0.1260.0>:mb_mnesia:init:255]Found backup. Restoring Mnesia database. [error_logger:info,2012-11-02T18:26:10.111,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mnesia_sup} started: [{pid,<0.1293.0>}, {name,mnesia_event}, {mfargs,{mnesia_sup,start_event,[]}}, {restart_type,permanent}, {shutdown,30000}, {child_type,worker}] [error_logger:info,2012-11-02T18:26:10.112,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mnesia_kernel_sup} started: [{pid,<0.1295.0>}, {name,mnesia_monitor}, {mfargs,{mnesia_monitor,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] [error_logger:info,2012-11-02T18:26:10.112,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mnesia_kernel_sup} started: [{pid,<0.1296.0>}, {name,mnesia_subscr}, {mfargs,{mnesia_subscr,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] [error_logger:info,2012-11-02T18:26:10.112,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mnesia_kernel_sup} started: [{pid,<0.1297.0>}, {name,mnesia_locker}, {mfargs,{mnesia_locker,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] [error_logger:info,2012-11-02T18:26:10.112,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mnesia_kernel_sup} started: [{pid,<0.1298.0>}, {name,mnesia_recover}, {mfargs,{mnesia_recover,start,[]}}, {restart_type,permanent}, {shutdown,180000}, {child_type,worker}] [error_logger:info,2012-11-02T18:26:10.138,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mnesia_kernel_sup} started: [{pid,<0.1299.0>}, {name,mnesia_tm}, {mfargs,{mnesia_tm,start,[]}}, {restart_type,permanent}, {shutdown,30000}, {child_type,worker}] [error_logger:info,2012-11-02T18:26:10.138,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mnesia_kernel_sup} started: [{pid,<0.1342.0>}, {name,mnesia_checkpoint_sup}, {mfargs,{mnesia_checkpoint_sup,start,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:26:10.138,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mnesia_kernel_sup} started: [{pid,<0.1343.0>}, {name,mnesia_snmp_sup}, {mfargs,{mnesia_snmp_sup,start,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:26:10.139,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mnesia_kernel_sup} started: [{pid,<0.1344.0>}, {name,mnesia_controller}, {mfargs,{mnesia_controller,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] [error_logger:info,2012-11-02T18:26:10.139,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mnesia_kernel_sup} started: [{pid,<0.1345.0>}, {name,mnesia_late_loader}, {mfargs,{mnesia_late_loader,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] [error_logger:info,2012-11-02T18:26:10.139,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mnesia_sup} started: [{pid,<0.1294.0>}, {name,mnesia_kernel_sup}, {mfargs,{mnesia_kernel_sup,start,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:26:10.139,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= application: mnesia started_at: 'ns_1@10.2.1.67' [ns_server:info,2012-11-02T18:26:10.139,ns_1@10.2.1.67:ns_node_disco_events<0.720.0>:ns_node_disco_log:handle_event:46]ns_node_disco_log: nodes changed: ['ns_1@10.2.1.67'] [ns_server:info,2012-11-02T18:26:10.140,ns_1@10.2.1.67:mb_mnesia<0.617.0>:mb_mnesia:handle_info:192]Saw Mnesia go down on 'ns_1@10.2.1.67' [error_logger:info,2012-11-02T18:26:10.141,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.1359.0>}, {name,ns_doctor}, {mfargs,{ns_doctor,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [ns_server:info,2012-11-02T18:26:10.141,ns_1@10.2.1.67:mb_master<0.748.0>:mb_master:terminate:288]Synchronously shutting down child mb_master_sup [user:info,2012-11-02T18:26:10.142,ns_1@10.2.1.67:mb_master<0.1383.0>:mb_master:init:89]I'm the only node, so I'm the master. [error_logger:info,2012-11-02T18:26:10.142,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mb_master_sup} started: [{pid,<0.1386.0>}, {name,ns_orchestrator}, {mfargs,{ns_orchestrator,start_link,[]}}, {restart_type,permanent}, {shutdown,20}, {child_type,worker}] [error_logger:info,2012-11-02T18:26:10.143,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mb_master_sup} started: [{pid,<0.1387.0>}, {name,ns_tick}, {mfargs,{ns_tick,start_link,[]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] [error_logger:info,2012-11-02T18:26:10.143,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mb_master_sup} started: [{pid,<0.1388.0>}, {name,auto_failover}, {mfargs,{auto_failover,start_link,[]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] [error_logger:info,2012-11-02T18:26:10.143,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.1383.0>}, {name,mb_master}, {mfargs,{mb_master,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [cluster:info,2012-11-02T18:26:10.143,ns_1@10.2.1.67:ns_cluster<0.614.0>:ns_cluster:do_change_address:287]Renamed node. New name is 'ns_1@10.2.1.67'. [user:info,2012-11-02T18:26:10.143,ns_1@10.2.1.67:ns_cookie_manager<0.613.0>:ns_cookie_manager:do_cookie_init:91]Initial otp cookie generated: rqmnpylwkdihztcr [user:info,2012-11-02T18:26:10.164,ns_1@10.2.1.67:ns_cluster<0.614.0>:ns_cluster:perform_actual_join:630]Node 'ns_1@10.2.1.67' is joining cluster via node 'ns_1@10.2.1.65'. [error_logger:error,2012-11-02T18:26:10.165,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_msg:76]** Connection attempt from disallowed node 'ns_1@10.2.1.65' ** [error_logger:error,2012-11-02T18:26:10.165,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_msg:76]** Connection attempt from disallowed node 'ns_1@10.2.1.66' ** [error_logger:error,2012-11-02T18:26:10.165,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,ns_bucket_sup} Context: shutdown_error Reason: normal Offender: [{pid,<0.811.0>}, {name,buckets_observing_subscription}, {mfargs,{ns_bucket_sup,subscribe_on_config_events,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [ns_server:info,2012-11-02T18:26:10.367,ns_1@10.2.1.67:ns_port_memcached<0.794.0>:ns_port_server:log:171]memcached<0.794.0>: EOL on stdin. Initiating shutdown [ns_server:info,2012-11-02T18:26:10.632,ns_1@10.2.1.67:ns_port_memcached<0.794.0>:ns_port_server:handle_info:104]Port server memcached exited with status 0 [ns_server:info,2012-11-02T18:26:10.632,ns_1@10.2.1.67:<0.792.0>:ns_port_server:handle_info:104]Port server moxi exited with status 0 [ns_server:info,2012-11-02T18:26:10.633,ns_1@10.2.1.67:<0.792.0>:ns_port_server:log:171]moxi<0.792.0>: EOL on stdin. Exiting [ns_server:info,2012-11-02T18:26:10.633,ns_1@10.2.1.67:mb_master<0.1383.0>:mb_master:terminate:288]Synchronously shutting down child mb_master_sup [user:info,2012-11-02T18:26:10.638,ns_1@10.2.1.67:ns_cookie_manager<0.613.0>:ns_cookie_manager:do_cookie_sync:135]Node 'ns_1@10.2.1.67' synchronized otp cookie khsxbujfzniwernw from cluster [ns_server:warn,2012-11-02T18:26:10.644,ns_1@10.2.1.67:ns_log<0.1427.0>:ns_log:read_logs:69]Couldn't load logs from "/opt/couchbase/var/lib/couchbase/data/ns_log" (perhaps it's first startup): {error, enoent} [error_logger:info,2012-11-02T18:26:10.645,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.1427.0>}, {name,ns_log}, {mfargs,{ns_log,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:26:10.645,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.1428.0>}, {name,ns_config_ets_dup}, {mfargs,{ns_config_ets_dup,start_link,[]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:26:10.645,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.1429.0>}, {name,ns_log_events}, {mfargs,{gen_event,start_link,[{local,ns_log_events}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:26:10.645,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_node_disco_sup} started: [{pid,<0.1432.0>}, {name,ns_node_disco_events}, {mfargs, {gen_event,start_link, [{local,ns_node_disco_events}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:26:10.647,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_node_disco_sup} started: [{pid,<0.1433.0>}, {name,ns_node_disco}, {mfargs,{ns_node_disco,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [ns_server:info,2012-11-02T18:26:10.648,ns_1@10.2.1.67:ns_config_rep<0.1439.0>:ns_config_rep:do_pull:341]Pulling config from: 'ns_1@10.2.1.65' [error_logger:info,2012-11-02T18:26:10.648,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_node_disco_sup} started: [{pid,<0.1436.0>}, {name,ns_node_disco_log}, {mfargs,{ns_node_disco_log,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:26:10.648,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_node_disco_sup} started: [{pid,<0.1437.0>}, {name,ns_node_disco_conf_events}, {mfargs,{ns_node_disco_conf_events,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:26:10.648,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_node_disco_sup} started: [{pid,<0.1438.0>}, {name,ns_config_rep_merger}, {mfargs,{ns_config_rep,start_link_merger,[]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [ns_server:info,2012-11-02T18:26:10.651,ns_1@10.2.1.67:ns_config_log<0.709.0>:ns_config_log:handle_info:57]config change: rest_creds -> ******** [error_logger:info,2012-11-02T18:26:10.656,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_node_disco_sup} started: [{pid,<0.1439.0>}, {name,ns_config_rep}, {mfargs,{ns_config_rep,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:26:10.656,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.1431.0>}, {name,ns_node_disco_sup}, {mfargs,{ns_node_disco_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:26:10.657,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.1447.0>}, {name,vbucket_map_mirror}, {mfargs,{vbucket_map_mirror,start_link,[]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:26:10.657,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.1453.0>}, {name,ns_tick_event}, {mfargs,{gen_event,start_link,[{local,ns_tick_event}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:26:10.657,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.1454.0>}, {name,mb_master_events}, {mfargs, {gen_event,start_link,[{local,mb_master_events}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:26:10.657,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.1455.0>}, {name,buckets_events}, {mfargs, {gen_event,start_link,[{local,buckets_events}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:26:10.657,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_mail_sup} started: [{pid,<0.1457.0>}, {name,ns_mail_log}, {mfargs,{ns_mail_log,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:26:10.657,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.1456.0>}, {name,ns_mail_sup}, {mfargs,{ns_mail_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [ns_server:info,2012-11-02T18:26:10.657,ns_1@10.2.1.67:remote_clusters_info<0.1464.0>:remote_clusters_info:read_or_create_table:384]Reading remote_clusters_info content from /opt/couchbase/var/lib/couchbase/remote_clusters_cache [ns_server:error,2012-11-02T18:26:10.657,ns_1@10.2.1.67:ns_heart<0.1459.0>:ns_heart:grab_samples_loading_tasks:298]Failed to grab samples loader tasks: {exit, {noproc, {gen_server,call, [samples_loader_tasks,get_tasks, 2000]}}, [{gen_server,call,3}, {ns_heart,grab_samples_loading_tasks,0}, {ns_heart,current_status,0}, {ns_heart,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} [error_logger:info,2012-11-02T18:26:10.657,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.1458.0>}, {name,ns_stats_event}, {mfargs, {gen_event,start_link,[{local,ns_stats_event}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:26:10.658,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.1459.0>}, {name,ns_heart}, {mfargs,{ns_heart,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:26:10.658,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.1461.0>}, {name,ns_doctor}, {mfargs,{ns_doctor,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [ns_server:info,2012-11-02T18:26:10.658,ns_1@10.2.1.67:ns_config_log<0.709.0>:ns_config_log:handle_info:57]config change: rest_creds -> ******** [error_logger:info,2012-11-02T18:26:10.658,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.1464.0>}, {name,remote_clusters_info}, {mfargs,{remote_clusters_info,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [user:info,2012-11-02T18:26:10.659,ns_1@10.2.1.67:ns_node_disco<0.1433.0>:ns_node_disco:handle_info:162]Node 'ns_1@10.2.1.67' saw that node 'ns_1@10.2.1.66' came up. [ns_server:error,2012-11-02T18:26:10.659,ns_1@10.2.1.67:ns_heart<0.1459.0>:ns_heart:grab_samples_loading_tasks:298]Failed to grab samples loader tasks: {exit, {noproc, {gen_server,call, [samples_loader_tasks,get_tasks, 2000]}}, [{gen_server,call,3}, {ns_heart,grab_samples_loading_tasks,0}, {ns_heart,current_status,0}, {ns_heart,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} [ns_server:info,2012-11-02T18:26:10.659,ns_1@10.2.1.67:ns_node_disco_events<0.1432.0>:ns_node_disco_log:handle_event:46]ns_node_disco_log: nodes changed: ['ns_1@10.2.1.65','ns_1@10.2.1.66', 'ns_1@10.2.1.67'] [ns_server:info,2012-11-02T18:26:10.659,ns_1@10.2.1.67:ns_config_rep<0.1439.0>:ns_config_rep:handle_info:220]Replicating config to/from: ['ns_1@10.2.1.66'] [ns_server:info,2012-11-02T18:26:10.660,ns_1@10.2.1.67:ns_config_rep<0.1439.0>:ns_config_rep:do_pull:341]Pulling config from: 'ns_1@10.2.1.66' [error_logger:info,2012-11-02T18:26:10.660,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.1479.0>}, {name,mb_master}, {mfargs,{mb_master,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:26:10.660,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.1481.0>}, {name,master_activity_events}, {mfargs, {gen_event,start_link, [{local,master_activity_events}]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:26:10.661,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.1482.0>}, {name,master_activity_events_ingress}, {mfargs, {gen_event,start_link, [{local,master_activity_events_ingress}]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:26:10.661,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.1483.0>}, {name,master_activity_events_timestamper}, {mfargs, {master_activity_events,start_link_timestamper,[]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:26:10.661,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.1484.0>}, {name,master_activity_events_pids_watcher}, {mfargs, {master_activity_events_pids_watcher,start_link, []}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:26:10.661,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.1485.0>}, {name,master_activity_events_keeper}, {mfargs,{master_activity_events_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [user:info,2012-11-02T18:26:10.661,ns_1@10.2.1.67:ns_server_sup<0.1426.0>:menelaus_sup:start_link:44]Couchbase Server has started on web port 8091 on node 'ns_1@10.2.1.67'. [error_logger:info,2012-11-02T18:26:10.662,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.1488.0>}, {name,menelaus_web}, {mfargs,{menelaus_web,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info,2012-11-02T18:26:10.662,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.1505.0>}, {name,menelaus_event}, {mfargs,{menelaus_event,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:info,2012-11-02T18:26:10.662,ns_1@10.2.1.67:<0.1511.0>:mc_tcp_listener:init:24]mccouch is listening on port 11213 [error_logger:info,2012-11-02T18:26:10.662,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.1506.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info,2012-11-02T18:26:10.662,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.1507.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info,2012-11-02T18:26:10.662,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.1487.0>}, {name,menelaus}, {mfargs,{menelaus_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:26:10.662,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mc_sup} started: [{pid,<0.1509.0>}, {name,mc_couch_events}, {mfargs, {gen_event,start_link,[{local,mc_couch_events}]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:26:10.662,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mc_sup} started: [{pid,<0.1510.0>}, {name,mc_conn_sup}, {mfargs,{mc_conn_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:26:10.663,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mc_sup} started: [{pid,<0.1511.0>}, {name,mc_tcp_listener}, {mfargs,{mc_tcp_listener,start_link,[11213]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:26:10.663,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.1508.0>}, {name,mc_sup}, {mfargs,{mc_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:26:10.663,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_port_sup} started: [{pid,<0.1515.0>}, {name,ns_port_init}, {mfargs,{ns_port_init,start_link,[]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] [error_logger:info,2012-11-02T18:26:10.664,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_port_sup} started: [{pid,<0.1516.0>}, {name, {moxi,"/opt/couchbase/bin/moxi", ["-Z", "port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", "-z", "url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming", "-p","0","-Y","y","-O","stderr",[]], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR","Administrator"}, {"MOXI_SASL_PLAIN_PWD","password"}]}, use_stdio,exit_status,port_server_send_eol, stderr_to_stdout,stream]}}, {mfargs, {supervisor_cushion,start_link, [moxi,5000,10000,ns_port_server,start_link, [moxi,"/opt/couchbase/bin/moxi", ["-Z", "port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", "-z", "url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming", "-p","0","-Y","y","-O","stderr",[]], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR","Administrator"}, {"MOXI_SASL_PLAIN_PWD","password"}]}, use_stdio,exit_status,port_server_send_eol, stderr_to_stdout,stream]]]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [error_logger:info,2012-11-02T18:26:10.668,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_port_sup} started: [{pid,<0.1518.0>}, {name, {memcached,"/opt/couchbase/bin/memcached", ["-X", "/opt/couchbase/lib/memcached/stdin_term_handler.so", "-X", "/opt/couchbase/lib/memcached/file_logger.so,cyclesize=104857600;sleeptime=19;filename=/opt/couchbase/var/lib/couchbase/logs/memcached.log", "-l","0.0.0.0:11210,0.0.0.0:11209:1000","-p", "11210","-E", "/opt/couchbase/lib/memcached/bucket_engine.so", "-B","binary","-r","-c","10000","-e", "admin=_admin;default_bucket_name=default;auto_create=false", []], [{env, [{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE", "/opt/couchbase/var/lib/couchbase/data/isasl.pw"}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,exit_status, port_server_send_eol,stream]}}, {mfargs, {erlang,apply, [#Fun, [memcached,"/opt/couchbase/bin/memcached", ["-X", "/opt/couchbase/lib/memcached/stdin_term_handler.so", "-X", "/opt/couchbase/lib/memcached/file_logger.so,cyclesize=104857600;sleeptime=19;filename=/opt/couchbase/var/lib/couchbase/logs/memcached.log", "-l","0.0.0.0:11210,0.0.0.0:11209:1000", "-p","11210","-E", "/opt/couchbase/lib/memcached/bucket_engine.so", "-B","binary","-r","-c","10000","-e", "admin=_admin;default_bucket_name=default;auto_create=false", []], [{env, [{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE", "/opt/couchbase/var/lib/couchbase/data/isasl.pw"}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,exit_status, port_server_send_eol,stream]]]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [ns_server:info,2012-11-02T18:26:10.668,ns_1@10.2.1.67:<0.1521.0>:ns_memcached_log_rotator:init:28]Starting log rotator on "/opt/couchbase/var/lib/couchbase/logs"/"memcached.log"* with an initial period of 39003ms [error_logger:info,2012-11-02T18:26:10.668,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.1512.0>}, {name,ns_port_sup}, {mfargs,{ns_port_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:26:10.669,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.1520.0>}, {name,ns_port_memcached_killer}, {mfargs,{ns_port_sup,start_memcached_force_killer,[]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:26:10.670,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.1521.0>}, {name,ns_memcached_log_rotator}, {mfargs,{ns_memcached_log_rotator,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:26:10.670,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.1523.0>}, {name,ns_bucket_worker}, {mfargs,{work_queue,start_link,[ns_bucket_worker]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:26:10.670,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.1524.0>}, {name,xdc_replication_sup}, {mfargs,{xdc_replication_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:26:10.670,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.1525.0>}, {name,xdc_rep_manager}, {mfargs,{xdc_rep_manager,start_link,[]}}, {restart_type,permanent}, {shutdown,30000}, {child_type,worker}] [error_logger:info,2012-11-02T18:26:10.671,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_bucket_sup} started: [{pid,<0.1528.0>}, {name,buckets_observing_subscription}, {mfargs,{ns_bucket_sup,subscribe_on_config_events,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:26:10.671,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.1527.0>}, {name,ns_bucket_sup}, {mfargs,{ns_bucket_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:26:10.673,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.1530.0>}, {name,system_stats_collector}, {mfargs,{system_stats_collector,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:26:10.673,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.1532.0>}, {name,{stats_archiver,"@system"}}, {mfargs,{stats_archiver,start_link,["@system"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:26:10.673,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.1534.0>}, {name,{stats_reader,"@system"}}, {mfargs,{stats_reader,start_link,["@system"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:26:10.673,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.1535.0>}, {name,ns_moxi_sup_work_queue}, {mfargs, {work_queue,start_link,[ns_moxi_sup_work_queue]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:26:10.673,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.1536.0>}, {name,ns_moxi_sup}, {mfargs,{ns_moxi_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:26:10.674,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.1538.0>}, {name,compaction_daemon}, {mfargs, {supervisor_cushion,start_link, [compaction_daemon,3000,1000,compaction_daemon, start_link,[]]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [error_logger:info,2012-11-02T18:26:10.674,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.1541.0>}, {name,xdc_rdoc_replication_srv}, {mfargs,{xdc_rdoc_replication_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [ns_server:info,2012-11-02T18:26:10.674,ns_1@10.2.1.67:set_view_update_daemon<0.1543.0>:set_view_update_daemon:init:50]Set view update daemon, starting with the following settings: update interval: 5000ms minimum number of changes: 5000 [user:info,2012-11-02T18:26:10.675,ns_1@10.2.1.67:ns_cluster<0.614.0>:ns_cluster:perform_actual_join:676]Node ns_1@10.2.1.67 joined cluster [error_logger:info,2012-11-02T18:26:10.675,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.1543.0>}, {name,set_view_update_daemon}, {mfargs,{set_view_update_daemon,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:26:10.675,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.1545.0>}, {name,samples_loader_tasks}, {mfargs,{samples_loader_tasks,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:26:10.675,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_cluster_sup} started: [{pid,<0.1426.0>}, {name,ns_server_sup}, {mfargs,{ns_server_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [ns_server:info,2012-11-02T18:26:10.878,ns_1@10.2.1.67:<0.1517.0>:ns_port_server:log:171]moxi<0.1517.0>: 2012-11-02 18:26:10: (cproxy_config.c.317) env: MOXI_SASL_PLAIN_USR (13) moxi<0.1517.0>: 2012-11-02 18:26:10: (cproxy_config.c.326) env: MOXI_SASL_PLAIN_PWD (8) [ns_server:info,2012-11-02T18:26:12.536,ns_1@10.2.1.67:mb_master<0.1479.0>:mb_master:candidate:351]Changing master from undefined to 'ns_1@10.2.1.65' [user:info,2012-11-02T18:26:19.020,ns_1@10.2.1.67:ns_node_disco<0.1433.0>:ns_node_disco:handle_info:162]Node 'ns_1@10.2.1.67' saw that node 'ns_1@10.2.1.68' came up. [ns_server:info,2012-11-02T18:26:19.020,ns_1@10.2.1.67:ns_node_disco_events<0.1432.0>:ns_node_disco_log:handle_event:46]ns_node_disco_log: nodes changed: ['ns_1@10.2.1.65','ns_1@10.2.1.66', 'ns_1@10.2.1.67','ns_1@10.2.1.68'] [ns_server:warn,2012-11-02T18:26:19.021,ns_1@10.2.1.67:xdc_rdoc_replication_srv<0.1541.0>:xdc_rdoc_replication_srv:handle_info:128]Remote server node {xdc_rdoc_replication_srv,'ns_1@10.2.1.68'} process down: noproc [ns_server:info,2012-11-02T18:26:19.899,ns_1@10.2.1.67:<0.1595.0>:diag_handler:log_all_tap_and_checkpoint_stats:127]logging tap & checkpoint stats [ns_server:info,2012-11-02T18:26:19.902,ns_1@10.2.1.67:<0.1595.0>:diag_handler:log_all_tap_and_checkpoint_stats:131]end of logging tap & checkpoint stats [error_logger:info,2012-11-02T18:26:32.230,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_bucket_sup} started: [{pid,<0.1662.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:26:32.239,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,'single_bucket_sup-default'} started: [{pid,<0.1664.0>}, {name,{capi_set_view_manager,"default"}}, {mfargs,{capi_set_view_manager,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [ns_server:info,2012-11-02T18:26:32.247,ns_1@10.2.1.67:ns_memcached-default<0.1676.0>:ns_memcached:ensure_bucket:1119]Created bucket "default" with config string "ht_size=3079;ht_locks=5;tap_noop_interval=20;max_txn_size=10000;max_size=5242880000;tap_keepalive=300;dbname=/data/default;allow_data_loss_during_shutdown=true;backend=couchdb;couch_bucket=default;couch_port=11213;max_vbuckets=512;alog_path=/data/default/access.log;data_traffic_enabled=false;vb0=false;waitforwarmup=false;failpartialwarmup=false;" [error_logger:info,2012-11-02T18:26:32.248,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,'single_bucket_sup-default'} started: [{pid,<0.1676.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [error_logger:info,2012-11-02T18:26:32.249,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,'single_bucket_sup-default'} started: [{pid,<0.1682.0>}, {name,{tap_replication_manager,"default"}}, {mfargs, {tap_replication_manager,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:26:32.251,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,'single_bucket_sup-default'} started: [{pid,<0.1683.0>}, {name,{ns_vbm_new_sup,"default"}}, {mfargs,{ns_vbm_new_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:26:32.252,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,'single_bucket_sup-default'} started: [{pid,<0.1684.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,supervisor}] [ns_server:info,2012-11-02T18:26:32.255,ns_1@10.2.1.67:janitor_agent-default<0.1685.0>:janitor_agent:read_flush_counter:764]Loading flushseq failed: {error,enoent}. Assuming it's equal to global config. [ns_server:info,2012-11-02T18:26:32.255,ns_1@10.2.1.67:janitor_agent-default<0.1685.0>:janitor_agent:read_flush_counter_from_config:771]Initialized flushseq 0 from bucket config [error_logger:info,2012-11-02T18:26:32.255,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,'single_bucket_sup-default'} started: [{pid,<0.1685.0>}, {name,{janitor_agent,"default"}}, {mfargs,{janitor_agent,start_link,["default"]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:26:32.256,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,'single_bucket_sup-default'} started: [{pid,<0.1686.0>}, {name,{couch_stats_reader,"default"}}, {mfargs,{couch_stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:26:32.260,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,'single_bucket_sup-default'} started: [{pid,<0.1687.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:26:32.261,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,'single_bucket_sup-default'} started: [{pid,<0.1689.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:26:32.261,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,'single_bucket_sup-default'} started: [{pid,<0.1691.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:26:32.261,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,'single_bucket_sup-default'} started: [{pid,<0.1692.0>}, {name,{failover_safeness_level,"default"}}, {mfargs, {failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [ns_server:info,2012-11-02T18:26:32.443,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:26:32.243313 PDT 3: Trying to connect to mccouch: "localhost:11213" memcached<0.1519.0>: Fri Nov 2 18:26:32.243805 PDT 3: Connected to mccouch: "localhost:11213" memcached<0.1519.0>: Fri Nov 2 18:26:32.247033 PDT 3: Warning: failed to load the engine session stats due to IO exception "basic_ios::clear" memcached<0.1519.0>: Fri Nov 2 18:26:32.247125 PDT 3: Extension support isn't implemented in this version of bucket_engine memcached<0.1519.0>: Fri Nov 2 18:26:32.247180 PDT 3: Failed to load mutation log, falling back to key dump memcached<0.1519.0>: Fri Nov 2 18:26:32.247286 PDT 3: metadata loaded in 643 usec memcached<0.1519.0>: Fri Nov 2 18:26:32.247450 PDT 3: warmup completed in 725 usec [ns_server:info,2012-11-02T18:26:32.477,ns_1@10.2.1.67:<0.1517.0>:ns_port_server:log:171]moxi<0.1517.0>: 2012-11-02 18:26:34: (agent_config.c.705) ERROR: bad JSON configuration from http://127.0.0.1:8091/pools/default/saslBucketsStreaming: Number of vBuckets must be a power of two > 0 and <= 65536 ({ moxi<0.1517.0>: "name": "default", moxi<0.1517.0>: "nodeLocator": "vbucket", moxi<0.1517.0>: "saslPassword": "", moxi<0.1517.0>: "nodes": [{ moxi<0.1517.0>: "hostname": "10.2.1.65:8091", moxi<0.1517.0>: "ports": { moxi<0.1517.0>: "direct": 11210, moxi<0.1517.0>: "proxy": 11211 moxi<0.1517.0>: } moxi<0.1517.0>: }, { moxi<0.1517.0>: "hostname": "10.2.1.66:8091", moxi<0.1517.0>: "ports": { moxi<0.1517.0>: "direct": 11210, moxi<0.1517.0>: "proxy": 11211 moxi<0.1517.0>: } moxi<0.1517.0>: }, { moxi<0.1517.0>: "hostname": "10.2.1.67:8091", moxi<0.1517.0>: "ports": { moxi<0.1517.0>: "direct": 11210, moxi<0.1517.0>: "proxy": 11211 moxi<0.1517.0>: } moxi<0.1517.0>: }, { moxi<0.1517.0>: "hostname": "10.2.1.68:8091", moxi<0.1517.0>: "ports": { moxi<0.1517.0>: "direct": 11210, moxi<0.1517.0>: "proxy": 11211 moxi<0.1517.0>: } moxi<0.1517.0>: }], moxi<0.1517.0>: "vBucketServerMap": { moxi<0.1517.0>: "hashAlgorithm": "CRC", moxi<0.1517.0>: "numReplicas": 0, moxi<0.1517.0>: "serverList": ["10.2.1.67:11210", "10.2.1.65:11210", "10.2.1.66:11210", "10.2.1.68:11210"], moxi<0.1517.0>: "vBucketMap": [] moxi<0.1517.0>: } moxi<0.1517.0>: }) [user:info,2012-11-02T18:26:32.741,ns_1@10.2.1.67:ns_memcached-default<0.1676.0>:ns_memcached:handle_cast:581]Bucket "default" loaded on node 'ns_1@10.2.1.67' in 0 seconds. [ns_server:info,2012-11-02T18:26:33.248,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 383 state to active [ns_server:info,2012-11-02T18:26:33.248,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 382 state to active [ns_server:info,2012-11-02T18:26:33.249,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 381 state to active [ns_server:info,2012-11-02T18:26:33.249,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 380 state to active [ns_server:info,2012-11-02T18:26:33.250,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 379 state to active [ns_server:info,2012-11-02T18:26:33.250,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 378 state to active [ns_server:info,2012-11-02T18:26:33.250,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 377 state to active [ns_server:info,2012-11-02T18:26:33.251,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 376 state to active [ns_server:info,2012-11-02T18:26:33.251,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 375 state to active [ns_server:info,2012-11-02T18:26:33.251,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 374 state to active [ns_server:info,2012-11-02T18:26:33.252,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 373 state to active [ns_server:info,2012-11-02T18:26:33.252,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 372 state to active [ns_server:info,2012-11-02T18:26:33.253,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 371 state to active [ns_server:info,2012-11-02T18:26:33.253,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 370 state to active [ns_server:info,2012-11-02T18:26:33.253,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 369 state to active [ns_server:info,2012-11-02T18:26:33.254,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 368 state to active [ns_server:info,2012-11-02T18:26:33.254,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 367 state to active [ns_server:info,2012-11-02T18:26:33.255,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 366 state to active [ns_server:info,2012-11-02T18:26:33.255,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 365 state to active [ns_server:info,2012-11-02T18:26:33.255,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 364 state to active [ns_server:info,2012-11-02T18:26:33.256,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 363 state to active [ns_server:info,2012-11-02T18:26:33.256,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 362 state to active [ns_server:info,2012-11-02T18:26:33.256,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 361 state to active [ns_server:info,2012-11-02T18:26:33.257,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 360 state to active [ns_server:info,2012-11-02T18:26:33.257,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 359 state to active [ns_server:info,2012-11-02T18:26:33.257,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 358 state to active [ns_server:info,2012-11-02T18:26:33.257,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 357 state to active [ns_server:info,2012-11-02T18:26:33.258,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 356 state to active [ns_server:info,2012-11-02T18:26:33.258,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 355 state to active [ns_server:info,2012-11-02T18:26:33.258,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 354 state to active [ns_server:info,2012-11-02T18:26:33.259,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 353 state to active [ns_server:info,2012-11-02T18:26:33.259,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 352 state to active [ns_server:info,2012-11-02T18:26:33.259,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 351 state to active [ns_server:info,2012-11-02T18:26:33.260,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 350 state to active [ns_server:info,2012-11-02T18:26:33.260,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 349 state to active [ns_server:info,2012-11-02T18:26:33.260,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 348 state to active [ns_server:info,2012-11-02T18:26:33.261,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 347 state to active [ns_server:info,2012-11-02T18:26:33.261,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 346 state to active [ns_server:info,2012-11-02T18:26:33.261,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 345 state to active [ns_server:info,2012-11-02T18:26:33.261,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 344 state to active [ns_server:info,2012-11-02T18:26:33.262,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 343 state to active [ns_server:info,2012-11-02T18:26:33.262,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 342 state to active [ns_server:info,2012-11-02T18:26:33.262,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 341 state to active [ns_server:info,2012-11-02T18:26:33.263,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 340 state to active [ns_server:info,2012-11-02T18:26:33.263,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 339 state to active [ns_server:info,2012-11-02T18:26:33.264,ns_1@10.2.1.67:<0.1681.0>:ns_memcached:do_handle_call:521]Changed vbucket 338 state to active [ns_server:info,2012-11-02T18:26:33.264,ns_1@10.2.1.67:<0.1681.0>:ns_memcached:do_handle_call:521]Changed vbucket 337 state to active [ns_server:info,2012-11-02T18:26:33.264,ns_1@10.2.1.67:<0.1681.0>:ns_memcached:do_handle_call:521]Changed vbucket 336 state to active [ns_server:info,2012-11-02T18:26:33.265,ns_1@10.2.1.67:<0.1681.0>:ns_memcached:do_handle_call:521]Changed vbucket 335 state to active [ns_server:info,2012-11-02T18:26:33.265,ns_1@10.2.1.67:<0.1681.0>:ns_memcached:do_handle_call:521]Changed vbucket 334 state to active [ns_server:info,2012-11-02T18:26:33.265,ns_1@10.2.1.67:<0.1681.0>:ns_memcached:do_handle_call:521]Changed vbucket 333 state to active [ns_server:info,2012-11-02T18:26:33.266,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 332 state to active [ns_server:info,2012-11-02T18:26:33.266,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 331 state to active [ns_server:info,2012-11-02T18:26:33.267,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 330 state to active [ns_server:info,2012-11-02T18:26:33.267,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 329 state to active [ns_server:info,2012-11-02T18:26:33.267,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 328 state to active [ns_server:info,2012-11-02T18:26:33.268,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 327 state to active [ns_server:info,2012-11-02T18:26:33.268,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 326 state to active [ns_server:info,2012-11-02T18:26:33.268,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 325 state to active [ns_server:info,2012-11-02T18:26:33.269,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 324 state to active [ns_server:info,2012-11-02T18:26:33.269,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 323 state to active [ns_server:info,2012-11-02T18:26:33.270,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 322 state to active [ns_server:info,2012-11-02T18:26:33.270,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 321 state to active [ns_server:info,2012-11-02T18:26:33.270,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 320 state to active [ns_server:info,2012-11-02T18:26:33.271,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 319 state to active [ns_server:info,2012-11-02T18:26:33.271,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 318 state to active [ns_server:info,2012-11-02T18:26:33.271,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 317 state to active [ns_server:info,2012-11-02T18:26:33.271,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 316 state to active [ns_server:info,2012-11-02T18:26:33.272,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 315 state to active [ns_server:info,2012-11-02T18:26:33.272,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 314 state to active [ns_server:info,2012-11-02T18:26:33.273,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 313 state to active [ns_server:info,2012-11-02T18:26:33.273,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 312 state to active [ns_server:info,2012-11-02T18:26:33.273,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 311 state to active [ns_server:info,2012-11-02T18:26:33.273,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 310 state to active [ns_server:info,2012-11-02T18:26:33.274,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 309 state to active [ns_server:info,2012-11-02T18:26:33.274,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 308 state to active [ns_server:info,2012-11-02T18:26:33.274,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 307 state to active [ns_server:info,2012-11-02T18:26:33.275,ns_1@10.2.1.67:<0.1678.0>:ns_memcached:do_handle_call:521]Changed vbucket 306 state to active [ns_server:info,2012-11-02T18:26:33.275,ns_1@10.2.1.67:<0.1681.0>:ns_memcached:do_handle_call:521]Changed vbucket 305 state to active [ns_server:info,2012-11-02T18:26:33.276,ns_1@10.2.1.67:<0.1681.0>:ns_memcached:do_handle_call:521]Changed vbucket 304 state to active [ns_server:info,2012-11-02T18:26:33.276,ns_1@10.2.1.67:<0.1681.0>:ns_memcached:do_handle_call:521]Changed vbucket 303 state to active [ns_server:info,2012-11-02T18:26:33.276,ns_1@10.2.1.67:<0.1681.0>:ns_memcached:do_handle_call:521]Changed vbucket 302 state to active [ns_server:info,2012-11-02T18:26:33.277,ns_1@10.2.1.67:<0.1681.0>:ns_memcached:do_handle_call:521]Changed vbucket 301 state to active [ns_server:info,2012-11-02T18:26:33.277,ns_1@10.2.1.67:<0.1681.0>:ns_memcached:do_handle_call:521]Changed vbucket 300 state to active [ns_server:info,2012-11-02T18:26:33.277,ns_1@10.2.1.67:<0.1681.0>:ns_memcached:do_handle_call:521]Changed vbucket 299 state to active [ns_server:info,2012-11-02T18:26:33.277,ns_1@10.2.1.67:<0.1681.0>:ns_memcached:do_handle_call:521]Changed vbucket 298 state to active [ns_server:info,2012-11-02T18:26:33.278,ns_1@10.2.1.67:<0.1681.0>:ns_memcached:do_handle_call:521]Changed vbucket 297 state to active [ns_server:info,2012-11-02T18:26:33.278,ns_1@10.2.1.67:<0.1681.0>:ns_memcached:do_handle_call:521]Changed vbucket 296 state to active [ns_server:info,2012-11-02T18:26:33.278,ns_1@10.2.1.67:<0.1681.0>:ns_memcached:do_handle_call:521]Changed vbucket 295 state to active [ns_server:info,2012-11-02T18:26:33.279,ns_1@10.2.1.67:<0.1681.0>:ns_memcached:do_handle_call:521]Changed vbucket 294 state to active [ns_server:info,2012-11-02T18:26:33.279,ns_1@10.2.1.67:<0.1681.0>:ns_memcached:do_handle_call:521]Changed vbucket 293 state to active [ns_server:info,2012-11-02T18:26:33.279,ns_1@10.2.1.67:<0.1681.0>:ns_memcached:do_handle_call:521]Changed vbucket 292 state to active [ns_server:info,2012-11-02T18:26:33.279,ns_1@10.2.1.67:<0.1681.0>:ns_memcached:do_handle_call:521]Changed vbucket 291 state to active [ns_server:info,2012-11-02T18:26:33.280,ns_1@10.2.1.67:<0.1681.0>:ns_memcached:do_handle_call:521]Changed vbucket 290 state to active [ns_server:info,2012-11-02T18:26:33.280,ns_1@10.2.1.67:<0.1681.0>:ns_memcached:do_handle_call:521]Changed vbucket 289 state to active [ns_server:info,2012-11-02T18:26:33.280,ns_1@10.2.1.67:<0.1681.0>:ns_memcached:do_handle_call:521]Changed vbucket 288 state to active [ns_server:info,2012-11-02T18:26:33.281,ns_1@10.2.1.67:<0.1681.0>:ns_memcached:do_handle_call:521]Changed vbucket 287 state to active [ns_server:info,2012-11-02T18:26:33.281,ns_1@10.2.1.67:<0.1681.0>:ns_memcached:do_handle_call:521]Changed vbucket 286 state to active [ns_server:info,2012-11-02T18:26:33.281,ns_1@10.2.1.67:<0.1681.0>:ns_memcached:do_handle_call:521]Changed vbucket 285 state to active [ns_server:info,2012-11-02T18:26:33.282,ns_1@10.2.1.67:<0.1681.0>:ns_memcached:do_handle_call:521]Changed vbucket 284 state to active [ns_server:info,2012-11-02T18:26:33.282,ns_1@10.2.1.67:<0.1681.0>:ns_memcached:do_handle_call:521]Changed vbucket 283 state to active [ns_server:info,2012-11-02T18:26:33.282,ns_1@10.2.1.67:<0.1681.0>:ns_memcached:do_handle_call:521]Changed vbucket 282 state to active [ns_server:info,2012-11-02T18:26:33.282,ns_1@10.2.1.67:<0.1681.0>:ns_memcached:do_handle_call:521]Changed vbucket 281 state to active [ns_server:info,2012-11-02T18:26:33.283,ns_1@10.2.1.67:<0.1681.0>:ns_memcached:do_handle_call:521]Changed vbucket 280 state to active [ns_server:info,2012-11-02T18:26:33.283,ns_1@10.2.1.67:<0.1681.0>:ns_memcached:do_handle_call:521]Changed vbucket 279 state to active [ns_server:info,2012-11-02T18:26:33.283,ns_1@10.2.1.67:<0.1681.0>:ns_memcached:do_handle_call:521]Changed vbucket 278 state to active [ns_server:info,2012-11-02T18:26:33.284,ns_1@10.2.1.67:<0.1681.0>:ns_memcached:do_handle_call:521]Changed vbucket 277 state to active [ns_server:info,2012-11-02T18:26:33.284,ns_1@10.2.1.67:<0.1681.0>:ns_memcached:do_handle_call:521]Changed vbucket 276 state to active [ns_server:info,2012-11-02T18:26:33.284,ns_1@10.2.1.67:<0.1681.0>:ns_memcached:do_handle_call:521]Changed vbucket 275 state to active [ns_server:info,2012-11-02T18:26:33.284,ns_1@10.2.1.67:<0.1681.0>:ns_memcached:do_handle_call:521]Changed vbucket 274 state to active [ns_server:info,2012-11-02T18:26:33.285,ns_1@10.2.1.67:<0.1681.0>:ns_memcached:do_handle_call:521]Changed vbucket 273 state to active [ns_server:info,2012-11-02T18:26:33.285,ns_1@10.2.1.67:<0.1681.0>:ns_memcached:do_handle_call:521]Changed vbucket 272 state to active [ns_server:info,2012-11-02T18:26:33.285,ns_1@10.2.1.67:<0.1681.0>:ns_memcached:do_handle_call:521]Changed vbucket 271 state to active [ns_server:info,2012-11-02T18:26:33.285,ns_1@10.2.1.67:<0.1681.0>:ns_memcached:do_handle_call:521]Changed vbucket 270 state to active [ns_server:info,2012-11-02T18:26:33.286,ns_1@10.2.1.67:<0.1681.0>:ns_memcached:do_handle_call:521]Changed vbucket 269 state to active [ns_server:info,2012-11-02T18:26:33.286,ns_1@10.2.1.67:<0.1681.0>:ns_memcached:do_handle_call:521]Changed vbucket 268 state to active [ns_server:info,2012-11-02T18:26:33.286,ns_1@10.2.1.67:<0.1681.0>:ns_memcached:do_handle_call:521]Changed vbucket 267 state to active [ns_server:info,2012-11-02T18:26:33.287,ns_1@10.2.1.67:<0.1681.0>:ns_memcached:do_handle_call:521]Changed vbucket 266 state to active [ns_server:info,2012-11-02T18:26:33.287,ns_1@10.2.1.67:<0.1681.0>:ns_memcached:do_handle_call:521]Changed vbucket 265 state to active [ns_server:info,2012-11-02T18:26:33.287,ns_1@10.2.1.67:<0.1681.0>:ns_memcached:do_handle_call:521]Changed vbucket 264 state to active [ns_server:info,2012-11-02T18:26:33.287,ns_1@10.2.1.67:<0.1681.0>:ns_memcached:do_handle_call:521]Changed vbucket 263 state to active [ns_server:info,2012-11-02T18:26:33.288,ns_1@10.2.1.67:<0.1681.0>:ns_memcached:do_handle_call:521]Changed vbucket 262 state to active [ns_server:info,2012-11-02T18:26:33.288,ns_1@10.2.1.67:<0.1681.0>:ns_memcached:do_handle_call:521]Changed vbucket 261 state to active [ns_server:info,2012-11-02T18:26:33.288,ns_1@10.2.1.67:<0.1681.0>:ns_memcached:do_handle_call:521]Changed vbucket 260 state to active [ns_server:info,2012-11-02T18:26:33.289,ns_1@10.2.1.67:<0.1681.0>:ns_memcached:do_handle_call:521]Changed vbucket 259 state to active [ns_server:info,2012-11-02T18:26:33.289,ns_1@10.2.1.67:<0.1681.0>:ns_memcached:do_handle_call:521]Changed vbucket 258 state to active [ns_server:info,2012-11-02T18:26:33.289,ns_1@10.2.1.67:<0.1681.0>:ns_memcached:do_handle_call:521]Changed vbucket 257 state to active [ns_server:info,2012-11-02T18:26:33.289,ns_1@10.2.1.67:<0.1681.0>:ns_memcached:do_handle_call:521]Changed vbucket 256 state to active [ns_server:info,2012-11-02T18:26:33.295,ns_1@10.2.1.67:ns_memcached-default<0.1676.0>:ns_memcached:handle_call:244]Enabling traffic to bucket "default" [ns_server:info,2012-11-02T18:26:33.295,ns_1@10.2.1.67:ns_memcached-default<0.1676.0>:ns_memcached:handle_call:248]Bucket "default" marked as warmed in 1 seconds [ns_server:info,2012-11-02T18:26:34.014,ns_1@10.2.1.67:ns_doctor<0.1461.0>:ns_doctor:update_status:211]The following buckets became ready on node 'ns_1@10.2.1.68': ["default"] [ns_server:info,2012-11-02T18:26:34.049,ns_1@10.2.1.67:ns_doctor<0.1461.0>:ns_doctor:update_status:211]The following buckets became ready on node 'ns_1@10.2.1.65': ["default"] [ns_server:info,2012-11-02T18:26:35.659,ns_1@10.2.1.67:ns_doctor<0.1461.0>:ns_doctor:update_status:211]The following buckets became ready on node 'ns_1@10.2.1.67': ["default"] [ns_server:info,2012-11-02T18:26:36.686,ns_1@10.2.1.67:ns_doctor<0.1461.0>:ns_doctor:update_status:211]The following buckets became ready on node 'ns_1@10.2.1.66': ["default"] [ns_server:info,2012-11-02T18:26:40.053,ns_1@10.2.1.67:ns_config_rep<0.1439.0>:ns_config_rep:do_pull:341]Pulling config from: 'ns_1@10.2.1.68' [ns_server:info,2012-11-02T18:26:40.676,ns_1@10.2.1.67:<0.2567.0>:compaction_daemon:try_to_cleanup_indexes:439]Cleaning up indexes for bucket `default` [ns_server:info,2012-11-02T18:26:40.679,ns_1@10.2.1.67:<0.2567.0>:compaction_daemon:spawn_bucket_compactor:404]Compacting bucket default with config: [{database_fragmentation_threshold,{30,undefined}}, {view_fragmentation_threshold,{30,undefined}}] [ns_server:info,2012-11-02T18:27:10.687,ns_1@10.2.1.67:<0.2713.0>:compaction_daemon:try_to_cleanup_indexes:439]Cleaning up indexes for bucket `default` [ns_server:info,2012-11-02T18:27:10.687,ns_1@10.2.1.67:<0.2713.0>:compaction_daemon:spawn_bucket_compactor:404]Compacting bucket default with config: [{database_fragmentation_threshold,{30,undefined}}, {view_fragmentation_threshold,{30,undefined}}] [ns_server:info,2012-11-02T18:27:24.820,ns_1@10.2.1.67:ns_config_rep<0.1439.0>:ns_config_rep:do_pull:341]Pulling config from: 'ns_1@10.2.1.68' [ns_server:info,2012-11-02T18:27:40.694,ns_1@10.2.1.67:<0.2869.0>:compaction_daemon:try_to_cleanup_indexes:439]Cleaning up indexes for bucket `default` [ns_server:info,2012-11-02T18:27:40.694,ns_1@10.2.1.67:<0.2869.0>:compaction_daemon:spawn_bucket_compactor:404]Compacting bucket default with config: [{parallel_db_and_view_compaction,false}, {database_fragmentation_threshold,{30,undefined}}, {view_fragmentation_threshold,{100,undefined}}] [ns_server:info,2012-11-02T18:28:02.679,ns_1@10.2.1.67:ns_config_rep<0.1439.0>:ns_config_rep:do_pull:341]Pulling config from: 'ns_1@10.2.1.65' [ns_server:info,2012-11-02T18:28:10.701,ns_1@10.2.1.67:<0.3017.0>:compaction_daemon:try_to_cleanup_indexes:439]Cleaning up indexes for bucket `default` [ns_server:info,2012-11-02T18:28:10.702,ns_1@10.2.1.67:<0.3017.0>:compaction_daemon:spawn_bucket_compactor:404]Compacting bucket default with config: [{parallel_db_and_view_compaction,false}, {database_fragmentation_threshold,{30,undefined}}, {view_fragmentation_threshold,{100,undefined}}] [ns_server:info,2012-11-02T18:28:40.487,ns_1@10.2.1.67:ns_config_rep<0.1439.0>:ns_config_rep:do_pull:341]Pulling config from: 'ns_1@10.2.1.66' [ns_server:info,2012-11-02T18:28:40.709,ns_1@10.2.1.67:<0.3162.0>:compaction_daemon:try_to_cleanup_indexes:439]Cleaning up indexes for bucket `default` [ns_server:info,2012-11-02T18:28:40.710,ns_1@10.2.1.67:<0.3162.0>:compaction_daemon:spawn_bucket_compactor:404]Compacting bucket default with config: [{parallel_db_and_view_compaction,false}, {database_fragmentation_threshold,{30,undefined}}, {view_fragmentation_threshold,{100,undefined}}] [ns_server:info,2012-11-02T18:28:54.276,ns_1@10.2.1.67:ns_config_rep<0.1439.0>:ns_config_rep:do_pull:341]Pulling config from: 'ns_1@10.2.1.65' [ns_server:info,2012-11-02T18:29:10.716,ns_1@10.2.1.67:<0.3343.0>:compaction_daemon:try_to_cleanup_indexes:439]Cleaning up indexes for bucket `default` [ns_server:info,2012-11-02T18:29:10.717,ns_1@10.2.1.67:<0.3343.0>:compaction_daemon:spawn_bucket_compactor:404]Compacting bucket default with config: [{parallel_db_and_view_compaction,false}, {database_fragmentation_threshold,{30,undefined}}, {view_fragmentation_threshold,{100,undefined}}] [ns_server:info,2012-11-02T18:29:22.453,ns_1@10.2.1.67:ns_config_rep<0.1439.0>:ns_config_rep:do_pull:341]Pulling config from: 'ns_1@10.2.1.66' [ns_server:info,2012-11-02T18:29:40.724,ns_1@10.2.1.67:<0.3488.0>:compaction_daemon:try_to_cleanup_indexes:439]Cleaning up indexes for bucket `default` [ns_server:info,2012-11-02T18:29:40.725,ns_1@10.2.1.67:<0.3488.0>:compaction_daemon:spawn_bucket_compactor:404]Compacting bucket default with config: [{parallel_db_and_view_compaction,false}, {database_fragmentation_threshold,{30,undefined}}, {view_fragmentation_threshold,{100,undefined}}] [ns_server:info,2012-11-02T18:29:49.505,ns_1@10.2.1.67:ns_config_rep<0.1439.0>:ns_config_rep:do_pull:341]Pulling config from: 'ns_1@10.2.1.66' [ns_server:info,2012-11-02T18:30:10.732,ns_1@10.2.1.67:<0.3645.0>:compaction_daemon:try_to_cleanup_indexes:439]Cleaning up indexes for bucket `default` [ns_server:info,2012-11-02T18:30:10.732,ns_1@10.2.1.67:<0.3645.0>:compaction_daemon:spawn_bucket_compactor:404]Compacting bucket default with config: [{parallel_db_and_view_compaction,false}, {database_fragmentation_threshold,{30,undefined}}, {view_fragmentation_threshold,{100,undefined}}] [ns_server:info,2012-11-02T18:30:10.736,ns_1@10.2.1.67:<0.3648.0>:compaction_daemon:spawn_dbs_compactor:544]Compacting databases for bucket default [ns_server:info,2012-11-02T18:30:10.738,ns_1@10.2.1.67:<0.3649.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/master">> [ns_server:info,2012-11-02T18:30:10.749,ns_1@10.2.1.67:<0.3660.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/256">> [ns_server:info,2012-11-02T18:30:10.763,ns_1@10.2.1.67:<0.3671.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/257">> [ns_server:info,2012-11-02T18:30:10.775,ns_1@10.2.1.67:<0.3682.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/258">> [ns_server:info,2012-11-02T18:30:10.787,ns_1@10.2.1.67:<0.3693.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/259">> [ns_server:info,2012-11-02T18:30:10.804,ns_1@10.2.1.67:<0.3704.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/260">> [ns_server:info,2012-11-02T18:30:10.818,ns_1@10.2.1.67:<0.3715.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/261">> [ns_server:info,2012-11-02T18:30:10.832,ns_1@10.2.1.67:<0.3726.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/262">> [ns_server:info,2012-11-02T18:30:10.847,ns_1@10.2.1.67:<0.3737.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/263">> [ns_server:info,2012-11-02T18:30:10.868,ns_1@10.2.1.67:<0.3748.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/264">> [ns_server:info,2012-11-02T18:30:10.887,ns_1@10.2.1.67:<0.3759.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/265">> [ns_server:info,2012-11-02T18:30:10.903,ns_1@10.2.1.67:<0.3770.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/266">> [ns_server:info,2012-11-02T18:30:10.921,ns_1@10.2.1.67:<0.3781.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/267">> [ns_server:info,2012-11-02T18:30:10.937,ns_1@10.2.1.67:<0.3792.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/268">> [ns_server:info,2012-11-02T18:30:10.961,ns_1@10.2.1.67:<0.3811.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/269">> [ns_server:info,2012-11-02T18:30:10.980,ns_1@10.2.1.67:<0.3822.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/270">> [ns_server:info,2012-11-02T18:30:10.995,ns_1@10.2.1.67:<0.3833.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/271">> [ns_server:info,2012-11-02T18:30:11.012,ns_1@10.2.1.67:<0.3844.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/272">> [ns_server:info,2012-11-02T18:30:11.031,ns_1@10.2.1.67:<0.3855.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/273">> [ns_server:info,2012-11-02T18:30:11.051,ns_1@10.2.1.67:<0.3866.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/274">> [ns_server:info,2012-11-02T18:30:11.069,ns_1@10.2.1.67:<0.3877.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/275">> [ns_server:info,2012-11-02T18:30:11.089,ns_1@10.2.1.67:<0.3888.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/276">> [ns_server:info,2012-11-02T18:30:11.108,ns_1@10.2.1.67:<0.3899.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/277">> [ns_server:info,2012-11-02T18:30:11.129,ns_1@10.2.1.67:<0.3910.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/278">> [ns_server:info,2012-11-02T18:30:11.148,ns_1@10.2.1.67:<0.3921.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/279">> [ns_server:info,2012-11-02T18:30:11.165,ns_1@10.2.1.67:<0.3932.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/280">> [ns_server:info,2012-11-02T18:30:11.190,ns_1@10.2.1.67:<0.3943.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/281">> [ns_server:info,2012-11-02T18:30:11.208,ns_1@10.2.1.67:<0.3954.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/282">> [ns_server:info,2012-11-02T18:30:11.222,ns_1@10.2.1.67:<0.3965.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/283">> [ns_server:info,2012-11-02T18:30:11.234,ns_1@10.2.1.67:<0.3976.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/284">> [ns_server:info,2012-11-02T18:30:11.255,ns_1@10.2.1.67:<0.3995.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/285">> [ns_server:info,2012-11-02T18:30:11.268,ns_1@10.2.1.67:<0.4006.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/286">> [ns_server:info,2012-11-02T18:30:11.289,ns_1@10.2.1.67:<0.4017.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/287">> [ns_server:info,2012-11-02T18:30:11.310,ns_1@10.2.1.67:<0.4028.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/288">> [ns_server:info,2012-11-02T18:30:11.329,ns_1@10.2.1.67:<0.4039.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/289">> [ns_server:info,2012-11-02T18:30:11.348,ns_1@10.2.1.67:<0.4050.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/290">> [ns_server:info,2012-11-02T18:30:11.365,ns_1@10.2.1.67:<0.4061.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/291">> [ns_server:info,2012-11-02T18:30:11.382,ns_1@10.2.1.67:<0.4072.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/292">> [ns_server:info,2012-11-02T18:30:11.398,ns_1@10.2.1.67:<0.4083.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/293">> [ns_server:info,2012-11-02T18:30:11.416,ns_1@10.2.1.67:<0.4094.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/294">> [ns_server:info,2012-11-02T18:30:11.427,ns_1@10.2.1.67:<0.4105.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/295">> [ns_server:info,2012-11-02T18:30:11.441,ns_1@10.2.1.67:<0.4116.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/296">> [ns_server:info,2012-11-02T18:30:11.459,ns_1@10.2.1.67:<0.4127.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/297">> [ns_server:info,2012-11-02T18:30:11.480,ns_1@10.2.1.67:<0.4138.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/298">> [ns_server:info,2012-11-02T18:30:11.495,ns_1@10.2.1.67:<0.4149.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/299">> [ns_server:info,2012-11-02T18:30:11.507,ns_1@10.2.1.67:<0.4160.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/300">> [ns_server:info,2012-11-02T18:30:11.526,ns_1@10.2.1.67:<0.4171.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/301">> [ns_server:info,2012-11-02T18:30:11.551,ns_1@10.2.1.67:<0.4190.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/302">> [ns_server:info,2012-11-02T18:30:11.561,ns_1@10.2.1.67:<0.4201.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/303">> [ns_server:info,2012-11-02T18:30:11.578,ns_1@10.2.1.67:<0.4212.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/304">> [ns_server:info,2012-11-02T18:30:11.598,ns_1@10.2.1.67:<0.4223.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/305">> [ns_server:info,2012-11-02T18:30:11.621,ns_1@10.2.1.67:<0.4234.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/306">> [ns_server:info,2012-11-02T18:30:11.645,ns_1@10.2.1.67:<0.4245.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/307">> [ns_server:info,2012-11-02T18:30:11.667,ns_1@10.2.1.67:<0.4256.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/308">> [ns_server:info,2012-11-02T18:30:11.686,ns_1@10.2.1.67:<0.4267.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/309">> [ns_server:info,2012-11-02T18:30:11.706,ns_1@10.2.1.67:<0.4278.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/310">> [ns_server:info,2012-11-02T18:30:11.722,ns_1@10.2.1.67:<0.4289.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/311">> [ns_server:info,2012-11-02T18:30:11.744,ns_1@10.2.1.67:<0.4300.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/312">> [ns_server:info,2012-11-02T18:30:11.762,ns_1@10.2.1.67:<0.4311.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/313">> [ns_server:info,2012-11-02T18:30:11.782,ns_1@10.2.1.67:<0.4322.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/314">> [ns_server:info,2012-11-02T18:30:11.799,ns_1@10.2.1.67:<0.4333.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/315">> [ns_server:info,2012-11-02T18:30:11.819,ns_1@10.2.1.67:<0.4344.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/316">> [ns_server:info,2012-11-02T18:30:11.838,ns_1@10.2.1.67:<0.4355.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/317">> [ns_server:info,2012-11-02T18:30:11.854,ns_1@10.2.1.67:<0.4366.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/318">> [ns_server:info,2012-11-02T18:30:11.864,ns_1@10.2.1.67:<0.4377.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/319">> [ns_server:info,2012-11-02T18:30:11.892,ns_1@10.2.1.67:<0.4396.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/320">> [ns_server:info,2012-11-02T18:30:11.914,ns_1@10.2.1.67:<0.4407.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/321">> [ns_server:info,2012-11-02T18:30:11.936,ns_1@10.2.1.67:<0.4418.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/322">> [ns_server:info,2012-11-02T18:30:11.959,ns_1@10.2.1.67:<0.4429.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/323">> [ns_server:info,2012-11-02T18:30:11.980,ns_1@10.2.1.67:<0.4440.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/324">> [ns_server:info,2012-11-02T18:30:12.000,ns_1@10.2.1.67:<0.4451.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/325">> [ns_server:info,2012-11-02T18:30:12.019,ns_1@10.2.1.67:<0.4462.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/326">> [ns_server:info,2012-11-02T18:30:12.035,ns_1@10.2.1.67:<0.4473.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/327">> [ns_server:info,2012-11-02T18:30:12.059,ns_1@10.2.1.67:<0.4484.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/328">> [ns_server:info,2012-11-02T18:30:12.079,ns_1@10.2.1.67:<0.4495.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/329">> [ns_server:info,2012-11-02T18:30:12.098,ns_1@10.2.1.67:<0.4506.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/330">> [ns_server:info,2012-11-02T18:30:12.120,ns_1@10.2.1.67:<0.4517.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/331">> [ns_server:info,2012-11-02T18:30:12.141,ns_1@10.2.1.67:<0.4528.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/332">> [ns_server:info,2012-11-02T18:30:12.167,ns_1@10.2.1.67:<0.4539.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/333">> [ns_server:info,2012-11-02T18:30:12.188,ns_1@10.2.1.67:<0.4550.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/334">> [ns_server:info,2012-11-02T18:30:12.205,ns_1@10.2.1.67:<0.4561.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/335">> [ns_server:info,2012-11-02T18:30:12.226,ns_1@10.2.1.67:<0.4572.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/336">> [ns_server:info,2012-11-02T18:30:12.246,ns_1@10.2.1.67:<0.4583.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/337">> [ns_server:info,2012-11-02T18:30:12.274,ns_1@10.2.1.67:<0.4594.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/338">> [ns_server:info,2012-11-02T18:30:12.293,ns_1@10.2.1.67:<0.4605.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/339">> [ns_server:info,2012-11-02T18:30:12.314,ns_1@10.2.1.67:<0.4616.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/340">> [ns_server:info,2012-11-02T18:30:12.336,ns_1@10.2.1.67:<0.4627.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/341">> [ns_server:info,2012-11-02T18:30:12.356,ns_1@10.2.1.67:<0.4638.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/342">> [ns_server:info,2012-11-02T18:30:12.376,ns_1@10.2.1.67:<0.4649.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/343">> [ns_server:info,2012-11-02T18:30:12.398,ns_1@10.2.1.67:<0.4660.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/344">> [ns_server:info,2012-11-02T18:30:12.418,ns_1@10.2.1.67:<0.4671.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/345">> [ns_server:info,2012-11-02T18:30:12.442,ns_1@10.2.1.67:<0.4682.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/346">> [ns_server:info,2012-11-02T18:30:12.465,ns_1@10.2.1.67:<0.4693.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/347">> [ns_server:info,2012-11-02T18:30:12.488,ns_1@10.2.1.67:<0.4704.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/348">> [ns_server:info,2012-11-02T18:30:12.511,ns_1@10.2.1.67:<0.4715.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/349">> [ns_server:info,2012-11-02T18:30:12.532,ns_1@10.2.1.67:<0.4726.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/350">> [ns_server:info,2012-11-02T18:30:12.553,ns_1@10.2.1.67:<0.4737.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/351">> [ns_server:info,2012-11-02T18:30:12.567,ns_1@10.2.1.67:<0.4748.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/352">> [ns_server:info,2012-11-02T18:30:12.591,ns_1@10.2.1.67:<0.4759.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/353">> [ns_server:info,2012-11-02T18:30:12.612,ns_1@10.2.1.67:<0.4770.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/354">> [ns_server:info,2012-11-02T18:30:12.648,ns_1@10.2.1.67:<0.4789.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/355">> [ns_server:info,2012-11-02T18:30:12.670,ns_1@10.2.1.67:<0.4805.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/356">> [ns_server:info,2012-11-02T18:30:12.693,ns_1@10.2.1.67:<0.4816.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/357">> [ns_server:info,2012-11-02T18:30:12.714,ns_1@10.2.1.67:<0.4827.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/358">> [ns_server:info,2012-11-02T18:30:12.739,ns_1@10.2.1.67:<0.4838.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/359">> [ns_server:info,2012-11-02T18:30:12.762,ns_1@10.2.1.67:<0.4849.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/360">> [ns_server:info,2012-11-02T18:30:12.783,ns_1@10.2.1.67:<0.4860.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/361">> [ns_server:info,2012-11-02T18:30:12.804,ns_1@10.2.1.67:<0.4871.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/362">> [ns_server:info,2012-11-02T18:30:12.827,ns_1@10.2.1.67:<0.4882.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/363">> [ns_server:info,2012-11-02T18:30:12.850,ns_1@10.2.1.67:<0.4893.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/364">> [ns_server:info,2012-11-02T18:30:12.874,ns_1@10.2.1.67:<0.4904.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/365">> [ns_server:info,2012-11-02T18:30:12.895,ns_1@10.2.1.67:<0.4915.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/366">> [ns_server:info,2012-11-02T18:30:12.916,ns_1@10.2.1.67:<0.4926.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/367">> [ns_server:info,2012-11-02T18:30:12.937,ns_1@10.2.1.67:<0.4937.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/368">> [ns_server:info,2012-11-02T18:30:12.956,ns_1@10.2.1.67:<0.4948.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/369">> [ns_server:info,2012-11-02T18:30:12.980,ns_1@10.2.1.67:<0.4959.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/370">> [ns_server:info,2012-11-02T18:30:13.004,ns_1@10.2.1.67:<0.4970.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/371">> [ns_server:info,2012-11-02T18:30:13.021,ns_1@10.2.1.67:<0.4981.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/372">> [ns_server:info,2012-11-02T18:30:13.041,ns_1@10.2.1.67:<0.4992.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/373">> [ns_server:info,2012-11-02T18:30:13.063,ns_1@10.2.1.67:<0.5003.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/374">> [ns_server:info,2012-11-02T18:30:13.086,ns_1@10.2.1.67:<0.5014.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/375">> [ns_server:info,2012-11-02T18:30:13.103,ns_1@10.2.1.67:<0.5025.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/376">> [ns_server:info,2012-11-02T18:30:13.121,ns_1@10.2.1.67:<0.5036.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/377">> [ns_server:info,2012-11-02T18:30:13.141,ns_1@10.2.1.67:<0.5047.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/378">> [ns_server:info,2012-11-02T18:30:13.163,ns_1@10.2.1.67:<0.5058.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/379">> [ns_server:info,2012-11-02T18:30:13.187,ns_1@10.2.1.67:<0.5069.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/380">> [ns_server:info,2012-11-02T18:30:13.209,ns_1@10.2.1.67:<0.5080.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/381">> [ns_server:info,2012-11-02T18:30:13.232,ns_1@10.2.1.67:<0.5091.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/382">> [ns_server:info,2012-11-02T18:30:13.250,ns_1@10.2.1.67:<0.5102.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/383">> [ns_server:info,2012-11-02T18:30:13.271,ns_1@10.2.1.67:<0.3648.0>:compaction_daemon:spawn_dbs_compactor:576]Finished compaction of databases for bucket default [ns_server:info,2012-11-02T18:30:40.274,ns_1@10.2.1.67:<0.5233.0>:compaction_daemon:try_to_cleanup_indexes:439]Cleaning up indexes for bucket `default` [ns_server:info,2012-11-02T18:30:40.275,ns_1@10.2.1.67:<0.5233.0>:compaction_daemon:spawn_bucket_compactor:404]Compacting bucket default with config: [{parallel_db_and_view_compaction,false}, {database_fragmentation_threshold,{30,undefined}}, {view_fragmentation_threshold,{100,undefined}}] [ns_server:info,2012-11-02T18:30:40.281,ns_1@10.2.1.67:<0.5236.0>:compaction_daemon:spawn_dbs_compactor:544]Compacting databases for bucket default [ns_server:info,2012-11-02T18:30:40.282,ns_1@10.2.1.67:<0.5237.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/master">> [ns_server:info,2012-11-02T18:30:40.295,ns_1@10.2.1.67:<0.5248.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/256">> [ns_server:info,2012-11-02T18:30:40.328,ns_1@10.2.1.67:<0.5259.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/257">> [ns_server:info,2012-11-02T18:30:40.361,ns_1@10.2.1.67:<0.5270.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/258">> [ns_server:info,2012-11-02T18:30:40.400,ns_1@10.2.1.67:<0.5281.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/259">> [ns_server:info,2012-11-02T18:30:40.439,ns_1@10.2.1.67:<0.5292.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/260">> [ns_server:info,2012-11-02T18:30:40.464,ns_1@10.2.1.67:<0.5303.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/261">> [ns_server:info,2012-11-02T18:30:40.502,ns_1@10.2.1.67:<0.5322.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/262">> [ns_server:info,2012-11-02T18:30:40.541,ns_1@10.2.1.67:<0.5333.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/263">> [ns_server:info,2012-11-02T18:30:40.571,ns_1@10.2.1.67:<0.5344.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/264">> [ns_server:info,2012-11-02T18:30:40.605,ns_1@10.2.1.67:<0.5355.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/265">> [ns_server:info,2012-11-02T18:30:40.636,ns_1@10.2.1.67:<0.5366.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/266">> [ns_server:info,2012-11-02T18:30:40.660,ns_1@10.2.1.67:<0.5383.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/267">> [ns_server:info,2012-11-02T18:30:40.683,ns_1@10.2.1.67:<0.5401.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/268">> [ns_server:info,2012-11-02T18:30:40.718,ns_1@10.2.1.67:<0.5413.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/269">> [ns_server:info,2012-11-02T18:30:40.754,ns_1@10.2.1.67:<0.5424.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/270">> [ns_server:info,2012-11-02T18:30:40.796,ns_1@10.2.1.67:<0.5435.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/271">> [ns_server:info,2012-11-02T18:30:40.825,ns_1@10.2.1.67:<0.5454.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/272">> [ns_server:info,2012-11-02T18:30:40.864,ns_1@10.2.1.67:<0.5465.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/273">> [ns_server:info,2012-11-02T18:30:40.902,ns_1@10.2.1.67:<0.5476.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/274">> [ns_server:info,2012-11-02T18:30:40.930,ns_1@10.2.1.67:<0.5487.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/275">> [ns_server:info,2012-11-02T18:30:40.963,ns_1@10.2.1.67:<0.5498.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/276">> [ns_server:info,2012-11-02T18:30:41.004,ns_1@10.2.1.67:<0.5509.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/277">> [ns_server:info,2012-11-02T18:30:41.042,ns_1@10.2.1.67:<0.5520.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/278">> [ns_server:info,2012-11-02T18:30:41.081,ns_1@10.2.1.67:<0.5531.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/279">> [ns_server:info,2012-11-02T18:30:41.117,ns_1@10.2.1.67:<0.5542.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/280">> [ns_server:info,2012-11-02T18:30:41.156,ns_1@10.2.1.67:<0.5553.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/281">> [ns_server:info,2012-11-02T18:30:41.188,ns_1@10.2.1.67:<0.5564.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/282">> [ns_server:info,2012-11-02T18:30:41.229,ns_1@10.2.1.67:<0.5583.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/283">> [ns_server:info,2012-11-02T18:30:41.263,ns_1@10.2.1.67:<0.5594.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/284">> [ns_server:info,2012-11-02T18:30:41.301,ns_1@10.2.1.67:<0.5605.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/285">> [ns_server:info,2012-11-02T18:30:41.341,ns_1@10.2.1.67:<0.5616.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/286">> [ns_server:info,2012-11-02T18:30:41.376,ns_1@10.2.1.67:<0.5627.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/287">> [ns_server:info,2012-11-02T18:30:41.415,ns_1@10.2.1.67:<0.5638.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/288">> [ns_server:info,2012-11-02T18:30:41.444,ns_1@10.2.1.67:<0.5649.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/289">> [ns_server:info,2012-11-02T18:30:41.470,ns_1@10.2.1.67:<0.5660.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/290">> [ns_server:info,2012-11-02T18:30:41.500,ns_1@10.2.1.67:<0.5671.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/291">> [ns_server:info,2012-11-02T18:30:41.522,ns_1@10.2.1.67:<0.5682.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/292">> [ns_server:info,2012-11-02T18:30:41.561,ns_1@10.2.1.67:<0.5693.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/293">> [ns_server:info,2012-11-02T18:30:41.594,ns_1@10.2.1.67:<0.5704.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/294">> [ns_server:info,2012-11-02T18:30:41.636,ns_1@10.2.1.67:<0.5723.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/295">> [ns_server:info,2012-11-02T18:30:41.667,ns_1@10.2.1.67:<0.5734.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/296">> [ns_server:info,2012-11-02T18:30:41.690,ns_1@10.2.1.67:<0.5745.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/297">> [ns_server:info,2012-11-02T18:30:41.720,ns_1@10.2.1.67:<0.5756.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/298">> [ns_server:info,2012-11-02T18:30:41.751,ns_1@10.2.1.67:<0.5767.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/299">> [ns_server:info,2012-11-02T18:30:41.788,ns_1@10.2.1.67:<0.5778.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/300">> [ns_server:info,2012-11-02T18:30:41.823,ns_1@10.2.1.67:<0.5789.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/301">> [ns_server:info,2012-11-02T18:30:41.869,ns_1@10.2.1.67:<0.5800.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/302">> [ns_server:info,2012-11-02T18:30:41.894,ns_1@10.2.1.67:<0.5811.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/303">> [ns_server:info,2012-11-02T18:30:41.929,ns_1@10.2.1.67:<0.5822.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/304">> [ns_server:info,2012-11-02T18:30:41.968,ns_1@10.2.1.67:<0.5833.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/305">> [ns_server:info,2012-11-02T18:30:41.997,ns_1@10.2.1.67:<0.5844.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/306">> [ns_server:info,2012-11-02T18:30:42.040,ns_1@10.2.1.67:<0.5863.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/307">> [ns_server:info,2012-11-02T18:30:42.084,ns_1@10.2.1.67:<0.5874.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/308">> [ns_server:info,2012-11-02T18:30:42.124,ns_1@10.2.1.67:<0.5885.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/309">> [ns_server:info,2012-11-02T18:30:42.159,ns_1@10.2.1.67:<0.5896.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/310">> [ns_server:info,2012-11-02T18:30:42.201,ns_1@10.2.1.67:<0.5907.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/311">> [ns_server:info,2012-11-02T18:30:42.240,ns_1@10.2.1.67:<0.5918.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/312">> [ns_server:info,2012-11-02T18:30:42.271,ns_1@10.2.1.67:<0.5929.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/313">> [ns_server:info,2012-11-02T18:30:42.305,ns_1@10.2.1.67:<0.5940.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/314">> [ns_server:info,2012-11-02T18:30:42.346,ns_1@10.2.1.67:<0.5951.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/315">> [ns_server:info,2012-11-02T18:30:42.382,ns_1@10.2.1.67:<0.5962.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/316">> [ns_server:info,2012-11-02T18:30:42.407,ns_1@10.2.1.67:<0.5973.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/317">> [ns_server:info,2012-11-02T18:30:42.440,ns_1@10.2.1.67:<0.5984.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/318">> [ns_server:info,2012-11-02T18:30:42.483,ns_1@10.2.1.67:<0.5995.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/319">> [ns_server:info,2012-11-02T18:30:42.512,ns_1@10.2.1.67:<0.6006.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/320">> [ns_server:info,2012-11-02T18:30:42.548,ns_1@10.2.1.67:<0.6017.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/321">> [ns_server:info,2012-11-02T18:30:42.581,ns_1@10.2.1.67:<0.6028.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/322">> [ns_server:info,2012-11-02T18:30:42.634,ns_1@10.2.1.67:<0.6039.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/323">> [ns_server:info,2012-11-02T18:30:42.671,ns_1@10.2.1.67:<0.6055.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/324">> [ns_server:info,2012-11-02T18:30:42.715,ns_1@10.2.1.67:<0.6066.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/325">> [ns_server:info,2012-11-02T18:30:42.755,ns_1@10.2.1.67:<0.6077.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/326">> [ns_server:info,2012-11-02T18:30:42.782,ns_1@10.2.1.67:<0.6088.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/327">> [ns_server:info,2012-11-02T18:30:42.825,ns_1@10.2.1.67:<0.6099.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/328">> [ns_server:info,2012-11-02T18:30:42.865,ns_1@10.2.1.67:<0.6110.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/329">> [ns_server:info,2012-11-02T18:30:42.909,ns_1@10.2.1.67:<0.6129.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/330">> [ns_server:info,2012-11-02T18:30:42.947,ns_1@10.2.1.67:<0.6140.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/331">> [ns_server:info,2012-11-02T18:30:42.979,ns_1@10.2.1.67:<0.6151.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/332">> [ns_server:info,2012-11-02T18:30:43.024,ns_1@10.2.1.67:<0.6162.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/333">> [ns_server:info,2012-11-02T18:30:43.063,ns_1@10.2.1.67:<0.6173.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/334">> [ns_server:info,2012-11-02T18:30:43.093,ns_1@10.2.1.67:<0.6184.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/335">> [ns_server:info,2012-11-02T18:30:43.125,ns_1@10.2.1.67:<0.6195.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/336">> [ns_server:info,2012-11-02T18:30:43.163,ns_1@10.2.1.67:<0.6206.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/337">> [ns_server:info,2012-11-02T18:30:43.199,ns_1@10.2.1.67:<0.6217.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/338">> [ns_server:info,2012-11-02T18:30:43.231,ns_1@10.2.1.67:<0.6228.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/339">> [ns_server:info,2012-11-02T18:30:43.263,ns_1@10.2.1.67:<0.6239.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/340">> [ns_server:info,2012-11-02T18:30:43.296,ns_1@10.2.1.67:<0.6250.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/341">> [ns_server:info,2012-11-02T18:30:43.351,ns_1@10.2.1.67:<0.6261.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/342">> [ns_server:info,2012-11-02T18:30:43.387,ns_1@10.2.1.67:<0.6272.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/343">> [ns_server:info,2012-11-02T18:30:43.428,ns_1@10.2.1.67:<0.6283.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/344">> [ns_server:info,2012-11-02T18:30:43.466,ns_1@10.2.1.67:<0.6294.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/345">> [ns_server:info,2012-11-02T18:30:43.506,ns_1@10.2.1.67:<0.6305.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/346">> [ns_server:info,2012-11-02T18:30:43.549,ns_1@10.2.1.67:<0.6316.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/347">> [ns_server:info,2012-11-02T18:30:43.581,ns_1@10.2.1.67:<0.6327.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/348">> [ns_server:info,2012-11-02T18:30:43.603,ns_1@10.2.1.67:<0.6338.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/349">> [ns_server:info,2012-11-02T18:30:43.633,ns_1@10.2.1.67:<0.6349.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/350">> [ns_server:info,2012-11-02T18:30:43.673,ns_1@10.2.1.67:<0.6361.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/351">> [ns_server:info,2012-11-02T18:30:43.710,ns_1@10.2.1.67:<0.6372.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/352">> [ns_server:info,2012-11-02T18:30:43.764,ns_1@10.2.1.67:<0.6391.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/353">> [ns_server:info,2012-11-02T18:30:43.804,ns_1@10.2.1.67:<0.6402.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/354">> [ns_server:info,2012-11-02T18:30:43.836,ns_1@10.2.1.67:<0.6413.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/355">> [ns_server:info,2012-11-02T18:30:43.886,ns_1@10.2.1.67:<0.6424.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/356">> [ns_server:info,2012-11-02T18:30:43.921,ns_1@10.2.1.67:<0.6435.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/357">> [ns_server:info,2012-11-02T18:30:43.961,ns_1@10.2.1.67:<0.6446.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/358">> [ns_server:info,2012-11-02T18:30:44.005,ns_1@10.2.1.67:<0.6457.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/359">> [ns_server:info,2012-11-02T18:30:44.043,ns_1@10.2.1.67:<0.6468.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/360">> [ns_server:info,2012-11-02T18:30:44.083,ns_1@10.2.1.67:<0.6479.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/361">> [ns_server:info,2012-11-02T18:30:44.119,ns_1@10.2.1.67:<0.6490.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/362">> [ns_server:info,2012-11-02T18:30:44.167,ns_1@10.2.1.67:<0.6509.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/363">> [ns_server:info,2012-11-02T18:30:44.207,ns_1@10.2.1.67:<0.6520.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/364">> [ns_server:info,2012-11-02T18:30:44.246,ns_1@10.2.1.67:<0.6531.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/365">> [ns_server:info,2012-11-02T18:30:44.283,ns_1@10.2.1.67:<0.6542.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/366">> [ns_server:info,2012-11-02T18:30:44.322,ns_1@10.2.1.67:<0.6553.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/367">> [ns_server:info,2012-11-02T18:30:44.354,ns_1@10.2.1.67:<0.6564.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/368">> [ns_server:info,2012-11-02T18:30:44.398,ns_1@10.2.1.67:<0.6575.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/369">> [ns_server:info,2012-11-02T18:30:44.439,ns_1@10.2.1.67:<0.6586.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/370">> [ns_server:info,2012-11-02T18:30:44.470,ns_1@10.2.1.67:<0.6598.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/371">> [ns_server:info,2012-11-02T18:30:44.505,ns_1@10.2.1.67:<0.6609.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/372">> [ns_server:info,2012-11-02T18:30:44.552,ns_1@10.2.1.67:<0.6628.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/373">> [ns_server:info,2012-11-02T18:30:44.592,ns_1@10.2.1.67:<0.6639.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/374">> [ns_server:info,2012-11-02T18:30:44.621,ns_1@10.2.1.67:<0.6650.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/375">> [ns_server:info,2012-11-02T18:30:44.658,ns_1@10.2.1.67:<0.6661.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/376">> [ns_server:info,2012-11-02T18:30:44.696,ns_1@10.2.1.67:<0.6677.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/377">> [ns_server:info,2012-11-02T18:30:44.727,ns_1@10.2.1.67:<0.6688.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/378">> [ns_server:info,2012-11-02T18:30:44.760,ns_1@10.2.1.67:<0.6699.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/379">> [ns_server:info,2012-11-02T18:30:44.793,ns_1@10.2.1.67:<0.6710.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/380">> [ns_server:info,2012-11-02T18:30:45.286,ns_1@10.2.1.67:<0.6721.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/381">> [ns_server:info,2012-11-02T18:30:45.323,ns_1@10.2.1.67:<0.6732.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/382">> [ns_server:info,2012-11-02T18:30:45.384,ns_1@10.2.1.67:<0.6751.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/383">> [ns_server:info,2012-11-02T18:30:45.432,ns_1@10.2.1.67:<0.5236.0>:compaction_daemon:spawn_dbs_compactor:576]Finished compaction of databases for bucket default [ns_server:info,2012-11-02T18:30:48.917,ns_1@10.2.1.67:ns_config_rep<0.1439.0>:ns_config_rep:do_pull:341]Pulling config from: 'ns_1@10.2.1.68' [ns_server:info,2012-11-02T18:30:55.764,ns_1@10.2.1.67:ns_config_rep<0.1439.0>:ns_config_rep:do_pull:341]Pulling config from: 'ns_1@10.2.1.68' [ns_server:info,2012-11-02T18:31:10.435,ns_1@10.2.1.67:<0.6879.0>:compaction_daemon:try_to_cleanup_indexes:439]Cleaning up indexes for bucket `default` [ns_server:info,2012-11-02T18:31:10.435,ns_1@10.2.1.67:<0.6879.0>:compaction_daemon:spawn_bucket_compactor:404]Compacting bucket default with config: [{parallel_db_and_view_compaction,false}, {database_fragmentation_threshold,{30,undefined}}, {view_fragmentation_threshold,{100,undefined}}] [ns_server:info,2012-11-02T18:31:18.804,ns_1@10.2.1.67:ns_config_rep<0.1439.0>:ns_config_rep:do_pull:341]Pulling config from: 'ns_1@10.2.1.65' [ns_server:info,2012-11-02T18:31:25.225,ns_1@10.2.1.67:ns_config_rep<0.1439.0>:ns_config_rep:do_pull:341]Pulling config from: 'ns_1@10.2.1.66' [ns_server:info,2012-11-02T18:31:40.443,ns_1@10.2.1.67:<0.7029.0>:compaction_daemon:try_to_cleanup_indexes:439]Cleaning up indexes for bucket `default` [ns_server:info,2012-11-02T18:31:40.443,ns_1@10.2.1.67:<0.7029.0>:compaction_daemon:spawn_bucket_compactor:404]Compacting bucket default with config: [{parallel_db_and_view_compaction,false}, {database_fragmentation_threshold,{30,undefined}}, {view_fragmentation_threshold,{100,undefined}}] [ns_server:info,2012-11-02T18:31:40.448,ns_1@10.2.1.67:<0.7032.0>:compaction_daemon:spawn_dbs_compactor:544]Compacting databases for bucket default [ns_server:info,2012-11-02T18:31:40.449,ns_1@10.2.1.67:<0.7033.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/master">> [ns_server:info,2012-11-02T18:31:40.844,ns_1@10.2.1.67:<0.7058.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/256">> [ns_server:info,2012-11-02T18:31:41.872,ns_1@10.2.1.67:<0.7069.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/257">> [ns_server:info,2012-11-02T18:31:42.909,ns_1@10.2.1.67:<0.7085.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/258">> [ns_server:info,2012-11-02T18:31:45.474,ns_1@10.2.1.67:<0.7103.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/259">> [ns_server:info,2012-11-02T18:31:45.552,ns_1@10.2.1.67:<0.7114.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/260">> [ns_server:info,2012-11-02T18:31:45.623,ns_1@10.2.1.67:<0.7125.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/261">> [ns_server:info,2012-11-02T18:31:45.683,ns_1@10.2.1.67:<0.7143.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/262">> [ns_server:info,2012-11-02T18:31:45.749,ns_1@10.2.1.67:<0.7155.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/263">> [ns_server:info,2012-11-02T18:31:45.852,ns_1@10.2.1.67:<0.7174.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/264">> [ns_server:info,2012-11-02T18:31:45.903,ns_1@10.2.1.67:<0.7185.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/265">> [ns_server:info,2012-11-02T18:31:45.989,ns_1@10.2.1.67:<0.7196.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/266">> [ns_server:info,2012-11-02T18:31:46.056,ns_1@10.2.1.67:<0.7207.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/267">> [ns_server:info,2012-11-02T18:31:46.125,ns_1@10.2.1.67:<0.7218.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/268">> [ns_server:info,2012-11-02T18:31:46.192,ns_1@10.2.1.67:<0.7229.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/269">> [ns_server:info,2012-11-02T18:31:46.260,ns_1@10.2.1.67:<0.7240.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/270">> [ns_server:info,2012-11-02T18:31:46.334,ns_1@10.2.1.67:<0.7251.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/271">> [ns_server:info,2012-11-02T18:31:46.397,ns_1@10.2.1.67:<0.7262.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/272">> [ns_server:info,2012-11-02T18:31:46.468,ns_1@10.2.1.67:<0.7273.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/273">> [ns_server:info,2012-11-02T18:31:46.546,ns_1@10.2.1.67:<0.7284.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/274">> [ns_server:info,2012-11-02T18:31:46.615,ns_1@10.2.1.67:<0.7295.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/275">> [ns_server:info,2012-11-02T18:31:47.270,ns_1@10.2.1.67:<0.7312.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/276">> [ns_server:info,2012-11-02T18:31:49.777,ns_1@10.2.1.67:<0.7330.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/277">> [ns_server:info,2012-11-02T18:31:52.432,ns_1@10.2.1.67:<0.7354.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/278">> [ns_server:info,2012-11-02T18:31:52.496,ns_1@10.2.1.67:<0.7365.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/279">> [ns_server:info,2012-11-02T18:31:52.563,ns_1@10.2.1.67:<0.7376.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/280">> [ns_server:info,2012-11-02T18:31:52.632,ns_1@10.2.1.67:<0.7387.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/281">> [ns_server:info,2012-11-02T18:31:52.711,ns_1@10.2.1.67:<0.7404.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/282">> [ns_server:info,2012-11-02T18:31:52.779,ns_1@10.2.1.67:<0.7415.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/283">> [ns_server:info,2012-11-02T18:31:52.840,ns_1@10.2.1.67:<0.7426.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/284">> [ns_server:info,2012-11-02T18:31:52.915,ns_1@10.2.1.67:<0.7437.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/285">> [ns_server:info,2012-11-02T18:31:52.976,ns_1@10.2.1.67:<0.7448.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/286">> [ns_server:info,2012-11-02T18:31:53.045,ns_1@10.2.1.67:<0.7459.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/287">> [ns_server:info,2012-11-02T18:31:53.108,ns_1@10.2.1.67:<0.7470.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/288">> [ns_server:info,2012-11-02T18:31:53.192,ns_1@10.2.1.67:<0.7481.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/289">> [ns_server:info,2012-11-02T18:31:53.261,ns_1@10.2.1.67:<0.7492.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/290">> [ns_server:info,2012-11-02T18:31:53.326,ns_1@10.2.1.67:<0.7503.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/291">> [ns_server:info,2012-11-02T18:31:53.386,ns_1@10.2.1.67:<0.7514.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/292">> [ns_server:info,2012-11-02T18:31:53.443,ns_1@10.2.1.67:<0.7525.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/293">> [ns_server:info,2012-11-02T18:31:53.510,ns_1@10.2.1.67:<0.7536.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/294">> [ns_server:info,2012-11-02T18:31:53.572,ns_1@10.2.1.67:<0.7547.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/295">> [ns_server:info,2012-11-02T18:31:53.640,ns_1@10.2.1.67:<0.7558.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/296">> [ns_server:info,2012-11-02T18:31:59.202,ns_1@10.2.1.67:<0.7595.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/297">> [ns_server:info,2012-11-02T18:31:59.286,ns_1@10.2.1.67:<0.7606.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/298">> [ns_server:info,2012-11-02T18:31:59.349,ns_1@10.2.1.67:<0.7617.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/299">> [ns_server:info,2012-11-02T18:31:59.411,ns_1@10.2.1.67:<0.7628.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/300">> [ns_server:info,2012-11-02T18:31:59.484,ns_1@10.2.1.67:<0.7640.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/301">> [ns_server:info,2012-11-02T18:31:59.544,ns_1@10.2.1.67:<0.7651.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/302">> [ns_server:info,2012-11-02T18:31:59.610,ns_1@10.2.1.67:<0.7662.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/303">> [ns_server:info,2012-11-02T18:31:59.683,ns_1@10.2.1.67:<0.7673.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/304">> [ns_server:info,2012-11-02T18:31:59.762,ns_1@10.2.1.67:<0.7684.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/305">> [ns_server:info,2012-11-02T18:31:59.821,ns_1@10.2.1.67:<0.7695.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/306">> [ns_server:info,2012-11-02T18:31:59.899,ns_1@10.2.1.67:<0.7706.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/307">> [ns_server:info,2012-11-02T18:31:59.962,ns_1@10.2.1.67:<0.7717.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/308">> [ns_server:info,2012-11-02T18:32:00.031,ns_1@10.2.1.67:<0.7728.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/309">> [ns_server:info,2012-11-02T18:32:00.107,ns_1@10.2.1.67:<0.7739.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/310">> [ns_server:info,2012-11-02T18:32:00.188,ns_1@10.2.1.67:<0.7750.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/311">> [ns_server:info,2012-11-02T18:32:00.254,ns_1@10.2.1.67:<0.7761.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/312">> [ns_server:info,2012-11-02T18:32:00.325,ns_1@10.2.1.67:<0.7772.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/313">> [ns_server:info,2012-11-02T18:32:00.397,ns_1@10.2.1.67:<0.7783.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/314">> [ns_server:info,2012-11-02T18:32:00.472,ns_1@10.2.1.67:<0.7794.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/315">> [ns_server:info,2012-11-02T18:32:00.549,ns_1@10.2.1.67:<0.7805.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/316">> [ns_server:info,2012-11-02T18:32:01.112,ns_1@10.2.1.67:ns_config_rep<0.1439.0>:ns_config_rep:do_pull:341]Pulling config from: 'ns_1@10.2.1.68' [ns_server:info,2012-11-02T18:32:01.736,ns_1@10.2.1.67:<0.7832.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/317">> [ns_server:info,2012-11-02T18:32:01.799,ns_1@10.2.1.67:<0.7843.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/318">> [ns_server:info,2012-11-02T18:32:01.863,ns_1@10.2.1.67:<0.7854.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/319">> [ns_server:info,2012-11-02T18:32:01.924,ns_1@10.2.1.67:<0.7865.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/320">> [ns_server:info,2012-11-02T18:32:01.999,ns_1@10.2.1.67:<0.7876.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/321">> [ns_server:info,2012-11-02T18:32:02.073,ns_1@10.2.1.67:<0.7887.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/322">> [ns_server:info,2012-11-02T18:32:02.140,ns_1@10.2.1.67:<0.7898.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/323">> [ns_server:info,2012-11-02T18:32:02.220,ns_1@10.2.1.67:<0.7909.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/324">> [ns_server:info,2012-11-02T18:32:02.306,ns_1@10.2.1.67:<0.7920.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/325">> [ns_server:info,2012-11-02T18:32:02.385,ns_1@10.2.1.67:<0.7931.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/326">> [ns_server:info,2012-11-02T18:32:02.465,ns_1@10.2.1.67:<0.7942.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/327">> [ns_server:info,2012-11-02T18:32:02.547,ns_1@10.2.1.67:<0.7953.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/328">> [ns_server:info,2012-11-02T18:32:02.625,ns_1@10.2.1.67:<0.7964.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/329">> [ns_server:info,2012-11-02T18:32:02.695,ns_1@10.2.1.67:<0.7980.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/330">> [ns_server:info,2012-11-02T18:32:02.767,ns_1@10.2.1.67:<0.7991.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/331">> [ns_server:info,2012-11-02T18:32:02.842,ns_1@10.2.1.67:<0.8002.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/332">> [ns_server:info,2012-11-02T18:32:02.900,ns_1@10.2.1.67:<0.8013.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/333">> [ns_server:info,2012-11-02T18:32:02.955,ns_1@10.2.1.67:<0.8024.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/334">> [ns_server:info,2012-11-02T18:32:03.024,ns_1@10.2.1.67:<0.8035.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/335">> [ns_server:info,2012-11-02T18:32:03.087,ns_1@10.2.1.67:<0.8046.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/336">> [ns_server:info,2012-11-02T18:32:06.473,ns_1@10.2.1.67:<0.8072.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/337">> [ns_server:info,2012-11-02T18:32:07.751,ns_1@10.2.1.67:<0.8089.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/338">> [ns_server:info,2012-11-02T18:32:07.838,ns_1@10.2.1.67:<0.8100.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/339">> [ns_server:info,2012-11-02T18:32:07.920,ns_1@10.2.1.67:<0.8111.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/340">> [ns_server:info,2012-11-02T18:32:07.987,ns_1@10.2.1.67:<0.8122.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/341">> [ns_server:info,2012-11-02T18:32:08.056,ns_1@10.2.1.67:<0.8133.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/342">> [ns_server:info,2012-11-02T18:32:08.134,ns_1@10.2.1.67:<0.8144.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/343">> [ns_server:info,2012-11-02T18:32:08.215,ns_1@10.2.1.67:<0.8155.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/344">> [ns_server:info,2012-11-02T18:32:08.296,ns_1@10.2.1.67:<0.8166.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/345">> [ns_server:info,2012-11-02T18:32:08.386,ns_1@10.2.1.67:<0.8177.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/346">> [ns_server:info,2012-11-02T18:32:08.465,ns_1@10.2.1.67:<0.8188.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/347">> [ns_server:info,2012-11-02T18:32:08.551,ns_1@10.2.1.67:<0.8199.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/348">> [ns_server:info,2012-11-02T18:32:08.629,ns_1@10.2.1.67:<0.8210.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/349">> [ns_server:info,2012-11-02T18:32:08.700,ns_1@10.2.1.67:<0.8226.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/350">> [ns_server:info,2012-11-02T18:32:08.760,ns_1@10.2.1.67:<0.8237.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/351">> [ns_server:info,2012-11-02T18:32:08.827,ns_1@10.2.1.67:<0.8248.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/352">> [ns_server:info,2012-11-02T18:32:08.895,ns_1@10.2.1.67:<0.8259.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/353">> [ns_server:info,2012-11-02T18:32:08.985,ns_1@10.2.1.67:<0.8270.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/354">> [ns_server:info,2012-11-02T18:32:09.070,ns_1@10.2.1.67:<0.8281.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/355">> [ns_server:info,2012-11-02T18:32:09.554,ns_1@10.2.1.67:<0.8293.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/356">> [ns_server:info,2012-11-02T18:32:11.512,ns_1@10.2.1.67:<0.8342.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/357">> [ns_server:info,2012-11-02T18:32:11.588,ns_1@10.2.1.67:<0.8353.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/358">> [ns_server:info,2012-11-02T18:32:11.665,ns_1@10.2.1.67:<0.8364.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/359">> [ns_server:info,2012-11-02T18:32:11.739,ns_1@10.2.1.67:<0.8375.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/360">> [ns_server:info,2012-11-02T18:32:11.801,ns_1@10.2.1.67:<0.8386.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/361">> [ns_server:info,2012-11-02T18:32:11.881,ns_1@10.2.1.67:<0.8397.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/362">> [ns_server:info,2012-11-02T18:32:11.964,ns_1@10.2.1.67:<0.8408.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/363">> [ns_server:info,2012-11-02T18:32:12.055,ns_1@10.2.1.67:<0.8419.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/364">> [ns_server:info,2012-11-02T18:32:12.119,ns_1@10.2.1.67:<0.8430.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/365">> [ns_server:info,2012-11-02T18:32:12.199,ns_1@10.2.1.67:<0.8441.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/366">> [ns_server:info,2012-11-02T18:32:12.256,ns_1@10.2.1.67:<0.8452.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/367">> [ns_server:info,2012-11-02T18:32:12.326,ns_1@10.2.1.67:<0.8463.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/368">> [ns_server:info,2012-11-02T18:32:12.392,ns_1@10.2.1.67:<0.8474.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/369">> [ns_server:info,2012-11-02T18:32:12.454,ns_1@10.2.1.67:<0.8485.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/370">> [ns_server:info,2012-11-02T18:32:12.529,ns_1@10.2.1.67:<0.8496.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/371">> [ns_server:info,2012-11-02T18:32:12.608,ns_1@10.2.1.67:<0.8507.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/372">> [ns_server:info,2012-11-02T18:32:12.694,ns_1@10.2.1.67:<0.8523.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/373">> [ns_server:info,2012-11-02T18:32:12.771,ns_1@10.2.1.67:<0.8534.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/374">> [ns_server:info,2012-11-02T18:32:12.843,ns_1@10.2.1.67:<0.8545.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/375">> [ns_server:info,2012-11-02T18:32:15.295,ns_1@10.2.1.67:<0.8563.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/376">> [ns_server:info,2012-11-02T18:32:15.364,ns_1@10.2.1.67:<0.8574.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/377">> [ns_server:info,2012-11-02T18:32:15.443,ns_1@10.2.1.67:<0.8585.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/378">> [ns_server:info,2012-11-02T18:32:15.514,ns_1@10.2.1.67:<0.8596.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/379">> [ns_server:info,2012-11-02T18:32:15.581,ns_1@10.2.1.67:<0.8607.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/380">> [ns_server:info,2012-11-02T18:32:15.651,ns_1@10.2.1.67:<0.8618.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/381">> [ns_server:info,2012-11-02T18:32:15.727,ns_1@10.2.1.67:<0.8637.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/382">> [ns_server:info,2012-11-02T18:32:15.807,ns_1@10.2.1.67:<0.8648.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/383">> [ns_server:info,2012-11-02T18:32:15.882,ns_1@10.2.1.67:<0.7032.0>:compaction_daemon:spawn_dbs_compactor:576]Finished compaction of databases for bucket default [ns_server:info,2012-11-02T18:32:15.884,ns_1@10.2.1.67:<0.8659.0>:compaction_daemon:try_to_cleanup_indexes:439]Cleaning up indexes for bucket `default` [ns_server:info,2012-11-02T18:32:15.884,ns_1@10.2.1.67:<0.8659.0>:compaction_daemon:spawn_bucket_compactor:404]Compacting bucket default with config: [{parallel_db_and_view_compaction,false}, {database_fragmentation_threshold,{30,undefined}}, {view_fragmentation_threshold,{100,undefined}}] [ns_server:info,2012-11-02T18:32:45.891,ns_1@10.2.1.67:<0.8802.0>:compaction_daemon:try_to_cleanup_indexes:439]Cleaning up indexes for bucket `default` [ns_server:info,2012-11-02T18:32:45.891,ns_1@10.2.1.67:<0.8802.0>:compaction_daemon:spawn_bucket_compactor:404]Compacting bucket default with config: [{parallel_db_and_view_compaction,false}, {database_fragmentation_threshold,{30,undefined}}, {view_fragmentation_threshold,{100,undefined}}] [ns_server:info,2012-11-02T18:32:54.298,ns_1@10.2.1.67:ns_config_rep<0.1439.0>:ns_config_rep:do_pull:341]Pulling config from: 'ns_1@10.2.1.66' [ns_server:info,2012-11-02T18:33:15.474,ns_1@10.2.1.67:ns_config_rep<0.1439.0>:ns_config_rep:do_pull:341]Pulling config from: 'ns_1@10.2.1.65' [ns_server:info,2012-11-02T18:33:15.897,ns_1@10.2.1.67:<0.8952.0>:compaction_daemon:try_to_cleanup_indexes:439]Cleaning up indexes for bucket `default` [ns_server:info,2012-11-02T18:33:15.898,ns_1@10.2.1.67:<0.8952.0>:compaction_daemon:spawn_bucket_compactor:404]Compacting bucket default with config: [{parallel_db_and_view_compaction,false}, {database_fragmentation_threshold,{30,undefined}}, {view_fragmentation_threshold,{100,undefined}}] [ns_server:info,2012-11-02T18:33:24.568,ns_1@10.2.1.67:ns_config_rep<0.1439.0>:ns_config_rep:do_pull:341]Pulling config from: 'ns_1@10.2.1.65' [ns_server:info,2012-11-02T18:33:45.906,ns_1@10.2.1.67:<0.9097.0>:compaction_daemon:try_to_cleanup_indexes:439]Cleaning up indexes for bucket `default` [ns_server:info,2012-11-02T18:33:45.906,ns_1@10.2.1.67:<0.9097.0>:compaction_daemon:spawn_bucket_compactor:404]Compacting bucket default with config: [{parallel_db_and_view_compaction,false}, {database_fragmentation_threshold,{30,undefined}}, {view_fragmentation_threshold,{100,undefined}}] [ns_server:info,2012-11-02T18:34:04.999,ns_1@10.2.1.67:ns_config_rep<0.1439.0>:ns_config_rep:do_pull:341]Pulling config from: 'ns_1@10.2.1.68' [ns_server:info,2012-11-02T18:34:10.094,ns_1@10.2.1.67:ns_config_rep<0.1439.0>:ns_config_rep:do_pull:341]Pulling config from: 'ns_1@10.2.1.66' [ns_server:info,2012-11-02T18:34:15.914,ns_1@10.2.1.67:<0.9247.0>:compaction_daemon:try_to_cleanup_indexes:439]Cleaning up indexes for bucket `default` [ns_server:info,2012-11-02T18:34:15.914,ns_1@10.2.1.67:<0.9247.0>:compaction_daemon:spawn_bucket_compactor:404]Compacting bucket default with config: [{parallel_db_and_view_compaction,false}, {database_fragmentation_threshold,{30,undefined}}, {view_fragmentation_threshold,{100,undefined}}] [ns_server:info,2012-11-02T18:34:37.838,ns_1@10.2.1.67:ns_config_rep<0.1439.0>:ns_config_rep:do_pull:341]Pulling config from: 'ns_1@10.2.1.65' [ns_server:info,2012-11-02T18:34:45.922,ns_1@10.2.1.67:<0.9392.0>:compaction_daemon:try_to_cleanup_indexes:439]Cleaning up indexes for bucket `default` [ns_server:info,2012-11-02T18:34:45.923,ns_1@10.2.1.67:<0.9392.0>:compaction_daemon:spawn_bucket_compactor:404]Compacting bucket default with config: [{parallel_db_and_view_compaction,false}, {database_fragmentation_threshold,{30,undefined}}, {view_fragmentation_threshold,{100,undefined}}] [ns_server:info,2012-11-02T18:35:15.931,ns_1@10.2.1.67:<0.9563.0>:compaction_daemon:try_to_cleanup_indexes:439]Cleaning up indexes for bucket `default` [ns_server:info,2012-11-02T18:35:15.932,ns_1@10.2.1.67:<0.9563.0>:compaction_daemon:spawn_bucket_compactor:404]Compacting bucket default with config: [{parallel_db_and_view_compaction,false}, {database_fragmentation_threshold,{30,undefined}}, {view_fragmentation_threshold,{100,undefined}}] [ns_server:info,2012-11-02T18:35:15.937,ns_1@10.2.1.67:<0.9566.0>:compaction_daemon:spawn_dbs_compactor:544]Compacting databases for bucket default [ns_server:info,2012-11-02T18:35:15.937,ns_1@10.2.1.67:<0.9567.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/master">> [ns_server:info,2012-11-02T18:35:16.453,ns_1@10.2.1.67:<0.9578.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/256">> [ns_server:info,2012-11-02T18:35:19.395,ns_1@10.2.1.67:<0.9600.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/257">> [ns_server:info,2012-11-02T18:35:19.602,ns_1@10.2.1.67:<0.9620.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/258">> [ns_server:info,2012-11-02T18:35:19.765,ns_1@10.2.1.67:<0.9632.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/259">> [ns_server:info,2012-11-02T18:35:19.952,ns_1@10.2.1.67:<0.9643.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/260">> [ns_server:info,2012-11-02T18:35:20.094,ns_1@10.2.1.67:<0.9654.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/261">> [ns_server:info,2012-11-02T18:35:20.239,ns_1@10.2.1.67:<0.9665.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/262">> [ns_server:info,2012-11-02T18:35:21.203,ns_1@10.2.1.67:<0.9689.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/263">> [ns_server:info,2012-11-02T18:35:25.407,ns_1@10.2.1.67:<0.9712.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/264">> [ns_server:info,2012-11-02T18:35:25.563,ns_1@10.2.1.67:<0.9723.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/265">> [ns_server:info,2012-11-02T18:35:25.717,ns_1@10.2.1.67:<0.9743.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/266">> [ns_server:info,2012-11-02T18:35:25.859,ns_1@10.2.1.67:<0.9754.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/267">> [ns_server:info,2012-11-02T18:35:26.000,ns_1@10.2.1.67:<0.9765.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/268">> [ns_server:info,2012-11-02T18:35:26.147,ns_1@10.2.1.67:<0.9776.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/269">> [ns_server:info,2012-11-02T18:35:28.622,ns_1@10.2.1.67:<0.9792.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/270">> [ns_server:info,2012-11-02T18:35:33.205,ns_1@10.2.1.67:<0.9829.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/271">> [ns_server:info,2012-11-02T18:35:33.350,ns_1@10.2.1.67:<0.9840.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/272">> [ns_server:info,2012-11-02T18:35:33.498,ns_1@10.2.1.67:<0.9851.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/273">> [ns_server:info,2012-11-02T18:35:33.639,ns_1@10.2.1.67:<0.9862.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/274">> [ns_server:info,2012-11-02T18:35:33.833,ns_1@10.2.1.67:<0.9873.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/275">> [ns_server:info,2012-11-02T18:35:33.980,ns_1@10.2.1.67:<0.9884.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/276">> [ns_server:info,2012-11-02T18:35:34.221,ns_1@10.2.1.67:ns_config_rep<0.1439.0>:ns_config_rep:do_pull:341]Pulling config from: 'ns_1@10.2.1.65' [ns_server:info,2012-11-02T18:35:34.967,ns_1@10.2.1.67:<0.9904.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/277">> [ns_server:info,2012-11-02T18:35:39.892,ns_1@10.2.1.67:<0.9935.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/278">> [ns_server:info,2012-11-02T18:35:40.043,ns_1@10.2.1.67:<0.9946.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/279">> [ns_server:info,2012-11-02T18:35:40.179,ns_1@10.2.1.67:<0.9957.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/280">> [ns_server:info,2012-11-02T18:35:40.359,ns_1@10.2.1.67:<0.9968.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/281">> [ns_server:info,2012-11-02T18:35:40.564,ns_1@10.2.1.67:<0.9979.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/282">> [ns_server:info,2012-11-02T18:35:40.780,ns_1@10.2.1.67:<0.10004.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/283">> [ns_server:info,2012-11-02T18:35:40.990,ns_1@10.2.1.67:<0.10015.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/284">> [ns_server:info,2012-11-02T18:35:43.206,ns_1@10.2.1.67:<0.10031.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/285">> [ns_server:info,2012-11-02T18:35:43.405,ns_1@10.2.1.67:<0.10042.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/286">> [ns_server:info,2012-11-02T18:35:43.574,ns_1@10.2.1.67:<0.10053.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/287">> [ns_server:info,2012-11-02T18:35:43.759,ns_1@10.2.1.67:<0.10065.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/288">> [ns_server:info,2012-11-02T18:35:43.952,ns_1@10.2.1.67:<0.10076.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/289">> [ns_server:info,2012-11-02T18:35:44.110,ns_1@10.2.1.67:<0.10087.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/290">> [ns_server:info,2012-11-02T18:35:44.317,ns_1@10.2.1.67:<0.10098.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/291">> [ns_server:info,2012-11-02T18:35:47.408,ns_1@10.2.1.67:<0.10129.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/292">> [ns_server:info,2012-11-02T18:35:48.543,ns_1@10.2.1.67:<0.10140.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/293">> [ns_server:info,2012-11-02T18:35:48.738,ns_1@10.2.1.67:<0.10156.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/294">> [ns_server:info,2012-11-02T18:35:48.956,ns_1@10.2.1.67:<0.10167.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/295">> [ns_server:info,2012-11-02T18:35:49.125,ns_1@10.2.1.67:<0.10178.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/296">> [ns_server:info,2012-11-02T18:35:50.063,ns_1@10.2.1.67:<0.10191.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/297">> [ns_server:info,2012-11-02T18:35:50.232,ns_1@10.2.1.67:<0.10202.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/298">> [ns_server:info,2012-11-02T18:35:52.392,ns_1@10.2.1.67:<0.10226.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/299">> [ns_server:info,2012-11-02T18:35:52.547,ns_1@10.2.1.67:<0.10237.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/300">> [ns_server:info,2012-11-02T18:35:52.699,ns_1@10.2.1.67:<0.10254.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/301">> [ns_server:info,2012-11-02T18:35:52.848,ns_1@10.2.1.67:<0.10265.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/302">> [ns_server:info,2012-11-02T18:35:53.015,ns_1@10.2.1.67:<0.10276.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/303">> [ns_server:info,2012-11-02T18:35:53.179,ns_1@10.2.1.67:<0.10287.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/304">> [ns_server:info,2012-11-02T18:35:53.377,ns_1@10.2.1.67:<0.10298.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/305">> [ns_server:info,2012-11-02T18:35:57.441,ns_1@10.2.1.67:<0.10329.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/306">> [ns_server:info,2012-11-02T18:35:57.674,ns_1@10.2.1.67:<0.10348.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/307">> [ns_server:info,2012-11-02T18:35:57.842,ns_1@10.2.1.67:<0.10359.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/308">> [ns_server:info,2012-11-02T18:35:58.069,ns_1@10.2.1.67:<0.10370.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/309">> [ns_server:info,2012-11-02T18:35:58.248,ns_1@10.2.1.67:<0.10381.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/310">> [ns_server:info,2012-11-02T18:35:58.438,ns_1@10.2.1.67:<0.10392.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/311">> [ns_server:info,2012-11-02T18:36:02.473,ns_1@10.2.1.67:<0.10424.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/312">> [ns_server:info,2012-11-02T18:36:02.642,ns_1@10.2.1.67:<0.10435.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/313">> [ns_server:info,2012-11-02T18:36:02.868,ns_1@10.2.1.67:<0.10451.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/314">> [ns_server:info,2012-11-02T18:36:03.105,ns_1@10.2.1.67:<0.10462.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/315">> [ns_server:info,2012-11-02T18:36:03.324,ns_1@10.2.1.67:<0.10473.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/316">> [ns_server:info,2012-11-02T18:36:03.554,ns_1@10.2.1.67:<0.10484.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/317">> [ns_server:info,2012-11-02T18:36:04.515,ns_1@10.2.1.67:<0.10496.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/318">> [ns_server:info,2012-11-02T18:36:05.810,ns_1@10.2.1.67:<0.10521.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/319">> [ns_server:info,2012-11-02T18:36:06.013,ns_1@10.2.1.67:<0.10532.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/320">> [ns_server:info,2012-11-02T18:36:06.217,ns_1@10.2.1.67:<0.10543.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/321">> [ns_server:info,2012-11-02T18:36:06.376,ns_1@10.2.1.67:<0.10554.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/322">> [ns_server:info,2012-11-02T18:36:06.548,ns_1@10.2.1.67:<0.10565.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/323">> [ns_server:info,2012-11-02T18:36:06.761,ns_1@10.2.1.67:<0.10581.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/324">> [ns_server:info,2012-11-02T18:36:06.965,ns_1@10.2.1.67:<0.10592.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/325">> [ns_server:info,2012-11-02T18:36:11.945,ns_1@10.2.1.67:<0.10627.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/326">> [ns_server:info,2012-11-02T18:36:12.190,ns_1@10.2.1.67:<0.10638.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/327">> [ns_server:info,2012-11-02T18:36:12.349,ns_1@10.2.1.67:<0.10649.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/328">> [ns_server:info,2012-11-02T18:36:12.508,ns_1@10.2.1.67:<0.10660.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/329">> [ns_server:info,2012-11-02T18:36:13.551,ns_1@10.2.1.67:<0.10676.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/330">> [ns_server:info,2012-11-02T18:36:13.732,ns_1@10.2.1.67:<0.10688.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/331">> [ns_server:info,2012-11-02T18:36:14.193,ns_1@10.2.1.67:<0.10699.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/332">> [ns_server:info,2012-11-02T18:36:16.616,ns_1@10.2.1.67:<0.10724.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/333">> [ns_server:info,2012-11-02T18:36:16.826,ns_1@10.2.1.67:<0.10741.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/334">> [ns_server:info,2012-11-02T18:36:17.035,ns_1@10.2.1.67:<0.10752.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/335">> [ns_server:info,2012-11-02T18:36:17.243,ns_1@10.2.1.67:<0.10763.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/336">> [ns_server:info,2012-11-02T18:36:17.453,ns_1@10.2.1.67:<0.10774.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/337">> [ns_server:info,2012-11-02T18:36:17.613,ns_1@10.2.1.67:<0.10785.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/338">> [ns_server:info,2012-11-02T18:36:17.824,ns_1@10.2.1.67:<0.10796.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/339">> [ns_server:info,2012-11-02T18:36:20.971,ns_1@10.2.1.67:<0.10827.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/340">> [ns_server:info,2012-11-02T18:36:21.172,ns_1@10.2.1.67:<0.10838.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/341">> [ns_server:info,2012-11-02T18:36:21.380,ns_1@10.2.1.67:<0.10849.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/342">> [ns_server:info,2012-11-02T18:36:21.587,ns_1@10.2.1.67:<0.10860.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/343">> [ns_server:info,2012-11-02T18:36:21.790,ns_1@10.2.1.67:<0.10871.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/344">> [ns_server:info,2012-11-02T18:36:22.019,ns_1@10.2.1.67:<0.10882.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/345">> [ns_server:info,2012-11-02T18:36:22.230,ns_1@10.2.1.67:<0.10893.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/346">> [ns_server:info,2012-11-02T18:36:26.170,ns_1@10.2.1.67:<0.10925.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/347">> [ns_server:info,2012-11-02T18:36:26.394,ns_1@10.2.1.67:<0.10936.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/348">> [ns_server:info,2012-11-02T18:36:26.640,ns_1@10.2.1.67:<0.10947.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/349">> [ns_server:info,2012-11-02T18:36:26.687,ns_1@10.2.1.67:ns_config_rep<0.1439.0>:ns_config_rep:do_pull:341]Pulling config from: 'ns_1@10.2.1.66' [ns_server:info,2012-11-02T18:36:26.794,ns_1@10.2.1.67:<0.10965.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/350">> [ns_server:info,2012-11-02T18:36:26.980,ns_1@10.2.1.67:<0.10976.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/351">> [ns_server:info,2012-11-02T18:36:27.210,ns_1@10.2.1.67:<0.10987.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/352">> [ns_server:info,2012-11-02T18:36:27.593,ns_1@10.2.1.67:<0.10998.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/353">> [ns_server:info,2012-11-02T18:36:27.793,ns_1@10.2.1.67:<0.11009.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/354">> [ns_server:info,2012-11-02T18:36:27.955,ns_1@10.2.1.67:<0.11020.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/355">> [ns_server:info,2012-11-02T18:36:28.172,ns_1@10.2.1.67:<0.11031.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/356">> [ns_server:info,2012-11-02T18:36:28.357,ns_1@10.2.1.67:<0.11042.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/357">> [ns_server:info,2012-11-02T18:36:28.567,ns_1@10.2.1.67:<0.11053.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/358">> [ns_server:info,2012-11-02T18:36:28.767,ns_1@10.2.1.67:<0.11070.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/359">> [ns_server:info,2012-11-02T18:36:29.936,ns_1@10.2.1.67:<0.11082.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/360">> [ns_server:info,2012-11-02T18:36:30.138,ns_1@10.2.1.67:<0.11093.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/361">> [ns_server:info,2012-11-02T18:36:30.343,ns_1@10.2.1.67:<0.11104.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/362">> [ns_server:info,2012-11-02T18:36:30.549,ns_1@10.2.1.67:<0.11115.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/363">> [ns_server:info,2012-11-02T18:36:30.756,ns_1@10.2.1.67:<0.11139.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/364">> [ns_server:info,2012-11-02T18:36:30.956,ns_1@10.2.1.67:<0.11150.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/365">> [ns_server:info,2012-11-02T18:36:34.199,ns_1@10.2.1.67:<0.11167.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/366">> [ns_server:info,2012-11-02T18:36:34.413,ns_1@10.2.1.67:<0.11178.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/367">> [ns_server:info,2012-11-02T18:36:34.595,ns_1@10.2.1.67:<0.11190.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/368">> [ns_server:info,2012-11-02T18:36:34.943,ns_1@10.2.1.67:<0.11207.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/369">> [ns_server:info,2012-11-02T18:36:35.145,ns_1@10.2.1.67:<0.11218.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/370">> [ns_server:info,2012-11-02T18:36:35.358,ns_1@10.2.1.67:<0.11229.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/371">> [ns_server:info,2012-11-02T18:36:35.986,ns_1@10.2.1.67:<0.11248.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/372">> [ns_server:info,2012-11-02T18:36:36.197,ns_1@10.2.1.67:<0.11259.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/373">> [ns_server:info,2012-11-02T18:36:36.409,ns_1@10.2.1.67:<0.11270.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/374">> [ns_server:info,2012-11-02T18:36:36.616,ns_1@10.2.1.67:<0.11281.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/375">> [ns_server:info,2012-11-02T18:36:36.803,ns_1@10.2.1.67:<0.11297.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/376">> [ns_server:info,2012-11-02T18:36:36.991,ns_1@10.2.1.67:<0.11308.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/377">> [ns_server:info,2012-11-02T18:36:37.178,ns_1@10.2.1.67:<0.11319.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/378">> [ns_server:info,2012-11-02T18:36:39.295,ns_1@10.2.1.67:<0.11336.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/379">> [ns_server:info,2012-11-02T18:36:39.495,ns_1@10.2.1.67:<0.11347.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/380">> [ns_server:info,2012-11-02T18:36:39.671,ns_1@10.2.1.67:<0.11359.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/381">> [ns_server:info,2012-11-02T18:36:39.868,ns_1@10.2.1.67:<0.11370.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/382">> [ns_server:info,2012-11-02T18:36:40.071,ns_1@10.2.1.67:<0.11381.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/383">> [ns_server:info,2012-11-02T18:36:40.287,ns_1@10.2.1.67:<0.9566.0>:compaction_daemon:spawn_dbs_compactor:576]Finished compaction of databases for bucket default [ns_server:info,2012-11-02T18:36:40.292,ns_1@10.2.1.67:<0.11392.0>:compaction_daemon:try_to_cleanup_indexes:439]Cleaning up indexes for bucket `default` [ns_server:info,2012-11-02T18:36:40.292,ns_1@10.2.1.67:<0.11392.0>:compaction_daemon:spawn_bucket_compactor:404]Compacting bucket default with config: [{parallel_db_and_view_compaction,false}, {database_fragmentation_threshold,{30,undefined}}, {view_fragmentation_threshold,{100,undefined}}] [ns_server:info,2012-11-02T18:37:03.755,ns_1@10.2.1.67:ns_config_rep<0.1439.0>:ns_config_rep:do_pull:341]Pulling config from: 'ns_1@10.2.1.65' [ns_server:info,2012-11-02T18:37:10.298,ns_1@10.2.1.67:<0.11537.0>:compaction_daemon:try_to_cleanup_indexes:439]Cleaning up indexes for bucket `default` [ns_server:info,2012-11-02T18:37:10.300,ns_1@10.2.1.67:<0.11537.0>:compaction_daemon:spawn_bucket_compactor:404]Compacting bucket default with config: [{parallel_db_and_view_compaction,false}, {database_fragmentation_threshold,{30,undefined}}, {view_fragmentation_threshold,{100,undefined}}] [ns_server:info,2012-11-02T18:37:17.877,ns_1@10.2.1.67:ns_config_rep<0.1439.0>:ns_config_rep:do_pull:341]Pulling config from: 'ns_1@10.2.1.68' [ns_server:info,2012-11-02T18:37:40.306,ns_1@10.2.1.67:<0.11685.0>:compaction_daemon:try_to_cleanup_indexes:439]Cleaning up indexes for bucket `default` [ns_server:info,2012-11-02T18:37:40.307,ns_1@10.2.1.67:<0.11685.0>:compaction_daemon:spawn_bucket_compactor:404]Compacting bucket default with config: [{parallel_db_and_view_compaction,false}, {database_fragmentation_threshold,{30,undefined}}, {view_fragmentation_threshold,{100,undefined}}] [ns_server:info,2012-11-02T18:37:41.227,ns_1@10.2.1.67:ns_config_rep<0.1439.0>:ns_config_rep:do_pull:341]Pulling config from: 'ns_1@10.2.1.68' [ns_server:info,2012-11-02T18:38:10.313,ns_1@10.2.1.67:<0.11851.0>:compaction_daemon:try_to_cleanup_indexes:439]Cleaning up indexes for bucket `default` [ns_server:info,2012-11-02T18:38:10.313,ns_1@10.2.1.67:<0.11851.0>:compaction_daemon:spawn_bucket_compactor:404]Compacting bucket default with config: [{parallel_db_and_view_compaction,false}, {database_fragmentation_threshold,{30,undefined}}, {view_fragmentation_threshold,{100,undefined}}] [ns_server:info,2012-11-02T18:38:18.375,ns_1@10.2.1.67:ns_config_rep<0.1439.0>:ns_config_rep:do_pull:341]Pulling config from: 'ns_1@10.2.1.65' [ns_server:info,2012-11-02T18:38:40.320,ns_1@10.2.1.67:<0.11999.0>:compaction_daemon:try_to_cleanup_indexes:439]Cleaning up indexes for bucket `default` [ns_server:info,2012-11-02T18:38:40.321,ns_1@10.2.1.67:<0.11999.0>:compaction_daemon:spawn_bucket_compactor:404]Compacting bucket default with config: [{parallel_db_and_view_compaction,false}, {database_fragmentation_threshold,{30,undefined}}, {view_fragmentation_threshold,{100,undefined}}] [stats:warn,2012-11-02T18:38:59.866,ns_1@10.2.1.67:system_stats_collector<0.1530.0>:system_stats_collector:handle_info:133]lost 2 ticks [stats:warn,2012-11-02T18:38:59.870,ns_1@10.2.1.67:<0.1687.0>:stats_collector:latest_tick:223]Dropped 2 ticks [ns_server:info,2012-11-02T18:39:10.328,ns_1@10.2.1.67:<0.12142.0>:compaction_daemon:try_to_cleanup_indexes:439]Cleaning up indexes for bucket `default` [ns_server:info,2012-11-02T18:39:10.328,ns_1@10.2.1.67:<0.12142.0>:compaction_daemon:spawn_bucket_compactor:404]Compacting bucket default with config: [{parallel_db_and_view_compaction,false}, {database_fragmentation_threshold,{30,undefined}}, {view_fragmentation_threshold,{100,undefined}}] [ns_server:info,2012-11-02T18:39:17.705,ns_1@10.2.1.67:ns_config_rep<0.1439.0>:ns_config_rep:do_pull:341]Pulling config from: 'ns_1@10.2.1.65' [ns_server:info,2012-11-02T18:39:23.512,ns_1@10.2.1.67:ns_config_rep<0.1439.0>:ns_config_rep:do_pull:341]Pulling config from: 'ns_1@10.2.1.66' [ns_server:info,2012-11-02T18:39:29.868,ns_1@10.2.1.67:ns_config_rep<0.1439.0>:ns_config_rep:do_pull:341]Pulling config from: 'ns_1@10.2.1.65' [ns_server:info,2012-11-02T18:39:40.333,ns_1@10.2.1.67:<0.12294.0>:compaction_daemon:try_to_cleanup_indexes:439]Cleaning up indexes for bucket `default` [ns_server:info,2012-11-02T18:39:40.334,ns_1@10.2.1.67:<0.12294.0>:compaction_daemon:spawn_bucket_compactor:404]Compacting bucket default with config: [{parallel_db_and_view_compaction,false}, {database_fragmentation_threshold,{30,undefined}}, {view_fragmentation_threshold,{100,undefined}}] [stats:warn,2012-11-02T18:39:57.397,ns_1@10.2.1.67:system_stats_collector<0.1530.0>:system_stats_collector:handle_info:133]lost 2 ticks [stats:warn,2012-11-02T18:39:57.401,ns_1@10.2.1.67:<0.1687.0>:stats_collector:latest_tick:223]Dropped 2 ticks [ns_server:info,2012-11-02T18:40:08.876,ns_1@10.2.1.67:ns_config_rep<0.1439.0>:ns_config_rep:do_pull:341]Pulling config from: 'ns_1@10.2.1.68' [ns_server:info,2012-11-02T18:40:10.342,ns_1@10.2.1.67:<0.12439.0>:compaction_daemon:try_to_cleanup_indexes:439]Cleaning up indexes for bucket `default` [ns_server:info,2012-11-02T18:40:10.342,ns_1@10.2.1.67:<0.12439.0>:compaction_daemon:spawn_bucket_compactor:404]Compacting bucket default with config: [{parallel_db_and_view_compaction,false}, {database_fragmentation_threshold,{30,undefined}}, {view_fragmentation_threshold,{100,undefined}}] [ns_server:info,2012-11-02T18:40:40.350,ns_1@10.2.1.67:<0.12585.0>:compaction_daemon:try_to_cleanup_indexes:439]Cleaning up indexes for bucket `default` [ns_server:info,2012-11-02T18:40:40.350,ns_1@10.2.1.67:<0.12585.0>:compaction_daemon:spawn_bucket_compactor:404]Compacting bucket default with config: [{parallel_db_and_view_compaction,false}, {database_fragmentation_threshold,{30,undefined}}, {view_fragmentation_threshold,{100,undefined}}] [ns_server:info,2012-11-02T18:41:05.633,ns_1@10.2.1.67:ns_config_rep<0.1439.0>:ns_config_rep:do_pull:341]Pulling config from: 'ns_1@10.2.1.66' [ns_server:info,2012-11-02T18:41:10.357,ns_1@10.2.1.67:<0.12755.0>:compaction_daemon:try_to_cleanup_indexes:439]Cleaning up indexes for bucket `default` [ns_server:info,2012-11-02T18:41:10.357,ns_1@10.2.1.67:<0.12755.0>:compaction_daemon:spawn_bucket_compactor:404]Compacting bucket default with config: [{parallel_db_and_view_compaction,false}, {database_fragmentation_threshold,{30,undefined}}, {view_fragmentation_threshold,{100,undefined}}] [ns_server:info,2012-11-02T18:41:40.363,ns_1@10.2.1.67:<0.12901.0>:compaction_daemon:try_to_cleanup_indexes:439]Cleaning up indexes for bucket `default` [ns_server:info,2012-11-02T18:41:40.364,ns_1@10.2.1.67:<0.12901.0>:compaction_daemon:spawn_bucket_compactor:404]Compacting bucket default with config: [{parallel_db_and_view_compaction,false}, {database_fragmentation_threshold,{30,undefined}}, {view_fragmentation_threshold,{100,undefined}}] [ns_server:info,2012-11-02T18:41:54.563,ns_1@10.2.1.67:ns_config_rep<0.1439.0>:ns_config_rep:do_pull:341]Pulling config from: 'ns_1@10.2.1.65' [ns_server:info,2012-11-02T18:42:10.370,ns_1@10.2.1.67:<0.13046.0>:compaction_daemon:try_to_cleanup_indexes:439]Cleaning up indexes for bucket `default` [ns_server:info,2012-11-02T18:42:10.371,ns_1@10.2.1.67:<0.13046.0>:compaction_daemon:spawn_bucket_compactor:404]Compacting bucket default with config: [{parallel_db_and_view_compaction,false}, {database_fragmentation_threshold,{30,undefined}}, {view_fragmentation_threshold,{100,undefined}}] [ns_server:info,2012-11-02T18:42:21.467,ns_1@10.2.1.67:ns_config_rep<0.1439.0>:ns_config_rep:do_pull:341]Pulling config from: 'ns_1@10.2.1.65' [ns_server:info,2012-11-02T18:42:40.375,ns_1@10.2.1.67:<0.13194.0>:compaction_daemon:try_to_cleanup_indexes:439]Cleaning up indexes for bucket `default` [ns_server:info,2012-11-02T18:42:40.376,ns_1@10.2.1.67:<0.13194.0>:compaction_daemon:spawn_bucket_compactor:404]Compacting bucket default with config: [{parallel_db_and_view_compaction,false}, {database_fragmentation_threshold,{30,undefined}}, {view_fragmentation_threshold,{100,undefined}}] [ns_server:info,2012-11-02T18:42:56.845,ns_1@10.2.1.67:ns_config_rep<0.1439.0>:ns_config_rep:do_pull:341]Pulling config from: 'ns_1@10.2.1.65' [ns_server:info,2012-11-02T18:43:10.381,ns_1@10.2.1.67:<0.13339.0>:compaction_daemon:try_to_cleanup_indexes:439]Cleaning up indexes for bucket `default` [ns_server:info,2012-11-02T18:43:10.382,ns_1@10.2.1.67:<0.13339.0>:compaction_daemon:spawn_bucket_compactor:404]Compacting bucket default with config: [{parallel_db_and_view_compaction,false}, {database_fragmentation_threshold,{30,undefined}}, {view_fragmentation_threshold,{100,undefined}}] [ns_server:info,2012-11-02T18:43:40.388,ns_1@10.2.1.67:<0.13485.0>:compaction_daemon:try_to_cleanup_indexes:439]Cleaning up indexes for bucket `default` [ns_server:info,2012-11-02T18:43:40.389,ns_1@10.2.1.67:<0.13485.0>:compaction_daemon:spawn_bucket_compactor:404]Compacting bucket default with config: [{parallel_db_and_view_compaction,false}, {database_fragmentation_threshold,{30,undefined}}, {view_fragmentation_threshold,{100,undefined}}] [ns_server:info,2012-11-02T18:43:49.498,ns_1@10.2.1.67:ns_config_rep<0.1439.0>:ns_config_rep:do_pull:341]Pulling config from: 'ns_1@10.2.1.65' [ns_server:info,2012-11-02T18:44:10.397,ns_1@10.2.1.67:<0.13651.0>:compaction_daemon:try_to_cleanup_indexes:439]Cleaning up indexes for bucket `default` [ns_server:info,2012-11-02T18:44:10.397,ns_1@10.2.1.67:<0.13651.0>:compaction_daemon:spawn_bucket_compactor:404]Compacting bucket default with config: [{parallel_db_and_view_compaction,false}, {database_fragmentation_threshold,{30,undefined}}, {view_fragmentation_threshold,{100,undefined}}] [ns_server:info,2012-11-02T18:44:10.402,ns_1@10.2.1.67:<0.13654.0>:compaction_daemon:spawn_dbs_compactor:544]Compacting databases for bucket default [ns_server:info,2012-11-02T18:44:10.402,ns_1@10.2.1.67:<0.13655.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/master">> [ns_server:info,2012-11-02T18:44:10.825,ns_1@10.2.1.67:<0.13683.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/256">> [ns_server:info,2012-11-02T18:44:14.159,ns_1@10.2.1.67:<0.13700.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/257">> [ns_server:info,2012-11-02T18:44:14.556,ns_1@10.2.1.67:<0.13719.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/258">> [ns_server:info,2012-11-02T18:44:20.460,ns_1@10.2.1.67:<0.13757.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/259">> [ns_server:info,2012-11-02T18:44:20.871,ns_1@10.2.1.67:<0.13781.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/260">> [ns_server:info,2012-11-02T18:44:21.325,ns_1@10.2.1.67:<0.13792.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/261">> [ns_server:info,2012-11-02T18:44:25.475,ns_1@10.2.1.67:<0.13815.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/262">> [ns_server:info,2012-11-02T18:44:25.891,ns_1@10.2.1.67:<0.13835.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/263">> [ns_server:info,2012-11-02T18:44:25.897,ns_1@10.2.1.67:ns_config_rep<0.1439.0>:ns_config_rep:do_pull:341]Pulling config from: 'ns_1@10.2.1.65' [ns_server:info,2012-11-02T18:44:29.264,ns_1@10.2.1.67:<0.13859.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/264">> [ns_server:info,2012-11-02T18:44:29.678,ns_1@10.2.1.67:<0.13870.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/265">> [ns_server:info,2012-11-02T18:44:30.025,ns_1@10.2.1.67:<0.13882.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/266">> [ns_server:info,2012-11-02T18:44:33.860,ns_1@10.2.1.67:<0.13912.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/267">> [ns_server:info,2012-11-02T18:44:34.241,ns_1@10.2.1.67:<0.13923.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/268">> [ns_server:info,2012-11-02T18:44:34.607,ns_1@10.2.1.67:<0.13934.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/269">> [ns_server:info,2012-11-02T18:44:37.176,ns_1@10.2.1.67:<0.13965.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/270">> [ns_server:info,2012-11-02T18:44:39.218,ns_1@10.2.1.67:<0.13982.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/271">> [ns_server:info,2012-11-02T18:44:41.729,ns_1@10.2.1.67:<0.14008.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/272">> [ns_server:info,2012-11-02T18:44:43.650,ns_1@10.2.1.67:<0.14024.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/273">> [ns_server:info,2012-11-02T18:44:43.995,ns_1@10.2.1.67:<0.14036.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/274">> [ns_server:info,2012-11-02T18:44:47.463,ns_1@10.2.1.67:<0.14067.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/275">> [ns_server:info,2012-11-02T18:44:47.714,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:44:47.513789 PDT 3: Retry notify CouchDB of update, vbucket=274 rev=5 [ns_server:info,2012-11-02T18:44:48.482,ns_1@10.2.1.67:<0.14078.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/276">> [ns_server:info,2012-11-02T18:44:49.095,ns_1@10.2.1.67:<0.14102.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/277">> [ns_server:info,2012-11-02T18:44:53.278,ns_1@10.2.1.67:<0.14134.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/278">> [ns_server:info,2012-11-02T18:44:53.779,ns_1@10.2.1.67:<0.14145.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/279">> [ns_server:info,2012-11-02T18:44:54.580,ns_1@10.2.1.67:<0.14156.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/280">> [ns_server:info,2012-11-02T18:44:57.280,ns_1@10.2.1.67:<0.14187.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/281">> [ns_server:info,2012-11-02T18:44:57.652,ns_1@10.2.1.67:<0.14198.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/282">> [ns_server:info,2012-11-02T18:45:00.648,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:45:00.448184 PDT 3: 56 Closing connection due to read error: Connection reset by peer [ns_server:info,2012-11-02T18:45:00.918,ns_1@10.2.1.67:<0.14238.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/283">> [ns_server:info,2012-11-02T18:45:02.974,ns_1@10.2.1.67:<0.14255.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/284">> [ns_server:info,2012-11-02T18:45:04.253,ns_1@10.2.1.67:<0.14266.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/285">> [ns_server:info,2012-11-02T18:45:08.586,ns_1@10.2.1.67:<0.14302.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/286">> [ns_server:info,2012-11-02T18:45:08.963,ns_1@10.2.1.67:<0.14318.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/287">> [ns_server:info,2012-11-02T18:45:09.442,ns_1@10.2.1.67:<0.14329.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/288">> [ns_server:info,2012-11-02T18:45:13.488,ns_1@10.2.1.67:<0.14367.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/289">> [ns_server:info,2012-11-02T18:45:14.791,ns_1@10.2.1.67:ns_config_rep<0.1439.0>:ns_config_rep:do_pull:341]Pulling config from: 'ns_1@10.2.1.66' [ns_server:info,2012-11-02T18:45:14.867,ns_1@10.2.1.67:<0.14386.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/290">> [ns_server:info,2012-11-02T18:45:17.277,ns_1@10.2.1.67:<0.14416.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/291">> [ns_server:info,2012-11-02T18:45:18.507,ns_1@10.2.1.67:<0.14427.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/292">> [ns_server:info,2012-11-02T18:45:20.554,ns_1@10.2.1.67:<0.14445.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/293">> [ns_server:info,2012-11-02T18:45:24.812,ns_1@10.2.1.67:<0.14484.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/294">> [ns_server:info,2012-11-02T18:45:25.314,ns_1@10.2.1.67:<0.14496.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/295">> [ns_server:info,2012-11-02T18:45:25.688,ns_1@10.2.1.67:<0.14515.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/296">> [ns_server:info,2012-11-02T18:45:28.128,ns_1@10.2.1.67:<0.14536.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/297">> [ns_server:info,2012-11-02T18:45:29.412,ns_1@10.2.1.67:<0.14553.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/298">> [ns_server:info,2012-11-02T18:45:31.806,ns_1@10.2.1.67:<0.14579.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/299">> [ns_server:info,2012-11-02T18:45:33.597,ns_1@10.2.1.67:<0.14599.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/300">> [ns_server:info,2012-11-02T18:45:34.107,ns_1@10.2.1.67:<0.14610.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/301">> [ns_server:info,2012-11-02T18:45:34.990,ns_1@10.2.1.67:ns_config_rep<0.1439.0>:ns_config_rep:do_pull:341]Pulling config from: 'ns_1@10.2.1.68' [ns_server:info,2012-11-02T18:45:35.790,ns_1@10.2.1.67:<0.14638.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/302">> [ns_server:info,2012-11-02T18:45:37.849,ns_1@10.2.1.67:<0.14659.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/303">> [ns_server:info,2012-11-02T18:45:38.349,ns_1@10.2.1.67:<0.14670.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/304">> [ns_server:info,2012-11-02T18:45:41.600,ns_1@10.2.1.67:<0.14701.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/305">> [ns_server:info,2012-11-02T18:45:43.659,ns_1@10.2.1.67:<0.14721.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/306">> [ns_server:info,2012-11-02T18:45:44.197,ns_1@10.2.1.67:<0.14733.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/307">> [ns_server:info,2012-11-02T18:45:46.057,ns_1@10.2.1.67:ns_config_rep<0.1439.0>:ns_config_rep:do_pull:341]Pulling config from: 'ns_1@10.2.1.65' [ns_server:info,2012-11-02T18:45:47.516,ns_1@10.2.1.67:<0.14766.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/308">> [ns_server:info,2012-11-02T18:45:47.869,ns_1@10.2.1.67:<0.14777.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/309">> [ns_server:info,2012-11-02T18:45:50.271,ns_1@10.2.1.67:<0.14799.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/310">> [ns_server:info,2012-11-02T18:45:52.660,ns_1@10.2.1.67:<0.14823.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/311">> [ns_server:info,2012-11-02T18:45:53.008,ns_1@10.2.1.67:<0.14840.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/312">> [ns_server:info,2012-11-02T18:45:55.401,ns_1@10.2.1.67:<0.14861.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/313">> [ns_server:info,2012-11-02T18:45:57.455,ns_1@10.2.1.67:<0.14886.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/314">> [ns_server:info,2012-11-02T18:45:57.820,ns_1@10.2.1.67:<0.14897.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/315">> [ns_server:info,2012-11-02T18:46:00.365,ns_1@10.2.1.67:<0.14919.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/316">> [ns_server:info,2012-11-02T18:46:00.864,ns_1@10.2.1.67:<0.14943.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/317">> [ns_server:info,2012-11-02T18:46:04.145,ns_1@10.2.1.67:<0.14964.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/318">> [ns_server:info,2012-11-02T18:46:04.655,ns_1@10.2.1.67:<0.14975.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/319">> [ns_server:info,2012-11-02T18:46:05.019,ns_1@10.2.1.67:<0.14992.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/320">> [ns_server:info,2012-11-02T18:46:07.751,ns_1@10.2.1.67:<0.15018.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/321">> [ns_server:info,2012-11-02T18:46:08.690,ns_1@10.2.1.67:<0.15034.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/322">> [ns_server:info,2012-11-02T18:46:09.205,ns_1@10.2.1.67:<0.15049.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/323">> [ns_server:info,2012-11-02T18:46:12.620,ns_1@10.2.1.67:<0.15078.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/324">> [ns_server:info,2012-11-02T18:46:13.126,ns_1@10.2.1.67:<0.15094.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/325">> [ns_server:info,2012-11-02T18:46:13.637,ns_1@10.2.1.67:<0.15105.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/326">> [ns_server:info,2012-11-02T18:46:17.191,ns_1@10.2.1.67:<0.15141.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/327">> [ns_server:info,2012-11-02T18:46:17.545,ns_1@10.2.1.67:<0.15152.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/328">> [ns_server:info,2012-11-02T18:46:19.140,ns_1@10.2.1.67:<0.15168.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/329">> [ns_server:info,2012-11-02T18:46:22.022,ns_1@10.2.1.67:<0.15198.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/330">> [ns_server:info,2012-11-02T18:46:24.091,ns_1@10.2.1.67:<0.15215.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/331">> [ns_server:info,2012-11-02T18:46:26.690,ns_1@10.2.1.67:<0.15250.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/332">> [ns_server:info,2012-11-02T18:46:30.598,ns_1@10.2.1.67:<0.15268.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/333">> [ns_server:info,2012-11-02T18:46:31.090,ns_1@10.2.1.67:<0.15292.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/334">> [ns_server:info,2012-11-02T18:46:35.334,ns_1@10.2.1.67:<0.15316.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/335">> [ns_server:info,2012-11-02T18:46:35.824,ns_1@10.2.1.67:<0.15335.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/336">> [ns_server:info,2012-11-02T18:46:36.323,ns_1@10.2.1.67:<0.15346.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/337">> [ns_server:info,2012-11-02T18:46:36.558,ns_1@10.2.1.67:ns_config_rep<0.1439.0>:ns_config_rep:do_pull:341]Pulling config from: 'ns_1@10.2.1.65' [ns_server:info,2012-11-02T18:46:41.663,ns_1@10.2.1.67:<0.15385.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/338">> [ns_server:info,2012-11-02T18:46:43.723,ns_1@10.2.1.67:<0.15402.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/339">> [ns_server:info,2012-11-02T18:46:44.932,ns_1@10.2.1.67:<0.15418.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/340">> [ns_server:info,2012-11-02T18:46:45.287,ns_1@10.2.1.67:<0.15430.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/341">> [ns_server:info,2012-11-02T18:46:45.784,ns_1@10.2.1.67:<0.15449.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/342">> [ns_server:info,2012-11-02T18:46:46.510,ns_1@10.2.1.67:<0.15460.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/343">> [ns_server:info,2012-11-02T18:46:46.869,ns_1@10.2.1.67:<0.15477.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/344">> [ns_server:info,2012-11-02T18:46:47.361,ns_1@10.2.1.67:<0.15488.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/345">> [ns_server:info,2012-11-02T18:46:50.578,ns_1@10.2.1.67:<0.15506.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/346">> [ns_server:info,2012-11-02T18:46:50.962,ns_1@10.2.1.67:<0.15530.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/347">> [ns_server:info,2012-11-02T18:46:51.139,ns_1@10.2.1.67:ns_config_rep<0.1439.0>:ns_config_rep:do_pull:341]Pulling config from: 'ns_1@10.2.1.66' [ns_server:info,2012-11-02T18:46:56.950,ns_1@10.2.1.67:<0.15569.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/348">> [ns_server:info,2012-11-02T18:46:57.464,ns_1@10.2.1.67:<0.15580.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/349">> [ns_server:info,2012-11-02T18:46:57.831,ns_1@10.2.1.67:<0.15591.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/350">> [ns_server:info,2012-11-02T18:46:58.188,ns_1@10.2.1.67:<0.15602.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/351">> [ns_server:info,2012-11-02T18:46:58.528,ns_1@10.2.1.67:<0.15613.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/352">> [ns_server:info,2012-11-02T18:47:00.219,ns_1@10.2.1.67:<0.15631.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/353">> [ns_server:info,2012-11-02T18:47:03.865,ns_1@10.2.1.67:ns_config_rep<0.1439.0>:ns_config_rep:do_pull:341]Pulling config from: 'ns_1@10.2.1.68' [ns_server:info,2012-11-02T18:47:03.979,ns_1@10.2.1.67:<0.15663.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/354">> [ns_server:info,2012-11-02T18:47:06.027,ns_1@10.2.1.67:<0.15689.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/355">> [stats:warn,2012-11-02T18:47:09.551,ns_1@10.2.1.67:<0.1687.0>:stats_collector:latest_tick:223]Dropped 1 ticks [ns_server:info,2012-11-02T18:47:09.668,ns_1@10.2.1.67:<0.15711.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/356">> [ns_server:info,2012-11-02T18:47:12.344,ns_1@10.2.1.67:<0.15765.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/357">> [ns_server:info,2012-11-02T18:47:12.747,ns_1@10.2.1.67:<0.15781.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/358">> [ns_server:info,2012-11-02T18:47:16.909,ns_1@10.2.1.67:<0.15813.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/359">> [ns_server:info,2012-11-02T18:47:17.274,ns_1@10.2.1.67:<0.15824.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/360">> [ns_server:info,2012-11-02T18:47:20.697,ns_1@10.2.1.67:<0.15854.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/361">> [ns_server:info,2012-11-02T18:47:22.744,ns_1@10.2.1.67:<0.15872.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/362">> [ns_server:info,2012-11-02T18:47:23.249,ns_1@10.2.1.67:<0.15883.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/363">> [ns_server:info,2012-11-02T18:47:27.215,ns_1@10.2.1.67:<0.15914.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/364">> [ns_server:info,2012-11-02T18:47:27.617,ns_1@10.2.1.67:<0.15925.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/365">> [ns_server:info,2012-11-02T18:47:28.127,ns_1@10.2.1.67:<0.15936.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/366">> [ns_server:info,2012-11-02T18:47:31.398,ns_1@10.2.1.67:<0.15967.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/367">> [ns_server:info,2012-11-02T18:47:31.920,ns_1@10.2.1.67:<0.15979.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/368">> [ns_server:info,2012-11-02T18:47:32.437,ns_1@10.2.1.67:<0.15990.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/369">> [ns_server:info,2012-11-02T18:47:36.865,ns_1@10.2.1.67:<0.16026.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/370">> [ns_server:info,2012-11-02T18:47:37.407,ns_1@10.2.1.67:<0.16037.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/371">> [ns_server:info,2012-11-02T18:47:40.690,ns_1@10.2.1.67:<0.16068.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/372">> [ns_server:info,2012-11-02T18:47:42.732,ns_1@10.2.1.67:<0.16085.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/373">> [ns_server:info,2012-11-02T18:47:43.244,ns_1@10.2.1.67:<0.16096.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/374">> [ns_server:info,2012-11-02T18:47:45.336,ns_1@10.2.1.67:ns_config_rep<0.1439.0>:ns_config_rep:do_pull:341]Pulling config from: 'ns_1@10.2.1.66' [ns_server:info,2012-11-02T18:47:48.274,ns_1@10.2.1.67:<0.16130.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/375">> [ns_server:info,2012-11-02T18:47:48.775,ns_1@10.2.1.67:<0.16146.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/376">> [ns_server:info,2012-11-02T18:47:49.275,ns_1@10.2.1.67:<0.16157.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/377">> [ns_server:info,2012-11-02T18:47:52.790,ns_1@10.2.1.67:<0.16189.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/378">> [ns_server:info,2012-11-02T18:47:53.331,ns_1@10.2.1.67:<0.16200.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/379">> [ns_server:info,2012-11-02T18:47:53.878,ns_1@10.2.1.67:<0.16211.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/380">> [ns_server:info,2012-11-02T18:47:57.130,ns_1@10.2.1.67:<0.16242.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/381">> [ns_server:info,2012-11-02T18:47:57.536,ns_1@10.2.1.67:<0.16253.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/382">> [ns_server:info,2012-11-02T18:47:59.859,ns_1@10.2.1.67:<0.16270.0>:compaction_daemon:spawn_vbucket_compactor:639]Compacting <<"default/383">> [ns_server:info,2012-11-02T18:48:00.313,ns_1@10.2.1.67:ns_config_rep<0.1439.0>:ns_config_rep:do_pull:341]Pulling config from: 'ns_1@10.2.1.66' [ns_server:info,2012-11-02T18:48:01.107,ns_1@10.2.1.67:<0.13654.0>:compaction_daemon:spawn_dbs_compactor:576]Finished compaction of databases for bucket default [ns_server:info,2012-11-02T18:48:01.110,ns_1@10.2.1.67:<0.16297.0>:compaction_daemon:try_to_cleanup_indexes:439]Cleaning up indexes for bucket `default` [ns_server:info,2012-11-02T18:48:01.110,ns_1@10.2.1.67:<0.16297.0>:compaction_daemon:spawn_bucket_compactor:404]Compacting bucket default with config: [{parallel_db_and_view_compaction,false}, {database_fragmentation_threshold,{30,undefined}}, {view_fragmentation_threshold,{100,undefined}}] [ns_server:info,2012-11-02T18:48:09.450,ns_1@10.2.1.67:ns_config_rep<0.1439.0>:ns_config_rep:do_pull:341]Pulling config from: 'ns_1@10.2.1.68' [ns_server:info,2012-11-02T18:48:13.761,ns_1@10.2.1.67:capi_set_view_manager-default<0.1664.0>:capi_set_view_manager:handle_info:353]Processing update_ddoc _design/A (false) [ns_server:info,2012-11-02T18:48:17.026,ns_1@10.2.1.67:ns_config_rep<0.1439.0>:ns_config_rep:do_pull:341]Pulling config from: 'ns_1@10.2.1.68' [ns_server:info,2012-11-02T18:48:31.117,ns_1@10.2.1.67:<0.16491.0>:compaction_daemon:try_to_cleanup_indexes:439]Cleaning up indexes for bucket `default` [ns_server:info,2012-11-02T18:48:31.122,ns_1@10.2.1.67:<0.16491.0>:compaction_daemon:spawn_bucket_compactor:404]Compacting bucket default with config: [{parallel_db_and_view_compaction,false}, {database_fragmentation_threshold,{30,undefined}}, {view_fragmentation_threshold,{100,undefined}}] [ns_server:info,2012-11-02T18:49:01.128,ns_1@10.2.1.67:<0.16640.0>:compaction_daemon:try_to_cleanup_indexes:439]Cleaning up indexes for bucket `default` [ns_server:info,2012-11-02T18:49:01.129,ns_1@10.2.1.67:<0.16640.0>:compaction_daemon:spawn_bucket_compactor:404]Compacting bucket default with config: [{parallel_db_and_view_compaction,false}, {database_fragmentation_threshold,{30,undefined}}, {view_fragmentation_threshold,{100,undefined}}] [ns_server:info,2012-11-02T18:49:10.377,ns_1@10.2.1.67:ns_config_rep<0.1439.0>:ns_config_rep:do_pull:341]Pulling config from: 'ns_1@10.2.1.66' [ns_server:info,2012-11-02T18:49:31.133,ns_1@10.2.1.67:<0.16794.0>:compaction_daemon:try_to_cleanup_indexes:439]Cleaning up indexes for bucket `default` [ns_server:info,2012-11-02T18:49:31.134,ns_1@10.2.1.67:<0.16794.0>:compaction_daemon:spawn_bucket_compactor:404]Compacting bucket default with config: [{parallel_db_and_view_compaction,false}, {database_fragmentation_threshold,{30,undefined}}, {view_fragmentation_threshold,{100,undefined}}] [ns_server:info,2012-11-02T18:49:32.664,ns_1@10.2.1.67:ns_config_rep<0.1439.0>:ns_config_rep:do_pull:341]Pulling config from: 'ns_1@10.2.1.65' [ns_server:info,2012-11-02T18:50:01.139,ns_1@10.2.1.67:<0.16947.0>:compaction_daemon:try_to_cleanup_indexes:439]Cleaning up indexes for bucket `default` [ns_server:info,2012-11-02T18:50:01.139,ns_1@10.2.1.67:<0.16947.0>:compaction_daemon:spawn_bucket_compactor:404]Compacting bucket default with config: [{parallel_db_and_view_compaction,false}, {database_fragmentation_threshold,{30,undefined}}, {view_fragmentation_threshold,{100,undefined}}] [ns_server:info,2012-11-02T18:50:31.146,ns_1@10.2.1.67:<0.17117.0>:compaction_daemon:try_to_cleanup_indexes:439]Cleaning up indexes for bucket `default` [ns_server:info,2012-11-02T18:50:31.147,ns_1@10.2.1.67:<0.17117.0>:compaction_daemon:spawn_bucket_compactor:404]Compacting bucket default with config: [{parallel_db_and_view_compaction,false}, {database_fragmentation_threshold,{30,undefined}}, {view_fragmentation_threshold,{100,undefined}}] [ns_server:info,2012-11-02T18:50:32.311,ns_1@10.2.1.67:ns_config_rep<0.1439.0>:ns_config_rep:do_pull:341]Pulling config from: 'ns_1@10.2.1.66' [ns_server:info,2012-11-02T18:50:59.842,ns_1@10.2.1.67:ns_config_rep<0.1439.0>:ns_config_rep:do_pull:341]Pulling config from: 'ns_1@10.2.1.68' [ns_server:info,2012-11-02T18:51:01.155,ns_1@10.2.1.67:<0.17267.0>:compaction_daemon:try_to_cleanup_indexes:439]Cleaning up indexes for bucket `default` [ns_server:info,2012-11-02T18:51:01.156,ns_1@10.2.1.67:<0.17267.0>:compaction_daemon:spawn_bucket_compactor:404]Compacting bucket default with config: [{parallel_db_and_view_compaction,false}, {database_fragmentation_threshold,{30,undefined}}, {view_fragmentation_threshold,{100,undefined}}] [ns_server:info,2012-11-02T18:51:14.421,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:51:14.220914 PDT 3: TAP (Producer) eq_tapq:replication_building_256_'ns_1@10.2.1.65' - Schedule the backfill for vbucket 256 memcached<0.1519.0>: Fri Nov 2 18:51:14.220957 PDT 3: TAP (Producer) eq_tapq:replication_building_256_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:14.220977 PDT 3: TAP (Producer) eq_tapq:replication_building_256_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:14.220988 PDT 3: TAP (Producer) eq_tapq:replication_building_256_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 256 [ns_server:info,2012-11-02T18:51:15.045,ns_1@10.2.1.67:<0.17342.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.65",11209} vbucket 256 to state replica [rebalance:info,2012-11-02T18:51:15.053,ns_1@10.2.1.67:<0.17342.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[256]}, {checkpoints,[{256,10}]}, {name,<<"rebalance_256">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.65",11209}, [{username,"default"}, {password,[]}, {vbuckets,[256]}, {takeover,true}, {suffix,"256"}]} [rebalance:info,2012-11-02T18:51:15.056,ns_1@10.2.1.67:<0.17342.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:51:15.061,ns_1@10.2.1.67:<0.17342.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:51:15.115,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/256 [ns_server:info,2012-11-02T18:51:15.167,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:51:14.966711 PDT 3: TAP (Producer) eq_tapq:replication_building_256_'ns_1@10.2.1.65' - Backfill is completed with VBuckets 256, memcached<0.1519.0>: Fri Nov 2 18:51:14.966741 PDT 3: TAP (Producer) eq_tapq:replication_building_256_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 256 memcached<0.1519.0>: Fri Nov 2 18:51:15.040161 PDT 3: TAP (Producer) eq_tapq:replication_building_256_'ns_1@10.2.1.65' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:51:15.054140 PDT 3: TAP (Producer) eq_tapq:rebalance_256 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:15.054163 PDT 3: TAP (Producer) eq_tapq:rebalance_256 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:15.054174 PDT 3: TAP (Producer) eq_tapq:rebalance_256 - Sending TAP_VBUCKET_SET with vbucket 256 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:51:15.060321 PDT 3: TAP (Producer) eq_tapq:rebalance_256 - VBucket <256> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:51:15.060347 PDT 3: TAP (Producer) eq_tapq:rebalance_256 - Sending TAP_VBUCKET_SET with vbucket 256 and state "active" memcached<0.1519.0>: Fri Nov 2 18:51:15.060775 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:51:15.060818 PDT 3: TAP (Producer) eq_tapq:rebalance_256 - disconnected memcached<0.1519.0>: Fri Nov 2 18:51:15.060887 PDT 3: Schedule cleanup of "eq_tapq:rebalance_256" memcached<0.1519.0>: Fri Nov 2 18:51:15.060976 PDT 3: TAP (Producer) eq_tapq:rebalance_256 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:51:15.068115 PDT 3: TAP (Producer) eq_tapq:replication_building_256_'ns_1@10.2.1.65' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:51:15.074831 PDT 3: Deletion of vbucket 256 was completed. memcached<0.1519.0>: Fri Nov 2 18:51:15.086193 PDT 3: TAP (Producer) eq_tapq:replication_building_257_'ns_1@10.2.1.65' - Schedule the backfill for vbucket 257 memcached<0.1519.0>: Fri Nov 2 18:51:15.086239 PDT 3: TAP (Producer) eq_tapq:replication_building_257_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:15.086255 PDT 3: TAP (Producer) eq_tapq:replication_building_257_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:15.086265 PDT 3: TAP (Producer) eq_tapq:replication_building_257_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 257 memcached<0.1519.0>: Fri Nov 2 18:51:15.086434 PDT 3: Schedule cleanup of "eq_tapq:anon_1" memcached<0.1519.0>: Fri Nov 2 18:51:15.104758 PDT 3: TAP (Producer) eq_tapq:replication_building_256_'ns_1@10.2.1.65' - Clear the tap queues by force [ns_server:info,2012-11-02T18:51:15.745,ns_1@10.2.1.67:<0.17356.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.65",11209} vbucket 257 to state replica [rebalance:info,2012-11-02T18:51:15.749,ns_1@10.2.1.67:<0.17356.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[257]}, {checkpoints,[{257,10}]}, {name,<<"rebalance_257">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.65",11209}, [{username,"default"}, {password,[]}, {vbuckets,[257]}, {takeover,true}, {suffix,"257"}]} [rebalance:info,2012-11-02T18:51:15.754,ns_1@10.2.1.67:<0.17356.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:51:15.757,ns_1@10.2.1.67:<0.17356.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:51:15.816,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/257 [ns_server:info,2012-11-02T18:51:15.929,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:51:15.729487 PDT 3: TAP (Producer) eq_tapq:replication_building_257_'ns_1@10.2.1.65' - Backfill is completed with VBuckets 257, memcached<0.1519.0>: Fri Nov 2 18:51:15.729508 PDT 3: TAP (Producer) eq_tapq:replication_building_257_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 257 memcached<0.1519.0>: Fri Nov 2 18:51:15.742838 PDT 3: TAP (Producer) eq_tapq:replication_building_257_'ns_1@10.2.1.65' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:51:15.750353 PDT 3: TAP (Producer) eq_tapq:rebalance_257 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:15.750369 PDT 3: TAP (Producer) eq_tapq:rebalance_257 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:15.750375 PDT 3: TAP (Producer) eq_tapq:rebalance_257 - Sending TAP_VBUCKET_SET with vbucket 257 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:51:15.757175 PDT 3: TAP (Producer) eq_tapq:rebalance_257 - VBucket <257> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:51:15.757196 PDT 3: TAP (Producer) eq_tapq:rebalance_257 - Sending TAP_VBUCKET_SET with vbucket 257 and state "active" memcached<0.1519.0>: Fri Nov 2 18:51:15.757605 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:51:15.757636 PDT 3: TAP (Producer) eq_tapq:rebalance_257 - disconnected memcached<0.1519.0>: Fri Nov 2 18:51:15.764176 PDT 3: TAP (Producer) eq_tapq:replication_building_257_'ns_1@10.2.1.65' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:51:15.771553 PDT 3: Deletion of vbucket 257 was completed. memcached<0.1519.0>: Fri Nov 2 18:51:15.781714 PDT 3: TAP (Producer) eq_tapq:replication_building_258_'ns_1@10.2.1.65' - Schedule the backfill for vbucket 258 memcached<0.1519.0>: Fri Nov 2 18:51:15.781741 PDT 3: TAP (Producer) eq_tapq:replication_building_258_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:15.781749 PDT 3: TAP (Producer) eq_tapq:replication_building_258_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:15.781753 PDT 3: TAP (Producer) eq_tapq:replication_building_258_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 258 memcached<0.1519.0>: Fri Nov 2 18:51:15.782666 PDT 3: Schedule cleanup of "eq_tapq:anon_2" memcached<0.1519.0>: Fri Nov 2 18:51:15.782678 PDT 3: Schedule cleanup of "eq_tapq:rebalance_257" memcached<0.1519.0>: Fri Nov 2 18:51:15.789612 PDT 3: TAP (Producer) eq_tapq:replication_building_257_'ns_1@10.2.1.65' - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:51:15.789639 PDT 3: TAP (Producer) eq_tapq:rebalance_257 - Clear the tap queues by force [ns_server:info,2012-11-02T18:51:16.451,ns_1@10.2.1.67:<0.17361.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.65",11209} vbucket 258 to state replica [rebalance:info,2012-11-02T18:51:16.462,ns_1@10.2.1.67:<0.17361.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[258]}, {checkpoints,[{258,10}]}, {name,<<"rebalance_258">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.65",11209}, [{username,"default"}, {password,[]}, {vbuckets,[258]}, {takeover,true}, {suffix,"258"}]} [rebalance:info,2012-11-02T18:51:16.463,ns_1@10.2.1.67:<0.17361.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:51:16.467,ns_1@10.2.1.67:<0.17361.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:51:16.520,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/258 [ns_server:info,2012-11-02T18:51:16.627,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:51:16.426348 PDT 3: TAP (Producer) eq_tapq:replication_building_258_'ns_1@10.2.1.65' - Backfill is completed with VBuckets 258, memcached<0.1519.0>: Fri Nov 2 18:51:16.426388 PDT 3: TAP (Producer) eq_tapq:replication_building_258_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 258 memcached<0.1519.0>: Fri Nov 2 18:51:16.448920 PDT 3: TAP (Producer) eq_tapq:replication_building_258_'ns_1@10.2.1.65' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:51:16.462692 PDT 3: TAP (Producer) eq_tapq:rebalance_258 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:16.462706 PDT 3: TAP (Producer) eq_tapq:rebalance_258 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:16.462711 PDT 3: TAP (Producer) eq_tapq:rebalance_258 - Sending TAP_VBUCKET_SET with vbucket 258 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:51:16.466851 PDT 3: TAP (Producer) eq_tapq:rebalance_258 - VBucket <258> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:51:16.466867 PDT 3: TAP (Producer) eq_tapq:rebalance_258 - Sending TAP_VBUCKET_SET with vbucket 258 and state "active" memcached<0.1519.0>: Fri Nov 2 18:51:16.467092 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:51:16.467118 PDT 3: TAP (Producer) eq_tapq:rebalance_258 - disconnected memcached<0.1519.0>: Fri Nov 2 18:51:16.467463 PDT 3: Schedule cleanup of "eq_tapq:rebalance_258" memcached<0.1519.0>: Fri Nov 2 18:51:16.467616 PDT 3: TAP (Producer) eq_tapq:rebalance_258 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:51:16.473872 PDT 3: TAP (Producer) eq_tapq:replication_building_258_'ns_1@10.2.1.65' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:51:16.480159 PDT 3: Deletion of vbucket 258 was completed. memcached<0.1519.0>: Fri Nov 2 18:51:16.491310 PDT 3: TAP (Producer) eq_tapq:replication_building_259_'ns_1@10.2.1.65' - Schedule the backfill for vbucket 259 memcached<0.1519.0>: Fri Nov 2 18:51:16.491345 PDT 3: TAP (Producer) eq_tapq:replication_building_259_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:16.491352 PDT 3: TAP (Producer) eq_tapq:replication_building_259_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:16.491357 PDT 3: TAP (Producer) eq_tapq:replication_building_259_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 259 memcached<0.1519.0>: Fri Nov 2 18:51:16.491388 PDT 3: Schedule cleanup of "eq_tapq:anon_3" memcached<0.1519.0>: Fri Nov 2 18:51:16.498318 PDT 3: TAP (Producer) eq_tapq:replication_building_258_'ns_1@10.2.1.65' - Clear the tap queues by force [ns_server:info,2012-11-02T18:51:17.277,ns_1@10.2.1.67:<0.17372.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.65",11209} vbucket 259 to state replica [rebalance:info,2012-11-02T18:51:17.292,ns_1@10.2.1.67:<0.17372.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[259]}, {checkpoints,[{259,10}]}, {name,<<"rebalance_259">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.65",11209}, [{username,"default"}, {password,[]}, {vbuckets,[259]}, {takeover,true}, {suffix,"259"}]} [rebalance:info,2012-11-02T18:51:17.293,ns_1@10.2.1.67:<0.17372.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:51:17.297,ns_1@10.2.1.67:<0.17372.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:51:17.375,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/259 [ns_server:info,2012-11-02T18:51:17.389,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:51:17.180205 PDT 3: TAP (Producer) eq_tapq:replication_building_259_'ns_1@10.2.1.65' - Backfill is completed with VBuckets 259, memcached<0.1519.0>: Fri Nov 2 18:51:17.180234 PDT 3: TAP (Producer) eq_tapq:replication_building_259_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 259 memcached<0.1519.0>: Fri Nov 2 18:51:17.275751 PDT 3: TAP (Producer) eq_tapq:replication_building_259_'ns_1@10.2.1.65' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:51:17.292642 PDT 3: TAP (Producer) eq_tapq:rebalance_259 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:17.292657 PDT 3: TAP (Producer) eq_tapq:rebalance_259 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:17.292662 PDT 3: TAP (Producer) eq_tapq:rebalance_259 - Sending TAP_VBUCKET_SET with vbucket 259 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:51:17.296676 PDT 3: TAP (Producer) eq_tapq:rebalance_259 - VBucket <259> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:51:17.296693 PDT 3: TAP (Producer) eq_tapq:rebalance_259 - Sending TAP_VBUCKET_SET with vbucket 259 and state "active" memcached<0.1519.0>: Fri Nov 2 18:51:17.297090 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:51:17.297127 PDT 3: TAP (Producer) eq_tapq:rebalance_259 - disconnected memcached<0.1519.0>: Fri Nov 2 18:51:17.297169 PDT 3: Schedule cleanup of "eq_tapq:rebalance_259" memcached<0.1519.0>: Fri Nov 2 18:51:17.297216 PDT 3: TAP (Producer) eq_tapq:rebalance_259 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:51:17.311249 PDT 3: TAP (Producer) eq_tapq:replication_building_259_'ns_1@10.2.1.65' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:51:17.320126 PDT 3: Deletion of vbucket 259 was completed. memcached<0.1519.0>: Fri Nov 2 18:51:17.329968 PDT 3: TAP (Producer) eq_tapq:replication_building_260_'ns_1@10.2.1.65' - Schedule the backfill for vbucket 260 memcached<0.1519.0>: Fri Nov 2 18:51:17.329999 PDT 3: TAP (Producer) eq_tapq:replication_building_260_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:17.330009 PDT 3: TAP (Producer) eq_tapq:replication_building_260_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:17.330017 PDT 3: TAP (Producer) eq_tapq:replication_building_260_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 260 memcached<0.1519.0>: Fri Nov 2 18:51:17.330080 PDT 3: Schedule cleanup of "eq_tapq:anon_4" memcached<0.1519.0>: Fri Nov 2 18:51:17.345240 PDT 3: TAP (Producer) eq_tapq:replication_building_259_'ns_1@10.2.1.65' - Clear the tap queues by force [ns_server:info,2012-11-02T18:51:18.125,ns_1@10.2.1.67:<0.17377.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.65",11209} vbucket 260 to state replica [rebalance:info,2012-11-02T18:51:18.132,ns_1@10.2.1.67:<0.17377.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[260]}, {checkpoints,[{260,10}]}, {name,<<"rebalance_260">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.65",11209}, [{username,"default"}, {password,[]}, {vbuckets,[260]}, {takeover,true}, {suffix,"260"}]} [rebalance:info,2012-11-02T18:51:18.138,ns_1@10.2.1.67:<0.17377.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:51:18.145,ns_1@10.2.1.67:<0.17377.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:51:18.227,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/260 [ns_server:info,2012-11-02T18:51:18.232,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:51:18.031532 PDT 3: TAP (Producer) eq_tapq:replication_building_260_'ns_1@10.2.1.65' - Backfill is completed with VBuckets 260, memcached<0.1519.0>: Fri Nov 2 18:51:18.031554 PDT 3: TAP (Producer) eq_tapq:replication_building_260_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 260 memcached<0.1519.0>: Fri Nov 2 18:51:18.123604 PDT 3: TAP (Producer) eq_tapq:replication_building_260_'ns_1@10.2.1.65' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:51:18.137233 PDT 3: TAP (Producer) eq_tapq:rebalance_260 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:18.137259 PDT 3: TAP (Producer) eq_tapq:rebalance_260 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:18.137269 PDT 3: TAP (Producer) eq_tapq:rebalance_260 - Sending TAP_VBUCKET_SET with vbucket 260 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:51:18.144781 PDT 3: TAP (Producer) eq_tapq:rebalance_260 - VBucket <260> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:51:18.144816 PDT 3: TAP (Producer) eq_tapq:rebalance_260 - Sending TAP_VBUCKET_SET with vbucket 260 and state "active" memcached<0.1519.0>: Fri Nov 2 18:51:18.145184 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:51:18.145271 PDT 3: TAP (Producer) eq_tapq:rebalance_260 - disconnected memcached<0.1519.0>: Fri Nov 2 18:51:18.145307 PDT 3: Schedule cleanup of "eq_tapq:rebalance_260" memcached<0.1519.0>: Fri Nov 2 18:51:18.145331 PDT 3: TAP (Producer) eq_tapq:rebalance_260 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:51:18.166151 PDT 3: TAP (Producer) eq_tapq:replication_building_260_'ns_1@10.2.1.65' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:51:18.173069 PDT 3: Deletion of vbucket 260 was completed. memcached<0.1519.0>: Fri Nov 2 18:51:18.183439 PDT 3: TAP (Producer) eq_tapq:replication_building_261_'ns_1@10.2.1.65' - Schedule the backfill for vbucket 261 memcached<0.1519.0>: Fri Nov 2 18:51:18.183465 PDT 3: TAP (Producer) eq_tapq:replication_building_261_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:18.183473 PDT 3: TAP (Producer) eq_tapq:replication_building_261_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:18.183479 PDT 3: TAP (Producer) eq_tapq:replication_building_261_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 261 memcached<0.1519.0>: Fri Nov 2 18:51:18.183626 PDT 3: Schedule cleanup of "eq_tapq:anon_5" memcached<0.1519.0>: Fri Nov 2 18:51:18.198830 PDT 3: TAP (Producer) eq_tapq:replication_building_260_'ns_1@10.2.1.65' - Clear the tap queues by force [ns_server:info,2012-11-02T18:51:18.922,ns_1@10.2.1.67:<0.17387.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.65",11209} vbucket 261 to state replica [rebalance:info,2012-11-02T18:51:18.926,ns_1@10.2.1.67:<0.17387.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[261]}, {checkpoints,[{261,9}]}, {name,<<"rebalance_261">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.65",11209}, [{username,"default"}, {password,[]}, {vbuckets,[261]}, {takeover,true}, {suffix,"261"}]} [rebalance:info,2012-11-02T18:51:18.927,ns_1@10.2.1.67:<0.17387.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:51:18.931,ns_1@10.2.1.67:<0.17387.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:51:18.993,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/261 [ns_server:info,2012-11-02T18:51:19.104,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:51:18.903965 PDT 3: TAP (Producer) eq_tapq:replication_building_261_'ns_1@10.2.1.65' - Backfill is completed with VBuckets 261, memcached<0.1519.0>: Fri Nov 2 18:51:18.903987 PDT 3: TAP (Producer) eq_tapq:replication_building_261_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 261 memcached<0.1519.0>: Fri Nov 2 18:51:18.920667 PDT 3: TAP (Producer) eq_tapq:replication_building_261_'ns_1@10.2.1.65' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:51:18.926927 PDT 3: TAP (Producer) eq_tapq:rebalance_261 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:18.926940 PDT 3: TAP (Producer) eq_tapq:rebalance_261 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:18.926946 PDT 3: TAP (Producer) eq_tapq:rebalance_261 - Sending TAP_VBUCKET_SET with vbucket 261 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:51:18.930906 PDT 3: TAP (Producer) eq_tapq:rebalance_261 - VBucket <261> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:51:18.930924 PDT 3: TAP (Producer) eq_tapq:rebalance_261 - Sending TAP_VBUCKET_SET with vbucket 261 and state "active" memcached<0.1519.0>: Fri Nov 2 18:51:18.931433 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:51:18.931456 PDT 3: TAP (Producer) eq_tapq:rebalance_261 - disconnected memcached<0.1519.0>: Fri Nov 2 18:51:18.931489 PDT 3: Schedule cleanup of "eq_tapq:rebalance_261" memcached<0.1519.0>: Fri Nov 2 18:51:18.931889 PDT 3: TAP (Producer) eq_tapq:rebalance_261 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:51:18.936138 PDT 3: TAP (Producer) eq_tapq:replication_building_261_'ns_1@10.2.1.65' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:51:18.942675 PDT 3: Deletion of vbucket 261 was completed. memcached<0.1519.0>: Fri Nov 2 18:51:18.963205 PDT 3: TAP (Producer) eq_tapq:replication_building_262_'ns_1@10.2.1.65' - Schedule the backfill for vbucket 262 memcached<0.1519.0>: Fri Nov 2 18:51:18.963247 PDT 3: TAP (Producer) eq_tapq:replication_building_262_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:18.963261 PDT 3: TAP (Producer) eq_tapq:replication_building_262_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:18.963271 PDT 3: TAP (Producer) eq_tapq:replication_building_262_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 262 memcached<0.1519.0>: Fri Nov 2 18:51:18.963385 PDT 3: Schedule cleanup of "eq_tapq:anon_6" memcached<0.1519.0>: Fri Nov 2 18:51:18.968546 PDT 3: TAP (Producer) eq_tapq:replication_building_261_'ns_1@10.2.1.65' - Clear the tap queues by force [ns_server:info,2012-11-02T18:51:19.854,ns_1@10.2.1.67:<0.17393.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.65",11209} vbucket 262 to state replica [rebalance:info,2012-11-02T18:51:19.860,ns_1@10.2.1.67:<0.17393.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[262]}, {checkpoints,[{262,10}]}, {name,<<"rebalance_262">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.65",11209}, [{username,"default"}, {password,[]}, {vbuckets,[262]}, {takeover,true}, {suffix,"262"}]} [rebalance:info,2012-11-02T18:51:19.867,ns_1@10.2.1.67:<0.17393.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:51:19.870,ns_1@10.2.1.67:<0.17393.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:51:19.942,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/262 [ns_server:info,2012-11-02T18:51:19.952,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:51:19.751348 PDT 3: TAP (Producer) eq_tapq:replication_building_262_'ns_1@10.2.1.65' - Backfill is completed with VBuckets 262, memcached<0.1519.0>: Fri Nov 2 18:51:19.751369 PDT 3: TAP (Producer) eq_tapq:replication_building_262_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 262 memcached<0.1519.0>: Fri Nov 2 18:51:19.850052 PDT 3: TAP (Producer) eq_tapq:replication_building_262_'ns_1@10.2.1.65' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:51:19.860979 PDT 3: TAP (Producer) eq_tapq:rebalance_262 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:19.860993 PDT 3: TAP (Producer) eq_tapq:rebalance_262 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:19.860998 PDT 3: TAP (Producer) eq_tapq:rebalance_262 - Sending TAP_VBUCKET_SET with vbucket 262 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:51:19.870016 PDT 3: TAP (Producer) eq_tapq:rebalance_262 - VBucket <262> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:51:19.870036 PDT 3: TAP (Producer) eq_tapq:rebalance_262 - Sending TAP_VBUCKET_SET with vbucket 262 and state "active" memcached<0.1519.0>: Fri Nov 2 18:51:19.870295 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:51:19.870318 PDT 3: TAP (Producer) eq_tapq:rebalance_262 - disconnected memcached<0.1519.0>: Fri Nov 2 18:51:19.870349 PDT 3: Schedule cleanup of "eq_tapq:rebalance_262" memcached<0.1519.0>: Fri Nov 2 18:51:19.870402 PDT 3: TAP (Producer) eq_tapq:rebalance_262 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:51:19.880326 PDT 3: TAP (Producer) eq_tapq:replication_building_262_'ns_1@10.2.1.65' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:51:19.888025 PDT 3: Deletion of vbucket 262 was completed. memcached<0.1519.0>: Fri Nov 2 18:51:19.902923 PDT 3: TAP (Producer) eq_tapq:replication_building_263_'ns_1@10.2.1.65' - Schedule the backfill for vbucket 263 memcached<0.1519.0>: Fri Nov 2 18:51:19.902950 PDT 3: TAP (Producer) eq_tapq:replication_building_263_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:19.902965 PDT 3: TAP (Producer) eq_tapq:replication_building_263_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:19.902971 PDT 3: TAP (Producer) eq_tapq:replication_building_263_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 263 memcached<0.1519.0>: Fri Nov 2 18:51:19.903023 PDT 3: Schedule cleanup of "eq_tapq:anon_7" memcached<0.1519.0>: Fri Nov 2 18:51:19.906336 PDT 3: TAP (Producer) eq_tapq:replication_building_262_'ns_1@10.2.1.65' - Clear the tap queues by force [ns_server:info,2012-11-02T18:51:20.696,ns_1@10.2.1.67:<0.17411.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.65",11209} vbucket 263 to state replica [rebalance:info,2012-11-02T18:51:20.706,ns_1@10.2.1.67:<0.17411.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[263]}, {checkpoints,[{263,10}]}, {name,<<"rebalance_263">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.65",11209}, [{username,"default"}, {password,[]}, {vbuckets,[263]}, {takeover,true}, {suffix,"263"}]} [rebalance:info,2012-11-02T18:51:20.708,ns_1@10.2.1.67:<0.17411.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:51:20.711,ns_1@10.2.1.67:<0.17411.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:51:20.774,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/263 [ns_server:info,2012-11-02T18:51:20.846,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:51:20.644651 PDT 3: TAP (Producer) eq_tapq:replication_building_263_'ns_1@10.2.1.65' - Backfill is completed with VBuckets 263, memcached<0.1519.0>: Fri Nov 2 18:51:20.644674 PDT 3: TAP (Producer) eq_tapq:replication_building_263_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 263 memcached<0.1519.0>: Fri Nov 2 18:51:20.692848 PDT 3: TAP (Producer) eq_tapq:replication_building_263_'ns_1@10.2.1.65' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:51:20.707978 PDT 3: TAP (Producer) eq_tapq:rebalance_263 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:20.707997 PDT 3: TAP (Producer) eq_tapq:rebalance_263 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:20.708002 PDT 3: TAP (Producer) eq_tapq:rebalance_263 - Sending TAP_VBUCKET_SET with vbucket 263 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:51:20.710895 PDT 3: TAP (Producer) eq_tapq:rebalance_263 - VBucket <263> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:51:20.710915 PDT 3: TAP (Producer) eq_tapq:rebalance_263 - Sending TAP_VBUCKET_SET with vbucket 263 and state "active" memcached<0.1519.0>: Fri Nov 2 18:51:20.711119 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:51:20.711144 PDT 3: TAP (Producer) eq_tapq:rebalance_263 - disconnected memcached<0.1519.0>: Fri Nov 2 18:51:20.719997 PDT 3: TAP (Producer) eq_tapq:replication_building_263_'ns_1@10.2.1.65' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:51:20.727989 PDT 3: Deletion of vbucket 263 was completed. memcached<0.1519.0>: Fri Nov 2 18:51:20.755084 PDT 3: TAP (Producer) eq_tapq:replication_building_264_'ns_1@10.2.1.65' - Schedule the backfill for vbucket 264 memcached<0.1519.0>: Fri Nov 2 18:51:20.755129 PDT 3: TAP (Producer) eq_tapq:replication_building_264_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:20.755150 PDT 3: TAP (Producer) eq_tapq:replication_building_264_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:20.755160 PDT 3: TAP (Producer) eq_tapq:replication_building_264_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 264 memcached<0.1519.0>: Fri Nov 2 18:51:20.755252 PDT 3: Schedule cleanup of "eq_tapq:anon_8" memcached<0.1519.0>: Fri Nov 2 18:51:20.755266 PDT 3: Schedule cleanup of "eq_tapq:rebalance_263" memcached<0.1519.0>: Fri Nov 2 18:51:20.755346 PDT 3: TAP (Producer) eq_tapq:replication_building_263_'ns_1@10.2.1.65' - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:51:20.755361 PDT 3: TAP (Producer) eq_tapq:rebalance_263 - Clear the tap queues by force [ns_server:info,2012-11-02T18:51:21.682,ns_1@10.2.1.67:<0.17417.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.65",11209} vbucket 264 to state replica [rebalance:info,2012-11-02T18:51:21.691,ns_1@10.2.1.67:<0.17417.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[264]}, {checkpoints,[{264,10}]}, {name,<<"rebalance_264">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.65",11209}, [{username,"default"}, {password,[]}, {vbuckets,[264]}, {takeover,true}, {suffix,"264"}]} [rebalance:info,2012-11-02T18:51:21.693,ns_1@10.2.1.67:<0.17417.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:51:21.697,ns_1@10.2.1.67:<0.17417.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:51:21.774,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/264 [ns_server:info,2012-11-02T18:51:21.780,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:51:21.579675 PDT 3: TAP (Producer) eq_tapq:replication_building_264_'ns_1@10.2.1.65' - Backfill is completed with VBuckets 264, memcached<0.1519.0>: Fri Nov 2 18:51:21.579697 PDT 3: TAP (Producer) eq_tapq:replication_building_264_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 264 memcached<0.1519.0>: Fri Nov 2 18:51:21.680222 PDT 3: TAP (Producer) eq_tapq:replication_building_264_'ns_1@10.2.1.65' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:51:21.692093 PDT 3: TAP (Producer) eq_tapq:rebalance_264 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:21.692108 PDT 3: TAP (Producer) eq_tapq:rebalance_264 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:21.692113 PDT 3: TAP (Producer) eq_tapq:rebalance_264 - Sending TAP_VBUCKET_SET with vbucket 264 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:51:21.696908 PDT 3: TAP (Producer) eq_tapq:rebalance_264 - VBucket <264> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:51:21.696943 PDT 3: TAP (Producer) eq_tapq:rebalance_264 - Sending TAP_VBUCKET_SET with vbucket 264 and state "active" memcached<0.1519.0>: Fri Nov 2 18:51:21.697365 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:51:21.697394 PDT 3: TAP (Producer) eq_tapq:rebalance_264 - disconnected memcached<0.1519.0>: Fri Nov 2 18:51:21.697427 PDT 3: Schedule cleanup of "eq_tapq:rebalance_264" memcached<0.1519.0>: Fri Nov 2 18:51:21.697499 PDT 3: TAP (Producer) eq_tapq:rebalance_264 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:51:21.707073 PDT 3: TAP (Producer) eq_tapq:replication_building_264_'ns_1@10.2.1.65' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:51:21.714000 PDT 3: Deletion of vbucket 264 was completed. memcached<0.1519.0>: Fri Nov 2 18:51:21.730072 PDT 3: TAP (Producer) eq_tapq:replication_building_265_'ns_1@10.2.1.65' - Schedule the backfill for vbucket 265 memcached<0.1519.0>: Fri Nov 2 18:51:21.730113 PDT 3: TAP (Producer) eq_tapq:replication_building_265_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:21.730128 PDT 3: TAP (Producer) eq_tapq:replication_building_265_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:21.730140 PDT 3: TAP (Producer) eq_tapq:replication_building_265_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 265 memcached<0.1519.0>: Fri Nov 2 18:51:21.730945 PDT 3: Schedule cleanup of "eq_tapq:anon_9" memcached<0.1519.0>: Fri Nov 2 18:51:21.739770 PDT 3: TAP (Producer) eq_tapq:replication_building_264_'ns_1@10.2.1.65' - Clear the tap queues by force [ns_server:info,2012-11-02T18:51:22.720,ns_1@10.2.1.67:<0.17428.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.65",11209} vbucket 265 to state replica [rebalance:info,2012-11-02T18:51:22.732,ns_1@10.2.1.67:<0.17428.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[265]}, {checkpoints,[{265,10}]}, {name,<<"rebalance_265">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.65",11209}, [{username,"default"}, {password,[]}, {vbuckets,[265]}, {takeover,true}, {suffix,"265"}]} [rebalance:info,2012-11-02T18:51:22.734,ns_1@10.2.1.67:<0.17428.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:51:22.739,ns_1@10.2.1.67:<0.17428.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:51:22.838,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:51:22.637068 PDT 3: TAP (Producer) eq_tapq:replication_building_265_'ns_1@10.2.1.65' - Backfill is completed with VBuckets 265, memcached<0.1519.0>: Fri Nov 2 18:51:22.637088 PDT 3: TAP (Producer) eq_tapq:replication_building_265_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 265 memcached<0.1519.0>: Fri Nov 2 18:51:22.715732 PDT 3: TAP (Producer) eq_tapq:replication_building_265_'ns_1@10.2.1.65' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:51:22.733201 PDT 3: TAP (Producer) eq_tapq:rebalance_265 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:22.733216 PDT 3: TAP (Producer) eq_tapq:rebalance_265 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:22.733224 PDT 3: TAP (Producer) eq_tapq:rebalance_265 - Sending TAP_VBUCKET_SET with vbucket 265 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:51:22.739275 PDT 3: TAP (Producer) eq_tapq:rebalance_265 - VBucket <265> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:51:22.739296 PDT 3: TAP (Producer) eq_tapq:rebalance_265 - Sending TAP_VBUCKET_SET with vbucket 265 and state "active" memcached<0.1519.0>: Fri Nov 2 18:51:22.739471 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:51:22.739493 PDT 3: TAP (Producer) eq_tapq:rebalance_265 - disconnected memcached<0.1519.0>: Fri Nov 2 18:51:22.830249 PDT 3: TAP (Producer) eq_tapq:replication_building_265_'ns_1@10.2.1.65' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:51:22.837470 PDT 3: Deletion of vbucket 265 was completed. [ns_server:info,2012-11-02T18:51:22.899,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/265 [ns_server:info,2012-11-02T18:51:23.096,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:51:22.867205 PDT 3: TAP (Producer) eq_tapq:replication_building_266_'ns_1@10.2.1.65' - Schedule the backfill for vbucket 266 memcached<0.1519.0>: Fri Nov 2 18:51:22.867248 PDT 3: TAP (Producer) eq_tapq:replication_building_266_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:22.867262 PDT 3: TAP (Producer) eq_tapq:replication_building_266_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:22.867272 PDT 3: TAP (Producer) eq_tapq:replication_building_266_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 266 memcached<0.1519.0>: Fri Nov 2 18:51:22.867400 PDT 3: Schedule cleanup of "eq_tapq:anon_10" memcached<0.1519.0>: Fri Nov 2 18:51:22.867430 PDT 3: Schedule cleanup of "eq_tapq:rebalance_265" memcached<0.1519.0>: Fri Nov 2 18:51:22.867490 PDT 3: TAP (Producer) eq_tapq:replication_building_265_'ns_1@10.2.1.65' - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:51:22.867545 PDT 3: TAP (Producer) eq_tapq:rebalance_265 - Clear the tap queues by force [ns_server:info,2012-11-02T18:51:23.820,ns_1@10.2.1.67:<0.17433.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.65",11209} vbucket 266 to state replica [rebalance:info,2012-11-02T18:51:23.824,ns_1@10.2.1.67:<0.17433.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[266]}, {checkpoints,[{266,9}]}, {name,<<"rebalance_266">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.65",11209}, [{username,"default"}, {password,[]}, {vbuckets,[266]}, {takeover,true}, {suffix,"266"}]} [rebalance:info,2012-11-02T18:51:23.825,ns_1@10.2.1.67:<0.17433.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:51:23.830,ns_1@10.2.1.67:<0.17433.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:51:23.900,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/266 [ns_server:info,2012-11-02T18:51:23.929,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:51:23.728221 PDT 3: TAP (Producer) eq_tapq:replication_building_266_'ns_1@10.2.1.65' - Backfill is completed with VBuckets 266, memcached<0.1519.0>: Fri Nov 2 18:51:23.728243 PDT 3: TAP (Producer) eq_tapq:replication_building_266_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 266 memcached<0.1519.0>: Fri Nov 2 18:51:23.811701 PDT 3: TAP (Producer) eq_tapq:replication_building_266_'ns_1@10.2.1.65' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:51:23.824786 PDT 3: TAP (Producer) eq_tapq:rebalance_266 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:23.824798 PDT 3: TAP (Producer) eq_tapq:rebalance_266 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:23.824803 PDT 3: TAP (Producer) eq_tapq:rebalance_266 - Sending TAP_VBUCKET_SET with vbucket 266 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:51:23.829333 PDT 3: TAP (Producer) eq_tapq:rebalance_266 - VBucket <266> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:51:23.829348 PDT 3: TAP (Producer) eq_tapq:rebalance_266 - Sending TAP_VBUCKET_SET with vbucket 266 and state "active" memcached<0.1519.0>: Fri Nov 2 18:51:23.830003 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:51:23.830027 PDT 3: TAP (Producer) eq_tapq:rebalance_266 - disconnected memcached<0.1519.0>: Fri Nov 2 18:51:23.830059 PDT 3: Schedule cleanup of "eq_tapq:rebalance_266" memcached<0.1519.0>: Fri Nov 2 18:51:23.830086 PDT 3: TAP (Producer) eq_tapq:rebalance_266 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:51:23.838763 PDT 3: TAP (Producer) eq_tapq:replication_building_266_'ns_1@10.2.1.65' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:51:23.845624 PDT 3: Deletion of vbucket 266 was completed. memcached<0.1519.0>: Fri Nov 2 18:51:23.859406 PDT 3: TAP (Producer) eq_tapq:replication_building_267_'ns_1@10.2.1.65' - Schedule the backfill for vbucket 267 memcached<0.1519.0>: Fri Nov 2 18:51:23.859432 PDT 3: TAP (Producer) eq_tapq:replication_building_267_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:23.859440 PDT 3: TAP (Producer) eq_tapq:replication_building_267_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:23.859445 PDT 3: TAP (Producer) eq_tapq:replication_building_267_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 267 memcached<0.1519.0>: Fri Nov 2 18:51:23.859548 PDT 3: Schedule cleanup of "eq_tapq:anon_11" memcached<0.1519.0>: Fri Nov 2 18:51:23.863604 PDT 3: TAP (Producer) eq_tapq:replication_building_266_'ns_1@10.2.1.65' - Clear the tap queues by force [ns_server:info,2012-11-02T18:51:24.605,ns_1@10.2.1.67:<0.17438.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.65",11209} vbucket 267 to state replica [rebalance:info,2012-11-02T18:51:24.616,ns_1@10.2.1.67:<0.17438.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[267]}, {checkpoints,[{267,10}]}, {name,<<"rebalance_267">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.65",11209}, [{username,"default"}, {password,[]}, {vbuckets,[267]}, {takeover,true}, {suffix,"267"}]} [rebalance:info,2012-11-02T18:51:24.617,ns_1@10.2.1.67:<0.17438.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:51:24.620,ns_1@10.2.1.67:<0.17438.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:51:24.684,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/267 [ns_server:info,2012-11-02T18:51:24.773,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:51:24.572538 PDT 3: TAP (Producer) eq_tapq:replication_building_267_'ns_1@10.2.1.65' - Backfill is completed with VBuckets 267, memcached<0.1519.0>: Fri Nov 2 18:51:24.572561 PDT 3: TAP (Producer) eq_tapq:replication_building_267_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 267 memcached<0.1519.0>: Fri Nov 2 18:51:24.601808 PDT 3: TAP (Producer) eq_tapq:replication_building_267_'ns_1@10.2.1.65' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:51:24.616473 PDT 3: TAP (Producer) eq_tapq:rebalance_267 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:24.616488 PDT 3: TAP (Producer) eq_tapq:rebalance_267 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:24.616493 PDT 3: TAP (Producer) eq_tapq:rebalance_267 - Sending TAP_VBUCKET_SET with vbucket 267 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:51:24.620362 PDT 3: TAP (Producer) eq_tapq:rebalance_267 - VBucket <267> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:51:24.620378 PDT 3: TAP (Producer) eq_tapq:rebalance_267 - Sending TAP_VBUCKET_SET with vbucket 267 and state "active" memcached<0.1519.0>: Fri Nov 2 18:51:24.620548 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:51:24.620860 PDT 3: TAP (Producer) eq_tapq:rebalance_267 - disconnected memcached<0.1519.0>: Fri Nov 2 18:51:24.629282 PDT 3: TAP (Producer) eq_tapq:replication_building_267_'ns_1@10.2.1.65' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:51:24.635082 PDT 3: Deletion of vbucket 267 was completed. memcached<0.1519.0>: Fri Nov 2 18:51:24.645453 PDT 3: TAP (Producer) eq_tapq:replication_building_268_'ns_1@10.2.1.65' - Schedule the backfill for vbucket 268 memcached<0.1519.0>: Fri Nov 2 18:51:24.645495 PDT 3: TAP (Producer) eq_tapq:replication_building_268_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:24.645509 PDT 3: TAP (Producer) eq_tapq:replication_building_268_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:24.645519 PDT 3: TAP (Producer) eq_tapq:replication_building_268_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 268 memcached<0.1519.0>: Fri Nov 2 18:51:24.645719 PDT 3: Schedule cleanup of "eq_tapq:anon_12" memcached<0.1519.0>: Fri Nov 2 18:51:24.645749 PDT 3: Schedule cleanup of "eq_tapq:rebalance_267" memcached<0.1519.0>: Fri Nov 2 18:51:24.662255 PDT 3: TAP (Producer) eq_tapq:replication_building_267_'ns_1@10.2.1.65' - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:51:24.662301 PDT 3: TAP (Producer) eq_tapq:rebalance_267 - Clear the tap queues by force [ns_server:info,2012-11-02T18:51:25.394,ns_1@10.2.1.67:<0.17449.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.65",11209} vbucket 268 to state replica [rebalance:info,2012-11-02T18:51:25.397,ns_1@10.2.1.67:<0.17449.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[268]}, {checkpoints,[{268,10}]}, {name,<<"rebalance_268">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.65",11209}, [{username,"default"}, {password,[]}, {vbuckets,[268]}, {takeover,true}, {suffix,"268"}]} [rebalance:info,2012-11-02T18:51:25.399,ns_1@10.2.1.67:<0.17449.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:51:25.403,ns_1@10.2.1.67:<0.17449.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:51:25.463,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/268 [ns_server:info,2012-11-02T18:51:25.498,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:51:25.298133 PDT 3: TAP (Producer) eq_tapq:replication_building_268_'ns_1@10.2.1.65' - Backfill is completed with VBuckets 268, memcached<0.1519.0>: Fri Nov 2 18:51:25.298155 PDT 3: TAP (Producer) eq_tapq:replication_building_268_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 268 memcached<0.1519.0>: Fri Nov 2 18:51:25.390258 PDT 3: TAP (Producer) eq_tapq:replication_building_268_'ns_1@10.2.1.65' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:51:25.398209 PDT 3: TAP (Producer) eq_tapq:rebalance_268 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:25.398221 PDT 3: TAP (Producer) eq_tapq:rebalance_268 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:25.398227 PDT 3: TAP (Producer) eq_tapq:rebalance_268 - Sending TAP_VBUCKET_SET with vbucket 268 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:51:25.402876 PDT 3: TAP (Producer) eq_tapq:rebalance_268 - VBucket <268> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:51:25.402898 PDT 3: TAP (Producer) eq_tapq:rebalance_268 - Sending TAP_VBUCKET_SET with vbucket 268 and state "active" memcached<0.1519.0>: Fri Nov 2 18:51:25.403185 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:51:25.403209 PDT 3: TAP (Producer) eq_tapq:rebalance_268 - disconnected memcached<0.1519.0>: Fri Nov 2 18:51:25.403243 PDT 3: Schedule cleanup of "eq_tapq:rebalance_268" memcached<0.1519.0>: Fri Nov 2 18:51:25.403266 PDT 3: TAP (Producer) eq_tapq:rebalance_268 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:51:25.409974 PDT 3: TAP (Producer) eq_tapq:replication_building_268_'ns_1@10.2.1.65' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:51:25.417967 PDT 3: Deletion of vbucket 268 was completed. memcached<0.1519.0>: Fri Nov 2 18:51:25.434813 PDT 3: TAP (Producer) eq_tapq:replication_building_269_'ns_1@10.2.1.65' - Schedule the backfill for vbucket 269 memcached<0.1519.0>: Fri Nov 2 18:51:25.434856 PDT 3: TAP (Producer) eq_tapq:replication_building_269_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:25.434870 PDT 3: TAP (Producer) eq_tapq:replication_building_269_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:25.434879 PDT 3: TAP (Producer) eq_tapq:replication_building_269_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 269 memcached<0.1519.0>: Fri Nov 2 18:51:25.434986 PDT 3: Schedule cleanup of "eq_tapq:anon_13" memcached<0.1519.0>: Fri Nov 2 18:51:25.441292 PDT 3: TAP (Producer) eq_tapq:replication_building_268_'ns_1@10.2.1.65' - Clear the tap queues by force [ns_server:info,2012-11-02T18:51:26.100,ns_1@10.2.1.67:<0.17463.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.65",11209} vbucket 269 to state replica [rebalance:info,2012-11-02T18:51:26.111,ns_1@10.2.1.67:<0.17463.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[269]}, {checkpoints,[{269,10}]}, {name,<<"rebalance_269">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.65",11209}, [{username,"default"}, {password,[]}, {vbuckets,[269]}, {takeover,true}, {suffix,"269"}]} [rebalance:info,2012-11-02T18:51:26.112,ns_1@10.2.1.67:<0.17463.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:51:26.115,ns_1@10.2.1.67:<0.17463.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:51:26.162,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/269 [ns_server:info,2012-11-02T18:51:26.271,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:51:26.065856 PDT 3: TAP (Producer) eq_tapq:replication_building_269_'ns_1@10.2.1.65' - Backfill is completed with VBuckets 269, memcached<0.1519.0>: Fri Nov 2 18:51:26.065879 PDT 3: TAP (Producer) eq_tapq:replication_building_269_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 269 memcached<0.1519.0>: Fri Nov 2 18:51:26.098765 PDT 3: TAP (Producer) eq_tapq:replication_building_269_'ns_1@10.2.1.65' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:51:26.111992 PDT 3: TAP (Producer) eq_tapq:rebalance_269 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:26.112007 PDT 3: TAP (Producer) eq_tapq:rebalance_269 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:26.112012 PDT 3: TAP (Producer) eq_tapq:rebalance_269 - Sending TAP_VBUCKET_SET with vbucket 269 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:51:26.115141 PDT 3: TAP (Producer) eq_tapq:rebalance_269 - VBucket <269> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:51:26.115159 PDT 3: TAP (Producer) eq_tapq:rebalance_269 - Sending TAP_VBUCKET_SET with vbucket 269 and state "active" memcached<0.1519.0>: Fri Nov 2 18:51:26.115331 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:51:26.115360 PDT 3: TAP (Producer) eq_tapq:rebalance_269 - disconnected memcached<0.1519.0>: Fri Nov 2 18:51:26.115393 PDT 3: Schedule cleanup of "eq_tapq:rebalance_269" memcached<0.1519.0>: Fri Nov 2 18:51:26.115433 PDT 3: TAP (Producer) eq_tapq:rebalance_269 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:51:26.118955 PDT 3: TAP (Producer) eq_tapq:replication_building_269_'ns_1@10.2.1.65' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:51:26.125916 PDT 3: Deletion of vbucket 269 was completed. memcached<0.1519.0>: Fri Nov 2 18:51:26.135014 PDT 3: TAP (Producer) eq_tapq:replication_building_270_'ns_1@10.2.1.65' - Schedule the backfill for vbucket 270 memcached<0.1519.0>: Fri Nov 2 18:51:26.135040 PDT 3: TAP (Producer) eq_tapq:replication_building_270_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:26.135047 PDT 3: TAP (Producer) eq_tapq:replication_building_270_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:26.135052 PDT 3: TAP (Producer) eq_tapq:replication_building_270_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 270 memcached<0.1519.0>: Fri Nov 2 18:51:26.135106 PDT 3: Schedule cleanup of "eq_tapq:anon_14" memcached<0.1519.0>: Fri Nov 2 18:51:26.143416 PDT 3: TAP (Producer) eq_tapq:replication_building_269_'ns_1@10.2.1.65' - Clear the tap queues by force [ns_server:info,2012-11-02T18:51:27.008,ns_1@10.2.1.67:<0.17473.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.65",11209} vbucket 270 to state replica [rebalance:info,2012-11-02T18:51:27.012,ns_1@10.2.1.67:<0.17473.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[270]}, {checkpoints,[{270,9}]}, {name,<<"rebalance_270">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.65",11209}, [{username,"default"}, {password,[]}, {vbuckets,[270]}, {takeover,true}, {suffix,"270"}]} [rebalance:info,2012-11-02T18:51:27.018,ns_1@10.2.1.67:<0.17473.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:51:27.021,ns_1@10.2.1.67:<0.17473.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:51:27.091,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/270 [ns_server:info,2012-11-02T18:51:27.112,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:51:26.911623 PDT 3: TAP (Producer) eq_tapq:replication_building_270_'ns_1@10.2.1.65' - Backfill is completed with VBuckets 270, memcached<0.1519.0>: Fri Nov 2 18:51:26.911647 PDT 3: TAP (Producer) eq_tapq:replication_building_270_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 270 memcached<0.1519.0>: Fri Nov 2 18:51:27.005702 PDT 3: TAP (Producer) eq_tapq:replication_building_270_'ns_1@10.2.1.65' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:51:27.017581 PDT 3: TAP (Producer) eq_tapq:rebalance_270 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:27.017596 PDT 3: TAP (Producer) eq_tapq:rebalance_270 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:27.017601 PDT 3: TAP (Producer) eq_tapq:rebalance_270 - Sending TAP_VBUCKET_SET with vbucket 270 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:51:27.020869 PDT 3: TAP (Producer) eq_tapq:rebalance_270 - VBucket <270> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:51:27.020884 PDT 3: TAP (Producer) eq_tapq:rebalance_270 - Sending TAP_VBUCKET_SET with vbucket 270 and state "active" memcached<0.1519.0>: Fri Nov 2 18:51:27.021073 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:51:27.021093 PDT 3: TAP (Producer) eq_tapq:rebalance_270 - disconnected memcached<0.1519.0>: Fri Nov 2 18:51:27.021135 PDT 3: Schedule cleanup of "eq_tapq:rebalance_270" memcached<0.1519.0>: Fri Nov 2 18:51:27.021167 PDT 3: TAP (Producer) eq_tapq:rebalance_270 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:51:27.030563 PDT 3: TAP (Producer) eq_tapq:replication_building_270_'ns_1@10.2.1.65' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:51:27.037291 PDT 3: Deletion of vbucket 270 was completed. memcached<0.1519.0>: Fri Nov 2 18:51:27.056292 PDT 3: TAP (Producer) eq_tapq:replication_building_271_'ns_1@10.2.1.65' - Schedule the backfill for vbucket 271 memcached<0.1519.0>: Fri Nov 2 18:51:27.056326 PDT 3: TAP (Producer) eq_tapq:replication_building_271_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:27.056342 PDT 3: TAP (Producer) eq_tapq:replication_building_271_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:27.056346 PDT 3: TAP (Producer) eq_tapq:replication_building_271_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 271 memcached<0.1519.0>: Fri Nov 2 18:51:27.056406 PDT 3: Schedule cleanup of "eq_tapq:anon_15" memcached<0.1519.0>: Fri Nov 2 18:51:27.058058 PDT 3: TAP (Producer) eq_tapq:replication_building_270_'ns_1@10.2.1.65' - Clear the tap queues by force [ns_server:info,2012-11-02T18:51:28.097,ns_1@10.2.1.67:<0.17478.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.65",11209} vbucket 271 to state replica [rebalance:info,2012-11-02T18:51:28.101,ns_1@10.2.1.67:<0.17478.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[271]}, {checkpoints,[{271,10}]}, {name,<<"rebalance_271">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.65",11209}, [{username,"default"}, {password,[]}, {vbuckets,[271]}, {takeover,true}, {suffix,"271"}]} [rebalance:info,2012-11-02T18:51:28.103,ns_1@10.2.1.67:<0.17478.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:51:28.111,ns_1@10.2.1.67:<0.17478.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:51:28.187,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/271 [ns_server:info,2012-11-02T18:51:28.267,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:51:28.066706 PDT 3: TAP (Producer) eq_tapq:replication_building_271_'ns_1@10.2.1.65' - Backfill is completed with VBuckets 271, memcached<0.1519.0>: Fri Nov 2 18:51:28.066729 PDT 3: TAP (Producer) eq_tapq:replication_building_271_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 271 memcached<0.1519.0>: Fri Nov 2 18:51:28.094271 PDT 3: TAP (Producer) eq_tapq:replication_building_271_'ns_1@10.2.1.65' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:51:28.102194 PDT 3: TAP (Producer) eq_tapq:rebalance_271 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:28.102208 PDT 3: TAP (Producer) eq_tapq:rebalance_271 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:28.102213 PDT 3: TAP (Producer) eq_tapq:rebalance_271 - Sending TAP_VBUCKET_SET with vbucket 271 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:51:28.110906 PDT 3: TAP (Producer) eq_tapq:rebalance_271 - VBucket <271> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:51:28.110936 PDT 3: TAP (Producer) eq_tapq:rebalance_271 - Sending TAP_VBUCKET_SET with vbucket 271 and state "active" memcached<0.1519.0>: Fri Nov 2 18:51:28.111724 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:51:28.111768 PDT 3: TAP (Producer) eq_tapq:rebalance_271 - disconnected memcached<0.1519.0>: Fri Nov 2 18:51:28.116013 PDT 3: TAP (Producer) eq_tapq:replication_building_271_'ns_1@10.2.1.65' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:51:28.123729 PDT 3: Deletion of vbucket 271 was completed. memcached<0.1519.0>: Fri Nov 2 18:51:28.148709 PDT 3: TAP (Producer) eq_tapq:replication_building_272_'ns_1@10.2.1.65' - Schedule the backfill for vbucket 272 memcached<0.1519.0>: Fri Nov 2 18:51:28.148738 PDT 3: TAP (Producer) eq_tapq:replication_building_272_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:28.148746 PDT 3: TAP (Producer) eq_tapq:replication_building_272_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:28.148751 PDT 3: TAP (Producer) eq_tapq:replication_building_272_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 272 memcached<0.1519.0>: Fri Nov 2 18:51:28.148829 PDT 3: Schedule cleanup of "eq_tapq:anon_16" memcached<0.1519.0>: Fri Nov 2 18:51:28.148841 PDT 3: Schedule cleanup of "eq_tapq:rebalance_271" memcached<0.1519.0>: Fri Nov 2 18:51:28.149913 PDT 3: TAP (Producer) eq_tapq:replication_building_271_'ns_1@10.2.1.65' - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:51:28.149930 PDT 3: TAP (Producer) eq_tapq:rebalance_271 - Clear the tap queues by force [ns_server:info,2012-11-02T18:51:29.006,ns_1@10.2.1.67:<0.17489.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.65",11209} vbucket 272 to state replica [rebalance:info,2012-11-02T18:51:29.015,ns_1@10.2.1.67:<0.17489.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[272]}, {checkpoints,[{272,10}]}, {name,<<"rebalance_272">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.65",11209}, [{username,"default"}, {password,[]}, {vbuckets,[272]}, {takeover,true}, {suffix,"272"}]} [rebalance:info,2012-11-02T18:51:29.016,ns_1@10.2.1.67:<0.17489.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:51:29.022,ns_1@10.2.1.67:<0.17489.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:51:29.102,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/272 [ns_server:info,2012-11-02T18:51:29.164,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:51:28.963812 PDT 3: TAP (Producer) eq_tapq:replication_building_272_'ns_1@10.2.1.65' - Backfill is completed with VBuckets 272, memcached<0.1519.0>: Fri Nov 2 18:51:28.963836 PDT 3: TAP (Producer) eq_tapq:replication_building_272_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 272 memcached<0.1519.0>: Fri Nov 2 18:51:28.999630 PDT 3: TAP (Producer) eq_tapq:replication_building_272_'ns_1@10.2.1.65' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:51:29.016173 PDT 3: TAP (Producer) eq_tapq:rebalance_272 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:29.016187 PDT 3: TAP (Producer) eq_tapq:rebalance_272 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:29.016192 PDT 3: TAP (Producer) eq_tapq:rebalance_272 - Sending TAP_VBUCKET_SET with vbucket 272 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:51:29.021452 PDT 3: TAP (Producer) eq_tapq:rebalance_272 - VBucket <272> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:51:29.021490 PDT 3: TAP (Producer) eq_tapq:rebalance_272 - Sending TAP_VBUCKET_SET with vbucket 272 and state "active" memcached<0.1519.0>: Fri Nov 2 18:51:29.022031 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:51:29.022088 PDT 3: TAP (Producer) eq_tapq:rebalance_272 - disconnected memcached<0.1519.0>: Fri Nov 2 18:51:29.037001 PDT 3: TAP (Producer) eq_tapq:replication_building_272_'ns_1@10.2.1.65' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:51:29.044916 PDT 3: Deletion of vbucket 272 was completed. memcached<0.1519.0>: Fri Nov 2 18:51:29.063603 PDT 3: TAP (Producer) eq_tapq:replication_building_273_'ns_1@10.2.1.65' - Schedule the backfill for vbucket 273 memcached<0.1519.0>: Fri Nov 2 18:51:29.063627 PDT 3: TAP (Producer) eq_tapq:replication_building_273_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:29.063634 PDT 3: TAP (Producer) eq_tapq:replication_building_273_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:29.063639 PDT 3: TAP (Producer) eq_tapq:replication_building_273_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 273 memcached<0.1519.0>: Fri Nov 2 18:51:29.063724 PDT 3: Schedule cleanup of "eq_tapq:anon_17" memcached<0.1519.0>: Fri Nov 2 18:51:29.063737 PDT 3: Schedule cleanup of "eq_tapq:rebalance_272" memcached<0.1519.0>: Fri Nov 2 18:51:29.063838 PDT 3: TAP (Producer) eq_tapq:replication_building_272_'ns_1@10.2.1.65' - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:51:29.063862 PDT 3: TAP (Producer) eq_tapq:rebalance_272 - Clear the tap queues by force [ns_server:info,2012-11-02T18:51:30.029,ns_1@10.2.1.67:<0.17494.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.65",11209} vbucket 273 to state replica [rebalance:info,2012-11-02T18:51:30.034,ns_1@10.2.1.67:<0.17494.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[273]}, {checkpoints,[{273,10}]}, {name,<<"rebalance_273">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.65",11209}, [{username,"default"}, {password,[]}, {vbuckets,[273]}, {takeover,true}, {suffix,"273"}]} [rebalance:info,2012-11-02T18:51:30.040,ns_1@10.2.1.67:<0.17494.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:51:30.043,ns_1@10.2.1.67:<0.17494.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:51:30.122,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/273 [ns_server:info,2012-11-02T18:51:30.128,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:51:29.925900 PDT 3: TAP (Producer) eq_tapq:replication_building_273_'ns_1@10.2.1.65' - Backfill is completed with VBuckets 273, memcached<0.1519.0>: Fri Nov 2 18:51:29.925921 PDT 3: TAP (Producer) eq_tapq:replication_building_273_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 273 memcached<0.1519.0>: Fri Nov 2 18:51:30.026623 PDT 3: TAP (Producer) eq_tapq:replication_building_273_'ns_1@10.2.1.65' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:51:30.035207 PDT 3: TAP (Producer) eq_tapq:rebalance_273 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:30.035222 PDT 3: TAP (Producer) eq_tapq:rebalance_273 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:30.035227 PDT 3: TAP (Producer) eq_tapq:rebalance_273 - Sending TAP_VBUCKET_SET with vbucket 273 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:51:30.043559 PDT 3: TAP (Producer) eq_tapq:rebalance_273 - VBucket <273> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:51:30.043577 PDT 3: TAP (Producer) eq_tapq:rebalance_273 - Sending TAP_VBUCKET_SET with vbucket 273 and state "active" memcached<0.1519.0>: Fri Nov 2 18:51:30.043817 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:51:30.043857 PDT 3: TAP (Producer) eq_tapq:rebalance_273 - disconnected memcached<0.1519.0>: Fri Nov 2 18:51:30.056755 PDT 3: TAP (Producer) eq_tapq:replication_building_273_'ns_1@10.2.1.65' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:51:30.063169 PDT 3: Deletion of vbucket 273 was completed. memcached<0.1519.0>: Fri Nov 2 18:51:30.077530 PDT 3: TAP (Producer) eq_tapq:replication_building_274_'ns_1@10.2.1.65' - Schedule the backfill for vbucket 274 memcached<0.1519.0>: Fri Nov 2 18:51:30.077557 PDT 3: TAP (Producer) eq_tapq:replication_building_274_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:30.077565 PDT 3: TAP (Producer) eq_tapq:replication_building_274_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:30.077569 PDT 3: TAP (Producer) eq_tapq:replication_building_274_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 274 memcached<0.1519.0>: Fri Nov 2 18:51:30.077646 PDT 3: Schedule cleanup of "eq_tapq:anon_18" memcached<0.1519.0>: Fri Nov 2 18:51:30.077663 PDT 3: Schedule cleanup of "eq_tapq:rebalance_273" memcached<0.1519.0>: Fri Nov 2 18:51:30.080219 PDT 3: TAP (Producer) eq_tapq:replication_building_273_'ns_1@10.2.1.65' - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:51:30.080238 PDT 3: TAP (Producer) eq_tapq:rebalance_273 - Clear the tap queues by force [ns_server:info,2012-11-02T18:51:31.033,ns_1@10.2.1.67:<0.17513.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.65",11209} vbucket 274 to state replica [rebalance:info,2012-11-02T18:51:31.042,ns_1@10.2.1.67:<0.17513.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[274]}, {checkpoints,[{274,10}]}, {name,<<"rebalance_274">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.65",11209}, [{username,"default"}, {password,[]}, {vbuckets,[274]}, {takeover,true}, {suffix,"274"}]} [rebalance:info,2012-11-02T18:51:31.044,ns_1@10.2.1.67:<0.17513.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:51:31.047,ns_1@10.2.1.67:<0.17513.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:51:31.120,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/274 [ns_server:info,2012-11-02T18:51:31.155,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:51:30.955489 PDT 3: TAP (Producer) eq_tapq:replication_building_274_'ns_1@10.2.1.65' - Backfill is completed with VBuckets 274, memcached<0.1519.0>: Fri Nov 2 18:51:30.955512 PDT 3: TAP (Producer) eq_tapq:replication_building_274_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 274 memcached<0.1519.0>: Fri Nov 2 18:51:31.032202 PDT 3: TAP (Producer) eq_tapq:replication_building_274_'ns_1@10.2.1.65' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:51:31.043205 PDT 3: TAP (Producer) eq_tapq:rebalance_274 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:31.043226 PDT 3: TAP (Producer) eq_tapq:rebalance_274 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:31.043232 PDT 3: TAP (Producer) eq_tapq:rebalance_274 - Sending TAP_VBUCKET_SET with vbucket 274 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:51:31.047047 PDT 3: TAP (Producer) eq_tapq:rebalance_274 - VBucket <274> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:51:31.047073 PDT 3: TAP (Producer) eq_tapq:rebalance_274 - Sending TAP_VBUCKET_SET with vbucket 274 and state "active" memcached<0.1519.0>: Fri Nov 2 18:51:31.047449 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:51:31.047472 PDT 3: TAP (Producer) eq_tapq:rebalance_274 - disconnected memcached<0.1519.0>: Fri Nov 2 18:51:31.054770 PDT 3: TAP (Producer) eq_tapq:replication_building_274_'ns_1@10.2.1.65' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:51:31.061979 PDT 3: Deletion of vbucket 274 was completed. memcached<0.1519.0>: Fri Nov 2 18:51:31.075343 PDT 3: TAP (Producer) eq_tapq:replication_building_275_'ns_1@10.2.1.65' - Schedule the backfill for vbucket 275 memcached<0.1519.0>: Fri Nov 2 18:51:31.075371 PDT 3: TAP (Producer) eq_tapq:replication_building_275_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:31.075383 PDT 3: TAP (Producer) eq_tapq:replication_building_275_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:31.075388 PDT 3: TAP (Producer) eq_tapq:replication_building_275_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 275 memcached<0.1519.0>: Fri Nov 2 18:51:31.075457 PDT 3: Schedule cleanup of "eq_tapq:anon_19" memcached<0.1519.0>: Fri Nov 2 18:51:31.075474 PDT 3: Schedule cleanup of "eq_tapq:rebalance_274" memcached<0.1519.0>: Fri Nov 2 18:51:31.079025 PDT 3: TAP (Producer) eq_tapq:replication_building_274_'ns_1@10.2.1.65' - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:51:31.079046 PDT 3: TAP (Producer) eq_tapq:rebalance_274 - Clear the tap queues by force [ns_server:info,2012-11-02T18:51:31.163,ns_1@10.2.1.67:<0.17518.0>:compaction_daemon:try_to_cleanup_indexes:439]Cleaning up indexes for bucket `default` [ns_server:info,2012-11-02T18:51:31.164,ns_1@10.2.1.67:<0.17518.0>:compaction_daemon:spawn_bucket_compactor:404]Compacting bucket default with config: [{parallel_db_and_view_compaction,false}, {database_fragmentation_threshold,{30,undefined}}, {view_fragmentation_threshold,{100,undefined}}] [ns_server:info,2012-11-02T18:51:31.974,ns_1@10.2.1.67:<0.17526.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.65",11209} vbucket 275 to state replica [rebalance:info,2012-11-02T18:51:31.983,ns_1@10.2.1.67:<0.17526.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[275]}, {checkpoints,[{275,10}]}, {name,<<"rebalance_275">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.65",11209}, [{username,"default"}, {password,[]}, {vbuckets,[275]}, {takeover,true}, {suffix,"275"}]} [rebalance:info,2012-11-02T18:51:31.984,ns_1@10.2.1.67:<0.17526.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:51:31.988,ns_1@10.2.1.67:<0.17526.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:51:32.057,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/275 [ns_server:info,2012-11-02T18:51:32.124,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:51:31.923594 PDT 3: TAP (Producer) eq_tapq:replication_building_275_'ns_1@10.2.1.65' - Backfill is completed with VBuckets 275, memcached<0.1519.0>: Fri Nov 2 18:51:31.923622 PDT 3: TAP (Producer) eq_tapq:replication_building_275_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 275 memcached<0.1519.0>: Fri Nov 2 18:51:31.971621 PDT 3: TAP (Producer) eq_tapq:replication_building_275_'ns_1@10.2.1.65' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:51:31.983842 PDT 3: TAP (Producer) eq_tapq:rebalance_275 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:31.983856 PDT 3: TAP (Producer) eq_tapq:rebalance_275 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:31.983861 PDT 3: TAP (Producer) eq_tapq:rebalance_275 - Sending TAP_VBUCKET_SET with vbucket 275 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:51:31.987953 PDT 3: TAP (Producer) eq_tapq:rebalance_275 - VBucket <275> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:51:31.987969 PDT 3: TAP (Producer) eq_tapq:rebalance_275 - Sending TAP_VBUCKET_SET with vbucket 275 and state "active" memcached<0.1519.0>: Fri Nov 2 18:51:31.988266 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:51:31.988290 PDT 3: TAP (Producer) eq_tapq:rebalance_275 - disconnected memcached<0.1519.0>: Fri Nov 2 18:51:31.988335 PDT 3: Schedule cleanup of "eq_tapq:rebalance_275" memcached<0.1519.0>: Fri Nov 2 18:51:31.990202 PDT 3: TAP (Producer) eq_tapq:rebalance_275 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:51:31.996794 PDT 3: TAP (Producer) eq_tapq:replication_building_275_'ns_1@10.2.1.65' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:51:32.004470 PDT 3: Deletion of vbucket 275 was completed. memcached<0.1519.0>: Fri Nov 2 18:51:32.016653 PDT 3: TAP (Producer) eq_tapq:replication_building_276_'ns_1@10.2.1.65' - Schedule the backfill for vbucket 276 memcached<0.1519.0>: Fri Nov 2 18:51:32.016680 PDT 3: TAP (Producer) eq_tapq:replication_building_276_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:32.016688 PDT 3: TAP (Producer) eq_tapq:replication_building_276_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:32.016692 PDT 3: TAP (Producer) eq_tapq:replication_building_276_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 276 memcached<0.1519.0>: Fri Nov 2 18:51:32.016738 PDT 3: Schedule cleanup of "eq_tapq:anon_20" memcached<0.1519.0>: Fri Nov 2 18:51:32.021677 PDT 3: TAP (Producer) eq_tapq:replication_building_275_'ns_1@10.2.1.65' - Clear the tap queues by force [ns_server:info,2012-11-02T18:51:32.867,ns_1@10.2.1.67:<0.17536.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.65",11209} vbucket 276 to state replica [rebalance:info,2012-11-02T18:51:32.873,ns_1@10.2.1.67:<0.17536.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[276]}, {checkpoints,[{276,10}]}, {name,<<"rebalance_276">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.65",11209}, [{username,"default"}, {password,[]}, {vbuckets,[276]}, {takeover,true}, {suffix,"276"}]} [rebalance:info,2012-11-02T18:51:32.877,ns_1@10.2.1.67:<0.17536.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:51:32.880,ns_1@10.2.1.67:<0.17536.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:51:32.950,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/276 [ns_server:info,2012-11-02T18:51:33.050,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:51:32.850649 PDT 3: TAP (Producer) eq_tapq:replication_building_276_'ns_1@10.2.1.65' - Backfill is completed with VBuckets 276, memcached<0.1519.0>: Fri Nov 2 18:51:32.850671 PDT 3: TAP (Producer) eq_tapq:replication_building_276_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 276 memcached<0.1519.0>: Fri Nov 2 18:51:32.864418 PDT 3: TAP (Producer) eq_tapq:replication_building_276_'ns_1@10.2.1.65' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:51:32.876184 PDT 3: TAP (Producer) eq_tapq:rebalance_276 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:32.876198 PDT 3: TAP (Producer) eq_tapq:rebalance_276 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:32.876203 PDT 3: TAP (Producer) eq_tapq:rebalance_276 - Sending TAP_VBUCKET_SET with vbucket 276 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:51:32.880311 PDT 3: TAP (Producer) eq_tapq:rebalance_276 - VBucket <276> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:51:32.880332 PDT 3: TAP (Producer) eq_tapq:rebalance_276 - Sending TAP_VBUCKET_SET with vbucket 276 and state "active" memcached<0.1519.0>: Fri Nov 2 18:51:32.880581 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:51:32.880603 PDT 3: TAP (Producer) eq_tapq:rebalance_276 - disconnected memcached<0.1519.0>: Fri Nov 2 18:51:32.880635 PDT 3: Schedule cleanup of "eq_tapq:rebalance_276" memcached<0.1519.0>: Fri Nov 2 18:51:32.880667 PDT 3: TAP (Producer) eq_tapq:rebalance_276 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:51:32.892197 PDT 3: TAP (Producer) eq_tapq:replication_building_276_'ns_1@10.2.1.65' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:51:32.899321 PDT 3: Deletion of vbucket 276 was completed. memcached<0.1519.0>: Fri Nov 2 18:51:32.914229 PDT 3: TAP (Producer) eq_tapq:replication_building_277_'ns_1@10.2.1.65' - Schedule the backfill for vbucket 277 memcached<0.1519.0>: Fri Nov 2 18:51:32.914264 PDT 3: TAP (Producer) eq_tapq:replication_building_277_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:32.914275 PDT 3: TAP (Producer) eq_tapq:replication_building_277_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:32.914282 PDT 3: TAP (Producer) eq_tapq:replication_building_277_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 277 memcached<0.1519.0>: Fri Nov 2 18:51:32.915166 PDT 3: Schedule cleanup of "eq_tapq:anon_21" memcached<0.1519.0>: Fri Nov 2 18:51:32.922387 PDT 3: TAP (Producer) eq_tapq:replication_building_276_'ns_1@10.2.1.65' - Clear the tap queues by force [ns_server:info,2012-11-02T18:51:33.882,ns_1@10.2.1.67:<0.17541.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.65",11209} vbucket 277 to state replica [rebalance:info,2012-11-02T18:51:33.893,ns_1@10.2.1.67:<0.17541.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[277]}, {checkpoints,[{277,9}]}, {name,<<"rebalance_277">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.65",11209}, [{username,"default"}, {password,[]}, {vbuckets,[277]}, {takeover,true}, {suffix,"277"}]} [rebalance:info,2012-11-02T18:51:33.895,ns_1@10.2.1.67:<0.17541.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:51:33.897,ns_1@10.2.1.67:<0.17541.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:51:33.975,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/277 [ns_server:info,2012-11-02T18:51:34.070,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:51:33.870000 PDT 3: TAP (Producer) eq_tapq:replication_building_277_'ns_1@10.2.1.65' - Backfill is completed with VBuckets 277, memcached<0.1519.0>: Fri Nov 2 18:51:33.870021 PDT 3: TAP (Producer) eq_tapq:replication_building_277_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 277 memcached<0.1519.0>: Fri Nov 2 18:51:33.878796 PDT 3: TAP (Producer) eq_tapq:replication_building_277_'ns_1@10.2.1.65' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:51:33.893401 PDT 3: TAP (Producer) eq_tapq:rebalance_277 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:33.893415 PDT 3: TAP (Producer) eq_tapq:rebalance_277 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:33.893420 PDT 3: TAP (Producer) eq_tapq:rebalance_277 - Sending TAP_VBUCKET_SET with vbucket 277 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:51:33.897543 PDT 3: TAP (Producer) eq_tapq:rebalance_277 - VBucket <277> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:51:33.897561 PDT 3: TAP (Producer) eq_tapq:rebalance_277 - Sending TAP_VBUCKET_SET with vbucket 277 and state "active" memcached<0.1519.0>: Fri Nov 2 18:51:33.897822 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:51:33.897844 PDT 3: TAP (Producer) eq_tapq:rebalance_277 - disconnected memcached<0.1519.0>: Fri Nov 2 18:51:33.909019 PDT 3: TAP (Producer) eq_tapq:replication_building_277_'ns_1@10.2.1.65' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:51:33.917119 PDT 3: Deletion of vbucket 277 was completed. memcached<0.1519.0>: Fri Nov 2 18:51:33.929334 PDT 3: TAP (Producer) eq_tapq:replication_building_278_'ns_1@10.2.1.65' - Schedule the backfill for vbucket 278 memcached<0.1519.0>: Fri Nov 2 18:51:33.929361 PDT 3: TAP (Producer) eq_tapq:replication_building_278_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:33.929369 PDT 3: TAP (Producer) eq_tapq:replication_building_278_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:33.929382 PDT 3: TAP (Producer) eq_tapq:replication_building_278_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 278 memcached<0.1519.0>: Fri Nov 2 18:51:33.929465 PDT 3: Schedule cleanup of "eq_tapq:anon_22" memcached<0.1519.0>: Fri Nov 2 18:51:33.929492 PDT 3: Schedule cleanup of "eq_tapq:rebalance_277" memcached<0.1519.0>: Fri Nov 2 18:51:33.934286 PDT 3: TAP (Producer) eq_tapq:replication_building_277_'ns_1@10.2.1.65' - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:51:33.934310 PDT 3: TAP (Producer) eq_tapq:rebalance_277 - Clear the tap queues by force [ns_server:info,2012-11-02T18:51:34.891,ns_1@10.2.1.67:<0.17552.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.65",11209} vbucket 278 to state replica [rebalance:info,2012-11-02T18:51:34.900,ns_1@10.2.1.67:<0.17552.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[278]}, {checkpoints,[{278,9}]}, {name,<<"rebalance_278">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.65",11209}, [{username,"default"}, {password,[]}, {vbuckets,[278]}, {takeover,true}, {suffix,"278"}]} [rebalance:info,2012-11-02T18:51:34.902,ns_1@10.2.1.67:<0.17552.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:51:34.906,ns_1@10.2.1.67:<0.17552.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:51:34.986,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/278 [ns_server:info,2012-11-02T18:51:35.059,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:51:34.858500 PDT 3: TAP (Producer) eq_tapq:replication_building_278_'ns_1@10.2.1.65' - Backfill is completed with VBuckets 278, memcached<0.1519.0>: Fri Nov 2 18:51:34.858530 PDT 3: TAP (Producer) eq_tapq:replication_building_278_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 278 memcached<0.1519.0>: Fri Nov 2 18:51:34.886885 PDT 3: TAP (Producer) eq_tapq:replication_building_278_'ns_1@10.2.1.65' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:51:34.900850 PDT 3: TAP (Producer) eq_tapq:rebalance_278 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:34.900865 PDT 3: TAP (Producer) eq_tapq:rebalance_278 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:34.900870 PDT 3: TAP (Producer) eq_tapq:rebalance_278 - Sending TAP_VBUCKET_SET with vbucket 278 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:51:34.905295 PDT 3: TAP (Producer) eq_tapq:rebalance_278 - VBucket <278> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:51:34.905312 PDT 3: TAP (Producer) eq_tapq:rebalance_278 - Sending TAP_VBUCKET_SET with vbucket 278 and state "active" memcached<0.1519.0>: Fri Nov 2 18:51:34.905691 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:51:34.905719 PDT 3: TAP (Producer) eq_tapq:rebalance_278 - disconnected memcached<0.1519.0>: Fri Nov 2 18:51:34.916124 PDT 3: TAP (Producer) eq_tapq:replication_building_278_'ns_1@10.2.1.65' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:51:34.926121 PDT 3: Deletion of vbucket 278 was completed. memcached<0.1519.0>: Fri Nov 2 18:51:34.941657 PDT 3: TAP (Producer) eq_tapq:replication_building_279_'ns_1@10.2.1.65' - Schedule the backfill for vbucket 279 memcached<0.1519.0>: Fri Nov 2 18:51:34.941697 PDT 3: TAP (Producer) eq_tapq:replication_building_279_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:34.941713 PDT 3: TAP (Producer) eq_tapq:replication_building_279_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:34.941722 PDT 3: TAP (Producer) eq_tapq:replication_building_279_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 279 memcached<0.1519.0>: Fri Nov 2 18:51:34.941878 PDT 3: Schedule cleanup of "eq_tapq:anon_23" memcached<0.1519.0>: Fri Nov 2 18:51:34.941900 PDT 3: Schedule cleanup of "eq_tapq:rebalance_278" memcached<0.1519.0>: Fri Nov 2 18:51:34.953238 PDT 3: TAP (Producer) eq_tapq:replication_building_278_'ns_1@10.2.1.65' - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:51:34.953265 PDT 3: TAP (Producer) eq_tapq:rebalance_278 - Clear the tap queues by force [ns_server:info,2012-11-02T18:51:35.877,ns_1@10.2.1.67:<0.17566.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.65",11209} vbucket 279 to state replica [rebalance:info,2012-11-02T18:51:35.887,ns_1@10.2.1.67:<0.17566.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[279]}, {checkpoints,[{279,9}]}, {name,<<"rebalance_279">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.65",11209}, [{username,"default"}, {password,[]}, {vbuckets,[279]}, {takeover,true}, {suffix,"279"}]} [rebalance:info,2012-11-02T18:51:35.888,ns_1@10.2.1.67:<0.17566.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:51:35.892,ns_1@10.2.1.67:<0.17566.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:51:35.970,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/279 [ns_server:info,2012-11-02T18:51:36.029,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:51:35.828084 PDT 3: TAP (Producer) eq_tapq:replication_building_279_'ns_1@10.2.1.65' - Backfill is completed with VBuckets 279, memcached<0.1519.0>: Fri Nov 2 18:51:35.828107 PDT 3: TAP (Producer) eq_tapq:replication_building_279_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 279 memcached<0.1519.0>: Fri Nov 2 18:51:35.872960 PDT 3: TAP (Producer) eq_tapq:replication_building_279_'ns_1@10.2.1.65' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:51:35.887735 PDT 3: TAP (Producer) eq_tapq:rebalance_279 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:35.887750 PDT 3: TAP (Producer) eq_tapq:rebalance_279 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:35.887755 PDT 3: TAP (Producer) eq_tapq:rebalance_279 - Sending TAP_VBUCKET_SET with vbucket 279 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:51:35.891646 PDT 3: TAP (Producer) eq_tapq:rebalance_279 - VBucket <279> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:51:35.891674 PDT 3: TAP (Producer) eq_tapq:rebalance_279 - Sending TAP_VBUCKET_SET with vbucket 279 and state "active" memcached<0.1519.0>: Fri Nov 2 18:51:35.891960 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:51:35.892015 PDT 3: TAP (Producer) eq_tapq:rebalance_279 - disconnected memcached<0.1519.0>: Fri Nov 2 18:51:35.892377 PDT 3: Schedule cleanup of "eq_tapq:rebalance_279" memcached<0.1519.0>: Fri Nov 2 18:51:35.892678 PDT 3: TAP (Producer) eq_tapq:rebalance_279 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:51:35.901148 PDT 3: TAP (Producer) eq_tapq:replication_building_279_'ns_1@10.2.1.65' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:51:35.907322 PDT 3: Deletion of vbucket 279 was completed. memcached<0.1519.0>: Fri Nov 2 18:51:35.922051 PDT 3: TAP (Producer) eq_tapq:replication_building_280_'ns_1@10.2.1.65' - Schedule the backfill for vbucket 280 memcached<0.1519.0>: Fri Nov 2 18:51:35.922092 PDT 3: TAP (Producer) eq_tapq:replication_building_280_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:35.922107 PDT 3: TAP (Producer) eq_tapq:replication_building_280_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:35.922117 PDT 3: TAP (Producer) eq_tapq:replication_building_280_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 280 memcached<0.1519.0>: Fri Nov 2 18:51:35.922979 PDT 3: Schedule cleanup of "eq_tapq:anon_24" memcached<0.1519.0>: Fri Nov 2 18:51:35.933728 PDT 3: TAP (Producer) eq_tapq:replication_building_279_'ns_1@10.2.1.65' - Clear the tap queues by force [ns_server:info,2012-11-02T18:51:36.704,ns_1@10.2.1.67:ns_config_rep<0.1439.0>:ns_config_rep:do_pull:341]Pulling config from: 'ns_1@10.2.1.65' [ns_server:info,2012-11-02T18:51:36.890,ns_1@10.2.1.67:<0.17578.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.65",11209} vbucket 280 to state replica [rebalance:info,2012-11-02T18:51:36.893,ns_1@10.2.1.67:<0.17578.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[280]}, {checkpoints,[{280,10}]}, {name,<<"rebalance_280">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.65",11209}, [{username,"default"}, {password,[]}, {vbuckets,[280]}, {takeover,true}, {suffix,"280"}]} [rebalance:info,2012-11-02T18:51:36.900,ns_1@10.2.1.67:<0.17578.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:51:36.903,ns_1@10.2.1.67:<0.17578.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:51:36.997,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/280 [ns_server:info,2012-11-02T18:51:37.018,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:51:36.818030 PDT 3: TAP (Producer) eq_tapq:replication_building_280_'ns_1@10.2.1.65' - Backfill is completed with VBuckets 280, memcached<0.1519.0>: Fri Nov 2 18:51:36.818051 PDT 3: TAP (Producer) eq_tapq:replication_building_280_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 280 memcached<0.1519.0>: Fri Nov 2 18:51:36.888628 PDT 3: TAP (Producer) eq_tapq:replication_building_280_'ns_1@10.2.1.65' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:51:36.894333 PDT 3: TAP (Producer) eq_tapq:rebalance_280 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:36.894345 PDT 3: TAP (Producer) eq_tapq:rebalance_280 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:36.894350 PDT 3: TAP (Producer) eq_tapq:rebalance_280 - Sending TAP_VBUCKET_SET with vbucket 280 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:51:36.902931 PDT 3: TAP (Producer) eq_tapq:rebalance_280 - VBucket <280> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:51:36.902952 PDT 3: TAP (Producer) eq_tapq:rebalance_280 - Sending TAP_VBUCKET_SET with vbucket 280 and state "active" memcached<0.1519.0>: Fri Nov 2 18:51:36.903196 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:51:36.903219 PDT 3: TAP (Producer) eq_tapq:rebalance_280 - disconnected memcached<0.1519.0>: Fri Nov 2 18:51:36.903251 PDT 3: Schedule cleanup of "eq_tapq:rebalance_280" memcached<0.1519.0>: Fri Nov 2 18:51:36.903296 PDT 3: TAP (Producer) eq_tapq:rebalance_280 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:51:36.913013 PDT 3: TAP (Producer) eq_tapq:replication_building_280_'ns_1@10.2.1.65' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:51:36.919970 PDT 3: Deletion of vbucket 280 was completed. memcached<0.1519.0>: Fri Nov 2 18:51:36.931159 PDT 3: TAP (Producer) eq_tapq:replication_building_281_'ns_1@10.2.1.65' - Schedule the backfill for vbucket 281 memcached<0.1519.0>: Fri Nov 2 18:51:36.931199 PDT 3: TAP (Producer) eq_tapq:replication_building_281_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:36.931213 PDT 3: TAP (Producer) eq_tapq:replication_building_281_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:36.931223 PDT 3: TAP (Producer) eq_tapq:replication_building_281_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 281 memcached<0.1519.0>: Fri Nov 2 18:51:36.932114 PDT 3: Schedule cleanup of "eq_tapq:anon_25" memcached<0.1519.0>: Fri Nov 2 18:51:36.948634 PDT 3: TAP (Producer) eq_tapq:replication_building_280_'ns_1@10.2.1.65' - Clear the tap queues by force [ns_server:info,2012-11-02T18:51:37.988,ns_1@10.2.1.67:<0.17584.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.65",11209} vbucket 281 to state replica [rebalance:info,2012-11-02T18:51:37.993,ns_1@10.2.1.67:<0.17584.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[281]}, {checkpoints,[{281,10}]}, {name,<<"rebalance_281">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.65",11209}, [{username,"default"}, {password,[]}, {vbuckets,[281]}, {takeover,true}, {suffix,"281"}]} [rebalance:info,2012-11-02T18:51:37.999,ns_1@10.2.1.67:<0.17584.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:51:38.006,ns_1@10.2.1.67:<0.17584.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:51:38.093,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/281 [ns_server:info,2012-11-02T18:51:38.157,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:51:37.957480 PDT 3: TAP (Producer) eq_tapq:replication_building_281_'ns_1@10.2.1.65' - Backfill is completed with VBuckets 281, memcached<0.1519.0>: Fri Nov 2 18:51:37.957505 PDT 3: TAP (Producer) eq_tapq:replication_building_281_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 281 memcached<0.1519.0>: Fri Nov 2 18:51:37.985652 PDT 3: TAP (Producer) eq_tapq:replication_building_281_'ns_1@10.2.1.65' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:51:37.994063 PDT 3: TAP (Producer) eq_tapq:rebalance_281 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:37.994076 PDT 3: TAP (Producer) eq_tapq:rebalance_281 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:37.994081 PDT 3: TAP (Producer) eq_tapq:rebalance_281 - Sending TAP_VBUCKET_SET with vbucket 281 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:51:38.005838 PDT 3: TAP (Producer) eq_tapq:rebalance_281 - VBucket <281> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:51:38.005877 PDT 3: TAP (Producer) eq_tapq:rebalance_281 - Sending TAP_VBUCKET_SET with vbucket 281 and state "active" memcached<0.1519.0>: Fri Nov 2 18:51:38.006406 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:51:38.006457 PDT 3: TAP (Producer) eq_tapq:rebalance_281 - disconnected memcached<0.1519.0>: Fri Nov 2 18:51:38.006527 PDT 3: Schedule cleanup of "eq_tapq:rebalance_281" memcached<0.1519.0>: Fri Nov 2 18:51:38.006608 PDT 3: TAP (Producer) eq_tapq:rebalance_281 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:51:38.019748 PDT 3: TAP (Producer) eq_tapq:replication_building_281_'ns_1@10.2.1.65' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:51:38.026760 PDT 3: Deletion of vbucket 281 was completed. memcached<0.1519.0>: Fri Nov 2 18:51:38.049150 PDT 3: TAP (Producer) eq_tapq:replication_building_282_'ns_1@10.2.1.65' - Schedule the backfill for vbucket 282 memcached<0.1519.0>: Fri Nov 2 18:51:38.049179 PDT 3: TAP (Producer) eq_tapq:replication_building_282_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:38.049186 PDT 3: TAP (Producer) eq_tapq:replication_building_282_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:38.049191 PDT 3: TAP (Producer) eq_tapq:replication_building_282_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 282 memcached<0.1519.0>: Fri Nov 2 18:51:38.049238 PDT 3: Schedule cleanup of "eq_tapq:anon_26" memcached<0.1519.0>: Fri Nov 2 18:51:38.051246 PDT 3: TAP (Producer) eq_tapq:replication_building_281_'ns_1@10.2.1.65' - Clear the tap queues by force [ns_server:info,2012-11-02T18:51:38.972,ns_1@10.2.1.67:<0.17594.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.65",11209} vbucket 282 to state replica [rebalance:info,2012-11-02T18:51:38.975,ns_1@10.2.1.67:<0.17594.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[282]}, {checkpoints,[{282,9}]}, {name,<<"rebalance_282">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.65",11209}, [{username,"default"}, {password,[]}, {vbuckets,[282]}, {takeover,true}, {suffix,"282"}]} [rebalance:info,2012-11-02T18:51:38.981,ns_1@10.2.1.67:<0.17594.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:51:38.986,ns_1@10.2.1.67:<0.17594.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:51:39.069,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/282 [ns_server:info,2012-11-02T18:51:39.124,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:51:38.923138 PDT 3: TAP (Producer) eq_tapq:replication_building_282_'ns_1@10.2.1.65' - Backfill is completed with VBuckets 282, memcached<0.1519.0>: Fri Nov 2 18:51:38.923160 PDT 3: TAP (Producer) eq_tapq:replication_building_282_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 282 memcached<0.1519.0>: Fri Nov 2 18:51:38.966823 PDT 3: TAP (Producer) eq_tapq:replication_building_282_'ns_1@10.2.1.65' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:51:38.976113 PDT 3: TAP (Producer) eq_tapq:rebalance_282 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:38.976126 PDT 3: TAP (Producer) eq_tapq:rebalance_282 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:38.976131 PDT 3: TAP (Producer) eq_tapq:rebalance_282 - Sending TAP_VBUCKET_SET with vbucket 282 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:51:38.985799 PDT 3: TAP (Producer) eq_tapq:rebalance_282 - VBucket <282> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:51:38.985816 PDT 3: TAP (Producer) eq_tapq:rebalance_282 - Sending TAP_VBUCKET_SET with vbucket 282 and state "active" memcached<0.1519.0>: Fri Nov 2 18:51:38.985997 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:51:38.986027 PDT 3: TAP (Producer) eq_tapq:rebalance_282 - disconnected memcached<0.1519.0>: Fri Nov 2 18:51:38.999252 PDT 3: TAP (Producer) eq_tapq:replication_building_282_'ns_1@10.2.1.65' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:51:39.007558 PDT 3: Deletion of vbucket 282 was completed. memcached<0.1519.0>: Fri Nov 2 18:51:39.018954 PDT 3: TAP (Producer) eq_tapq:replication_building_283_'ns_1@10.2.1.65' - Schedule the backfill for vbucket 283 memcached<0.1519.0>: Fri Nov 2 18:51:39.018994 PDT 3: TAP (Producer) eq_tapq:replication_building_283_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:39.019004 PDT 3: TAP (Producer) eq_tapq:replication_building_283_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:39.019012 PDT 3: TAP (Producer) eq_tapq:replication_building_283_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 283 memcached<0.1519.0>: Fri Nov 2 18:51:39.019113 PDT 3: Schedule cleanup of "eq_tapq:anon_27" memcached<0.1519.0>: Fri Nov 2 18:51:39.019129 PDT 3: Schedule cleanup of "eq_tapq:rebalance_282" memcached<0.1519.0>: Fri Nov 2 18:51:39.032164 PDT 3: TAP (Producer) eq_tapq:replication_building_282_'ns_1@10.2.1.65' - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:51:39.032194 PDT 3: TAP (Producer) eq_tapq:rebalance_282 - Clear the tap queues by force [ns_server:info,2012-11-02T18:51:39.961,ns_1@10.2.1.67:<0.17599.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.65",11209} vbucket 283 to state replica [rebalance:info,2012-11-02T18:51:39.966,ns_1@10.2.1.67:<0.17599.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[283]}, {checkpoints,[{283,10}]}, {name,<<"rebalance_283">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.65",11209}, [{username,"default"}, {password,[]}, {vbuckets,[283]}, {takeover,true}, {suffix,"283"}]} [rebalance:info,2012-11-02T18:51:39.967,ns_1@10.2.1.67:<0.17599.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:51:39.971,ns_1@10.2.1.67:<0.17599.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:51:40.053,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/283 [ns_server:info,2012-11-02T18:51:40.073,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:51:39.872474 PDT 3: TAP (Producer) eq_tapq:replication_building_283_'ns_1@10.2.1.65' - Backfill is completed with VBuckets 283, memcached<0.1519.0>: Fri Nov 2 18:51:39.872504 PDT 3: TAP (Producer) eq_tapq:replication_building_283_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 283 memcached<0.1519.0>: Fri Nov 2 18:51:39.957708 PDT 3: TAP (Producer) eq_tapq:replication_building_283_'ns_1@10.2.1.65' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:51:39.966893 PDT 3: TAP (Producer) eq_tapq:rebalance_283 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:39.966906 PDT 3: TAP (Producer) eq_tapq:rebalance_283 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:39.966911 PDT 3: TAP (Producer) eq_tapq:rebalance_283 - Sending TAP_VBUCKET_SET with vbucket 283 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:51:39.970534 PDT 3: TAP (Producer) eq_tapq:rebalance_283 - VBucket <283> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:51:39.970550 PDT 3: TAP (Producer) eq_tapq:rebalance_283 - Sending TAP_VBUCKET_SET with vbucket 283 and state "active" memcached<0.1519.0>: Fri Nov 2 18:51:39.970811 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:51:39.970833 PDT 3: TAP (Producer) eq_tapq:rebalance_283 - disconnected memcached<0.1519.0>: Fri Nov 2 18:51:39.970867 PDT 3: Schedule cleanup of "eq_tapq:rebalance_283" memcached<0.1519.0>: Fri Nov 2 18:51:39.970895 PDT 3: TAP (Producer) eq_tapq:rebalance_283 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:51:39.986661 PDT 3: TAP (Producer) eq_tapq:replication_building_283_'ns_1@10.2.1.65' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:51:39.995828 PDT 3: Deletion of vbucket 283 was completed. memcached<0.1519.0>: Fri Nov 2 18:51:40.007809 PDT 3: TAP (Producer) eq_tapq:replication_building_284_'ns_1@10.2.1.65' - Schedule the backfill for vbucket 284 memcached<0.1519.0>: Fri Nov 2 18:51:40.007836 PDT 3: TAP (Producer) eq_tapq:replication_building_284_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:40.007844 PDT 3: TAP (Producer) eq_tapq:replication_building_284_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:40.007849 PDT 3: TAP (Producer) eq_tapq:replication_building_284_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 284 memcached<0.1519.0>: Fri Nov 2 18:51:40.007890 PDT 3: Schedule cleanup of "eq_tapq:anon_28" memcached<0.1519.0>: Fri Nov 2 18:51:40.016456 PDT 3: TAP (Producer) eq_tapq:replication_building_283_'ns_1@10.2.1.65' - Clear the tap queues by force [ns_server:info,2012-11-02T18:51:41.041,ns_1@10.2.1.67:<0.17619.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.65",11209} vbucket 284 to state replica [rebalance:info,2012-11-02T18:51:41.050,ns_1@10.2.1.67:<0.17619.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[284]}, {checkpoints,[{284,10}]}, {name,<<"rebalance_284">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.65",11209}, [{username,"default"}, {password,[]}, {vbuckets,[284]}, {takeover,true}, {suffix,"284"}]} [rebalance:info,2012-11-02T18:51:41.051,ns_1@10.2.1.67:<0.17619.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:51:41.056,ns_1@10.2.1.67:<0.17619.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:51:41.138,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:51:40.919877 PDT 3: TAP (Producer) eq_tapq:replication_building_284_'ns_1@10.2.1.65' - Backfill is completed with VBuckets 284, memcached<0.1519.0>: Fri Nov 2 18:51:40.919901 PDT 3: TAP (Producer) eq_tapq:replication_building_284_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 284 memcached<0.1519.0>: Fri Nov 2 18:51:41.015310 PDT 3: TAP (Producer) eq_tapq:replication_building_284_'ns_1@10.2.1.65' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:51:41.051042 PDT 3: TAP (Producer) eq_tapq:rebalance_284 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:41.051059 PDT 3: TAP (Producer) eq_tapq:rebalance_284 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:41.051065 PDT 3: TAP (Producer) eq_tapq:rebalance_284 - Sending TAP_VBUCKET_SET with vbucket 284 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:51:41.055664 PDT 3: TAP (Producer) eq_tapq:rebalance_284 - VBucket <284> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:51:41.055684 PDT 3: TAP (Producer) eq_tapq:rebalance_284 - Sending TAP_VBUCKET_SET with vbucket 284 and state "active" memcached<0.1519.0>: Fri Nov 2 18:51:41.056036 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:51:41.056069 PDT 3: TAP (Producer) eq_tapq:rebalance_284 - disconnected memcached<0.1519.0>: Fri Nov 2 18:51:41.069311 PDT 3: TAP (Producer) eq_tapq:replication_building_284_'ns_1@10.2.1.65' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:51:41.079847 PDT 3: Deletion of vbucket 284 was completed. memcached<0.1519.0>: Fri Nov 2 18:51:41.091704 PDT 3: TAP (Producer) eq_tapq:replication_building_285_'ns_1@10.2.1.65' - Schedule the backfill for vbucket 285 memcached<0.1519.0>: Fri Nov 2 18:51:41.091739 PDT 3: TAP (Producer) eq_tapq:replication_building_285_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:41.091747 PDT 3: TAP (Producer) eq_tapq:replication_building_285_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:41.091752 PDT 3: TAP (Producer) eq_tapq:replication_building_285_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 285 memcached<0.1519.0>: Fri Nov 2 18:51:41.091812 PDT 3: Schedule cleanup of "eq_tapq:anon_29" memcached<0.1519.0>: Fri Nov 2 18:51:41.091831 PDT 3: Schedule cleanup of "eq_tapq:rebalance_284" memcached<0.1519.0>: Fri Nov 2 18:51:41.106653 PDT 3: TAP (Producer) eq_tapq:replication_building_284_'ns_1@10.2.1.65' - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:51:41.106699 PDT 3: TAP (Producer) eq_tapq:rebalance_284 - Clear the tap queues by force [ns_server:info,2012-11-02T18:51:41.140,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/284 [ns_server:info,2012-11-02T18:51:42.089,ns_1@10.2.1.67:<0.17624.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.65",11209} vbucket 285 to state replica [rebalance:info,2012-11-02T18:51:42.093,ns_1@10.2.1.67:<0.17624.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[285]}, {checkpoints,[{285,10}]}, {name,<<"rebalance_285">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.65",11209}, [{username,"default"}, {password,[]}, {vbuckets,[285]}, {takeover,true}, {suffix,"285"}]} [rebalance:info,2012-11-02T18:51:42.100,ns_1@10.2.1.67:<0.17624.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:51:42.104,ns_1@10.2.1.67:<0.17624.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:51:42.182,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/285 [ns_server:info,2012-11-02T18:51:42.197,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:51:41.996994 PDT 3: TAP (Producer) eq_tapq:replication_building_285_'ns_1@10.2.1.65' - Backfill is completed with VBuckets 285, memcached<0.1519.0>: Fri Nov 2 18:51:41.997030 PDT 3: TAP (Producer) eq_tapq:replication_building_285_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 285 memcached<0.1519.0>: Fri Nov 2 18:51:42.087514 PDT 3: TAP (Producer) eq_tapq:replication_building_285_'ns_1@10.2.1.65' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:51:42.093432 PDT 3: TAP (Producer) eq_tapq:rebalance_285 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:42.093444 PDT 3: TAP (Producer) eq_tapq:rebalance_285 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:42.093450 PDT 3: TAP (Producer) eq_tapq:rebalance_285 - Sending TAP_VBUCKET_SET with vbucket 285 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:51:42.103721 PDT 3: TAP (Producer) eq_tapq:rebalance_285 - VBucket <285> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:51:42.103753 PDT 3: TAP (Producer) eq_tapq:rebalance_285 - Sending TAP_VBUCKET_SET with vbucket 285 and state "active" memcached<0.1519.0>: Fri Nov 2 18:51:42.104267 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:51:42.104314 PDT 3: TAP (Producer) eq_tapq:rebalance_285 - disconnected memcached<0.1519.0>: Fri Nov 2 18:51:42.104375 PDT 3: Schedule cleanup of "eq_tapq:rebalance_285" memcached<0.1519.0>: Fri Nov 2 18:51:42.104447 PDT 3: TAP (Producer) eq_tapq:rebalance_285 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:51:42.115396 PDT 3: TAP (Producer) eq_tapq:replication_building_285_'ns_1@10.2.1.65' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:51:42.123181 PDT 3: Deletion of vbucket 285 was completed. memcached<0.1519.0>: Fri Nov 2 18:51:42.132339 PDT 3: TAP (Producer) eq_tapq:replication_building_286_'ns_1@10.2.1.65' - Schedule the backfill for vbucket 286 memcached<0.1519.0>: Fri Nov 2 18:51:42.132364 PDT 3: TAP (Producer) eq_tapq:replication_building_286_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:42.132372 PDT 3: TAP (Producer) eq_tapq:replication_building_286_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:42.132383 PDT 3: TAP (Producer) eq_tapq:replication_building_286_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 286 memcached<0.1519.0>: Fri Nov 2 18:51:42.132566 PDT 3: Schedule cleanup of "eq_tapq:anon_30" memcached<0.1519.0>: Fri Nov 2 18:51:42.148054 PDT 3: TAP (Producer) eq_tapq:replication_building_285_'ns_1@10.2.1.65' - Clear the tap queues by force [ns_server:info,2012-11-02T18:51:43.608,ns_1@10.2.1.67:<0.17634.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.65",11209} vbucket 286 to state replica [rebalance:info,2012-11-02T18:51:43.619,ns_1@10.2.1.67:<0.17634.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[286]}, {checkpoints,[{286,10}]}, {name,<<"rebalance_286">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.65",11209}, [{username,"default"}, {password,[]}, {vbuckets,[286]}, {takeover,true}, {suffix,"286"}]} [rebalance:info,2012-11-02T18:51:43.622,ns_1@10.2.1.67:<0.17634.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:51:43.625,ns_1@10.2.1.67:<0.17634.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:51:43.735,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/286 [ns_server:info,2012-11-02T18:51:43.763,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:51:43.563177 PDT 3: TAP (Producer) eq_tapq:replication_building_286_'ns_1@10.2.1.65' - Backfill is completed with VBuckets 286, memcached<0.1519.0>: Fri Nov 2 18:51:43.563200 PDT 3: TAP (Producer) eq_tapq:replication_building_286_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 286 memcached<0.1519.0>: Fri Nov 2 18:51:43.607523 PDT 3: TAP (Producer) eq_tapq:replication_building_286_'ns_1@10.2.1.65' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:51:43.620341 PDT 3: TAP (Producer) eq_tapq:rebalance_286 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:43.620356 PDT 3: TAP (Producer) eq_tapq:rebalance_286 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:43.620360 PDT 3: TAP (Producer) eq_tapq:rebalance_286 - Sending TAP_VBUCKET_SET with vbucket 286 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:51:43.625041 PDT 3: TAP (Producer) eq_tapq:rebalance_286 - VBucket <286> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:51:43.625068 PDT 3: TAP (Producer) eq_tapq:rebalance_286 - Sending TAP_VBUCKET_SET with vbucket 286 and state "active" memcached<0.1519.0>: Fri Nov 2 18:51:43.625390 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:51:43.625433 PDT 3: TAP (Producer) eq_tapq:rebalance_286 - disconnected memcached<0.1519.0>: Fri Nov 2 18:51:43.637172 PDT 3: TAP (Producer) eq_tapq:replication_building_286_'ns_1@10.2.1.65' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:51:43.646123 PDT 3: Deletion of vbucket 286 was completed. memcached<0.1519.0>: Fri Nov 2 18:51:43.656923 PDT 3: TAP (Producer) eq_tapq:replication_building_287_'ns_1@10.2.1.65' - Schedule the backfill for vbucket 287 memcached<0.1519.0>: Fri Nov 2 18:51:43.656959 PDT 3: TAP (Producer) eq_tapq:replication_building_287_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:43.656967 PDT 3: TAP (Producer) eq_tapq:replication_building_287_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:43.656972 PDT 3: TAP (Producer) eq_tapq:replication_building_287_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 287 memcached<0.1519.0>: Fri Nov 2 18:51:43.657057 PDT 3: Schedule cleanup of "eq_tapq:anon_31" memcached<0.1519.0>: Fri Nov 2 18:51:43.657076 PDT 3: Schedule cleanup of "eq_tapq:rebalance_286" memcached<0.1519.0>: Fri Nov 2 18:51:43.667688 PDT 3: TAP (Producer) eq_tapq:replication_building_286_'ns_1@10.2.1.65' - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:51:43.667731 PDT 3: TAP (Producer) eq_tapq:rebalance_286 - Clear the tap queues by force [ns_server:info,2012-11-02T18:51:44.625,ns_1@10.2.1.67:<0.17640.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.65",11209} vbucket 287 to state replica [rebalance:info,2012-11-02T18:51:44.631,ns_1@10.2.1.67:<0.17640.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[287]}, {checkpoints,[{287,10}]}, {name,<<"rebalance_287">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.65",11209}, [{username,"default"}, {password,[]}, {vbuckets,[287]}, {takeover,true}, {suffix,"287"}]} [rebalance:info,2012-11-02T18:51:44.632,ns_1@10.2.1.67:<0.17640.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:51:44.636,ns_1@10.2.1.67:<0.17640.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:51:44.710,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/287 [ns_server:info,2012-11-02T18:51:44.715,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:51:44.515008 PDT 3: TAP (Producer) eq_tapq:replication_building_287_'ns_1@10.2.1.65' - Backfill is completed with VBuckets 287, memcached<0.1519.0>: Fri Nov 2 18:51:44.515032 PDT 3: TAP (Producer) eq_tapq:replication_building_287_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 287 memcached<0.1519.0>: Fri Nov 2 18:51:44.619002 PDT 3: TAP (Producer) eq_tapq:replication_building_287_'ns_1@10.2.1.65' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:51:44.631808 PDT 3: TAP (Producer) eq_tapq:rebalance_287 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:44.631822 PDT 3: TAP (Producer) eq_tapq:rebalance_287 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:44.631827 PDT 3: TAP (Producer) eq_tapq:rebalance_287 - Sending TAP_VBUCKET_SET with vbucket 287 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:51:44.635147 PDT 3: TAP (Producer) eq_tapq:rebalance_287 - VBucket <287> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:51:44.635175 PDT 3: TAP (Producer) eq_tapq:rebalance_287 - Sending TAP_VBUCKET_SET with vbucket 287 and state "active" memcached<0.1519.0>: Fri Nov 2 18:51:44.636057 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:51:44.636104 PDT 3: TAP (Producer) eq_tapq:rebalance_287 - disconnected memcached<0.1519.0>: Fri Nov 2 18:51:44.636173 PDT 3: Schedule cleanup of "eq_tapq:rebalance_287" memcached<0.1519.0>: Fri Nov 2 18:51:44.637232 PDT 3: TAP (Producer) eq_tapq:rebalance_287 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:51:44.646401 PDT 3: TAP (Producer) eq_tapq:replication_building_287_'ns_1@10.2.1.65' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:51:44.654231 PDT 3: Deletion of vbucket 287 was completed. memcached<0.1519.0>: Fri Nov 2 18:51:44.676885 PDT 3: TAP (Producer) eq_tapq:replication_building_288_'ns_1@10.2.1.65' - Schedule the backfill for vbucket 288 memcached<0.1519.0>: Fri Nov 2 18:51:44.676926 PDT 3: TAP (Producer) eq_tapq:replication_building_288_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:44.676941 PDT 3: TAP (Producer) eq_tapq:replication_building_288_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:44.676951 PDT 3: TAP (Producer) eq_tapq:replication_building_288_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 288 memcached<0.1519.0>: Fri Nov 2 18:51:44.677796 PDT 3: Schedule cleanup of "eq_tapq:anon_32" memcached<0.1519.0>: Fri Nov 2 18:51:44.677954 PDT 3: TAP (Producer) eq_tapq:replication_building_287_'ns_1@10.2.1.65' - Clear the tap queues by force [ns_server:info,2012-11-02T18:51:45.615,ns_1@10.2.1.67:<0.17651.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.65",11209} vbucket 288 to state replica [rebalance:info,2012-11-02T18:51:45.618,ns_1@10.2.1.67:<0.17651.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[288]}, {checkpoints,[{288,10}]}, {name,<<"rebalance_288">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.65",11209}, [{username,"default"}, {password,[]}, {vbuckets,[288]}, {takeover,true}, {suffix,"288"}]} [rebalance:info,2012-11-02T18:51:45.620,ns_1@10.2.1.67:<0.17651.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:51:45.628,ns_1@10.2.1.67:<0.17651.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:51:45.718,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:51:45.481928 PDT 3: TAP (Producer) eq_tapq:replication_building_288_'ns_1@10.2.1.65' - Backfill is completed with VBuckets 288, memcached<0.1519.0>: Fri Nov 2 18:51:45.481950 PDT 3: TAP (Producer) eq_tapq:replication_building_288_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 288 memcached<0.1519.0>: Fri Nov 2 18:51:45.580588 PDT 3: TAP (Producer) eq_tapq:replication_building_288_'ns_1@10.2.1.65' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:51:45.619167 PDT 3: TAP (Producer) eq_tapq:rebalance_288 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:45.619181 PDT 3: TAP (Producer) eq_tapq:rebalance_288 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:45.619186 PDT 3: TAP (Producer) eq_tapq:rebalance_288 - Sending TAP_VBUCKET_SET with vbucket 288 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:51:45.625016 PDT 3: TAP (Producer) eq_tapq:rebalance_288 - VBucket <288> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:51:45.625035 PDT 3: TAP (Producer) eq_tapq:rebalance_288 - Sending TAP_VBUCKET_SET with vbucket 288 and state "active" memcached<0.1519.0>: Fri Nov 2 18:51:45.628461 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:51:45.628486 PDT 3: TAP (Producer) eq_tapq:rebalance_288 - disconnected memcached<0.1519.0>: Fri Nov 2 18:51:45.628522 PDT 3: Schedule cleanup of "eq_tapq:rebalance_288" memcached<0.1519.0>: Fri Nov 2 18:51:45.628546 PDT 3: TAP (Producer) eq_tapq:rebalance_288 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:51:45.639998 PDT 3: TAP (Producer) eq_tapq:replication_building_288_'ns_1@10.2.1.65' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:51:45.648949 PDT 3: Deletion of vbucket 288 was completed. [ns_server:info,2012-11-02T18:51:45.719,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/288 [ns_server:info,2012-11-02T18:51:45.918,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:51:45.662349 PDT 3: TAP (Producer) eq_tapq:replication_building_289_'ns_1@10.2.1.65' - Schedule the backfill for vbucket 289 memcached<0.1519.0>: Fri Nov 2 18:51:45.662377 PDT 3: TAP (Producer) eq_tapq:replication_building_289_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:45.662385 PDT 3: TAP (Producer) eq_tapq:replication_building_289_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:45.662389 PDT 3: TAP (Producer) eq_tapq:replication_building_289_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 289 memcached<0.1519.0>: Fri Nov 2 18:51:45.662433 PDT 3: Schedule cleanup of "eq_tapq:anon_33" memcached<0.1519.0>: Fri Nov 2 18:51:45.666262 PDT 3: TAP (Producer) eq_tapq:replication_building_288_'ns_1@10.2.1.65' - Clear the tap queues by force [ns_server:info,2012-11-02T18:51:46.698,ns_1@10.2.1.67:<0.17670.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.65",11209} vbucket 289 to state replica [rebalance:info,2012-11-02T18:51:46.710,ns_1@10.2.1.67:<0.17670.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[289]}, {checkpoints,[{289,10}]}, {name,<<"rebalance_289">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.65",11209}, [{username,"default"}, {password,[]}, {vbuckets,[289]}, {takeover,true}, {suffix,"289"}]} [rebalance:info,2012-11-02T18:51:46.711,ns_1@10.2.1.67:<0.17670.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:51:46.715,ns_1@10.2.1.67:<0.17670.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:51:46.792,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/289 [ns_server:info,2012-11-02T18:51:46.799,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:51:46.598791 PDT 3: TAP (Producer) eq_tapq:replication_building_289_'ns_1@10.2.1.65' - Backfill is completed with VBuckets 289, memcached<0.1519.0>: Fri Nov 2 18:51:46.598809 PDT 3: TAP (Producer) eq_tapq:replication_building_289_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 289 memcached<0.1519.0>: Fri Nov 2 18:51:46.695913 PDT 3: TAP (Producer) eq_tapq:replication_building_289_'ns_1@10.2.1.65' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:51:46.711036 PDT 3: TAP (Producer) eq_tapq:rebalance_289 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:46.711050 PDT 3: TAP (Producer) eq_tapq:rebalance_289 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:46.711056 PDT 3: TAP (Producer) eq_tapq:rebalance_289 - Sending TAP_VBUCKET_SET with vbucket 289 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:51:46.714427 PDT 3: TAP (Producer) eq_tapq:rebalance_289 - VBucket <289> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:51:46.714443 PDT 3: TAP (Producer) eq_tapq:rebalance_289 - Sending TAP_VBUCKET_SET with vbucket 289 and state "active" memcached<0.1519.0>: Fri Nov 2 18:51:46.714906 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:51:46.714928 PDT 3: TAP (Producer) eq_tapq:rebalance_289 - disconnected memcached<0.1519.0>: Fri Nov 2 18:51:46.723512 PDT 3: TAP (Producer) eq_tapq:replication_building_289_'ns_1@10.2.1.65' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:51:46.732594 PDT 3: Deletion of vbucket 289 was completed. memcached<0.1519.0>: Fri Nov 2 18:51:46.746984 PDT 3: TAP (Producer) eq_tapq:replication_building_290_'ns_1@10.2.1.65' - Schedule the backfill for vbucket 290 memcached<0.1519.0>: Fri Nov 2 18:51:46.747009 PDT 3: TAP (Producer) eq_tapq:replication_building_290_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:46.747016 PDT 3: TAP (Producer) eq_tapq:replication_building_290_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:46.747021 PDT 3: TAP (Producer) eq_tapq:replication_building_290_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 290 memcached<0.1519.0>: Fri Nov 2 18:51:46.747094 PDT 3: Schedule cleanup of "eq_tapq:anon_34" memcached<0.1519.0>: Fri Nov 2 18:51:46.747109 PDT 3: Schedule cleanup of "eq_tapq:rebalance_289" memcached<0.1519.0>: Fri Nov 2 18:51:46.750562 PDT 3: TAP (Producer) eq_tapq:replication_building_289_'ns_1@10.2.1.65' - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:51:46.750582 PDT 3: TAP (Producer) eq_tapq:rebalance_289 - Clear the tap queues by force [ns_server:info,2012-11-02T18:51:47.173,ns_1@10.2.1.67:ns_config_rep<0.1439.0>:ns_config_rep:do_pull:341]Pulling config from: 'ns_1@10.2.1.65' [ns_server:info,2012-11-02T18:51:47.640,ns_1@10.2.1.67:<0.17677.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.65",11209} vbucket 290 to state replica [rebalance:info,2012-11-02T18:51:47.645,ns_1@10.2.1.67:<0.17677.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[290]}, {checkpoints,[{290,10}]}, {name,<<"rebalance_290">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.65",11209}, [{username,"default"}, {password,[]}, {vbuckets,[290]}, {takeover,true}, {suffix,"290"}]} [rebalance:info,2012-11-02T18:51:47.648,ns_1@10.2.1.67:<0.17677.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:51:47.656,ns_1@10.2.1.67:<0.17677.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:51:47.721,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/290 [ns_server:info,2012-11-02T18:51:47.762,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:51:47.561698 PDT 3: TAP (Producer) eq_tapq:replication_building_290_'ns_1@10.2.1.65' - Backfill is completed with VBuckets 290, memcached<0.1519.0>: Fri Nov 2 18:51:47.561721 PDT 3: TAP (Producer) eq_tapq:replication_building_290_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 290 memcached<0.1519.0>: Fri Nov 2 18:51:47.637346 PDT 3: TAP (Producer) eq_tapq:replication_building_290_'ns_1@10.2.1.65' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:51:47.646277 PDT 3: TAP (Producer) eq_tapq:rebalance_290 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:47.646289 PDT 3: TAP (Producer) eq_tapq:rebalance_290 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:47.646294 PDT 3: TAP (Producer) eq_tapq:rebalance_290 - Sending TAP_VBUCKET_SET with vbucket 290 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:51:47.655198 PDT 3: TAP (Producer) eq_tapq:rebalance_290 - VBucket <290> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:51:47.655232 PDT 3: TAP (Producer) eq_tapq:rebalance_290 - Sending TAP_VBUCKET_SET with vbucket 290 and state "active" memcached<0.1519.0>: Fri Nov 2 18:51:47.656445 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:51:47.656505 PDT 3: TAP (Producer) eq_tapq:rebalance_290 - disconnected memcached<0.1519.0>: Fri Nov 2 18:51:47.665678 PDT 3: TAP (Producer) eq_tapq:replication_building_290_'ns_1@10.2.1.65' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:51:47.674305 PDT 3: Deletion of vbucket 290 was completed. memcached<0.1519.0>: Fri Nov 2 18:51:47.683822 PDT 3: TAP (Producer) eq_tapq:replication_building_291_'ns_1@10.2.1.65' - Schedule the backfill for vbucket 291 memcached<0.1519.0>: Fri Nov 2 18:51:47.683848 PDT 3: TAP (Producer) eq_tapq:replication_building_291_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:47.683856 PDT 3: TAP (Producer) eq_tapq:replication_building_291_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:47.683860 PDT 3: TAP (Producer) eq_tapq:replication_building_291_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 291 memcached<0.1519.0>: Fri Nov 2 18:51:47.683938 PDT 3: Schedule cleanup of "eq_tapq:anon_35" memcached<0.1519.0>: Fri Nov 2 18:51:47.683949 PDT 3: Schedule cleanup of "eq_tapq:rebalance_290" memcached<0.1519.0>: Fri Nov 2 18:51:47.699955 PDT 3: TAP (Producer) eq_tapq:replication_building_290_'ns_1@10.2.1.65' - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:51:47.700000 PDT 3: TAP (Producer) eq_tapq:rebalance_290 - Clear the tap queues by force [ns_server:info,2012-11-02T18:51:48.450,ns_1@10.2.1.67:<0.17682.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.65",11209} vbucket 291 to state replica [rebalance:info,2012-11-02T18:51:48.460,ns_1@10.2.1.67:<0.17682.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[291]}, {checkpoints,[{291,10}]}, {name,<<"rebalance_291">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.65",11209}, [{username,"default"}, {password,[]}, {vbuckets,[291]}, {takeover,true}, {suffix,"291"}]} [rebalance:info,2012-11-02T18:51:48.462,ns_1@10.2.1.67:<0.17682.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:51:48.465,ns_1@10.2.1.67:<0.17682.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:51:48.508,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/291 [ns_server:info,2012-11-02T18:51:48.576,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:51:48.376483 PDT 3: TAP (Producer) eq_tapq:replication_building_291_'ns_1@10.2.1.65' - Backfill is completed with VBuckets 291, memcached<0.1519.0>: Fri Nov 2 18:51:48.376506 PDT 3: TAP (Producer) eq_tapq:replication_building_291_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 291 memcached<0.1519.0>: Fri Nov 2 18:51:48.447700 PDT 3: TAP (Producer) eq_tapq:replication_building_291_'ns_1@10.2.1.65' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:51:48.461177 PDT 3: TAP (Producer) eq_tapq:rebalance_291 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:48.461201 PDT 3: TAP (Producer) eq_tapq:rebalance_291 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:48.461211 PDT 3: TAP (Producer) eq_tapq:rebalance_291 - Sending TAP_VBUCKET_SET with vbucket 291 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:51:48.465082 PDT 3: TAP (Producer) eq_tapq:rebalance_291 - VBucket <291> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:51:48.465114 PDT 3: TAP (Producer) eq_tapq:rebalance_291 - Sending TAP_VBUCKET_SET with vbucket 291 and state "active" memcached<0.1519.0>: Fri Nov 2 18:51:48.465575 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:51:48.465622 PDT 3: TAP (Producer) eq_tapq:rebalance_291 - disconnected memcached<0.1519.0>: Fri Nov 2 18:51:48.470132 PDT 3: TAP (Producer) eq_tapq:replication_building_291_'ns_1@10.2.1.65' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:51:48.477381 PDT 3: Deletion of vbucket 291 was completed. memcached<0.1519.0>: Fri Nov 2 18:51:48.492926 PDT 3: TAP (Producer) eq_tapq:replication_building_292_'ns_1@10.2.1.65' - Schedule the backfill for vbucket 292 memcached<0.1519.0>: Fri Nov 2 18:51:48.492957 PDT 3: TAP (Producer) eq_tapq:replication_building_292_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:48.492966 PDT 3: TAP (Producer) eq_tapq:replication_building_292_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:48.492972 PDT 3: TAP (Producer) eq_tapq:replication_building_292_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 292 memcached<0.1519.0>: Fri Nov 2 18:51:48.493190 PDT 3: Schedule cleanup of "eq_tapq:anon_36" memcached<0.1519.0>: Fri Nov 2 18:51:48.493206 PDT 3: Schedule cleanup of "eq_tapq:rebalance_291" memcached<0.1519.0>: Fri Nov 2 18:51:48.502297 PDT 3: TAP (Producer) eq_tapq:replication_building_291_'ns_1@10.2.1.65' - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:51:48.502339 PDT 3: TAP (Producer) eq_tapq:rebalance_291 - Clear the tap queues by force [ns_server:info,2012-11-02T18:51:49.152,ns_1@10.2.1.67:<0.17692.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.65",11209} vbucket 292 to state replica [rebalance:info,2012-11-02T18:51:49.155,ns_1@10.2.1.67:<0.17692.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[292]}, {checkpoints,[{292,10}]}, {name,<<"rebalance_292">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.65",11209}, [{username,"default"}, {password,[]}, {vbuckets,[292]}, {takeover,true}, {suffix,"292"}]} [rebalance:info,2012-11-02T18:51:49.156,ns_1@10.2.1.67:<0.17692.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:51:49.164,ns_1@10.2.1.67:<0.17692.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:51:49.222,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/292 [ns_server:info,2012-11-02T18:51:49.333,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:51:49.132274 PDT 3: TAP (Producer) eq_tapq:replication_building_292_'ns_1@10.2.1.65' - Backfill is completed with VBuckets 292, memcached<0.1519.0>: Fri Nov 2 18:51:49.132296 PDT 3: TAP (Producer) eq_tapq:replication_building_292_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 292 memcached<0.1519.0>: Fri Nov 2 18:51:49.150053 PDT 3: TAP (Producer) eq_tapq:replication_building_292_'ns_1@10.2.1.65' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:51:49.156020 PDT 3: TAP (Producer) eq_tapq:rebalance_292 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:49.156035 PDT 3: TAP (Producer) eq_tapq:rebalance_292 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:49.156040 PDT 3: TAP (Producer) eq_tapq:rebalance_292 - Sending TAP_VBUCKET_SET with vbucket 292 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:51:49.160002 PDT 3: TAP (Producer) eq_tapq:rebalance_292 - VBucket <292> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:51:49.160038 PDT 3: TAP (Producer) eq_tapq:rebalance_292 - Sending TAP_VBUCKET_SET with vbucket 292 and state "active" memcached<0.1519.0>: Fri Nov 2 18:51:49.164051 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:51:49.164095 PDT 3: TAP (Producer) eq_tapq:rebalance_292 - disconnected memcached<0.1519.0>: Fri Nov 2 18:51:49.168455 PDT 3: TAP (Producer) eq_tapq:replication_building_292_'ns_1@10.2.1.65' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:51:49.173191 PDT 3: Deletion of vbucket 292 was completed. memcached<0.1519.0>: Fri Nov 2 18:51:49.181808 PDT 3: TAP (Producer) eq_tapq:replication_building_293_'ns_1@10.2.1.65' - Schedule the backfill for vbucket 293 memcached<0.1519.0>: Fri Nov 2 18:51:49.181833 PDT 3: TAP (Producer) eq_tapq:replication_building_293_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:49.181841 PDT 3: TAP (Producer) eq_tapq:replication_building_293_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:49.181846 PDT 3: TAP (Producer) eq_tapq:replication_building_293_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 293 memcached<0.1519.0>: Fri Nov 2 18:51:49.181919 PDT 3: Schedule cleanup of "eq_tapq:anon_37" memcached<0.1519.0>: Fri Nov 2 18:51:49.181936 PDT 3: Schedule cleanup of "eq_tapq:rebalance_292" memcached<0.1519.0>: Fri Nov 2 18:51:49.189904 PDT 3: TAP (Producer) eq_tapq:replication_building_292_'ns_1@10.2.1.65' - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:51:49.189931 PDT 3: TAP (Producer) eq_tapq:rebalance_292 - Clear the tap queues by force [ns_server:info,2012-11-02T18:51:49.900,ns_1@10.2.1.67:<0.17698.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.65",11209} vbucket 293 to state replica [rebalance:info,2012-11-02T18:51:49.908,ns_1@10.2.1.67:<0.17698.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[293]}, {checkpoints,[{293,10}]}, {name,<<"rebalance_293">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.65",11209}, [{username,"default"}, {password,[]}, {vbuckets,[293]}, {takeover,true}, {suffix,"293"}]} [rebalance:info,2012-11-02T18:51:49.909,ns_1@10.2.1.67:<0.17698.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:51:49.915,ns_1@10.2.1.67:<0.17698.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:51:50.002,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/293 [ns_server:info,2012-11-02T18:51:50.049,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:51:49.849014 PDT 3: TAP (Producer) eq_tapq:replication_building_293_'ns_1@10.2.1.65' - Backfill is completed with VBuckets 293, memcached<0.1519.0>: Fri Nov 2 18:51:49.849036 PDT 3: TAP (Producer) eq_tapq:replication_building_293_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 293 memcached<0.1519.0>: Fri Nov 2 18:51:49.896308 PDT 3: TAP (Producer) eq_tapq:replication_building_293_'ns_1@10.2.1.65' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:51:49.908452 PDT 3: TAP (Producer) eq_tapq:rebalance_293 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:49.908470 PDT 3: TAP (Producer) eq_tapq:rebalance_293 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:49.908475 PDT 3: TAP (Producer) eq_tapq:rebalance_293 - Sending TAP_VBUCKET_SET with vbucket 293 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:51:49.914898 PDT 3: TAP (Producer) eq_tapq:rebalance_293 - VBucket <293> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:51:49.914930 PDT 3: TAP (Producer) eq_tapq:rebalance_293 - Sending TAP_VBUCKET_SET with vbucket 293 and state "active" memcached<0.1519.0>: Fri Nov 2 18:51:49.915421 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:51:49.915456 PDT 3: TAP (Producer) eq_tapq:rebalance_293 - disconnected memcached<0.1519.0>: Fri Nov 2 18:51:49.915585 PDT 3: Schedule cleanup of "eq_tapq:rebalance_293" memcached<0.1519.0>: Fri Nov 2 18:51:49.915614 PDT 3: TAP (Producer) eq_tapq:rebalance_293 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:51:49.944438 PDT 3: TAP (Producer) eq_tapq:replication_building_293_'ns_1@10.2.1.65' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:51:49.953213 PDT 3: Deletion of vbucket 293 was completed. memcached<0.1519.0>: Fri Nov 2 18:51:49.962789 PDT 3: TAP (Producer) eq_tapq:replication_building_294_'ns_1@10.2.1.65' - Schedule the backfill for vbucket 294 memcached<0.1519.0>: Fri Nov 2 18:51:49.962814 PDT 3: TAP (Producer) eq_tapq:replication_building_294_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:49.962821 PDT 3: TAP (Producer) eq_tapq:replication_building_294_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:49.962826 PDT 3: TAP (Producer) eq_tapq:replication_building_294_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 294 memcached<0.1519.0>: Fri Nov 2 18:51:49.962847 PDT 3: Schedule cleanup of "eq_tapq:anon_38" memcached<0.1519.0>: Fri Nov 2 18:51:49.975665 PDT 3: TAP (Producer) eq_tapq:replication_building_293_'ns_1@10.2.1.65' - Clear the tap queues by force [ns_server:info,2012-11-02T18:51:51.020,ns_1@10.2.1.67:<0.17717.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.65",11209} vbucket 294 to state replica [rebalance:info,2012-11-02T18:51:51.029,ns_1@10.2.1.67:<0.17717.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[294]}, {checkpoints,[{294,10}]}, {name,<<"rebalance_294">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.65",11209}, [{username,"default"}, {password,[]}, {vbuckets,[294]}, {takeover,true}, {suffix,"294"}]} [rebalance:info,2012-11-02T18:51:51.030,ns_1@10.2.1.67:<0.17717.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:51:51.035,ns_1@10.2.1.67:<0.17717.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:51:51.118,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/294 [ns_server:info,2012-11-02T18:51:51.167,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:51:50.966704 PDT 3: TAP (Producer) eq_tapq:replication_building_294_'ns_1@10.2.1.65' - Backfill is completed with VBuckets 294, memcached<0.1519.0>: Fri Nov 2 18:51:50.966726 PDT 3: TAP (Producer) eq_tapq:replication_building_294_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 294 memcached<0.1519.0>: Fri Nov 2 18:51:51.016511 PDT 3: TAP (Producer) eq_tapq:replication_building_294_'ns_1@10.2.1.65' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:51:51.029328 PDT 3: TAP (Producer) eq_tapq:rebalance_294 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:51.029342 PDT 3: TAP (Producer) eq_tapq:rebalance_294 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:51.029347 PDT 3: TAP (Producer) eq_tapq:rebalance_294 - Sending TAP_VBUCKET_SET with vbucket 294 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:51:51.034376 PDT 3: TAP (Producer) eq_tapq:rebalance_294 - VBucket <294> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:51:51.034403 PDT 3: TAP (Producer) eq_tapq:rebalance_294 - Sending TAP_VBUCKET_SET with vbucket 294 and state "active" memcached<0.1519.0>: Fri Nov 2 18:51:51.034800 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:51:51.034843 PDT 3: TAP (Producer) eq_tapq:rebalance_294 - disconnected memcached<0.1519.0>: Fri Nov 2 18:51:51.034999 PDT 3: Schedule cleanup of "eq_tapq:rebalance_294" memcached<0.1519.0>: Fri Nov 2 18:51:51.035109 PDT 3: TAP (Producer) eq_tapq:rebalance_294 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:51:51.048760 PDT 3: TAP (Producer) eq_tapq:replication_building_294_'ns_1@10.2.1.65' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:51:51.055283 PDT 3: Deletion of vbucket 294 was completed. memcached<0.1519.0>: Fri Nov 2 18:51:51.066114 PDT 3: TAP (Producer) eq_tapq:replication_building_295_'ns_1@10.2.1.65' - Schedule the backfill for vbucket 295 memcached<0.1519.0>: Fri Nov 2 18:51:51.066140 PDT 3: TAP (Producer) eq_tapq:replication_building_295_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:51.066148 PDT 3: TAP (Producer) eq_tapq:replication_building_295_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:51.066153 PDT 3: TAP (Producer) eq_tapq:replication_building_295_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 295 memcached<0.1519.0>: Fri Nov 2 18:51:51.066205 PDT 3: Schedule cleanup of "eq_tapq:anon_39" memcached<0.1519.0>: Fri Nov 2 18:51:51.072167 PDT 3: TAP (Producer) eq_tapq:replication_building_294_'ns_1@10.2.1.65' - Clear the tap queues by force [ns_server:info,2012-11-02T18:51:52.114,ns_1@10.2.1.67:<0.17722.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.65",11209} vbucket 295 to state replica [rebalance:info,2012-11-02T18:51:52.117,ns_1@10.2.1.67:<0.17722.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[295]}, {checkpoints,[{295,10}]}, {name,<<"rebalance_295">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.65",11209}, [{username,"default"}, {password,[]}, {vbuckets,[295]}, {takeover,true}, {suffix,"295"}]} [rebalance:info,2012-11-02T18:51:52.122,ns_1@10.2.1.67:<0.17722.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:51:52.125,ns_1@10.2.1.67:<0.17722.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:51:52.206,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/295 [ns_server:info,2012-11-02T18:51:52.228,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:51:52.027994 PDT 3: TAP (Producer) eq_tapq:replication_building_295_'ns_1@10.2.1.65' - Backfill is completed with VBuckets 295, memcached<0.1519.0>: Fri Nov 2 18:51:52.028051 PDT 3: TAP (Producer) eq_tapq:replication_building_295_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 295 memcached<0.1519.0>: Fri Nov 2 18:51:52.111145 PDT 3: TAP (Producer) eq_tapq:replication_building_295_'ns_1@10.2.1.65' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:51:52.117493 PDT 3: TAP (Producer) eq_tapq:rebalance_295 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:52.117505 PDT 3: TAP (Producer) eq_tapq:rebalance_295 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:52.117510 PDT 3: TAP (Producer) eq_tapq:rebalance_295 - Sending TAP_VBUCKET_SET with vbucket 295 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:51:52.125485 PDT 3: TAP (Producer) eq_tapq:rebalance_295 - VBucket <295> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:51:52.125503 PDT 3: TAP (Producer) eq_tapq:rebalance_295 - Sending TAP_VBUCKET_SET with vbucket 295 and state "active" memcached<0.1519.0>: Fri Nov 2 18:51:52.125648 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:51:52.125670 PDT 3: TAP (Producer) eq_tapq:rebalance_295 - disconnected memcached<0.1519.0>: Fri Nov 2 18:51:52.136793 PDT 3: TAP (Producer) eq_tapq:replication_building_295_'ns_1@10.2.1.65' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:51:52.146452 PDT 3: Deletion of vbucket 295 was completed. memcached<0.1519.0>: Fri Nov 2 18:51:52.156781 PDT 3: TAP (Producer) eq_tapq:replication_building_296_'ns_1@10.2.1.65' - Schedule the backfill for vbucket 296 memcached<0.1519.0>: Fri Nov 2 18:51:52.156819 PDT 3: TAP (Producer) eq_tapq:replication_building_296_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:52.156835 PDT 3: TAP (Producer) eq_tapq:replication_building_296_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:52.156845 PDT 3: TAP (Producer) eq_tapq:replication_building_296_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 296 memcached<0.1519.0>: Fri Nov 2 18:51:52.156950 PDT 3: Schedule cleanup of "eq_tapq:anon_40" memcached<0.1519.0>: Fri Nov 2 18:51:52.156983 PDT 3: Schedule cleanup of "eq_tapq:rebalance_295" memcached<0.1519.0>: Fri Nov 2 18:51:52.172071 PDT 3: TAP (Producer) eq_tapq:replication_building_295_'ns_1@10.2.1.65' - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:51:52.172102 PDT 3: TAP (Producer) eq_tapq:rebalance_295 - Clear the tap queues by force [ns_server:info,2012-11-02T18:51:53.416,ns_1@10.2.1.67:<0.17733.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.65",11209} vbucket 296 to state replica [rebalance:info,2012-11-02T18:51:53.421,ns_1@10.2.1.67:<0.17733.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[296]}, {checkpoints,[{296,9}]}, {name,<<"rebalance_296">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.65",11209}, [{username,"default"}, {password,[]}, {vbuckets,[296]}, {takeover,true}, {suffix,"296"}]} [rebalance:info,2012-11-02T18:51:53.424,ns_1@10.2.1.67:<0.17733.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:51:53.431,ns_1@10.2.1.67:<0.17733.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:51:53.559,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:51:53.358734 PDT 3: TAP (Producer) eq_tapq:replication_building_296_'ns_1@10.2.1.65' - Backfill is completed with VBuckets 296, memcached<0.1519.0>: Fri Nov 2 18:51:53.358772 PDT 3: TAP (Producer) eq_tapq:replication_building_296_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 296 memcached<0.1519.0>: Fri Nov 2 18:51:53.414505 PDT 3: TAP (Producer) eq_tapq:replication_building_296_'ns_1@10.2.1.65' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:51:53.421608 PDT 3: TAP (Producer) eq_tapq:rebalance_296 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:53.421620 PDT 3: TAP (Producer) eq_tapq:rebalance_296 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:53.421625 PDT 3: TAP (Producer) eq_tapq:rebalance_296 - Sending TAP_VBUCKET_SET with vbucket 296 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:51:53.427188 PDT 3: TAP (Producer) eq_tapq:rebalance_296 - VBucket <296> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:51:53.427209 PDT 3: TAP (Producer) eq_tapq:rebalance_296 - Sending TAP_VBUCKET_SET with vbucket 296 and state "active" memcached<0.1519.0>: Fri Nov 2 18:51:53.429800 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:51:53.429851 PDT 3: TAP (Producer) eq_tapq:rebalance_296 - disconnected memcached<0.1519.0>: Fri Nov 2 18:51:53.429980 PDT 3: Schedule cleanup of "eq_tapq:rebalance_296" memcached<0.1519.0>: Fri Nov 2 18:51:53.430112 PDT 3: TAP (Producer) eq_tapq:rebalance_296 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:51:53.444473 PDT 3: TAP (Producer) eq_tapq:replication_building_296_'ns_1@10.2.1.65' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:51:53.452685 PDT 3: Deletion of vbucket 296 was completed. memcached<0.1519.0>: Fri Nov 2 18:51:53.474867 PDT 3: TAP (Producer) eq_tapq:replication_building_297_'ns_1@10.2.1.65' - Schedule the backfill for vbucket 297 memcached<0.1519.0>: Fri Nov 2 18:51:53.474907 PDT 3: TAP (Producer) eq_tapq:replication_building_297_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:53.474922 PDT 3: TAP (Producer) eq_tapq:replication_building_297_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:53.474932 PDT 3: TAP (Producer) eq_tapq:replication_building_297_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 297 memcached<0.1519.0>: Fri Nov 2 18:51:53.475016 PDT 3: Schedule cleanup of "eq_tapq:anon_41" memcached<0.1519.0>: Fri Nov 2 18:51:53.475122 PDT 3: TAP (Producer) eq_tapq:replication_building_296_'ns_1@10.2.1.65' - Clear the tap queues by force [ns_server:info,2012-11-02T18:51:53.591,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/296 [ns_server:info,2012-11-02T18:51:54.434,ns_1@10.2.1.67:<0.17738.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.65",11209} vbucket 297 to state replica [rebalance:info,2012-11-02T18:51:54.452,ns_1@10.2.1.67:<0.17738.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[297]}, {checkpoints,[{297,10}]}, {name,<<"rebalance_297">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.65",11209}, [{username,"default"}, {password,[]}, {vbuckets,[297]}, {takeover,true}, {suffix,"297"}]} [rebalance:info,2012-11-02T18:51:54.454,ns_1@10.2.1.67:<0.17738.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:51:54.457,ns_1@10.2.1.67:<0.17738.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:51:54.565,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/297 [ns_server:info,2012-11-02T18:51:54.615,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:51:54.415074 PDT 3: TAP (Producer) eq_tapq:replication_building_297_'ns_1@10.2.1.65' - Backfill is completed with VBuckets 297, memcached<0.1519.0>: Fri Nov 2 18:51:54.415113 PDT 3: TAP (Producer) eq_tapq:replication_building_297_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 297 memcached<0.1519.0>: Fri Nov 2 18:51:54.431779 PDT 3: TAP (Producer) eq_tapq:replication_building_297_'ns_1@10.2.1.65' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:51:54.452476 PDT 3: TAP (Producer) eq_tapq:rebalance_297 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:54.452490 PDT 3: TAP (Producer) eq_tapq:rebalance_297 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:54.452495 PDT 3: TAP (Producer) eq_tapq:rebalance_297 - Sending TAP_VBUCKET_SET with vbucket 297 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:51:54.456954 PDT 3: TAP (Producer) eq_tapq:rebalance_297 - VBucket <297> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:51:54.456985 PDT 3: TAP (Producer) eq_tapq:rebalance_297 - Sending TAP_VBUCKET_SET with vbucket 297 and state "active" memcached<0.1519.0>: Fri Nov 2 18:51:54.457710 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:51:54.457740 PDT 3: TAP (Producer) eq_tapq:rebalance_297 - disconnected memcached<0.1519.0>: Fri Nov 2 18:51:54.466815 PDT 3: TAP (Producer) eq_tapq:replication_building_297_'ns_1@10.2.1.65' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:51:54.475506 PDT 3: Deletion of vbucket 297 was completed. memcached<0.1519.0>: Fri Nov 2 18:51:54.485520 PDT 3: TAP (Producer) eq_tapq:replication_building_298_'ns_1@10.2.1.65' - Schedule the backfill for vbucket 298 memcached<0.1519.0>: Fri Nov 2 18:51:54.485544 PDT 3: TAP (Producer) eq_tapq:replication_building_298_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:54.485552 PDT 3: TAP (Producer) eq_tapq:replication_building_298_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:54.485557 PDT 3: TAP (Producer) eq_tapq:replication_building_298_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 298 memcached<0.1519.0>: Fri Nov 2 18:51:54.485668 PDT 3: Schedule cleanup of "eq_tapq:anon_42" memcached<0.1519.0>: Fri Nov 2 18:51:54.485698 PDT 3: Schedule cleanup of "eq_tapq:rebalance_297" memcached<0.1519.0>: Fri Nov 2 18:51:54.501975 PDT 3: TAP (Producer) eq_tapq:replication_building_297_'ns_1@10.2.1.65' - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:51:54.502024 PDT 3: TAP (Producer) eq_tapq:rebalance_297 - Clear the tap queues by force [ns_server:info,2012-11-02T18:51:55.558,ns_1@10.2.1.67:<0.17749.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.65",11209} vbucket 298 to state replica [rebalance:info,2012-11-02T18:51:55.567,ns_1@10.2.1.67:<0.17749.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[298]}, {checkpoints,[{298,9}]}, {name,<<"rebalance_298">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.65",11209}, [{username,"default"}, {password,[]}, {vbuckets,[298]}, {takeover,true}, {suffix,"298"}]} [rebalance:info,2012-11-02T18:51:55.571,ns_1@10.2.1.67:<0.17749.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:51:55.574,ns_1@10.2.1.67:<0.17749.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:51:55.660,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:51:55.460186 PDT 3: TAP (Producer) eq_tapq:replication_building_298_'ns_1@10.2.1.65' - Backfill is completed with VBuckets 298, memcached<0.1519.0>: Fri Nov 2 18:51:55.460206 PDT 3: TAP (Producer) eq_tapq:replication_building_298_'ns_1@10.2.1.65' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 298 memcached<0.1519.0>: Fri Nov 2 18:51:55.555130 PDT 3: TAP (Producer) eq_tapq:replication_building_298_'ns_1@10.2.1.65' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:51:55.567614 PDT 3: TAP (Producer) eq_tapq:rebalance_298 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:55.567639 PDT 3: TAP (Producer) eq_tapq:rebalance_298 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:55.567650 PDT 3: TAP (Producer) eq_tapq:rebalance_298 - Sending TAP_VBUCKET_SET with vbucket 298 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:51:55.574303 PDT 3: TAP (Producer) eq_tapq:rebalance_298 - VBucket <298> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:51:55.574337 PDT 3: TAP (Producer) eq_tapq:rebalance_298 - Sending TAP_VBUCKET_SET with vbucket 298 and state "active" memcached<0.1519.0>: Fri Nov 2 18:51:55.574786 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:51:55.574817 PDT 3: TAP (Producer) eq_tapq:rebalance_298 - disconnected memcached<0.1519.0>: Fri Nov 2 18:51:55.574851 PDT 3: Schedule cleanup of "eq_tapq:rebalance_298" memcached<0.1519.0>: Fri Nov 2 18:51:55.574898 PDT 3: TAP (Producer) eq_tapq:rebalance_298 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:51:55.591302 PDT 3: TAP (Producer) eq_tapq:replication_building_298_'ns_1@10.2.1.65' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:51:55.598635 PDT 3: Deletion of vbucket 298 was completed. memcached<0.1519.0>: Fri Nov 2 18:51:55.617739 PDT 3: TAP (Producer) eq_tapq:replication_building_299_'ns_1@10.2.1.66' - Schedule the backfill for vbucket 299 memcached<0.1519.0>: Fri Nov 2 18:51:55.617766 PDT 3: TAP (Producer) eq_tapq:replication_building_299_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:55.617775 PDT 3: TAP (Producer) eq_tapq:replication_building_299_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:55.617780 PDT 3: TAP (Producer) eq_tapq:replication_building_299_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 299 memcached<0.1519.0>: Fri Nov 2 18:51:55.617850 PDT 3: Schedule cleanup of "eq_tapq:anon_43" memcached<0.1519.0>: Fri Nov 2 18:51:55.621650 PDT 3: TAP (Producer) eq_tapq:replication_building_298_'ns_1@10.2.1.65' - Clear the tap queues by force [ns_server:info,2012-11-02T18:51:55.663,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/298 [ns_server:info,2012-11-02T18:51:56.313,ns_1@10.2.1.67:<0.17763.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.66",11209} vbucket 299 to state replica [rebalance:info,2012-11-02T18:51:56.319,ns_1@10.2.1.67:<0.17763.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[299]}, {checkpoints,[{299,10}]}, {name,<<"rebalance_299">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.66",11209}, [{username,"default"}, {password,[]}, {vbuckets,[299]}, {takeover,true}, {suffix,"299"}]} [rebalance:info,2012-11-02T18:51:56.320,ns_1@10.2.1.67:<0.17763.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:51:56.325,ns_1@10.2.1.67:<0.17763.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:51:56.384,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/299 [ns_server:info,2012-11-02T18:51:56.472,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:51:56.272258 PDT 3: TAP (Producer) eq_tapq:replication_building_299_'ns_1@10.2.1.66' - Backfill is completed with VBuckets 299, memcached<0.1519.0>: Fri Nov 2 18:51:56.272280 PDT 3: TAP (Producer) eq_tapq:replication_building_299_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 299 memcached<0.1519.0>: Fri Nov 2 18:51:56.310444 PDT 3: TAP (Producer) eq_tapq:replication_building_299_'ns_1@10.2.1.66' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:51:56.319392 PDT 3: TAP (Producer) eq_tapq:rebalance_299 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:56.319405 PDT 3: TAP (Producer) eq_tapq:rebalance_299 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:56.319410 PDT 3: TAP (Producer) eq_tapq:rebalance_299 - Sending TAP_VBUCKET_SET with vbucket 299 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:51:56.325084 PDT 3: TAP (Producer) eq_tapq:rebalance_299 - VBucket <299> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:51:56.325117 PDT 3: TAP (Producer) eq_tapq:rebalance_299 - Sending TAP_VBUCKET_SET with vbucket 299 and state "active" memcached<0.1519.0>: Fri Nov 2 18:51:56.325391 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:51:56.325415 PDT 3: TAP (Producer) eq_tapq:rebalance_299 - disconnected memcached<0.1519.0>: Fri Nov 2 18:51:56.325448 PDT 3: Schedule cleanup of "eq_tapq:rebalance_299" memcached<0.1519.0>: Fri Nov 2 18:51:56.325692 PDT 3: TAP (Producer) eq_tapq:rebalance_299 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:51:56.333388 PDT 3: TAP (Producer) eq_tapq:replication_building_299_'ns_1@10.2.1.66' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:51:56.343195 PDT 3: Deletion of vbucket 299 was completed. memcached<0.1519.0>: Fri Nov 2 18:51:56.352511 PDT 3: TAP (Producer) eq_tapq:replication_building_300_'ns_1@10.2.1.66' - Schedule the backfill for vbucket 300 memcached<0.1519.0>: Fri Nov 2 18:51:56.352556 PDT 3: TAP (Producer) eq_tapq:replication_building_300_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:56.352570 PDT 3: TAP (Producer) eq_tapq:replication_building_300_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:56.352580 PDT 3: TAP (Producer) eq_tapq:replication_building_300_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 300 memcached<0.1519.0>: Fri Nov 2 18:51:56.352656 PDT 3: Schedule cleanup of "eq_tapq:anon_44" memcached<0.1519.0>: Fri Nov 2 18:51:56.370175 PDT 3: TAP (Producer) eq_tapq:replication_building_299_'ns_1@10.2.1.66' - Clear the tap queues by force [ns_server:info,2012-11-02T18:51:57.125,ns_1@10.2.1.67:<0.17773.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.66",11209} vbucket 300 to state replica [rebalance:info,2012-11-02T18:51:57.128,ns_1@10.2.1.67:<0.17773.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[300]}, {checkpoints,[{300,9}]}, {name,<<"rebalance_300">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.66",11209}, [{username,"default"}, {password,[]}, {vbuckets,[300]}, {takeover,true}, {suffix,"300"}]} [rebalance:info,2012-11-02T18:51:57.134,ns_1@10.2.1.67:<0.17773.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:51:57.138,ns_1@10.2.1.67:<0.17773.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:51:57.201,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/300 [ns_server:info,2012-11-02T18:51:57.273,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:51:57.072826 PDT 3: TAP (Producer) eq_tapq:replication_building_300_'ns_1@10.2.1.66' - Backfill is completed with VBuckets 300, memcached<0.1519.0>: Fri Nov 2 18:51:57.072847 PDT 3: TAP (Producer) eq_tapq:replication_building_300_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 300 memcached<0.1519.0>: Fri Nov 2 18:51:57.122128 PDT 3: TAP (Producer) eq_tapq:replication_building_300_'ns_1@10.2.1.66' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:51:57.128823 PDT 3: TAP (Producer) eq_tapq:rebalance_300 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:57.128835 PDT 3: TAP (Producer) eq_tapq:rebalance_300 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:57.128840 PDT 3: TAP (Producer) eq_tapq:rebalance_300 - Sending TAP_VBUCKET_SET with vbucket 300 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:51:57.137570 PDT 3: TAP (Producer) eq_tapq:rebalance_300 - VBucket <300> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:51:57.137591 PDT 3: TAP (Producer) eq_tapq:rebalance_300 - Sending TAP_VBUCKET_SET with vbucket 300 and state "active" memcached<0.1519.0>: Fri Nov 2 18:51:57.137963 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:51:57.137985 PDT 3: TAP (Producer) eq_tapq:rebalance_300 - disconnected memcached<0.1519.0>: Fri Nov 2 18:51:57.138016 PDT 3: Schedule cleanup of "eq_tapq:rebalance_300" memcached<0.1519.0>: Fri Nov 2 18:51:57.138039 PDT 3: TAP (Producer) eq_tapq:rebalance_300 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:51:57.145729 PDT 3: TAP (Producer) eq_tapq:replication_building_300_'ns_1@10.2.1.66' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:51:57.154415 PDT 3: Deletion of vbucket 300 was completed. memcached<0.1519.0>: Fri Nov 2 18:51:57.164628 PDT 3: TAP (Producer) eq_tapq:replication_building_301_'ns_1@10.2.1.66' - Schedule the backfill for vbucket 301 memcached<0.1519.0>: Fri Nov 2 18:51:57.164652 PDT 3: TAP (Producer) eq_tapq:replication_building_301_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:57.164660 PDT 3: TAP (Producer) eq_tapq:replication_building_301_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:57.164664 PDT 3: TAP (Producer) eq_tapq:replication_building_301_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 301 memcached<0.1519.0>: Fri Nov 2 18:51:57.164727 PDT 3: Schedule cleanup of "eq_tapq:anon_45" memcached<0.1519.0>: Fri Nov 2 18:51:57.182752 PDT 3: TAP (Producer) eq_tapq:replication_building_300_'ns_1@10.2.1.66' - Clear the tap queues by force [ns_server:info,2012-11-02T18:51:57.844,ns_1@10.2.1.67:<0.17778.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.66",11209} vbucket 301 to state replica [rebalance:info,2012-11-02T18:51:57.847,ns_1@10.2.1.67:<0.17778.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[301]}, {checkpoints,[{301,10}]}, {name,<<"rebalance_301">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.66",11209}, [{username,"default"}, {password,[]}, {vbuckets,[301]}, {takeover,true}, {suffix,"301"}]} [rebalance:info,2012-11-02T18:51:57.848,ns_1@10.2.1.67:<0.17778.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:51:57.851,ns_1@10.2.1.67:<0.17778.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:51:57.914,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/301 [ns_server:info,2012-11-02T18:51:58.031,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:51:57.830909 PDT 3: TAP (Producer) eq_tapq:replication_building_301_'ns_1@10.2.1.66' - Backfill is completed with VBuckets 301, memcached<0.1519.0>: Fri Nov 2 18:51:57.830932 PDT 3: TAP (Producer) eq_tapq:replication_building_301_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 301 memcached<0.1519.0>: Fri Nov 2 18:51:57.841530 PDT 3: TAP (Producer) eq_tapq:replication_building_301_'ns_1@10.2.1.66' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:51:57.847895 PDT 3: TAP (Producer) eq_tapq:rebalance_301 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:57.847908 PDT 3: TAP (Producer) eq_tapq:rebalance_301 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:57.847913 PDT 3: TAP (Producer) eq_tapq:rebalance_301 - Sending TAP_VBUCKET_SET with vbucket 301 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:51:57.851186 PDT 3: TAP (Producer) eq_tapq:rebalance_301 - VBucket <301> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:51:57.851204 PDT 3: TAP (Producer) eq_tapq:rebalance_301 - Sending TAP_VBUCKET_SET with vbucket 301 and state "active" memcached<0.1519.0>: Fri Nov 2 18:51:57.851685 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:51:57.851709 PDT 3: TAP (Producer) eq_tapq:rebalance_301 - disconnected memcached<0.1519.0>: Fri Nov 2 18:51:57.851743 PDT 3: Schedule cleanup of "eq_tapq:rebalance_301" memcached<0.1519.0>: Fri Nov 2 18:51:57.852130 PDT 3: TAP (Producer) eq_tapq:rebalance_301 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:51:57.860859 PDT 3: TAP (Producer) eq_tapq:replication_building_301_'ns_1@10.2.1.66' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:51:57.872580 PDT 3: Deletion of vbucket 301 was completed. memcached<0.1519.0>: Fri Nov 2 18:51:57.881338 PDT 3: TAP (Producer) eq_tapq:replication_building_302_'ns_1@10.2.1.66' - Schedule the backfill for vbucket 302 memcached<0.1519.0>: Fri Nov 2 18:51:57.881500 PDT 3: TAP (Producer) eq_tapq:replication_building_302_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:57.881511 PDT 3: TAP (Producer) eq_tapq:replication_building_302_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:57.881519 PDT 3: TAP (Producer) eq_tapq:replication_building_302_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 302 memcached<0.1519.0>: Fri Nov 2 18:51:57.881577 PDT 3: Schedule cleanup of "eq_tapq:anon_46" memcached<0.1519.0>: Fri Nov 2 18:51:57.893078 PDT 3: TAP (Producer) eq_tapq:replication_building_301_'ns_1@10.2.1.66' - Clear the tap queues by force [ns_server:info,2012-11-02T18:51:58.567,ns_1@10.2.1.67:<0.17783.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.66",11209} vbucket 302 to state replica [rebalance:info,2012-11-02T18:51:58.578,ns_1@10.2.1.67:<0.17783.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[302]}, {checkpoints,[{302,10}]}, {name,<<"rebalance_302">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.66",11209}, [{username,"default"}, {password,[]}, {vbuckets,[302]}, {takeover,true}, {suffix,"302"}]} [rebalance:info,2012-11-02T18:51:58.579,ns_1@10.2.1.67:<0.17783.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:51:58.582,ns_1@10.2.1.67:<0.17783.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:51:58.643,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/302 [ns_server:info,2012-11-02T18:51:58.719,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:51:58.518587 PDT 3: TAP (Producer) eq_tapq:replication_building_302_'ns_1@10.2.1.66' - Backfill is completed with VBuckets 302, memcached<0.1519.0>: Fri Nov 2 18:51:58.518608 PDT 3: TAP (Producer) eq_tapq:replication_building_302_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 302 memcached<0.1519.0>: Fri Nov 2 18:51:58.563743 PDT 3: TAP (Producer) eq_tapq:replication_building_302_'ns_1@10.2.1.66' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:51:58.578304 PDT 3: TAP (Producer) eq_tapq:rebalance_302 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:58.578316 PDT 3: TAP (Producer) eq_tapq:rebalance_302 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:58.578321 PDT 3: TAP (Producer) eq_tapq:rebalance_302 - Sending TAP_VBUCKET_SET with vbucket 302 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:51:58.582161 PDT 3: TAP (Producer) eq_tapq:rebalance_302 - VBucket <302> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:51:58.582176 PDT 3: TAP (Producer) eq_tapq:rebalance_302 - Sending TAP_VBUCKET_SET with vbucket 302 and state "active" memcached<0.1519.0>: Fri Nov 2 18:51:58.582454 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:51:58.582476 PDT 3: TAP (Producer) eq_tapq:rebalance_302 - disconnected memcached<0.1519.0>: Fri Nov 2 18:51:58.582531 PDT 3: Schedule cleanup of "eq_tapq:rebalance_302" memcached<0.1519.0>: Fri Nov 2 18:51:58.582581 PDT 3: TAP (Producer) eq_tapq:rebalance_302 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:51:58.588903 PDT 3: TAP (Producer) eq_tapq:replication_building_302_'ns_1@10.2.1.66' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:51:58.595661 PDT 3: Deletion of vbucket 302 was completed. memcached<0.1519.0>: Fri Nov 2 18:51:58.610501 PDT 3: TAP (Producer) eq_tapq:replication_building_303_'ns_1@10.2.1.66' - Schedule the backfill for vbucket 303 memcached<0.1519.0>: Fri Nov 2 18:51:58.610528 PDT 3: TAP (Producer) eq_tapq:replication_building_303_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:58.610535 PDT 3: TAP (Producer) eq_tapq:replication_building_303_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:58.610540 PDT 3: TAP (Producer) eq_tapq:replication_building_303_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 303 memcached<0.1519.0>: Fri Nov 2 18:51:58.610593 PDT 3: Schedule cleanup of "eq_tapq:anon_47" memcached<0.1519.0>: Fri Nov 2 18:51:58.614070 PDT 3: TAP (Producer) eq_tapq:replication_building_302_'ns_1@10.2.1.66' - Clear the tap queues by force [ns_server:info,2012-11-02T18:51:59.396,ns_1@10.2.1.67:<0.17794.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.66",11209} vbucket 303 to state replica [rebalance:info,2012-11-02T18:51:59.400,ns_1@10.2.1.67:<0.17794.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[303]}, {checkpoints,[{303,9}]}, {name,<<"rebalance_303">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.66",11209}, [{username,"default"}, {password,[]}, {vbuckets,[303]}, {takeover,true}, {suffix,"303"}]} [rebalance:info,2012-11-02T18:51:59.405,ns_1@10.2.1.67:<0.17794.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:51:59.409,ns_1@10.2.1.67:<0.17794.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:51:59.489,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/303 [ns_server:info,2012-11-02T18:51:59.504,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:51:59.290789 PDT 3: TAP (Producer) eq_tapq:replication_building_303_'ns_1@10.2.1.66' - Backfill is completed with VBuckets 303, memcached<0.1519.0>: Fri Nov 2 18:51:59.290816 PDT 3: TAP (Producer) eq_tapq:replication_building_303_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 303 memcached<0.1519.0>: Fri Nov 2 18:51:59.393394 PDT 3: TAP (Producer) eq_tapq:replication_building_303_'ns_1@10.2.1.66' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:51:59.400578 PDT 3: TAP (Producer) eq_tapq:rebalance_303 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:59.400591 PDT 3: TAP (Producer) eq_tapq:rebalance_303 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:59.400596 PDT 3: TAP (Producer) eq_tapq:rebalance_303 - Sending TAP_VBUCKET_SET with vbucket 303 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:51:59.408815 PDT 3: TAP (Producer) eq_tapq:rebalance_303 - VBucket <303> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:51:59.408832 PDT 3: TAP (Producer) eq_tapq:rebalance_303 - Sending TAP_VBUCKET_SET with vbucket 303 and state "active" memcached<0.1519.0>: Fri Nov 2 18:51:59.409095 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:51:59.409118 PDT 3: TAP (Producer) eq_tapq:rebalance_303 - disconnected memcached<0.1519.0>: Fri Nov 2 18:51:59.409174 PDT 3: Schedule cleanup of "eq_tapq:rebalance_303" memcached<0.1519.0>: Fri Nov 2 18:51:59.409201 PDT 3: TAP (Producer) eq_tapq:rebalance_303 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:51:59.426066 PDT 3: TAP (Producer) eq_tapq:replication_building_303_'ns_1@10.2.1.66' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:51:59.434446 PDT 3: Deletion of vbucket 303 was completed. memcached<0.1519.0>: Fri Nov 2 18:51:59.446070 PDT 3: TAP (Producer) eq_tapq:replication_building_304_'ns_1@10.2.1.66' - Schedule the backfill for vbucket 304 memcached<0.1519.0>: Fri Nov 2 18:51:59.446111 PDT 3: TAP (Producer) eq_tapq:replication_building_304_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:59.446124 PDT 3: TAP (Producer) eq_tapq:replication_building_304_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:51:59.446134 PDT 3: TAP (Producer) eq_tapq:replication_building_304_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 304 memcached<0.1519.0>: Fri Nov 2 18:51:59.446415 PDT 3: Schedule cleanup of "eq_tapq:anon_48" memcached<0.1519.0>: Fri Nov 2 18:51:59.460546 PDT 3: TAP (Producer) eq_tapq:replication_building_303_'ns_1@10.2.1.66' - Clear the tap queues by force [ns_server:info,2012-11-02T18:52:00.265,ns_1@10.2.1.67:<0.17800.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.66",11209} vbucket 304 to state replica [rebalance:info,2012-11-02T18:52:00.268,ns_1@10.2.1.67:<0.17800.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[304]}, {checkpoints,[{304,10}]}, {name,<<"rebalance_304">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.66",11209}, [{username,"default"}, {password,[]}, {vbuckets,[304]}, {takeover,true}, {suffix,"304"}]} [rebalance:info,2012-11-02T18:52:00.274,ns_1@10.2.1.67:<0.17800.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:52:00.277,ns_1@10.2.1.67:<0.17800.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:52:00.346,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/304 [ns_server:info,2012-11-02T18:52:00.379,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:52:00.179002 PDT 3: TAP (Producer) eq_tapq:replication_building_304_'ns_1@10.2.1.66' - Backfill is completed with VBuckets 304, memcached<0.1519.0>: Fri Nov 2 18:52:00.179023 PDT 3: TAP (Producer) eq_tapq:replication_building_304_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 304 memcached<0.1519.0>: Fri Nov 2 18:52:00.262244 PDT 3: TAP (Producer) eq_tapq:replication_building_304_'ns_1@10.2.1.66' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:52:00.269033 PDT 3: TAP (Producer) eq_tapq:rebalance_304 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:00.269046 PDT 3: TAP (Producer) eq_tapq:rebalance_304 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:00.269051 PDT 3: TAP (Producer) eq_tapq:rebalance_304 - Sending TAP_VBUCKET_SET with vbucket 304 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:52:00.277208 PDT 3: TAP (Producer) eq_tapq:rebalance_304 - VBucket <304> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:52:00.277230 PDT 3: TAP (Producer) eq_tapq:rebalance_304 - Sending TAP_VBUCKET_SET with vbucket 304 and state "active" memcached<0.1519.0>: Fri Nov 2 18:52:00.277520 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:52:00.277544 PDT 3: TAP (Producer) eq_tapq:rebalance_304 - disconnected memcached<0.1519.0>: Fri Nov 2 18:52:00.277576 PDT 3: Schedule cleanup of "eq_tapq:rebalance_304" memcached<0.1519.0>: Fri Nov 2 18:52:00.277648 PDT 3: TAP (Producer) eq_tapq:rebalance_304 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:52:00.287243 PDT 3: TAP (Producer) eq_tapq:replication_building_304_'ns_1@10.2.1.66' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:52:00.297392 PDT 3: Deletion of vbucket 304 was completed. memcached<0.1519.0>: Fri Nov 2 18:52:00.310695 PDT 3: TAP (Producer) eq_tapq:replication_building_305_'ns_1@10.2.1.66' - Schedule the backfill for vbucket 305 memcached<0.1519.0>: Fri Nov 2 18:52:00.310723 PDT 3: TAP (Producer) eq_tapq:replication_building_305_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:00.310731 PDT 3: TAP (Producer) eq_tapq:replication_building_305_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:00.310736 PDT 3: TAP (Producer) eq_tapq:replication_building_305_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 305 memcached<0.1519.0>: Fri Nov 2 18:52:00.311333 PDT 3: Schedule cleanup of "eq_tapq:anon_49" memcached<0.1519.0>: Fri Nov 2 18:52:00.314390 PDT 3: TAP (Producer) eq_tapq:replication_building_304_'ns_1@10.2.1.66' - Clear the tap queues by force [ns_server:info,2012-11-02T18:52:01.097,ns_1@10.2.1.67:<0.17818.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.66",11209} vbucket 305 to state replica [rebalance:info,2012-11-02T18:52:01.106,ns_1@10.2.1.67:<0.17818.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[305]}, {checkpoints,[{305,10}]}, {name,<<"rebalance_305">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.66",11209}, [{username,"default"}, {password,[]}, {vbuckets,[305]}, {takeover,true}, {suffix,"305"}]} [rebalance:info,2012-11-02T18:52:01.107,ns_1@10.2.1.67:<0.17818.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:52:01.112,ns_1@10.2.1.67:<0.17818.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:52:01.192,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/305 [ns_server:info,2012-11-02T18:52:01.193,ns_1@10.2.1.67:<0.17822.0>:compaction_daemon:try_to_cleanup_indexes:439]Cleaning up indexes for bucket `default` [ns_server:info,2012-11-02T18:52:01.194,ns_1@10.2.1.67:<0.17822.0>:compaction_daemon:spawn_bucket_compactor:404]Compacting bucket default with config: [{parallel_db_and_view_compaction,false}, {database_fragmentation_threshold,{30,undefined}}, {view_fragmentation_threshold,{100,undefined}}] [ns_server:info,2012-11-02T18:52:01.201,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:52:00.999712 PDT 3: TAP (Producer) eq_tapq:replication_building_305_'ns_1@10.2.1.66' - Backfill is completed with VBuckets 305, memcached<0.1519.0>: Fri Nov 2 18:52:00.999738 PDT 3: TAP (Producer) eq_tapq:replication_building_305_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 305 memcached<0.1519.0>: Fri Nov 2 18:52:01.092629 PDT 3: TAP (Producer) eq_tapq:replication_building_305_'ns_1@10.2.1.66' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:52:01.106901 PDT 3: TAP (Producer) eq_tapq:rebalance_305 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:01.106914 PDT 3: TAP (Producer) eq_tapq:rebalance_305 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:01.106919 PDT 3: TAP (Producer) eq_tapq:rebalance_305 - Sending TAP_VBUCKET_SET with vbucket 305 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:52:01.111783 PDT 3: TAP (Producer) eq_tapq:rebalance_305 - VBucket <305> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:52:01.111800 PDT 3: TAP (Producer) eq_tapq:rebalance_305 - Sending TAP_VBUCKET_SET with vbucket 305 and state "active" memcached<0.1519.0>: Fri Nov 2 18:52:01.112057 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:52:01.112081 PDT 3: TAP (Producer) eq_tapq:rebalance_305 - disconnected memcached<0.1519.0>: Fri Nov 2 18:52:01.112273 PDT 3: Schedule cleanup of "eq_tapq:rebalance_305" memcached<0.1519.0>: Fri Nov 2 18:52:01.112332 PDT 3: TAP (Producer) eq_tapq:rebalance_305 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:52:01.138296 PDT 3: TAP (Producer) eq_tapq:replication_building_305_'ns_1@10.2.1.66' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:52:01.146816 PDT 3: Deletion of vbucket 305 was completed. memcached<0.1519.0>: Fri Nov 2 18:52:01.163316 PDT 3: TAP (Producer) eq_tapq:replication_building_306_'ns_1@10.2.1.66' - Schedule the backfill for vbucket 306 memcached<0.1519.0>: Fri Nov 2 18:52:01.163360 PDT 3: TAP (Producer) eq_tapq:replication_building_306_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:01.163374 PDT 3: TAP (Producer) eq_tapq:replication_building_306_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:01.163384 PDT 3: TAP (Producer) eq_tapq:replication_building_306_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 306 memcached<0.1519.0>: Fri Nov 2 18:52:01.163444 PDT 3: Schedule cleanup of "eq_tapq:anon_50" memcached<0.1519.0>: Fri Nov 2 18:52:01.169613 PDT 3: TAP (Producer) eq_tapq:replication_building_305_'ns_1@10.2.1.66' - Clear the tap queues by force [ns_server:info,2012-11-02T18:52:02.240,ns_1@10.2.1.67:<0.17831.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.66",11209} vbucket 306 to state replica [rebalance:info,2012-11-02T18:52:02.244,ns_1@10.2.1.67:<0.17831.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[306]}, {checkpoints,[{306,10}]}, {name,<<"rebalance_306">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.66",11209}, [{username,"default"}, {password,[]}, {vbuckets,[306]}, {takeover,true}, {suffix,"306"}]} [rebalance:info,2012-11-02T18:52:02.245,ns_1@10.2.1.67:<0.17831.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:52:02.248,ns_1@10.2.1.67:<0.17831.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:52:02.327,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/306 [ns_server:info,2012-11-02T18:52:02.350,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:52:02.149282 PDT 3: TAP (Producer) eq_tapq:replication_building_306_'ns_1@10.2.1.66' - Backfill is completed with VBuckets 306, memcached<0.1519.0>: Fri Nov 2 18:52:02.149318 PDT 3: TAP (Producer) eq_tapq:replication_building_306_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 306 memcached<0.1519.0>: Fri Nov 2 18:52:02.237229 PDT 3: TAP (Producer) eq_tapq:replication_building_306_'ns_1@10.2.1.66' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:52:02.244900 PDT 3: TAP (Producer) eq_tapq:rebalance_306 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:02.244912 PDT 3: TAP (Producer) eq_tapq:rebalance_306 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:02.244917 PDT 3: TAP (Producer) eq_tapq:rebalance_306 - Sending TAP_VBUCKET_SET with vbucket 306 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:52:02.248282 PDT 3: TAP (Producer) eq_tapq:rebalance_306 - VBucket <306> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:52:02.248298 PDT 3: TAP (Producer) eq_tapq:rebalance_306 - Sending TAP_VBUCKET_SET with vbucket 306 and state "active" memcached<0.1519.0>: Fri Nov 2 18:52:02.248695 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:52:02.248718 PDT 3: TAP (Producer) eq_tapq:rebalance_306 - disconnected memcached<0.1519.0>: Fri Nov 2 18:52:02.248750 PDT 3: Schedule cleanup of "eq_tapq:rebalance_306" memcached<0.1519.0>: Fri Nov 2 18:52:02.249081 PDT 3: TAP (Producer) eq_tapq:rebalance_306 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:52:02.257249 PDT 3: TAP (Producer) eq_tapq:replication_building_306_'ns_1@10.2.1.66' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:52:02.268243 PDT 3: Deletion of vbucket 306 was completed. memcached<0.1519.0>: Fri Nov 2 18:52:02.289601 PDT 3: TAP (Producer) eq_tapq:replication_building_307_'ns_1@10.2.1.66' - Schedule the backfill for vbucket 307 memcached<0.1519.0>: Fri Nov 2 18:52:02.289642 PDT 3: TAP (Producer) eq_tapq:replication_building_307_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:02.289656 PDT 3: TAP (Producer) eq_tapq:replication_building_307_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:02.289666 PDT 3: TAP (Producer) eq_tapq:replication_building_307_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 307 memcached<0.1519.0>: Fri Nov 2 18:52:02.289726 PDT 3: Schedule cleanup of "eq_tapq:anon_51" memcached<0.1519.0>: Fri Nov 2 18:52:02.289828 PDT 3: TAP (Producer) eq_tapq:replication_building_306_'ns_1@10.2.1.66' - Clear the tap queues by force [ns_server:info,2012-11-02T18:52:03.372,ns_1@10.2.1.67:<0.17841.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.66",11209} vbucket 307 to state replica [rebalance:info,2012-11-02T18:52:03.375,ns_1@10.2.1.67:<0.17841.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[307]}, {checkpoints,[{307,10}]}, {name,<<"rebalance_307">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.66",11209}, [{username,"default"}, {password,[]}, {vbuckets,[307]}, {takeover,true}, {suffix,"307"}]} [rebalance:info,2012-11-02T18:52:03.380,ns_1@10.2.1.67:<0.17841.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:52:03.383,ns_1@10.2.1.67:<0.17841.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:52:03.486,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/307 [ns_server:info,2012-11-02T18:52:03.494,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:52:03.289466 PDT 3: TAP (Producer) eq_tapq:replication_building_307_'ns_1@10.2.1.66' - Backfill is completed with VBuckets 307, memcached<0.1519.0>: Fri Nov 2 18:52:03.289488 PDT 3: TAP (Producer) eq_tapq:replication_building_307_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 307 memcached<0.1519.0>: Fri Nov 2 18:52:03.369131 PDT 3: TAP (Producer) eq_tapq:replication_building_307_'ns_1@10.2.1.66' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:52:03.375555 PDT 3: TAP (Producer) eq_tapq:rebalance_307 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:03.375568 PDT 3: TAP (Producer) eq_tapq:rebalance_307 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:03.375573 PDT 3: TAP (Producer) eq_tapq:rebalance_307 - Sending TAP_VBUCKET_SET with vbucket 307 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:52:03.383293 PDT 3: TAP (Producer) eq_tapq:rebalance_307 - VBucket <307> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:52:03.383311 PDT 3: TAP (Producer) eq_tapq:rebalance_307 - Sending TAP_VBUCKET_SET with vbucket 307 and state "active" memcached<0.1519.0>: Fri Nov 2 18:52:03.383579 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:52:03.383609 PDT 3: TAP (Producer) eq_tapq:rebalance_307 - disconnected memcached<0.1519.0>: Fri Nov 2 18:52:03.383641 PDT 3: Schedule cleanup of "eq_tapq:rebalance_307" memcached<0.1519.0>: Fri Nov 2 18:52:03.383721 PDT 3: TAP (Producer) eq_tapq:rebalance_307 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:52:03.408618 PDT 3: TAP (Producer) eq_tapq:replication_building_307_'ns_1@10.2.1.66' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:52:03.422234 PDT 3: Deletion of vbucket 307 was completed. memcached<0.1519.0>: Fri Nov 2 18:52:03.436733 PDT 3: TAP (Producer) eq_tapq:replication_building_308_'ns_1@10.2.1.66' - Schedule the backfill for vbucket 308 memcached<0.1519.0>: Fri Nov 2 18:52:03.436763 PDT 3: TAP (Producer) eq_tapq:replication_building_308_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:03.436771 PDT 3: TAP (Producer) eq_tapq:replication_building_308_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:03.436777 PDT 3: TAP (Producer) eq_tapq:replication_building_308_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 308 memcached<0.1519.0>: Fri Nov 2 18:52:03.437096 PDT 3: Schedule cleanup of "eq_tapq:anon_52" memcached<0.1519.0>: Fri Nov 2 18:52:03.445257 PDT 3: TAP (Producer) eq_tapq:replication_building_307_'ns_1@10.2.1.66' - Clear the tap queues by force [ns_server:info,2012-11-02T18:52:04.663,ns_1@10.2.1.67:<0.17852.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.66",11209} vbucket 308 to state replica [rebalance:info,2012-11-02T18:52:04.667,ns_1@10.2.1.67:<0.17852.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[308]}, {checkpoints,[{308,10}]}, {name,<<"rebalance_308">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.66",11209}, [{username,"default"}, {password,[]}, {vbuckets,[308]}, {takeover,true}, {suffix,"308"}]} [rebalance:info,2012-11-02T18:52:04.673,ns_1@10.2.1.67:<0.17852.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:52:04.678,ns_1@10.2.1.67:<0.17852.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:52:04.873,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:52:04.567966 PDT 3: TAP (Producer) eq_tapq:replication_building_308_'ns_1@10.2.1.66' - Backfill is completed with VBuckets 308, memcached<0.1519.0>: Fri Nov 2 18:52:04.568000 PDT 3: TAP (Producer) eq_tapq:replication_building_308_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 308 memcached<0.1519.0>: Fri Nov 2 18:52:04.659811 PDT 3: TAP (Producer) eq_tapq:replication_building_308_'ns_1@10.2.1.66' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:52:04.667994 PDT 3: TAP (Producer) eq_tapq:rebalance_308 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:04.668008 PDT 3: TAP (Producer) eq_tapq:rebalance_308 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:04.668013 PDT 3: TAP (Producer) eq_tapq:rebalance_308 - Sending TAP_VBUCKET_SET with vbucket 308 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:52:04.677492 PDT 3: TAP (Producer) eq_tapq:rebalance_308 - VBucket <308> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:52:04.677530 PDT 3: TAP (Producer) eq_tapq:rebalance_308 - Sending TAP_VBUCKET_SET with vbucket 308 and state "active" memcached<0.1519.0>: Fri Nov 2 18:52:04.677903 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:52:04.677947 PDT 3: TAP (Producer) eq_tapq:rebalance_308 - disconnected memcached<0.1519.0>: Fri Nov 2 18:52:04.678016 PDT 3: Schedule cleanup of "eq_tapq:rebalance_308" memcached<0.1519.0>: Fri Nov 2 18:52:04.678167 PDT 3: TAP (Producer) eq_tapq:rebalance_308 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:52:04.697912 PDT 3: TAP (Producer) eq_tapq:replication_building_308_'ns_1@10.2.1.66' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:52:04.706749 PDT 3: Deletion of vbucket 308 was completed. memcached<0.1519.0>: Fri Nov 2 18:52:04.722437 PDT 3: TAP (Producer) eq_tapq:replication_building_309_'ns_1@10.2.1.66' - Schedule the backfill for vbucket 309 memcached<0.1519.0>: Fri Nov 2 18:52:04.722478 PDT 3: TAP (Producer) eq_tapq:replication_building_309_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:04.722493 PDT 3: TAP (Producer) eq_tapq:replication_building_309_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:04.722504 PDT 3: TAP (Producer) eq_tapq:replication_building_309_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 309 memcached<0.1519.0>: Fri Nov 2 18:52:04.722609 PDT 3: Schedule cleanup of "eq_tapq:anon_53" memcached<0.1519.0>: Fri Nov 2 18:52:04.730278 PDT 3: TAP (Producer) eq_tapq:replication_building_308_'ns_1@10.2.1.66' - Clear the tap queues by force [ns_server:info,2012-11-02T18:52:04.875,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/308 [ns_server:info,2012-11-02T18:52:05.924,ns_1@10.2.1.67:<0.17866.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.66",11209} vbucket 309 to state replica [rebalance:info,2012-11-02T18:52:05.927,ns_1@10.2.1.67:<0.17866.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[309]}, {checkpoints,[{309,10}]}, {name,<<"rebalance_309">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.66",11209}, [{username,"default"}, {password,[]}, {vbuckets,[309]}, {takeover,true}, {suffix,"309"}]} [rebalance:info,2012-11-02T18:52:05.933,ns_1@10.2.1.67:<0.17866.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:52:05.938,ns_1@10.2.1.67:<0.17866.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:52:06.031,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/309 [ns_server:info,2012-11-02T18:52:06.049,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:52:05.848951 PDT 3: TAP (Producer) eq_tapq:replication_building_309_'ns_1@10.2.1.66' - Backfill is completed with VBuckets 309, memcached<0.1519.0>: Fri Nov 2 18:52:05.848971 PDT 3: TAP (Producer) eq_tapq:replication_building_309_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 309 memcached<0.1519.0>: Fri Nov 2 18:52:05.921171 PDT 3: TAP (Producer) eq_tapq:replication_building_309_'ns_1@10.2.1.66' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:52:05.927638 PDT 3: TAP (Producer) eq_tapq:rebalance_309 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:05.927651 PDT 3: TAP (Producer) eq_tapq:rebalance_309 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:05.927656 PDT 3: TAP (Producer) eq_tapq:rebalance_309 - Sending TAP_VBUCKET_SET with vbucket 309 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:52:05.937576 PDT 3: TAP (Producer) eq_tapq:rebalance_309 - VBucket <309> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:52:05.937615 PDT 3: TAP (Producer) eq_tapq:rebalance_309 - Sending TAP_VBUCKET_SET with vbucket 309 and state "active" memcached<0.1519.0>: Fri Nov 2 18:52:05.938060 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:52:05.938124 PDT 3: TAP (Producer) eq_tapq:rebalance_309 - disconnected memcached<0.1519.0>: Fri Nov 2 18:52:05.938204 PDT 3: Schedule cleanup of "eq_tapq:rebalance_309" memcached<0.1519.0>: Fri Nov 2 18:52:05.938275 PDT 3: TAP (Producer) eq_tapq:rebalance_309 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:52:05.954218 PDT 3: TAP (Producer) eq_tapq:replication_building_309_'ns_1@10.2.1.66' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:52:05.965717 PDT 3: Deletion of vbucket 309 was completed. memcached<0.1519.0>: Fri Nov 2 18:52:05.981984 PDT 3: TAP (Producer) eq_tapq:replication_building_310_'ns_1@10.2.1.66' - Schedule the backfill for vbucket 310 memcached<0.1519.0>: Fri Nov 2 18:52:05.982025 PDT 3: TAP (Producer) eq_tapq:replication_building_310_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:05.982040 PDT 3: TAP (Producer) eq_tapq:replication_building_310_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:05.982050 PDT 3: TAP (Producer) eq_tapq:replication_building_310_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 310 memcached<0.1519.0>: Fri Nov 2 18:52:05.982182 PDT 3: Schedule cleanup of "eq_tapq:anon_54" memcached<0.1519.0>: Fri Nov 2 18:52:05.990268 PDT 3: TAP (Producer) eq_tapq:replication_building_309_'ns_1@10.2.1.66' - Clear the tap queues by force [ns_server:info,2012-11-02T18:52:07.231,ns_1@10.2.1.67:<0.17876.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.66",11209} vbucket 310 to state replica [rebalance:info,2012-11-02T18:52:07.234,ns_1@10.2.1.67:<0.17876.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[310]}, {checkpoints,[{310,10}]}, {name,<<"rebalance_310">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.66",11209}, [{username,"default"}, {password,[]}, {vbuckets,[310]}, {takeover,true}, {suffix,"310"}]} [rebalance:info,2012-11-02T18:52:07.239,ns_1@10.2.1.67:<0.17876.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:52:07.242,ns_1@10.2.1.67:<0.17876.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:52:07.326,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:52:07.125324 PDT 3: TAP (Producer) eq_tapq:replication_building_310_'ns_1@10.2.1.66' - Backfill is completed with VBuckets 310, memcached<0.1519.0>: Fri Nov 2 18:52:07.125349 PDT 3: TAP (Producer) eq_tapq:replication_building_310_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 310 memcached<0.1519.0>: Fri Nov 2 18:52:07.225588 PDT 3: TAP (Producer) eq_tapq:replication_building_310_'ns_1@10.2.1.66' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:52:07.238921 PDT 3: TAP (Producer) eq_tapq:rebalance_310 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:07.238936 PDT 3: TAP (Producer) eq_tapq:rebalance_310 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:07.238941 PDT 3: TAP (Producer) eq_tapq:rebalance_310 - Sending TAP_VBUCKET_SET with vbucket 310 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:52:07.242320 PDT 3: TAP (Producer) eq_tapq:rebalance_310 - VBucket <310> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:52:07.242336 PDT 3: TAP (Producer) eq_tapq:rebalance_310 - Sending TAP_VBUCKET_SET with vbucket 310 and state "active" memcached<0.1519.0>: Fri Nov 2 18:52:07.242607 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:52:07.242630 PDT 3: TAP (Producer) eq_tapq:rebalance_310 - disconnected memcached<0.1519.0>: Fri Nov 2 18:52:07.258926 PDT 3: TAP (Producer) eq_tapq:replication_building_310_'ns_1@10.2.1.66' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:52:07.273936 PDT 3: Deletion of vbucket 310 was completed. memcached<0.1519.0>: Fri Nov 2 18:52:07.291215 PDT 3: TAP (Producer) eq_tapq:replication_building_311_'ns_1@10.2.1.66' - Schedule the backfill for vbucket 311 memcached<0.1519.0>: Fri Nov 2 18:52:07.291242 PDT 3: TAP (Producer) eq_tapq:replication_building_311_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:07.291250 PDT 3: TAP (Producer) eq_tapq:replication_building_311_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:07.291254 PDT 3: TAP (Producer) eq_tapq:replication_building_311_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 311 memcached<0.1519.0>: Fri Nov 2 18:52:07.291354 PDT 3: Schedule cleanup of "eq_tapq:anon_55" memcached<0.1519.0>: Fri Nov 2 18:52:07.291368 PDT 3: Schedule cleanup of "eq_tapq:rebalance_310" memcached<0.1519.0>: Fri Nov 2 18:52:07.292665 PDT 3: TAP (Producer) eq_tapq:replication_building_310_'ns_1@10.2.1.66' - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:52:07.292685 PDT 3: TAP (Producer) eq_tapq:rebalance_310 - Clear the tap queues by force [ns_server:info,2012-11-02T18:52:07.342,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/310 [ns_server:info,2012-11-02T18:52:08.538,ns_1@10.2.1.67:<0.17882.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.66",11209} vbucket 311 to state replica [rebalance:info,2012-11-02T18:52:08.548,ns_1@10.2.1.67:<0.17882.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[311]}, {checkpoints,[{311,10}]}, {name,<<"rebalance_311">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.66",11209}, [{username,"default"}, {password,[]}, {vbuckets,[311]}, {takeover,true}, {suffix,"311"}]} [rebalance:info,2012-11-02T18:52:08.549,ns_1@10.2.1.67:<0.17882.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:52:08.552,ns_1@10.2.1.67:<0.17882.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:52:08.639,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/311 [ns_server:info,2012-11-02T18:52:08.656,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:52:08.455964 PDT 3: TAP (Producer) eq_tapq:replication_building_311_'ns_1@10.2.1.66' - Backfill is completed with VBuckets 311, memcached<0.1519.0>: Fri Nov 2 18:52:08.455986 PDT 3: TAP (Producer) eq_tapq:replication_building_311_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 311 memcached<0.1519.0>: Fri Nov 2 18:52:08.535241 PDT 3: TAP (Producer) eq_tapq:replication_building_311_'ns_1@10.2.1.66' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:52:08.548638 PDT 3: TAP (Producer) eq_tapq:rebalance_311 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:08.548662 PDT 3: TAP (Producer) eq_tapq:rebalance_311 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:08.548673 PDT 3: TAP (Producer) eq_tapq:rebalance_311 - Sending TAP_VBUCKET_SET with vbucket 311 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:52:08.552155 PDT 3: TAP (Producer) eq_tapq:rebalance_311 - VBucket <311> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:52:08.552172 PDT 3: TAP (Producer) eq_tapq:rebalance_311 - Sending TAP_VBUCKET_SET with vbucket 311 and state "active" memcached<0.1519.0>: Fri Nov 2 18:52:08.552690 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:52:08.552737 PDT 3: TAP (Producer) eq_tapq:rebalance_311 - disconnected memcached<0.1519.0>: Fri Nov 2 18:52:08.552790 PDT 3: Schedule cleanup of "eq_tapq:rebalance_311" memcached<0.1519.0>: Fri Nov 2 18:52:08.552838 PDT 3: TAP (Producer) eq_tapq:rebalance_311 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:52:08.575804 PDT 3: TAP (Producer) eq_tapq:replication_building_311_'ns_1@10.2.1.66' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:52:08.583414 PDT 3: Deletion of vbucket 311 was completed. memcached<0.1519.0>: Fri Nov 2 18:52:08.603652 PDT 3: TAP (Producer) eq_tapq:replication_building_312_'ns_1@10.2.1.66' - Schedule the backfill for vbucket 312 memcached<0.1519.0>: Fri Nov 2 18:52:08.603677 PDT 3: TAP (Producer) eq_tapq:replication_building_312_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:08.603685 PDT 3: TAP (Producer) eq_tapq:replication_building_312_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:08.603690 PDT 3: TAP (Producer) eq_tapq:replication_building_312_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 312 memcached<0.1519.0>: Fri Nov 2 18:52:08.603747 PDT 3: Schedule cleanup of "eq_tapq:anon_56" memcached<0.1519.0>: Fri Nov 2 18:52:08.603799 PDT 3: TAP (Producer) eq_tapq:replication_building_311_'ns_1@10.2.1.66' - Clear the tap queues by force [ns_server:info,2012-11-02T18:52:09.847,ns_1@10.2.1.67:<0.17892.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.66",11209} vbucket 312 to state replica [rebalance:info,2012-11-02T18:52:09.851,ns_1@10.2.1.67:<0.17892.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[312]}, {checkpoints,[{312,10}]}, {name,<<"rebalance_312">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.66",11209}, [{username,"default"}, {password,[]}, {vbuckets,[312]}, {takeover,true}, {suffix,"312"}]} [rebalance:info,2012-11-02T18:52:09.852,ns_1@10.2.1.67:<0.17892.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:52:09.859,ns_1@10.2.1.67:<0.17892.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:52:09.955,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/312 [ns_server:info,2012-11-02T18:52:10.018,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:52:09.817469 PDT 3: TAP (Producer) eq_tapq:replication_building_312_'ns_1@10.2.1.66' - Backfill is completed with VBuckets 312, memcached<0.1519.0>: Fri Nov 2 18:52:09.817491 PDT 3: TAP (Producer) eq_tapq:replication_building_312_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 312 memcached<0.1519.0>: Fri Nov 2 18:52:09.843419 PDT 3: TAP (Producer) eq_tapq:replication_building_312_'ns_1@10.2.1.66' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:52:09.851821 PDT 3: TAP (Producer) eq_tapq:rebalance_312 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:09.851836 PDT 3: TAP (Producer) eq_tapq:rebalance_312 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:09.851841 PDT 3: TAP (Producer) eq_tapq:rebalance_312 - Sending TAP_VBUCKET_SET with vbucket 312 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:52:09.855335 PDT 3: TAP (Producer) eq_tapq:rebalance_312 - VBucket <312> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:52:09.855368 PDT 3: TAP (Producer) eq_tapq:rebalance_312 - Sending TAP_VBUCKET_SET with vbucket 312 and state "active" memcached<0.1519.0>: Fri Nov 2 18:52:09.858863 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:52:09.858920 PDT 3: TAP (Producer) eq_tapq:rebalance_312 - disconnected memcached<0.1519.0>: Fri Nov 2 18:52:09.879583 PDT 3: TAP (Producer) eq_tapq:replication_building_312_'ns_1@10.2.1.66' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:52:09.889616 PDT 3: Deletion of vbucket 312 was completed. memcached<0.1519.0>: Fri Nov 2 18:52:09.901221 PDT 3: TAP (Producer) eq_tapq:replication_building_313_'ns_1@10.2.1.66' - Schedule the backfill for vbucket 313 memcached<0.1519.0>: Fri Nov 2 18:52:09.901248 PDT 3: TAP (Producer) eq_tapq:replication_building_313_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:09.901256 PDT 3: TAP (Producer) eq_tapq:replication_building_313_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:09.901261 PDT 3: TAP (Producer) eq_tapq:replication_building_313_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 313 memcached<0.1519.0>: Fri Nov 2 18:52:09.901335 PDT 3: Schedule cleanup of "eq_tapq:anon_57" memcached<0.1519.0>: Fri Nov 2 18:52:09.901347 PDT 3: Schedule cleanup of "eq_tapq:rebalance_312" memcached<0.1519.0>: Fri Nov 2 18:52:09.910068 PDT 3: TAP (Producer) eq_tapq:replication_building_312_'ns_1@10.2.1.66' - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:52:09.910092 PDT 3: TAP (Producer) eq_tapq:rebalance_312 - Clear the tap queues by force [ns_server:info,2012-11-02T18:52:11.059,ns_1@10.2.1.67:<0.17915.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.66",11209} vbucket 313 to state replica [rebalance:info,2012-11-02T18:52:11.069,ns_1@10.2.1.67:<0.17915.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[313]}, {checkpoints,[{313,10}]}, {name,<<"rebalance_313">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.66",11209}, [{username,"default"}, {password,[]}, {vbuckets,[313]}, {takeover,true}, {suffix,"313"}]} [rebalance:info,2012-11-02T18:52:11.072,ns_1@10.2.1.67:<0.17915.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:52:11.075,ns_1@10.2.1.67:<0.17915.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:52:11.166,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/313 [ns_server:info,2012-11-02T18:52:11.175,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:52:10.974274 PDT 3: TAP (Producer) eq_tapq:replication_building_313_'ns_1@10.2.1.66' - Backfill is completed with VBuckets 313, memcached<0.1519.0>: Fri Nov 2 18:52:10.974295 PDT 3: TAP (Producer) eq_tapq:replication_building_313_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 313 memcached<0.1519.0>: Fri Nov 2 18:52:11.055752 PDT 3: TAP (Producer) eq_tapq:replication_building_313_'ns_1@10.2.1.66' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:52:11.070916 PDT 3: TAP (Producer) eq_tapq:rebalance_313 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:11.070941 PDT 3: TAP (Producer) eq_tapq:rebalance_313 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:11.070953 PDT 3: TAP (Producer) eq_tapq:rebalance_313 - Sending TAP_VBUCKET_SET with vbucket 313 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:52:11.075058 PDT 3: TAP (Producer) eq_tapq:rebalance_313 - VBucket <313> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:52:11.075074 PDT 3: TAP (Producer) eq_tapq:rebalance_313 - Sending TAP_VBUCKET_SET with vbucket 313 and state "active" memcached<0.1519.0>: Fri Nov 2 18:52:11.075278 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:52:11.075301 PDT 3: TAP (Producer) eq_tapq:rebalance_313 - disconnected memcached<0.1519.0>: Fri Nov 2 18:52:11.075363 PDT 3: Schedule cleanup of "eq_tapq:rebalance_313" memcached<0.1519.0>: Fri Nov 2 18:52:11.075609 PDT 3: TAP (Producer) eq_tapq:rebalance_313 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:52:11.088244 PDT 3: TAP (Producer) eq_tapq:replication_building_313_'ns_1@10.2.1.66' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:52:11.096157 PDT 3: Deletion of vbucket 313 was completed. memcached<0.1519.0>: Fri Nov 2 18:52:11.112139 PDT 3: TAP (Producer) eq_tapq:replication_building_314_'ns_1@10.2.1.66' - Schedule the backfill for vbucket 314 memcached<0.1519.0>: Fri Nov 2 18:52:11.112167 PDT 3: TAP (Producer) eq_tapq:replication_building_314_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:11.112175 PDT 3: TAP (Producer) eq_tapq:replication_building_314_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:11.112180 PDT 3: TAP (Producer) eq_tapq:replication_building_314_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 314 memcached<0.1519.0>: Fri Nov 2 18:52:11.112220 PDT 3: Schedule cleanup of "eq_tapq:anon_58" memcached<0.1519.0>: Fri Nov 2 18:52:11.116942 PDT 3: TAP (Producer) eq_tapq:replication_building_313_'ns_1@10.2.1.66' - Clear the tap queues by force [ns_server:info,2012-11-02T18:52:12.270,ns_1@10.2.1.67:<0.17920.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.66",11209} vbucket 314 to state replica [rebalance:info,2012-11-02T18:52:12.280,ns_1@10.2.1.67:<0.17920.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[314]}, {checkpoints,[{314,10}]}, {name,<<"rebalance_314">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.66",11209}, [{username,"default"}, {password,[]}, {vbuckets,[314]}, {takeover,true}, {suffix,"314"}]} [rebalance:info,2012-11-02T18:52:12.282,ns_1@10.2.1.67:<0.17920.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:52:12.285,ns_1@10.2.1.67:<0.17920.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:52:12.376,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:52:12.175927 PDT 3: TAP (Producer) eq_tapq:replication_building_314_'ns_1@10.2.1.66' - Backfill is completed with VBuckets 314, memcached<0.1519.0>: Fri Nov 2 18:52:12.175948 PDT 3: TAP (Producer) eq_tapq:replication_building_314_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 314 memcached<0.1519.0>: Fri Nov 2 18:52:12.262419 PDT 3: TAP (Producer) eq_tapq:replication_building_314_'ns_1@10.2.1.66' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:52:12.281269 PDT 3: TAP (Producer) eq_tapq:rebalance_314 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:12.281294 PDT 3: TAP (Producer) eq_tapq:rebalance_314 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:12.281309 PDT 3: TAP (Producer) eq_tapq:rebalance_314 - Sending TAP_VBUCKET_SET with vbucket 314 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:52:12.285356 PDT 3: TAP (Producer) eq_tapq:rebalance_314 - VBucket <314> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:52:12.285389 PDT 3: TAP (Producer) eq_tapq:rebalance_314 - Sending TAP_VBUCKET_SET with vbucket 314 and state "active" memcached<0.1519.0>: Fri Nov 2 18:52:12.285596 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:52:12.285620 PDT 3: TAP (Producer) eq_tapq:rebalance_314 - disconnected memcached<0.1519.0>: Fri Nov 2 18:52:12.310244 PDT 3: TAP (Producer) eq_tapq:replication_building_314_'ns_1@10.2.1.66' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:52:12.320404 PDT 3: Deletion of vbucket 314 was completed. memcached<0.1519.0>: Fri Nov 2 18:52:12.336616 PDT 3: TAP (Producer) eq_tapq:replication_building_315_'ns_1@10.2.1.66' - Schedule the backfill for vbucket 315 memcached<0.1519.0>: Fri Nov 2 18:52:12.336646 PDT 3: TAP (Producer) eq_tapq:replication_building_315_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:12.336655 PDT 3: TAP (Producer) eq_tapq:replication_building_315_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:12.336661 PDT 3: TAP (Producer) eq_tapq:replication_building_315_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 315 memcached<0.1519.0>: Fri Nov 2 18:52:12.336748 PDT 3: Schedule cleanup of "eq_tapq:anon_59" memcached<0.1519.0>: Fri Nov 2 18:52:12.336764 PDT 3: Schedule cleanup of "eq_tapq:rebalance_314" memcached<0.1519.0>: Fri Nov 2 18:52:12.339776 PDT 3: TAP (Producer) eq_tapq:replication_building_314_'ns_1@10.2.1.66' - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:52:12.339819 PDT 3: TAP (Producer) eq_tapq:rebalance_314 - Clear the tap queues by force [ns_server:info,2012-11-02T18:52:12.384,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/314 [ns_server:info,2012-11-02T18:52:13.684,ns_1@10.2.1.67:<0.17931.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.66",11209} vbucket 315 to state replica [rebalance:info,2012-11-02T18:52:13.693,ns_1@10.2.1.67:<0.17931.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[315]}, {checkpoints,[{315,9}]}, {name,<<"rebalance_315">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.66",11209}, [{username,"default"}, {password,[]}, {vbuckets,[315]}, {takeover,true}, {suffix,"315"}]} [rebalance:info,2012-11-02T18:52:13.696,ns_1@10.2.1.67:<0.17931.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:52:13.699,ns_1@10.2.1.67:<0.17931.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:52:13.814,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/315 [ns_server:info,2012-11-02T18:52:13.832,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:52:13.632087 PDT 3: TAP (Producer) eq_tapq:replication_building_315_'ns_1@10.2.1.66' - Backfill is completed with VBuckets 315, memcached<0.1519.0>: Fri Nov 2 18:52:13.632114 PDT 3: TAP (Producer) eq_tapq:replication_building_315_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 315 memcached<0.1519.0>: Fri Nov 2 18:52:13.681534 PDT 3: TAP (Producer) eq_tapq:replication_building_315_'ns_1@10.2.1.66' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:52:13.693879 PDT 3: TAP (Producer) eq_tapq:rebalance_315 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:13.693894 PDT 3: TAP (Producer) eq_tapq:rebalance_315 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:13.693899 PDT 3: TAP (Producer) eq_tapq:rebalance_315 - Sending TAP_VBUCKET_SET with vbucket 315 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:52:13.699518 PDT 3: TAP (Producer) eq_tapq:rebalance_315 - VBucket <315> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:52:13.699539 PDT 3: TAP (Producer) eq_tapq:rebalance_315 - Sending TAP_VBUCKET_SET with vbucket 315 and state "active" memcached<0.1519.0>: Fri Nov 2 18:52:13.699749 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:52:13.699788 PDT 3: TAP (Producer) eq_tapq:rebalance_315 - disconnected memcached<0.1519.0>: Fri Nov 2 18:52:13.731198 PDT 3: TAP (Producer) eq_tapq:replication_building_315_'ns_1@10.2.1.66' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:52:13.740547 PDT 3: Deletion of vbucket 315 was completed. memcached<0.1519.0>: Fri Nov 2 18:52:13.753281 PDT 3: TAP (Producer) eq_tapq:replication_building_316_'ns_1@10.2.1.66' - Schedule the backfill for vbucket 316 memcached<0.1519.0>: Fri Nov 2 18:52:13.753310 PDT 3: TAP (Producer) eq_tapq:replication_building_316_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:13.753317 PDT 3: TAP (Producer) eq_tapq:replication_building_316_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:13.753322 PDT 3: TAP (Producer) eq_tapq:replication_building_316_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 316 memcached<0.1519.0>: Fri Nov 2 18:52:13.753404 PDT 3: Schedule cleanup of "eq_tapq:anon_60" memcached<0.1519.0>: Fri Nov 2 18:52:13.753421 PDT 3: Schedule cleanup of "eq_tapq:rebalance_315" memcached<0.1519.0>: Fri Nov 2 18:52:13.758604 PDT 3: TAP (Producer) eq_tapq:replication_building_315_'ns_1@10.2.1.66' - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:52:13.758627 PDT 3: TAP (Producer) eq_tapq:rebalance_315 - Clear the tap queues by force [ns_server:info,2012-11-02T18:52:15.005,ns_1@10.2.1.67:<0.17941.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.66",11209} vbucket 316 to state replica [rebalance:info,2012-11-02T18:52:15.008,ns_1@10.2.1.67:<0.17941.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[316]}, {checkpoints,[{316,10}]}, {name,<<"rebalance_316">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.66",11209}, [{username,"default"}, {password,[]}, {vbuckets,[316]}, {takeover,true}, {suffix,"316"}]} [rebalance:info,2012-11-02T18:52:15.013,ns_1@10.2.1.67:<0.17941.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:52:15.017,ns_1@10.2.1.67:<0.17941.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:52:15.115,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:52:14.902473 PDT 3: TAP (Producer) eq_tapq:replication_building_316_'ns_1@10.2.1.66' - Backfill is completed with VBuckets 316, memcached<0.1519.0>: Fri Nov 2 18:52:14.902501 PDT 3: TAP (Producer) eq_tapq:replication_building_316_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 316 memcached<0.1519.0>: Fri Nov 2 18:52:15.001401 PDT 3: TAP (Producer) eq_tapq:replication_building_316_'ns_1@10.2.1.66' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:52:15.008472 PDT 3: TAP (Producer) eq_tapq:rebalance_316 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:15.008485 PDT 3: TAP (Producer) eq_tapq:rebalance_316 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:15.008490 PDT 3: TAP (Producer) eq_tapq:rebalance_316 - Sending TAP_VBUCKET_SET with vbucket 316 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:52:15.016543 PDT 3: TAP (Producer) eq_tapq:rebalance_316 - VBucket <316> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:52:15.016567 PDT 3: TAP (Producer) eq_tapq:rebalance_316 - Sending TAP_VBUCKET_SET with vbucket 316 and state "active" memcached<0.1519.0>: Fri Nov 2 18:52:15.017006 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:52:15.017064 PDT 3: TAP (Producer) eq_tapq:rebalance_316 - disconnected memcached<0.1519.0>: Fri Nov 2 18:52:15.042850 PDT 3: TAP (Producer) eq_tapq:replication_building_316_'ns_1@10.2.1.66' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:52:15.054585 PDT 3: Deletion of vbucket 316 was completed. memcached<0.1519.0>: Fri Nov 2 18:52:15.078373 PDT 3: TAP (Producer) eq_tapq:replication_building_317_'ns_1@10.2.1.66' - Schedule the backfill for vbucket 317 memcached<0.1519.0>: Fri Nov 2 18:52:15.078396 PDT 3: TAP (Producer) eq_tapq:replication_building_317_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:15.078404 PDT 3: TAP (Producer) eq_tapq:replication_building_317_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:15.078409 PDT 3: TAP (Producer) eq_tapq:replication_building_317_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 317 memcached<0.1519.0>: Fri Nov 2 18:52:15.078486 PDT 3: Schedule cleanup of "eq_tapq:anon_61" memcached<0.1519.0>: Fri Nov 2 18:52:15.078498 PDT 3: Schedule cleanup of "eq_tapq:rebalance_316" memcached<0.1519.0>: Fri Nov 2 18:52:15.078577 PDT 3: TAP (Producer) eq_tapq:replication_building_316_'ns_1@10.2.1.66' - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:52:15.078592 PDT 3: TAP (Producer) eq_tapq:rebalance_316 - Clear the tap queues by force [ns_server:info,2012-11-02T18:52:15.117,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/316 [ns_server:info,2012-11-02T18:52:16.357,ns_1@10.2.1.67:<0.17955.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.66",11209} vbucket 317 to state replica [rebalance:info,2012-11-02T18:52:16.360,ns_1@10.2.1.67:<0.17955.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[317]}, {checkpoints,[{317,10}]}, {name,<<"rebalance_317">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.66",11209}, [{username,"default"}, {password,[]}, {vbuckets,[317]}, {takeover,true}, {suffix,"317"}]} [rebalance:info,2012-11-02T18:52:16.366,ns_1@10.2.1.67:<0.17955.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:52:16.369,ns_1@10.2.1.67:<0.17955.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:52:16.456,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:52:16.239974 PDT 3: TAP (Producer) eq_tapq:replication_building_317_'ns_1@10.2.1.66' - Backfill is completed with VBuckets 317, memcached<0.1519.0>: Fri Nov 2 18:52:16.239993 PDT 3: TAP (Producer) eq_tapq:replication_building_317_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 317 memcached<0.1519.0>: Fri Nov 2 18:52:16.353731 PDT 3: TAP (Producer) eq_tapq:replication_building_317_'ns_1@10.2.1.66' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:52:16.360458 PDT 3: TAP (Producer) eq_tapq:rebalance_317 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:16.360472 PDT 3: TAP (Producer) eq_tapq:rebalance_317 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:16.360477 PDT 3: TAP (Producer) eq_tapq:rebalance_317 - Sending TAP_VBUCKET_SET with vbucket 317 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:52:16.368876 PDT 3: TAP (Producer) eq_tapq:rebalance_317 - VBucket <317> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:52:16.368909 PDT 3: TAP (Producer) eq_tapq:rebalance_317 - Sending TAP_VBUCKET_SET with vbucket 317 and state "active" memcached<0.1519.0>: Fri Nov 2 18:52:16.369130 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:52:16.369164 PDT 3: TAP (Producer) eq_tapq:rebalance_317 - disconnected memcached<0.1519.0>: Fri Nov 2 18:52:16.369263 PDT 3: Schedule cleanup of "eq_tapq:rebalance_317" memcached<0.1519.0>: Fri Nov 2 18:52:16.369290 PDT 3: TAP (Producer) eq_tapq:rebalance_317 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:52:16.384359 PDT 3: TAP (Producer) eq_tapq:replication_building_317_'ns_1@10.2.1.66' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:52:16.391868 PDT 3: Deletion of vbucket 317 was completed. [ns_server:info,2012-11-02T18:52:16.456,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/317 [ns_server:info,2012-11-02T18:52:16.656,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:52:16.405726 PDT 3: TAP (Producer) eq_tapq:replication_building_318_'ns_1@10.2.1.66' - Schedule the backfill for vbucket 318 memcached<0.1519.0>: Fri Nov 2 18:52:16.405753 PDT 3: TAP (Producer) eq_tapq:replication_building_318_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:16.405761 PDT 3: TAP (Producer) eq_tapq:replication_building_318_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:16.405765 PDT 3: TAP (Producer) eq_tapq:replication_building_318_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 318 memcached<0.1519.0>: Fri Nov 2 18:52:16.405807 PDT 3: Schedule cleanup of "eq_tapq:anon_62" memcached<0.1519.0>: Fri Nov 2 18:52:16.409617 PDT 3: TAP (Producer) eq_tapq:replication_building_317_'ns_1@10.2.1.66' - Clear the tap queues by force [ns_server:info,2012-11-02T18:52:17.810,ns_1@10.2.1.67:<0.17966.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.66",11209} vbucket 318 to state replica [rebalance:info,2012-11-02T18:52:17.814,ns_1@10.2.1.67:<0.17966.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[318]}, {checkpoints,[{318,10}]}, {name,<<"rebalance_318">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.66",11209}, [{username,"default"}, {password,[]}, {vbuckets,[318]}, {takeover,true}, {suffix,"318"}]} [rebalance:info,2012-11-02T18:52:17.815,ns_1@10.2.1.67:<0.17966.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:52:17.822,ns_1@10.2.1.67:<0.17966.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:52:17.917,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:52:17.699764 PDT 3: TAP (Producer) eq_tapq:replication_building_318_'ns_1@10.2.1.66' - Backfill is completed with VBuckets 318, memcached<0.1519.0>: Fri Nov 2 18:52:17.699784 PDT 3: TAP (Producer) eq_tapq:replication_building_318_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 318 memcached<0.1519.0>: Fri Nov 2 18:52:17.807155 PDT 3: TAP (Producer) eq_tapq:replication_building_318_'ns_1@10.2.1.66' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:52:17.814807 PDT 3: TAP (Producer) eq_tapq:rebalance_318 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:17.814820 PDT 3: TAP (Producer) eq_tapq:rebalance_318 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:17.814825 PDT 3: TAP (Producer) eq_tapq:rebalance_318 - Sending TAP_VBUCKET_SET with vbucket 318 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:52:17.821223 PDT 3: TAP (Producer) eq_tapq:rebalance_318 - VBucket <318> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:52:17.821241 PDT 3: TAP (Producer) eq_tapq:rebalance_318 - Sending TAP_VBUCKET_SET with vbucket 318 and state "active" memcached<0.1519.0>: Fri Nov 2 18:52:17.822269 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:52:17.822296 PDT 3: TAP (Producer) eq_tapq:rebalance_318 - disconnected memcached<0.1519.0>: Fri Nov 2 18:52:17.836556 PDT 3: TAP (Producer) eq_tapq:replication_building_318_'ns_1@10.2.1.66' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:52:17.845492 PDT 3: Deletion of vbucket 318 was completed. [ns_server:info,2012-11-02T18:52:17.919,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/318 [ns_server:info,2012-11-02T18:52:18.118,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:52:17.896509 PDT 3: TAP (Producer) eq_tapq:replication_building_319_'ns_1@10.2.1.66' - Schedule the backfill for vbucket 319 memcached<0.1519.0>: Fri Nov 2 18:52:17.896551 PDT 3: TAP (Producer) eq_tapq:replication_building_319_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:17.896565 PDT 3: TAP (Producer) eq_tapq:replication_building_319_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:17.896575 PDT 3: TAP (Producer) eq_tapq:replication_building_319_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 319 memcached<0.1519.0>: Fri Nov 2 18:52:17.896734 PDT 3: Schedule cleanup of "eq_tapq:anon_63" memcached<0.1519.0>: Fri Nov 2 18:52:17.896756 PDT 3: Schedule cleanup of "eq_tapq:rebalance_318" memcached<0.1519.0>: Fri Nov 2 18:52:17.896935 PDT 3: TAP (Producer) eq_tapq:replication_building_318_'ns_1@10.2.1.66' - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:52:17.896966 PDT 3: TAP (Producer) eq_tapq:rebalance_318 - Clear the tap queues by force [ns_server:info,2012-11-02T18:52:19.086,ns_1@10.2.1.67:<0.17976.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.66",11209} vbucket 319 to state replica [rebalance:info,2012-11-02T18:52:19.088,ns_1@10.2.1.67:<0.17976.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[319]}, {checkpoints,[{319,10}]}, {name,<<"rebalance_319">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.66",11209}, [{username,"default"}, {password,[]}, {vbuckets,[319]}, {takeover,true}, {suffix,"319"}]} [rebalance:info,2012-11-02T18:52:19.096,ns_1@10.2.1.67:<0.17976.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:52:19.098,ns_1@10.2.1.67:<0.17976.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:52:19.201,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/319 [ns_server:info,2012-11-02T18:52:19.233,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:52:19.032779 PDT 3: TAP (Producer) eq_tapq:replication_building_319_'ns_1@10.2.1.66' - Backfill is completed with VBuckets 319, memcached<0.1519.0>: Fri Nov 2 18:52:19.032801 PDT 3: TAP (Producer) eq_tapq:replication_building_319_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 319 memcached<0.1519.0>: Fri Nov 2 18:52:19.082833 PDT 3: TAP (Producer) eq_tapq:replication_building_319_'ns_1@10.2.1.66' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:52:19.089384 PDT 3: TAP (Producer) eq_tapq:rebalance_319 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:19.089397 PDT 3: TAP (Producer) eq_tapq:rebalance_319 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:19.089402 PDT 3: TAP (Producer) eq_tapq:rebalance_319 - Sending TAP_VBUCKET_SET with vbucket 319 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:52:19.098611 PDT 3: TAP (Producer) eq_tapq:rebalance_319 - VBucket <319> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:52:19.098628 PDT 3: TAP (Producer) eq_tapq:rebalance_319 - Sending TAP_VBUCKET_SET with vbucket 319 and state "active" memcached<0.1519.0>: Fri Nov 2 18:52:19.098788 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:52:19.098810 PDT 3: TAP (Producer) eq_tapq:rebalance_319 - disconnected memcached<0.1519.0>: Fri Nov 2 18:52:19.098842 PDT 3: Schedule cleanup of "eq_tapq:rebalance_319" memcached<0.1519.0>: Fri Nov 2 18:52:19.099184 PDT 3: TAP (Producer) eq_tapq:rebalance_319 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:52:19.123125 PDT 3: TAP (Producer) eq_tapq:replication_building_319_'ns_1@10.2.1.66' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:52:19.138415 PDT 3: Deletion of vbucket 319 was completed. memcached<0.1519.0>: Fri Nov 2 18:52:19.154047 PDT 3: TAP (Producer) eq_tapq:replication_building_320_'ns_1@10.2.1.66' - Schedule the backfill for vbucket 320 memcached<0.1519.0>: Fri Nov 2 18:52:19.154079 PDT 3: TAP (Producer) eq_tapq:replication_building_320_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:19.154087 PDT 3: TAP (Producer) eq_tapq:replication_building_320_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:19.154092 PDT 3: TAP (Producer) eq_tapq:replication_building_320_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 320 memcached<0.1519.0>: Fri Nov 2 18:52:19.154144 PDT 3: Schedule cleanup of "eq_tapq:anon_64" memcached<0.1519.0>: Fri Nov 2 18:52:19.162511 PDT 3: TAP (Producer) eq_tapq:replication_building_319_'ns_1@10.2.1.66' - Clear the tap queues by force [ns_server:info,2012-11-02T18:52:20.552,ns_1@10.2.1.67:<0.17983.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.66",11209} vbucket 320 to state replica [rebalance:info,2012-11-02T18:52:20.555,ns_1@10.2.1.67:<0.17983.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[320]}, {checkpoints,[{320,10}]}, {name,<<"rebalance_320">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.66",11209}, [{username,"default"}, {password,[]}, {vbuckets,[320]}, {takeover,true}, {suffix,"320"}]} [rebalance:info,2012-11-02T18:52:20.556,ns_1@10.2.1.67:<0.17983.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:52:20.562,ns_1@10.2.1.67:<0.17983.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:52:20.654,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/320 [ns_server:info,2012-11-02T18:52:20.720,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:52:20.514990 PDT 3: TAP (Producer) eq_tapq:replication_building_320_'ns_1@10.2.1.66' - Backfill is completed with VBuckets 320, memcached<0.1519.0>: Fri Nov 2 18:52:20.515016 PDT 3: TAP (Producer) eq_tapq:replication_building_320_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 320 memcached<0.1519.0>: Fri Nov 2 18:52:20.550163 PDT 3: TAP (Producer) eq_tapq:replication_building_320_'ns_1@10.2.1.66' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:52:20.555516 PDT 3: TAP (Producer) eq_tapq:rebalance_320 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:20.555528 PDT 3: TAP (Producer) eq_tapq:rebalance_320 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:20.555533 PDT 3: TAP (Producer) eq_tapq:rebalance_320 - Sending TAP_VBUCKET_SET with vbucket 320 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:52:20.562149 PDT 3: TAP (Producer) eq_tapq:rebalance_320 - VBucket <320> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:52:20.562172 PDT 3: TAP (Producer) eq_tapq:rebalance_320 - Sending TAP_VBUCKET_SET with vbucket 320 and state "active" memcached<0.1519.0>: Fri Nov 2 18:52:20.562431 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:52:20.562453 PDT 3: TAP (Producer) eq_tapq:rebalance_320 - disconnected memcached<0.1519.0>: Fri Nov 2 18:52:20.562484 PDT 3: Schedule cleanup of "eq_tapq:rebalance_320" memcached<0.1519.0>: Fri Nov 2 18:52:20.562559 PDT 3: TAP (Producer) eq_tapq:rebalance_320 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:52:20.575336 PDT 3: TAP (Producer) eq_tapq:replication_building_320_'ns_1@10.2.1.66' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:52:20.585983 PDT 3: Deletion of vbucket 320 was completed. memcached<0.1519.0>: Fri Nov 2 18:52:20.603507 PDT 3: TAP (Producer) eq_tapq:replication_building_321_'ns_1@10.2.1.66' - Schedule the backfill for vbucket 321 memcached<0.1519.0>: Fri Nov 2 18:52:20.603533 PDT 3: TAP (Producer) eq_tapq:replication_building_321_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:20.603540 PDT 3: TAP (Producer) eq_tapq:replication_building_321_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:20.603545 PDT 3: TAP (Producer) eq_tapq:replication_building_321_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 321 memcached<0.1519.0>: Fri Nov 2 18:52:20.603597 PDT 3: Schedule cleanup of "eq_tapq:anon_65" memcached<0.1519.0>: Fri Nov 2 18:52:20.612865 PDT 3: TAP (Producer) eq_tapq:replication_building_320_'ns_1@10.2.1.66' - Clear the tap queues by force [ns_server:info,2012-11-02T18:52:21.827,ns_1@10.2.1.67:<0.18001.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.66",11209} vbucket 321 to state replica [rebalance:info,2012-11-02T18:52:21.830,ns_1@10.2.1.67:<0.18001.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[321]}, {checkpoints,[{321,10}]}, {name,<<"rebalance_321">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.66",11209}, [{username,"default"}, {password,[]}, {vbuckets,[321]}, {takeover,true}, {suffix,"321"}]} [rebalance:info,2012-11-02T18:52:21.836,ns_1@10.2.1.67:<0.18001.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:52:21.839,ns_1@10.2.1.67:<0.18001.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:52:21.929,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/321 [ns_server:info,2012-11-02T18:52:21.988,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:52:21.788294 PDT 3: TAP (Producer) eq_tapq:replication_building_321_'ns_1@10.2.1.66' - Backfill is completed with VBuckets 321, memcached<0.1519.0>: Fri Nov 2 18:52:21.788316 PDT 3: TAP (Producer) eq_tapq:replication_building_321_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 321 memcached<0.1519.0>: Fri Nov 2 18:52:21.823969 PDT 3: TAP (Producer) eq_tapq:replication_building_321_'ns_1@10.2.1.66' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:52:21.835950 PDT 3: TAP (Producer) eq_tapq:rebalance_321 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:21.835964 PDT 3: TAP (Producer) eq_tapq:rebalance_321 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:21.835969 PDT 3: TAP (Producer) eq_tapq:rebalance_321 - Sending TAP_VBUCKET_SET with vbucket 321 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:52:21.839011 PDT 3: TAP (Producer) eq_tapq:rebalance_321 - VBucket <321> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:52:21.839027 PDT 3: TAP (Producer) eq_tapq:rebalance_321 - Sending TAP_VBUCKET_SET with vbucket 321 and state "active" memcached<0.1519.0>: Fri Nov 2 18:52:21.839322 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:52:21.839344 PDT 3: TAP (Producer) eq_tapq:rebalance_321 - disconnected memcached<0.1519.0>: Fri Nov 2 18:52:21.854513 PDT 3: TAP (Producer) eq_tapq:replication_building_321_'ns_1@10.2.1.66' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:52:21.865354 PDT 3: Deletion of vbucket 321 was completed. memcached<0.1519.0>: Fri Nov 2 18:52:21.899936 PDT 3: TAP (Producer) eq_tapq:replication_building_322_'ns_1@10.2.1.66' - Schedule the backfill for vbucket 322 memcached<0.1519.0>: Fri Nov 2 18:52:21.899967 PDT 3: TAP (Producer) eq_tapq:replication_building_322_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:21.899978 PDT 3: TAP (Producer) eq_tapq:replication_building_322_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:21.899985 PDT 3: TAP (Producer) eq_tapq:replication_building_322_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 322 memcached<0.1519.0>: Fri Nov 2 18:52:21.900087 PDT 3: Schedule cleanup of "eq_tapq:anon_66" memcached<0.1519.0>: Fri Nov 2 18:52:21.900112 PDT 3: Schedule cleanup of "eq_tapq:rebalance_321" memcached<0.1519.0>: Fri Nov 2 18:52:21.900222 PDT 3: TAP (Producer) eq_tapq:replication_building_321_'ns_1@10.2.1.66' - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:52:21.900237 PDT 3: TAP (Producer) eq_tapq:rebalance_321 - Clear the tap queues by force [ns_server:info,2012-11-02T18:52:23.164,ns_1@10.2.1.67:<0.18012.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.66",11209} vbucket 322 to state replica [rebalance:info,2012-11-02T18:52:23.171,ns_1@10.2.1.67:<0.18012.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[322]}, {checkpoints,[{322,10}]}, {name,<<"rebalance_322">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.66",11209}, [{username,"default"}, {password,[]}, {vbuckets,[322]}, {takeover,true}, {suffix,"322"}]} [rebalance:info,2012-11-02T18:52:23.172,ns_1@10.2.1.67:<0.18012.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:52:23.176,ns_1@10.2.1.67:<0.18012.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:52:23.282,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/322 [ns_server:info,2012-11-02T18:52:23.333,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:52:23.133005 PDT 3: TAP (Producer) eq_tapq:replication_building_322_'ns_1@10.2.1.66' - Backfill is completed with VBuckets 322, memcached<0.1519.0>: Fri Nov 2 18:52:23.133028 PDT 3: TAP (Producer) eq_tapq:replication_building_322_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 322 memcached<0.1519.0>: Fri Nov 2 18:52:23.160257 PDT 3: TAP (Producer) eq_tapq:replication_building_322_'ns_1@10.2.1.66' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:52:23.172190 PDT 3: TAP (Producer) eq_tapq:rebalance_322 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:23.172203 PDT 3: TAP (Producer) eq_tapq:rebalance_322 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:23.172208 PDT 3: TAP (Producer) eq_tapq:rebalance_322 - Sending TAP_VBUCKET_SET with vbucket 322 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:52:23.175796 PDT 3: TAP (Producer) eq_tapq:rebalance_322 - VBucket <322> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:52:23.175813 PDT 3: TAP (Producer) eq_tapq:rebalance_322 - Sending TAP_VBUCKET_SET with vbucket 322 and state "active" memcached<0.1519.0>: Fri Nov 2 18:52:23.176155 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:52:23.176179 PDT 3: TAP (Producer) eq_tapq:rebalance_322 - disconnected memcached<0.1519.0>: Fri Nov 2 18:52:23.176210 PDT 3: Schedule cleanup of "eq_tapq:rebalance_322" memcached<0.1519.0>: Fri Nov 2 18:52:23.176281 PDT 3: TAP (Producer) eq_tapq:rebalance_322 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:52:23.208321 PDT 3: TAP (Producer) eq_tapq:replication_building_322_'ns_1@10.2.1.66' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:52:23.219537 PDT 3: Deletion of vbucket 322 was completed. memcached<0.1519.0>: Fri Nov 2 18:52:23.249214 PDT 3: TAP (Producer) eq_tapq:replication_building_323_'ns_1@10.2.1.66' - Schedule the backfill for vbucket 323 memcached<0.1519.0>: Fri Nov 2 18:52:23.249257 PDT 3: TAP (Producer) eq_tapq:replication_building_323_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:23.249271 PDT 3: TAP (Producer) eq_tapq:replication_building_323_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:23.249281 PDT 3: TAP (Producer) eq_tapq:replication_building_323_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 323 memcached<0.1519.0>: Fri Nov 2 18:52:23.249339 PDT 3: Schedule cleanup of "eq_tapq:anon_67" memcached<0.1519.0>: Fri Nov 2 18:52:23.249558 PDT 3: TAP (Producer) eq_tapq:replication_building_322_'ns_1@10.2.1.66' - Clear the tap queues by force [ns_server:info,2012-11-02T18:52:24.541,ns_1@10.2.1.67:<0.18017.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.66",11209} vbucket 323 to state replica [rebalance:info,2012-11-02T18:52:24.548,ns_1@10.2.1.67:<0.18017.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[323]}, {checkpoints,[{323,10}]}, {name,<<"rebalance_323">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.66",11209}, [{username,"default"}, {password,[]}, {vbuckets,[323]}, {takeover,true}, {suffix,"323"}]} [rebalance:info,2012-11-02T18:52:24.549,ns_1@10.2.1.67:<0.18017.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:52:24.552,ns_1@10.2.1.67:<0.18017.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:52:24.654,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/323 [ns_server:info,2012-11-02T18:52:24.691,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:52:24.490812 PDT 3: TAP (Producer) eq_tapq:replication_building_323_'ns_1@10.2.1.66' - Backfill is completed with VBuckets 323, memcached<0.1519.0>: Fri Nov 2 18:52:24.490834 PDT 3: TAP (Producer) eq_tapq:replication_building_323_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 323 memcached<0.1519.0>: Fri Nov 2 18:52:24.537635 PDT 3: TAP (Producer) eq_tapq:replication_building_323_'ns_1@10.2.1.66' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:52:24.549127 PDT 3: TAP (Producer) eq_tapq:rebalance_323 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:24.549140 PDT 3: TAP (Producer) eq_tapq:rebalance_323 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:24.549145 PDT 3: TAP (Producer) eq_tapq:rebalance_323 - Sending TAP_VBUCKET_SET with vbucket 323 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:52:24.552337 PDT 3: TAP (Producer) eq_tapq:rebalance_323 - VBucket <323> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:52:24.552353 PDT 3: TAP (Producer) eq_tapq:rebalance_323 - Sending TAP_VBUCKET_SET with vbucket 323 and state "active" memcached<0.1519.0>: Fri Nov 2 18:52:24.552541 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:52:24.552564 PDT 3: TAP (Producer) eq_tapq:rebalance_323 - disconnected memcached<0.1519.0>: Fri Nov 2 18:52:24.570598 PDT 3: TAP (Producer) eq_tapq:replication_building_323_'ns_1@10.2.1.66' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:52:24.580692 PDT 3: Deletion of vbucket 323 was completed. memcached<0.1519.0>: Fri Nov 2 18:52:24.595444 PDT 3: TAP (Producer) eq_tapq:replication_building_324_'ns_1@10.2.1.66' - Schedule the backfill for vbucket 324 memcached<0.1519.0>: Fri Nov 2 18:52:24.595487 PDT 3: TAP (Producer) eq_tapq:replication_building_324_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:24.595502 PDT 3: TAP (Producer) eq_tapq:replication_building_324_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:24.595512 PDT 3: TAP (Producer) eq_tapq:replication_building_324_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 324 memcached<0.1519.0>: Fri Nov 2 18:52:24.595665 PDT 3: Schedule cleanup of "eq_tapq:anon_68" memcached<0.1519.0>: Fri Nov 2 18:52:24.595690 PDT 3: Schedule cleanup of "eq_tapq:rebalance_323" memcached<0.1519.0>: Fri Nov 2 18:52:24.608218 PDT 3: TAP (Producer) eq_tapq:replication_building_323_'ns_1@10.2.1.66' - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:52:24.608262 PDT 3: TAP (Producer) eq_tapq:rebalance_323 - Clear the tap queues by force [ns_server:info,2012-11-02T18:52:25.216,ns_1@10.2.1.67:ns_config_rep<0.1439.0>:ns_config_rep:do_pull:341]Pulling config from: 'ns_1@10.2.1.66' [ns_server:info,2012-11-02T18:52:25.811,ns_1@10.2.1.67:<0.18039.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.66",11209} vbucket 324 to state replica [rebalance:info,2012-11-02T18:52:25.813,ns_1@10.2.1.67:<0.18039.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[324]}, {checkpoints,[{324,10}]}, {name,<<"rebalance_324">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.66",11209}, [{username,"default"}, {password,[]}, {vbuckets,[324]}, {takeover,true}, {suffix,"324"}]} [rebalance:info,2012-11-02T18:52:25.819,ns_1@10.2.1.67:<0.18039.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:52:25.823,ns_1@10.2.1.67:<0.18039.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:52:26.020,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:52:25.785079 PDT 3: TAP (Producer) eq_tapq:replication_building_324_'ns_1@10.2.1.66' - Backfill is completed with VBuckets 324, memcached<0.1519.0>: Fri Nov 2 18:52:25.785100 PDT 3: TAP (Producer) eq_tapq:replication_building_324_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 324 memcached<0.1519.0>: Fri Nov 2 18:52:25.808193 PDT 3: TAP (Producer) eq_tapq:replication_building_324_'ns_1@10.2.1.66' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:52:25.814111 PDT 3: TAP (Producer) eq_tapq:rebalance_324 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:25.814125 PDT 3: TAP (Producer) eq_tapq:rebalance_324 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:25.814130 PDT 3: TAP (Producer) eq_tapq:rebalance_324 - Sending TAP_VBUCKET_SET with vbucket 324 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:52:25.822773 PDT 3: TAP (Producer) eq_tapq:rebalance_324 - VBucket <324> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:52:25.822794 PDT 3: TAP (Producer) eq_tapq:rebalance_324 - Sending TAP_VBUCKET_SET with vbucket 324 and state "active" memcached<0.1519.0>: Fri Nov 2 18:52:25.823159 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:52:25.823189 PDT 3: TAP (Producer) eq_tapq:rebalance_324 - disconnected memcached<0.1519.0>: Fri Nov 2 18:52:25.854716 PDT 3: TAP (Producer) eq_tapq:replication_building_324_'ns_1@10.2.1.66' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:52:25.867846 PDT 3: Deletion of vbucket 324 was completed. memcached<0.1519.0>: Fri Nov 2 18:52:25.899709 PDT 3: TAP (Producer) eq_tapq:replication_building_325_'ns_1@10.2.1.66' - Schedule the backfill for vbucket 325 memcached<0.1519.0>: Fri Nov 2 18:52:25.899733 PDT 3: TAP (Producer) eq_tapq:replication_building_325_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:25.899740 PDT 3: TAP (Producer) eq_tapq:replication_building_325_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:25.899745 PDT 3: TAP (Producer) eq_tapq:replication_building_325_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 325 memcached<0.1519.0>: Fri Nov 2 18:52:25.899875 PDT 3: Schedule cleanup of "eq_tapq:anon_69" memcached<0.1519.0>: Fri Nov 2 18:52:25.899910 PDT 3: Schedule cleanup of "eq_tapq:rebalance_324" memcached<0.1519.0>: Fri Nov 2 18:52:25.899936 PDT 3: TAP (Producer) eq_tapq:replication_building_324_'ns_1@10.2.1.66' - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:52:25.899971 PDT 3: TAP (Producer) eq_tapq:rebalance_324 - Clear the tap queues by force [ns_server:info,2012-11-02T18:52:26.021,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/324 [ns_server:info,2012-11-02T18:52:27.166,ns_1@10.2.1.67:<0.18049.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.66",11209} vbucket 325 to state replica [rebalance:info,2012-11-02T18:52:27.169,ns_1@10.2.1.67:<0.18049.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[325]}, {checkpoints,[{325,10}]}, {name,<<"rebalance_325">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.66",11209}, [{username,"default"}, {password,[]}, {vbuckets,[325]}, {takeover,true}, {suffix,"325"}]} [rebalance:info,2012-11-02T18:52:27.170,ns_1@10.2.1.67:<0.18049.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:52:27.173,ns_1@10.2.1.67:<0.18049.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:52:27.329,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/325 [ns_server:info,2012-11-02T18:52:27.354,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:52:27.151361 PDT 3: TAP (Producer) eq_tapq:replication_building_325_'ns_1@10.2.1.66' - Backfill is completed with VBuckets 325, memcached<0.1519.0>: Fri Nov 2 18:52:27.151385 PDT 3: TAP (Producer) eq_tapq:replication_building_325_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 325 memcached<0.1519.0>: Fri Nov 2 18:52:27.162173 PDT 3: TAP (Producer) eq_tapq:replication_building_325_'ns_1@10.2.1.66' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:52:27.169719 PDT 3: TAP (Producer) eq_tapq:rebalance_325 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:27.169730 PDT 3: TAP (Producer) eq_tapq:rebalance_325 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:27.169736 PDT 3: TAP (Producer) eq_tapq:rebalance_325 - Sending TAP_VBUCKET_SET with vbucket 325 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:52:27.173231 PDT 3: TAP (Producer) eq_tapq:rebalance_325 - VBucket <325> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:52:27.173249 PDT 3: TAP (Producer) eq_tapq:rebalance_325 - Sending TAP_VBUCKET_SET with vbucket 325 and state "active" memcached<0.1519.0>: Fri Nov 2 18:52:27.173735 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:52:27.173757 PDT 3: TAP (Producer) eq_tapq:rebalance_325 - disconnected memcached<0.1519.0>: Fri Nov 2 18:52:27.174142 PDT 3: Schedule cleanup of "eq_tapq:rebalance_325" memcached<0.1519.0>: Fri Nov 2 18:52:27.174379 PDT 3: TAP (Producer) eq_tapq:rebalance_325 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:52:27.198756 PDT 3: TAP (Producer) eq_tapq:replication_building_325_'ns_1@10.2.1.66' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:52:27.211622 PDT 3: Deletion of vbucket 325 was completed. memcached<0.1519.0>: Fri Nov 2 18:52:27.229127 PDT 3: TAP (Producer) eq_tapq:replication_building_326_'ns_1@10.2.1.66' - Schedule the backfill for vbucket 326 memcached<0.1519.0>: Fri Nov 2 18:52:27.229154 PDT 3: TAP (Producer) eq_tapq:replication_building_326_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:27.229161 PDT 3: TAP (Producer) eq_tapq:replication_building_326_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:27.229165 PDT 3: TAP (Producer) eq_tapq:replication_building_326_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 326 memcached<0.1519.0>: Fri Nov 2 18:52:27.229217 PDT 3: Schedule cleanup of "eq_tapq:anon_70" memcached<0.1519.0>: Fri Nov 2 18:52:27.229854 PDT 3: TAP (Producer) eq_tapq:replication_building_325_'ns_1@10.2.1.66' - Clear the tap queues by force [ns_server:info,2012-11-02T18:52:28.605,ns_1@10.2.1.67:<0.18054.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.66",11209} vbucket 326 to state replica [rebalance:info,2012-11-02T18:52:28.612,ns_1@10.2.1.67:<0.18054.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[326]}, {checkpoints,[{326,10}]}, {name,<<"rebalance_326">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.66",11209}, [{username,"default"}, {password,[]}, {vbuckets,[326]}, {takeover,true}, {suffix,"326"}]} [rebalance:info,2012-11-02T18:52:28.613,ns_1@10.2.1.67:<0.18054.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:52:28.616,ns_1@10.2.1.67:<0.18054.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:52:28.712,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/326 [ns_server:info,2012-11-02T18:52:28.777,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:52:28.576632 PDT 3: TAP (Producer) eq_tapq:replication_building_326_'ns_1@10.2.1.66' - Backfill is completed with VBuckets 326, memcached<0.1519.0>: Fri Nov 2 18:52:28.576654 PDT 3: TAP (Producer) eq_tapq:replication_building_326_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 326 memcached<0.1519.0>: Fri Nov 2 18:52:28.601793 PDT 3: TAP (Producer) eq_tapq:replication_building_326_'ns_1@10.2.1.66' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:52:28.612815 PDT 3: TAP (Producer) eq_tapq:rebalance_326 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:28.612829 PDT 3: TAP (Producer) eq_tapq:rebalance_326 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:28.612841 PDT 3: TAP (Producer) eq_tapq:rebalance_326 - Sending TAP_VBUCKET_SET with vbucket 326 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:52:28.615925 PDT 3: TAP (Producer) eq_tapq:rebalance_326 - VBucket <326> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:52:28.615952 PDT 3: TAP (Producer) eq_tapq:rebalance_326 - Sending TAP_VBUCKET_SET with vbucket 326 and state "active" memcached<0.1519.0>: Fri Nov 2 18:52:28.616306 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:52:28.616348 PDT 3: TAP (Producer) eq_tapq:rebalance_326 - disconnected memcached<0.1519.0>: Fri Nov 2 18:52:28.616556 PDT 3: Schedule cleanup of "eq_tapq:rebalance_326" memcached<0.1519.0>: Fri Nov 2 18:52:28.616715 PDT 3: TAP (Producer) eq_tapq:rebalance_326 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:52:28.629073 PDT 3: TAP (Producer) eq_tapq:replication_building_326_'ns_1@10.2.1.66' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:52:28.639604 PDT 3: Deletion of vbucket 326 was completed. memcached<0.1519.0>: Fri Nov 2 18:52:28.656733 PDT 3: TAP (Producer) eq_tapq:replication_building_327_'ns_1@10.2.1.66' - Schedule the backfill for vbucket 327 memcached<0.1519.0>: Fri Nov 2 18:52:28.656760 PDT 3: TAP (Producer) eq_tapq:replication_building_327_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:28.656768 PDT 3: TAP (Producer) eq_tapq:replication_building_327_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:28.656773 PDT 3: TAP (Producer) eq_tapq:replication_building_327_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 327 memcached<0.1519.0>: Fri Nov 2 18:52:28.657491 PDT 3: Schedule cleanup of "eq_tapq:anon_71" memcached<0.1519.0>: Fri Nov 2 18:52:28.667444 PDT 3: TAP (Producer) eq_tapq:replication_building_326_'ns_1@10.2.1.66' - Clear the tap queues by force [ns_server:info,2012-11-02T18:52:29.960,ns_1@10.2.1.67:<0.18065.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.66",11209} vbucket 327 to state replica [rebalance:info,2012-11-02T18:52:29.963,ns_1@10.2.1.67:<0.18065.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[327]}, {checkpoints,[{327,10}]}, {name,<<"rebalance_327">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.66",11209}, [{username,"default"}, {password,[]}, {vbuckets,[327]}, {takeover,true}, {suffix,"327"}]} [rebalance:info,2012-11-02T18:52:29.964,ns_1@10.2.1.67:<0.18065.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:52:29.968,ns_1@10.2.1.67:<0.18065.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:52:30.082,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/327 [ns_server:info,2012-11-02T18:52:30.132,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:52:29.931897 PDT 3: TAP (Producer) eq_tapq:replication_building_327_'ns_1@10.2.1.66' - Backfill is completed with VBuckets 327, memcached<0.1519.0>: Fri Nov 2 18:52:29.931921 PDT 3: TAP (Producer) eq_tapq:replication_building_327_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 327 memcached<0.1519.0>: Fri Nov 2 18:52:29.955356 PDT 3: TAP (Producer) eq_tapq:replication_building_327_'ns_1@10.2.1.66' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:52:29.964023 PDT 3: TAP (Producer) eq_tapq:rebalance_327 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:29.964036 PDT 3: TAP (Producer) eq_tapq:rebalance_327 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:29.964042 PDT 3: TAP (Producer) eq_tapq:rebalance_327 - Sending TAP_VBUCKET_SET with vbucket 327 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:52:29.967996 PDT 3: TAP (Producer) eq_tapq:rebalance_327 - VBucket <327> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:52:29.968014 PDT 3: TAP (Producer) eq_tapq:rebalance_327 - Sending TAP_VBUCKET_SET with vbucket 327 and state "active" memcached<0.1519.0>: Fri Nov 2 18:52:29.968326 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:52:29.968350 PDT 3: TAP (Producer) eq_tapq:rebalance_327 - disconnected memcached<0.1519.0>: Fri Nov 2 18:52:29.968385 PDT 3: Schedule cleanup of "eq_tapq:rebalance_327" memcached<0.1519.0>: Fri Nov 2 18:52:29.968456 PDT 3: TAP (Producer) eq_tapq:rebalance_327 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:52:29.996527 PDT 3: TAP (Producer) eq_tapq:replication_building_327_'ns_1@10.2.1.66' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:52:30.007881 PDT 3: Deletion of vbucket 327 was completed. memcached<0.1519.0>: Fri Nov 2 18:52:30.028361 PDT 3: TAP (Producer) eq_tapq:replication_building_328_'ns_1@10.2.1.66' - Schedule the backfill for vbucket 328 memcached<0.1519.0>: Fri Nov 2 18:52:30.028386 PDT 3: TAP (Producer) eq_tapq:replication_building_328_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:30.028394 PDT 3: TAP (Producer) eq_tapq:replication_building_328_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:30.028399 PDT 3: TAP (Producer) eq_tapq:replication_building_328_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 328 memcached<0.1519.0>: Fri Nov 2 18:52:30.028440 PDT 3: Schedule cleanup of "eq_tapq:anon_72" memcached<0.1519.0>: Fri Nov 2 18:52:30.030830 PDT 3: TAP (Producer) eq_tapq:replication_building_327_'ns_1@10.2.1.66' - Clear the tap queues by force [ns_server:info,2012-11-02T18:52:31.205,ns_1@10.2.1.67:<0.18084.0>:compaction_daemon:try_to_cleanup_indexes:439]Cleaning up indexes for bucket `default` [ns_server:info,2012-11-02T18:52:31.205,ns_1@10.2.1.67:<0.18084.0>:compaction_daemon:spawn_bucket_compactor:404]Compacting bucket default with config: [{parallel_db_and_view_compaction,false}, {database_fragmentation_threshold,{30,undefined}}, {view_fragmentation_threshold,{100,undefined}}] [ns_server:info,2012-11-02T18:52:31.320,ns_1@10.2.1.67:<0.18091.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.66",11209} vbucket 328 to state replica [rebalance:info,2012-11-02T18:52:31.323,ns_1@10.2.1.67:<0.18091.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[328]}, {checkpoints,[{328,10}]}, {name,<<"rebalance_328">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.66",11209}, [{username,"default"}, {password,[]}, {vbuckets,[328]}, {takeover,true}, {suffix,"328"}]} [rebalance:info,2012-11-02T18:52:31.324,ns_1@10.2.1.67:<0.18091.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:52:31.329,ns_1@10.2.1.67:<0.18091.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:52:31.419,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/328 [ns_server:info,2012-11-02T18:52:31.473,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:52:31.272521 PDT 3: TAP (Producer) eq_tapq:replication_building_328_'ns_1@10.2.1.66' - Backfill is completed with VBuckets 328, memcached<0.1519.0>: Fri Nov 2 18:52:31.272544 PDT 3: TAP (Producer) eq_tapq:replication_building_328_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 328 memcached<0.1519.0>: Fri Nov 2 18:52:31.317455 PDT 3: TAP (Producer) eq_tapq:replication_building_328_'ns_1@10.2.1.66' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:52:31.323664 PDT 3: TAP (Producer) eq_tapq:rebalance_328 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:31.323676 PDT 3: TAP (Producer) eq_tapq:rebalance_328 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:31.323682 PDT 3: TAP (Producer) eq_tapq:rebalance_328 - Sending TAP_VBUCKET_SET with vbucket 328 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:52:31.329376 PDT 3: TAP (Producer) eq_tapq:rebalance_328 - VBucket <328> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:52:31.329396 PDT 3: TAP (Producer) eq_tapq:rebalance_328 - Sending TAP_VBUCKET_SET with vbucket 328 and state "active" memcached<0.1519.0>: Fri Nov 2 18:52:31.329594 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:52:31.329616 PDT 3: TAP (Producer) eq_tapq:rebalance_328 - disconnected memcached<0.1519.0>: Fri Nov 2 18:52:31.329647 PDT 3: Schedule cleanup of "eq_tapq:rebalance_328" memcached<0.1519.0>: Fri Nov 2 18:52:31.329693 PDT 3: TAP (Producer) eq_tapq:rebalance_328 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:52:31.341527 PDT 3: TAP (Producer) eq_tapq:replication_building_328_'ns_1@10.2.1.66' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:52:31.354150 PDT 3: Deletion of vbucket 328 was completed. memcached<0.1519.0>: Fri Nov 2 18:52:31.366455 PDT 3: TAP (Producer) eq_tapq:replication_building_329_'ns_1@10.2.1.66' - Schedule the backfill for vbucket 329 memcached<0.1519.0>: Fri Nov 2 18:52:31.366483 PDT 3: TAP (Producer) eq_tapq:replication_building_329_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:31.366491 PDT 3: TAP (Producer) eq_tapq:replication_building_329_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:31.366495 PDT 3: TAP (Producer) eq_tapq:replication_building_329_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 329 memcached<0.1519.0>: Fri Nov 2 18:52:31.366524 PDT 3: Schedule cleanup of "eq_tapq:anon_73" memcached<0.1519.0>: Fri Nov 2 18:52:31.379155 PDT 3: TAP (Producer) eq_tapq:replication_building_328_'ns_1@10.2.1.66' - Clear the tap queues by force [ns_server:info,2012-11-02T18:52:32.699,ns_1@10.2.1.67:<0.18102.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.66",11209} vbucket 329 to state replica [rebalance:info,2012-11-02T18:52:32.703,ns_1@10.2.1.67:<0.18102.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[329]}, {checkpoints,[{329,10}]}, {name,<<"rebalance_329">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.66",11209}, [{username,"default"}, {password,[]}, {vbuckets,[329]}, {takeover,true}, {suffix,"329"}]} [rebalance:info,2012-11-02T18:52:32.704,ns_1@10.2.1.67:<0.18102.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:52:32.713,ns_1@10.2.1.67:<0.18102.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:52:32.810,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/329 [ns_server:info,2012-11-02T18:52:32.861,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:52:32.660686 PDT 3: TAP (Producer) eq_tapq:replication_building_329_'ns_1@10.2.1.66' - Backfill is completed with VBuckets 329, memcached<0.1519.0>: Fri Nov 2 18:52:32.660709 PDT 3: TAP (Producer) eq_tapq:replication_building_329_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 329 memcached<0.1519.0>: Fri Nov 2 18:52:32.697478 PDT 3: TAP (Producer) eq_tapq:replication_building_329_'ns_1@10.2.1.66' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:52:32.703522 PDT 3: TAP (Producer) eq_tapq:rebalance_329 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:32.703535 PDT 3: TAP (Producer) eq_tapq:rebalance_329 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:32.703540 PDT 3: TAP (Producer) eq_tapq:rebalance_329 - Sending TAP_VBUCKET_SET with vbucket 329 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:52:32.712399 PDT 3: TAP (Producer) eq_tapq:rebalance_329 - VBucket <329> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:52:32.712434 PDT 3: TAP (Producer) eq_tapq:rebalance_329 - Sending TAP_VBUCKET_SET with vbucket 329 and state "active" memcached<0.1519.0>: Fri Nov 2 18:52:32.713253 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:52:32.713311 PDT 3: TAP (Producer) eq_tapq:rebalance_329 - disconnected memcached<0.1519.0>: Fri Nov 2 18:52:32.734331 PDT 3: TAP (Producer) eq_tapq:replication_building_329_'ns_1@10.2.1.66' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:52:32.743688 PDT 3: Deletion of vbucket 329 was completed. memcached<0.1519.0>: Fri Nov 2 18:52:32.763267 PDT 3: TAP (Producer) eq_tapq:replication_building_330_'ns_1@10.2.1.66' - Schedule the backfill for vbucket 330 memcached<0.1519.0>: Fri Nov 2 18:52:32.763295 PDT 3: TAP (Producer) eq_tapq:replication_building_330_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:32.763303 PDT 3: TAP (Producer) eq_tapq:replication_building_330_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:32.763308 PDT 3: TAP (Producer) eq_tapq:replication_building_330_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 330 memcached<0.1519.0>: Fri Nov 2 18:52:32.763407 PDT 3: Schedule cleanup of "eq_tapq:anon_74" memcached<0.1519.0>: Fri Nov 2 18:52:32.763435 PDT 3: Schedule cleanup of "eq_tapq:rebalance_329" memcached<0.1519.0>: Fri Nov 2 18:52:32.770784 PDT 3: TAP (Producer) eq_tapq:replication_building_329_'ns_1@10.2.1.66' - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:52:32.770837 PDT 3: TAP (Producer) eq_tapq:rebalance_329 - Clear the tap queues by force [ns_server:info,2012-11-02T18:52:34.166,ns_1@10.2.1.67:<0.18107.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.66",11209} vbucket 330 to state replica [rebalance:info,2012-11-02T18:52:34.174,ns_1@10.2.1.67:<0.18107.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[330]}, {checkpoints,[{330,10}]}, {name,<<"rebalance_330">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.66",11209}, [{username,"default"}, {password,[]}, {vbuckets,[330]}, {takeover,true}, {suffix,"330"}]} [rebalance:info,2012-11-02T18:52:34.175,ns_1@10.2.1.67:<0.18107.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:52:34.178,ns_1@10.2.1.67:<0.18107.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:52:34.276,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:52:34.075831 PDT 3: TAP (Producer) eq_tapq:replication_building_330_'ns_1@10.2.1.66' - Backfill is completed with VBuckets 330, memcached<0.1519.0>: Fri Nov 2 18:52:34.075858 PDT 3: TAP (Producer) eq_tapq:replication_building_330_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 330 memcached<0.1519.0>: Fri Nov 2 18:52:34.164085 PDT 3: TAP (Producer) eq_tapq:replication_building_330_'ns_1@10.2.1.66' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:52:34.174553 PDT 3: TAP (Producer) eq_tapq:rebalance_330 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:34.174567 PDT 3: TAP (Producer) eq_tapq:rebalance_330 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:34.174572 PDT 3: TAP (Producer) eq_tapq:rebalance_330 - Sending TAP_VBUCKET_SET with vbucket 330 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:52:34.178287 PDT 3: TAP (Producer) eq_tapq:rebalance_330 - VBucket <330> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:52:34.178321 PDT 3: TAP (Producer) eq_tapq:rebalance_330 - Sending TAP_VBUCKET_SET with vbucket 330 and state "active" memcached<0.1519.0>: Fri Nov 2 18:52:34.178743 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:52:34.178773 PDT 3: TAP (Producer) eq_tapq:rebalance_330 - disconnected memcached<0.1519.0>: Fri Nov 2 18:52:34.178883 PDT 3: Schedule cleanup of "eq_tapq:rebalance_330" memcached<0.1519.0>: Fri Nov 2 18:52:34.178913 PDT 3: TAP (Producer) eq_tapq:rebalance_330 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:52:34.202760 PDT 3: TAP (Producer) eq_tapq:replication_building_330_'ns_1@10.2.1.66' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:52:34.213787 PDT 3: Deletion of vbucket 330 was completed. memcached<0.1519.0>: Fri Nov 2 18:52:34.227160 PDT 3: TAP (Producer) eq_tapq:replication_building_331_'ns_1@10.2.1.66' - Schedule the backfill for vbucket 331 memcached<0.1519.0>: Fri Nov 2 18:52:34.227187 PDT 3: TAP (Producer) eq_tapq:replication_building_331_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:34.227195 PDT 3: TAP (Producer) eq_tapq:replication_building_331_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:34.227200 PDT 3: TAP (Producer) eq_tapq:replication_building_331_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 331 memcached<0.1519.0>: Fri Nov 2 18:52:34.228291 PDT 3: Schedule cleanup of "eq_tapq:anon_75" memcached<0.1519.0>: Fri Nov 2 18:52:34.242344 PDT 3: TAP (Producer) eq_tapq:replication_building_330_'ns_1@10.2.1.66' - Clear the tap queues by force [ns_server:info,2012-11-02T18:52:34.283,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/330 [ns_server:info,2012-11-02T18:52:35.574,ns_1@10.2.1.67:<0.18119.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.66",11209} vbucket 331 to state replica [rebalance:info,2012-11-02T18:52:35.579,ns_1@10.2.1.67:<0.18119.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[331]}, {checkpoints,[{331,10}]}, {name,<<"rebalance_331">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.66",11209}, [{username,"default"}, {password,[]}, {vbuckets,[331]}, {takeover,true}, {suffix,"331"}]} [rebalance:info,2012-11-02T18:52:35.580,ns_1@10.2.1.67:<0.18119.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:52:35.583,ns_1@10.2.1.67:<0.18119.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:52:35.685,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:52:35.484487 PDT 3: TAP (Producer) eq_tapq:replication_building_331_'ns_1@10.2.1.66' - Backfill is completed with VBuckets 331, memcached<0.1519.0>: Fri Nov 2 18:52:35.484508 PDT 3: TAP (Producer) eq_tapq:replication_building_331_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 331 memcached<0.1519.0>: Fri Nov 2 18:52:35.572066 PDT 3: TAP (Producer) eq_tapq:replication_building_331_'ns_1@10.2.1.66' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:52:35.579352 PDT 3: TAP (Producer) eq_tapq:rebalance_331 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:35.579365 PDT 3: TAP (Producer) eq_tapq:rebalance_331 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:35.579370 PDT 3: TAP (Producer) eq_tapq:rebalance_331 - Sending TAP_VBUCKET_SET with vbucket 331 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:52:35.583137 PDT 3: TAP (Producer) eq_tapq:rebalance_331 - VBucket <331> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:52:35.583154 PDT 3: TAP (Producer) eq_tapq:rebalance_331 - Sending TAP_VBUCKET_SET with vbucket 331 and state "active" memcached<0.1519.0>: Fri Nov 2 18:52:35.583529 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:52:35.583551 PDT 3: TAP (Producer) eq_tapq:rebalance_331 - disconnected memcached<0.1519.0>: Fri Nov 2 18:52:35.583582 PDT 3: Schedule cleanup of "eq_tapq:rebalance_331" memcached<0.1519.0>: Fri Nov 2 18:52:35.583607 PDT 3: TAP (Producer) eq_tapq:rebalance_331 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:52:35.611059 PDT 3: TAP (Producer) eq_tapq:replication_building_331_'ns_1@10.2.1.66' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:52:35.618381 PDT 3: Deletion of vbucket 331 was completed. memcached<0.1519.0>: Fri Nov 2 18:52:35.633496 PDT 3: TAP (Producer) eq_tapq:replication_building_332_'ns_1@10.2.1.66' - Schedule the backfill for vbucket 332 memcached<0.1519.0>: Fri Nov 2 18:52:35.633538 PDT 3: TAP (Producer) eq_tapq:replication_building_332_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:35.633553 PDT 3: TAP (Producer) eq_tapq:replication_building_332_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:35.633563 PDT 3: TAP (Producer) eq_tapq:replication_building_332_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 332 memcached<0.1519.0>: Fri Nov 2 18:52:35.633639 PDT 3: Schedule cleanup of "eq_tapq:anon_76" memcached<0.1519.0>: Fri Nov 2 18:52:35.642233 PDT 3: TAP (Producer) eq_tapq:replication_building_331_'ns_1@10.2.1.66' - Clear the tap queues by force [ns_server:info,2012-11-02T18:52:35.690,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/331 [ns_server:info,2012-11-02T18:52:37.034,ns_1@10.2.1.67:<0.18137.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.66",11209} vbucket 332 to state replica [rebalance:info,2012-11-02T18:52:37.038,ns_1@10.2.1.67:<0.18137.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[332]}, {checkpoints,[{332,10}]}, {name,<<"rebalance_332">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.66",11209}, [{username,"default"}, {password,[]}, {vbuckets,[332]}, {takeover,true}, {suffix,"332"}]} [rebalance:info,2012-11-02T18:52:37.039,ns_1@10.2.1.67:<0.18137.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:52:37.043,ns_1@10.2.1.67:<0.18137.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:52:37.146,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/332 [ns_server:info,2012-11-02T18:52:37.171,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:52:36.970966 PDT 3: TAP (Producer) eq_tapq:replication_building_332_'ns_1@10.2.1.66' - Backfill is completed with VBuckets 332, memcached<0.1519.0>: Fri Nov 2 18:52:36.970987 PDT 3: TAP (Producer) eq_tapq:replication_building_332_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 332 memcached<0.1519.0>: Fri Nov 2 18:52:37.030886 PDT 3: TAP (Producer) eq_tapq:replication_building_332_'ns_1@10.2.1.66' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:52:37.038651 PDT 3: TAP (Producer) eq_tapq:rebalance_332 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:37.038664 PDT 3: TAP (Producer) eq_tapq:rebalance_332 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:37.038669 PDT 3: TAP (Producer) eq_tapq:rebalance_332 - Sending TAP_VBUCKET_SET with vbucket 332 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:52:37.042638 PDT 3: TAP (Producer) eq_tapq:rebalance_332 - VBucket <332> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:52:37.042659 PDT 3: TAP (Producer) eq_tapq:rebalance_332 - Sending TAP_VBUCKET_SET with vbucket 332 and state "active" memcached<0.1519.0>: Fri Nov 2 18:52:37.042959 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:52:37.042982 PDT 3: TAP (Producer) eq_tapq:rebalance_332 - disconnected memcached<0.1519.0>: Fri Nov 2 18:52:37.043014 PDT 3: Schedule cleanup of "eq_tapq:rebalance_332" memcached<0.1519.0>: Fri Nov 2 18:52:37.043036 PDT 3: TAP (Producer) eq_tapq:rebalance_332 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:52:37.062910 PDT 3: TAP (Producer) eq_tapq:replication_building_332_'ns_1@10.2.1.66' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:52:37.079005 PDT 3: Deletion of vbucket 332 was completed. memcached<0.1519.0>: Fri Nov 2 18:52:37.105979 PDT 3: TAP (Producer) eq_tapq:replication_building_333_'ns_1@10.2.1.66' - Schedule the backfill for vbucket 333 memcached<0.1519.0>: Fri Nov 2 18:52:37.106004 PDT 3: TAP (Producer) eq_tapq:replication_building_333_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:37.106012 PDT 3: TAP (Producer) eq_tapq:replication_building_333_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:37.106016 PDT 3: TAP (Producer) eq_tapq:replication_building_333_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 333 memcached<0.1519.0>: Fri Nov 2 18:52:37.106148 PDT 3: Schedule cleanup of "eq_tapq:anon_77" memcached<0.1519.0>: Fri Nov 2 18:52:37.106222 PDT 3: TAP (Producer) eq_tapq:replication_building_332_'ns_1@10.2.1.66' - Clear the tap queues by force [ns_server:info,2012-11-02T18:52:38.342,ns_1@10.2.1.67:<0.18143.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.66",11209} vbucket 333 to state replica [rebalance:info,2012-11-02T18:52:38.344,ns_1@10.2.1.67:<0.18143.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[333]}, {checkpoints,[{333,10}]}, {name,<<"rebalance_333">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.66",11209}, [{username,"default"}, {password,[]}, {vbuckets,[333]}, {takeover,true}, {suffix,"333"}]} [rebalance:info,2012-11-02T18:52:38.346,ns_1@10.2.1.67:<0.18143.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:52:38.349,ns_1@10.2.1.67:<0.18143.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:52:38.466,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/333 [ns_server:info,2012-11-02T18:52:38.490,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:52:38.289888 PDT 3: TAP (Producer) eq_tapq:replication_building_333_'ns_1@10.2.1.66' - Backfill is completed with VBuckets 333, memcached<0.1519.0>: Fri Nov 2 18:52:38.289911 PDT 3: TAP (Producer) eq_tapq:replication_building_333_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 333 memcached<0.1519.0>: Fri Nov 2 18:52:38.337177 PDT 3: TAP (Producer) eq_tapq:replication_building_333_'ns_1@10.2.1.66' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:52:38.344997 PDT 3: TAP (Producer) eq_tapq:rebalance_333 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:38.345009 PDT 3: TAP (Producer) eq_tapq:rebalance_333 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:38.345014 PDT 3: TAP (Producer) eq_tapq:rebalance_333 - Sending TAP_VBUCKET_SET with vbucket 333 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:52:38.348967 PDT 3: TAP (Producer) eq_tapq:rebalance_333 - VBucket <333> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:52:38.348982 PDT 3: TAP (Producer) eq_tapq:rebalance_333 - Sending TAP_VBUCKET_SET with vbucket 333 and state "active" memcached<0.1519.0>: Fri Nov 2 18:52:38.349378 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:52:38.349401 PDT 3: TAP (Producer) eq_tapq:rebalance_333 - disconnected memcached<0.1519.0>: Fri Nov 2 18:52:38.349433 PDT 3: Schedule cleanup of "eq_tapq:rebalance_333" memcached<0.1519.0>: Fri Nov 2 18:52:38.349456 PDT 3: TAP (Producer) eq_tapq:rebalance_333 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:52:38.372422 PDT 3: TAP (Producer) eq_tapq:replication_building_333_'ns_1@10.2.1.66' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:52:38.388859 PDT 3: Deletion of vbucket 333 was completed. memcached<0.1519.0>: Fri Nov 2 18:52:38.401011 PDT 3: TAP (Producer) eq_tapq:replication_building_334_'ns_1@10.2.1.66' - Schedule the backfill for vbucket 334 memcached<0.1519.0>: Fri Nov 2 18:52:38.401039 PDT 3: TAP (Producer) eq_tapq:replication_building_334_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:38.401048 PDT 3: TAP (Producer) eq_tapq:replication_building_334_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:38.401054 PDT 3: TAP (Producer) eq_tapq:replication_building_334_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 334 memcached<0.1519.0>: Fri Nov 2 18:52:38.401127 PDT 3: Schedule cleanup of "eq_tapq:anon_78" memcached<0.1519.0>: Fri Nov 2 18:52:38.413841 PDT 3: TAP (Producer) eq_tapq:replication_building_333_'ns_1@10.2.1.66' - Clear the tap queues by force [ns_server:info,2012-11-02T18:52:39.705,ns_1@10.2.1.67:<0.18153.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.66",11209} vbucket 334 to state replica [rebalance:info,2012-11-02T18:52:39.707,ns_1@10.2.1.67:<0.18153.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[334]}, {checkpoints,[{334,10}]}, {name,<<"rebalance_334">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.66",11209}, [{username,"default"}, {password,[]}, {vbuckets,[334]}, {takeover,true}, {suffix,"334"}]} [rebalance:info,2012-11-02T18:52:39.709,ns_1@10.2.1.67:<0.18153.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:52:39.712,ns_1@10.2.1.67:<0.18153.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:52:39.798,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/334 [ns_server:info,2012-11-02T18:52:39.842,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:52:39.642676 PDT 3: TAP (Producer) eq_tapq:replication_building_334_'ns_1@10.2.1.66' - Backfill is completed with VBuckets 334, memcached<0.1519.0>: Fri Nov 2 18:52:39.642706 PDT 3: TAP (Producer) eq_tapq:replication_building_334_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 334 memcached<0.1519.0>: Fri Nov 2 18:52:39.702732 PDT 3: TAP (Producer) eq_tapq:replication_building_334_'ns_1@10.2.1.66' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:52:39.708202 PDT 3: TAP (Producer) eq_tapq:rebalance_334 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:39.708214 PDT 3: TAP (Producer) eq_tapq:rebalance_334 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:39.708219 PDT 3: TAP (Producer) eq_tapq:rebalance_334 - Sending TAP_VBUCKET_SET with vbucket 334 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:52:39.712473 PDT 3: TAP (Producer) eq_tapq:rebalance_334 - VBucket <334> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:52:39.712489 PDT 3: TAP (Producer) eq_tapq:rebalance_334 - Sending TAP_VBUCKET_SET with vbucket 334 and state "active" memcached<0.1519.0>: Fri Nov 2 18:52:39.712752 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:52:39.712773 PDT 3: TAP (Producer) eq_tapq:rebalance_334 - disconnected memcached<0.1519.0>: Fri Nov 2 18:52:39.712805 PDT 3: Schedule cleanup of "eq_tapq:rebalance_334" memcached<0.1519.0>: Fri Nov 2 18:52:39.712886 PDT 3: TAP (Producer) eq_tapq:rebalance_334 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:52:39.726474 PDT 3: TAP (Producer) eq_tapq:replication_building_334_'ns_1@10.2.1.66' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:52:39.734982 PDT 3: Deletion of vbucket 334 was completed. memcached<0.1519.0>: Fri Nov 2 18:52:39.757499 PDT 3: TAP (Producer) eq_tapq:replication_building_335_'ns_1@10.2.1.66' - Schedule the backfill for vbucket 335 memcached<0.1519.0>: Fri Nov 2 18:52:39.757541 PDT 3: TAP (Producer) eq_tapq:replication_building_335_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:39.757556 PDT 3: TAP (Producer) eq_tapq:replication_building_335_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:39.757566 PDT 3: TAP (Producer) eq_tapq:replication_building_335_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 335 memcached<0.1519.0>: Fri Nov 2 18:52:39.757895 PDT 3: Schedule cleanup of "eq_tapq:anon_79" memcached<0.1519.0>: Fri Nov 2 18:52:39.761517 PDT 3: TAP (Producer) eq_tapq:replication_building_334_'ns_1@10.2.1.66' - Clear the tap queues by force [ns_server:info,2012-11-02T18:52:40.986,ns_1@10.2.1.67:<0.18173.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.66",11209} vbucket 335 to state replica [rebalance:info,2012-11-02T18:52:40.989,ns_1@10.2.1.67:<0.18173.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[335]}, {checkpoints,[{335,10}]}, {name,<<"rebalance_335">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.66",11209}, [{username,"default"}, {password,[]}, {vbuckets,[335]}, {takeover,true}, {suffix,"335"}]} [rebalance:info,2012-11-02T18:52:40.990,ns_1@10.2.1.67:<0.18173.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:52:40.993,ns_1@10.2.1.67:<0.18173.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:52:41.071,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:52:40.870271 PDT 3: TAP (Producer) eq_tapq:replication_building_335_'ns_1@10.2.1.66' - Backfill is completed with VBuckets 335, memcached<0.1519.0>: Fri Nov 2 18:52:40.870288 PDT 3: TAP (Producer) eq_tapq:replication_building_335_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 335 memcached<0.1519.0>: Fri Nov 2 18:52:40.983365 PDT 3: TAP (Producer) eq_tapq:replication_building_335_'ns_1@10.2.1.66' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:52:40.989349 PDT 3: TAP (Producer) eq_tapq:rebalance_335 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:40.989361 PDT 3: TAP (Producer) eq_tapq:rebalance_335 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:40.989366 PDT 3: TAP (Producer) eq_tapq:rebalance_335 - Sending TAP_VBUCKET_SET with vbucket 335 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:52:40.992513 PDT 3: TAP (Producer) eq_tapq:rebalance_335 - VBucket <335> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:52:40.992532 PDT 3: TAP (Producer) eq_tapq:rebalance_335 - Sending TAP_VBUCKET_SET with vbucket 335 and state "active" memcached<0.1519.0>: Fri Nov 2 18:52:40.993049 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:52:40.993073 PDT 3: TAP (Producer) eq_tapq:rebalance_335 - disconnected memcached<0.1519.0>: Fri Nov 2 18:52:40.993105 PDT 3: Schedule cleanup of "eq_tapq:rebalance_335" memcached<0.1519.0>: Fri Nov 2 18:52:40.993156 PDT 3: TAP (Producer) eq_tapq:rebalance_335 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:52:40.998790 PDT 3: TAP (Producer) eq_tapq:replication_building_335_'ns_1@10.2.1.66' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:52:41.009192 PDT 3: Deletion of vbucket 335 was completed. memcached<0.1519.0>: Fri Nov 2 18:52:41.025745 PDT 3: TAP (Producer) eq_tapq:replication_building_336_'ns_1@10.2.1.66' - Schedule the backfill for vbucket 336 memcached<0.1519.0>: Fri Nov 2 18:52:41.025787 PDT 3: TAP (Producer) eq_tapq:replication_building_336_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:41.025800 PDT 3: TAP (Producer) eq_tapq:replication_building_336_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:41.025810 PDT 3: TAP (Producer) eq_tapq:replication_building_336_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 336 memcached<0.1519.0>: Fri Nov 2 18:52:41.025991 PDT 3: Schedule cleanup of "eq_tapq:anon_80" memcached<0.1519.0>: Fri Nov 2 18:52:41.032656 PDT 3: TAP (Producer) eq_tapq:replication_building_335_'ns_1@10.2.1.66' - Clear the tap queues by force [ns_server:info,2012-11-02T18:52:41.075,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/335 [ns_server:info,2012-11-02T18:52:42.086,ns_1@10.2.1.67:<0.18178.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.66",11209} vbucket 336 to state replica [rebalance:info,2012-11-02T18:52:42.088,ns_1@10.2.1.67:<0.18178.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[336]}, {checkpoints,[{336,10}]}, {name,<<"rebalance_336">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.66",11209}, [{username,"default"}, {password,[]}, {vbuckets,[336]}, {takeover,true}, {suffix,"336"}]} [rebalance:info,2012-11-02T18:52:42.090,ns_1@10.2.1.67:<0.18178.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:52:42.093,ns_1@10.2.1.67:<0.18178.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:52:42.196,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/336 [ns_server:info,2012-11-02T18:52:42.231,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:52:42.030802 PDT 3: TAP (Producer) eq_tapq:replication_building_336_'ns_1@10.2.1.66' - Backfill is completed with VBuckets 336, memcached<0.1519.0>: Fri Nov 2 18:52:42.030838 PDT 3: TAP (Producer) eq_tapq:replication_building_336_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 336 memcached<0.1519.0>: Fri Nov 2 18:52:42.080635 PDT 3: TAP (Producer) eq_tapq:replication_building_336_'ns_1@10.2.1.66' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:52:42.089125 PDT 3: TAP (Producer) eq_tapq:rebalance_336 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:42.089137 PDT 3: TAP (Producer) eq_tapq:rebalance_336 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:42.089143 PDT 3: TAP (Producer) eq_tapq:rebalance_336 - Sending TAP_VBUCKET_SET with vbucket 336 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:52:42.092631 PDT 3: TAP (Producer) eq_tapq:rebalance_336 - VBucket <336> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:52:42.092649 PDT 3: TAP (Producer) eq_tapq:rebalance_336 - Sending TAP_VBUCKET_SET with vbucket 336 and state "active" memcached<0.1519.0>: Fri Nov 2 18:52:42.093174 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:52:42.093196 PDT 3: TAP (Producer) eq_tapq:rebalance_336 - disconnected memcached<0.1519.0>: Fri Nov 2 18:52:42.093226 PDT 3: Schedule cleanup of "eq_tapq:rebalance_336" memcached<0.1519.0>: Fri Nov 2 18:52:42.093645 PDT 3: TAP (Producer) eq_tapq:rebalance_336 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:52:42.117200 PDT 3: TAP (Producer) eq_tapq:replication_building_336_'ns_1@10.2.1.66' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:52:42.141597 PDT 3: Deletion of vbucket 336 was completed. memcached<0.1519.0>: Fri Nov 2 18:52:42.152954 PDT 3: TAP (Producer) eq_tapq:replication_building_337_'ns_1@10.2.1.66' - Schedule the backfill for vbucket 337 memcached<0.1519.0>: Fri Nov 2 18:52:42.152977 PDT 3: TAP (Producer) eq_tapq:replication_building_337_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:42.152986 PDT 3: TAP (Producer) eq_tapq:replication_building_337_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:42.152990 PDT 3: TAP (Producer) eq_tapq:replication_building_337_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 337 memcached<0.1519.0>: Fri Nov 2 18:52:42.153041 PDT 3: Schedule cleanup of "eq_tapq:anon_81" memcached<0.1519.0>: Fri Nov 2 18:52:42.167196 PDT 3: TAP (Producer) eq_tapq:replication_building_336_'ns_1@10.2.1.66' - Clear the tap queues by force [ns_server:info,2012-11-02T18:52:43.444,ns_1@10.2.1.67:<0.18188.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.66",11209} vbucket 337 to state replica [rebalance:info,2012-11-02T18:52:43.446,ns_1@10.2.1.67:<0.18188.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[337]}, {checkpoints,[{337,10}]}, {name,<<"rebalance_337">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.66",11209}, [{username,"default"}, {password,[]}, {vbuckets,[337]}, {takeover,true}, {suffix,"337"}]} [rebalance:info,2012-11-02T18:52:43.447,ns_1@10.2.1.67:<0.18188.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:52:43.454,ns_1@10.2.1.67:<0.18188.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:52:43.538,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/337 [ns_server:info,2012-11-02T18:52:43.557,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:52:43.357586 PDT 3: TAP (Producer) eq_tapq:replication_building_337_'ns_1@10.2.1.66' - Backfill is completed with VBuckets 337, memcached<0.1519.0>: Fri Nov 2 18:52:43.357605 PDT 3: TAP (Producer) eq_tapq:replication_building_337_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 337 memcached<0.1519.0>: Fri Nov 2 18:52:43.440566 PDT 3: TAP (Producer) eq_tapq:replication_building_337_'ns_1@10.2.1.66' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:52:43.446856 PDT 3: TAP (Producer) eq_tapq:rebalance_337 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:43.446869 PDT 3: TAP (Producer) eq_tapq:rebalance_337 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:43.446874 PDT 3: TAP (Producer) eq_tapq:rebalance_337 - Sending TAP_VBUCKET_SET with vbucket 337 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:52:43.453755 PDT 3: TAP (Producer) eq_tapq:rebalance_337 - VBucket <337> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:52:43.453790 PDT 3: TAP (Producer) eq_tapq:rebalance_337 - Sending TAP_VBUCKET_SET with vbucket 337 and state "active" memcached<0.1519.0>: Fri Nov 2 18:52:43.454522 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:52:43.454555 PDT 3: TAP (Producer) eq_tapq:rebalance_337 - disconnected memcached<0.1519.0>: Fri Nov 2 18:52:43.454616 PDT 3: Schedule cleanup of "eq_tapq:rebalance_337" memcached<0.1519.0>: Fri Nov 2 18:52:43.455276 PDT 3: TAP (Producer) eq_tapq:rebalance_337 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:52:43.460288 PDT 3: TAP (Producer) eq_tapq:replication_building_337_'ns_1@10.2.1.66' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:52:43.472687 PDT 3: Deletion of vbucket 337 was completed. memcached<0.1519.0>: Fri Nov 2 18:52:43.497594 PDT 3: TAP (Producer) eq_tapq:replication_building_338_'ns_1@10.2.1.66' - Schedule the backfill for vbucket 338 memcached<0.1519.0>: Fri Nov 2 18:52:43.497625 PDT 3: TAP (Producer) eq_tapq:replication_building_338_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:43.497635 PDT 3: TAP (Producer) eq_tapq:replication_building_338_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:43.497642 PDT 3: TAP (Producer) eq_tapq:replication_building_338_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 338 memcached<0.1519.0>: Fri Nov 2 18:52:43.497708 PDT 3: Schedule cleanup of "eq_tapq:anon_82" memcached<0.1519.0>: Fri Nov 2 18:52:43.497827 PDT 3: TAP (Producer) eq_tapq:replication_building_337_'ns_1@10.2.1.66' - Clear the tap queues by force [ns_server:info,2012-11-02T18:52:43.803,ns_1@10.2.1.67:ns_config_rep<0.1439.0>:ns_config_rep:do_pull:341]Pulling config from: 'ns_1@10.2.1.68' [ns_server:info,2012-11-02T18:52:44.295,ns_1@10.2.1.67:<0.18196.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.66",11209} vbucket 338 to state replica [rebalance:info,2012-11-02T18:52:44.302,ns_1@10.2.1.67:<0.18196.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[338]}, {checkpoints,[{338,10}]}, {name,<<"rebalance_338">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.66",11209}, [{username,"default"}, {password,[]}, {vbuckets,[338]}, {takeover,true}, {suffix,"338"}]} [rebalance:info,2012-11-02T18:52:44.303,ns_1@10.2.1.67:<0.18196.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:52:44.306,ns_1@10.2.1.67:<0.18196.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:52:44.357,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/338 [ns_server:info,2012-11-02T18:52:44.387,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:52:44.185594 PDT 3: TAP (Producer) eq_tapq:replication_building_338_'ns_1@10.2.1.66' - Backfill is completed with VBuckets 338, memcached<0.1519.0>: Fri Nov 2 18:52:44.185614 PDT 3: TAP (Producer) eq_tapq:replication_building_338_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 338 memcached<0.1519.0>: Fri Nov 2 18:52:44.291846 PDT 3: TAP (Producer) eq_tapq:replication_building_338_'ns_1@10.2.1.66' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:52:44.302858 PDT 3: TAP (Producer) eq_tapq:rebalance_338 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:44.302872 PDT 3: TAP (Producer) eq_tapq:rebalance_338 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:44.302877 PDT 3: TAP (Producer) eq_tapq:rebalance_338 - Sending TAP_VBUCKET_SET with vbucket 338 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:52:44.306021 PDT 3: TAP (Producer) eq_tapq:rebalance_338 - VBucket <338> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:52:44.306038 PDT 3: TAP (Producer) eq_tapq:rebalance_338 - Sending TAP_VBUCKET_SET with vbucket 338 and state "active" memcached<0.1519.0>: Fri Nov 2 18:52:44.306363 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:52:44.306395 PDT 3: TAP (Producer) eq_tapq:rebalance_338 - disconnected memcached<0.1519.0>: Fri Nov 2 18:52:44.313033 PDT 3: TAP (Producer) eq_tapq:replication_building_338_'ns_1@10.2.1.66' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:52:44.321062 PDT 3: Deletion of vbucket 338 was completed. memcached<0.1519.0>: Fri Nov 2 18:52:44.337279 PDT 3: TAP (Producer) eq_tapq:replication_building_339_'ns_1@10.2.1.66' - Schedule the backfill for vbucket 339 memcached<0.1519.0>: Fri Nov 2 18:52:44.337320 PDT 3: TAP (Producer) eq_tapq:replication_building_339_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:44.337335 PDT 3: TAP (Producer) eq_tapq:replication_building_339_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:44.337345 PDT 3: TAP (Producer) eq_tapq:replication_building_339_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 339 memcached<0.1519.0>: Fri Nov 2 18:52:44.337449 PDT 3: Schedule cleanup of "eq_tapq:anon_83" memcached<0.1519.0>: Fri Nov 2 18:52:44.337468 PDT 3: Schedule cleanup of "eq_tapq:rebalance_338" memcached<0.1519.0>: Fri Nov 2 18:52:44.347083 PDT 3: TAP (Producer) eq_tapq:replication_building_338_'ns_1@10.2.1.66' - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:52:44.347125 PDT 3: TAP (Producer) eq_tapq:rebalance_338 - Clear the tap queues by force [ns_server:info,2012-11-02T18:52:45.926,ns_1@10.2.1.67:<0.18215.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.66",11209} vbucket 339 to state replica [rebalance:info,2012-11-02T18:52:45.929,ns_1@10.2.1.67:<0.18215.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[339]}, {checkpoints,[{339,10}]}, {name,<<"rebalance_339">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.66",11209}, [{username,"default"}, {password,[]}, {vbuckets,[339]}, {takeover,true}, {suffix,"339"}]} [rebalance:info,2012-11-02T18:52:45.936,ns_1@10.2.1.67:<0.18215.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:52:45.941,ns_1@10.2.1.67:<0.18215.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:52:46.049,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:52:45.835052 PDT 3: TAP (Producer) eq_tapq:replication_building_339_'ns_1@10.2.1.66' - Backfill is completed with VBuckets 339, memcached<0.1519.0>: Fri Nov 2 18:52:45.835073 PDT 3: TAP (Producer) eq_tapq:replication_building_339_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 339 memcached<0.1519.0>: Fri Nov 2 18:52:45.923644 PDT 3: TAP (Producer) eq_tapq:replication_building_339_'ns_1@10.2.1.66' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:52:45.929524 PDT 3: TAP (Producer) eq_tapq:rebalance_339 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:45.929536 PDT 3: TAP (Producer) eq_tapq:rebalance_339 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:45.929541 PDT 3: TAP (Producer) eq_tapq:rebalance_339 - Sending TAP_VBUCKET_SET with vbucket 339 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:52:45.940358 PDT 3: TAP (Producer) eq_tapq:rebalance_339 - VBucket <339> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:52:45.940396 PDT 3: TAP (Producer) eq_tapq:rebalance_339 - Sending TAP_VBUCKET_SET with vbucket 339 and state "active" memcached<0.1519.0>: Fri Nov 2 18:52:45.941317 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:52:45.941367 PDT 3: TAP (Producer) eq_tapq:rebalance_339 - disconnected memcached<0.1519.0>: Fri Nov 2 18:52:45.964425 PDT 3: TAP (Producer) eq_tapq:replication_building_339_'ns_1@10.2.1.66' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:52:45.972853 PDT 3: Deletion of vbucket 339 was completed. memcached<0.1519.0>: Fri Nov 2 18:52:45.986693 PDT 3: TAP (Producer) eq_tapq:replication_building_340_'ns_1@10.2.1.66' - Schedule the backfill for vbucket 340 memcached<0.1519.0>: Fri Nov 2 18:52:45.986734 PDT 3: TAP (Producer) eq_tapq:replication_building_340_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:45.986748 PDT 3: TAP (Producer) eq_tapq:replication_building_340_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:45.986757 PDT 3: TAP (Producer) eq_tapq:replication_building_340_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 340 memcached<0.1519.0>: Fri Nov 2 18:52:45.986963 PDT 3: Schedule cleanup of "eq_tapq:anon_84" memcached<0.1519.0>: Fri Nov 2 18:52:45.986978 PDT 3: Schedule cleanup of "eq_tapq:rebalance_339" memcached<0.1519.0>: Fri Nov 2 18:52:45.993155 PDT 3: TAP (Producer) eq_tapq:replication_building_339_'ns_1@10.2.1.66' - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:52:45.993180 PDT 3: TAP (Producer) eq_tapq:rebalance_339 - Clear the tap queues by force [ns_server:info,2012-11-02T18:52:46.052,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/339 [ns_server:info,2012-11-02T18:52:47.296,ns_1@10.2.1.67:<0.18226.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.66",11209} vbucket 340 to state replica [rebalance:info,2012-11-02T18:52:47.298,ns_1@10.2.1.67:<0.18226.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[340]}, {checkpoints,[{340,10}]}, {name,<<"rebalance_340">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.66",11209}, [{username,"default"}, {password,[]}, {vbuckets,[340]}, {takeover,true}, {suffix,"340"}]} [rebalance:info,2012-11-02T18:52:47.304,ns_1@10.2.1.67:<0.18226.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:52:47.309,ns_1@10.2.1.67:<0.18226.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:52:47.386,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/340 [ns_server:info,2012-11-02T18:52:47.402,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:52:47.201853 PDT 3: TAP (Producer) eq_tapq:replication_building_340_'ns_1@10.2.1.66' - Backfill is completed with VBuckets 340, memcached<0.1519.0>: Fri Nov 2 18:52:47.201873 PDT 3: TAP (Producer) eq_tapq:replication_building_340_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 340 memcached<0.1519.0>: Fri Nov 2 18:52:47.292434 PDT 3: TAP (Producer) eq_tapq:replication_building_340_'ns_1@10.2.1.66' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:52:47.299181 PDT 3: TAP (Producer) eq_tapq:rebalance_340 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:47.299193 PDT 3: TAP (Producer) eq_tapq:rebalance_340 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:47.299198 PDT 3: TAP (Producer) eq_tapq:rebalance_340 - Sending TAP_VBUCKET_SET with vbucket 340 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:52:47.307675 PDT 3: TAP (Producer) eq_tapq:rebalance_340 - VBucket <340> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:52:47.307713 PDT 3: TAP (Producer) eq_tapq:rebalance_340 - Sending TAP_VBUCKET_SET with vbucket 340 and state "active" memcached<0.1519.0>: Fri Nov 2 18:52:47.308798 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:52:47.308846 PDT 3: TAP (Producer) eq_tapq:rebalance_340 - disconnected memcached<0.1519.0>: Fri Nov 2 18:52:47.317335 PDT 3: TAP (Producer) eq_tapq:replication_building_340_'ns_1@10.2.1.66' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:52:47.326259 PDT 3: Deletion of vbucket 340 was completed. memcached<0.1519.0>: Fri Nov 2 18:52:47.333792 PDT 3: TAP (Producer) eq_tapq:replication_building_341_'ns_1@10.2.1.66' - Schedule the backfill for vbucket 341 memcached<0.1519.0>: Fri Nov 2 18:52:47.333817 PDT 3: TAP (Producer) eq_tapq:replication_building_341_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:47.333825 PDT 3: TAP (Producer) eq_tapq:replication_building_341_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:47.333830 PDT 3: TAP (Producer) eq_tapq:replication_building_341_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 341 memcached<0.1519.0>: Fri Nov 2 18:52:47.334257 PDT 3: Schedule cleanup of "eq_tapq:anon_85" memcached<0.1519.0>: Fri Nov 2 18:52:47.334271 PDT 3: Schedule cleanup of "eq_tapq:rebalance_340" memcached<0.1519.0>: Fri Nov 2 18:52:47.347940 PDT 3: TAP (Producer) eq_tapq:replication_building_340_'ns_1@10.2.1.66' - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:52:47.347975 PDT 3: TAP (Producer) eq_tapq:rebalance_340 - Clear the tap queues by force [ns_server:info,2012-11-02T18:52:48.196,ns_1@10.2.1.67:<0.18231.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.66",11209} vbucket 341 to state replica [rebalance:info,2012-11-02T18:52:48.208,ns_1@10.2.1.67:<0.18231.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[341]}, {checkpoints,[{341,10}]}, {name,<<"rebalance_341">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.66",11209}, [{username,"default"}, {password,[]}, {vbuckets,[341]}, {takeover,true}, {suffix,"341"}]} [rebalance:info,2012-11-02T18:52:48.210,ns_1@10.2.1.67:<0.18231.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:52:48.213,ns_1@10.2.1.67:<0.18231.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:52:48.275,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/341 [ns_server:info,2012-11-02T18:52:48.292,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:52:48.092342 PDT 3: TAP (Producer) eq_tapq:replication_building_341_'ns_1@10.2.1.66' - Backfill is completed with VBuckets 341, memcached<0.1519.0>: Fri Nov 2 18:52:48.092363 PDT 3: TAP (Producer) eq_tapq:replication_building_341_'ns_1@10.2.1.66' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 341 memcached<0.1519.0>: Fri Nov 2 18:52:48.192001 PDT 3: TAP (Producer) eq_tapq:replication_building_341_'ns_1@10.2.1.66' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:52:48.208979 PDT 3: TAP (Producer) eq_tapq:rebalance_341 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:48.209010 PDT 3: TAP (Producer) eq_tapq:rebalance_341 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:48.209021 PDT 3: TAP (Producer) eq_tapq:rebalance_341 - Sending TAP_VBUCKET_SET with vbucket 341 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:52:48.212746 PDT 3: TAP (Producer) eq_tapq:rebalance_341 - VBucket <341> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:52:48.212762 PDT 3: TAP (Producer) eq_tapq:rebalance_341 - Sending TAP_VBUCKET_SET with vbucket 341 and state "active" memcached<0.1519.0>: Fri Nov 2 18:52:48.212932 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:52:48.212956 PDT 3: TAP (Producer) eq_tapq:rebalance_341 - disconnected memcached<0.1519.0>: Fri Nov 2 18:52:48.227884 PDT 3: TAP (Producer) eq_tapq:replication_building_341_'ns_1@10.2.1.66' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:52:48.239159 PDT 3: Deletion of vbucket 341 was completed. memcached<0.1519.0>: Fri Nov 2 18:52:48.253575 PDT 3: TAP (Producer) eq_tapq:replication_building_342_'ns_1@10.2.1.68' - Schedule the backfill for vbucket 342 memcached<0.1519.0>: Fri Nov 2 18:52:48.253617 PDT 3: TAP (Producer) eq_tapq:replication_building_342_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:48.253632 PDT 3: TAP (Producer) eq_tapq:replication_building_342_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:48.253642 PDT 3: TAP (Producer) eq_tapq:replication_building_342_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 342 memcached<0.1519.0>: Fri Nov 2 18:52:48.253793 PDT 3: Schedule cleanup of "eq_tapq:anon_86" memcached<0.1519.0>: Fri Nov 2 18:52:48.253814 PDT 3: Schedule cleanup of "eq_tapq:rebalance_341" memcached<0.1519.0>: Fri Nov 2 18:52:48.265495 PDT 3: TAP (Producer) eq_tapq:replication_building_341_'ns_1@10.2.1.66' - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:52:48.265540 PDT 3: TAP (Producer) eq_tapq:rebalance_341 - Clear the tap queues by force [ns_server:info,2012-11-02T18:52:49.002,ns_1@10.2.1.67:<0.18241.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.68",11209} vbucket 342 to state replica [rebalance:info,2012-11-02T18:52:49.012,ns_1@10.2.1.67:<0.18241.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[342]}, {checkpoints,[{342,10}]}, {name,<<"rebalance_342">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.68",11209}, [{username,"default"}, {password,[]}, {vbuckets,[342]}, {takeover,true}, {suffix,"342"}]} [rebalance:info,2012-11-02T18:52:49.013,ns_1@10.2.1.67:<0.18241.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:52:49.017,ns_1@10.2.1.67:<0.18241.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:52:49.394,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/342 [ns_server:info,2012-11-02T18:52:49.395,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:52:48.950501 PDT 3: TAP (Producer) eq_tapq:replication_building_342_'ns_1@10.2.1.68' - Backfill is completed with VBuckets 342, memcached<0.1519.0>: Fri Nov 2 18:52:48.950523 PDT 3: TAP (Producer) eq_tapq:replication_building_342_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 342 memcached<0.1519.0>: Fri Nov 2 18:52:48.998565 PDT 3: TAP (Producer) eq_tapq:replication_building_342_'ns_1@10.2.1.68' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:52:49.012556 PDT 3: TAP (Producer) eq_tapq:rebalance_342 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:49.012569 PDT 3: TAP (Producer) eq_tapq:rebalance_342 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:49.012574 PDT 3: TAP (Producer) eq_tapq:rebalance_342 - Sending TAP_VBUCKET_SET with vbucket 342 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:52:49.016761 PDT 3: TAP (Producer) eq_tapq:rebalance_342 - VBucket <342> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:52:49.016779 PDT 3: TAP (Producer) eq_tapq:rebalance_342 - Sending TAP_VBUCKET_SET with vbucket 342 and state "active" memcached<0.1519.0>: Fri Nov 2 18:52:49.017004 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:52:49.017037 PDT 3: TAP (Producer) eq_tapq:rebalance_342 - disconnected memcached<0.1519.0>: Fri Nov 2 18:52:49.031424 PDT 3: TAP (Producer) eq_tapq:replication_building_342_'ns_1@10.2.1.68' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:52:49.041542 PDT 3: Deletion of vbucket 342 was completed. [ns_server:info,2012-11-02T18:52:49.598,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:52:49.055246 PDT 3: TAP (Producer) eq_tapq:replication_building_343_'ns_1@10.2.1.68' - Schedule the backfill for vbucket 343 memcached<0.1519.0>: Fri Nov 2 18:52:49.055286 PDT 3: TAP (Producer) eq_tapq:replication_building_343_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:49.055300 PDT 3: TAP (Producer) eq_tapq:replication_building_343_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:49.055310 PDT 3: TAP (Producer) eq_tapq:replication_building_343_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 343 memcached<0.1519.0>: Fri Nov 2 18:52:49.055415 PDT 3: Schedule cleanup of "eq_tapq:anon_87" memcached<0.1519.0>: Fri Nov 2 18:52:49.055442 PDT 3: Schedule cleanup of "eq_tapq:rebalance_342" memcached<0.1519.0>: Fri Nov 2 18:52:49.068469 PDT 3: TAP (Producer) eq_tapq:replication_building_342_'ns_1@10.2.1.68' - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:52:49.068523 PDT 3: TAP (Producer) eq_tapq:rebalance_342 - Clear the tap queues by force [ns_server:info,2012-11-02T18:52:49.842,ns_1@10.2.1.67:<0.18247.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.68",11209} vbucket 343 to state replica [rebalance:info,2012-11-02T18:52:49.850,ns_1@10.2.1.67:<0.18247.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[343]}, {checkpoints,[{343,10}]}, {name,<<"rebalance_343">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.68",11209}, [{username,"default"}, {password,[]}, {vbuckets,[343]}, {takeover,true}, {suffix,"343"}]} [rebalance:info,2012-11-02T18:52:49.851,ns_1@10.2.1.67:<0.18247.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:52:49.854,ns_1@10.2.1.67:<0.18247.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:52:49.911,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/343 [ns_server:info,2012-11-02T18:52:49.951,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:52:49.750934 PDT 3: TAP (Producer) eq_tapq:replication_building_343_'ns_1@10.2.1.68' - Backfill is completed with VBuckets 343, memcached<0.1519.0>: Fri Nov 2 18:52:49.750955 PDT 3: TAP (Producer) eq_tapq:replication_building_343_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 343 memcached<0.1519.0>: Fri Nov 2 18:52:49.836583 PDT 3: TAP (Producer) eq_tapq:replication_building_343_'ns_1@10.2.1.68' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:52:49.850425 PDT 3: TAP (Producer) eq_tapq:rebalance_343 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:49.850439 PDT 3: TAP (Producer) eq_tapq:rebalance_343 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:49.850449 PDT 3: TAP (Producer) eq_tapq:rebalance_343 - Sending TAP_VBUCKET_SET with vbucket 343 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:52:49.853646 PDT 3: TAP (Producer) eq_tapq:rebalance_343 - VBucket <343> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:52:49.853663 PDT 3: TAP (Producer) eq_tapq:rebalance_343 - Sending TAP_VBUCKET_SET with vbucket 343 and state "active" memcached<0.1519.0>: Fri Nov 2 18:52:49.853940 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:52:49.853962 PDT 3: TAP (Producer) eq_tapq:rebalance_343 - disconnected memcached<0.1519.0>: Fri Nov 2 18:52:49.854066 PDT 3: Schedule cleanup of "eq_tapq:rebalance_343" memcached<0.1519.0>: Fri Nov 2 18:52:49.854104 PDT 3: TAP (Producer) eq_tapq:rebalance_343 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:52:49.869148 PDT 3: TAP (Producer) eq_tapq:replication_building_343_'ns_1@10.2.1.68' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:52:49.882196 PDT 3: Deletion of vbucket 343 was completed. memcached<0.1519.0>: Fri Nov 2 18:52:49.896702 PDT 3: TAP (Producer) eq_tapq:replication_building_344_'ns_1@10.2.1.68' - Schedule the backfill for vbucket 344 memcached<0.1519.0>: Fri Nov 2 18:52:49.896730 PDT 3: TAP (Producer) eq_tapq:replication_building_344_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:49.896738 PDT 3: TAP (Producer) eq_tapq:replication_building_344_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:49.896743 PDT 3: TAP (Producer) eq_tapq:replication_building_344_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 344 memcached<0.1519.0>: Fri Nov 2 18:52:49.897059 PDT 3: Schedule cleanup of "eq_tapq:anon_88" memcached<0.1519.0>: Fri Nov 2 18:52:49.901001 PDT 3: TAP (Producer) eq_tapq:replication_building_343_'ns_1@10.2.1.68' - Clear the tap queues by force [ns_server:info,2012-11-02T18:52:50.863,ns_1@10.2.1.67:<0.18266.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.68",11209} vbucket 344 to state replica [rebalance:info,2012-11-02T18:52:50.865,ns_1@10.2.1.67:<0.18266.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[344]}, {checkpoints,[{344,10}]}, {name,<<"rebalance_344">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.68",11209}, [{username,"default"}, {password,[]}, {vbuckets,[344]}, {takeover,true}, {suffix,"344"}]} [rebalance:info,2012-11-02T18:52:50.867,ns_1@10.2.1.67:<0.18266.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:52:50.870,ns_1@10.2.1.67:<0.18266.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:52:50.946,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/344 [ns_server:info,2012-11-02T18:52:50.977,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:52:50.777376 PDT 3: TAP (Producer) eq_tapq:replication_building_344_'ns_1@10.2.1.68' - Backfill is completed with VBuckets 344, memcached<0.1519.0>: Fri Nov 2 18:52:50.777397 PDT 3: TAP (Producer) eq_tapq:replication_building_344_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 344 memcached<0.1519.0>: Fri Nov 2 18:52:50.860100 PDT 3: TAP (Producer) eq_tapq:replication_building_344_'ns_1@10.2.1.68' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:52:50.866210 PDT 3: TAP (Producer) eq_tapq:rebalance_344 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:50.866223 PDT 3: TAP (Producer) eq_tapq:rebalance_344 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:50.866229 PDT 3: TAP (Producer) eq_tapq:rebalance_344 - Sending TAP_VBUCKET_SET with vbucket 344 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:52:50.869867 PDT 3: TAP (Producer) eq_tapq:rebalance_344 - VBucket <344> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:52:50.869886 PDT 3: TAP (Producer) eq_tapq:rebalance_344 - Sending TAP_VBUCKET_SET with vbucket 344 and state "active" memcached<0.1519.0>: Fri Nov 2 18:52:50.870272 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:52:50.870296 PDT 3: TAP (Producer) eq_tapq:rebalance_344 - disconnected memcached<0.1519.0>: Fri Nov 2 18:52:50.870328 PDT 3: Schedule cleanup of "eq_tapq:rebalance_344" memcached<0.1519.0>: Fri Nov 2 18:52:50.870695 PDT 3: TAP (Producer) eq_tapq:rebalance_344 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:52:50.878805 PDT 3: TAP (Producer) eq_tapq:replication_building_344_'ns_1@10.2.1.68' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:52:50.888785 PDT 3: Deletion of vbucket 344 was completed. memcached<0.1519.0>: Fri Nov 2 18:52:50.897262 PDT 3: TAP (Producer) eq_tapq:replication_building_345_'ns_1@10.2.1.68' - Schedule the backfill for vbucket 345 memcached<0.1519.0>: Fri Nov 2 18:52:50.897300 PDT 3: TAP (Producer) eq_tapq:replication_building_345_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:50.897314 PDT 3: TAP (Producer) eq_tapq:replication_building_345_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:50.897324 PDT 3: TAP (Producer) eq_tapq:replication_building_345_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 345 memcached<0.1519.0>: Fri Nov 2 18:52:50.898003 PDT 3: Schedule cleanup of "eq_tapq:anon_89" memcached<0.1519.0>: Fri Nov 2 18:52:50.914133 PDT 3: TAP (Producer) eq_tapq:replication_building_344_'ns_1@10.2.1.68' - Clear the tap queues by force [ns_server:info,2012-11-02T18:52:51.888,ns_1@10.2.1.67:<0.18271.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.68",11209} vbucket 345 to state replica [rebalance:info,2012-11-02T18:52:51.897,ns_1@10.2.1.67:<0.18271.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[345]}, {checkpoints,[{345,10}]}, {name,<<"rebalance_345">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.68",11209}, [{username,"default"}, {password,[]}, {vbuckets,[345]}, {takeover,true}, {suffix,"345"}]} [rebalance:info,2012-11-02T18:52:51.898,ns_1@10.2.1.67:<0.18271.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:52:51.902,ns_1@10.2.1.67:<0.18271.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:52:51.975,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/345 [ns_server:info,2012-11-02T18:52:52.030,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:52:51.829781 PDT 3: TAP (Producer) eq_tapq:replication_building_345_'ns_1@10.2.1.68' - Backfill is completed with VBuckets 345, memcached<0.1519.0>: Fri Nov 2 18:52:51.829803 PDT 3: TAP (Producer) eq_tapq:replication_building_345_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 345 memcached<0.1519.0>: Fri Nov 2 18:52:51.885607 PDT 3: TAP (Producer) eq_tapq:replication_building_345_'ns_1@10.2.1.68' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:52:51.897804 PDT 3: TAP (Producer) eq_tapq:rebalance_345 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:51.897817 PDT 3: TAP (Producer) eq_tapq:rebalance_345 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:51.897822 PDT 3: TAP (Producer) eq_tapq:rebalance_345 - Sending TAP_VBUCKET_SET with vbucket 345 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:52:51.901624 PDT 3: TAP (Producer) eq_tapq:rebalance_345 - VBucket <345> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:52:51.901651 PDT 3: TAP (Producer) eq_tapq:rebalance_345 - Sending TAP_VBUCKET_SET with vbucket 345 and state "active" memcached<0.1519.0>: Fri Nov 2 18:52:51.901933 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:52:51.901958 PDT 3: TAP (Producer) eq_tapq:rebalance_345 - disconnected memcached<0.1519.0>: Fri Nov 2 18:52:51.911075 PDT 3: TAP (Producer) eq_tapq:replication_building_345_'ns_1@10.2.1.68' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:52:51.920760 PDT 3: Deletion of vbucket 345 was completed. memcached<0.1519.0>: Fri Nov 2 18:52:51.936720 PDT 3: TAP (Producer) eq_tapq:replication_building_346_'ns_1@10.2.1.68' - Schedule the backfill for vbucket 346 memcached<0.1519.0>: Fri Nov 2 18:52:51.936762 PDT 3: TAP (Producer) eq_tapq:replication_building_346_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:51.936776 PDT 3: TAP (Producer) eq_tapq:replication_building_346_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:51.936786 PDT 3: TAP (Producer) eq_tapq:replication_building_346_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 346 memcached<0.1519.0>: Fri Nov 2 18:52:51.936893 PDT 3: Schedule cleanup of "eq_tapq:anon_90" memcached<0.1519.0>: Fri Nov 2 18:52:51.936919 PDT 3: Schedule cleanup of "eq_tapq:rebalance_345" memcached<0.1519.0>: Fri Nov 2 18:52:51.947606 PDT 3: TAP (Producer) eq_tapq:replication_building_345_'ns_1@10.2.1.68' - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:52:51.947649 PDT 3: TAP (Producer) eq_tapq:rebalance_345 - Clear the tap queues by force [ns_server:info,2012-11-02T18:52:52.812,ns_1@10.2.1.67:<0.18282.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.68",11209} vbucket 346 to state replica [rebalance:info,2012-11-02T18:52:52.814,ns_1@10.2.1.67:<0.18282.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[346]}, {checkpoints,[{346,10}]}, {name,<<"rebalance_346">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.68",11209}, [{username,"default"}, {password,[]}, {vbuckets,[346]}, {takeover,true}, {suffix,"346"}]} [rebalance:info,2012-11-02T18:52:52.815,ns_1@10.2.1.67:<0.18282.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:52:52.823,ns_1@10.2.1.67:<0.18282.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:52:52.962,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:52:52.761139 PDT 3: TAP (Producer) eq_tapq:replication_building_346_'ns_1@10.2.1.68' - Backfill is completed with VBuckets 346, memcached<0.1519.0>: Fri Nov 2 18:52:52.761162 PDT 3: TAP (Producer) eq_tapq:replication_building_346_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 346 memcached<0.1519.0>: Fri Nov 2 18:52:52.807888 PDT 3: TAP (Producer) eq_tapq:replication_building_346_'ns_1@10.2.1.68' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:52:52.814728 PDT 3: TAP (Producer) eq_tapq:rebalance_346 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:52.814740 PDT 3: TAP (Producer) eq_tapq:rebalance_346 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:52.814745 PDT 3: TAP (Producer) eq_tapq:rebalance_346 - Sending TAP_VBUCKET_SET with vbucket 346 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:52:52.822701 PDT 3: TAP (Producer) eq_tapq:rebalance_346 - VBucket <346> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:52:52.822733 PDT 3: TAP (Producer) eq_tapq:rebalance_346 - Sending TAP_VBUCKET_SET with vbucket 346 and state "active" memcached<0.1519.0>: Fri Nov 2 18:52:52.822982 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:52:52.823008 PDT 3: TAP (Producer) eq_tapq:rebalance_346 - disconnected memcached<0.1519.0>: Fri Nov 2 18:52:52.823042 PDT 3: Schedule cleanup of "eq_tapq:rebalance_346" memcached<0.1519.0>: Fri Nov 2 18:52:52.823065 PDT 3: TAP (Producer) eq_tapq:rebalance_346 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:52:52.834260 PDT 3: TAP (Producer) eq_tapq:replication_building_346_'ns_1@10.2.1.68' - Connection is closed by force. [ns_server:info,2012-11-02T18:52:53.040,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/346 [ns_server:info,2012-11-02T18:52:53.189,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:52:52.988071 PDT 3: Deletion of vbucket 346 was completed. memcached<0.1519.0>: Fri Nov 2 18:52:52.999872 PDT 3: TAP (Producer) eq_tapq:replication_building_347_'ns_1@10.2.1.68' - Schedule the backfill for vbucket 347 memcached<0.1519.0>: Fri Nov 2 18:52:52.999914 PDT 3: TAP (Producer) eq_tapq:replication_building_347_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:52.999928 PDT 3: TAP (Producer) eq_tapq:replication_building_347_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:52.999939 PDT 3: TAP (Producer) eq_tapq:replication_building_347_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 347 memcached<0.1519.0>: Fri Nov 2 18:52:52.999997 PDT 3: Schedule cleanup of "eq_tapq:anon_91" memcached<0.1519.0>: Fri Nov 2 18:52:53.011543 PDT 3: TAP (Producer) eq_tapq:replication_building_346_'ns_1@10.2.1.68' - Clear the tap queues by force [ns_server:info,2012-11-02T18:52:53.937,ns_1@10.2.1.67:<0.18287.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.68",11209} vbucket 347 to state replica [rebalance:info,2012-11-02T18:52:53.945,ns_1@10.2.1.67:<0.18287.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[347]}, {checkpoints,[{347,10}]}, {name,<<"rebalance_347">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.68",11209}, [{username,"default"}, {password,[]}, {vbuckets,[347]}, {takeover,true}, {suffix,"347"}]} [rebalance:info,2012-11-02T18:52:53.948,ns_1@10.2.1.67:<0.18287.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:52:53.958,ns_1@10.2.1.67:<0.18287.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:52:54.030,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:52:53.829576 PDT 3: TAP (Producer) eq_tapq:replication_building_347_'ns_1@10.2.1.68' - Backfill is completed with VBuckets 347, memcached<0.1519.0>: Fri Nov 2 18:52:53.829596 PDT 3: TAP (Producer) eq_tapq:replication_building_347_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 347 memcached<0.1519.0>: Fri Nov 2 18:52:53.932345 PDT 3: TAP (Producer) eq_tapq:replication_building_347_'ns_1@10.2.1.68' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:52:53.946130 PDT 3: TAP (Producer) eq_tapq:rebalance_347 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:53.946148 PDT 3: TAP (Producer) eq_tapq:rebalance_347 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:53.946153 PDT 3: TAP (Producer) eq_tapq:rebalance_347 - Sending TAP_VBUCKET_SET with vbucket 347 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:52:53.957535 PDT 3: TAP (Producer) eq_tapq:rebalance_347 - VBucket <347> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:52:53.957565 PDT 3: TAP (Producer) eq_tapq:rebalance_347 - Sending TAP_VBUCKET_SET with vbucket 347 and state "active" memcached<0.1519.0>: Fri Nov 2 18:52:53.958044 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:52:53.958073 PDT 3: TAP (Producer) eq_tapq:rebalance_347 - disconnected [ns_server:info,2012-11-02T18:52:54.114,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/347 [ns_server:info,2012-11-02T18:52:54.252,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:52:54.051855 PDT 3: TAP (Producer) eq_tapq:replication_building_347_'ns_1@10.2.1.68' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:52:54.059529 PDT 3: Deletion of vbucket 347 was completed. memcached<0.1519.0>: Fri Nov 2 18:52:54.081531 PDT 3: TAP (Producer) eq_tapq:replication_building_348_'ns_1@10.2.1.68' - Schedule the backfill for vbucket 348 memcached<0.1519.0>: Fri Nov 2 18:52:54.081559 PDT 3: TAP (Producer) eq_tapq:replication_building_348_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:54.081566 PDT 3: TAP (Producer) eq_tapq:replication_building_348_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:54.081571 PDT 3: TAP (Producer) eq_tapq:replication_building_348_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 348 memcached<0.1519.0>: Fri Nov 2 18:52:54.081648 PDT 3: Schedule cleanup of "eq_tapq:anon_92" memcached<0.1519.0>: Fri Nov 2 18:52:54.081660 PDT 3: Schedule cleanup of "eq_tapq:rebalance_347" memcached<0.1519.0>: Fri Nov 2 18:52:54.085060 PDT 3: TAP (Producer) eq_tapq:replication_building_347_'ns_1@10.2.1.68' - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:52:54.085106 PDT 3: TAP (Producer) eq_tapq:rebalance_347 - Clear the tap queues by force [ns_server:info,2012-11-02T18:52:55.049,ns_1@10.2.1.67:<0.18297.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.68",11209} vbucket 348 to state replica [rebalance:info,2012-11-02T18:52:55.052,ns_1@10.2.1.67:<0.18297.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[348]}, {checkpoints,[{348,10}]}, {name,<<"rebalance_348">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.68",11209}, [{username,"default"}, {password,[]}, {vbuckets,[348]}, {takeover,true}, {suffix,"348"}]} [rebalance:info,2012-11-02T18:52:55.061,ns_1@10.2.1.67:<0.18297.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:52:55.064,ns_1@10.2.1.67:<0.18297.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:52:55.133,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/348 [ns_server:info,2012-11-02T18:52:55.145,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:52:54.944889 PDT 3: TAP (Producer) eq_tapq:replication_building_348_'ns_1@10.2.1.68' - Backfill is completed with VBuckets 348, memcached<0.1519.0>: Fri Nov 2 18:52:54.944910 PDT 3: TAP (Producer) eq_tapq:replication_building_348_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 348 memcached<0.1519.0>: Fri Nov 2 18:52:55.046100 PDT 3: TAP (Producer) eq_tapq:replication_building_348_'ns_1@10.2.1.68' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:52:55.059370 PDT 3: TAP (Producer) eq_tapq:rebalance_348 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:55.059385 PDT 3: TAP (Producer) eq_tapq:rebalance_348 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:55.059390 PDT 3: TAP (Producer) eq_tapq:rebalance_348 - Sending TAP_VBUCKET_SET with vbucket 348 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:52:55.063924 PDT 3: TAP (Producer) eq_tapq:rebalance_348 - VBucket <348> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:52:55.063956 PDT 3: TAP (Producer) eq_tapq:rebalance_348 - Sending TAP_VBUCKET_SET with vbucket 348 and state "active" memcached<0.1519.0>: Fri Nov 2 18:52:55.064519 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:52:55.064562 PDT 3: TAP (Producer) eq_tapq:rebalance_348 - disconnected memcached<0.1519.0>: Fri Nov 2 18:52:55.064621 PDT 3: Schedule cleanup of "eq_tapq:rebalance_348" memcached<0.1519.0>: Fri Nov 2 18:52:55.064666 PDT 3: TAP (Producer) eq_tapq:rebalance_348 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:52:55.073848 PDT 3: TAP (Producer) eq_tapq:replication_building_348_'ns_1@10.2.1.68' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:52:55.086063 PDT 3: Deletion of vbucket 348 was completed. memcached<0.1519.0>: Fri Nov 2 18:52:55.094634 PDT 3: TAP (Producer) eq_tapq:replication_building_349_'ns_1@10.2.1.68' - Schedule the backfill for vbucket 349 memcached<0.1519.0>: Fri Nov 2 18:52:55.094678 PDT 3: TAP (Producer) eq_tapq:replication_building_349_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:55.094693 PDT 3: TAP (Producer) eq_tapq:replication_building_349_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:55.094704 PDT 3: TAP (Producer) eq_tapq:replication_building_349_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 349 memcached<0.1519.0>: Fri Nov 2 18:52:55.094776 PDT 3: Schedule cleanup of "eq_tapq:anon_93" memcached<0.1519.0>: Fri Nov 2 18:52:55.110550 PDT 3: TAP (Producer) eq_tapq:replication_building_348_'ns_1@10.2.1.68' - Clear the tap queues by force [ns_server:info,2012-11-02T18:52:55.861,ns_1@10.2.1.67:<0.18312.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.68",11209} vbucket 349 to state replica [rebalance:info,2012-11-02T18:52:55.868,ns_1@10.2.1.67:<0.18312.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[349]}, {checkpoints,[{349,10}]}, {name,<<"rebalance_349">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.68",11209}, [{username,"default"}, {password,[]}, {vbuckets,[349]}, {takeover,true}, {suffix,"349"}]} [rebalance:info,2012-11-02T18:52:55.869,ns_1@10.2.1.67:<0.18312.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:52:55.873,ns_1@10.2.1.67:<0.18312.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:52:55.924,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/349 [ns_server:info,2012-11-02T18:52:56.028,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:52:55.827283 PDT 3: TAP (Producer) eq_tapq:replication_building_349_'ns_1@10.2.1.68' - Backfill is completed with VBuckets 349, memcached<0.1519.0>: Fri Nov 2 18:52:55.827304 PDT 3: TAP (Producer) eq_tapq:replication_building_349_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 349 memcached<0.1519.0>: Fri Nov 2 18:52:55.855962 PDT 3: TAP (Producer) eq_tapq:replication_building_349_'ns_1@10.2.1.68' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:52:55.868819 PDT 3: TAP (Producer) eq_tapq:rebalance_349 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:55.868832 PDT 3: TAP (Producer) eq_tapq:rebalance_349 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:55.868844 PDT 3: TAP (Producer) eq_tapq:rebalance_349 - Sending TAP_VBUCKET_SET with vbucket 349 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:52:55.872773 PDT 3: TAP (Producer) eq_tapq:rebalance_349 - VBucket <349> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:52:55.872791 PDT 3: TAP (Producer) eq_tapq:rebalance_349 - Sending TAP_VBUCKET_SET with vbucket 349 and state "active" memcached<0.1519.0>: Fri Nov 2 18:52:55.873143 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:52:55.873169 PDT 3: TAP (Producer) eq_tapq:rebalance_349 - disconnected memcached<0.1519.0>: Fri Nov 2 18:52:55.880507 PDT 3: TAP (Producer) eq_tapq:replication_building_349_'ns_1@10.2.1.68' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:52:55.892660 PDT 3: Deletion of vbucket 349 was completed. memcached<0.1519.0>: Fri Nov 2 18:52:55.908145 PDT 3: TAP (Producer) eq_tapq:replication_building_350_'ns_1@10.2.1.68' - Schedule the backfill for vbucket 350 memcached<0.1519.0>: Fri Nov 2 18:52:55.908173 PDT 3: TAP (Producer) eq_tapq:replication_building_350_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:55.908181 PDT 3: TAP (Producer) eq_tapq:replication_building_350_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:55.908186 PDT 3: TAP (Producer) eq_tapq:replication_building_350_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 350 memcached<0.1519.0>: Fri Nov 2 18:52:55.908268 PDT 3: Schedule cleanup of "eq_tapq:anon_94" memcached<0.1519.0>: Fri Nov 2 18:52:55.908291 PDT 3: Schedule cleanup of "eq_tapq:rebalance_349" memcached<0.1519.0>: Fri Nov 2 18:52:55.918593 PDT 3: TAP (Producer) eq_tapq:replication_building_349_'ns_1@10.2.1.68' - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:52:55.918634 PDT 3: TAP (Producer) eq_tapq:rebalance_349 - Clear the tap queues by force [ns_server:info,2012-11-02T18:52:56.874,ns_1@10.2.1.67:<0.18322.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.68",11209} vbucket 350 to state replica [rebalance:info,2012-11-02T18:52:56.876,ns_1@10.2.1.67:<0.18322.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[350]}, {checkpoints,[{350,10}]}, {name,<<"rebalance_350">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.68",11209}, [{username,"default"}, {password,[]}, {vbuckets,[350]}, {takeover,true}, {suffix,"350"}]} [rebalance:info,2012-11-02T18:52:56.882,ns_1@10.2.1.67:<0.18322.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:52:56.886,ns_1@10.2.1.67:<0.18322.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:52:56.958,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/350 [ns_server:info,2012-11-02T18:52:56.983,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:52:56.782346 PDT 3: TAP (Producer) eq_tapq:replication_building_350_'ns_1@10.2.1.68' - Backfill is completed with VBuckets 350, memcached<0.1519.0>: Fri Nov 2 18:52:56.782367 PDT 3: TAP (Producer) eq_tapq:replication_building_350_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 350 memcached<0.1519.0>: Fri Nov 2 18:52:56.870624 PDT 3: TAP (Producer) eq_tapq:replication_building_350_'ns_1@10.2.1.68' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:52:56.876910 PDT 3: TAP (Producer) eq_tapq:rebalance_350 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:56.876923 PDT 3: TAP (Producer) eq_tapq:rebalance_350 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:56.876928 PDT 3: TAP (Producer) eq_tapq:rebalance_350 - Sending TAP_VBUCKET_SET with vbucket 350 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:52:56.885088 PDT 3: TAP (Producer) eq_tapq:rebalance_350 - VBucket <350> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:52:56.885108 PDT 3: TAP (Producer) eq_tapq:rebalance_350 - Sending TAP_VBUCKET_SET with vbucket 350 and state "active" memcached<0.1519.0>: Fri Nov 2 18:52:56.885982 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:52:56.886006 PDT 3: TAP (Producer) eq_tapq:rebalance_350 - disconnected memcached<0.1519.0>: Fri Nov 2 18:52:56.897799 PDT 3: TAP (Producer) eq_tapq:replication_building_350_'ns_1@10.2.1.68' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:52:56.908866 PDT 3: Deletion of vbucket 350 was completed. memcached<0.1519.0>: Fri Nov 2 18:52:56.921943 PDT 3: TAP (Producer) eq_tapq:replication_building_351_'ns_1@10.2.1.68' - Schedule the backfill for vbucket 351 memcached<0.1519.0>: Fri Nov 2 18:52:56.921984 PDT 3: TAP (Producer) eq_tapq:replication_building_351_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:56.921998 PDT 3: TAP (Producer) eq_tapq:replication_building_351_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:56.922008 PDT 3: TAP (Producer) eq_tapq:replication_building_351_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 351 memcached<0.1519.0>: Fri Nov 2 18:52:56.922125 PDT 3: Schedule cleanup of "eq_tapq:anon_95" memcached<0.1519.0>: Fri Nov 2 18:52:56.922155 PDT 3: Schedule cleanup of "eq_tapq:rebalance_350" memcached<0.1519.0>: Fri Nov 2 18:52:56.934225 PDT 3: TAP (Producer) eq_tapq:replication_building_350_'ns_1@10.2.1.68' - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:52:56.934252 PDT 3: TAP (Producer) eq_tapq:rebalance_350 - Clear the tap queues by force [ns_server:info,2012-11-02T18:52:57.888,ns_1@10.2.1.67:<0.18327.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.68",11209} vbucket 351 to state replica [rebalance:info,2012-11-02T18:52:57.894,ns_1@10.2.1.67:<0.18327.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[351]}, {checkpoints,[{351,10}]}, {name,<<"rebalance_351">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.68",11209}, [{username,"default"}, {password,[]}, {vbuckets,[351]}, {takeover,true}, {suffix,"351"}]} [rebalance:info,2012-11-02T18:52:57.895,ns_1@10.2.1.67:<0.18327.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:52:57.898,ns_1@10.2.1.67:<0.18327.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:52:57.963,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/351 [ns_server:info,2012-11-02T18:52:58.052,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:52:57.852045 PDT 3: TAP (Producer) eq_tapq:replication_building_351_'ns_1@10.2.1.68' - Backfill is completed with VBuckets 351, memcached<0.1519.0>: Fri Nov 2 18:52:57.852083 PDT 3: TAP (Producer) eq_tapq:replication_building_351_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 351 memcached<0.1519.0>: Fri Nov 2 18:52:57.883421 PDT 3: TAP (Producer) eq_tapq:replication_building_351_'ns_1@10.2.1.68' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:52:57.895142 PDT 3: TAP (Producer) eq_tapq:rebalance_351 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:57.895154 PDT 3: TAP (Producer) eq_tapq:rebalance_351 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:57.895166 PDT 3: TAP (Producer) eq_tapq:rebalance_351 - Sending TAP_VBUCKET_SET with vbucket 351 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:52:57.898548 PDT 3: TAP (Producer) eq_tapq:rebalance_351 - VBucket <351> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:52:57.898566 PDT 3: TAP (Producer) eq_tapq:rebalance_351 - Sending TAP_VBUCKET_SET with vbucket 351 and state "active" memcached<0.1519.0>: Fri Nov 2 18:52:57.898817 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:52:57.898854 PDT 3: TAP (Producer) eq_tapq:rebalance_351 - disconnected memcached<0.1519.0>: Fri Nov 2 18:52:57.898887 PDT 3: Schedule cleanup of "eq_tapq:rebalance_351" memcached<0.1519.0>: Fri Nov 2 18:52:57.898917 PDT 3: TAP (Producer) eq_tapq:rebalance_351 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:52:57.909331 PDT 3: TAP (Producer) eq_tapq:replication_building_351_'ns_1@10.2.1.68' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:52:57.916579 PDT 3: Deletion of vbucket 351 was completed. memcached<0.1519.0>: Fri Nov 2 18:52:57.927916 PDT 3: TAP (Producer) eq_tapq:replication_building_352_'ns_1@10.2.1.68' - Schedule the backfill for vbucket 352 memcached<0.1519.0>: Fri Nov 2 18:52:57.927941 PDT 3: TAP (Producer) eq_tapq:replication_building_352_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:57.927949 PDT 3: TAP (Producer) eq_tapq:replication_building_352_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:57.927953 PDT 3: TAP (Producer) eq_tapq:replication_building_352_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 352 memcached<0.1519.0>: Fri Nov 2 18:52:57.928006 PDT 3: Schedule cleanup of "eq_tapq:anon_96" memcached<0.1519.0>: Fri Nov 2 18:52:57.934495 PDT 3: TAP (Producer) eq_tapq:replication_building_351_'ns_1@10.2.1.68' - Clear the tap queues by force [ns_server:info,2012-11-02T18:52:58.805,ns_1@10.2.1.67:<0.18338.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.68",11209} vbucket 352 to state replica [rebalance:info,2012-11-02T18:52:58.807,ns_1@10.2.1.67:<0.18338.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[352]}, {checkpoints,[{352,10}]}, {name,<<"rebalance_352">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.68",11209}, [{username,"default"}, {password,[]}, {vbuckets,[352]}, {takeover,true}, {suffix,"352"}]} [rebalance:info,2012-11-02T18:52:58.812,ns_1@10.2.1.67:<0.18338.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:52:58.819,ns_1@10.2.1.67:<0.18338.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:52:58.869,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/352 [ns_server:info,2012-11-02T18:52:58.914,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:52:58.714003 PDT 3: TAP (Producer) eq_tapq:replication_building_352_'ns_1@10.2.1.68' - Backfill is completed with VBuckets 352, memcached<0.1519.0>: Fri Nov 2 18:52:58.714023 PDT 3: TAP (Producer) eq_tapq:replication_building_352_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 352 memcached<0.1519.0>: Fri Nov 2 18:52:58.803062 PDT 3: TAP (Producer) eq_tapq:replication_building_352_'ns_1@10.2.1.68' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:52:58.807881 PDT 3: TAP (Producer) eq_tapq:rebalance_352 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:58.807893 PDT 3: TAP (Producer) eq_tapq:rebalance_352 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:58.807899 PDT 3: TAP (Producer) eq_tapq:rebalance_352 - Sending TAP_VBUCKET_SET with vbucket 352 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:52:58.818456 PDT 3: TAP (Producer) eq_tapq:rebalance_352 - VBucket <352> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:52:58.818477 PDT 3: TAP (Producer) eq_tapq:rebalance_352 - Sending TAP_VBUCKET_SET with vbucket 352 and state "active" memcached<0.1519.0>: Fri Nov 2 18:52:58.818799 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:52:58.818821 PDT 3: TAP (Producer) eq_tapq:rebalance_352 - disconnected memcached<0.1519.0>: Fri Nov 2 18:52:58.818852 PDT 3: Schedule cleanup of "eq_tapq:rebalance_352" memcached<0.1519.0>: Fri Nov 2 18:52:58.818900 PDT 3: TAP (Producer) eq_tapq:rebalance_352 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:52:58.826432 PDT 3: TAP (Producer) eq_tapq:replication_building_352_'ns_1@10.2.1.68' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:52:58.833318 PDT 3: Deletion of vbucket 352 was completed. memcached<0.1519.0>: Fri Nov 2 18:52:58.843264 PDT 3: TAP (Producer) eq_tapq:replication_building_353_'ns_1@10.2.1.68' - Schedule the backfill for vbucket 353 memcached<0.1519.0>: Fri Nov 2 18:52:58.843289 PDT 3: TAP (Producer) eq_tapq:replication_building_353_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:58.843297 PDT 3: TAP (Producer) eq_tapq:replication_building_353_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:58.843302 PDT 3: TAP (Producer) eq_tapq:replication_building_353_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 353 memcached<0.1519.0>: Fri Nov 2 18:52:58.843330 PDT 3: Schedule cleanup of "eq_tapq:anon_97" memcached<0.1519.0>: Fri Nov 2 18:52:58.863150 PDT 3: TAP (Producer) eq_tapq:replication_building_352_'ns_1@10.2.1.68' - Clear the tap queues by force [ns_server:info,2012-11-02T18:52:59.914,ns_1@10.2.1.67:<0.18343.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.68",11209} vbucket 353 to state replica [rebalance:info,2012-11-02T18:52:59.916,ns_1@10.2.1.67:<0.18343.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[353]}, {checkpoints,[{353,10}]}, {name,<<"rebalance_353">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.68",11209}, [{username,"default"}, {password,[]}, {vbuckets,[353]}, {takeover,true}, {suffix,"353"}]} [rebalance:info,2012-11-02T18:52:59.917,ns_1@10.2.1.67:<0.18343.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:52:59.923,ns_1@10.2.1.67:<0.18343.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:52:59.992,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/353 [ns_server:info,2012-11-02T18:53:00.005,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:52:59.805270 PDT 3: TAP (Producer) eq_tapq:replication_building_353_'ns_1@10.2.1.68' - Backfill is completed with VBuckets 353, memcached<0.1519.0>: Fri Nov 2 18:52:59.805304 PDT 3: TAP (Producer) eq_tapq:replication_building_353_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 353 memcached<0.1519.0>: Fri Nov 2 18:52:59.910561 PDT 3: TAP (Producer) eq_tapq:replication_building_353_'ns_1@10.2.1.68' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:52:59.916811 PDT 3: TAP (Producer) eq_tapq:rebalance_353 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:59.916823 PDT 3: TAP (Producer) eq_tapq:rebalance_353 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:59.916833 PDT 3: TAP (Producer) eq_tapq:rebalance_353 - Sending TAP_VBUCKET_SET with vbucket 353 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:52:59.920354 PDT 3: TAP (Producer) eq_tapq:rebalance_353 - VBucket <353> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:52:59.920373 PDT 3: TAP (Producer) eq_tapq:rebalance_353 - Sending TAP_VBUCKET_SET with vbucket 353 and state "active" memcached<0.1519.0>: Fri Nov 2 18:52:59.923483 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:52:59.923508 PDT 3: TAP (Producer) eq_tapq:rebalance_353 - disconnected memcached<0.1519.0>: Fri Nov 2 18:52:59.923555 PDT 3: Schedule cleanup of "eq_tapq:rebalance_353" memcached<0.1519.0>: Fri Nov 2 18:52:59.923587 PDT 3: TAP (Producer) eq_tapq:rebalance_353 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:52:59.934230 PDT 3: TAP (Producer) eq_tapq:replication_building_353_'ns_1@10.2.1.68' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:52:59.941564 PDT 3: Deletion of vbucket 353 was completed. memcached<0.1519.0>: Fri Nov 2 18:52:59.954219 PDT 3: TAP (Producer) eq_tapq:replication_building_354_'ns_1@10.2.1.68' - Schedule the backfill for vbucket 354 memcached<0.1519.0>: Fri Nov 2 18:52:59.954258 PDT 3: TAP (Producer) eq_tapq:replication_building_354_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:59.954272 PDT 3: TAP (Producer) eq_tapq:replication_building_354_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:52:59.954282 PDT 3: TAP (Producer) eq_tapq:replication_building_354_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 354 memcached<0.1519.0>: Fri Nov 2 18:52:59.954471 PDT 3: Schedule cleanup of "eq_tapq:anon_98" memcached<0.1519.0>: Fri Nov 2 18:52:59.969994 PDT 3: TAP (Producer) eq_tapq:replication_building_353_'ns_1@10.2.1.68' - Clear the tap queues by force [ns_server:info,2012-11-02T18:53:00.892,ns_1@10.2.1.67:<0.18362.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.68",11209} vbucket 354 to state replica [rebalance:info,2012-11-02T18:53:00.894,ns_1@10.2.1.67:<0.18362.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[354]}, {checkpoints,[{354,10}]}, {name,<<"rebalance_354">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.68",11209}, [{username,"default"}, {password,[]}, {vbuckets,[354]}, {takeover,true}, {suffix,"354"}]} [rebalance:info,2012-11-02T18:53:00.895,ns_1@10.2.1.67:<0.18362.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:53:00.898,ns_1@10.2.1.67:<0.18362.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:53:00.971,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/354 [ns_server:info,2012-11-02T18:53:01.056,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:53:00.855772 PDT 3: TAP (Producer) eq_tapq:replication_building_354_'ns_1@10.2.1.68' - Backfill is completed with VBuckets 354, memcached<0.1519.0>: Fri Nov 2 18:53:00.855794 PDT 3: TAP (Producer) eq_tapq:replication_building_354_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 354 memcached<0.1519.0>: Fri Nov 2 18:53:00.883382 PDT 3: TAP (Producer) eq_tapq:replication_building_354_'ns_1@10.2.1.68' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:53:00.894793 PDT 3: TAP (Producer) eq_tapq:rebalance_354 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:00.894805 PDT 3: TAP (Producer) eq_tapq:rebalance_354 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:00.894810 PDT 3: TAP (Producer) eq_tapq:rebalance_354 - Sending TAP_VBUCKET_SET with vbucket 354 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:53:00.898208 PDT 3: TAP (Producer) eq_tapq:rebalance_354 - VBucket <354> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:53:00.898225 PDT 3: TAP (Producer) eq_tapq:rebalance_354 - Sending TAP_VBUCKET_SET with vbucket 354 and state "active" memcached<0.1519.0>: Fri Nov 2 18:53:00.898539 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:53:00.898566 PDT 3: TAP (Producer) eq_tapq:rebalance_354 - disconnected memcached<0.1519.0>: Fri Nov 2 18:53:00.898596 PDT 3: Schedule cleanup of "eq_tapq:rebalance_354" memcached<0.1519.0>: Fri Nov 2 18:53:00.898890 PDT 3: TAP (Producer) eq_tapq:rebalance_354 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:53:00.917651 PDT 3: TAP (Producer) eq_tapq:replication_building_354_'ns_1@10.2.1.68' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:53:00.923153 PDT 3: Deletion of vbucket 354 was completed. memcached<0.1519.0>: Fri Nov 2 18:53:00.931714 PDT 3: TAP (Producer) eq_tapq:replication_building_355_'ns_1@10.2.1.68' - Schedule the backfill for vbucket 355 memcached<0.1519.0>: Fri Nov 2 18:53:00.931738 PDT 3: TAP (Producer) eq_tapq:replication_building_355_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:00.931746 PDT 3: TAP (Producer) eq_tapq:replication_building_355_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:00.931751 PDT 3: TAP (Producer) eq_tapq:replication_building_355_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 355 memcached<0.1519.0>: Fri Nov 2 18:53:00.931798 PDT 3: Schedule cleanup of "eq_tapq:anon_99" memcached<0.1519.0>: Fri Nov 2 18:53:00.950335 PDT 3: TAP (Producer) eq_tapq:replication_building_354_'ns_1@10.2.1.68' - Clear the tap queues by force [ns_server:info,2012-11-02T18:53:01.216,ns_1@10.2.1.67:<0.18367.0>:compaction_daemon:try_to_cleanup_indexes:439]Cleaning up indexes for bucket `default` [ns_server:info,2012-11-02T18:53:01.216,ns_1@10.2.1.67:<0.18367.0>:compaction_daemon:spawn_bucket_compactor:404]Compacting bucket default with config: [{parallel_db_and_view_compaction,false}, {database_fragmentation_threshold,{30,undefined}}, {view_fragmentation_threshold,{100,undefined}}] [ns_server:info,2012-11-02T18:53:01.919,ns_1@10.2.1.67:<0.18375.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.68",11209} vbucket 355 to state replica [rebalance:info,2012-11-02T18:53:01.921,ns_1@10.2.1.67:<0.18375.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[355]}, {checkpoints,[{355,10}]}, {name,<<"rebalance_355">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.68",11209}, [{username,"default"}, {password,[]}, {vbuckets,[355]}, {takeover,true}, {suffix,"355"}]} [rebalance:info,2012-11-02T18:53:01.922,ns_1@10.2.1.67:<0.18375.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:53:01.935,ns_1@10.2.1.67:<0.18375.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:53:02.010,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/355 [ns_server:info,2012-11-02T18:53:02.094,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:53:01.894429 PDT 3: TAP (Producer) eq_tapq:replication_building_355_'ns_1@10.2.1.68' - Backfill is completed with VBuckets 355, memcached<0.1519.0>: Fri Nov 2 18:53:01.894450 PDT 3: TAP (Producer) eq_tapq:replication_building_355_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 355 memcached<0.1519.0>: Fri Nov 2 18:53:01.911395 PDT 3: TAP (Producer) eq_tapq:replication_building_355_'ns_1@10.2.1.68' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:53:01.921750 PDT 3: TAP (Producer) eq_tapq:rebalance_355 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:01.921762 PDT 3: TAP (Producer) eq_tapq:rebalance_355 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:01.921767 PDT 3: TAP (Producer) eq_tapq:rebalance_355 - Sending TAP_VBUCKET_SET with vbucket 355 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:53:01.935314 PDT 3: TAP (Producer) eq_tapq:rebalance_355 - VBucket <355> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:53:01.935331 PDT 3: TAP (Producer) eq_tapq:rebalance_355 - Sending TAP_VBUCKET_SET with vbucket 355 and state "active" memcached<0.1519.0>: Fri Nov 2 18:53:01.935595 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:53:01.935624 PDT 3: TAP (Producer) eq_tapq:rebalance_355 - disconnected memcached<0.1519.0>: Fri Nov 2 18:53:01.935655 PDT 3: Schedule cleanup of "eq_tapq:rebalance_355" memcached<0.1519.0>: Fri Nov 2 18:53:01.935675 PDT 3: TAP (Producer) eq_tapq:rebalance_355 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:53:01.948340 PDT 3: TAP (Producer) eq_tapq:replication_building_355_'ns_1@10.2.1.68' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:53:01.959242 PDT 3: Deletion of vbucket 355 was completed. memcached<0.1519.0>: Fri Nov 2 18:53:01.969677 PDT 3: TAP (Producer) eq_tapq:replication_building_356_'ns_1@10.2.1.68' - Schedule the backfill for vbucket 356 memcached<0.1519.0>: Fri Nov 2 18:53:01.969701 PDT 3: TAP (Producer) eq_tapq:replication_building_356_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:01.969709 PDT 3: TAP (Producer) eq_tapq:replication_building_356_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:01.969713 PDT 3: TAP (Producer) eq_tapq:replication_building_356_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 356 memcached<0.1519.0>: Fri Nov 2 18:53:01.969764 PDT 3: Schedule cleanup of "eq_tapq:anon_100" memcached<0.1519.0>: Fri Nov 2 18:53:01.987107 PDT 3: TAP (Producer) eq_tapq:replication_building_355_'ns_1@10.2.1.68' - Clear the tap queues by force [ns_server:info,2012-11-02T18:53:03.111,ns_1@10.2.1.67:<0.18385.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.68",11209} vbucket 356 to state replica [rebalance:info,2012-11-02T18:53:03.113,ns_1@10.2.1.67:<0.18385.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[356]}, {checkpoints,[{356,10}]}, {name,<<"rebalance_356">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.68",11209}, [{username,"default"}, {password,[]}, {vbuckets,[356]}, {takeover,true}, {suffix,"356"}]} [rebalance:info,2012-11-02T18:53:03.119,ns_1@10.2.1.67:<0.18385.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:53:03.126,ns_1@10.2.1.67:<0.18385.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:53:03.190,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/356 [ns_server:info,2012-11-02T18:53:03.293,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:53:03.092750 PDT 3: TAP (Producer) eq_tapq:replication_building_356_'ns_1@10.2.1.68' - Backfill is completed with VBuckets 356, memcached<0.1519.0>: Fri Nov 2 18:53:03.092772 PDT 3: TAP (Producer) eq_tapq:replication_building_356_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 356 memcached<0.1519.0>: Fri Nov 2 18:53:03.108957 PDT 3: TAP (Producer) eq_tapq:replication_building_356_'ns_1@10.2.1.68' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:53:03.113616 PDT 3: TAP (Producer) eq_tapq:rebalance_356 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:03.113629 PDT 3: TAP (Producer) eq_tapq:rebalance_356 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:03.113634 PDT 3: TAP (Producer) eq_tapq:rebalance_356 - Sending TAP_VBUCKET_SET with vbucket 356 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:53:03.125874 PDT 3: TAP (Producer) eq_tapq:rebalance_356 - VBucket <356> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:53:03.125899 PDT 3: TAP (Producer) eq_tapq:rebalance_356 - Sending TAP_VBUCKET_SET with vbucket 356 and state "active" memcached<0.1519.0>: Fri Nov 2 18:53:03.126193 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:53:03.126224 PDT 3: TAP (Producer) eq_tapq:rebalance_356 - disconnected memcached<0.1519.0>: Fri Nov 2 18:53:03.126254 PDT 3: Schedule cleanup of "eq_tapq:rebalance_356" memcached<0.1519.0>: Fri Nov 2 18:53:03.126300 PDT 3: TAP (Producer) eq_tapq:rebalance_356 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:53:03.134015 PDT 3: TAP (Producer) eq_tapq:replication_building_356_'ns_1@10.2.1.68' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:53:03.140962 PDT 3: Deletion of vbucket 356 was completed. memcached<0.1519.0>: Fri Nov 2 18:53:03.159704 PDT 3: TAP (Producer) eq_tapq:replication_building_357_'ns_1@10.2.1.68' - Schedule the backfill for vbucket 357 memcached<0.1519.0>: Fri Nov 2 18:53:03.159747 PDT 3: TAP (Producer) eq_tapq:replication_building_357_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:03.159762 PDT 3: TAP (Producer) eq_tapq:replication_building_357_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:03.159771 PDT 3: TAP (Producer) eq_tapq:replication_building_357_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 357 memcached<0.1519.0>: Fri Nov 2 18:53:03.159827 PDT 3: Schedule cleanup of "eq_tapq:anon_101" memcached<0.1519.0>: Fri Nov 2 18:53:03.159879 PDT 3: TAP (Producer) eq_tapq:replication_building_356_'ns_1@10.2.1.68' - Clear the tap queues by force [ns_server:info,2012-11-02T18:53:04.140,ns_1@10.2.1.67:<0.18390.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.68",11209} vbucket 357 to state replica [rebalance:info,2012-11-02T18:53:04.142,ns_1@10.2.1.67:<0.18390.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[357]}, {checkpoints,[{357,10}]}, {name,<<"rebalance_357">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.68",11209}, [{username,"default"}, {password,[]}, {vbuckets,[357]}, {takeover,true}, {suffix,"357"}]} [rebalance:info,2012-11-02T18:53:04.143,ns_1@10.2.1.67:<0.18390.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:53:04.148,ns_1@10.2.1.67:<0.18390.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:53:04.218,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/357 [ns_server:info,2012-11-02T18:53:04.292,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:53:04.091348 PDT 3: TAP (Producer) eq_tapq:replication_building_357_'ns_1@10.2.1.68' - Backfill is completed with VBuckets 357, memcached<0.1519.0>: Fri Nov 2 18:53:04.091370 PDT 3: TAP (Producer) eq_tapq:replication_building_357_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 357 memcached<0.1519.0>: Fri Nov 2 18:53:04.137820 PDT 3: TAP (Producer) eq_tapq:replication_building_357_'ns_1@10.2.1.68' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:53:04.142637 PDT 3: TAP (Producer) eq_tapq:rebalance_357 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:04.142648 PDT 3: TAP (Producer) eq_tapq:rebalance_357 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:04.142653 PDT 3: TAP (Producer) eq_tapq:rebalance_357 - Sending TAP_VBUCKET_SET with vbucket 357 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:53:04.147649 PDT 3: TAP (Producer) eq_tapq:rebalance_357 - VBucket <357> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:53:04.147667 PDT 3: TAP (Producer) eq_tapq:rebalance_357 - Sending TAP_VBUCKET_SET with vbucket 357 and state "active" memcached<0.1519.0>: Fri Nov 2 18:53:04.147909 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:53:04.148008 PDT 3: TAP (Producer) eq_tapq:rebalance_357 - disconnected memcached<0.1519.0>: Fri Nov 2 18:53:04.148046 PDT 3: Schedule cleanup of "eq_tapq:rebalance_357" memcached<0.1519.0>: Fri Nov 2 18:53:04.148120 PDT 3: TAP (Producer) eq_tapq:rebalance_357 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:53:04.158262 PDT 3: TAP (Producer) eq_tapq:replication_building_357_'ns_1@10.2.1.68' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:53:04.166974 PDT 3: Deletion of vbucket 357 was completed. memcached<0.1519.0>: Fri Nov 2 18:53:04.176194 PDT 3: TAP (Producer) eq_tapq:replication_building_358_'ns_1@10.2.1.68' - Schedule the backfill for vbucket 358 memcached<0.1519.0>: Fri Nov 2 18:53:04.176220 PDT 3: TAP (Producer) eq_tapq:replication_building_358_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:04.176227 PDT 3: TAP (Producer) eq_tapq:replication_building_358_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:04.176232 PDT 3: TAP (Producer) eq_tapq:replication_building_358_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 358 memcached<0.1519.0>: Fri Nov 2 18:53:04.176301 PDT 3: Schedule cleanup of "eq_tapq:anon_102" memcached<0.1519.0>: Fri Nov 2 18:53:04.192620 PDT 3: TAP (Producer) eq_tapq:replication_building_357_'ns_1@10.2.1.68' - Clear the tap queues by force [ns_server:info,2012-11-02T18:53:05.135,ns_1@10.2.1.67:<0.18401.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.68",11209} vbucket 358 to state replica [rebalance:info,2012-11-02T18:53:05.137,ns_1@10.2.1.67:<0.18401.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[358]}, {checkpoints,[{358,10}]}, {name,<<"rebalance_358">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.68",11209}, [{username,"default"}, {password,[]}, {vbuckets,[358]}, {takeover,true}, {suffix,"358"}]} [rebalance:info,2012-11-02T18:53:05.145,ns_1@10.2.1.67:<0.18401.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:53:05.149,ns_1@10.2.1.67:<0.18401.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:53:05.202,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/358 [ns_server:info,2012-11-02T18:53:05.265,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:53:05.064468 PDT 3: TAP (Producer) eq_tapq:replication_building_358_'ns_1@10.2.1.68' - Backfill is completed with VBuckets 358, memcached<0.1519.0>: Fri Nov 2 18:53:05.064489 PDT 3: TAP (Producer) eq_tapq:replication_building_358_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 358 memcached<0.1519.0>: Fri Nov 2 18:53:05.131447 PDT 3: TAP (Producer) eq_tapq:replication_building_358_'ns_1@10.2.1.68' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:53:05.137960 PDT 3: TAP (Producer) eq_tapq:rebalance_358 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:05.137976 PDT 3: TAP (Producer) eq_tapq:rebalance_358 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:05.137983 PDT 3: TAP (Producer) eq_tapq:rebalance_358 - Sending TAP_VBUCKET_SET with vbucket 358 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:53:05.148358 PDT 3: TAP (Producer) eq_tapq:rebalance_358 - VBucket <358> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:53:05.148380 PDT 3: TAP (Producer) eq_tapq:rebalance_358 - Sending TAP_VBUCKET_SET with vbucket 358 and state "active" memcached<0.1519.0>: Fri Nov 2 18:53:05.149037 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:53:05.149072 PDT 3: TAP (Producer) eq_tapq:rebalance_358 - disconnected memcached<0.1519.0>: Fri Nov 2 18:53:05.149108 PDT 3: Schedule cleanup of "eq_tapq:rebalance_358" memcached<0.1519.0>: Fri Nov 2 18:53:05.149158 PDT 3: TAP (Producer) eq_tapq:rebalance_358 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:53:05.158304 PDT 3: TAP (Producer) eq_tapq:replication_building_358_'ns_1@10.2.1.68' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:53:05.165017 PDT 3: Deletion of vbucket 358 was completed. memcached<0.1519.0>: Fri Nov 2 18:53:05.174011 PDT 3: TAP (Producer) eq_tapq:replication_building_359_'ns_1@10.2.1.68' - Schedule the backfill for vbucket 359 memcached<0.1519.0>: Fri Nov 2 18:53:05.174052 PDT 3: TAP (Producer) eq_tapq:replication_building_359_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:05.174066 PDT 3: TAP (Producer) eq_tapq:replication_building_359_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:05.174076 PDT 3: TAP (Producer) eq_tapq:replication_building_359_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 359 memcached<0.1519.0>: Fri Nov 2 18:53:05.176039 PDT 3: Schedule cleanup of "eq_tapq:anon_103" memcached<0.1519.0>: Fri Nov 2 18:53:05.193542 PDT 3: TAP (Producer) eq_tapq:replication_building_358_'ns_1@10.2.1.68' - Clear the tap queues by force [ns_server:info,2012-11-02T18:53:06.120,ns_1@10.2.1.67:<0.18415.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.68",11209} vbucket 359 to state replica [rebalance:info,2012-11-02T18:53:06.121,ns_1@10.2.1.67:<0.18415.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[359]}, {checkpoints,[{359,10}]}, {name,<<"rebalance_359">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.68",11209}, [{username,"default"}, {password,[]}, {vbuckets,[359]}, {takeover,true}, {suffix,"359"}]} [rebalance:info,2012-11-02T18:53:06.127,ns_1@10.2.1.67:<0.18415.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:53:06.130,ns_1@10.2.1.67:<0.18415.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:53:06.182,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/359 [ns_server:info,2012-11-02T18:53:06.228,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:53:06.028004 PDT 3: TAP (Producer) eq_tapq:replication_building_359_'ns_1@10.2.1.68' - Backfill is completed with VBuckets 359, memcached<0.1519.0>: Fri Nov 2 18:53:06.028024 PDT 3: TAP (Producer) eq_tapq:replication_building_359_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 359 memcached<0.1519.0>: Fri Nov 2 18:53:06.113946 PDT 3: TAP (Producer) eq_tapq:replication_building_359_'ns_1@10.2.1.68' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:53:06.122245 PDT 3: TAP (Producer) eq_tapq:rebalance_359 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:06.122256 PDT 3: TAP (Producer) eq_tapq:rebalance_359 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:06.122261 PDT 3: TAP (Producer) eq_tapq:rebalance_359 - Sending TAP_VBUCKET_SET with vbucket 359 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:53:06.130052 PDT 3: TAP (Producer) eq_tapq:rebalance_359 - VBucket <359> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:53:06.130068 PDT 3: TAP (Producer) eq_tapq:rebalance_359 - Sending TAP_VBUCKET_SET with vbucket 359 and state "active" memcached<0.1519.0>: Fri Nov 2 18:53:06.130337 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:53:06.130359 PDT 3: TAP (Producer) eq_tapq:rebalance_359 - disconnected memcached<0.1519.0>: Fri Nov 2 18:53:06.130390 PDT 3: Schedule cleanup of "eq_tapq:rebalance_359" memcached<0.1519.0>: Fri Nov 2 18:53:06.130787 PDT 3: TAP (Producer) eq_tapq:rebalance_359 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:53:06.138964 PDT 3: TAP (Producer) eq_tapq:replication_building_359_'ns_1@10.2.1.68' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:53:06.146781 PDT 3: Deletion of vbucket 359 was completed. memcached<0.1519.0>: Fri Nov 2 18:53:06.157490 PDT 3: TAP (Producer) eq_tapq:replication_building_360_'ns_1@10.2.1.68' - Schedule the backfill for vbucket 360 memcached<0.1519.0>: Fri Nov 2 18:53:06.157529 PDT 3: TAP (Producer) eq_tapq:replication_building_360_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:06.157543 PDT 3: TAP (Producer) eq_tapq:replication_building_360_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:06.157554 PDT 3: TAP (Producer) eq_tapq:replication_building_360_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 360 memcached<0.1519.0>: Fri Nov 2 18:53:06.158987 PDT 3: Schedule cleanup of "eq_tapq:anon_104" memcached<0.1519.0>: Fri Nov 2 18:53:06.172884 PDT 3: TAP (Producer) eq_tapq:replication_building_359_'ns_1@10.2.1.68' - Clear the tap queues by force [ns_server:info,2012-11-02T18:53:07.235,ns_1@10.2.1.67:<0.18425.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.68",11209} vbucket 360 to state replica [rebalance:info,2012-11-02T18:53:07.237,ns_1@10.2.1.67:<0.18425.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[360]}, {checkpoints,[{360,10}]}, {name,<<"rebalance_360">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.68",11209}, [{username,"default"}, {password,[]}, {vbuckets,[360]}, {takeover,true}, {suffix,"360"}]} [rebalance:info,2012-11-02T18:53:07.238,ns_1@10.2.1.67:<0.18425.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:53:07.245,ns_1@10.2.1.67:<0.18425.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:53:07.314,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/360 [ns_server:info,2012-11-02T18:53:07.331,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:53:07.130805 PDT 3: TAP (Producer) eq_tapq:replication_building_360_'ns_1@10.2.1.68' - Backfill is completed with VBuckets 360, memcached<0.1519.0>: Fri Nov 2 18:53:07.130834 PDT 3: TAP (Producer) eq_tapq:replication_building_360_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 360 memcached<0.1519.0>: Fri Nov 2 18:53:07.227194 PDT 3: TAP (Producer) eq_tapq:replication_building_360_'ns_1@10.2.1.68' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:53:07.237795 PDT 3: TAP (Producer) eq_tapq:rebalance_360 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:07.237808 PDT 3: TAP (Producer) eq_tapq:rebalance_360 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:07.237813 PDT 3: TAP (Producer) eq_tapq:rebalance_360 - Sending TAP_VBUCKET_SET with vbucket 360 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:53:07.244595 PDT 3: TAP (Producer) eq_tapq:rebalance_360 - VBucket <360> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:53:07.244632 PDT 3: TAP (Producer) eq_tapq:rebalance_360 - Sending TAP_VBUCKET_SET with vbucket 360 and state "active" memcached<0.1519.0>: Fri Nov 2 18:53:07.245159 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:53:07.245208 PDT 3: TAP (Producer) eq_tapq:rebalance_360 - disconnected memcached<0.1519.0>: Fri Nov 2 18:53:07.245274 PDT 3: Schedule cleanup of "eq_tapq:rebalance_360" memcached<0.1519.0>: Fri Nov 2 18:53:07.245468 PDT 3: TAP (Producer) eq_tapq:rebalance_360 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:53:07.258617 PDT 3: TAP (Producer) eq_tapq:replication_building_360_'ns_1@10.2.1.68' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:53:07.266778 PDT 3: Deletion of vbucket 360 was completed. memcached<0.1519.0>: Fri Nov 2 18:53:07.277214 PDT 3: TAP (Producer) eq_tapq:replication_building_361_'ns_1@10.2.1.68' - Schedule the backfill for vbucket 361 memcached<0.1519.0>: Fri Nov 2 18:53:07.277238 PDT 3: TAP (Producer) eq_tapq:replication_building_361_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:07.277247 PDT 3: TAP (Producer) eq_tapq:replication_building_361_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:07.277252 PDT 3: TAP (Producer) eq_tapq:replication_building_361_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 361 memcached<0.1519.0>: Fri Nov 2 18:53:07.277306 PDT 3: Schedule cleanup of "eq_tapq:anon_105" memcached<0.1519.0>: Fri Nov 2 18:53:07.291372 PDT 3: TAP (Producer) eq_tapq:replication_building_360_'ns_1@10.2.1.68' - Clear the tap queues by force [ns_server:info,2012-11-02T18:53:08.543,ns_1@10.2.1.67:<0.18431.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.68",11209} vbucket 361 to state replica [rebalance:info,2012-11-02T18:53:08.548,ns_1@10.2.1.67:<0.18431.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[361]}, {checkpoints,[{361,10}]}, {name,<<"rebalance_361">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.68",11209}, [{username,"default"}, {password,[]}, {vbuckets,[361]}, {takeover,true}, {suffix,"361"}]} [rebalance:info,2012-11-02T18:53:08.549,ns_1@10.2.1.67:<0.18431.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:53:08.557,ns_1@10.2.1.67:<0.18431.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:53:08.622,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/361 [ns_server:info,2012-11-02T18:53:08.623,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:53:08.422101 PDT 3: TAP (Producer) eq_tapq:replication_building_361_'ns_1@10.2.1.68' - Backfill is completed with VBuckets 361, memcached<0.1519.0>: Fri Nov 2 18:53:08.422119 PDT 3: TAP (Producer) eq_tapq:replication_building_361_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 361 memcached<0.1519.0>: Fri Nov 2 18:53:08.540954 PDT 3: TAP (Producer) eq_tapq:replication_building_361_'ns_1@10.2.1.68' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:53:08.548325 PDT 3: TAP (Producer) eq_tapq:rebalance_361 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:08.548338 PDT 3: TAP (Producer) eq_tapq:rebalance_361 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:08.548343 PDT 3: TAP (Producer) eq_tapq:rebalance_361 - Sending TAP_VBUCKET_SET with vbucket 361 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:53:08.556039 PDT 3: TAP (Producer) eq_tapq:rebalance_361 - VBucket <361> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:53:08.556059 PDT 3: TAP (Producer) eq_tapq:rebalance_361 - Sending TAP_VBUCKET_SET with vbucket 361 and state "active" memcached<0.1519.0>: Fri Nov 2 18:53:08.556693 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:53:08.556716 PDT 3: TAP (Producer) eq_tapq:rebalance_361 - disconnected memcached<0.1519.0>: Fri Nov 2 18:53:08.556762 PDT 3: Schedule cleanup of "eq_tapq:rebalance_361" memcached<0.1519.0>: Fri Nov 2 18:53:08.556814 PDT 3: TAP (Producer) eq_tapq:rebalance_361 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:53:08.565802 PDT 3: TAP (Producer) eq_tapq:replication_building_361_'ns_1@10.2.1.68' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:53:08.574995 PDT 3: Deletion of vbucket 361 was completed. memcached<0.1519.0>: Fri Nov 2 18:53:08.585715 PDT 3: TAP (Producer) eq_tapq:replication_building_362_'ns_1@10.2.1.68' - Schedule the backfill for vbucket 362 memcached<0.1519.0>: Fri Nov 2 18:53:08.585756 PDT 3: TAP (Producer) eq_tapq:replication_building_362_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:08.585773 PDT 3: TAP (Producer) eq_tapq:replication_building_362_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:08.585783 PDT 3: TAP (Producer) eq_tapq:replication_building_362_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 362 memcached<0.1519.0>: Fri Nov 2 18:53:08.585832 PDT 3: Schedule cleanup of "eq_tapq:anon_106" memcached<0.1519.0>: Fri Nov 2 18:53:08.598366 PDT 3: TAP (Producer) eq_tapq:replication_building_361_'ns_1@10.2.1.68' - Clear the tap queues by force [ns_server:info,2012-11-02T18:53:09.570,ns_1@10.2.1.67:<0.18441.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.68",11209} vbucket 362 to state replica [rebalance:info,2012-11-02T18:53:09.572,ns_1@10.2.1.67:<0.18441.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[362]}, {checkpoints,[{362,10}]}, {name,<<"rebalance_362">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.68",11209}, [{username,"default"}, {password,[]}, {vbuckets,[362]}, {takeover,true}, {suffix,"362"}]} [rebalance:info,2012-11-02T18:53:09.578,ns_1@10.2.1.67:<0.18441.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:53:09.581,ns_1@10.2.1.67:<0.18441.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:53:09.636,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/362 [ns_server:info,2012-11-02T18:53:09.737,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:53:09.536738 PDT 3: TAP (Producer) eq_tapq:replication_building_362_'ns_1@10.2.1.68' - Backfill is completed with VBuckets 362, memcached<0.1519.0>: Fri Nov 2 18:53:09.536760 PDT 3: TAP (Producer) eq_tapq:replication_building_362_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 362 memcached<0.1519.0>: Fri Nov 2 18:53:09.567743 PDT 3: TAP (Producer) eq_tapq:replication_building_362_'ns_1@10.2.1.68' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:53:09.572899 PDT 3: TAP (Producer) eq_tapq:rebalance_362 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:09.572910 PDT 3: TAP (Producer) eq_tapq:rebalance_362 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:09.572915 PDT 3: TAP (Producer) eq_tapq:rebalance_362 - Sending TAP_VBUCKET_SET with vbucket 362 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:53:09.581632 PDT 3: TAP (Producer) eq_tapq:rebalance_362 - VBucket <362> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:53:09.581651 PDT 3: TAP (Producer) eq_tapq:rebalance_362 - Sending TAP_VBUCKET_SET with vbucket 362 and state "active" memcached<0.1519.0>: Fri Nov 2 18:53:09.581826 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:53:09.581852 PDT 3: TAP (Producer) eq_tapq:rebalance_362 - disconnected memcached<0.1519.0>: Fri Nov 2 18:53:09.582783 PDT 3: Schedule cleanup of "eq_tapq:rebalance_362" memcached<0.1519.0>: Fri Nov 2 18:53:09.582809 PDT 3: TAP (Producer) eq_tapq:rebalance_362 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:53:09.590559 PDT 3: TAP (Producer) eq_tapq:replication_building_362_'ns_1@10.2.1.68' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:53:09.597766 PDT 3: Deletion of vbucket 362 was completed. memcached<0.1519.0>: Fri Nov 2 18:53:09.615710 PDT 3: TAP (Producer) eq_tapq:replication_building_363_'ns_1@10.2.1.68' - Schedule the backfill for vbucket 363 memcached<0.1519.0>: Fri Nov 2 18:53:09.615740 PDT 3: TAP (Producer) eq_tapq:replication_building_363_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:09.615747 PDT 3: TAP (Producer) eq_tapq:replication_building_363_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:09.615752 PDT 3: TAP (Producer) eq_tapq:replication_building_363_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 363 memcached<0.1519.0>: Fri Nov 2 18:53:09.615795 PDT 3: Schedule cleanup of "eq_tapq:anon_107" memcached<0.1519.0>: Fri Nov 2 18:53:09.617587 PDT 3: TAP (Producer) eq_tapq:replication_building_362_'ns_1@10.2.1.68' - Clear the tap queues by force [ns_server:info,2012-11-02T18:53:10.573,ns_1@10.2.1.67:<0.18472.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.68",11209} vbucket 363 to state replica [rebalance:info,2012-11-02T18:53:10.575,ns_1@10.2.1.67:<0.18472.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[363]}, {checkpoints,[{363,10}]}, {name,<<"rebalance_363">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.68",11209}, [{username,"default"}, {password,[]}, {vbuckets,[363]}, {takeover,true}, {suffix,"363"}]} [rebalance:info,2012-11-02T18:53:10.580,ns_1@10.2.1.67:<0.18472.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:53:10.584,ns_1@10.2.1.67:<0.18472.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:53:10.653,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/363 [ns_server:info,2012-11-02T18:53:10.664,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:53:10.463397 PDT 3: TAP (Producer) eq_tapq:replication_building_363_'ns_1@10.2.1.68' - Backfill is completed with VBuckets 363, memcached<0.1519.0>: Fri Nov 2 18:53:10.463415 PDT 3: TAP (Producer) eq_tapq:replication_building_363_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 363 memcached<0.1519.0>: Fri Nov 2 18:53:10.570641 PDT 3: TAP (Producer) eq_tapq:replication_building_363_'ns_1@10.2.1.68' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:53:10.575920 PDT 3: TAP (Producer) eq_tapq:rebalance_363 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:10.575933 PDT 3: TAP (Producer) eq_tapq:rebalance_363 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:10.575938 PDT 3: TAP (Producer) eq_tapq:rebalance_363 - Sending TAP_VBUCKET_SET with vbucket 363 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:53:10.583601 PDT 3: TAP (Producer) eq_tapq:rebalance_363 - VBucket <363> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:53:10.583621 PDT 3: TAP (Producer) eq_tapq:rebalance_363 - Sending TAP_VBUCKET_SET with vbucket 363 and state "active" memcached<0.1519.0>: Fri Nov 2 18:53:10.583934 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:53:10.583963 PDT 3: TAP (Producer) eq_tapq:rebalance_363 - disconnected memcached<0.1519.0>: Fri Nov 2 18:53:10.584057 PDT 3: Schedule cleanup of "eq_tapq:rebalance_363" memcached<0.1519.0>: Fri Nov 2 18:53:10.584133 PDT 3: TAP (Producer) eq_tapq:rebalance_363 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:53:10.595363 PDT 3: TAP (Producer) eq_tapq:replication_building_363_'ns_1@10.2.1.68' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:53:10.605755 PDT 3: Deletion of vbucket 363 was completed. memcached<0.1519.0>: Fri Nov 2 18:53:10.624666 PDT 3: TAP (Producer) eq_tapq:replication_building_364_'ns_1@10.2.1.68' - Schedule the backfill for vbucket 364 memcached<0.1519.0>: Fri Nov 2 18:53:10.624708 PDT 3: TAP (Producer) eq_tapq:replication_building_364_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:10.624722 PDT 3: TAP (Producer) eq_tapq:replication_building_364_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:10.624731 PDT 3: TAP (Producer) eq_tapq:replication_building_364_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 364 memcached<0.1519.0>: Fri Nov 2 18:53:10.625702 PDT 3: Schedule cleanup of "eq_tapq:anon_108" memcached<0.1519.0>: Fri Nov 2 18:53:10.634134 PDT 3: TAP (Producer) eq_tapq:replication_building_363_'ns_1@10.2.1.68' - Clear the tap queues by force [ns_server:info,2012-11-02T18:53:11.527,ns_1@10.2.1.67:<0.18494.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.68",11209} vbucket 364 to state replica [rebalance:info,2012-11-02T18:53:11.535,ns_1@10.2.1.67:<0.18494.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[364]}, {checkpoints,[{364,9}]}, {name,<<"rebalance_364">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.68",11209}, [{username,"default"}, {password,[]}, {vbuckets,[364]}, {takeover,true}, {suffix,"364"}]} [rebalance:info,2012-11-02T18:53:11.535,ns_1@10.2.1.67:<0.18494.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:53:11.538,ns_1@10.2.1.67:<0.18494.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:53:11.594,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/364 [ns_server:info,2012-11-02T18:53:11.705,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:53:11.505247 PDT 3: TAP (Producer) eq_tapq:replication_building_364_'ns_1@10.2.1.68' - Backfill is completed with VBuckets 364, memcached<0.1519.0>: Fri Nov 2 18:53:11.505276 PDT 3: TAP (Producer) eq_tapq:replication_building_364_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 364 memcached<0.1519.0>: Fri Nov 2 18:53:11.518180 PDT 3: TAP (Producer) eq_tapq:replication_building_364_'ns_1@10.2.1.68' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:53:11.535379 PDT 3: TAP (Producer) eq_tapq:rebalance_364 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:11.535395 PDT 3: TAP (Producer) eq_tapq:rebalance_364 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:11.535400 PDT 3: TAP (Producer) eq_tapq:rebalance_364 - Sending TAP_VBUCKET_SET with vbucket 364 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:53:11.538612 PDT 3: TAP (Producer) eq_tapq:rebalance_364 - VBucket <364> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:53:11.538632 PDT 3: TAP (Producer) eq_tapq:rebalance_364 - Sending TAP_VBUCKET_SET with vbucket 364 and state "active" memcached<0.1519.0>: Fri Nov 2 18:53:11.538908 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:53:11.538931 PDT 3: TAP (Producer) eq_tapq:rebalance_364 - disconnected memcached<0.1519.0>: Fri Nov 2 18:53:11.549493 PDT 3: TAP (Producer) eq_tapq:replication_building_364_'ns_1@10.2.1.68' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:53:11.560806 PDT 3: Deletion of vbucket 364 was completed. memcached<0.1519.0>: Fri Nov 2 18:53:11.572530 PDT 3: TAP (Producer) eq_tapq:replication_building_365_'ns_1@10.2.1.68' - Schedule the backfill for vbucket 365 memcached<0.1519.0>: Fri Nov 2 18:53:11.572562 PDT 3: TAP (Producer) eq_tapq:replication_building_365_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:11.572571 PDT 3: TAP (Producer) eq_tapq:replication_building_365_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:11.572577 PDT 3: TAP (Producer) eq_tapq:replication_building_365_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 365 memcached<0.1519.0>: Fri Nov 2 18:53:11.572650 PDT 3: Schedule cleanup of "eq_tapq:anon_109" memcached<0.1519.0>: Fri Nov 2 18:53:11.572667 PDT 3: Schedule cleanup of "eq_tapq:rebalance_364" memcached<0.1519.0>: Fri Nov 2 18:53:11.589602 PDT 3: TAP (Producer) eq_tapq:replication_building_364_'ns_1@10.2.1.68' - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:53:11.589871 PDT 3: TAP (Producer) eq_tapq:rebalance_364 - Clear the tap queues by force [ns_server:info,2012-11-02T18:53:12.619,ns_1@10.2.1.67:<0.18499.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.68",11209} vbucket 365 to state replica [rebalance:info,2012-11-02T18:53:12.623,ns_1@10.2.1.67:<0.18499.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[365]}, {checkpoints,[{365,10}]}, {name,<<"rebalance_365">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.68",11209}, [{username,"default"}, {password,[]}, {vbuckets,[365]}, {takeover,true}, {suffix,"365"}]} [rebalance:info,2012-11-02T18:53:12.625,ns_1@10.2.1.67:<0.18499.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:53:12.630,ns_1@10.2.1.67:<0.18499.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:53:12.714,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/365 [ns_server:info,2012-11-02T18:53:12.748,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:53:12.547677 PDT 3: TAP (Producer) eq_tapq:replication_building_365_'ns_1@10.2.1.68' - Backfill is completed with VBuckets 365, memcached<0.1519.0>: Fri Nov 2 18:53:12.547698 PDT 3: TAP (Producer) eq_tapq:replication_building_365_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 365 memcached<0.1519.0>: Fri Nov 2 18:53:12.613023 PDT 3: TAP (Producer) eq_tapq:replication_building_365_'ns_1@10.2.1.68' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:53:12.623901 PDT 3: TAP (Producer) eq_tapq:rebalance_365 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:12.623913 PDT 3: TAP (Producer) eq_tapq:rebalance_365 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:12.623918 PDT 3: TAP (Producer) eq_tapq:rebalance_365 - Sending TAP_VBUCKET_SET with vbucket 365 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:53:12.630514 PDT 3: TAP (Producer) eq_tapq:rebalance_365 - VBucket <365> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:53:12.630534 PDT 3: TAP (Producer) eq_tapq:rebalance_365 - Sending TAP_VBUCKET_SET with vbucket 365 and state "active" memcached<0.1519.0>: Fri Nov 2 18:53:12.630874 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:53:12.630897 PDT 3: TAP (Producer) eq_tapq:rebalance_365 - disconnected memcached<0.1519.0>: Fri Nov 2 18:53:12.630928 PDT 3: Schedule cleanup of "eq_tapq:rebalance_365" memcached<0.1519.0>: Fri Nov 2 18:53:12.630995 PDT 3: TAP (Producer) eq_tapq:rebalance_365 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:53:12.657687 PDT 3: TAP (Producer) eq_tapq:replication_building_365_'ns_1@10.2.1.68' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:53:12.668519 PDT 3: Deletion of vbucket 365 was completed. memcached<0.1519.0>: Fri Nov 2 18:53:12.680717 PDT 3: TAP (Producer) eq_tapq:replication_building_366_'ns_1@10.2.1.68' - Schedule the backfill for vbucket 366 memcached<0.1519.0>: Fri Nov 2 18:53:12.680743 PDT 3: TAP (Producer) eq_tapq:replication_building_366_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:12.680750 PDT 3: TAP (Producer) eq_tapq:replication_building_366_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:12.680755 PDT 3: TAP (Producer) eq_tapq:replication_building_366_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 366 memcached<0.1519.0>: Fri Nov 2 18:53:12.681570 PDT 3: Schedule cleanup of "eq_tapq:anon_110" memcached<0.1519.0>: Fri Nov 2 18:53:12.692684 PDT 3: TAP (Producer) eq_tapq:replication_building_365_'ns_1@10.2.1.68' - Clear the tap queues by force [ns_server:info,2012-11-02T18:53:13.760,ns_1@10.2.1.67:<0.18510.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.68",11209} vbucket 366 to state replica [rebalance:info,2012-11-02T18:53:13.766,ns_1@10.2.1.67:<0.18510.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[366]}, {checkpoints,[{366,10}]}, {name,<<"rebalance_366">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.68",11209}, [{username,"default"}, {password,[]}, {vbuckets,[366]}, {takeover,true}, {suffix,"366"}]} [rebalance:info,2012-11-02T18:53:13.768,ns_1@10.2.1.67:<0.18510.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:53:13.771,ns_1@10.2.1.67:<0.18510.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:53:13.832,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/366 [ns_server:info,2012-11-02T18:53:13.918,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:53:13.718549 PDT 3: TAP (Producer) eq_tapq:replication_building_366_'ns_1@10.2.1.68' - Backfill is completed with VBuckets 366, memcached<0.1519.0>: Fri Nov 2 18:53:13.718572 PDT 3: TAP (Producer) eq_tapq:replication_building_366_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 366 memcached<0.1519.0>: Fri Nov 2 18:53:13.757812 PDT 3: TAP (Producer) eq_tapq:replication_building_366_'ns_1@10.2.1.68' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:53:13.766840 PDT 3: TAP (Producer) eq_tapq:rebalance_366 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:13.766863 PDT 3: TAP (Producer) eq_tapq:rebalance_366 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:13.766874 PDT 3: TAP (Producer) eq_tapq:rebalance_366 - Sending TAP_VBUCKET_SET with vbucket 366 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:53:13.770887 PDT 3: TAP (Producer) eq_tapq:rebalance_366 - VBucket <366> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:53:13.770918 PDT 3: TAP (Producer) eq_tapq:rebalance_366 - Sending TAP_VBUCKET_SET with vbucket 366 and state "active" memcached<0.1519.0>: Fri Nov 2 18:53:13.771366 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:53:13.771401 PDT 3: TAP (Producer) eq_tapq:rebalance_366 - disconnected memcached<0.1519.0>: Fri Nov 2 18:53:13.771449 PDT 3: Schedule cleanup of "eq_tapq:rebalance_366" memcached<0.1519.0>: Fri Nov 2 18:53:13.771578 PDT 3: TAP (Producer) eq_tapq:rebalance_366 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:53:13.780544 PDT 3: TAP (Producer) eq_tapq:replication_building_366_'ns_1@10.2.1.68' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:53:13.790973 PDT 3: Deletion of vbucket 366 was completed. memcached<0.1519.0>: Fri Nov 2 18:53:13.802219 PDT 3: TAP (Producer) eq_tapq:replication_building_367_'ns_1@10.2.1.68' - Schedule the backfill for vbucket 367 memcached<0.1519.0>: Fri Nov 2 18:53:13.802260 PDT 3: TAP (Producer) eq_tapq:replication_building_367_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:13.802275 PDT 3: TAP (Producer) eq_tapq:replication_building_367_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:13.802284 PDT 3: TAP (Producer) eq_tapq:replication_building_367_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 367 memcached<0.1519.0>: Fri Nov 2 18:53:13.802339 PDT 3: Schedule cleanup of "eq_tapq:anon_111" memcached<0.1519.0>: Fri Nov 2 18:53:13.816829 PDT 3: TAP (Producer) eq_tapq:replication_building_366_'ns_1@10.2.1.68' - Clear the tap queues by force [ns_server:info,2012-11-02T18:53:14.787,ns_1@10.2.1.67:<0.18520.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.68",11209} vbucket 367 to state replica [rebalance:info,2012-11-02T18:53:14.788,ns_1@10.2.1.67:<0.18520.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[367]}, {checkpoints,[{367,10}]}, {name,<<"rebalance_367">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.68",11209}, [{username,"default"}, {password,[]}, {vbuckets,[367]}, {takeover,true}, {suffix,"367"}]} [rebalance:info,2012-11-02T18:53:14.789,ns_1@10.2.1.67:<0.18520.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:53:14.799,ns_1@10.2.1.67:<0.18520.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:53:14.857,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/367 [ns_server:info,2012-11-02T18:53:14.897,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:53:14.697537 PDT 3: TAP (Producer) eq_tapq:replication_building_367_'ns_1@10.2.1.68' - Backfill is completed with VBuckets 367, memcached<0.1519.0>: Fri Nov 2 18:53:14.697561 PDT 3: TAP (Producer) eq_tapq:replication_building_367_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 367 memcached<0.1519.0>: Fri Nov 2 18:53:14.783955 PDT 3: TAP (Producer) eq_tapq:replication_building_367_'ns_1@10.2.1.68' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:53:14.789003 PDT 3: TAP (Producer) eq_tapq:rebalance_367 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:14.789014 PDT 3: TAP (Producer) eq_tapq:rebalance_367 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:14.789020 PDT 3: TAP (Producer) eq_tapq:rebalance_367 - Sending TAP_VBUCKET_SET with vbucket 367 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:53:14.798884 PDT 3: TAP (Producer) eq_tapq:rebalance_367 - VBucket <367> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:53:14.798920 PDT 3: TAP (Producer) eq_tapq:rebalance_367 - Sending TAP_VBUCKET_SET with vbucket 367 and state "active" memcached<0.1519.0>: Fri Nov 2 18:53:14.799527 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:53:14.799572 PDT 3: TAP (Producer) eq_tapq:rebalance_367 - disconnected memcached<0.1519.0>: Fri Nov 2 18:53:14.810710 PDT 3: TAP (Producer) eq_tapq:replication_building_367_'ns_1@10.2.1.68' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:53:14.823041 PDT 3: Deletion of vbucket 367 was completed. memcached<0.1519.0>: Fri Nov 2 18:53:14.832963 PDT 3: TAP (Producer) eq_tapq:replication_building_368_'ns_1@10.2.1.68' - Schedule the backfill for vbucket 368 memcached<0.1519.0>: Fri Nov 2 18:53:14.832993 PDT 3: TAP (Producer) eq_tapq:replication_building_368_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:14.833003 PDT 3: TAP (Producer) eq_tapq:replication_building_368_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:14.833011 PDT 3: TAP (Producer) eq_tapq:replication_building_368_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 368 memcached<0.1519.0>: Fri Nov 2 18:53:14.834474 PDT 3: Schedule cleanup of "eq_tapq:anon_112" memcached<0.1519.0>: Fri Nov 2 18:53:14.834506 PDT 3: Schedule cleanup of "eq_tapq:rebalance_367" memcached<0.1519.0>: Fri Nov 2 18:53:14.850538 PDT 3: TAP (Producer) eq_tapq:replication_building_367_'ns_1@10.2.1.68' - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:53:14.850788 PDT 3: TAP (Producer) eq_tapq:rebalance_367 - Clear the tap queues by force [ns_server:info,2012-11-02T18:53:15.800,ns_1@10.2.1.67:<0.18534.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.68",11209} vbucket 368 to state replica [rebalance:info,2012-11-02T18:53:15.801,ns_1@10.2.1.67:<0.18534.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[368]}, {checkpoints,[{368,10}]}, {name,<<"rebalance_368">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.68",11209}, [{username,"default"}, {password,[]}, {vbuckets,[368]}, {takeover,true}, {suffix,"368"}]} [rebalance:info,2012-11-02T18:53:15.802,ns_1@10.2.1.67:<0.18534.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:53:15.810,ns_1@10.2.1.67:<0.18534.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:53:15.867,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/368 [ns_server:info,2012-11-02T18:53:15.939,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:53:15.739185 PDT 3: TAP (Producer) eq_tapq:replication_building_368_'ns_1@10.2.1.68' - Backfill is completed with VBuckets 368, memcached<0.1519.0>: Fri Nov 2 18:53:15.739206 PDT 3: TAP (Producer) eq_tapq:replication_building_368_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 368 memcached<0.1519.0>: Fri Nov 2 18:53:15.796667 PDT 3: TAP (Producer) eq_tapq:replication_building_368_'ns_1@10.2.1.68' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:53:15.802019 PDT 3: TAP (Producer) eq_tapq:rebalance_368 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:15.802030 PDT 3: TAP (Producer) eq_tapq:rebalance_368 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:15.802035 PDT 3: TAP (Producer) eq_tapq:rebalance_368 - Sending TAP_VBUCKET_SET with vbucket 368 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:53:15.809674 PDT 3: TAP (Producer) eq_tapq:rebalance_368 - VBucket <368> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:53:15.809709 PDT 3: TAP (Producer) eq_tapq:rebalance_368 - Sending TAP_VBUCKET_SET with vbucket 368 and state "active" memcached<0.1519.0>: Fri Nov 2 18:53:15.810734 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:53:15.810779 PDT 3: TAP (Producer) eq_tapq:rebalance_368 - disconnected memcached<0.1519.0>: Fri Nov 2 18:53:15.810845 PDT 3: Schedule cleanup of "eq_tapq:rebalance_368" memcached<0.1519.0>: Fri Nov 2 18:53:15.810911 PDT 3: TAP (Producer) eq_tapq:rebalance_368 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:53:15.819723 PDT 3: TAP (Producer) eq_tapq:replication_building_368_'ns_1@10.2.1.68' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:53:15.833181 PDT 3: Deletion of vbucket 368 was completed. memcached<0.1519.0>: Fri Nov 2 18:53:15.841768 PDT 3: TAP (Producer) eq_tapq:replication_building_369_'ns_1@10.2.1.68' - Schedule the backfill for vbucket 369 memcached<0.1519.0>: Fri Nov 2 18:53:15.841791 PDT 3: TAP (Producer) eq_tapq:replication_building_369_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:15.841799 PDT 3: TAP (Producer) eq_tapq:replication_building_369_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:15.841804 PDT 3: TAP (Producer) eq_tapq:replication_building_369_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 369 memcached<0.1519.0>: Fri Nov 2 18:53:15.841857 PDT 3: Schedule cleanup of "eq_tapq:anon_113" memcached<0.1519.0>: Fri Nov 2 18:53:15.863936 PDT 3: TAP (Producer) eq_tapq:replication_building_368_'ns_1@10.2.1.68' - Clear the tap queues by force [ns_server:info,2012-11-02T18:53:16.914,ns_1@10.2.1.67:<0.18545.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.68",11209} vbucket 369 to state replica [rebalance:info,2012-11-02T18:53:16.920,ns_1@10.2.1.67:<0.18545.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[369]}, {checkpoints,[{369,10}]}, {name,<<"rebalance_369">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.68",11209}, [{username,"default"}, {password,[]}, {vbuckets,[369]}, {takeover,true}, {suffix,"369"}]} [rebalance:info,2012-11-02T18:53:16.921,ns_1@10.2.1.67:<0.18545.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:53:16.926,ns_1@10.2.1.67:<0.18545.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:53:16.991,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/369 [ns_server:info,2012-11-02T18:53:17.020,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:53:16.819882 PDT 3: TAP (Producer) eq_tapq:replication_building_369_'ns_1@10.2.1.68' - Backfill is completed with VBuckets 369, memcached<0.1519.0>: Fri Nov 2 18:53:16.819905 PDT 3: TAP (Producer) eq_tapq:replication_building_369_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 369 memcached<0.1519.0>: Fri Nov 2 18:53:16.910727 PDT 3: TAP (Producer) eq_tapq:replication_building_369_'ns_1@10.2.1.68' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:53:16.920858 PDT 3: TAP (Producer) eq_tapq:rebalance_369 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:16.920870 PDT 3: TAP (Producer) eq_tapq:rebalance_369 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:16.920876 PDT 3: TAP (Producer) eq_tapq:rebalance_369 - Sending TAP_VBUCKET_SET with vbucket 369 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:53:16.926271 PDT 3: TAP (Producer) eq_tapq:rebalance_369 - VBucket <369> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:53:16.926307 PDT 3: TAP (Producer) eq_tapq:rebalance_369 - Sending TAP_VBUCKET_SET with vbucket 369 and state "active" memcached<0.1519.0>: Fri Nov 2 18:53:16.926602 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:53:16.926624 PDT 3: TAP (Producer) eq_tapq:rebalance_369 - disconnected memcached<0.1519.0>: Fri Nov 2 18:53:16.926654 PDT 3: Schedule cleanup of "eq_tapq:rebalance_369" memcached<0.1519.0>: Fri Nov 2 18:53:16.926702 PDT 3: TAP (Producer) eq_tapq:rebalance_369 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:53:16.935587 PDT 3: TAP (Producer) eq_tapq:replication_building_369_'ns_1@10.2.1.68' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:53:16.948137 PDT 3: Deletion of vbucket 369 was completed. memcached<0.1519.0>: Fri Nov 2 18:53:16.957160 PDT 3: TAP (Producer) eq_tapq:replication_building_370_'ns_1@10.2.1.68' - Schedule the backfill for vbucket 370 memcached<0.1519.0>: Fri Nov 2 18:53:16.957198 PDT 3: TAP (Producer) eq_tapq:replication_building_370_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:16.957215 PDT 3: TAP (Producer) eq_tapq:replication_building_370_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:16.957225 PDT 3: TAP (Producer) eq_tapq:replication_building_370_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 370 memcached<0.1519.0>: Fri Nov 2 18:53:16.957328 PDT 3: Schedule cleanup of "eq_tapq:anon_114" memcached<0.1519.0>: Fri Nov 2 18:53:16.973256 PDT 3: TAP (Producer) eq_tapq:replication_building_369_'ns_1@10.2.1.68' - Clear the tap queues by force [ns_server:info,2012-11-02T18:53:17.861,ns_1@10.2.1.67:<0.18550.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.68",11209} vbucket 370 to state replica [rebalance:info,2012-11-02T18:53:17.862,ns_1@10.2.1.67:<0.18550.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[370]}, {checkpoints,[{370,10}]}, {name,<<"rebalance_370">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.68",11209}, [{username,"default"}, {password,[]}, {vbuckets,[370]}, {takeover,true}, {suffix,"370"}]} [rebalance:info,2012-11-02T18:53:17.864,ns_1@10.2.1.67:<0.18550.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:53:17.869,ns_1@10.2.1.67:<0.18550.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:53:17.916,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/370 [ns_server:info,2012-11-02T18:53:17.960,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:53:17.759790 PDT 3: TAP (Producer) eq_tapq:replication_building_370_'ns_1@10.2.1.68' - Backfill is completed with VBuckets 370, memcached<0.1519.0>: Fri Nov 2 18:53:17.759811 PDT 3: TAP (Producer) eq_tapq:replication_building_370_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 370 memcached<0.1519.0>: Fri Nov 2 18:53:17.857151 PDT 3: TAP (Producer) eq_tapq:replication_building_370_'ns_1@10.2.1.68' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:53:17.863116 PDT 3: TAP (Producer) eq_tapq:rebalance_370 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:17.863128 PDT 3: TAP (Producer) eq_tapq:rebalance_370 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:17.863133 PDT 3: TAP (Producer) eq_tapq:rebalance_370 - Sending TAP_VBUCKET_SET with vbucket 370 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:53:17.869351 PDT 3: TAP (Producer) eq_tapq:rebalance_370 - VBucket <370> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:53:17.869368 PDT 3: TAP (Producer) eq_tapq:rebalance_370 - Sending TAP_VBUCKET_SET with vbucket 370 and state "active" memcached<0.1519.0>: Fri Nov 2 18:53:17.869722 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:53:17.869745 PDT 3: TAP (Producer) eq_tapq:rebalance_370 - disconnected memcached<0.1519.0>: Fri Nov 2 18:53:17.869778 PDT 3: Schedule cleanup of "eq_tapq:rebalance_370" memcached<0.1519.0>: Fri Nov 2 18:53:17.869827 PDT 3: TAP (Producer) eq_tapq:rebalance_370 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:53:17.875880 PDT 3: TAP (Producer) eq_tapq:replication_building_370_'ns_1@10.2.1.68' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:53:17.885297 PDT 3: Deletion of vbucket 370 was completed. memcached<0.1519.0>: Fri Nov 2 18:53:17.893048 PDT 3: TAP (Producer) eq_tapq:replication_building_371_'ns_1@10.2.1.68' - Schedule the backfill for vbucket 371 memcached<0.1519.0>: Fri Nov 2 18:53:17.893073 PDT 3: TAP (Producer) eq_tapq:replication_building_371_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:17.893081 PDT 3: TAP (Producer) eq_tapq:replication_building_371_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:17.893087 PDT 3: TAP (Producer) eq_tapq:replication_building_371_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 371 memcached<0.1519.0>: Fri Nov 2 18:53:17.893149 PDT 3: Schedule cleanup of "eq_tapq:anon_115" memcached<0.1519.0>: Fri Nov 2 18:53:17.911578 PDT 3: TAP (Producer) eq_tapq:replication_building_370_'ns_1@10.2.1.68' - Clear the tap queues by force [ns_server:info,2012-11-02T18:53:18.743,ns_1@10.2.1.67:<0.18560.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.68",11209} vbucket 371 to state replica [rebalance:info,2012-11-02T18:53:18.745,ns_1@10.2.1.67:<0.18560.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[371]}, {checkpoints,[{371,10}]}, {name,<<"rebalance_371">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.68",11209}, [{username,"default"}, {password,[]}, {vbuckets,[371]}, {takeover,true}, {suffix,"371"}]} [rebalance:info,2012-11-02T18:53:18.746,ns_1@10.2.1.67:<0.18560.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:53:18.758,ns_1@10.2.1.67:<0.18560.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:53:18.811,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/371 [ns_server:info,2012-11-02T18:53:18.884,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:53:18.683888 PDT 3: TAP (Producer) eq_tapq:replication_building_371_'ns_1@10.2.1.68' - Backfill is completed with VBuckets 371, memcached<0.1519.0>: Fri Nov 2 18:53:18.683917 PDT 3: TAP (Producer) eq_tapq:replication_building_371_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 371 memcached<0.1519.0>: Fri Nov 2 18:53:18.739552 PDT 3: TAP (Producer) eq_tapq:replication_building_371_'ns_1@10.2.1.68' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:53:18.745717 PDT 3: TAP (Producer) eq_tapq:rebalance_371 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:18.745729 PDT 3: TAP (Producer) eq_tapq:rebalance_371 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:18.745734 PDT 3: TAP (Producer) eq_tapq:rebalance_371 - Sending TAP_VBUCKET_SET with vbucket 371 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:53:18.757505 PDT 3: TAP (Producer) eq_tapq:rebalance_371 - VBucket <371> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:53:18.757538 PDT 3: TAP (Producer) eq_tapq:rebalance_371 - Sending TAP_VBUCKET_SET with vbucket 371 and state "active" memcached<0.1519.0>: Fri Nov 2 18:53:18.758006 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:53:18.758064 PDT 3: TAP (Producer) eq_tapq:rebalance_371 - disconnected memcached<0.1519.0>: Fri Nov 2 18:53:18.758131 PDT 3: Schedule cleanup of "eq_tapq:rebalance_371" memcached<0.1519.0>: Fri Nov 2 18:53:18.758172 PDT 3: TAP (Producer) eq_tapq:rebalance_371 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:53:18.769821 PDT 3: TAP (Producer) eq_tapq:replication_building_371_'ns_1@10.2.1.68' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:53:18.777292 PDT 3: Deletion of vbucket 371 was completed. memcached<0.1519.0>: Fri Nov 2 18:53:18.795734 PDT 3: TAP (Producer) eq_tapq:replication_building_372_'ns_1@10.2.1.68' - Schedule the backfill for vbucket 372 memcached<0.1519.0>: Fri Nov 2 18:53:18.795777 PDT 3: TAP (Producer) eq_tapq:replication_building_372_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:18.795791 PDT 3: TAP (Producer) eq_tapq:replication_building_372_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:18.795801 PDT 3: TAP (Producer) eq_tapq:replication_building_372_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 372 memcached<0.1519.0>: Fri Nov 2 18:53:18.795875 PDT 3: Schedule cleanup of "eq_tapq:anon_116" memcached<0.1519.0>: Fri Nov 2 18:53:18.809659 PDT 3: TAP (Producer) eq_tapq:replication_building_371_'ns_1@10.2.1.68' - Clear the tap queues by force [ns_server:info,2012-11-02T18:53:19.866,ns_1@10.2.1.67:<0.18566.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.68",11209} vbucket 372 to state replica [rebalance:info,2012-11-02T18:53:19.869,ns_1@10.2.1.67:<0.18566.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[372]}, {checkpoints,[{372,10}]}, {name,<<"rebalance_372">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.68",11209}, [{username,"default"}, {password,[]}, {vbuckets,[372]}, {takeover,true}, {suffix,"372"}]} [rebalance:info,2012-11-02T18:53:19.876,ns_1@10.2.1.67:<0.18566.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:53:19.882,ns_1@10.2.1.67:<0.18566.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:53:19.933,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/372 [ns_server:info,2012-11-02T18:53:19.955,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:53:19.755306 PDT 3: TAP (Producer) eq_tapq:replication_building_372_'ns_1@10.2.1.68' - Backfill is completed with VBuckets 372, memcached<0.1519.0>: Fri Nov 2 18:53:19.755340 PDT 3: TAP (Producer) eq_tapq:replication_building_372_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 372 memcached<0.1519.0>: Fri Nov 2 18:53:19.860061 PDT 3: TAP (Producer) eq_tapq:replication_building_372_'ns_1@10.2.1.68' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:53:19.870017 PDT 3: TAP (Producer) eq_tapq:rebalance_372 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:19.870030 PDT 3: TAP (Producer) eq_tapq:rebalance_372 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:19.870035 PDT 3: TAP (Producer) eq_tapq:rebalance_372 - Sending TAP_VBUCKET_SET with vbucket 372 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:53:19.881162 PDT 3: TAP (Producer) eq_tapq:rebalance_372 - VBucket <372> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:53:19.881203 PDT 3: TAP (Producer) eq_tapq:rebalance_372 - Sending TAP_VBUCKET_SET with vbucket 372 and state "active" memcached<0.1519.0>: Fri Nov 2 18:53:19.881576 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:53:19.881621 PDT 3: TAP (Producer) eq_tapq:rebalance_372 - disconnected memcached<0.1519.0>: Fri Nov 2 18:53:19.881704 PDT 3: Schedule cleanup of "eq_tapq:rebalance_372" memcached<0.1519.0>: Fri Nov 2 18:53:19.881755 PDT 3: TAP (Producer) eq_tapq:rebalance_372 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:53:19.891679 PDT 3: TAP (Producer) eq_tapq:replication_building_372_'ns_1@10.2.1.68' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:53:19.903298 PDT 3: Deletion of vbucket 372 was completed. memcached<0.1519.0>: Fri Nov 2 18:53:19.912389 PDT 3: TAP (Producer) eq_tapq:replication_building_373_'ns_1@10.2.1.68' - Schedule the backfill for vbucket 373 memcached<0.1519.0>: Fri Nov 2 18:53:19.912429 PDT 3: TAP (Producer) eq_tapq:replication_building_373_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:19.912444 PDT 3: TAP (Producer) eq_tapq:replication_building_373_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:19.912454 PDT 3: TAP (Producer) eq_tapq:replication_building_373_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 373 memcached<0.1519.0>: Fri Nov 2 18:53:19.912527 PDT 3: Schedule cleanup of "eq_tapq:anon_117" memcached<0.1519.0>: Fri Nov 2 18:53:19.932424 PDT 3: TAP (Producer) eq_tapq:replication_building_372_'ns_1@10.2.1.68' - Clear the tap queues by force [ns_server:info,2012-11-02T18:53:20.879,ns_1@10.2.1.67:<0.18585.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.68",11209} vbucket 373 to state replica [rebalance:info,2012-11-02T18:53:20.891,ns_1@10.2.1.67:<0.18585.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[373]}, {checkpoints,[{373,10}]}, {name,<<"rebalance_373">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.68",11209}, [{username,"default"}, {password,[]}, {vbuckets,[373]}, {takeover,true}, {suffix,"373"}]} [rebalance:info,2012-11-02T18:53:20.893,ns_1@10.2.1.67:<0.18585.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:53:20.896,ns_1@10.2.1.67:<0.18585.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:53:20.947,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/373 [ns_server:info,2012-11-02T18:53:20.970,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:53:20.769788 PDT 3: TAP (Producer) eq_tapq:replication_building_373_'ns_1@10.2.1.68' - Backfill is completed with VBuckets 373, memcached<0.1519.0>: Fri Nov 2 18:53:20.769807 PDT 3: TAP (Producer) eq_tapq:replication_building_373_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 373 memcached<0.1519.0>: Fri Nov 2 18:53:20.874225 PDT 3: TAP (Producer) eq_tapq:replication_building_373_'ns_1@10.2.1.68' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:53:20.892359 PDT 3: TAP (Producer) eq_tapq:rebalance_373 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:20.892379 PDT 3: TAP (Producer) eq_tapq:rebalance_373 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:20.892387 PDT 3: TAP (Producer) eq_tapq:rebalance_373 - Sending TAP_VBUCKET_SET with vbucket 373 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:53:20.895949 PDT 3: TAP (Producer) eq_tapq:rebalance_373 - VBucket <373> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:53:20.895972 PDT 3: TAP (Producer) eq_tapq:rebalance_373 - Sending TAP_VBUCKET_SET with vbucket 373 and state "active" memcached<0.1519.0>: Fri Nov 2 18:53:20.896616 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:53:20.896667 PDT 3: TAP (Producer) eq_tapq:rebalance_373 - disconnected memcached<0.1519.0>: Fri Nov 2 18:53:20.908109 PDT 3: TAP (Producer) eq_tapq:replication_building_373_'ns_1@10.2.1.68' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:53:20.920161 PDT 3: Deletion of vbucket 373 was completed. memcached<0.1519.0>: Fri Nov 2 18:53:20.929142 PDT 3: TAP (Producer) eq_tapq:replication_building_374_'ns_1@10.2.1.68' - Schedule the backfill for vbucket 374 memcached<0.1519.0>: Fri Nov 2 18:53:20.929166 PDT 3: TAP (Producer) eq_tapq:replication_building_374_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:20.929174 PDT 3: TAP (Producer) eq_tapq:replication_building_374_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:20.929179 PDT 3: TAP (Producer) eq_tapq:replication_building_374_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 374 memcached<0.1519.0>: Fri Nov 2 18:53:20.929244 PDT 3: Schedule cleanup of "eq_tapq:anon_118" memcached<0.1519.0>: Fri Nov 2 18:53:20.929258 PDT 3: Schedule cleanup of "eq_tapq:rebalance_373" memcached<0.1519.0>: Fri Nov 2 18:53:20.939834 PDT 3: TAP (Producer) eq_tapq:replication_building_373_'ns_1@10.2.1.68' - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:53:20.939864 PDT 3: TAP (Producer) eq_tapq:rebalance_373 - Clear the tap queues by force [ns_server:info,2012-11-02T18:53:21.988,ns_1@10.2.1.67:<0.18590.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.68",11209} vbucket 374 to state replica [rebalance:info,2012-11-02T18:53:21.989,ns_1@10.2.1.67:<0.18590.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[374]}, {checkpoints,[{374,10}]}, {name,<<"rebalance_374">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.68",11209}, [{username,"default"}, {password,[]}, {vbuckets,[374]}, {takeover,true}, {suffix,"374"}]} [rebalance:info,2012-11-02T18:53:21.995,ns_1@10.2.1.67:<0.18590.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:53:22.000,ns_1@10.2.1.67:<0.18590.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:53:22.050,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/374 [ns_server:info,2012-11-02T18:53:22.120,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:53:21.919140 PDT 3: TAP (Producer) eq_tapq:replication_building_374_'ns_1@10.2.1.68' - Backfill is completed with VBuckets 374, memcached<0.1519.0>: Fri Nov 2 18:53:21.919162 PDT 3: TAP (Producer) eq_tapq:replication_building_374_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 374 memcached<0.1519.0>: Fri Nov 2 18:53:21.985907 PDT 3: TAP (Producer) eq_tapq:replication_building_374_'ns_1@10.2.1.68' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:53:21.989967 PDT 3: TAP (Producer) eq_tapq:rebalance_374 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:21.989979 PDT 3: TAP (Producer) eq_tapq:rebalance_374 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:21.989984 PDT 3: TAP (Producer) eq_tapq:rebalance_374 - Sending TAP_VBUCKET_SET with vbucket 374 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:53:21.999722 PDT 3: TAP (Producer) eq_tapq:rebalance_374 - VBucket <374> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:53:21.999740 PDT 3: TAP (Producer) eq_tapq:rebalance_374 - Sending TAP_VBUCKET_SET with vbucket 374 and state "active" memcached<0.1519.0>: Fri Nov 2 18:53:21.999981 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:53:22.000003 PDT 3: TAP (Producer) eq_tapq:rebalance_374 - disconnected memcached<0.1519.0>: Fri Nov 2 18:53:22.000034 PDT 3: Schedule cleanup of "eq_tapq:rebalance_374" memcached<0.1519.0>: Fri Nov 2 18:53:22.000103 PDT 3: TAP (Producer) eq_tapq:rebalance_374 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:53:22.007734 PDT 3: TAP (Producer) eq_tapq:replication_building_374_'ns_1@10.2.1.68' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:53:22.017764 PDT 3: Deletion of vbucket 374 was completed. memcached<0.1519.0>: Fri Nov 2 18:53:22.027942 PDT 3: TAP (Producer) eq_tapq:replication_building_375_'ns_1@10.2.1.68' - Schedule the backfill for vbucket 375 memcached<0.1519.0>: Fri Nov 2 18:53:22.027971 PDT 3: TAP (Producer) eq_tapq:replication_building_375_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:22.027981 PDT 3: TAP (Producer) eq_tapq:replication_building_375_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:22.027987 PDT 3: TAP (Producer) eq_tapq:replication_building_375_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 375 memcached<0.1519.0>: Fri Nov 2 18:53:22.028931 PDT 3: Schedule cleanup of "eq_tapq:anon_119" memcached<0.1519.0>: Fri Nov 2 18:53:22.041505 PDT 3: TAP (Producer) eq_tapq:replication_building_374_'ns_1@10.2.1.68' - Clear the tap queues by force [ns_server:info,2012-11-02T18:53:23.504,ns_1@10.2.1.67:<0.18601.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.68",11209} vbucket 375 to state replica [rebalance:info,2012-11-02T18:53:23.511,ns_1@10.2.1.67:<0.18601.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[375]}, {checkpoints,[{375,10}]}, {name,<<"rebalance_375">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.68",11209}, [{username,"default"}, {password,[]}, {vbuckets,[375]}, {takeover,true}, {suffix,"375"}]} [rebalance:info,2012-11-02T18:53:23.512,ns_1@10.2.1.67:<0.18601.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:53:23.526,ns_1@10.2.1.67:<0.18601.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:53:23.579,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/375 [ns_server:info,2012-11-02T18:53:23.667,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:53:23.467081 PDT 3: TAP (Producer) eq_tapq:replication_building_375_'ns_1@10.2.1.68' - Backfill is completed with VBuckets 375, memcached<0.1519.0>: Fri Nov 2 18:53:23.467112 PDT 3: TAP (Producer) eq_tapq:replication_building_375_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 375 memcached<0.1519.0>: Fri Nov 2 18:53:23.501299 PDT 3: TAP (Producer) eq_tapq:replication_building_375_'ns_1@10.2.1.68' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:53:23.511435 PDT 3: TAP (Producer) eq_tapq:rebalance_375 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:23.511448 PDT 3: TAP (Producer) eq_tapq:rebalance_375 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:23.511459 PDT 3: TAP (Producer) eq_tapq:rebalance_375 - Sending TAP_VBUCKET_SET with vbucket 375 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:53:23.524930 PDT 3: TAP (Producer) eq_tapq:rebalance_375 - VBucket <375> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:53:23.524959 PDT 3: TAP (Producer) eq_tapq:rebalance_375 - Sending TAP_VBUCKET_SET with vbucket 375 and state "active" memcached<0.1519.0>: Fri Nov 2 18:53:23.526655 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:53:23.526680 PDT 3: TAP (Producer) eq_tapq:rebalance_375 - disconnected memcached<0.1519.0>: Fri Nov 2 18:53:23.526711 PDT 3: Schedule cleanup of "eq_tapq:rebalance_375" memcached<0.1519.0>: Fri Nov 2 18:53:23.526782 PDT 3: TAP (Producer) eq_tapq:rebalance_375 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:53:23.534339 PDT 3: TAP (Producer) eq_tapq:replication_building_375_'ns_1@10.2.1.68' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:53:23.546080 PDT 3: Deletion of vbucket 375 was completed. memcached<0.1519.0>: Fri Nov 2 18:53:23.560166 PDT 3: TAP (Producer) eq_tapq:replication_building_376_'ns_1@10.2.1.68' - Schedule the backfill for vbucket 376 memcached<0.1519.0>: Fri Nov 2 18:53:23.560206 PDT 3: TAP (Producer) eq_tapq:replication_building_376_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:23.560220 PDT 3: TAP (Producer) eq_tapq:replication_building_376_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:23.560229 PDT 3: TAP (Producer) eq_tapq:replication_building_376_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 376 memcached<0.1519.0>: Fri Nov 2 18:53:23.560998 PDT 3: Schedule cleanup of "eq_tapq:anon_120" memcached<0.1519.0>: Fri Nov 2 18:53:23.573509 PDT 3: TAP (Producer) eq_tapq:replication_building_375_'ns_1@10.2.1.68' - Clear the tap queues by force [ns_server:info,2012-11-02T18:53:24.120,ns_1@10.2.1.67:ns_config_rep<0.1439.0>:ns_config_rep:do_pull:341]Pulling config from: 'ns_1@10.2.1.66' [ns_server:info,2012-11-02T18:53:24.648,ns_1@10.2.1.67:<0.18608.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.68",11209} vbucket 376 to state replica [rebalance:info,2012-11-02T18:53:24.649,ns_1@10.2.1.67:<0.18608.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[376]}, {checkpoints,[{376,10}]}, {name,<<"rebalance_376">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.68",11209}, [{username,"default"}, {password,[]}, {vbuckets,[376]}, {takeover,true}, {suffix,"376"}]} [rebalance:info,2012-11-02T18:53:24.650,ns_1@10.2.1.67:<0.18608.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:53:24.655,ns_1@10.2.1.67:<0.18608.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:53:24.713,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/376 [ns_server:info,2012-11-02T18:53:24.733,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:53:24.532836 PDT 3: TAP (Producer) eq_tapq:replication_building_376_'ns_1@10.2.1.68' - Backfill is completed with VBuckets 376, memcached<0.1519.0>: Fri Nov 2 18:53:24.532865 PDT 3: TAP (Producer) eq_tapq:replication_building_376_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 376 memcached<0.1519.0>: Fri Nov 2 18:53:24.623154 PDT 3: TAP (Producer) eq_tapq:replication_building_376_'ns_1@10.2.1.68' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:53:24.649709 PDT 3: TAP (Producer) eq_tapq:rebalance_376 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:24.649722 PDT 3: TAP (Producer) eq_tapq:rebalance_376 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:24.649727 PDT 3: TAP (Producer) eq_tapq:rebalance_376 - Sending TAP_VBUCKET_SET with vbucket 376 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:53:24.655564 PDT 3: TAP (Producer) eq_tapq:rebalance_376 - VBucket <376> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:53:24.655584 PDT 3: TAP (Producer) eq_tapq:rebalance_376 - Sending TAP_VBUCKET_SET with vbucket 376 and state "active" memcached<0.1519.0>: Fri Nov 2 18:53:24.655854 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:53:24.655877 PDT 3: TAP (Producer) eq_tapq:rebalance_376 - disconnected memcached<0.1519.0>: Fri Nov 2 18:53:24.655907 PDT 3: Schedule cleanup of "eq_tapq:rebalance_376" memcached<0.1519.0>: Fri Nov 2 18:53:24.656197 PDT 3: TAP (Producer) eq_tapq:rebalance_376 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:53:24.665347 PDT 3: TAP (Producer) eq_tapq:replication_building_376_'ns_1@10.2.1.68' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:53:24.676831 PDT 3: Deletion of vbucket 376 was completed. memcached<0.1519.0>: Fri Nov 2 18:53:24.688020 PDT 3: TAP (Producer) eq_tapq:replication_building_377_'ns_1@10.2.1.68' - Schedule the backfill for vbucket 377 memcached<0.1519.0>: Fri Nov 2 18:53:24.688044 PDT 3: TAP (Producer) eq_tapq:replication_building_377_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:24.688052 PDT 3: TAP (Producer) eq_tapq:replication_building_377_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:24.688056 PDT 3: TAP (Producer) eq_tapq:replication_building_377_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 377 memcached<0.1519.0>: Fri Nov 2 18:53:24.688085 PDT 3: Schedule cleanup of "eq_tapq:anon_121" memcached<0.1519.0>: Fri Nov 2 18:53:24.708947 PDT 3: TAP (Producer) eq_tapq:replication_building_376_'ns_1@10.2.1.68' - Clear the tap queues by force [ns_server:info,2012-11-02T18:53:25.665,ns_1@10.2.1.67:<0.18627.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.68",11209} vbucket 377 to state replica [rebalance:info,2012-11-02T18:53:25.667,ns_1@10.2.1.67:<0.18627.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[377]}, {checkpoints,[{377,10}]}, {name,<<"rebalance_377">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.68",11209}, [{username,"default"}, {password,[]}, {vbuckets,[377]}, {takeover,true}, {suffix,"377"}]} [rebalance:info,2012-11-02T18:53:25.675,ns_1@10.2.1.67:<0.18627.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:53:25.678,ns_1@10.2.1.67:<0.18627.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:53:25.727,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/377 [ns_server:info,2012-11-02T18:53:25.794,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:53:25.593339 PDT 3: TAP (Producer) eq_tapq:replication_building_377_'ns_1@10.2.1.68' - Backfill is completed with VBuckets 377, memcached<0.1519.0>: Fri Nov 2 18:53:25.593360 PDT 3: TAP (Producer) eq_tapq:replication_building_377_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 377 memcached<0.1519.0>: Fri Nov 2 18:53:25.655489 PDT 3: TAP (Producer) eq_tapq:replication_building_377_'ns_1@10.2.1.68' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:53:25.672522 PDT 3: TAP (Producer) eq_tapq:rebalance_377 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:25.672540 PDT 3: TAP (Producer) eq_tapq:rebalance_377 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:25.672552 PDT 3: TAP (Producer) eq_tapq:rebalance_377 - Sending TAP_VBUCKET_SET with vbucket 377 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:53:25.677968 PDT 3: TAP (Producer) eq_tapq:rebalance_377 - VBucket <377> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:53:25.677992 PDT 3: TAP (Producer) eq_tapq:rebalance_377 - Sending TAP_VBUCKET_SET with vbucket 377 and state "active" memcached<0.1519.0>: Fri Nov 2 18:53:25.678255 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:53:25.678283 PDT 3: TAP (Producer) eq_tapq:rebalance_377 - disconnected memcached<0.1519.0>: Fri Nov 2 18:53:25.689064 PDT 3: TAP (Producer) eq_tapq:replication_building_377_'ns_1@10.2.1.68' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:53:25.702674 PDT 3: Deletion of vbucket 377 was completed. memcached<0.1519.0>: Fri Nov 2 18:53:25.712817 PDT 3: TAP (Producer) eq_tapq:replication_building_378_'ns_1@10.2.1.68' - Schedule the backfill for vbucket 378 memcached<0.1519.0>: Fri Nov 2 18:53:25.712843 PDT 3: TAP (Producer) eq_tapq:replication_building_378_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:25.712851 PDT 3: TAP (Producer) eq_tapq:replication_building_378_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:25.712855 PDT 3: TAP (Producer) eq_tapq:replication_building_378_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 378 memcached<0.1519.0>: Fri Nov 2 18:53:25.712936 PDT 3: Schedule cleanup of "eq_tapq:anon_122" memcached<0.1519.0>: Fri Nov 2 18:53:25.712947 PDT 3: Schedule cleanup of "eq_tapq:rebalance_377" memcached<0.1519.0>: Fri Nov 2 18:53:25.726812 PDT 3: TAP (Producer) eq_tapq:replication_building_377_'ns_1@10.2.1.68' - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:53:25.726842 PDT 3: TAP (Producer) eq_tapq:rebalance_377 - Clear the tap queues by force [ns_server:info,2012-11-02T18:53:26.754,ns_1@10.2.1.67:<0.18638.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.68",11209} vbucket 378 to state replica [rebalance:info,2012-11-02T18:53:26.762,ns_1@10.2.1.67:<0.18638.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[378]}, {checkpoints,[{378,10}]}, {name,<<"rebalance_378">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.68",11209}, [{username,"default"}, {password,[]}, {vbuckets,[378]}, {takeover,true}, {suffix,"378"}]} [rebalance:info,2012-11-02T18:53:26.764,ns_1@10.2.1.67:<0.18638.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:53:26.768,ns_1@10.2.1.67:<0.18638.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:53:26.809,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/378 [ns_server:info,2012-11-02T18:53:26.918,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:53:26.717080 PDT 3: TAP (Producer) eq_tapq:replication_building_378_'ns_1@10.2.1.68' - Backfill is completed with VBuckets 378, memcached<0.1519.0>: Fri Nov 2 18:53:26.717108 PDT 3: TAP (Producer) eq_tapq:replication_building_378_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 378 memcached<0.1519.0>: Fri Nov 2 18:53:26.749820 PDT 3: TAP (Producer) eq_tapq:replication_building_378_'ns_1@10.2.1.68' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:53:26.763200 PDT 3: TAP (Producer) eq_tapq:rebalance_378 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:26.763213 PDT 3: TAP (Producer) eq_tapq:rebalance_378 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:26.763218 PDT 3: TAP (Producer) eq_tapq:rebalance_378 - Sending TAP_VBUCKET_SET with vbucket 378 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:53:26.767680 PDT 3: TAP (Producer) eq_tapq:rebalance_378 - VBucket <378> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:53:26.767711 PDT 3: TAP (Producer) eq_tapq:rebalance_378 - Sending TAP_VBUCKET_SET with vbucket 378 and state "active" memcached<0.1519.0>: Fri Nov 2 18:53:26.768338 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:53:26.768390 PDT 3: TAP (Producer) eq_tapq:rebalance_378 - disconnected memcached<0.1519.0>: Fri Nov 2 18:53:26.777164 PDT 3: TAP (Producer) eq_tapq:replication_building_378_'ns_1@10.2.1.68' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:53:26.785864 PDT 3: Deletion of vbucket 378 was completed. memcached<0.1519.0>: Fri Nov 2 18:53:26.791761 PDT 3: TAP (Producer) eq_tapq:replication_building_379_'ns_1@10.2.1.68' - Schedule the backfill for vbucket 379 memcached<0.1519.0>: Fri Nov 2 18:53:26.791783 PDT 3: TAP (Producer) eq_tapq:replication_building_379_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:26.791790 PDT 3: TAP (Producer) eq_tapq:replication_building_379_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:26.791795 PDT 3: TAP (Producer) eq_tapq:replication_building_379_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 379 memcached<0.1519.0>: Fri Nov 2 18:53:26.791856 PDT 3: Schedule cleanup of "eq_tapq:anon_123" memcached<0.1519.0>: Fri Nov 2 18:53:26.791869 PDT 3: Schedule cleanup of "eq_tapq:rebalance_378" memcached<0.1519.0>: Fri Nov 2 18:53:26.806991 PDT 3: TAP (Producer) eq_tapq:replication_building_378_'ns_1@10.2.1.68' - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:53:26.807023 PDT 3: TAP (Producer) eq_tapq:rebalance_378 - Clear the tap queues by force [ns_server:info,2012-11-02T18:53:27.967,ns_1@10.2.1.67:<0.18643.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.68",11209} vbucket 379 to state replica [rebalance:info,2012-11-02T18:53:27.968,ns_1@10.2.1.67:<0.18643.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[379]}, {checkpoints,[{379,10}]}, {name,<<"rebalance_379">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.68",11209}, [{username,"default"}, {password,[]}, {vbuckets,[379]}, {takeover,true}, {suffix,"379"}]} [rebalance:info,2012-11-02T18:53:27.969,ns_1@10.2.1.67:<0.18643.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:53:27.977,ns_1@10.2.1.67:<0.18643.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:53:28.022,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/379 [ns_server:info,2012-11-02T18:53:28.135,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:53:27.934191 PDT 3: TAP (Producer) eq_tapq:replication_building_379_'ns_1@10.2.1.68' - Backfill is completed with VBuckets 379, memcached<0.1519.0>: Fri Nov 2 18:53:27.934214 PDT 3: TAP (Producer) eq_tapq:replication_building_379_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 379 memcached<0.1519.0>: Fri Nov 2 18:53:27.955668 PDT 3: TAP (Producer) eq_tapq:replication_building_379_'ns_1@10.2.1.68' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:53:27.969076 PDT 3: TAP (Producer) eq_tapq:rebalance_379 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:27.969088 PDT 3: TAP (Producer) eq_tapq:rebalance_379 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:27.969093 PDT 3: TAP (Producer) eq_tapq:rebalance_379 - Sending TAP_VBUCKET_SET with vbucket 379 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:53:27.976643 PDT 3: TAP (Producer) eq_tapq:rebalance_379 - VBucket <379> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:53:27.976669 PDT 3: TAP (Producer) eq_tapq:rebalance_379 - Sending TAP_VBUCKET_SET with vbucket 379 and state "active" memcached<0.1519.0>: Fri Nov 2 18:53:27.976955 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:53:27.976991 PDT 3: TAP (Producer) eq_tapq:rebalance_379 - disconnected memcached<0.1519.0>: Fri Nov 2 18:53:27.977099 PDT 3: Schedule cleanup of "eq_tapq:rebalance_379" memcached<0.1519.0>: Fri Nov 2 18:53:27.977137 PDT 3: TAP (Producer) eq_tapq:rebalance_379 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:53:27.986296 PDT 3: TAP (Producer) eq_tapq:replication_building_379_'ns_1@10.2.1.68' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:53:27.996548 PDT 3: Deletion of vbucket 379 was completed. memcached<0.1519.0>: Fri Nov 2 18:53:28.009703 PDT 3: TAP (Producer) eq_tapq:replication_building_380_'ns_1@10.2.1.68' - Schedule the backfill for vbucket 380 memcached<0.1519.0>: Fri Nov 2 18:53:28.009735 PDT 3: TAP (Producer) eq_tapq:replication_building_380_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:28.009743 PDT 3: TAP (Producer) eq_tapq:replication_building_380_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:28.009748 PDT 3: TAP (Producer) eq_tapq:replication_building_380_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 380 memcached<0.1519.0>: Fri Nov 2 18:53:28.009772 PDT 3: Schedule cleanup of "eq_tapq:anon_124" memcached<0.1519.0>: Fri Nov 2 18:53:28.021581 PDT 3: TAP (Producer) eq_tapq:replication_building_379_'ns_1@10.2.1.68' - Clear the tap queues by force [ns_server:info,2012-11-02T18:53:29.077,ns_1@10.2.1.67:<0.18654.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.68",11209} vbucket 380 to state replica [rebalance:info,2012-11-02T18:53:29.078,ns_1@10.2.1.67:<0.18654.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[380]}, {checkpoints,[{380,10}]}, {name,<<"rebalance_380">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.68",11209}, [{username,"default"}, {password,[]}, {vbuckets,[380]}, {takeover,true}, {suffix,"380"}]} [rebalance:info,2012-11-02T18:53:29.079,ns_1@10.2.1.67:<0.18654.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:53:29.082,ns_1@10.2.1.67:<0.18654.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:53:29.123,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/380 [ns_server:info,2012-11-02T18:53:29.257,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:53:29.057217 PDT 3: TAP (Producer) eq_tapq:replication_building_380_'ns_1@10.2.1.68' - Backfill is completed with VBuckets 380, memcached<0.1519.0>: Fri Nov 2 18:53:29.057240 PDT 3: TAP (Producer) eq_tapq:replication_building_380_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 380 memcached<0.1519.0>: Fri Nov 2 18:53:29.073419 PDT 3: TAP (Producer) eq_tapq:replication_building_380_'ns_1@10.2.1.68' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:53:29.078275 PDT 3: TAP (Producer) eq_tapq:rebalance_380 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:29.078286 PDT 3: TAP (Producer) eq_tapq:rebalance_380 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:29.078292 PDT 3: TAP (Producer) eq_tapq:rebalance_380 - Sending TAP_VBUCKET_SET with vbucket 380 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:53:29.081816 PDT 3: TAP (Producer) eq_tapq:rebalance_380 - VBucket <380> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:53:29.081835 PDT 3: TAP (Producer) eq_tapq:rebalance_380 - Sending TAP_VBUCKET_SET with vbucket 380 and state "active" memcached<0.1519.0>: Fri Nov 2 18:53:29.082206 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:53:29.082249 PDT 3: TAP (Producer) eq_tapq:rebalance_380 - disconnected memcached<0.1519.0>: Fri Nov 2 18:53:29.082314 PDT 3: Schedule cleanup of "eq_tapq:rebalance_380" memcached<0.1519.0>: Fri Nov 2 18:53:29.082380 PDT 3: TAP (Producer) eq_tapq:rebalance_380 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:53:29.092812 PDT 3: TAP (Producer) eq_tapq:replication_building_380_'ns_1@10.2.1.68' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:53:29.101208 PDT 3: Deletion of vbucket 380 was completed. memcached<0.1519.0>: Fri Nov 2 18:53:29.111056 PDT 3: TAP (Producer) eq_tapq:replication_building_381_'ns_1@10.2.1.68' - Schedule the backfill for vbucket 381 memcached<0.1519.0>: Fri Nov 2 18:53:29.111094 PDT 3: TAP (Producer) eq_tapq:replication_building_381_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:29.111108 PDT 3: TAP (Producer) eq_tapq:replication_building_381_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:29.111120 PDT 3: TAP (Producer) eq_tapq:replication_building_381_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 381 memcached<0.1519.0>: Fri Nov 2 18:53:29.111224 PDT 3: Schedule cleanup of "eq_tapq:anon_125" memcached<0.1519.0>: Fri Nov 2 18:53:29.135077 PDT 3: TAP (Producer) eq_tapq:replication_building_380_'ns_1@10.2.1.68' - Clear the tap queues by force [ns_server:info,2012-11-02T18:53:30.169,ns_1@10.2.1.67:<0.18659.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.68",11209} vbucket 381 to state replica [rebalance:info,2012-11-02T18:53:30.171,ns_1@10.2.1.67:<0.18659.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[381]}, {checkpoints,[{381,10}]}, {name,<<"rebalance_381">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.68",11209}, [{username,"default"}, {password,[]}, {vbuckets,[381]}, {takeover,true}, {suffix,"381"}]} [rebalance:info,2012-11-02T18:53:30.173,ns_1@10.2.1.67:<0.18659.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:53:30.177,ns_1@10.2.1.67:<0.18659.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:53:30.218,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/381 [ns_server:info,2012-11-02T18:53:30.274,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:53:30.073987 PDT 3: TAP (Producer) eq_tapq:replication_building_381_'ns_1@10.2.1.68' - Backfill is completed with VBuckets 381, memcached<0.1519.0>: Fri Nov 2 18:53:30.074009 PDT 3: TAP (Producer) eq_tapq:replication_building_381_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 381 memcached<0.1519.0>: Fri Nov 2 18:53:30.165691 PDT 3: TAP (Producer) eq_tapq:replication_building_381_'ns_1@10.2.1.68' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:53:30.171747 PDT 3: TAP (Producer) eq_tapq:rebalance_381 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:30.171769 PDT 3: TAP (Producer) eq_tapq:rebalance_381 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:30.171780 PDT 3: TAP (Producer) eq_tapq:rebalance_381 - Sending TAP_VBUCKET_SET with vbucket 381 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:53:30.176394 PDT 3: TAP (Producer) eq_tapq:rebalance_381 - VBucket <381> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:53:30.176421 PDT 3: TAP (Producer) eq_tapq:rebalance_381 - Sending TAP_VBUCKET_SET with vbucket 381 and state "active" memcached<0.1519.0>: Fri Nov 2 18:53:30.177114 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:53:30.177161 PDT 3: TAP (Producer) eq_tapq:rebalance_381 - disconnected memcached<0.1519.0>: Fri Nov 2 18:53:30.177227 PDT 3: Schedule cleanup of "eq_tapq:rebalance_381" memcached<0.1519.0>: Fri Nov 2 18:53:30.177846 PDT 3: TAP (Producer) eq_tapq:rebalance_381 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:53:30.185709 PDT 3: TAP (Producer) eq_tapq:replication_building_381_'ns_1@10.2.1.68' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:53:30.199163 PDT 3: Deletion of vbucket 381 was completed. memcached<0.1519.0>: Fri Nov 2 18:53:30.208465 PDT 3: TAP (Producer) eq_tapq:replication_building_382_'ns_1@10.2.1.68' - Schedule the backfill for vbucket 382 memcached<0.1519.0>: Fri Nov 2 18:53:30.208504 PDT 3: TAP (Producer) eq_tapq:replication_building_382_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:30.208518 PDT 3: TAP (Producer) eq_tapq:replication_building_382_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:30.208529 PDT 3: TAP (Producer) eq_tapq:replication_building_382_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 382 memcached<0.1519.0>: Fri Nov 2 18:53:30.208645 PDT 3: Schedule cleanup of "eq_tapq:anon_126" memcached<0.1519.0>: Fri Nov 2 18:53:30.234379 PDT 3: TAP (Producer) eq_tapq:replication_building_381_'ns_1@10.2.1.68' - Clear the tap queues by force [ns_server:info,2012-11-02T18:53:31.163,ns_1@10.2.1.67:<0.18678.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.68",11209} vbucket 382 to state replica [rebalance:info,2012-11-02T18:53:31.167,ns_1@10.2.1.67:<0.18678.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[382]}, {checkpoints,[{382,10}]}, {name,<<"rebalance_382">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.68",11209}, [{username,"default"}, {password,[]}, {vbuckets,[382]}, {takeover,true}, {suffix,"382"}]} [rebalance:info,2012-11-02T18:53:31.169,ns_1@10.2.1.67:<0.18678.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:53:31.176,ns_1@10.2.1.67:<0.18678.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:53:31.217,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/382 [ns_server:info,2012-11-02T18:53:31.223,ns_1@10.2.1.67:<0.18683.0>:compaction_daemon:try_to_cleanup_indexes:439]Cleaning up indexes for bucket `default` [ns_server:info,2012-11-02T18:53:31.224,ns_1@10.2.1.67:<0.18683.0>:compaction_daemon:spawn_bucket_compactor:404]Compacting bucket default with config: [{parallel_db_and_view_compaction,false}, {database_fragmentation_threshold,{30,undefined}}, {view_fragmentation_threshold,{100,undefined}}] [ns_server:info,2012-11-02T18:53:31.226,ns_1@10.2.1.67:<0.18688.0>:compaction_daemon:spawn_view_index_compactor:711]Compacting indexes for default/_design/A/main [ns_server:info,2012-11-02T18:53:31.285,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:53:31.084793 PDT 3: TAP (Producer) eq_tapq:replication_building_382_'ns_1@10.2.1.68' - Backfill is completed with VBuckets 382, memcached<0.1519.0>: Fri Nov 2 18:53:31.084812 PDT 3: TAP (Producer) eq_tapq:replication_building_382_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 382 memcached<0.1519.0>: Fri Nov 2 18:53:31.157823 PDT 3: TAP (Producer) eq_tapq:replication_building_382_'ns_1@10.2.1.68' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:53:31.167874 PDT 3: TAP (Producer) eq_tapq:rebalance_382 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:31.167897 PDT 3: TAP (Producer) eq_tapq:rebalance_382 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:31.167907 PDT 3: TAP (Producer) eq_tapq:rebalance_382 - Sending TAP_VBUCKET_SET with vbucket 382 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:53:31.174170 PDT 3: TAP (Producer) eq_tapq:rebalance_382 - VBucket <382> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:53:31.174197 PDT 3: TAP (Producer) eq_tapq:rebalance_382 - Sending TAP_VBUCKET_SET with vbucket 382 and state "active" memcached<0.1519.0>: Fri Nov 2 18:53:31.176453 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:53:31.176503 PDT 3: TAP (Producer) eq_tapq:rebalance_382 - disconnected memcached<0.1519.0>: Fri Nov 2 18:53:31.176569 PDT 3: Schedule cleanup of "eq_tapq:rebalance_382" memcached<0.1519.0>: Fri Nov 2 18:53:31.176998 PDT 3: TAP (Producer) eq_tapq:rebalance_382 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:53:31.188916 PDT 3: TAP (Producer) eq_tapq:replication_building_382_'ns_1@10.2.1.68' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:53:31.200915 PDT 3: Deletion of vbucket 382 was completed. memcached<0.1519.0>: Fri Nov 2 18:53:31.210444 PDT 3: TAP (Producer) eq_tapq:replication_building_383_'ns_1@10.2.1.68' - Schedule the backfill for vbucket 383 memcached<0.1519.0>: Fri Nov 2 18:53:31.210482 PDT 3: TAP (Producer) eq_tapq:replication_building_383_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:31.210496 PDT 3: TAP (Producer) eq_tapq:replication_building_383_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:31.210506 PDT 3: TAP (Producer) eq_tapq:replication_building_383_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 383 memcached<0.1519.0>: Fri Nov 2 18:53:31.210609 PDT 3: Schedule cleanup of "eq_tapq:anon_127" memcached<0.1519.0>: Fri Nov 2 18:53:31.236781 PDT 3: TAP (Producer) eq_tapq:replication_building_382_'ns_1@10.2.1.68' - Clear the tap queues by force [ns_server:info,2012-11-02T18:53:31.662,ns_1@10.2.1.67:<0.18688.0>:compaction_daemon:spawn_view_index_compactor:727]Finished compacting indexes for default/_design/A/main [ns_server:info,2012-11-02T18:53:32.226,ns_1@10.2.1.67:<0.18700.0>:ebucketmigrator_srv:init:479]Setting {"10.2.1.68",11209} vbucket 383 to state replica [rebalance:info,2012-11-02T18:53:32.229,ns_1@10.2.1.67:<0.18700.0>:ebucketmigrator_srv:init:538]Starting tap stream: [{vbuckets,[383]}, {checkpoints,[{383,10}]}, {name,<<"rebalance_383">>}, {takeover,true}] {{"10.2.1.67",11209}, {"10.2.1.68",11209}, [{username,"default"}, {password,[]}, {vbuckets,[383]}, {takeover,true}, {suffix,"383"}]} [rebalance:info,2012-11-02T18:53:32.231,ns_1@10.2.1.67:<0.18700.0>:ebucketmigrator_srv:process_upstream:920]TAP stream is not doing backfill [rebalance:info,2012-11-02T18:53:32.238,ns_1@10.2.1.67:<0.18700.0>:ebucketmigrator_srv:terminate:639]Skipping close ack for successfull takover [ns_server:info,2012-11-02T18:53:32.274,ns_1@10.2.1.67:janitor_agent-default<0.1685.0>:janitor_agent:handle_info:676]Undoing temporary vbucket states caused by rebalance [ns_server:info,2012-11-02T18:53:32.284,ns_1@10.2.1.67:<0.1677.0>:mc_connection:do_delete_vbucket:134]Deleting vbucket: default/383 [ns_server:info,2012-11-02T18:53:32.389,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:53:32.189333 PDT 3: TAP (Producer) eq_tapq:replication_building_383_'ns_1@10.2.1.68' - Backfill is completed with VBuckets 383, memcached<0.1519.0>: Fri Nov 2 18:53:32.189357 PDT 3: TAP (Producer) eq_tapq:replication_building_383_'ns_1@10.2.1.68' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 383 memcached<0.1519.0>: Fri Nov 2 18:53:32.221161 PDT 3: TAP (Producer) eq_tapq:replication_building_383_'ns_1@10.2.1.68' - disconnected, keep alive for 300 seconds memcached<0.1519.0>: Fri Nov 2 18:53:32.229767 PDT 3: TAP (Producer) eq_tapq:rebalance_383 - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:32.229788 PDT 3: TAP (Producer) eq_tapq:rebalance_383 - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0 memcached<0.1519.0>: Fri Nov 2 18:53:32.229799 PDT 3: TAP (Producer) eq_tapq:rebalance_383 - Sending TAP_VBUCKET_SET with vbucket 383 and state "pending" memcached<0.1519.0>: Fri Nov 2 18:53:32.236862 PDT 3: TAP (Producer) eq_tapq:rebalance_383 - VBucket <383> is going dead to complete vbucket takeover. memcached<0.1519.0>: Fri Nov 2 18:53:32.236891 PDT 3: TAP (Producer) eq_tapq:rebalance_383 - Sending TAP_VBUCKET_SET with vbucket 383 and state "active" memcached<0.1519.0>: Fri Nov 2 18:53:32.237664 PDT 3: TAP takeover is completed. Disconnecting tap stream memcached<0.1519.0>: Fri Nov 2 18:53:32.237707 PDT 3: TAP (Producer) eq_tapq:rebalance_383 - disconnected memcached<0.1519.0>: Fri Nov 2 18:53:32.237771 PDT 3: Schedule cleanup of "eq_tapq:rebalance_383" memcached<0.1519.0>: Fri Nov 2 18:53:32.238404 PDT 3: TAP (Producer) eq_tapq:rebalance_383 - Clear the tap queues by force memcached<0.1519.0>: Fri Nov 2 18:53:32.246469 PDT 3: TAP (Producer) eq_tapq:replication_building_383_'ns_1@10.2.1.68' - Connection is closed by force. memcached<0.1519.0>: Fri Nov 2 18:53:32.268912 PDT 3: Deletion of vbucket 383 was completed. [ns_server:info,2012-11-02T18:53:33.074,ns_1@10.2.1.67:ns_config_rep<0.1439.0>:ns_config_rep:do_pull:341]Pulling config from: 'ns_1@10.2.1.66' [ns_server:info,2012-11-02T18:53:33.439,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:53:33.239221 PDT 3: Schedule cleanup of "eq_tapq:anon_128" memcached<0.1519.0>: Fri Nov 2 18:53:33.239391 PDT 3: TAP (Producer) eq_tapq:replication_building_383_'ns_1@10.2.1.68' - Clear the tap queues by force [user:info,2012-11-02T18:53:42.282,ns_1@10.2.1.67:ns_memcached-default<0.1676.0>:ns_memcached:terminate:661]Shutting down bucket "default" on 'ns_1@10.2.1.67' for deletion [user:info,2012-11-02T18:53:42.289,ns_1@10.2.1.67:ns_cluster<0.614.0>:ns_cluster:handle_cast:137]Node 'ns_1@10.2.1.67' is leaving cluster. [ns_server:info,2012-11-02T18:53:42.289,ns_1@10.2.1.67:<0.18754.0>:diag_handler:log_all_tap_and_checkpoint_stats:127]logging tap & checkpoint stats [ns_server:info,2012-11-02T18:53:42.292,ns_1@10.2.1.67:<0.18754.0>:diag_handler:log_all_tap_and_checkpoint_stats:131]end of logging tap & checkpoint stats [error_logger:error,2012-11-02T18:53:42.294,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,ns_bucket_sup} Context: shutdown_error Reason: normal Offender: [{pid,<0.1528.0>}, {name,buckets_observing_subscription}, {mfargs,{ns_bucket_sup,subscribe_on_config_events,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:error,2012-11-02T18:53:42.295,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,ns_server_sup} Context: shutdown_error Reason: {shutdown, {gen_server,call, [ns_bucket_sup, {delete_child,{per_bucket_sup,"default"}}, infinity]}} Offender: [{pid,<0.1523.0>}, {name,ns_bucket_worker}, {mfargs,{work_queue,start_link,[ns_bucket_worker]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [ns_server:info,2012-11-02T18:53:42.489,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:log:171]memcached<0.1519.0>: Fri Nov 2 18:53:42.283116 PDT 3: Shutting down tap connections! memcached<0.1519.0>: EOL on stdin. Initiating shutdown [ns_server:error,2012-11-02T18:53:42.490,ns_1@10.2.1.67:ns_heart<0.1459.0>:ns_heart:grab_samples_loading_tasks:298]Failed to grab samples loader tasks: {exit, {noproc, {gen_server,call, [samples_loader_tasks,get_tasks, 2000]}}, [{gen_server,call,3}, {ns_heart,grab_samples_loading_tasks,0}, {ns_heart,current_status,0}, {ns_heart,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} [ns_server:info,2012-11-02T18:53:43.423,ns_1@10.2.1.67:<0.1677.0>:mc_connection:run_loop:201]mccouch connection was normally closed [ns_server:info,2012-11-02T18:53:43.423,ns_1@10.2.1.67:ns_port_memcached<0.1519.0>:ns_port_server:handle_info:104]Port server memcached exited with status 0 [ns_server:info,2012-11-02T18:53:43.424,ns_1@10.2.1.67:<0.1517.0>:ns_port_server:handle_info:104]Port server moxi exited with status 0 [ns_server:info,2012-11-02T18:53:43.424,ns_1@10.2.1.67:<0.1517.0>:ns_port_server:log:171]moxi<0.1517.0>: EOL on stdin. Exiting [error_logger:info,2012-11-02T18:53:43.432,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================INFO REPORT========================= application: mnesia exited: stopped type: temporary [error_logger:info,2012-11-02T18:53:43.436,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mnesia_sup} started: [{pid,<0.18774.0>}, {name,mnesia_event}, {mfargs,{mnesia_sup,start_event,[]}}, {restart_type,permanent}, {shutdown,30000}, {child_type,worker}] [error_logger:info,2012-11-02T18:53:43.436,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mnesia_kernel_sup} started: [{pid,<0.18776.0>}, {name,mnesia_monitor}, {mfargs,{mnesia_monitor,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] [error_logger:info,2012-11-02T18:53:43.436,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mnesia_kernel_sup} started: [{pid,<0.18777.0>}, {name,mnesia_subscr}, {mfargs,{mnesia_subscr,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] [error_logger:info,2012-11-02T18:53:43.436,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mnesia_kernel_sup} started: [{pid,<0.18778.0>}, {name,mnesia_locker}, {mfargs,{mnesia_locker,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] [error_logger:info,2012-11-02T18:53:43.437,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mnesia_kernel_sup} started: [{pid,<0.18779.0>}, {name,mnesia_recover}, {mfargs,{mnesia_recover,start,[]}}, {restart_type,permanent}, {shutdown,180000}, {child_type,worker}] [error_logger:info,2012-11-02T18:53:43.437,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mnesia_kernel_sup} started: [{pid,<0.18780.0>}, {name,mnesia_tm}, {mfargs,{mnesia_tm,start,[]}}, {restart_type,permanent}, {shutdown,30000}, {child_type,worker}] [error_logger:info,2012-11-02T18:53:43.437,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mnesia_kernel_sup} started: [{pid,<0.18781.0>}, {name,mnesia_checkpoint_sup}, {mfargs,{mnesia_checkpoint_sup,start,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:53:43.437,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mnesia_kernel_sup} started: [{pid,<0.18782.0>}, {name,mnesia_snmp_sup}, {mfargs,{mnesia_snmp_sup,start,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:53:43.437,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mnesia_kernel_sup} started: [{pid,<0.18783.0>}, {name,mnesia_controller}, {mfargs,{mnesia_controller,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] [error_logger:info,2012-11-02T18:53:43.437,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mnesia_kernel_sup} started: [{pid,<0.18784.0>}, {name,mnesia_late_loader}, {mfargs,{mnesia_late_loader,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] [error_logger:info,2012-11-02T18:53:43.437,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mnesia_sup} started: [{pid,<0.18775.0>}, {name,mnesia_kernel_sup}, {mfargs,{mnesia_kernel_sup,start,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:53:43.437,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= application: mnesia started_at: 'ns_1@10.2.1.67' [ns_server:info,2012-11-02T18:53:43.444,ns_1@10.2.1.67:ns_config<0.703.0>:ns_config:load_config:674]Loading static config from "/opt/couchbase/etc/couchbase/config" [ns_server:info,2012-11-02T18:53:43.444,ns_1@10.2.1.67:ns_config<0.703.0>:ns_config:load_config:688]Loading dynamic config from "/opt/couchbase/var/lib/couchbase/config/config.dat" [ns_server:info,2012-11-02T18:53:43.445,ns_1@10.2.1.67:ns_config<0.703.0>:ns_config:load_config:706]Here's full dynamic config we loaded + static & default config: [{auto_failover_cfg,[{enabled,false},{timeout,30},{max_nodes,1},{count,0}]}, {replication,[{enabled,true}]}, {email_alerts, [{recipients,["root@localhost"]}, {sender,"couchbase@localhost"}, {enabled,false}, {email_server, [{user,[]},{pass,[]},{host,"localhost"},{port,25},{encrypt,false}]}, {alerts, [auto_failover_node,auto_failover_maximum_reached, auto_failover_other_nodes_down,auto_failover_cluster_too_small,ip, disk,overhead,ep_oom_errors,ep_item_commit_failed]}]}, {{node,'ns_1@10.2.1.67',ns_log}, [{filename,"/opt/couchbase/var/lib/couchbase/data/ns_log"}]}, {{node,'ns_1@10.2.1.67',port_servers}, [{moxi,"/opt/couchbase/bin/moxi", ["-Z", {"port_listen=~B,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", [port]}, "-z", {"url=http://127.0.0.1:~B/pools/default/saslBucketsStreaming", [{misc,this_node_rest_port,[]}]}, "-p","0","-Y","y","-O","stderr", {"~s",[verbosity]}], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR",{"~s",[{ns_moxi_sup,rest_user,[]}]}}, {"MOXI_SASL_PLAIN_PWD",{"~s",[{ns_moxi_sup,rest_pass,[]}]}}]}, use_stdio,exit_status,port_server_send_eol,stderr_to_stdout,stream]}, {memcached,"/opt/couchbase/bin/memcached", ["-X","/opt/couchbase/lib/memcached/stdin_term_handler.so","-X", {"/opt/couchbase/lib/memcached/file_logger.so,cyclesize=~B;sleeptime=~B;filename=~s/~s", [log_cyclesize,log_sleeptime,log_path,log_prefix]}, "-l", {"0.0.0.0:~B,0.0.0.0:~B:1000",[port,dedicated_port]}, "-p", {"~B",[port]}, "-E","/opt/couchbase/lib/memcached/bucket_engine.so","-B","binary", "-r","-c","10000","-e", {"admin=~s;default_bucket_name=default;auto_create=false", [admin_user]}, {"~s",[verbosity]}], [{env, [{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE",{"~s",[{isasl,path}]}}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,exit_status,port_server_send_eol,stream]}]}, {{node,'ns_1@10.2.1.67',moxi},[{port,11211},{verbosity,[]}]}, {buckets,[{configs,[]}]}, {memory_quota,31074}, {{node,'ns_1@10.2.1.67',memcached}, [{'_vclock',[{'_',{1,0}}]}, {port,11210}, {mccouch_port,11213}, {dedicated_port,11209}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine,"/opt/couchbase/lib/memcached/bucket_engine.so"}, {engines, [{membase, [{engine,"/opt/couchbase/lib/memcached/ep.so"}, {static_config_string, "vb0=false;waitforwarmup=false;failpartialwarmup=false"}]}, {memcached, [{engine,"/opt/couchbase/lib/memcached/default_engine.so"}, {static_config_string,"vb0=true"}]}]}, {log_path,"/opt/couchbase/var/lib/couchbase/logs"}, {log_prefix,"memcached.log"}, {log_generations,10}, {log_cyclesize,104857600}, {log_sleeptime,19}, {log_rotation_period,39003}, {verbosity,[]}]}, {{node,'ns_1@10.2.1.67',isasl}, [{path,"/opt/couchbase/var/lib/couchbase/data/isasl.pw"}]}, {remote_clusters,[]}, {rest_creds,[{creds,[]}]}, {{node,'ns_1@10.2.1.67',capi_port},8092}, {{node,'ns_1@10.2.1.67',rest},[{port,8091},{port_meta,global}]}, {{couchdb,max_parallel_replica_indexers},2}, {{couchdb,max_parallel_indexers},4}, {rest,[{port,8091}]}, {{node,'ns_1@10.2.1.67',membership},active}, {nodes_wanted,['ns_1@10.2.1.67']}, {{node,'ns_1@10.2.1.67',compaction_daemon}, [{check_interval,30},{min_file_size,131072}]}, {fast_warmup, [{fast_warmup_enabled,true}, {min_memory_threshold,10}, {min_items_threshold,10}]}, {set_view_update_daemon, [{update_interval,5000}, {update_min_changes,5000}, {replica_update_min_changes,5000}]}, {autocompaction, [{database_fragmentation_threshold,{30,undefined}}, {view_fragmentation_threshold,{30,undefined}}]}, {max_bucket_count,10}, {index_aware_rebalance_disabled,false}, {xdcr_capi_checkpoint_timeout,10}, {xdcr_failure_restart_interval,30}, {xdcr_doc_batch_size_kb,512}, {xdcr_checkpoint_interval,1800}] [ns_server:info,2012-11-02T18:53:43.446,ns_1@10.2.1.67:ns_config<0.703.0>:ns_config_default:upgrade_config_from_1_7_to_1_7_1:296]Upgrading config from 1.7 to 1.7.1 [ns_server:info,2012-11-02T18:53:43.447,ns_1@10.2.1.67:ns_config<0.703.0>:ns_config_default:upgrade_config_from_1_7_1_to_1_7_2:307]Upgrading config from 1.7.1 to 1.7.2 [ns_server:info,2012-11-02T18:53:43.448,ns_1@10.2.1.67:ns_config<0.703.0>:ns_config_default:upgrade_config_from_1_7_2_to_1_8_0:361]Upgrading config from 1.7.2 to 1.8.0 [ns_server:info,2012-11-02T18:53:43.449,ns_1@10.2.1.67:ns_config<0.703.0>:ns_config_default:upgrade_config_from_1_8_0_to_1_8_1:398]Upgrading config from 1.8.0 to 1.8.1 [ns_server:info,2012-11-02T18:53:43.450,ns_1@10.2.1.67:ns_config<0.703.0>:ns_config_default:upgrade_config_from_1_8_1_to_2_0:427]Upgrading config from 1.8.1 to 2.0 [user:info,2012-11-02T18:53:43.470,ns_1@10.2.1.67:ns_cookie_manager<0.613.0>:ns_cookie_manager:do_cookie_sync:125]Node 'ns_1@10.2.1.67' inherited otp cookie omqsxfcmmozneqtc from cluster [error_logger:info,2012-11-02T18:53:44.476,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.18809.0>}, {name,ns_log}, {mfargs,{ns_log,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:53:44.477,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.18810.0>}, {name,ns_config_ets_dup}, {mfargs,{ns_config_ets_dup,start_link,[]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:53:44.477,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.18811.0>}, {name,ns_log_events}, {mfargs,{gen_event,start_link,[{local,ns_log_events}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:53:44.477,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_node_disco_sup} started: [{pid,<0.18814.0>}, {name,ns_node_disco_events}, {mfargs, {gen_event,start_link, [{local,ns_node_disco_events}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:53:44.480,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_node_disco_sup} started: [{pid,<0.18815.0>}, {name,ns_node_disco}, {mfargs,{ns_node_disco,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [ns_server:info,2012-11-02T18:53:44.480,ns_1@10.2.1.67:ns_node_disco_events<0.18814.0>:ns_node_disco_log:handle_event:46]ns_node_disco_log: nodes changed: ['ns_1@10.2.1.67'] [error_logger:info,2012-11-02T18:53:44.480,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_node_disco_sup} started: [{pid,<0.18818.0>}, {name,ns_node_disco_log}, {mfargs,{ns_node_disco_log,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:53:44.480,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_node_disco_sup} started: [{pid,<0.18819.0>}, {name,ns_node_disco_conf_events}, {mfargs,{ns_node_disco_conf_events,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:53:44.480,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_node_disco_sup} started: [{pid,<0.18820.0>}, {name,ns_config_rep_merger}, {mfargs,{ns_config_rep,start_link_merger,[]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:53:44.482,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_node_disco_sup} started: [{pid,<0.18821.0>}, {name,ns_config_rep}, {mfargs,{ns_config_rep,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:53:44.483,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.18813.0>}, {name,ns_node_disco_sup}, {mfargs,{ns_node_disco_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:53:44.483,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.18825.0>}, {name,vbucket_map_mirror}, {mfargs,{vbucket_map_mirror,start_link,[]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:53:44.484,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.18829.0>}, {name,ns_tick_event}, {mfargs,{gen_event,start_link,[{local,ns_tick_event}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:53:44.484,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.18830.0>}, {name,mb_master_events}, {mfargs, {gen_event,start_link,[{local,mb_master_events}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [ns_server:error,2012-11-02T18:53:44.484,ns_1@10.2.1.67:ns_heart<0.18835.0>:ns_heart:grab_samples_loading_tasks:298]Failed to grab samples loader tasks: {exit, {noproc, {gen_server,call, [samples_loader_tasks,get_tasks, 2000]}}, [{gen_server,call,3}, {ns_heart,grab_samples_loading_tasks,0}, {ns_heart,current_status,0}, {ns_heart,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} [error_logger:info,2012-11-02T18:53:44.484,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.18831.0>}, {name,buckets_events}, {mfargs, {gen_event,start_link,[{local,buckets_events}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [ns_server:info,2012-11-02T18:53:44.484,ns_1@10.2.1.67:ns_config_log<0.709.0>:ns_config_log:handle_info:57]config change: rest_creds -> ******** [error_logger:info,2012-11-02T18:53:44.485,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_mail_sup} started: [{pid,<0.18833.0>}, {name,ns_mail_log}, {mfargs,{ns_mail_log,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [ns_server:info,2012-11-02T18:53:44.485,ns_1@10.2.1.67:remote_clusters_info<0.18841.0>:remote_clusters_info:read_or_create_table:384]Reading remote_clusters_info content from /opt/couchbase/var/lib/couchbase/remote_clusters_cache [ns_server:warn,2012-11-02T18:53:44.485,ns_1@10.2.1.67:ns_config_isasl_sync<0.706.0>:ns_memcached:connect:1103]Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:info,2012-11-02T18:53:44.485,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.18832.0>}, {name,ns_mail_sup}, {mfargs,{ns_mail_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:53:44.486,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.18834.0>}, {name,ns_stats_event}, {mfargs, {gen_event,start_link,[{local,ns_stats_event}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:53:44.486,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.18835.0>}, {name,ns_heart}, {mfargs,{ns_heart,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [user:info,2012-11-02T18:53:44.487,ns_1@10.2.1.67:mb_master<0.18846.0>:mb_master:init:89]I'm the only node, so I'm the master. [error_logger:info,2012-11-02T18:53:44.487,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.18838.0>}, {name,ns_doctor}, {mfargs,{ns_doctor,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:53:44.487,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.18841.0>}, {name,remote_clusters_info}, {mfargs,{remote_clusters_info,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [ns_server:info,2012-11-02T18:53:44.488,ns_1@10.2.1.67:ns_config<0.703.0>:ns_online_config_upgrader:upgrade_config_on_join_from_pre_2_0_to_2_0:65]Adding some 2.0 specific keys to the config [ns_server:error,2012-11-02T18:53:44.488,ns_1@10.2.1.67:ns_heart<0.18835.0>:ns_heart:grab_samples_loading_tasks:298]Failed to grab samples loader tasks: {exit, {noproc, {gen_server,call, [samples_loader_tasks,get_tasks, 2000]}}, [{gen_server,call,3}, {ns_heart,grab_samples_loading_tasks,0}, {ns_heart,current_status,0}, {ns_heart,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} [user:warn,2012-11-02T18:53:44.492,ns_1@10.2.1.67:<0.18849.0>:ns_orchestrator:consider_switching_compat_mode:665]Changed cluster compat mode from undefined to [2,0] [ns_server:info,2012-11-02T18:53:44.492,ns_1@10.2.1.67:ns_config<0.703.0>:ns_online_config_upgrader:upgrade_config_from_pre_2_0_to_2_0:69]Performing online config upgrade to 2.0 version [error_logger:info,2012-11-02T18:53:44.493,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mb_master_sup} started: [{pid,<0.18849.0>}, {name,ns_orchestrator}, {mfargs,{ns_orchestrator,start_link,[]}}, {restart_type,permanent}, {shutdown,20}, {child_type,worker}] [error_logger:info,2012-11-02T18:53:44.493,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mb_master_sup} started: [{pid,<0.18861.0>}, {name,ns_tick}, {mfargs,{ns_tick,start_link,[]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] [error_logger:info,2012-11-02T18:53:44.494,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mb_master_sup} started: [{pid,<0.18863.0>}, {name,auto_failover}, {mfargs,{auto_failover,start_link,[]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] [error_logger:info,2012-11-02T18:53:44.494,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.18846.0>}, {name,mb_master}, {mfargs,{mb_master,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:53:44.494,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.18864.0>}, {name,master_activity_events}, {mfargs, {gen_event,start_link, [{local,master_activity_events}]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:53:44.494,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.18865.0>}, {name,master_activity_events_ingress}, {mfargs, {gen_event,start_link, [{local,master_activity_events_ingress}]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:53:44.495,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.18866.0>}, {name,master_activity_events_timestamper}, {mfargs, {master_activity_events,start_link_timestamper,[]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:53:44.495,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.18867.0>}, {name,master_activity_events_pids_watcher}, {mfargs, {master_activity_events_pids_watcher,start_link, []}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:53:44.495,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.18868.0>}, {name,master_activity_events_keeper}, {mfargs,{master_activity_events_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:53:44.495,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.18871.0>}, {name,menelaus_web}, {mfargs,{menelaus_web,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info,2012-11-02T18:53:44.495,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.18888.0>}, {name,menelaus_event}, {mfargs,{menelaus_event,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info,2012-11-02T18:53:44.496,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.18889.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [user:info,2012-11-02T18:53:44.496,ns_1@10.2.1.67:ns_server_sup<0.18808.0>:menelaus_sup:start_link:44]Couchbase Server has started on web port 8091 on node 'ns_1@10.2.1.67'. [error_logger:info,2012-11-02T18:53:44.496,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.18890.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info,2012-11-02T18:53:44.496,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.18870.0>}, {name,menelaus}, {mfargs,{menelaus_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:53:44.496,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mc_sup} started: [{pid,<0.18892.0>}, {name,mc_couch_events}, {mfargs, {gen_event,start_link,[{local,mc_couch_events}]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [ns_server:info,2012-11-02T18:53:44.497,ns_1@10.2.1.67:<0.18894.0>:mc_tcp_listener:init:24]mccouch is listening on port 11213 [error_logger:info,2012-11-02T18:53:44.497,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mc_sup} started: [{pid,<0.18893.0>}, {name,mc_conn_sup}, {mfargs,{mc_conn_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:53:44.497,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mc_sup} started: [{pid,<0.18894.0>}, {name,mc_tcp_listener}, {mfargs,{mc_tcp_listener,start_link,[11213]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:53:44.497,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.18891.0>}, {name,mc_sup}, {mfargs,{mc_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:53:44.498,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_port_sup} started: [{pid,<0.18896.0>}, {name,ns_port_init}, {mfargs,{ns_port_init,start_link,[]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] [error_logger:info,2012-11-02T18:53:44.501,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_port_sup} started: [{pid,<0.18897.0>}, {name, {moxi,"/opt/couchbase/bin/moxi", ["-Z", "port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", "-z", "url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming", "-p","0","-Y","y","-O","stderr",[]], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR",[]}, {"MOXI_SASL_PLAIN_PWD",[]}]}, use_stdio,exit_status,port_server_send_eol, stderr_to_stdout,stream]}}, {mfargs, {supervisor_cushion,start_link, [moxi,5000,10000,ns_port_server,start_link, [moxi,"/opt/couchbase/bin/moxi", ["-Z", "port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", "-z", "url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming", "-p","0","-Y","y","-O","stderr",[]], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR",[]}, {"MOXI_SASL_PLAIN_PWD",[]}]}, use_stdio,exit_status,port_server_send_eol, stderr_to_stdout,stream]]]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [ns_server:info,2012-11-02T18:53:44.502,ns_1@10.2.1.67:<0.18902.0>:ns_memcached_log_rotator:init:28]Starting log rotator on "/opt/couchbase/var/lib/couchbase/logs"/"memcached.log"* with an initial period of 39003ms [error_logger:info,2012-11-02T18:53:44.503,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_port_sup} started: [{pid,<0.18899.0>}, {name, {memcached,"/opt/couchbase/bin/memcached", ["-X", "/opt/couchbase/lib/memcached/stdin_term_handler.so", "-X", "/opt/couchbase/lib/memcached/file_logger.so,cyclesize=104857600;sleeptime=19;filename=/opt/couchbase/var/lib/couchbase/logs/memcached.log", "-l","0.0.0.0:11210,0.0.0.0:11209:1000","-p", "11210","-E", "/opt/couchbase/lib/memcached/bucket_engine.so", "-B","binary","-r","-c","10000","-e", "admin=_admin;default_bucket_name=default;auto_create=false", []], [{env, [{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE", "/opt/couchbase/var/lib/couchbase/data/isasl.pw"}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,exit_status, port_server_send_eol,stream]}}, {mfargs, {erlang,apply, [#Fun, [memcached,"/opt/couchbase/bin/memcached", ["-X", "/opt/couchbase/lib/memcached/stdin_term_handler.so", "-X", "/opt/couchbase/lib/memcached/file_logger.so,cyclesize=104857600;sleeptime=19;filename=/opt/couchbase/var/lib/couchbase/logs/memcached.log", "-l","0.0.0.0:11210,0.0.0.0:11209:1000", "-p","11210","-E", "/opt/couchbase/lib/memcached/bucket_engine.so", "-B","binary","-r","-c","10000","-e", "admin=_admin;default_bucket_name=default;auto_create=false", []], [{env, [{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE", "/opt/couchbase/var/lib/couchbase/data/isasl.pw"}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,exit_status, port_server_send_eol,stream]]]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [error_logger:info,2012-11-02T18:53:44.504,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.18895.0>}, {name,ns_port_sup}, {mfargs,{ns_port_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:53:44.504,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.18901.0>}, {name,ns_port_memcached_killer}, {mfargs,{ns_port_sup,start_memcached_force_killer,[]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:53:44.504,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.18902.0>}, {name,ns_memcached_log_rotator}, {mfargs,{ns_memcached_log_rotator,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:53:44.504,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.18904.0>}, {name,ns_bucket_worker}, {mfargs,{work_queue,start_link,[ns_bucket_worker]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:53:44.504,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.18905.0>}, {name,xdc_replication_sup}, {mfargs,{xdc_replication_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:53:44.511,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.18906.0>}, {name,xdc_rep_manager}, {mfargs,{xdc_rep_manager,start_link,[]}}, {restart_type,permanent}, {shutdown,30000}, {child_type,worker}] [error_logger:info,2012-11-02T18:53:44.511,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_bucket_sup} started: [{pid,<0.18916.0>}, {name,buckets_observing_subscription}, {mfargs,{ns_bucket_sup,subscribe_on_config_events,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:53:44.512,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.18915.0>}, {name,ns_bucket_sup}, {mfargs,{ns_bucket_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:53:44.512,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.18918.0>}, {name,system_stats_collector}, {mfargs,{system_stats_collector,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:53:44.512,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.18920.0>}, {name,{stats_archiver,"@system"}}, {mfargs,{stats_archiver,start_link,["@system"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:53:44.512,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.18922.0>}, {name,{stats_reader,"@system"}}, {mfargs,{stats_reader,start_link,["@system"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:53:44.512,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.18923.0>}, {name,ns_moxi_sup_work_queue}, {mfargs, {work_queue,start_link,[ns_moxi_sup_work_queue]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:53:44.512,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.18924.0>}, {name,ns_moxi_sup}, {mfargs,{ns_moxi_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:53:44.512,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.18928.0>}, {name,compaction_daemon}, {mfargs, {supervisor_cushion,start_link, [compaction_daemon,3000,1000,compaction_daemon, start_link,[]]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [error_logger:info,2012-11-02T18:53:44.513,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.18931.0>}, {name,xdc_rdoc_replication_srv}, {mfargs,{xdc_rdoc_replication_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [ns_server:info,2012-11-02T18:53:44.513,ns_1@10.2.1.67:set_view_update_daemon<0.18935.0>:set_view_update_daemon:init:50]Set view update daemon, starting with the following settings: update interval: 5000ms minimum number of changes: 5000 [error_logger:info,2012-11-02T18:53:44.513,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.18935.0>}, {name,set_view_update_daemon}, {mfargs,{set_view_update_daemon,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:53:44.513,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.18937.0>}, {name,samples_loader_tasks}, {mfargs,{samples_loader_tasks,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:53:44.513,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_cluster_sup} started: [{pid,<0.18808.0>}, {name,ns_server_sup}, {mfargs,{ns_server_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [user:info,2012-11-02T18:57:23.252,ns_1@10.2.1.67:ns_cookie_manager<0.613.0>:ns_cookie_manager:do_cookie_init:91]Initial otp cookie generated: ibsurenydgcnovwc [user:info,2012-11-02T18:57:23.265,ns_1@10.2.1.67:ns_cluster<0.614.0>:ns_cluster:perform_actual_join:630]Node 'ns_1@10.2.1.67' is joining cluster via node 'ns_1@10.2.1.65'. [error_logger:error,2012-11-02T18:57:23.269,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_msg:76]** Connection attempt from disallowed node 'ns_1@10.2.1.68' ** [error_logger:error,2012-11-02T18:57:23.270,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_msg:76]** Connection attempt from disallowed node 'ns_1@10.2.1.66' ** [error_logger:error,2012-11-02T18:57:23.270,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_msg:76]** Connection attempt from disallowed node 'ns_1@10.2.1.65' ** [error_logger:error,2012-11-02T18:57:23.270,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,ns_bucket_sup} Context: shutdown_error Reason: normal Offender: [{pid,<0.18916.0>}, {name,buckets_observing_subscription}, {mfargs,{ns_bucket_sup,subscribe_on_config_events,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [ns_server:info,2012-11-02T18:57:23.472,ns_1@10.2.1.67:ns_port_memcached<0.18900.0>:ns_port_server:log:171]memcached<0.18900.0>: EOL on stdin. Initiating shutdown [ns_server:info,2012-11-02T18:57:23.754,ns_1@10.2.1.67:ns_port_memcached<0.18900.0>:ns_port_server:handle_info:104]Port server memcached exited with status 0 [ns_server:info,2012-11-02T18:57:23.755,ns_1@10.2.1.67:<0.18898.0>:ns_port_server:handle_info:104]Port server moxi exited with status 0 [ns_server:info,2012-11-02T18:57:23.755,ns_1@10.2.1.67:<0.18898.0>:ns_port_server:log:171]moxi<0.18898.0>: EOL on stdin. Exiting [ns_server:info,2012-11-02T18:57:23.756,ns_1@10.2.1.67:mb_master<0.18846.0>:mb_master:terminate:288]Synchronously shutting down child mb_master_sup [user:info,2012-11-02T18:57:23.762,ns_1@10.2.1.67:ns_cookie_manager<0.613.0>:ns_cookie_manager:do_cookie_sync:135]Node 'ns_1@10.2.1.67' synchronized otp cookie khsxbujfzniwernw from cluster [ns_server:warn,2012-11-02T18:57:23.768,ns_1@10.2.1.67:ns_log<0.19574.0>:ns_log:read_logs:69]Couldn't load logs from "/opt/couchbase/var/lib/couchbase/data/ns_log" (perhaps it's first startup): {error, enoent} [error_logger:info,2012-11-02T18:57:23.769,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.19574.0>}, {name,ns_log}, {mfargs,{ns_log,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:57:23.769,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.19575.0>}, {name,ns_config_ets_dup}, {mfargs,{ns_config_ets_dup,start_link,[]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:57:23.769,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.19576.0>}, {name,ns_log_events}, {mfargs,{gen_event,start_link,[{local,ns_log_events}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:57:23.770,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_node_disco_sup} started: [{pid,<0.19579.0>}, {name,ns_node_disco_events}, {mfargs, {gen_event,start_link, [{local,ns_node_disco_events}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:57:23.774,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_node_disco_sup} started: [{pid,<0.19580.0>}, {name,ns_node_disco}, {mfargs,{ns_node_disco,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [ns_server:info,2012-11-02T18:57:23.774,ns_1@10.2.1.67:ns_node_disco_events<0.19579.0>:ns_node_disco_log:handle_event:46]ns_node_disco_log: nodes changed: ['ns_1@10.2.1.65','ns_1@10.2.1.67'] [error_logger:info,2012-11-02T18:57:23.774,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_node_disco_sup} started: [{pid,<0.19583.0>}, {name,ns_node_disco_log}, {mfargs,{ns_node_disco_log,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [ns_server:info,2012-11-02T18:57:23.775,ns_1@10.2.1.67:ns_config_rep<0.19586.0>:ns_config_rep:do_pull:341]Pulling config from: 'ns_1@10.2.1.65' [error_logger:info,2012-11-02T18:57:23.775,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_node_disco_sup} started: [{pid,<0.19584.0>}, {name,ns_node_disco_conf_events}, {mfargs,{ns_node_disco_conf_events,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:57:23.775,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_node_disco_sup} started: [{pid,<0.19585.0>}, {name,ns_config_rep_merger}, {mfargs,{ns_config_rep,start_link_merger,[]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [ns_server:info,2012-11-02T18:57:23.781,ns_1@10.2.1.67:ns_config_log<0.709.0>:ns_config_log:handle_info:57]config change: rest_creds -> ******** [error_logger:info,2012-11-02T18:57:23.784,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_node_disco_sup} started: [{pid,<0.19586.0>}, {name,ns_config_rep}, {mfargs,{ns_config_rep,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:57:23.784,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.19578.0>}, {name,ns_node_disco_sup}, {mfargs,{ns_node_disco_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:57:23.784,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.19596.0>}, {name,vbucket_map_mirror}, {mfargs,{vbucket_map_mirror,start_link,[]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:57:23.785,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.19598.0>}, {name,ns_tick_event}, {mfargs,{gen_event,start_link,[{local,ns_tick_event}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:57:23.785,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.19599.0>}, {name,mb_master_events}, {mfargs, {gen_event,start_link,[{local,mb_master_events}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:57:23.786,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.19600.0>}, {name,buckets_events}, {mfargs, {gen_event,start_link,[{local,buckets_events}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:57:23.786,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_mail_sup} started: [{pid,<0.19602.0>}, {name,ns_mail_log}, {mfargs,{ns_mail_log,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:57:23.786,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.19601.0>}, {name,ns_mail_sup}, {mfargs,{ns_mail_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:57:23.786,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.19603.0>}, {name,ns_stats_event}, {mfargs, {gen_event,start_link,[{local,ns_stats_event}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:57:23.786,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.19604.0>}, {name,ns_heart}, {mfargs,{ns_heart,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [ns_server:error,2012-11-02T18:57:23.787,ns_1@10.2.1.67:ns_heart<0.19604.0>:ns_heart:grab_samples_loading_tasks:298]Failed to grab samples loader tasks: {exit, {noproc, {gen_server,call, [samples_loader_tasks,get_tasks, 2000]}}, [{gen_server,call,3}, {ns_heart,grab_samples_loading_tasks,0}, {ns_heart,current_status,0}, {ns_heart,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} [error_logger:info,2012-11-02T18:57:23.787,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.19607.0>}, {name,ns_doctor}, {mfargs,{ns_doctor,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [ns_server:info,2012-11-02T18:57:23.788,ns_1@10.2.1.67:remote_clusters_info<0.19612.0>:remote_clusters_info:read_or_create_table:384]Reading remote_clusters_info content from /opt/couchbase/var/lib/couchbase/remote_clusters_cache [error_logger:info,2012-11-02T18:57:23.789,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.19612.0>}, {name,remote_clusters_info}, {mfargs,{remote_clusters_info,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [ns_server:error,2012-11-02T18:57:23.792,ns_1@10.2.1.67:ns_heart<0.19604.0>:ns_heart:grab_samples_loading_tasks:298]Failed to grab samples loader tasks: {exit, {noproc, {gen_server,call, [samples_loader_tasks,get_tasks, 2000]}}, [{gen_server,call,3}, {ns_heart,grab_samples_loading_tasks,0}, {ns_heart,current_status,0}, {ns_heart,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} [user:info,2012-11-02T18:57:23.792,ns_1@10.2.1.67:ns_node_disco<0.19580.0>:ns_node_disco:handle_info:162]Node 'ns_1@10.2.1.67' saw that node 'ns_1@10.2.1.66' came up. [error_logger:info,2012-11-02T18:57:23.792,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.19623.0>}, {name,mb_master}, {mfargs,{mb_master,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:57:23.793,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.19626.0>}, {name,master_activity_events}, {mfargs, {gen_event,start_link, [{local,master_activity_events}]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:57:23.793,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.19627.0>}, {name,master_activity_events_ingress}, {mfargs, {gen_event,start_link, [{local,master_activity_events_ingress}]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [ns_server:info,2012-11-02T18:57:23.794,ns_1@10.2.1.67:ns_config_rep<0.19586.0>:ns_config_rep:handle_info:220]Replicating config to/from: ['ns_1@10.2.1.66'] [ns_server:info,2012-11-02T18:57:23.794,ns_1@10.2.1.67:ns_node_disco_events<0.19579.0>:ns_node_disco_log:handle_event:46]ns_node_disco_log: nodes changed: ['ns_1@10.2.1.65','ns_1@10.2.1.66', 'ns_1@10.2.1.67'] [ns_server:info,2012-11-02T18:57:23.794,ns_1@10.2.1.67:ns_config_log<0.709.0>:ns_config_log:handle_info:57]config change: rest_creds -> ******** [error_logger:info,2012-11-02T18:57:23.794,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.19628.0>}, {name,master_activity_events_timestamper}, {mfargs, {master_activity_events,start_link_timestamper,[]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [ns_server:info,2012-11-02T18:57:23.794,ns_1@10.2.1.67:ns_config_rep<0.19586.0>:ns_config_rep:do_pull:341]Pulling config from: 'ns_1@10.2.1.66' [error_logger:info,2012-11-02T18:57:23.794,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.19629.0>}, {name,master_activity_events_pids_watcher}, {mfargs, {master_activity_events_pids_watcher,start_link, []}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:57:23.795,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.19631.0>}, {name,master_activity_events_keeper}, {mfargs,{master_activity_events_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:57:23.796,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.19635.0>}, {name,menelaus_web}, {mfargs,{menelaus_web,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info,2012-11-02T18:57:23.796,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.19655.0>}, {name,menelaus_event}, {mfargs,{menelaus_event,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info,2012-11-02T18:57:23.796,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.19656.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info,2012-11-02T18:57:23.797,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.19657.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [user:info,2012-11-02T18:57:23.797,ns_1@10.2.1.67:ns_server_sup<0.19573.0>:menelaus_sup:start_link:44]Couchbase Server has started on web port 8091 on node 'ns_1@10.2.1.67'. [error_logger:info,2012-11-02T18:57:23.797,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.19634.0>}, {name,menelaus}, {mfargs,{menelaus_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [user:info,2012-11-02T18:57:23.797,ns_1@10.2.1.67:ns_node_disco<0.19580.0>:ns_node_disco:handle_info:162]Node 'ns_1@10.2.1.67' saw that node 'ns_1@10.2.1.68' came up. [ns_server:info,2012-11-02T18:57:23.798,ns_1@10.2.1.67:<0.19662.0>:mc_tcp_listener:init:24]mccouch is listening on port 11213 [error_logger:info,2012-11-02T18:57:23.798,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mc_sup} started: [{pid,<0.19660.0>}, {name,mc_couch_events}, {mfargs, {gen_event,start_link,[{local,mc_couch_events}]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:57:23.798,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mc_sup} started: [{pid,<0.19661.0>}, {name,mc_conn_sup}, {mfargs,{mc_conn_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:57:23.798,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mc_sup} started: [{pid,<0.19662.0>}, {name,mc_tcp_listener}, {mfargs,{mc_tcp_listener,start_link,[11213]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:57:23.798,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.19659.0>}, {name,mc_sup}, {mfargs,{mc_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [ns_server:info,2012-11-02T18:57:23.799,ns_1@10.2.1.67:ns_node_disco_events<0.19579.0>:ns_node_disco_log:handle_event:46]ns_node_disco_log: nodes changed: ['ns_1@10.2.1.65','ns_1@10.2.1.66', 'ns_1@10.2.1.67','ns_1@10.2.1.68'] [error_logger:info,2012-11-02T18:57:23.801,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_port_sup} started: [{pid,<0.19664.0>}, {name,ns_port_init}, {mfargs,{ns_port_init,start_link,[]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] [error_logger:info,2012-11-02T18:57:23.802,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_port_sup} started: [{pid,<0.19665.0>}, {name, {moxi,"/opt/couchbase/bin/moxi", ["-Z", "port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", "-z", "url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming", "-p","0","-Y","y","-O","stderr",[]], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR","Administrator"}, {"MOXI_SASL_PLAIN_PWD","password"}]}, use_stdio,exit_status,port_server_send_eol, stderr_to_stdout,stream]}}, {mfargs, {supervisor_cushion,start_link, [moxi,5000,10000,ns_port_server,start_link, [moxi,"/opt/couchbase/bin/moxi", ["-Z", "port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", "-z", "url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming", "-p","0","-Y","y","-O","stderr",[]], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR","Administrator"}, {"MOXI_SASL_PLAIN_PWD","password"}]}, use_stdio,exit_status,port_server_send_eol, stderr_to_stdout,stream]]]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [error_logger:info,2012-11-02T18:57:23.804,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_port_sup} started: [{pid,<0.19669.0>}, {name, {memcached,"/opt/couchbase/bin/memcached", ["-X", "/opt/couchbase/lib/memcached/stdin_term_handler.so", "-X", "/opt/couchbase/lib/memcached/file_logger.so,cyclesize=104857600;sleeptime=19;filename=/opt/couchbase/var/lib/couchbase/logs/memcached.log", "-l","0.0.0.0:11210,0.0.0.0:11209:1000","-p", "11210","-E", "/opt/couchbase/lib/memcached/bucket_engine.so", "-B","binary","-r","-c","10000","-e", "admin=_admin;default_bucket_name=default;auto_create=false", []], [{env, [{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE", "/opt/couchbase/var/lib/couchbase/data/isasl.pw"}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,exit_status, port_server_send_eol,stream]}}, {mfargs, {erlang,apply, [#Fun, [memcached,"/opt/couchbase/bin/memcached", ["-X", "/opt/couchbase/lib/memcached/stdin_term_handler.so", "-X", "/opt/couchbase/lib/memcached/file_logger.so,cyclesize=104857600;sleeptime=19;filename=/opt/couchbase/var/lib/couchbase/logs/memcached.log", "-l","0.0.0.0:11210,0.0.0.0:11209:1000", "-p","11210","-E", "/opt/couchbase/lib/memcached/bucket_engine.so", "-B","binary","-r","-c","10000","-e", "admin=_admin;default_bucket_name=default;auto_create=false", []], [{env, [{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE", "/opt/couchbase/var/lib/couchbase/data/isasl.pw"}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,exit_status, port_server_send_eol,stream]]]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [ns_server:info,2012-11-02T18:57:23.805,ns_1@10.2.1.67:<0.19673.0>:ns_memcached_log_rotator:init:28]Starting log rotator on "/opt/couchbase/var/lib/couchbase/logs"/"memcached.log"* with an initial period of 39003ms [error_logger:info,2012-11-02T18:57:23.805,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.19663.0>}, {name,ns_port_sup}, {mfargs,{ns_port_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:57:23.806,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.19671.0>}, {name,ns_port_memcached_killer}, {mfargs,{ns_port_sup,start_memcached_force_killer,[]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:57:23.806,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.19673.0>}, {name,ns_memcached_log_rotator}, {mfargs,{ns_memcached_log_rotator,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:57:23.806,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.19674.0>}, {name,ns_bucket_worker}, {mfargs,{work_queue,start_link,[ns_bucket_worker]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:57:23.806,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.19675.0>}, {name,xdc_replication_sup}, {mfargs,{xdc_replication_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:57:23.807,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.19676.0>}, {name,xdc_rep_manager}, {mfargs,{xdc_rep_manager,start_link,[]}}, {restart_type,permanent}, {shutdown,30000}, {child_type,worker}] [error_logger:info,2012-11-02T18:57:23.808,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_bucket_sup} started: [{pid,<0.19679.0>}, {name,buckets_observing_subscription}, {mfargs,{ns_bucket_sup,subscribe_on_config_events,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:57:23.808,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.19678.0>}, {name,ns_bucket_sup}, {mfargs,{ns_bucket_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:57:23.810,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.19681.0>}, {name,system_stats_collector}, {mfargs,{system_stats_collector,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:57:23.811,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.19683.0>}, {name,{stats_archiver,"@system"}}, {mfargs,{stats_archiver,start_link,["@system"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:57:23.811,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.19685.0>}, {name,{stats_reader,"@system"}}, {mfargs,{stats_reader,start_link,["@system"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:57:23.811,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.19686.0>}, {name,ns_moxi_sup_work_queue}, {mfargs, {work_queue,start_link,[ns_moxi_sup_work_queue]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:57:23.811,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.19687.0>}, {name,ns_moxi_sup}, {mfargs,{ns_moxi_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:57:23.811,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.19689.0>}, {name,compaction_daemon}, {mfargs, {supervisor_cushion,start_link, [compaction_daemon,3000,1000,compaction_daemon, start_link,[]]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [error_logger:info,2012-11-02T18:57:23.812,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.19692.0>}, {name,xdc_rdoc_replication_srv}, {mfargs,{xdc_rdoc_replication_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [ns_server:info,2012-11-02T18:57:23.812,ns_1@10.2.1.67:set_view_update_daemon<0.19694.0>:set_view_update_daemon:init:50]Set view update daemon, starting with the following settings: update interval: 5000ms minimum number of changes: 5000 [error_logger:info,2012-11-02T18:57:23.812,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.19694.0>}, {name,set_view_update_daemon}, {mfargs,{set_view_update_daemon,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [user:info,2012-11-02T18:57:23.812,ns_1@10.2.1.67:ns_cluster<0.614.0>:ns_cluster:perform_actual_join:676]Node ns_1@10.2.1.67 joined cluster [error_logger:info,2012-11-02T18:57:23.812,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.19696.0>}, {name,samples_loader_tasks}, {mfargs,{samples_loader_tasks,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:57:23.812,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_cluster_sup} started: [{pid,<0.19573.0>}, {name,ns_server_sup}, {mfargs,{ns_server_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [ns_server:info,2012-11-02T18:57:24.015,ns_1@10.2.1.67:<0.19666.0>:ns_port_server:log:171]moxi<0.19666.0>: 2012-11-02 18:57:23: (cproxy_config.c.317) env: MOXI_SASL_PLAIN_USR (13) moxi<0.19666.0>: 2012-11-02 18:57:23: (cproxy_config.c.326) env: MOXI_SASL_PLAIN_PWD (8) [ns_server:info,2012-11-02T18:57:24.504,ns_1@10.2.1.67:mb_master<0.19623.0>:mb_master:candidate:351]Changing master from undefined to 'ns_1@10.2.1.65' [user:info,2012-11-02T18:57:25.120,ns_1@10.2.1.67:<0.19707.0>:ns_storage_conf:delete_unused_buckets_db_files:530]Deleting old data files of bucket "default" [ns_server:info,2012-11-02T18:57:25.122,ns_1@10.2.1.67:<0.19707.0>:ns_storage_conf:delete_couch_database:431]Deleting database <<"default/master">>: ok [ns_server:info,2012-11-02T18:57:25.122,ns_1@10.2.1.67:<0.19707.0>:ns_storage_conf:delete_databases_and_files:474]Couch dbs are deleted. Proceeding with bucket directory [ns_server:info,2012-11-02T18:57:25.132,ns_1@10.2.1.67:<0.19709.0>:diag_handler:log_all_tap_and_checkpoint_stats:127]logging tap & checkpoint stats [ns_server:info,2012-11-02T18:57:25.132,ns_1@10.2.1.67:<0.19709.0>:diag_handler:log_all_tap_and_checkpoint_stats:131]end of logging tap & checkpoint stats [error_logger:info,2012-11-02T18:57:39.631,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_bucket_sup} started: [{pid,<0.19781.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:57:39.633,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,'single_bucket_sup-default'} started: [{pid,<0.19783.0>}, {name,{capi_set_view_manager,"default"}}, {mfargs,{capi_set_view_manager,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [ns_server:info,2012-11-02T18:57:39.640,ns_1@10.2.1.67:ns_memcached-default<0.19796.0>:ns_memcached:ensure_bucket:1119]Created bucket "default" with config string "ht_size=3079;ht_locks=5;tap_noop_interval=20;max_txn_size=10000;max_size=5242880000;tap_keepalive=300;dbname=/data/default;allow_data_loss_during_shutdown=true;backend=couchdb;couch_bucket=default;couch_port=11213;max_vbuckets=512;alog_path=/data/default/access.log;data_traffic_enabled=false;vb0=false;waitforwarmup=false;failpartialwarmup=false;" [error_logger:info,2012-11-02T18:57:39.642,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,'single_bucket_sup-default'} started: [{pid,<0.19796.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [error_logger:info,2012-11-02T18:57:39.642,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,'single_bucket_sup-default'} started: [{pid,<0.19802.0>}, {name,{tap_replication_manager,"default"}}, {mfargs, {tap_replication_manager,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [ns_server:info,2012-11-02T18:57:39.643,ns_1@10.2.1.67:janitor_agent-default<0.19805.0>:janitor_agent:read_flush_counter:764]Loading flushseq failed: {error,enoent}. Assuming it's equal to global config. [error_logger:info,2012-11-02T18:57:39.643,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,'single_bucket_sup-default'} started: [{pid,<0.19803.0>}, {name,{ns_vbm_new_sup,"default"}}, {mfargs,{ns_vbm_new_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [ns_server:info,2012-11-02T18:57:39.643,ns_1@10.2.1.67:janitor_agent-default<0.19805.0>:janitor_agent:read_flush_counter_from_config:771]Initialized flushseq 0 from bucket config [error_logger:info,2012-11-02T18:57:39.643,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,'single_bucket_sup-default'} started: [{pid,<0.19804.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,supervisor}] [error_logger:info,2012-11-02T18:57:39.643,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,'single_bucket_sup-default'} started: [{pid,<0.19805.0>}, {name,{janitor_agent,"default"}}, {mfargs,{janitor_agent,start_link,["default"]}}, {restart_type,permanent}, {shutdown,brutal_kill}, {child_type,worker}] [error_logger:info,2012-11-02T18:57:39.643,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,'single_bucket_sup-default'} started: [{pid,<0.19806.0>}, {name,{couch_stats_reader,"default"}}, {mfargs,{couch_stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:57:39.644,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,'single_bucket_sup-default'} started: [{pid,<0.19807.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:57:39.644,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,'single_bucket_sup-default'} started: [{pid,<0.19809.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:57:39.644,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,'single_bucket_sup-default'} started: [{pid,<0.19811.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info,2012-11-02T18:57:39.644,ns_1@10.2.1.67:error_logger<0.5.0>:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,'single_bucket_sup-default'} started: [{pid,<0.19812.0>}, {name,{failover_safeness_level,"default"}}, {mfargs, {failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [ns_server:info,2012-11-02T18:57:39.838,ns_1@10.2.1.67:ns_port_memcached<0.19670.0>:ns_port_server:log:171]memcached<0.19670.0>: Fri Nov 2 18:57:39.638009 PDT 3: Trying to connect to mccouch: "localhost:11213" memcached<0.19670.0>: Fri Nov 2 18:57:39.638578 PDT 3: Connected to mccouch: "localhost:11213" memcached<0.19670.0>: Fri Nov 2 18:57:39.640824 PDT 3: Warning: failed to load the engine session stats due to IO exception "basic_ios::clear" memcached<0.19670.0>: Fri Nov 2 18:57:39.640909 PDT 3: Extension support isn't implemented in this version of bucket_engine memcached<0.19670.0>: Fri Nov 2 18:57:39.640964 PDT 3: Failed to load mutation log, falling back to key dump memcached<0.19670.0>: Fri Nov 2 18:57:39.641085 PDT 3: metadata loaded in 644 usec memcached<0.19670.0>: Fri Nov 2 18:57:39.641254 PDT 3: warmup completed in 729 usec [ns_server:info,2012-11-02T18:57:39.882,ns_1@10.2.1.67:<0.19666.0>:ns_port_server:log:171]moxi<0.19666.0>: 2012-11-02 18:57:41: (agent_config.c.705) ERROR: bad JSON configuration from http://127.0.0.1:8091/pools/default/saslBucketsStreaming: Number of vBuckets must be a power of two > 0 and <= 65536 ({ moxi<0.19666.0>: "name": "default", moxi<0.19666.0>: "nodeLocator": "vbucket", moxi<0.19666.0>: "saslPassword": "", moxi<0.19666.0>: "nodes": [{ moxi<0.19666.0>: "hostname": "10.2.1.65:8091", moxi<0.19666.0>: "ports": { moxi<0.19666.0>: "direct": 11210, moxi<0.19666.0>: "proxy": 11211 moxi<0.19666.0>: } moxi<0.19666.0>: }, { moxi<0.19666.0>: "hostname": "10.2.1.66:8091", moxi<0.19666.0>: "ports": { moxi<0.19666.0>: "direct": 11210, moxi<0.19666.0>: "proxy": 11211 moxi<0.19666.0>: } moxi<0.19666.0>: }, { moxi<0.19666.0>: "hostname": "10.2.1.67:8091", moxi<0.19666.0>: "ports": { moxi<0.19666.0>: "direct": 11210, moxi<0.19666.0>: "proxy": 11211 moxi<0.19666.0>: } moxi<0.19666.0>: }, { moxi<0.19666.0>: "hostname": "10.2.1.68:8091", moxi<0.19666.0>: "ports": { moxi<0.19666.0>: "direct": 11210, moxi<0.19666.0>: "proxy": 11211 moxi<0.19666.0>: } moxi<0.19666.0>: }], moxi<0.19666.0>: "vBucketServerMap": { moxi<0.19666.0>: "hashAlgorithm": "CRC", moxi<0.19666.0>: "numReplicas": 0, moxi<0.19666.0>: "serverList": ["10.2.1.67:11210", "10.2.1.65:11210", "10.2.1.66:11210", "10.2.1.68:11210"], moxi<0.19666.0>: "vBucketMap": [] moxi<0.19666.0>: } moxi<0.19666.0>: }) [user:info,2012-11-02T18:57:40.136,ns_1@10.2.1.67:ns_memcached-default<0.19796.0>:ns_memcached:handle_cast:581]Bucket "default" loaded on node 'ns_1@10.2.1.67' in 0 seconds. [ns_server:info,2012-11-02T18:57:40.659,ns_1@10.2.1.67:<0.19800.0>:ns_memcached:do_handle_call:521]Changed vbucket 383 state to active [ns_server:info,2012-11-02T18:57:40.659,ns_1@10.2.1.67:<0.19800.0>:ns_memcached:do_handle_call:521]Changed vbucket 382 state to active [ns_server:info,2012-11-02T18:57:40.660,ns_1@10.2.1.67:<0.19800.0>:ns_memcached:do_handle_call:521]Changed vbucket 381 state to active [ns_server:info,2012-11-02T18:57:40.660,ns_1@10.2.1.67:<0.19800.0>:ns_memcached:do_handle_call:521]Changed vbucket 380 state to active [ns_server:info,2012-11-02T18:57:40.660,ns_1@10.2.1.67:<0.19800.0>:ns_memcached:do_handle_call:521]Changed vbucket 379 state to active [ns_server:info,2012-11-02T18:57:40.661,ns_1@10.2.1.67:<0.19800.0>:ns_memcached:do_handle_call:521]Changed vbucket 378 state to active [ns_server:info,2012-11-02T18:57:40.661,ns_1@10.2.1.67:<0.19800.0>:ns_memcached:do_handle_call:521]Changed vbucket 377 state to active [ns_server:info,2012-11-02T18:57:40.661,ns_1@10.2.1.67:<0.19800.0>:ns_memcached:do_handle_call:521]Changed vbucket 376 state to active [ns_server:info,2012-11-02T18:57:40.662,ns_1@10.2.1.67:<0.19800.0>:ns_memcached:do_handle_call:521]Changed vbucket 375 state to active [ns_server:info,2012-11-02T18:57:40.663,ns_1@10.2.1.67:<0.19800.0>:ns_memcached:do_handle_call:521]Changed vbucket 374 state to active [ns_server:info,2012-11-02T18:57:40.663,ns_1@10.2.1.67:<0.19800.0>:ns_memcached:do_handle_call:521]Changed vbucket 373 state to active [ns_server:info,2012-11-02T18:57:40.663,ns_1@10.2.1.67:<0.19800.0>:ns_memcached:do_handle_call:521]Changed vbucket 372 state to active [ns_server:info,2012-11-02T18:57:40.664,ns_1@10.2.1.67:<0.19800.0>:ns_memcached:do_handle_call:521]Changed vbucket 371 state to active [ns_server:info,2012-11-02T18:57:40.664,ns_1@10.2.1.67:<0.19800.0>:ns_memcached:do_handle_call:521]Changed vbucket 370 state to active [ns_server:info,2012-11-02T18:57:40.664,ns_1@10.2.1.67:<0.19800.0>:ns_memcached:do_handle_call:521]Changed vbucket 369 state to active [ns_server:info,2012-11-02T18:57:40.665,ns_1@10.2.1.67:<0.19800.0>:ns_memcached:do_handle_call:521]Changed vbucket 368 state to active [ns_server:info,2012-11-02T18:57:40.665,ns_1@10.2.1.67:<0.19800.0>:ns_memcached:do_handle_call:521]Changed vbucket 367 state to active [ns_server:info,2012-11-02T18:57:40.665,ns_1@10.2.1.67:<0.19800.0>:ns_memcached:do_handle_call:521]Changed vbucket 366 state to active [ns_server:info,2012-11-02T18:57:40.666,ns_1@10.2.1.67:<0.19800.0>:ns_memcached:do_handle_call:521]Changed vbucket 365 state to active [ns_server:info,2012-11-02T18:57:40.666,ns_1@10.2.1.67:<0.19800.0>:ns_memcached:do_handle_call:521]Changed vbucket 364 state to active [ns_server:info,2012-11-02T18:57:40.666,ns_1@10.2.1.67:<0.19800.0>:ns_memcached:do_handle_call:521]Changed vbucket 363 state to active [ns_server:info,2012-11-02T18:57:40.667,ns_1@10.2.1.67:<0.19800.0>:ns_memcached:do_handle_call:521]Changed vbucket 362 state to active [ns_server:info,2012-11-02T18:57:40.667,ns_1@10.2.1.67:<0.19800.0>:ns_memcached:do_handle_call:521]Changed vbucket 361 state to active [ns_server:info