============================================================================== couchbase logs (stats) cbbrowse_logs stats ============================================================================== [ns_doctor:debug,2012-11-14T10:46:35.026,ns_1@127.0.0.1:ns_doctor<0.377.0>:ns_doctor:handle_info:133]Got initial status [{'ns_1@127.0.0.1', [{last_heard,{1352,918795,26405}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,39773104}, {processes,9023768}, {processes_used,9012144}, {system,30749336}, {atom,1171065}, {atom_used,1138716}, {binary,106280}, {code,11330364}, {ets,875032}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585772032}, {total_swap,10585923584}, {cached_memory,15307087872}, {buffered_memory,282624000}, {free_memory,17405005824}, {total_memory,33657663488}]}, {node_storage_conf, [{db_path, "/opt/couchbase/var/lib/couchbase/data/"}, {index_path, "/opt/couchbase/var/lib/couchbase/data/"}]}, {statistics, [{wall_clock,{1982,3}}, {context_switches,{26777,0}}, {garbage_collection,{7355,34756690,0}}, {io,{{input,6771392},{output,664104}}}, {reductions,{20428742,18057}}, {run_queue,0}, {runtime,{1250,10}}]}, {system_stats, [{cpu_utilization_rate,0}, {swap_total,0}, {swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,1}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1}, {memory_data, {33657663488,16232849408,{<0.6.0>,601272}}}, {disk_data, [{"/",49781912,18}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,2}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16997076 kB\nBuffers: 275992 kB\nCached: 14948336 kB\nSwapCached: 0 kB\nActive: 4333020 kB\nInactive: 11017736 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16997076 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337668 kB\nDirty: 419876 kB\nWriteback: 20 kB\nAnonPages: 126516 kB\nMapped: 14044 kB\nSlab: 483332 kB\nPageTables: 4820 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 1574652 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-14T10:46:52.323,ns_1@10.2.1.63:ns_doctor<0.375.0>:ns_doctor:handle_info:133]Got initial status [{'ns_1@10.2.1.63', [{last_heard,{1352,918812,323668}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,39607064}, {processes,8849432}, {processes_used,8837008}, {system,30757632}, {atom,1171873}, {atom_used,1138823}, {binary,134672}, {code,11311852}, {ets,875280}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585772032}, {total_swap,10585923584}, {cached_memory,15307206656}, {buffered_memory,282984448}, {free_memory,17409105920}, {total_memory,33657663488}]}, {node_storage_conf, [{db_path, "/opt/couchbase/var/lib/couchbase/data/"}, {index_path, "/opt/couchbase/var/lib/couchbase/data/"}]}, {statistics, [{wall_clock,{1859,3}}, {context_switches,{26262,0}}, {garbage_collection,{7197,33322025,0}}, {io,{{input,6772603},{output,660335}}}, {reductions,{19526293,20113}}, {run_queue,0}, {runtime,{1160,10}}]}, {system_stats, [{cpu_utilization_rate,0}, {swap_total,0}, {swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,1}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1}, {memory_data, {33657663488,16231763968,{<0.6.0>,601272}}}, {disk_data, [{"/",49781912,18}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,2}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 17001204 kB\nBuffers: 276352 kB\nCached: 14948444 kB\nSwapCached: 0 kB\nActive: 4334732 kB\nInactive: 11014608 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 17001204 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337668 kB\nDirty: 390848 kB\nWriteback: 20 kB\nAnonPages: 124208 kB\nMapped: 14088 kB\nSlab: 480860 kB\nPageTables: 4772 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 1573380 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-14T10:47:03.633,ns_1@10.2.1.63:ns_doctor<0.744.0>:ns_doctor:handle_info:133]Got initial status [{'ns_1@10.2.1.63', [{last_heard,{1352,918823,633311}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,42865344}, {processes,10397224}, {processes_used,10340704}, {system,32468120}, {atom,1242281}, {atom_used,1241108}, {binary,235160}, {code,12678909}, {ets,1002080}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585772032}, {total_swap,10585923584}, {cached_memory,627134464}, {buffered_memory,284397568}, {free_memory,32259448832}, {total_memory,33657663488}]}, {node_storage_conf, [{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{13168,3}}, {context_switches,{34285,0}}, {garbage_collection,{9308,37072814,0}}, {io,{{input,7663770},{output,1087428}}}, {reductions,{20887658,77092}}, {run_queue,2}, {runtime,{1890,10}}]}, {system_stats, [{cpu_utilization_rate,0}, {swap_total,0}, {swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,13}, {memory_data, {33657663488,16231763968,{<0.6.0>,601272}}}, {disk_data, [{"/",49781912,18}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,2}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 31503492 kB\nBuffers: 277732 kB\nCached: 612436 kB\nSwapCached: 0 kB\nActive: 593192 kB\nInactive: 426820 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 31503492 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337668 kB\nDirty: 214956 kB\nWriteback: 0 kB\nAnonPages: 129448 kB\nMapped: 15932 kB\nSlab: 307024 kB\nPageTables: 4780 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 1599812 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-14T10:47:21.330,ns_1@10.2.1.63:ns_doctor<0.408.0>:ns_doctor:handle_info:133]Got initial status [{'ns_1@10.2.1.63', [{last_heard,{1352,918841,330136}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,39817496}, {processes,9137576}, {processes_used,9133584}, {system,30679920}, {atom,1124441}, {atom_used,1122558}, {binary,176600}, {code,11184434}, {ets,930312}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585772032}, {total_swap,10585923584}, {cached_memory,627507200}, {buffered_memory,284839936}, {free_memory,32263532544}, {total_memory,33657663488}]}, {node_storage_conf, [{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1842,4}}, {context_switches,{26140,0}}, {garbage_collection,{7182,33101170,0}}, {io,{{input,6688130},{output,621308}}}, {reductions,{19545032,33861}}, {run_queue,0}, {runtime,{1110,10}}]}, {system_stats, [{cpu_utilization_rate,0}, {swap_total,0}, {swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1}, {memory_data, {33657663488,1372852224,{<0.6.0>,601352}}}, {disk_data, [{"/",49781912,18}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 31507480 kB\nBuffers: 278164 kB\nCached: 612800 kB\nSwapCached: 0 kB\nActive: 589488 kB\nInactive: 427412 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 31507480 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337668 kB\nDirty: 212700 kB\nWriteback: 0 kB\nAnonPages: 125964 kB\nMapped: 14196 kB\nSlab: 305484 kB\nPageTables: 5244 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 1582376 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-14T10:47:24.432,ns_1@10.2.1.63:ns_doctor<0.540.0>:ns_doctor:handle_info:133]Got initial status [{'ns_1@10.2.1.61', [{last_heard,{1352,918844,432697}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,47804392}, {processes,15030480}, {processes_used,15012336}, {system,32773912}, {atom,1244705}, {atom_used,1244165}, {binary,274152}, {code,12847120}, {ets,1050008}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684477440}, {total_swap,10684628992}, {cached_memory,628707328}, {buffered_memory,296083456}, {free_memory,32185946112}, {total_memory,33657663488}]}, {node_storage_conf, [{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{8535,13}}, {context_switches,{29970,0}}, {garbage_collection,{7826,35111208,0}}, {io,{{input,7612103},{output,809877}}}, {reductions,{20178758,53327}}, {run_queue,0}, {runtime,{1470,0}}]}, {system_stats, [{cpu_utilization_rate,2.75}, {swap_total,10684628992}, {swap_used,151552}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,8}, {memory_data, {33657663488,1387597824,{<17795.6.0>,601312}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 31431588 kB\nBuffers: 289144 kB\nCached: 613972 kB\nSwapCached: 0 kB\nActive: 650308 kB\nInactive: 441160 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 31431588 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434060 kB\nDirty: 216768 kB\nWriteback: 0 kB\nAnonPages: 188288 kB\nMapped: 14140 kB\nSlab: 307428 kB\nPageTables: 4992 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 1803552 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,918844,432684}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,42588992}, {processes,10281888}, {processes_used,10219496}, {system,32307104}, {atom,1234201}, {atom_used,1229651}, {binary,257704}, {code,12479739}, {ets,1013144}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585772032}, {total_swap,10585923584}, {cached_memory,627597312}, {buffered_memory,284946432}, {free_memory,32259301376}, {total_memory,33657663488}]}, {node_storage_conf, [{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{4945,2}}, {context_switches,{29666,0}}, {garbage_collection,{8073,34933181,0}}, {io,{{input,7415028},{output,821929}}}, {reductions,{20149613,38703}}, {run_queue,1}, {runtime,{1310,0}}]}, {system_stats, [{cpu_utilization_rate,0}, {swap_total,0}, {swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,4}, {memory_data, {33657663488,1372852224,{<0.6.0>,601352}}}, {disk_data, [{"/",49781912,18}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 31503224 kB\nBuffers: 278260 kB\nCached: 612896 kB\nSwapCached: 0 kB\nActive: 594652 kB\nInactive: 427580 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 31503224 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337668 kB\nDirty: 199756 kB\nWriteback: 0 kB\nAnonPages: 130820 kB\nMapped: 13616 kB\nSlab: 305484 kB\nPageTables: 4672 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 1581744 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-14T10:47:32.798,ns_1@10.2.1.63:<0.682.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,48,38}}) Stats for bucket "default": accepting_conns 1 auth_cmds 0 auth_errors 0 bucket_active_conns 1 bucket_conns 5 bytes 255904 bytes_read 196 bytes_written 120 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 0 conn_yields 0 connection_structures 5000 curr_connections 9 curr_conns_on_port_11209 6 curr_conns_on_port_11210 1 curr_items 0 curr_items_tot 0 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-15 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data2/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 0 ep_commit_time 0 ep_commit_time_total 0 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 0 ep_data_age_highwat 0 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data2/default ep_degraded_mode 1 ep_diskqueue_drain 0 ep_diskqueue_fill 0 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 0 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 0 ep_io_read_bytes 0 ep_io_write_bytes 0 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 0 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 0 ep_max_checkpoints 2 ep_max_data_size 12582912000 ep_max_item_size 20971520 ep_max_size 12582912000 ep_max_txn_size 10000 ep_max_vbuckets 1024 ep_mem_high_wat 9437184000 ep_mem_high_wat 9437184000 ep_mem_low_wat 7549747200 ep_mem_low_wat 7549747200 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 1152 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1352918852 ep_storage_age 0 ep_storage_age_highwat 0 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 0 ep_total_del_items 0 ep_total_enqueued 0 ep_total_new_items 0 ep_total_persisted 0 ep_uncommitted_items 0 ep_value_size 0 ep_vb0 0 ep_vb_snapshot_total 1 ep_vb_total 0 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_135_g40a391b ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 759 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 255904 pid 16783 pointer_size 64 rejected_conns 0 rusage_system 0.019996 rusage_user 0.028995 threads 4 time 1352918852 total_connections 10 uptime 10 vb_active_curr_items 0 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 0 vb_active_itm_memory 0 vb_active_meta_data_memory 0 vb_active_num 0 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 0 vb_active_ops_create 0 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 0 vb_active_queue_age 0 vb_active_queue_drain 0 vb_active_queue_fill 0 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_meta_data_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-14T10:48:24.428,ns_1@10.2.1.63:ns_doctor<0.540.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.61', [{last_heard,{1352,918902,777133}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,153808544}, {processes,112115456}, {processes_used,112102960}, {system,41693088}, {atom,1393705}, {atom_used,1362813}, {binary,6080632}, {code,13894422}, {ets,1987272}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,592027648}, {buffered_memory,47194112}, {free_memory,31226195968}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{66880,5002}}, {context_switches,{675544,0}}, {garbage_collection,{79465,327977081,0}}, {io,{{input,20112877},{output,590307417}}}, {reductions,{188441602,8190873}}, {run_queue,0}, {runtime,{15650,1420}}]}, {system_stats, [{cpu_utilization_rate,46.25}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,366847785}, {couch_docs_data_size,350205617}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,230559}, {curr_items_tot,456412}, {mem_used,1240463816}, {vb_replica_curr_items,225853}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,66}, {memory_data,{33657663488,1667715072,{<17795.67.0>,2172424}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 30494704 kB\nBuffers: 46088 kB\nCached: 578152 kB\nSwapCached: 0 kB\nActive: 1707496 kB\nInactive: 575200 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 30494704 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 5768 kB\nWriteback: 0 kB\nAnonPages: 1658116 kB\nMapped: 17280 kB\nSlab: 49732 kB\nPageTables: 8424 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 3365876 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,918899,428371}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.61',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,150618552}, {processes,109157592}, {processes_used,109142936}, {system,41460960}, {atom,1337385}, {atom_used,1332169}, {binary,6408176}, {code,13469049}, {ets,1924680}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,490827776}, {buffered_memory,46149632}, {free_memory,31551938560}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{59941,4999}}, {context_switches,{609023,0}}, {garbage_collection,{74730,276278467,0}}, {io,{{input,18986439},{output,493429276}}}, {reductions,{164599759,5904166}}, {run_queue,0}, {runtime,{14590,1390}}]}, {system_stats, [{cpu_utilization_rate,0.5}, {swap_total,10585923584}, {swap_used,151552}]}, {interesting_stats, [{couch_docs_actual_disk_size,394458921}, {couch_docs_data_size,377833940}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,194148}, {curr_items_tot,383474}, {mem_used,1045923976}, {vb_replica_curr_items,189326}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,59}, {memory_data,{33657663488,1372852224,{<0.6.0>,601352}}}, {disk_data, [{"/",49781912,18}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 30812440 kB\nBuffers: 45068 kB\nCached: 479324 kB\nSwapCached: 0 kB\nActive: 1486652 kB\nInactive: 477832 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 30812440 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 5288 kB\nWriteback: 352 kB\nAnonPages: 1439764 kB\nMapped: 17264 kB\nSlab: 49840 kB\nPageTables: 7872 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 3160724 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-14T10:49:12.798,ns_1@10.2.1.63:<0.682.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,48,38}}) Stats for bucket "default": accepting_conns 1 auth_cmds 7 auth_errors 0 bucket_active_conns 1 bucket_conns 14 bytes 4503764600 bytes_read 4083958270 bytes_written 2073652983 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 1 cmd_set 848818 conn_yields 15110 connection_structures 5000 curr_connections 17 curr_conns_on_port_11209 10 curr_conns_on_port_11210 5 curr_items 848818 curr_items_tot 1693814 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-15 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data2/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 135 ep_commit_time 16 ep_commit_time_total 2071 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 10 ep_data_age_highwat 33 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data2/default ep_degraded_mode 0 ep_diskqueue_drain 1312400 ep_diskqueue_fill 1693814 ep_diskqueue_items 381344 ep_diskqueue_memory 12205280 ep_diskqueue_pending 18307968 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 71 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 363259 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 1312200 ep_io_read_bytes 0 ep_io_write_bytes 3107253382 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 0 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 4143896978 ep_max_checkpoints 2 ep_max_data_size 12582912000 ep_max_item_size 20971520 ep_max_size 12582912000 ep_max_txn_size 10000 ep_max_vbuckets 1024 ep_mem_high_wat 9437184000 ep_mem_high_wat 9437184000 ep_mem_low_wat 7549747200 ep_mem_low_wat 7549747200 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 179186832 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 18085 ep_startup_time 1352918852 ep_storage_age 13 ep_storage_age_highwat 30 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 4123571210 ep_total_del_items 0 ep_total_enqueued 1693814 ep_total_new_items 1312200 ep_total_persisted 1312200 ep_uncommitted_items 4805 ep_value_size 4004226779 ep_vb0 0 ep_vb_snapshot_total 6 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_135_g40a391b ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 759 get_hits 0 get_misses 1 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 4503764600 pid 16783 pointer_size 64 rejected_conns 0 rusage_system 43.271421 rusage_user 63.672320 tap_checkpoint_start_received 512 tap_checkpoint_start_sent 512 tap_connect_received 1 tap_mutation_received 844996 tap_mutation_sent 845443 tap_opaque_received 1026 tap_opaque_sent 1026 threads 4 time 1352918952 total_connections 23 uptime 110 vb_active_curr_items 848818 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 12857344 vb_active_itm_memory 2066692582 vb_active_meta_data_memory 67056622 vb_active_num 512 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 848818 vb_active_ops_create 653615 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 100 vb_active_queue_age 1634076000 vb_active_queue_drain 653615 vb_active_queue_fill 848818 vb_active_queue_memory 6246528 vb_active_queue_pending 9369792 vb_active_queue_size 195204 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 844996 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 12857344 vb_replica_itm_memory 2056878628 vb_replica_meta_data_memory 66754684 vb_replica_num 512 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 844996 vb_replica_ops_create 658784 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 100 vb_replica_queue_age 1635033000 vb_replica_queue_drain 658785 vb_replica_queue_fill 844996 vb_replica_queue_memory 5958752 vb_replica_queue_pending 8938176 vb_replica_queue_size 186211 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-14T10:49:24.428,ns_1@10.2.1.63:ns_doctor<0.540.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.61', [{last_heard,{1352,918962,773680}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,185646496}, {processes,134011416}, {processes_used,133998840}, {system,51635080}, {atom,1393705}, {atom_used,1362813}, {binary,15568312}, {code,13894422}, {ets,2249760}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,3751354368}, {buffered_memory,86867968}, {free_memory,23880224768}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{126876,4999}}, {context_switches,{1692056,0}}, {garbage_collection,{138143,512139833,0}}, {io,{{input,23972333},{output,2355560570}}}, {reductions,{269971637,7817508}}, {run_queue,0}, {runtime,{30410,1250}}]}, {system_stats, [{cpu_utilization_rate,54.636591478696744}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,3433049937}, {couch_docs_data_size,3311023004}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,955676}, {curr_items_tot,1909479}, {mem_used,5084111464}, {vb_replica_curr_items,953803}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,126}, {memory_data,{33657663488,8949936128,{<17795.619.0>,4714984}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 23320656 kB\nBuffers: 84832 kB\nCached: 3663432 kB\nSwapCached: 0 kB\nActive: 5874532 kB\nInactive: 3488892 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 23320656 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 5428 kB\nWriteback: 1016 kB\nAnonPages: 5615172 kB\nMapped: 17284 kB\nSlab: 135124 kB\nPageTables: 16180 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 7335736 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,918959,428306}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.61',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,171783256}, {processes,130798200}, {processes_used,130783624}, {system,40985056}, {atom,1338193}, {atom_used,1333766}, {binary,5389592}, {code,13493302}, {ets,2227696}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,3732549632}, {buffered_memory,93126656}, {free_memory,24106065920}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{119941,4999}}, {context_switches,{1554017,0}}, {garbage_collection,{131292,452335164,0}}, {io,{{input,22924211},{output,2250827779}}}, {reductions,{231038678,5751713}}, {run_queue,0}, {runtime,{28830,1130}}]}, {system_stats, [{cpu_utilization_rate,53.88471177944862}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,3602128727}, {couch_docs_data_size,3455500291}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,920265}, {curr_items_tot,1837648}, {mem_used,4895026120}, {vb_replica_curr_items,917383}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,119}, {memory_data,{33657663488,2137214976,{<0.674.0>,4714984}}}, {disk_data, [{"/",49781912,18}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 23541204 kB\nBuffers: 90944 kB\nCached: 3645068 kB\nSwapCached: 0 kB\nActive: 5691884 kB\nInactive: 3451720 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 23541204 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 5660 kB\nWriteback: 0 kB\nAnonPages: 5407260 kB\nMapped: 17208 kB\nSlab: 134672 kB\nPageTables: 15696 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 7127920 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-14T10:50:24.428,ns_1@10.2.1.63:ns_doctor<0.540.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.61', [{last_heard,{1352,919022,775623}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,185312568}, {processes,140081368}, {processes_used,140066392}, {system,45231200}, {atom,1393705}, {atom_used,1362813}, {binary,8973176}, {code,13894422}, {ets,2435296}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7008292864}, {buffered_memory,113512448}, {free_memory,17541976064}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{186877,5000}}, {context_switches,{2467967,0}}, {garbage_collection,{185760,668998791,0}}, {io,{{input,30771968},{output,3650483980}}}, {reductions,{341306308,5074131}}, {run_queue,0}, {runtime,{41790,520}}]}, {system_stats, [{cpu_utilization_rate,18.75}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,6732468319}, {couch_docs_data_size,6401051048}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499534}, {curr_items_tot,3000000}, {mem_used,7952854856}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,186}, {memory_data,{33657663488,15775277056,{<17795.619.0>,2914384}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 17130960 kB\nBuffers: 110852 kB\nCached: 6844036 kB\nSwapCached: 0 kB\nActive: 9060876 kB\nInactive: 6398236 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 17130960 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 12444 kB\nWriteback: 32 kB\nAnonPages: 8504320 kB\nMapped: 17284 kB\nSlab: 222872 kB\nPageTables: 21912 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10205580 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,919019,429499}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.61',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,175205656}, {processes,131186752}, {processes_used,131172096}, {system,44018904}, {atom,1338193}, {atom_used,1333766}, {binary,8237824}, {code,13493302}, {ets,2410760}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7010332672}, {buffered_memory,118738944}, {free_memory,17515819008}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{179942,5000}}, {context_switches,{2326779,0}}, {garbage_collection,{177111,614140932,0}}, {io,{{input,27726995},{output,3641643158}}}, {reductions,{298780026,5043117}}, {run_queue,0}, {runtime,{40870,600}}]}, {system_stats, [{cpu_utilization_rate,20.25}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,6900628629}, {couch_docs_data_size,6504651329}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000000}, {mem_used,7952024728}, {vb_replica_curr_items,1499534}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,179}, {memory_data,{33657663488,9584357376,{<0.675.0>,4714984}}}, {disk_data, [{"/",49781912,18}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 17105292 kB\nBuffers: 115956 kB\nCached: 6846028 kB\nSwapCached: 0 kB\nActive: 9148556 kB\nInactive: 6335896 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 17105292 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 12200 kB\nWriteback: 104 kB\nAnonPages: 8522256 kB\nMapped: 17284 kB\nSlab: 223000 kB\nPageTables: 21976 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10219632 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-14T10:50:52.801,ns_1@10.2.1.63:<0.682.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,48,38}}) Stats for bucket "default": accepting_conns 1 auth_cmds 7 auth_errors 0 bucket_active_conns 1 bucket_conns 11 bytes 7948905768 bytes_read 7233362671 bytes_written 3682147533 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 1 cmd_set 1500466 conn_yields 26080 connection_structures 5000 curr_connections 14 curr_conns_on_port_11209 10 curr_conns_on_port_11210 2 curr_items 1500466 curr_items_tot 3000000 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-15 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data2/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 303 ep_commit_time 8 ep_commit_time_total 5822 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 33 ep_data_age_highwat 47 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data2/default ep_degraded_mode 0 ep_diskqueue_drain 3000000 ep_diskqueue_fill 3000000 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 162 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 3000000 ep_io_read_bytes 0 ep_io_write_bytes 7103890861 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 0 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 7339240416 ep_max_checkpoints 2 ep_max_data_size 12582912000 ep_max_item_size 20971520 ep_max_size 12582912000 ep_max_txn_size 10000 ep_max_vbuckets 1024 ep_mem_high_wat 9437184000 ep_mem_high_wat 9437184000 ep_mem_low_wat 7549747200 ep_mem_low_wat 7549747200 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 291224736 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1352918852 ep_storage_age 33 ep_storage_age_highwat 46 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 7303240416 ep_total_del_items 0 ep_total_enqueued 3000000 ep_total_new_items 3000000 ep_total_persisted 3000000 ep_uncommitted_items 0 ep_value_size 7091890861 ep_vb0 0 ep_vb_snapshot_total 6 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_135_g40a391b ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 759 get_hits 0 get_misses 1 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 7948905768 pid 16783 pointer_size 64 rejected_conns 0 rusage_system 77.902157 rusage_user 132.984783 tap_checkpoint_start_received 512 tap_checkpoint_start_sent 512 tap_connect_received 1 tap_mutation_received 1499534 tap_mutation_sent 1500466 tap_opaque_received 1026 tap_opaque_sent 1026 threads 4 time 1352919052 total_connections 24 uptime 210 vb_active_curr_items 1500466 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 12857056 vb_active_itm_memory 3653055086 vb_active_meta_data_memory 118536814 vb_active_num 512 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1500466 vb_active_ops_create 1500466 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 100 vb_active_queue_age 0 vb_active_queue_drain 1500466 vb_active_queue_fill 1500466 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 1499534 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 12857152 vb_replica_itm_memory 3650185330 vb_replica_meta_data_memory 118463186 vb_replica_num 512 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 1499534 vb_replica_ops_create 1499534 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 100 vb_replica_queue_age 0 vb_replica_queue_drain 1499534 vb_replica_queue_fill 1499534 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-14T10:51:24.428,ns_1@10.2.1.63:ns_doctor<0.540.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.61', [{last_heard,{1352,919082,774917}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,194191824}, {processes,150751968}, {processes_used,150736992}, {system,43439856}, {atom,1393705}, {atom_used,1362813}, {binary,7006208}, {code,13894422}, {ets,2610544}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7732658176}, {buffered_memory,116645888}, {free_memory,16787681280}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{246877,5000}}, {context_switches,{2700702,0}}, {garbage_collection,{198320,763465942,0}}, {io,{{input,35958750},{output,3654138443}}}, {reductions,{403018000,4807781}}, {run_queue,0}, {runtime,{49190,530}}]}, {system_stats, [{cpu_utilization_rate,2.255639097744361}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7702629509}, {couch_docs_data_size,7284522645}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499534}, {curr_items_tot,3000000}, {mem_used,7948905752}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,246}, {memory_data,{33657663488,16872300544,{<17795.620.0>,4714984}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16394220 kB\nBuffers: 113912 kB\nCached: 7551424 kB\nSwapCached: 0 kB\nActive: 9156584 kB\nInactive: 7021856 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16394220 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 8396 kB\nWriteback: 0 kB\nAnonPages: 8513164 kB\nMapped: 15404 kB\nSlab: 241016 kB\nPageTables: 21504 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10213668 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,919079,429921}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.61',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,188320584}, {processes,144085808}, {processes_used,144068752}, {system,44234776}, {atom,1338193}, {atom_used,1333766}, {binary,8276152}, {code,13493302}, {ets,2587912}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7799922688}, {buffered_memory,126025728}, {free_memory,16667049984}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{239942,5000}}, {context_switches,{2582406,0}}, {garbage_collection,{191622,707108216,0}}, {io,{{input,36994469},{output,3648333396}}}, {reductions,{353051635,2571257}}, {run_queue,0}, {runtime,{49340,560}}]}, {system_stats, [{cpu_utilization_rate,3.7593984962406015}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7776722048}, {couch_docs_data_size,7286663413}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000000}, {mem_used,7948905752}, {vb_replica_curr_items,1499534}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,239}, {memory_data,{33657663488,16158351360,{<0.674.0>,2914384}}}, {disk_data, [{"/",49781912,18}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16276416 kB\nBuffers: 123072 kB\nCached: 7617112 kB\nSwapCached: 0 kB\nActive: 9382272 kB\nInactive: 6910636 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16276416 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 7880 kB\nWriteback: 0 kB\nAnonPages: 8552780 kB\nMapped: 15592 kB\nSlab: 243228 kB\nPageTables: 21588 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10250936 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-14T10:52:24.427,ns_1@10.2.1.63:ns_doctor<0.540.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.61', [{last_heard,{1352,919142,774808}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,186173408}, {processes,141195288}, {processes_used,141180312}, {system,44978120}, {atom,1393705}, {atom_used,1362813}, {binary,8361392}, {code,13894422}, {ets,2792928}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,10358198272}, {buffered_memory,135983104}, {free_memory,14064672768}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{306877,4998}}, {context_switches,{4048755,0}}, {garbage_collection,{273251,1027207117,0}}, {io,{{input,41000645},{output,5822540540}}}, {reductions,{515589550,17539170}}, {run_queue,0}, {runtime,{66950,1750}}]}, {system_stats, [{cpu_utilization_rate,44.27860696517413}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,10009443464}, {couch_docs_data_size,7288895158}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499534}, {curr_items_tot,3000000}, {mem_used,7952312424}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,306}, {memory_data,{33657663488,19174096896,{<17795.619.0>,2914384}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 13735032 kB\nBuffers: 132796 kB\nCached: 10115428 kB\nSwapCached: 0 kB\nActive: 9690452 kB\nInactive: 9073024 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 13735032 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 12456 kB\nWriteback: 580 kB\nAnonPages: 8514976 kB\nMapped: 17284 kB\nSlab: 314176 kB\nPageTables: 21932 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10222804 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,919139,430097}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.61',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,184835480}, {processes,139775376}, {processes_used,139758400}, {system,45060104}, {atom,1338193}, {atom_used,1333766}, {binary,8917568}, {code,13493302}, {ets,2771224}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,10258509824}, {buffered_memory,141045760}, {free_memory,14122467328}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{299943,5002}}, {context_switches,{3730147,0}}, {garbage_collection,{256945,923525634,0}}, {io,{{input,41105470},{output,5730573662}}}, {reductions,{432831083,8533561}}, {run_queue,0}, {runtime,{64990,1500}}]}, {system_stats, [{cpu_utilization_rate,79.4486215538847}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,10132527588}, {couch_docs_data_size,7290815751}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000000}, {mem_used,7950288232}, {vb_replica_curr_items,1499534}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,299}, {memory_data,{33657663488,16990486528,{<0.674.0>,2914384}}}, {disk_data, [{"/",49781912,18}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 13791720 kB\nBuffers: 137740 kB\nCached: 10018076 kB\nSwapCached: 0 kB\nActive: 9825064 kB\nInactive: 8884692 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 13791720 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 12392 kB\nWriteback: 0 kB\nAnonPages: 8554044 kB\nMapped: 17284 kB\nSlab: 311588 kB\nPageTables: 22040 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10256976 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-14T10:52:32.800,ns_1@10.2.1.63:<0.682.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,48,39}}) Stats for bucket "default": accepting_conns 1 auth_cmds 19 auth_errors 0 bucket_active_conns 1 bucket_conns 11 bytes 7956108376 bytes_read 11575798008 bytes_written 5892593130 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 2 cmd_set 2400776 conn_yields 41473 connection_structures 5000 curr_connections 14 curr_conns_on_port_11209 10 curr_conns_on_port_11210 2 curr_items 1500466 curr_items_tot 3000000 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-15 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data2/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 405 ep_commit_time 96 ep_commit_time_total 8915 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 14 ep_data_age_highwat 48 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data2/default ep_degraded_mode 0 ep_diskqueue_drain 4001136 ep_diskqueue_fill 4800000 ep_diskqueue_items 798865 ep_diskqueue_memory 25563648 ep_diskqueue_pending 38345472 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 218 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 785507 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 4000278 ep_io_read_bytes 0 ep_io_write_bytes 9472006807 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 0 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 7339240416 ep_max_checkpoints 2 ep_max_data_size 12582912000 ep_max_item_size 20971520 ep_max_size 12582912000 ep_max_txn_size 10000 ep_max_vbuckets 1024 ep_mem_high_wat 9437184000 ep_mem_high_wat 9437184000 ep_mem_low_wat 7549747200 ep_mem_low_wat 7549747200 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 297508864 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 13358 ep_startup_time 1352918852 ep_storage_age 14 ep_storage_age_highwat 46 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 7303240416 ep_total_del_items 0 ep_total_enqueued 4800000 ep_total_new_items 3000000 ep_total_persisted 4000278 ep_uncommitted_items 2653 ep_value_size 7091890861 ep_vb0 0 ep_vb_snapshot_total 6 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_135_g40a391b ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 759 get_hits 1 get_misses 1 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 7956108376 pid 16783 pointer_size 64 rejected_conns 0 rusage_system 117.030208 rusage_user 201.472371 tap_checkpoint_start_received 512 tap_checkpoint_start_sent 512 tap_connect_received 1 tap_mutation_received 2399224 tap_mutation_sent 2400776 tap_opaque_received 1026 tap_opaque_sent 1026 threads 4 time 1352919152 total_connections 37 uptime 310 vb_active_curr_items 1500466 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 12857056 vb_active_itm_memory 3653302894 vb_active_meta_data_memory 118536814 vb_active_num 512 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1500466 vb_active_ops_create 1500466 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 457146 vb_active_perc_mem_resident 100 vb_active_queue_age 10944958000 vb_active_queue_drain 1957612 vb_active_queue_fill 2400776 vb_active_queue_memory 14181248 vb_active_queue_pending 21271872 vb_active_queue_size 443164 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 1499534 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 12857152 vb_replica_itm_memory 3649937522 vb_replica_meta_data_memory 118463186 vb_replica_num 512 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 1499534 vb_replica_ops_create 1499534 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 543989 vb_replica_perc_mem_resident 100 vb_replica_queue_age 8694685000 vb_replica_queue_drain 2043524 vb_replica_queue_fill 2399224 vb_replica_queue_memory 11382400 vb_replica_queue_pending 17073600 vb_replica_queue_size 355700 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-14T10:53:24.428,ns_1@10.2.1.63:ns_doctor<0.540.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.61', [{last_heard,{1352,919202,775812}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.9071.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,20}, {started_on,1352919104}, {total_vbuckets,1025}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1352919139}, {vbuckets_done,210}]]}, {memory, [{total,199950264}, {processes,150482840}, {processes_used,150468136}, {system,49467424}, {atom,1393705}, {atom_used,1362813}, {binary,12428352}, {code,13894422}, {ets,3178544}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,11220881408}, {buffered_memory,142946304}, {free_memory,13157601280}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{366877,4999}}, {context_switches,{4364859,0}}, {garbage_collection,{296749,1173637642,0}}, {io,{{input,58479828},{output,5852089725}}}, {reductions,{635908674,11295943}}, {run_queue,0}, {runtime,{77250,880}}]}, {system_stats, [{cpu_utilization_rate,24.25}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,11304856134}, {couch_docs_data_size,7290109677}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499534}, {curr_items_tot,3000000}, {mem_used,7951590344}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,366}, {memory_data,{33657663488,20651905024,{<17795.620.0>,4714984}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 12849220 kB\nBuffers: 139596 kB\nCached: 10957892 kB\nSwapCached: 0 kB\nActive: 9697724 kB\nInactive: 9927320 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 12849220 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 21104 kB\nWriteback: 208 kB\nAnonPages: 8527388 kB\nMapped: 17780 kB\nSlab: 338520 kB\nPageTables: 21980 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10225456 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,919199,429215}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.61',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.8315.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,29}, {started_on,1352919144}, {total_vbuckets,1025}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1352919199}, {vbuckets_done,305}]]}, {memory, [{total,196110472}, {processes,147738376}, {processes_used,147731712}, {system,48372096}, {atom,1338193}, {atom_used,1333766}, {binary,11709800}, {code,13493302}, {ets,3249184}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,10005041152}, {buffered_memory,146849792}, {free_memory,14376615936}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{359942,4999}}, {context_switches,{4038208,0}}, {garbage_collection,{279879,1019255746,0}}, {io,{{input,48839991},{output,5832988433}}}, {reductions,{478090693,3686243}}, {run_queue,0}, {runtime,{71970,570}}]}, {system_stats, [{cpu_utilization_rate,23.19201995012469}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,9987140905}, {couch_docs_data_size,7289922339}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000000}, {mem_used,7954152280}, {vb_replica_curr_items,1499534}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,359}, {memory_data,{33657663488,19563130880,{<0.674.0>,2914384}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 14039664 kB\nBuffers: 143408 kB\nCached: 9770548 kB\nSwapCached: 0 kB\nActive: 9572896 kB\nInactive: 8894012 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 14039664 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 20972 kB\nWriteback: 232 kB\nAnonPages: 8552656 kB\nMapped: 17276 kB\nSlab: 306236 kB\nPageTables: 22212 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10244088 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-14T10:54:12.799,ns_1@10.2.1.63:<0.682.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,48,39}}) Stats for bucket "default": accepting_conns 1 auth_cmds 19 auth_errors 0 bucket_active_conns 1 bucket_conns 11 bytes 7950673528 bytes_read 11575809166 bytes_written 5900066160 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 2 cmd_set 2400776 conn_yields 41473 connection_structures 5000 curr_connections 14 curr_conns_on_port_11209 10 curr_conns_on_port_11210 2 curr_items 1500466 curr_items_tot 3000000 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-15 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data2/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 474 ep_commit_time 89 ep_commit_time_total 17307 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 125 ep_data_age_highwat 137 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data2/default ep_degraded_mode 0 ep_diskqueue_drain 4717009 ep_diskqueue_fill 4800000 ep_diskqueue_items 82992 ep_diskqueue_memory 2655712 ep_diskqueue_pending 3983568 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 218 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 69634 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 4716195 ep_io_read_bytes 0 ep_io_write_bytes 11167111416 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 0 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 7339240416 ep_max_checkpoints 2 ep_max_data_size 12582912000 ep_max_item_size 20971520 ep_max_size 12582912000 ep_max_txn_size 10000 ep_max_vbuckets 1024 ep_mem_high_wat 9437184000 ep_mem_high_wat 9437184000 ep_mem_low_wat 7549747200 ep_mem_low_wat 7549747200 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 291781880 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 13358 ep_startup_time 1352918852 ep_storage_age 123 ep_storage_age_highwat 137 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 7303240416 ep_total_del_items 0 ep_total_enqueued 4800000 ep_total_new_items 3000000 ep_total_persisted 4716195 ep_uncommitted_items 4995 ep_value_size 7091890861 ep_vb0 0 ep_vb_snapshot_total 6 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_135_g40a391b ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 759 get_hits 1 get_misses 1 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 7950673528 pid 16783 pointer_size 64 rejected_conns 0 rusage_system 121.853475 rusage_user 227.749376 tap_checkpoint_start_received 512 tap_checkpoint_start_sent 512 tap_connect_received 1 tap_mutation_received 2399224 tap_mutation_sent 2400776 tap_opaque_received 1026 tap_opaque_sent 1026 threads 4 time 1352919252 total_connections 39 uptime 410 vb_active_curr_items 1500466 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 12857056 vb_active_itm_memory 3653302894 vb_active_meta_data_memory 118536814 vb_active_num 512 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1500466 vb_active_ops_create 1500466 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 825771 vb_active_perc_mem_resident 100 vb_active_queue_age 9227770000 vb_active_queue_drain 2326238 vb_active_queue_fill 2400776 vb_active_queue_memory 2385216 vb_active_queue_pending 3577824 vb_active_queue_size 74538 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 1499534 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 12857152 vb_replica_itm_memory 3649937522 vb_replica_meta_data_memory 118463186 vb_replica_num 512 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 1499534 vb_replica_ops_create 1499534 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 891237 vb_replica_perc_mem_resident 100 vb_replica_queue_age 923032000 vb_replica_queue_drain 2390771 vb_replica_queue_fill 2399224 vb_replica_queue_memory 270496 vb_replica_queue_pending 405744 vb_replica_queue_size 8453 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-14T10:54:24.428,ns_1@10.2.1.63:ns_doctor<0.540.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.61', [{last_heard,{1352,919262,776967}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.9071.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,59}, {started_on,1352919104}, {total_vbuckets,1025}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1352919199}, {vbuckets_done,611}]]}, {memory, [{total,195491112}, {processes,144946856}, {processes_used,144935608}, {system,50544256}, {atom,1393705}, {atom_used,1362813}, {binary,12862096}, {code,13894422}, {ets,3766312}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,9536462848}, {buffered_memory,149078016}, {free_memory,14871011328}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{426878,5001}}, {context_switches,{4680790,0}}, {garbage_collection,{322049,1318048290,0}}, {io,{{input,74398398},{output,5860077488}}}, {reductions,{755172155,11288672}}, {run_queue,0}, {runtime,{87730,850}}]}, {system_stats, [{cpu_utilization_rate,16.834170854271356}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,9714424889}, {couch_docs_data_size,7288684764}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499534}, {curr_items_tot,3000000}, {mem_used,7948932632}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,426}, {memory_data,{33657663488,19056668672,{<17795.620.0>,4714984}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 14518576 kB\nBuffers: 145584 kB\nCached: 9316888 kB\nSwapCached: 0 kB\nActive: 9230916 kB\nInactive: 8766516 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 14518576 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 17892 kB\nWriteback: 0 kB\nAnonPages: 8534876 kB\nMapped: 17312 kB\nSlab: 296892 kB\nPageTables: 21836 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10233432 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,919259,430159}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.61',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.8315.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,64}, {started_on,1352919144}, {total_vbuckets,1025}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1352919259}, {vbuckets_done,658}]]}, {memory, [{total,195731216}, {processes,146231976}, {processes_used,146219672}, {system,49499240}, {atom,1338193}, {atom_used,1333766}, {binary,12240520}, {code,13493302}, {ets,3792208}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,9216991232}, {buffered_memory,152141824}, {free_memory,15177867264}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{419943,5001}}, {context_switches,{4313015,0}}, {garbage_collection,{301964,1110591420,0}}, {io,{{input,60681661},{output,5837454139}}}, {reductions,{522622962,3729432}}, {run_queue,0}, {runtime,{79050,680}}]}, {system_stats, [{cpu_utilization_rate,22.80701754385965}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,9187789931}, {couch_docs_data_size,7288862399}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000000}, {mem_used,7950353640}, {vb_replica_curr_items,1499534}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,419}, {memory_data,{33657663488,19284094976,{<0.674.0>,2914384}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 14822136 kB\nBuffers: 148576 kB\nCached: 9000968 kB\nSwapCached: 0 kB\nActive: 9203508 kB\nInactive: 8500132 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 14822136 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 21076 kB\nWriteback: 0 kB\nAnonPages: 8554260 kB\nMapped: 17284 kB\nSlab: 287056 kB\nPageTables: 22020 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10244376 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-14T10:55:24.428,ns_1@10.2.1.63:ns_doctor<0.540.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.61', [{last_heard,{1352,919322,775424}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,200905296}, {processes,152273960}, {processes_used,152251584}, {system,48631336}, {atom,1393705}, {atom_used,1362813}, {binary,10302584}, {code,13894422}, {ets,4359536}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7327703040}, {buffered_memory,155234304}, {free_memory,17138479104}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{486877,5000}}, {context_switches,{4994371,0}}, {garbage_collection,{351158,1487492831,0}}, {io,{{input,89353536},{output,5867675127}}}, {reductions,{869693045,4763533}}, {run_queue,0}, {runtime,{97980,550}}]}, {system_stats, [{cpu_utilization_rate,2.5},{swap_total,10684628992},{swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7288347726}, {couch_docs_data_size,7286110552}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499534}, {curr_items_tot,3000000}, {mem_used,7948932616}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,486}, {memory_data,{33657663488,16521998336,{<17795.619.0>,2914384}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16736796 kB\nBuffers: 151596 kB\nCached: 7155960 kB\nSwapCached: 0 kB\nActive: 8665732 kB\nInactive: 7169328 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16736796 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 20404 kB\nWriteback: 0 kB\nAnonPages: 8527264 kB\nMapped: 15404 kB\nSlab: 241652 kB\nPageTables: 21500 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10214608 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,919319,428335}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.61',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.8315.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,99}, {started_on,1352919144}, {total_vbuckets,1025}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1352919319}, {vbuckets_done,1024}]]}, {memory, [{total,192276832}, {processes,145539944}, {processes_used,145119280}, {system,46736888}, {atom,1338193}, {atom_used,1333766}, {binary,9769152}, {code,13493302}, {ets,3544392}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7374376960}, {buffered_memory,158679040}, {free_memory,17063108608}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{479941,4999}}, {context_switches,{4637395,0}}, {garbage_collection,{329163,1256000584,0}}, {io,{{input,75471699},{output,5844488264}}}, {reductions,{620358408,3105851}}, {run_queue,0}, {runtime,{90400,510}}]}, {system_stats, [{cpu_utilization_rate,16.91919191919192}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7400823836}, {couch_docs_data_size,7286227702}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000000}, {mem_used,7949086088}, {vb_replica_curr_items,1499534}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,479}, {memory_data,{33657663488,18476744704,{<0.674.0>,2914384}}}, {disk_data, [{"/",49781912,18}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16663192 kB\nBuffers: 154960 kB\nCached: 7201540 kB\nSwapCached: 0 kB\nActive: 8837300 kB\nInactive: 7070384 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16663192 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 24088 kB\nWriteback: 100 kB\nAnonPages: 8552084 kB\nMapped: 15580 kB\nSlab: 241824 kB\nPageTables: 21348 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10235772 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-14T10:55:52.802,ns_1@10.2.1.63:<0.682.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,48,39}}) Stats for bucket "default": accepting_conns 1 auth_cmds 19 auth_errors 0 bucket_active_conns 1 bucket_conns 10 bytes 7949086088 bytes_read 11575820382 bytes_written 5907600846 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 2 cmd_set 2400776 conn_yields 41473 connection_structures 5000 curr_connections 13 curr_conns_on_port_11209 10 curr_conns_on_port_11210 1 curr_items 1500466 curr_items_tot 3000000 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-15 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data2/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 484 ep_commit_time 92 ep_commit_time_total 18491 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 150 ep_data_age_highwat 150 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data2/default ep_degraded_mode 0 ep_diskqueue_drain 4800000 ep_diskqueue_fill 4800000 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 368 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 4800000 ep_io_read_bytes 0 ep_io_write_bytes 11365781341 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 0 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 7339240416 ep_max_checkpoints 2 ep_max_data_size 12582912000 ep_max_item_size 20971520 ep_max_size 12582912000 ep_max_txn_size 10000 ep_max_vbuckets 1024 ep_mem_high_wat 9437184000 ep_mem_high_wat 9437184000 ep_mem_low_wat 7549747200 ep_mem_low_wat 7549747200 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 291224736 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1352918852 ep_storage_age 149 ep_storage_age_highwat 150 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 7303240416 ep_total_del_items 0 ep_total_enqueued 4800000 ep_total_new_items 3000000 ep_total_persisted 4800000 ep_uncommitted_items 0 ep_value_size 7091890861 ep_vb0 0 ep_vb_snapshot_total 6 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_135_g40a391b ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 759 get_hits 1 get_misses 1 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 7949086088 pid 16783 pointer_size 64 rejected_conns 0 rusage_system 123.359246 rusage_user 232.272689 tap_checkpoint_start_received 512 tap_checkpoint_start_sent 512 tap_connect_received 1 tap_mutation_received 2399224 tap_mutation_sent 2400776 tap_opaque_received 1026 tap_opaque_sent 1026 threads 4 time 1352919351 total_connections 41 uptime 509 vb_active_curr_items 1500466 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 12857056 vb_active_itm_memory 3653302894 vb_active_meta_data_memory 118536814 vb_active_num 512 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1500466 vb_active_ops_create 1500466 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 900310 vb_active_perc_mem_resident 100 vb_active_queue_age 0 vb_active_queue_drain 2400776 vb_active_queue_fill 2400776 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 1499534 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 12857152 vb_replica_itm_memory 3649937522 vb_replica_meta_data_memory 118463186 vb_replica_num 512 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 1499534 vb_replica_ops_create 1499534 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 899690 vb_replica_perc_mem_resident 100 vb_replica_queue_age 0 vb_replica_queue_drain 2399224 vb_replica_queue_fill 2399224 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-14T10:56:24.428,ns_1@10.2.1.63:ns_doctor<0.540.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,919380,27382}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',69379005}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,81085416}, {processes,45788952}, {processes_used,45782408}, {system,35296464}, {atom,1329305}, {atom_used,1319635}, {binary,1508496}, {code,13300329}, {ets,1683016}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,439586816}, {buffered_memory,13828096}, {free_memory,32659513344}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{542496,4999}}, {context_switches,{245301,0}}, {garbage_collection,{85164,151383809,0}}, {io,{{input,12230899},{output,475779442}}}, {reductions,{46664537,10884564}}, {run_queue,0}, {runtime,{29520,2390}}]}, {system_stats, [{cpu_utilization_rate,5.25}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,196378661}, {couch_docs_data_size,92358587}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,41090}, {curr_items_tot,76324}, {mem_used,202499224}, {vb_replica_curr_items,35234}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,542}, {memory_data,{33657663488,825831424,{<19415.367.0>,4114672}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 31894304 kB\nBuffers: 13504 kB\nCached: 429284 kB\nSwapCached: 0 kB\nActive: 499688 kB\nInactive: 392076 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 31894304 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 3544 kB\nWriteback: 0 kB\nAnonPages: 448856 kB\nMapped: 15584 kB\nSlab: 43392 kB\nPageTables: 5696 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 2158060 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,919382,779187}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',71935981}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,223161912}, {processes,176138448}, {processes_used,175943496}, {system,47023464}, {atom,1401785}, {atom_used,1374887}, {binary,8291120}, {code,14061510}, {ets,4576896}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7090450432}, {buffered_memory,157802496}, {free_memory,17343004672}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{546877,5000}}, {context_switches,{5245030,0}}, {garbage_collection,{410336,1814692816,0}}, {io,{{input,97196186},{output,5884677930}}}, {reductions,{940445721,11963869}}, {run_queue,0}, {runtime,{108920,2500}}]}, {system_stats, [{cpu_utilization_rate,32.66832917705736}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7160862143}, {couch_docs_data_size,7158257881}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1455877}, {curr_items_tot,2909413}, {mem_used,7713558856}, {vb_replica_curr_items,1453536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,546}, {memory_data,{33657663488,16467406848,{<17795.67.0>,12342376}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16936652 kB\nBuffers: 154104 kB\nCached: 6924268 kB\nSwapCached: 0 kB\nActive: 8698904 kB\nInactive: 6936672 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16936652 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 19804 kB\nWriteback: 0 kB\nAnonPages: 8557000 kB\nMapped: 16468 kB\nSlab: 240136 kB\nPageTables: 22024 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10275452 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,919379,429710}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default", [{'ns_1@10.2.1.58',62989170},{'ns_1@10.2.1.61',129307025}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,206483056}, {processes,159345824}, {processes_used,158924072}, {system,47137232}, {atom,1338193}, {atom_used,1334188}, {binary,9865624}, {code,13493302}, {ets,3806152}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7556280320}, {buffered_memory,160878592}, {free_memory,16861552640}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{539942,5000}}, {context_switches,{4910762,0}}, {garbage_collection,{391304,1418287610,0}}, {io,{{input,82366441},{output,6021518801}}}, {reductions,{668864578,12326287}}, {run_queue,0}, {runtime,{99080,1880}}]}, {system_stats, [{cpu_utilization_rate,40.25}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7469326028}, {couch_docs_data_size,7286026557}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1494469}, {curr_items_tot,3000003}, {mem_used,7953147272}, {vb_replica_curr_items,1505534}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,539}, {memory_data,{33657663488,16585338880,{<0.674.0>,2914384}}}, {disk_data, [{"/",49781912,18}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16466360 kB\nBuffers: 157108 kB\nCached: 7379180 kB\nSwapCached: 0 kB\nActive: 8910096 kB\nInactive: 7188048 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16466360 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 20936 kB\nWriteback: 0 kB\nAnonPages: 8561948 kB\nMapped: 16728 kB\nSlab: 247468 kB\nPageTables: 22308 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10265520 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-14T10:57:24.428,ns_1@10.2.1.63:ns_doctor<0.540.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,919440,26200}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',63506121}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,102494080}, {processes,64411568}, {processes_used,64396360}, {system,38082512}, {atom,1332537}, {atom_used,1324944}, {binary,3208088}, {code,13364289}, {ets,2296368}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,2027646976}, {buffered_memory,53514240}, {free_memory,29672099840}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{602495,5000}}, {context_switches,{1131379,0}}, {garbage_collection,{454097,715583485,0}}, {io,{{input,32226985},{output,3206691318}}}, {reductions,{160648431,8869472}}, {run_queue,0}, {runtime,{53580,1900}}]}, {system_stats, [{cpu_utilization_rate,5.25}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,1822012591}, {couch_docs_data_size,1227068174}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,277693}, {curr_items_tot,528998}, {mem_used,1399500240}, {vb_replica_curr_items,251305}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,602}, {memory_data,{33657663488,3829891072,{<19415.367.0>,5085816}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 28976784 kB\nBuffers: 52260 kB\nCached: 1980124 kB\nSwapCached: 0 kB\nActive: 1854372 kB\nInactive: 1907008 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 28976784 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 6260 kB\nWriteback: 968 kB\nAnonPages: 1728692 kB\nMapped: 15592 kB\nSlab: 88024 kB\nPageTables: 8272 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 3427716 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,919442,776906}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',87166797}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,242875176}, {processes,196823392}, {processes_used,195850472}, {system,46051784}, {atom,1401785}, {atom_used,1374887}, {binary,7548728}, {code,14061510}, {ets,4451472}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,5996150784}, {buffered_memory,160456704}, {free_memory,18413162496}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{606878,4999}}, {context_switches,{5589715,0}}, {garbage_collection,{612585,3997083928,0}}, {io,{{input,103899817},{output,5930436497}}}, {reductions,{1066064358,9249045}}, {run_queue,0}, {runtime,{143970,3000}}]}, {system_stats, [{cpu_utilization_rate,29.18781725888325}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,6032163531}, {couch_docs_data_size,6029874240}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1242485}, {curr_items_tot,2450486}, {mem_used,6529530248}, {vb_replica_curr_items,1208001}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,606}, {memory_data,{33657663488,15336648704,{<17795.427.0>,9356040}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 17981728 kB\nBuffers: 156696 kB\nCached: 5855616 kB\nSwapCached: 0 kB\nActive: 8740972 kB\nInactive: 5868744 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 17981728 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 21308 kB\nWriteback: 0 kB\nAnonPages: 8597636 kB\nMapped: 16492 kB\nSlab: 220724 kB\nPageTables: 22244 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10321572 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,919439,444613}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default", [{'ns_1@10.2.1.58',96745982},{'ns_1@10.2.1.61',74118087}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,209602080}, {processes,161593008}, {processes_used,161169456}, {system,48009072}, {atom,1338193}, {atom_used,1334188}, {binary,10543256}, {code,13493302}, {ets,3990136}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,8796856320}, {buffered_memory,167133184}, {free_memory,15543635968}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{599957,5015}}, {context_switches,{5571639,0}}, {garbage_collection,{733800,2046681159,0}}, {io,{{input,108764249},{output,7126779590}}}, {reductions,{802399097,9806940}}, {run_queue,0}, {runtime,{120480,1610}}]}, {system_stats, [{cpu_utilization_rate,48.593350383631716}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,8720585217}, {couch_docs_data_size,7285189553}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1470998}, {curr_items_tot,3000003}, {mem_used,7978989496}, {vb_replica_curr_items,1529005}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,599}, {memory_data,{33657663488,16794689536,{<0.368.0>,5085816}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 15178340 kB\nBuffers: 163216 kB\nCached: 8590680 kB\nSwapCached: 0 kB\nActive: 9432528 kB\nInactive: 7923972 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 15178340 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 18312 kB\nWriteback: 0 kB\nAnonPages: 8602368 kB\nMapped: 16404 kB\nSlab: 277144 kB\nPageTables: 22112 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10315616 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-14T10:57:32.810,ns_1@10.2.1.63:<0.682.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,48,39}}) Stats for bucket "default": accepting_conns 1 auth_cmds 27 auth_errors 0 bucket_active_conns 2 bucket_conns 25 bytes 7983616024 bytes_read 13033482370 bytes_written 8555670706 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 82 cmd_set 2400893 conn_yields 46404 connection_structures 5000 curr_connections 25 curr_conns_on_port_11209 17 curr_conns_on_port_11210 7 curr_items 1468053 curr_items_tot 3000003 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 17 delete_misses 1 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-15 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data2/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 3721 ep_commit_time 5 ep_commit_time_total 48235 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 1 ep_data_age_highwat 150 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data2/default ep_degraded_mode 0 ep_diskqueue_drain 6268998 ep_diskqueue_fill 5377819 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 406 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 5401904 ep_io_read_bytes 0 ep_io_write_bytes 12791590283 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 26134 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 7339251617 ep_max_checkpoints 2 ep_max_data_size 12582912000 ep_max_item_size 20971520 ep_max_size 12582912000 ep_max_txn_size 10000 ep_max_vbuckets 1024 ep_mem_high_wat 9437184000 ep_mem_high_wat 9437184000 ep_mem_low_wat 7549747200 ep_mem_low_wat 7549747200 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 284973190 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1352918852 ep_storage_age 0 ep_storage_age_highwat 560 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 7303251581 ep_total_del_items 2 ep_total_enqueued 5377819 ep_total_new_items 3000005 ep_total_persisted 5401904 ep_uncommitted_items 0 ep_value_size 7091901812 ep_vb0 0 ep_vb_snapshot_total 580 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_135_g40a391b ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 759 get_hits 69 get_misses 13 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 7983616024 pid 16783 pointer_size 64 rejected_conns 0 rusage_system 141.560479 rusage_user 300.490318 tap_checkpoint_end_received 206 tap_checkpoint_end_sent 318 tap_checkpoint_start_received 927 tap_checkpoint_start_sent 1088 tap_connect_received 227 tap_delete_received 1 tap_delete_sent 1 tap_mutation_received 3001088 tap_mutation_sent 3332171 tap_opaque_received 1959 tap_opaque_sent 1916 tap_vbucket_set_received 202 tap_vbucket_set_sent 224 threads 4 time 1352919451 total_connections 1337 uptime 609 vb_active_curr_items 1468053 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 12580824 vb_active_itm_memory 3574253171 vb_active_meta_data_memory 115976187 vb_active_num 501 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1468053 vb_active_ops_create 1468054 vb_active_ops_delete 1 vb_active_ops_reject 0 vb_active_ops_update 1176434 vb_active_perc_mem_resident 100 vb_active_queue_age 0 vb_active_queue_drain 3203195 vb_active_queue_fill 2630673 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 1531950 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 13133384 vb_replica_itm_memory 3728998410 vb_replica_meta_data_memory 121024050 vb_replica_num 523 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 1531951 vb_replica_ops_create 1531951 vb_replica_ops_delete 1 vb_replica_ops_reject 0 vb_replica_ops_update 1225463 vb_replica_perc_mem_resident 100 vb_replica_queue_age 0 vb_replica_queue_drain 3065803 vb_replica_queue_fill 2747146 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-14T10:58:24.428,ns_1@10.2.1.63:ns_doctor<0.540.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,919500,26172}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',128859274}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.10994.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,18}, {started_on,1352919504}, {total_vbuckets,288}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1352919512}, {vbuckets_done,54}]]}, {memory, [{total,115915424}, {processes,77419120}, {processes_used,77401496}, {system,38496304}, {atom,1332537}, {atom_used,1324944}, {binary,2918280}, {code,13364289}, {ets,2651552}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,2506522624}, {buffered_memory,77803520}, {free_memory,28180885504}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{662495,5000}}, {context_switches,{1849879,0}}, {garbage_collection,{719063,1152290106,0}}, {io,{{input,48444638},{output,5254627575}}}, {reductions,{253526341,7815246}}, {run_queue,0}, {runtime,{72520,1630}}]}, {system_stats, [{cpu_utilization_rate,49.25}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,2372405905}, {couch_docs_data_size,2073506068}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,465527}, {curr_items_tot,877903}, {mem_used,2323024208}, {vb_replica_curr_items,412376}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,662}, {memory_data,{33657663488,5408337920,{<19415.367.0>,5085816}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 27520644 kB\nBuffers: 75980 kB\nCached: 2447776 kB\nSwapCached: 0 kB\nActive: 2758508 kB\nInactive: 2442724 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 27520644 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 15688 kB\nWriteback: 0 kB\nAnonPages: 2677800 kB\nMapped: 15596 kB\nSlab: 102588 kB\nPageTables: 10244 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 4382068 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,919502,777150}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',507061}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,207284096}, {processes,162244488}, {processes_used,160690048}, {system,45039608}, {atom,1401785}, {atom_used,1374887}, {binary,6570160}, {code,14061510}, {ets,4499592}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,5169197056}, {buffered_memory,162361344}, {free_memory,19241656320}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{666878,5001}}, {context_switches,{5867454,0}}, {garbage_collection,{750994,6344620192,0}}, {io,{{input,109030123},{output,5964768484}}}, {reductions,{1162921670,7429718}}, {run_queue,0}, {runtime,{177180,2550}}]}, {system_stats, [{cpu_utilization_rate,27.75}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,5185249013}, {couch_docs_data_size,5183272872}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1081194}, {curr_items_tot,2110442}, {mem_used,5640341768}, {vb_replica_curr_items,1029248}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,666}, {memory_data,{33657663488,14503849984,{<17795.427.0>,25150688}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 18790680 kB\nBuffers: 158556 kB\nCached: 5048044 kB\nSwapCached: 0 kB\nActive: 8754176 kB\nInactive: 5061592 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 18790680 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 19144 kB\nWriteback: 0 kB\nAnonPages: 8609488 kB\nMapped: 16624 kB\nSlab: 205044 kB\nPageTables: 22332 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10336872 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,919499,430394}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default", [{'ns_1@10.2.1.58',8601115},{'ns_1@10.2.1.61',15785876}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,214879464}, {processes,165414328}, {processes_used,164990800}, {system,49465136}, {atom,1338193}, {atom_used,1334188}, {binary,11926400}, {code,13493302}, {ets,4056744}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,9815769088}, {buffered_memory,172728320}, {free_memory,14462664704}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{659943,5002}}, {context_switches,{6156196,0}}, {garbage_collection,{999382,2524322305,0}}, {io,{{input,124282666},{output,7973060816}}}, {reductions,{909498147,9013946}}, {run_queue,0}, {runtime,{138450,1450}}]}, {system_stats, [{cpu_utilization_rate,37.84461152882206}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,9723674798}, {couch_docs_data_size,7284646992}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1444455}, {curr_items_tot,3000003}, {mem_used,7998061352}, {vb_replica_curr_items,1555548}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,659}, {memory_data,{33657663488,18123296768,{<0.368.0>,5085816}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 14123696 kB\nBuffers: 168680 kB\nCached: 9585712 kB\nSwapCached: 0 kB\nActive: 9915816 kB\nInactive: 8469700 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 14123696 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 13308 kB\nWriteback: 0 kB\nAnonPages: 8631244 kB\nMapped: 16320 kB\nSlab: 302528 kB\nPageTables: 22548 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10344808 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-14T10:59:12.803,ns_1@10.2.1.63:<0.682.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,48,39}}) Stats for bucket "default": accepting_conns 1 auth_cmds 27 auth_errors 0 bucket_active_conns 2 bucket_conns 26 bytes 8012462456 bytes_read 14199592643 bytes_written 10694934604 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 82 cmd_set 2400893 conn_yields 50325 connection_structures 5000 curr_connections 27 curr_conns_on_port_11209 19 curr_conns_on_port_11210 7 curr_items 1441688 curr_items_tot 3000003 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 17 delete_misses 1 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-15 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data2/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 1 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 7077 ep_commit_time 138 ep_commit_time_total 95666 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 2 ep_data_age_highwat 150 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data2/default ep_degraded_mode 0 ep_diskqueue_drain 7441806 ep_diskqueue_fill 5834931 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 482 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 5883033 ep_io_read_bytes 0 ep_io_write_bytes 13929989139 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 58641 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 7339251617 ep_max_checkpoints 2 ep_max_data_size 12582912000 ep_max_item_size 20971520 ep_max_size 12582912000 ep_max_txn_size 10000 ep_max_vbuckets 1024 ep_mem_high_wat 9437184000 ep_mem_high_wat 9437184000 ep_mem_low_wat 7549747200 ep_mem_low_wat 7549747200 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 280388734 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1352918852 ep_storage_age 0 ep_storage_age_highwat 633 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 7303251581 ep_total_del_items 2 ep_total_enqueued 5834931 ep_total_new_items 3000005 ep_total_persisted 5883033 ep_uncommitted_items 186 ep_value_size 7091901812 ep_vb0 0 ep_vb_snapshot_total 985 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_135_g40a391b ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 759 get_hits 69 get_misses 13 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 8012462456 pid 16783 pointer_size 64 rejected_conns 0 rusage_system 157.651033 rusage_user 380.739118 tap_checkpoint_end_received 370 tap_checkpoint_end_sent 575 tap_checkpoint_start_received 1265 tap_checkpoint_start_sent 1548 tap_connect_received 409 tap_delete_received 1 tap_delete_sent 1 tap_mutation_received 3483220 tap_mutation_sent 4084601 tap_opaque_received 2717 tap_opaque_sent 2635 tap_vbucket_set_received 366 tap_vbucket_set_sent 406 threads 4 time 1352919552 total_connections 2376 uptime 710 vb_active_curr_items 1441688 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 12354816 vb_active_itm_memory 3509602952 vb_active_meta_data_memory 113893352 vb_active_num 492 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1441688 vb_active_ops_create 1441689 vb_active_ops_delete 1 vb_active_ops_reject 0 vb_active_ops_update 1400589 vb_active_perc_mem_resident 100 vb_active_queue_age 0 vb_active_queue_drain 3861549 vb_active_queue_fill 2816776 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 1558315 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 13359392 vb_replica_itm_memory 3793648629 vb_replica_meta_data_memory 123106885 vb_replica_num 532 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 1558316 vb_replica_ops_create 1558316 vb_replica_ops_delete 1 vb_replica_ops_reject 0 vb_replica_ops_update 1482623 vb_replica_perc_mem_resident 100 vb_replica_queue_age 0 vb_replica_queue_drain 3580257 vb_replica_queue_fill 3018155 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-14T10:59:24.427,ns_1@10.2.1.63:ns_doctor<0.540.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,919560,27222}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',51005697}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,132888000}, {processes,92638208}, {processes_used,92623176}, {system,40249792}, {atom,1332537}, {atom_used,1324944}, {binary,4307760}, {code,13364289}, {ets,2908384}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,3599450112}, {buffered_memory,100782080}, {free_memory,26234208256}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{722496,5001}}, {context_switches,{2473360,0}}, {garbage_collection,{928329,1515386287,0}}, {io,{{input,65066487},{output,6930216901}}}, {reductions,{332890264,6228366}}, {run_queue,0}, {runtime,{89490,1300}}]}, {system_stats, [{cpu_utilization_rate,38.0}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,3451531431}, {couch_docs_data_size,2771087243}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,609138}, {curr_items_tot,1163051}, {mem_used,3074758496}, {vb_replica_curr_items,553913}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,722}, {memory_data,{33657663488,7342944256,{<19415.367.0>,5085816}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 25619840 kB\nBuffers: 98420 kB\nCached: 3514760 kB\nSwapCached: 0 kB\nActive: 3640704 kB\nInactive: 3429796 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 25619840 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 16172 kB\nWriteback: 0 kB\nAnonPages: 3456840 kB\nMapped: 15592 kB\nSlab: 132268 kB\nPageTables: 11732 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 5171912 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,919562,809233}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',78692421}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,195345712}, {processes,151117680}, {processes_used,149088176}, {system,44228032}, {atom,1401785}, {atom_used,1374887}, {binary,5775640}, {code,14061510}, {ets,4546712}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,4497174528}, {buffered_memory,163926016}, {free_memory,19920121856}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{726910,5031}}, {context_switches,{6104611,0}}, {garbage_collection,{871162,8336054279,0}}, {io,{{input,116768157},{output,5994482691}}}, {reductions,{1243264948,6077526}}, {run_queue,0}, {runtime,{206540,2290}}]}, {system_stats, [{cpu_utilization_rate,15.288220551378446}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,4501302539}, {couch_docs_data_size,4499560955}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,946414}, {curr_items_tot,1834885}, {mem_used,4925155992}, {vb_replica_curr_items,888471}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,726}, {memory_data,{33657663488,13801148416,{<17795.427.0>,15599024}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 19453244 kB\nBuffers: 160084 kB\nCached: 4391772 kB\nSwapCached: 0 kB\nActive: 8761956 kB\nInactive: 4406548 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 19453244 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 10380 kB\nWriteback: 156 kB\nAnonPages: 8616764 kB\nMapped: 16492 kB\nSlab: 191096 kB\nPageTables: 22348 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10330664 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,919559,430395}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default", [{'ns_1@10.2.1.58',48285060},{'ns_1@10.2.1.61',5963133}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.24870.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,44}, {started_on,1352919499}, {total_vbuckets,1025}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1352919559}, {vbuckets_done,455}]]}, {memory, [{total,217085112}, {processes,167155992}, {processes_used,167126120}, {system,49929120}, {atom,1338193}, {atom_used,1334188}, {binary,11730640}, {code,13493302}, {ets,4653888}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,9214308352}, {buffered_memory,178786304}, {free_memory,15059300352}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{719943,5001}}, {context_switches,{6674913,0}}, {garbage_collection,{1220666,2930544759,0}}, {io,{{input,141823693},{output,8624579887}}}, {reductions,{1004770987,7230457}}, {run_queue,0}, {runtime,{153000,1100}}]}, {system_stats, [{cpu_utilization_rate,41.54228855721393}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,9136950563}, {couch_docs_data_size,7284951799}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1438661}, {curr_items_tot,2997107}, {mem_used,8013793264}, {vb_replica_curr_items,1558446}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,719}, {memory_data,{33657663488,19198943232,{<0.368.0>,5085816}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 14706348 kB\nBuffers: 174596 kB\nCached: 8998348 kB\nSwapCached: 0 kB\nActive: 9651252 kB\nInactive: 8166372 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 14706348 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 17144 kB\nWriteback: 0 kB\nAnonPages: 8644784 kB\nMapped: 16380 kB\nSlab: 288588 kB\nPageTables: 22096 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10358972 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-14T11:00:24.428,ns_1@10.2.1.63:ns_doctor<0.540.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,919620,27159}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',707371}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,141311872}, {processes,100391072}, {processes_used,100371960}, {system,40920800}, {atom,1332537}, {atom_used,1324944}, {binary,4597544}, {code,13364289}, {ets,3215824}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,3796017152}, {buffered_memory,118992896}, {free_memory,25328844800}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{782496,5000}}, {context_switches,{3044636,0}}, {garbage_collection,{1100042,1825601959,0}}, {io,{{input,77491804},{output,8304774887}}}, {reductions,{404274121,6218908}}, {run_queue,0}, {runtime,{103790,1380}}]}, {system_stats, [{cpu_utilization_rate,37.02770780856423}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,3805066384}, {couch_docs_data_size,3341141742}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,737724}, {curr_items_tot,1399454}, {mem_used,3701637112}, {vb_replica_curr_items,661730}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,782}, {memory_data,{33657663488,8310542336,{<19415.367.0>,5085816}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 24735324 kB\nBuffers: 116204 kB\nCached: 3707048 kB\nSwapCached: 0 kB\nActive: 4254964 kB\nInactive: 3691756 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 24735324 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 15404 kB\nWriteback: 0 kB\nAnonPages: 4123224 kB\nMapped: 15592 kB\nSlab: 139120 kB\nPageTables: 13104 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 5842796 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,919622,777433}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',14694477}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,176403440}, {processes,132686872}, {processes_used,130262872}, {system,43716568}, {atom,1401785}, {atom_used,1374887}, {binary,5243896}, {code,14061510}, {ets,4623096}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,3937509376}, {buffered_memory,165179392}, {free_memory,20519075840}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{786878,5001}}, {context_switches,{6309620,0}}, {garbage_collection,{965468,10024083535,0}}, {io,{{input,120594753},{output,6017990392}}}, {reductions,{1313265628,5223927}}, {run_queue,0}, {runtime,{232440,2050}}]}, {system_stats, [{cpu_utilization_rate,16.75}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,3924152233}, {couch_docs_data_size,3922623435}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,840757}, {curr_items_tot,1600549}, {mem_used,4314551832}, {vb_replica_curr_items,759792}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,786}, {memory_data,{33657663488,13209440256,{<17795.427.0>,15648032}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 20038160 kB\nBuffers: 161308 kB\nCached: 3845224 kB\nSwapCached: 0 kB\nActive: 8733248 kB\nInactive: 3860420 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 20038160 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 8284 kB\nWriteback: 0 kB\nAnonPages: 8586912 kB\nMapped: 16492 kB\nSlab: 180372 kB\nPageTables: 22312 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10295956 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,919619,429693}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default", [{'ns_1@10.2.1.58',60950656},{'ns_1@10.2.1.61',46947163}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.24870.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,84}, {started_on,1352919499}, {total_vbuckets,1025}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1352919619}, {vbuckets_done,871}]]}, {memory, [{total,219423448}, {processes,171368272}, {processes_used,171346536}, {system,48055176}, {atom,1338193}, {atom_used,1334188}, {binary,9343968}, {code,13493302}, {ets,5122136}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7930109952}, {buffered_memory,184655872}, {free_memory,16341536768}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{779942,4999}}, {context_switches,{7164068,0}}, {garbage_collection,{1411266,3288664374,0}}, {io,{{input,162236465},{output,9201307483}}}, {reductions,{1091313802,7043923}}, {run_queue,0}, {runtime,{166760,1070}}]}, {system_stats, [{cpu_utilization_rate,46.30541871921182}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7851850160}, {couch_docs_data_size,7285665697}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1415506}, {curr_items_tot,3000003}, {mem_used,8025669912}, {vb_replica_curr_items,1584497}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,779}, {memory_data,{33657663488,18613084160,{<0.368.0>,5085816}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 15958532 kB\nBuffers: 180328 kB\nCached: 7744248 kB\nSwapCached: 0 kB\nActive: 9055424 kB\nInactive: 7540412 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 15958532 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 22408 kB\nWriteback: 1096 kB\nAnonPages: 8671188 kB\nMapped: 16380 kB\nSlab: 257232 kB\nPageTables: 22496 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10388800 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-14T11:00:52.800,ns_1@10.2.1.63:<0.682.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,48,39}}) Stats for bucket "default": accepting_conns 1 auth_cmds 27 auth_errors 0 bucket_active_conns 1 bucket_conns 24 bytes 8032536056 bytes_read 15094868308 bytes_written 12332251551 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 82 cmd_set 2400893 conn_yields 53379 connection_structures 5000 curr_connections 25 curr_conns_on_port_11209 16 curr_conns_on_port_11210 7 curr_items 1421474 curr_items_tot 3000003 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 17 delete_misses 1 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-15 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data2/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 8869 ep_commit_time 3 ep_commit_time_total 150062 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 2 ep_data_age_highwat 150 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data2/default ep_degraded_mode 0 ep_diskqueue_drain 8291619 ep_diskqueue_fill 6157157 ep_diskqueue_items 9 ep_diskqueue_memory 288 ep_diskqueue_pending 432 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 541 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 6253325 ep_io_read_bytes 0 ep_io_write_bytes 14808077450 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 105473 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 7339251617 ep_max_checkpoints 2 ep_max_data_size 12582912000 ep_max_item_size 20971520 ep_max_size 12582912000 ep_max_txn_size 10000 ep_max_vbuckets 1024 ep_mem_high_wat 9437184000 ep_mem_high_wat 9437184000 ep_mem_low_wat 7549747200 ep_mem_low_wat 7549747200 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 276094730 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 9 ep_startup_time 1352918852 ep_storage_age 0 ep_storage_age_highwat 633 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 7303251581 ep_total_del_items 2 ep_total_enqueued 6157157 ep_total_new_items 3000005 ep_total_persisted 6253325 ep_uncommitted_items 0 ep_value_size 7091901812 ep_vb0 0 ep_vb_snapshot_total 1274 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_135_g40a391b ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 759 get_hits 69 get_misses 13 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 8032536056 pid 16783 pointer_size 64 rejected_conns 0 rusage_system 169.740195 rusage_user 454.229946 tap_checkpoint_end_received 496 tap_checkpoint_end_sent 770 tap_checkpoint_start_received 1523 tap_checkpoint_start_sent 1902 tap_connect_received 549 tap_delete_received 1 tap_delete_sent 1 tap_mutation_received 3852518 tap_mutation_sent 4658157 tap_opaque_received 3294 tap_opaque_sent 3187 tap_vbucket_set_received 492 tap_vbucket_set_sent 546 threads 4 time 1352919652 total_connections 3173 uptime 810 vb_active_curr_items 1421474 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 12179032 vb_active_itm_memory 3459907678 vb_active_meta_data_memory 112296446 vb_active_num 485 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1421474 vb_active_ops_create 1421475 vb_active_ops_delete 1 vb_active_ops_reject 0 vb_active_ops_update 1573124 vb_active_perc_mem_resident 100 vb_active_queue_age 0 vb_active_queue_drain 4327940 vb_active_queue_fill 2928531 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 1578529 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 13535176 vb_replica_itm_memory 3843343903 vb_replica_meta_data_memory 124703791 vb_replica_num 539 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 1578530 vb_replica_ops_create 1578530 vb_replica_ops_delete 1 vb_replica_ops_reject 0 vb_replica_ops_update 1680194 vb_replica_perc_mem_resident 100 vb_replica_queue_age 0 vb_replica_queue_drain 3963679 vb_replica_queue_fill 3228626 vb_replica_queue_memory 288 vb_replica_queue_pending 432 vb_replica_queue_size 9 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-14T11:01:24.428,ns_1@10.2.1.63:ns_doctor<0.540.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,919680,27371}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',11074484}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,157042256}, {processes,114792216}, {processes_used,114785320}, {system,42250040}, {atom,1332537}, {atom_used,1324944}, {binary,5739808}, {code,13364289}, {ets,3353112}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,4997160960}, {buffered_memory,138989568}, {free_memory,23457902592}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{842496,5000}}, {context_switches,{3583827,0}}, {garbage_collection,{1252369,2114243873,0}}, {io,{{input,86986156},{output,9603565409}}}, {reductions,{470416463,5795623}}, {run_queue,0}, {runtime,{118450,1260}}]}, {system_stats, [{cpu_utilization_rate,41.04477611940298}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,4848530844}, {couch_docs_data_size,3875458052}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,846247}, {curr_items_tot,1616391}, {mem_used,4274024904}, {vb_replica_curr_items,770144}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,842}, {memory_data,{33657663488,10106941440,{<19415.367.0>,5085816}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 22908232 kB\nBuffers: 135732 kB\nCached: 4880040 kB\nSwapCached: 0 kB\nActive: 5060476 kB\nInactive: 4680276 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 22908232 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 13000 kB\nWriteback: 0 kB\nAnonPages: 4724764 kB\nMapped: 15592 kB\nSlab: 171428 kB\nPageTables: 14252 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6425660 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,919682,777059}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',17940423}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,175409408}, {processes,132533104}, {processes_used,129738592}, {system,42876304}, {atom,1401785}, {atom_used,1374887}, {binary,4483736}, {code,14061510}, {ets,4595272}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,3405967360}, {buffered_memory,166453248}, {free_memory,21043994624}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{846878,5000}}, {context_switches,{6495784,0}}, {garbage_collection,{1053604,11627370351,0}}, {io,{{input,124106816},{output,6040238580}}}, {reductions,{1377528396,5135616}}, {run_queue,0}, {runtime,{256640,2020}}]}, {system_stats, [{cpu_utilization_rate,15.288220551378446}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,3396396382}, {couch_docs_data_size,3395061731}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,732330}, {curr_items_tot,1383612}, {mem_used,3751397384}, {vb_replica_curr_items,651282}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,846}, {memory_data,{33657663488,12657483776,{<17795.427.0>,15610592}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 20550776 kB\nBuffers: 162552 kB\nCached: 3326140 kB\nSwapCached: 0 kB\nActive: 8749420 kB\nInactive: 3341656 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 20550776 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 1960 kB\nWriteback: 92 kB\nAnonPages: 8602216 kB\nMapped: 16492 kB\nSlab: 170008 kB\nPageTables: 22252 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10316188 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,919679,428286}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default", [{'ns_1@10.2.1.58',131855548},{'ns_1@10.2.1.61',10361256}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,223750152}, {processes,174126168}, {processes_used,174033896}, {system,49623984}, {atom,1338193}, {atom_used,1334188}, {binary,10816080}, {code,13493302}, {ets,5222728}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,8494407680}, {buffered_memory,188309504}, {free_memory,15743373312}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{839941,4999}}, {context_switches,{7623282,0}}, {garbage_collection,{1579453,3608363480,0}}, {io,{{input,174029164},{output,9696165767}}}, {reductions,{1169997644,6497049}}, {run_queue,0}, {runtime,{179480,1000}}]}, {system_stats, [{cpu_utilization_rate,44.0}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,8405755314}, {couch_docs_data_size,7285320850}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1418481}, {curr_items_tot,3000003}, {mem_used,8036808952}, {vb_replica_curr_items,1581522}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,839}, {memory_data,{33657663488,17321205760,{<0.368.0>,5085816}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 15374512 kB\nBuffers: 183896 kB\nCached: 8295320 kB\nSwapCached: 0 kB\nActive: 9328748 kB\nInactive: 7838664 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 15374512 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 18984 kB\nWriteback: 80 kB\nAnonPages: 8687652 kB\nMapped: 16700 kB\nSlab: 270364 kB\nPageTables: 22536 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10404436 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-14T11:02:24.428,ns_1@10.2.1.63:ns_doctor<0.540.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,919740,26424}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',5667593}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,166527152}, {processes,123135896}, {processes_used,123131712}, {system,43391256}, {atom,1332537}, {atom_used,1324944}, {binary,6620824}, {code,13364289}, {ets,3571568}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,6153256960}, {buffered_memory,155324416}, {free_memory,21678342144}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{902495,4999}}, {context_switches,{4136931,0}}, {garbage_collection,{1389164,2386154449,0}}, {io,{{input,99622587},{output,10804987168}}}, {reductions,{534684175,5762868}}, {run_queue,0}, {runtime,{132830,1210}}]}, {system_stats, [{cpu_utilization_rate,42.35588972431078}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,5998265752}, {couch_docs_data_size,4373856096}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,957969}, {curr_items_tot,1821473}, {mem_used,4813832536}, {vb_replica_curr_items,863504}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,902}, {memory_data,{33657663488,11905548288,{<19415.367.0>,5085816}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 21170256 kB\nBuffers: 151684 kB\nCached: 6009040 kB\nSwapCached: 0 kB\nActive: 5849552 kB\nInactive: 5596968 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 21170256 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 7008 kB\nWriteback: 32 kB\nAnonPages: 5285208 kB\nMapped: 15592 kB\nSlab: 202968 kB\nPageTables: 15388 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 7001304 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,919742,776694}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',27872123}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,163295904}, {processes,120639344}, {processes_used,117506128}, {system,42656560}, {atom,1401785}, {atom_used,1374887}, {binary,4207240}, {code,14061510}, {ets,4698392}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,2924486656}, {buffered_memory,167833600}, {free_memory,21550247936}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{906877,4999}}, {context_switches,{6663715,0}}, {garbage_collection,{1126191,13101284677,0}}, {io,{{input,130964881},{output,6063718224}}}, {reductions,{1437379611,4630991}}, {run_queue,0}, {runtime,{279900,1660}}]}, {system_stats, [{cpu_utilization_rate,17.037037037037038}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,2897669243}, {couch_docs_data_size,2896516801}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,638609}, {curr_items_tot,1181106}, {mem_used,3225620824}, {vb_replica_curr_items,542497}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,906}, {memory_data,{33657663488,12171030528,{<17795.427.0>,15604608}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 21045164 kB\nBuffers: 163900 kB\nCached: 2855944 kB\nSwapCached: 0 kB\nActive: 8733436 kB\nInactive: 2872320 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 21045164 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 1408 kB\nWriteback: 16 kB\nAnonPages: 8586148 kB\nMapped: 16492 kB\nSlab: 161372 kB\nPageTables: 22292 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10306160 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,919739,429522}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default", [{'ns_1@10.2.1.58',74732187},{'ns_1@10.2.1.61',97283348}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,223252704}, {processes,172582824}, {processes_used,172424456}, {system,50669880}, {atom,1338193}, {atom_used,1334188}, {binary,11868144}, {code,13493302}, {ets,5223856}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,9049149440}, {buffered_memory,190545920}, {free_memory,15157903360}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{899942,5001}}, {context_switches,{8082072,0}}, {garbage_collection,{1739137,3910827627,0}}, {io,{{input,184608488},{output,10152894890}}}, {reductions,{1245410546,6713412}}, {run_queue,0}, {runtime,{192630,1150}}]}, {system_stats, [{cpu_utilization_rate,36.5}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,8957875650}, {couch_docs_data_size,7285017974}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1403545}, {curr_items_tot,3000003}, {mem_used,8047563720}, {vb_replica_curr_items,1596458}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,899}, {memory_data,{33657663488,17915916288,{<0.368.0>,5085816}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 14802764 kB\nBuffers: 186080 kB\nCached: 8837060 kB\nSwapCached: 0 kB\nActive: 9624748 kB\nInactive: 8101008 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 14802764 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 17396 kB\nWriteback: 0 kB\nAnonPages: 8702596 kB\nMapped: 16404 kB\nSlab: 283412 kB\nPageTables: 22344 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10419168 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-14T11:02:32.803,ns_1@10.2.1.63:<0.682.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,48,39}}) Stats for bucket "default": accepting_conns 1 auth_cmds 27 auth_errors 0 bucket_active_conns 1 bucket_conns 24 bytes 8050291160 bytes_read 15852738712 bytes_written 13772611660 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 82 cmd_set 2400893 conn_yields 55905 connection_structures 5000 curr_connections 25 curr_conns_on_port_11209 16 curr_conns_on_port_11210 7 curr_items 1400601 curr_items_tot 3000003 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 17 delete_misses 1 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-15 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data2/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 15277 ep_commit_time 3 ep_commit_time_total 178022 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 2 ep_data_age_highwat 150 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data2/default ep_degraded_mode 0 ep_diskqueue_drain 9034292 ep_diskqueue_fill 6441647 ep_diskqueue_items 12 ep_diskqueue_memory 384 ep_diskqueue_pending 576 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 575 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 6566346 ep_io_read_bytes 0 ep_io_write_bytes 15549384526 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 151985 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 7339251617 ep_max_checkpoints 2 ep_max_data_size 12582912000 ep_max_item_size 20971520 ep_max_size 12582912000 ep_max_txn_size 10000 ep_max_vbuckets 1024 ep_mem_high_wat 9437184000 ep_mem_high_wat 9437184000 ep_mem_low_wat 7549747200 ep_mem_low_wat 7549747200 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 272196462 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 12 ep_startup_time 1352918852 ep_storage_age 0 ep_storage_age_highwat 814 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 7303251581 ep_total_del_items 2 ep_total_enqueued 6441647 ep_total_new_items 3000005 ep_total_persisted 6566346 ep_uncommitted_items 0 ep_value_size 7091901812 ep_vb0 0 ep_vb_snapshot_total 1571 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_135_g40a391b ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 759 get_hits 69 get_misses 13 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 8050291160 pid 16783 pointer_size 64 rejected_conns 0 rusage_system 181.633387 rusage_user 537.886228 tap_checkpoint_end_received 602 tap_checkpoint_end_sent 942 tap_checkpoint_start_received 1750 tap_checkpoint_start_sent 2214 tap_connect_received 673 tap_delete_received 1 tap_delete_sent 1 tap_mutation_received 4165542 tap_mutation_sent 5163273 tap_opaque_received 3803 tap_opaque_sent 3676 tap_vbucket_set_received 602 tap_vbucket_set_sent 670 threads 4 time 1352919752 total_connections 3877 uptime 910 vb_active_curr_items 1400601 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 12003440 vb_active_itm_memory 3409930751 vb_active_meta_data_memory 110647479 vb_active_num 478 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1400601 vb_active_ops_create 1400601 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 1722013 vb_active_perc_mem_resident 100 vb_active_queue_age 0 vb_active_queue_drain 4736876 vb_active_queue_fill 3029429 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 1599402 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 13710768 vb_replica_itm_memory 3893320830 vb_replica_meta_data_memory 126352758 vb_replica_num 546 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 1599403 vb_replica_ops_create 1599404 vb_replica_ops_delete 2 vb_replica_ops_reject 0 vb_replica_ops_update 1844326 vb_replica_perc_mem_resident 100 vb_replica_queue_age 0 vb_replica_queue_drain 4297416 vb_replica_queue_fill 3412218 vb_replica_queue_memory 384 vb_replica_queue_pending 576 vb_replica_queue_size 12 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-14T11:03:24.428,ns_1@10.2.1.63:ns_doctor<0.540.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,919800,33829}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',11592181}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.26002.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,37}, {started_on,1352919780}, {total_vbuckets,654}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1352919812}, {vbuckets_done,242}]]}, {memory, [{total,170892592}, {processes,126930216}, {processes_used,126925752}, {system,43962376}, {atom,1332537}, {atom_used,1324944}, {binary,6979288}, {code,13364289}, {ets,3736648}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,6577197056}, {buffered_memory,165896192}, {free_memory,20707205120}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{962502,5007}}, {context_switches,{4673493,0}}, {garbage_collection,{1515837,2650812914,0}}, {io,{{input,110501000},{output,11926191705}}}, {reductions,{598143720,5437820}}, {run_queue,0}, {runtime,{145740,1070}}]}, {system_stats, [{cpu_utilization_rate,46.365914786967416}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,6806131499}, {couch_docs_data_size,4835646679}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1048945}, {curr_items_tot,2006112}, {mem_used,5300613144}, {vb_replica_curr_items,957167}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,962}, {memory_data,{33657663488,13134573568,{<19415.367.0>,5085816}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 20224360 kB\nBuffers: 162008 kB\nCached: 6420748 kB\nSwapCached: 0 kB\nActive: 6396500 kB\nInactive: 5981048 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 20224360 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 12192 kB\nWriteback: 0 kB\nAnonPages: 5794968 kB\nMapped: 15572 kB\nSlab: 217024 kB\nPageTables: 16356 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 7502872 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,919802,776899}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',23602083}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,155275672}, {processes,113036592}, {processes_used,109572008}, {system,42239080}, {atom,1401785}, {atom_used,1374887}, {binary,3851720}, {code,14061510}, {ets,4681048}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,2458927104}, {buffered_memory,168996864}, {free_memory,22031978496}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{966877,4999}}, {context_switches,{6816764,0}}, {garbage_collection,{1192655,14456987460,0}}, {io,{{input,133712981},{output,6083240959}}}, {reductions,{1493746846,4857221}}, {run_queue,0}, {runtime,{301890,2040}}]}, {system_stats, [{cpu_utilization_rate,13.533834586466165}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,2435768288}, {couch_docs_data_size,2434862799}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,542373}, {curr_items_tot,988184}, {mem_used,2725717704}, {vb_replica_curr_items,445811}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,966}, {memory_data,{33657663488,11684450304,{<17795.427.0>,15604480}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 21515604 kB\nBuffers: 165036 kB\nCached: 2401296 kB\nSwapCached: 0 kB\nActive: 8726296 kB\nInactive: 2418076 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 21515604 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 1368 kB\nWriteback: 144 kB\nAnonPages: 8577768 kB\nMapped: 16736 kB\nSlab: 152356 kB\nPageTables: 22392 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10291832 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,919799,428304}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default", [{'ns_1@10.2.1.58',49100432},{'ns_1@10.2.1.61',129613372}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.8415.1>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,32}, {started_on,1352919760}, {total_vbuckets,1025}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1352919799}, {vbuckets_done,332}]]}, {memory, [{total,222021440}, {processes,171412112}, {processes_used,171354656}, {system,50609328}, {atom,1338193}, {atom_used,1334188}, {binary,11510200}, {code,13493302}, {ets,5506784}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,8309166080}, {buffered_memory,194985984}, {free_memory,15892332544}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{959941,4998}}, {context_switches,{8540642,0}}, {garbage_collection,{1892880,4208717099,0}}, {io,{{input,201177643},{output,10601683635}}}, {reductions,{1321446832,6537960}}, {run_queue,0}, {runtime,{204570,950}}]}, {system_stats, [{cpu_utilization_rate,43.5}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,8272781460}, {couch_docs_data_size,7285582452}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1405818}, {curr_items_tot,3000003}, {mem_used,8056636424}, {vb_replica_curr_items,1594185}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,959}, {memory_data,{33657663488,18504839168,{<0.368.0>,5085816}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 15519980 kB\nBuffers: 190416 kB\nCached: 8114420 kB\nSwapCached: 0 kB\nActive: 9456600 kB\nInactive: 7563352 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 15519980 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 20696 kB\nWriteback: 0 kB\nAnonPages: 8714756 kB\nMapped: 16404 kB\nSlab: 272492 kB\nPageTables: 22372 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10430944 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-14T11:04:12.804,ns_1@10.2.1.63:<0.682.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,48,39}}) Stats for bucket "default": accepting_conns 1 auth_cmds 27 auth_errors 0 bucket_active_conns 1 bucket_conns 21 bytes 8065738424 bytes_read 16519087236 bytes_written 15021986476 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 82 cmd_set 2400893 conn_yields 58499 connection_structures 5000 curr_connections 23 curr_conns_on_port_11209 14 curr_conns_on_port_11210 7 curr_items 1392905 curr_items_tot 3000002 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 17 delete_misses 1 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-15 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data2/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 16969 ep_commit_time 165 ep_commit_time_total 230744 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 2 ep_data_age_highwat 150 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data2/default ep_degraded_mode 0 ep_diskqueue_drain 9698778 ep_diskqueue_fill 6700250 ep_diskqueue_items 40 ep_diskqueue_memory 960 ep_diskqueue_pending 1440 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 1 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 635 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 6841549 ep_io_read_bytes 0 ep_io_write_bytes 16200224366 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 199115 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 7339250270 ep_max_checkpoints 2 ep_max_data_size 12582912000 ep_max_item_size 20971520 ep_max_size 12582912000 ep_max_txn_size 10000 ep_max_vbuckets 1024 ep_mem_high_wat 9437184000 ep_mem_high_wat 9437184000 ep_mem_low_wat 7549747200 ep_mem_low_wat 7549747200 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 268849648 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 40 ep_startup_time 1352918852 ep_storage_age 0 ep_storage_age_highwat 814 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 7303250246 ep_total_del_items 4 ep_total_enqueued 6700250 ep_total_new_items 3000005 ep_total_persisted 6841549 ep_uncommitted_items 380 ep_value_size 7091900551 ep_vb0 0 ep_vb_snapshot_total 1801 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_135_g40a391b ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 759 get_hits 69 get_misses 13 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 8065738424 pid 16783 pointer_size 64 rejected_conns 0 rusage_system 189.817143 rusage_user 611.502037 tap_checkpoint_end_received 697 tap_checkpoint_end_sent 1090 tap_checkpoint_start_received 1956 tap_checkpoint_start_sent 2480 tap_connect_received 778 tap_delete_received 2 tap_delete_sent 1 tap_mutation_received 4441144 tap_mutation_sent 5600456 tap_opaque_received 4266 tap_opaque_sent 4094 tap_vbucket_set_received 704 tap_vbucket_set_sent 776 threads 4 time 1352919852 total_connections 4500 uptime 1010 vb_active_curr_items 1392905 vb_active_eject 0 vb_active_expired 1 vb_active_ht_memory 11953312 vb_active_itm_memory 3390549367 vb_active_meta_data_memory 110039495 vb_active_num 476 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1392907 vb_active_ops_create 1392907 vb_active_ops_delete 3 vb_active_ops_reject 0 vb_active_ops_update 1865821 vb_active_perc_mem_resident 100 vb_active_queue_age 0 vb_active_queue_drain 5141084 vb_active_queue_fill 3151500 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 1607097 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 13760896 vb_replica_itm_memory 3912700879 vb_replica_meta_data_memory 126960663 vb_replica_num 548 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 1607097 vb_replica_ops_create 1607098 vb_replica_ops_delete 1 vb_replica_ops_reject 0 vb_replica_ops_update 1976092 vb_replica_perc_mem_resident 100 vb_replica_queue_age 0 vb_replica_queue_drain 4557694 vb_replica_queue_fill 3548750 vb_replica_queue_memory 960 vb_replica_queue_pending 1440 vb_replica_queue_size 30 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-14T11:04:24.428,ns_1@10.2.1.63:ns_doctor<0.540.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,919860,26193}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',17400160}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,179826128}, {processes,135504648}, {processes_used,135499624}, {system,44321480}, {atom,1332537}, {atom_used,1325003}, {binary,6864736}, {code,13364289}, {ets,4150320}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,5864488960}, {buffered_memory,174825472}, {free_memory,20928032768}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1022495,4999}}, {context_switches,{5198153,0}}, {garbage_collection,{1643583,2906170546,0}}, {io,{{input,122138633},{output,12982746206}}}, {reductions,{660983863,5314460}}, {run_queue,0}, {runtime,{158380,1090}}]}, {system_stats, [{cpu_utilization_rate,45.6140350877193}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,5764249091}, {couch_docs_data_size,5269855543}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1146437}, {curr_items_tot,2188257}, {mem_used,5780969664}, {vb_replica_curr_items,1041820}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1022}, {memory_data,{33657663488,12650778624,{<19415.367.0>,5085816}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 20437656 kB\nBuffers: 170728 kB\nCached: 5727040 kB\nSwapCached: 0 kB\nActive: 6556644 kB\nInactive: 5624356 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 20437656 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 13164 kB\nWriteback: 0 kB\nAnonPages: 6282952 kB\nMapped: 15592 kB\nSlab: 198288 kB\nPageTables: 17344 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 7995376 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,919862,787210}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',3680157}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,152342400}, {processes,110653536}, {processes_used,106884776}, {system,41688864}, {atom,1401785}, {atom_used,1374887}, {binary,3224320}, {code,14061510}, {ets,4799744}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,2025955328}, {buffered_memory,170115072}, {free_memory,22468182016}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1026886,5008}}, {context_switches,{6953698,0}}, {garbage_collection,{1254071,15688059392,0}}, {io,{{input,136277856},{output,6101063893}}}, {reductions,{1546819920,5094319}}, {run_queue,0}, {runtime,{322630,1760}}]}, {system_stats, [{cpu_utilization_rate,31.829573934837093}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,2008264632}, {couch_docs_data_size,2007539238}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,457689}, {curr_items_tot,808767}, {mem_used,2260516968}, {vb_replica_curr_items,351078}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1026}, {memory_data,{33657663488,11256016896,{<17795.427.0>,15603072}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 21941584 kB\nBuffers: 166128 kB\nCached: 1978472 kB\nSwapCached: 0 kB\nActive: 8730036 kB\nInactive: 1995732 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 21941584 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 1264 kB\nWriteback: 0 kB\nAnonPages: 8581160 kB\nMapped: 16492 kB\nSlab: 144664 kB\nPageTables: 22196 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10300368 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,919859,429268}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default", [{'ns_1@10.2.1.58',65462301},{'ns_1@10.2.1.61',96688122}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.8415.1>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,76}, {started_on,1352919760}, {total_vbuckets,1025}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1352919859}, {vbuckets_done,789}]]}, {memory, [{total,220250400}, {processes,169042008}, {processes_used,168962472}, {system,51208392}, {atom,1338193}, {atom_used,1334188}, {binary,12070472}, {code,13493302}, {ets,5546352}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,8472784896}, {buffered_memory,200196096}, {free_memory,15704637440}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1019942,5000}}, {context_switches,{8977639,0}}, {garbage_collection,{2035045,4484139604,0}}, {io,{{input,213365750},{output,11005316160}}}, {reductions,{1393992917,5637307}}, {run_queue,0}, {runtime,{215680,810}}]}, {system_stats, [{cpu_utilization_rate,53.75}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,8475694565}, {couch_docs_data_size,7285426747}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1392913}, {curr_items_tot,3000002}, {mem_used,8066029848}, {vb_replica_curr_items,1607089}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1019}, {memory_data,{33657663488,17768759296,{<0.368.0>,5085816}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 15336560 kB\nBuffers: 195504 kB\nCached: 8274204 kB\nSwapCached: 0 kB\nActive: 9510696 kB\nInactive: 7690024 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 15336560 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 25980 kB\nWriteback: 0 kB\nAnonPages: 8731156 kB\nMapped: 16412 kB\nSlab: 274712 kB\nPageTables: 22552 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10454036 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-14T11:05:24.428,ns_1@10.2.1.63:ns_doctor<0.540.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,919920,26612}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',20555837}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,194866920}, {processes,149654672}, {processes_used,149650256}, {system,45212248}, {atom,1332537}, {atom_used,1325003}, {binary,7528872}, {code,13364289}, {ets,4341232}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,6821871616}, {buffered_memory,181035008}, {free_memory,19483308032}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1082495,4998}}, {context_switches,{5710886,0}}, {garbage_collection,{1748030,3145302746,0}}, {io,{{input,134146263},{output,13973564193}}}, {reductions,{719158214,4807866}}, {run_queue,0}, {runtime,{171340,1310}}]}, {system_stats, [{cpu_utilization_rate,30.048076923076923}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,6697845986}, {couch_docs_data_size,5668480359}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1219389}, {curr_items_tot,2348998}, {mem_used,6203893800}, {vb_replica_curr_items,1129609}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1082}, {memory_data,{33657663488,14136889344,{<19415.367.0>,5085816}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 19026668 kB\nBuffers: 176792 kB\nCached: 6661984 kB\nSwapCached: 0 kB\nActive: 7195856 kB\nInactive: 6370340 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 19026668 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 7168 kB\nWriteback: 0 kB\nAnonPages: 6727540 kB\nMapped: 15592 kB\nSlab: 222600 kB\nPageTables: 18196 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 8440296 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,919922,776622}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',61014766}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,146050304}, {processes,104584136}, {processes_used,100545472}, {system,41466168}, {atom,1401785}, {atom_used,1374887}, {binary,2942672}, {code,14061510}, {ets,4895992}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,1638891520}, {buffered_memory,171302912}, {free_memory,22856204288}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1086877,4999}}, {context_switches,{7074105,0}}, {garbage_collection,{1305158,16845874920,0}}, {io,{{input,142272705},{output,6120130289}}}, {reductions,{1594560149,3647639}}, {run_queue,0}, {runtime,{342130,1490}}]}, {system_stats, [{cpu_utilization_rate,9.5},{swap_total,10684628992},{swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,1602736380}, {couch_docs_data_size,1602134482}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,369927}, {curr_items_tot,651004}, {mem_used,1851506728}, {vb_replica_curr_items,281077}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1086}, {memory_data,{33657663488,10837528576,{<17795.427.0>,15584616}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 22320512 kB\nBuffers: 167288 kB\nCached: 1600480 kB\nSwapCached: 0 kB\nActive: 8737908 kB\nInactive: 1619088 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 22320512 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 1228 kB\nWriteback: 136 kB\nAnonPages: 8588904 kB\nMapped: 16492 kB\nSlab: 135200 kB\nPageTables: 22184 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10301016 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,919919,428457}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default", [{'ns_1@10.2.1.58',131550954},{'ns_1@10.2.1.61',10426706}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,224952376}, {processes,175046440}, {processes_used,174988168}, {system,49905936}, {atom,1338193}, {atom_used,1334188}, {binary,10523744}, {code,13493302}, {ets,5780560}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7979798528}, {buffered_memory,203735040}, {free_memory,16195932160}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1079941,4998}}, {context_switches,{9412139,0}}, {garbage_collection,{2160005,4740609712,0}}, {io,{{input,227783395},{output,11381507673}}}, {reductions,{1462905087,5306571}}, {run_queue,0}, {runtime,{226420,860}}]}, {system_stats, [{cpu_utilization_rate,22.443890274314214}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7875926849}, {couch_docs_data_size,7285758947}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1407749}, {curr_items_tot,3000002}, {mem_used,8073027160}, {vb_replica_curr_items,1592253}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1079}, {memory_data,{33657663488,17936011264,{<0.368.0>,5085816}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 15816464 kB\nBuffers: 198960 kB\nCached: 7792772 kB\nSwapCached: 0 kB\nActive: 9207992 kB\nInactive: 7528800 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 15816464 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 25920 kB\nWriteback: 48 kB\nAnonPages: 8744612 kB\nMapped: 16404 kB\nSlab: 259592 kB\nPageTables: 22428 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10466208 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:warn,2012-11-14T11:05:49.800,ns_1@10.2.1.63:<0.682.0>:stats_collector:latest_tick:223]Dropped 1 ticks [stats:warn,2012-11-14T11:05:51.805,ns_1@10.2.1.63:<0.682.0>:stats_collector:latest_tick:223]Dropped 1 ticks [stats:debug,2012-11-14T11:05:54.804,ns_1@10.2.1.63:<0.682.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,48,39}}) Stats for bucket "default": accepting_conns 1 auth_cmds 27 auth_errors 0 bucket_active_conns 1 bucket_conns 26 bytes 8076403608 bytes_read 17138770024 bytes_written 16115122328 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 82 cmd_set 2400893 conn_yields 60466 connection_structures 5000 curr_connections 27 curr_conns_on_port_11209 18 curr_conns_on_port_11210 7 curr_items 1416876 curr_items_tot 3000002 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 17 delete_misses 1 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-15 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data2/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 22441 ep_commit_time 3 ep_commit_time_total 269527 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 0 ep_data_age_highwat 150 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data2/default ep_degraded_mode 0 ep_diskqueue_drain 10311414 ep_diskqueue_fill 6944147 ep_diskqueue_items 12 ep_diskqueue_memory 384 ep_diskqueue_pending 576 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 1 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 684 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 12 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 7097853 ep_io_read_bytes 0 ep_io_write_bytes 16807323554 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 248560 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 7339250270 ep_max_checkpoints 2 ep_max_data_size 12582912000 ep_max_item_size 20971520 ep_max_size 12582912000 ep_max_txn_size 10000 ep_max_vbuckets 1024 ep_mem_high_wat 9437184000 ep_mem_high_wat 9437184000 ep_mem_low_wat 7549747200 ep_mem_low_wat 7549747200 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 265493968 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1352918852 ep_storage_age 0 ep_storage_age_highwat 1011 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 2 ep_too_young 0 ep_total_cache_size 7303250246 ep_total_del_items 4 ep_total_enqueued 6944147 ep_total_new_items 3000005 ep_total_persisted 7097853 ep_uncommitted_items 0 ep_value_size 7091900551 ep_vb0 0 ep_vb_snapshot_total 2044 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_135_g40a391b ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 759 get_hits 69 get_misses 13 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 8076403608 pid 16783 pointer_size 64 rejected_conns 0 rusage_system 198.941756 rusage_user 696.962045 tap_checkpoint_end_received 784 tap_checkpoint_end_sent 1220 tap_checkpoint_start_received 2150 tap_checkpoint_start_sent 2719 tap_connect_received 863 tap_delete_received 2 tap_delete_sent 1 tap_mutation_received 4697047 tap_mutation_sent 5983010 tap_opaque_received 4704 tap_opaque_sent 4443 tap_vbucket_set_received 804 tap_vbucket_set_sent 860 threads 4 time 1352919954 total_connections 5059 uptime 1112 vb_active_curr_items 1416876 vb_active_eject 0 vb_active_expired 1 vb_active_ht_memory 12154208 vb_active_itm_memory 3448895892 vb_active_meta_data_memory 111933204 vb_active_num 484 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1416878 vb_active_ops_create 1416878 vb_active_ops_delete 3 vb_active_ops_reject 0 vb_active_ops_update 2026505 vb_active_perc_mem_resident 100 vb_active_queue_age 0 vb_active_queue_drain 5592532 vb_active_queue_fill 3327703 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 1583126 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 13560000 vb_replica_itm_memory 3854354354 vb_replica_meta_data_memory 125066954 vb_replica_num 540 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 1583126 vb_replica_ops_create 1583127 vb_replica_ops_delete 1 vb_replica_ops_reject 0 vb_replica_ops_update 2071339 vb_replica_perc_mem_resident 100 vb_replica_queue_age 0 vb_replica_queue_drain 4718882 vb_replica_queue_fill 3616444 vb_replica_queue_memory 384 vb_replica_queue_pending 576 vb_replica_queue_size 12 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-14T11:06:24.428,ns_1@10.2.1.63:ns_doctor<0.540.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,919980,27601}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',103007495}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.4647.1>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,3}, {started_on,1352919988}, {total_vbuckets,850}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1352919991}, {vbuckets_done,29}]]}, {memory, [{total,212207760}, {processes,164204496}, {processes_used,164193776}, {system,48003264}, {atom,1332537}, {atom_used,1325003}, {binary,10110968}, {code,13364289}, {ets,4515392}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7632773120}, {buffered_memory,184885248}, {free_memory,18217746432}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1142496,5000}}, {context_switches,{6198017,0}}, {garbage_collection,{1843205,3366081739,0}}, {io,{{input,142402052},{output,14866280042}}}, {reductions,{774799408,4652044}}, {run_queue,0}, {runtime,{183380,920}}]}, {system_stats, [{cpu_utilization_rate,25.628140703517587}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7510383649}, {couch_docs_data_size,6029251133}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1286387}, {curr_items_tot,2497824}, {mem_used,6598827008}, {vb_replica_curr_items,1208566}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1142}, {memory_data,{33657663488,15422648320,{<19415.67.0>,11371232}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 17790768 kB\nBuffers: 180552 kB\nCached: 7453880 kB\nSwapCached: 0 kB\nActive: 7766100 kB\nInactive: 7013032 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 17790768 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 3476 kB\nWriteback: 328 kB\nAnonPages: 7144548 kB\nMapped: 15596 kB\nSlab: 244700 kB\nPageTables: 19120 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 8870888 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,919982,777651}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',50992096}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,143443984}, {processes,102358728}, {processes_used,98074112}, {system,41085256}, {atom,1401785}, {atom_used,1374887}, {binary,2474104}, {code,14061510}, {ets,5018840}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,1287553024}, {buffered_memory,172220416}, {free_memory,23226712064}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1146878,5000}}, {context_switches,{7181055,0}}, {garbage_collection,{1353668,17832443756,0}}, {io,{{input,144147525},{output,6135027923}}}, {reductions,{1640205277,3129714}}, {run_queue,0}, {runtime,{359440,1090}}]}, {system_stats, [{cpu_utilization_rate,8.0},{swap_total,10684628992},{swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,1241611974}, {couch_docs_data_size,1241128842}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,290970}, {curr_items_tot,502178}, {mem_used,1466583544}, {vb_replica_curr_items,211208}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1146}, {memory_data,{33657663488,10465226752,{<17795.427.0>,15576440}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 22682336 kB\nBuffers: 168184 kB\nCached: 1257376 kB\nSwapCached: 0 kB\nActive: 8725240 kB\nInactive: 1276340 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 22682336 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 1812 kB\nWriteback: 0 kB\nAnonPages: 8575908 kB\nMapped: 16492 kB\nSlab: 128492 kB\nPageTables: 22220 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10295284 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,919979,429148}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default", [{'ns_1@10.2.1.58',104740915},{'ns_1@10.2.1.61',50002795}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,223860904}, {processes,172582832}, {processes_used,172477760}, {system,51278072}, {atom,1338193}, {atom_used,1334188}, {binary,11764560}, {code,13493302}, {ets,5915584}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,8445337600}, {buffered_memory,205168640}, {free_memory,15704907776}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1139942,4997}}, {context_switches,{9874465,0}}, {garbage_collection,{2270063,4982089623,0}}, {io,{{input,240659701},{output,11749535420}}}, {reductions,{1529808924,5450194}}, {run_queue,0}, {runtime,{238120,1020}}]}, {system_stats, [{cpu_utilization_rate,29.17705735660848}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,8334705757}, {couch_docs_data_size,7285548158}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1419793}, {curr_items_tot,3000002}, {mem_used,8079103976}, {vb_replica_curr_items,1580209}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1139}, {memory_data,{33657663488,17469222912,{<0.368.0>,5085816}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 15336824 kB\nBuffers: 200360 kB\nCached: 8247400 kB\nSwapCached: 0 kB\nActive: 9479540 kB\nInactive: 7726292 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 15336824 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 20544 kB\nWriteback: 0 kB\nAnonPages: 8758064 kB\nMapped: 16352 kB\nSlab: 269684 kB\nPageTables: 22636 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10472408 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-14T11:07:24.428,ns_1@10.2.1.63:ns_doctor<0.540.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,920040,26744}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',111474371}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.4647.1>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,59}, {started_on,1352919988}, {total_vbuckets,850}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1352920051}, {vbuckets_done,503}]]}, {memory, [{total,202912104}, {processes,156274304}, {processes_used,156259040}, {system,46637800}, {atom,1332537}, {atom_used,1325003}, {binary,8509496}, {code,13364289}, {ets,4703176}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7056048128}, {buffered_memory,191963136}, {free_memory,18401218560}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1202495,5000}}, {context_switches,{6709687,0}}, {garbage_collection,{1949309,3593562127,0}}, {io,{{input,153828566},{output,15745053781}}}, {reductions,{833306506,4891133}}, {run_queue,0}, {runtime,{194480,980}}]}, {system_stats, [{cpu_utilization_rate,42.64339152119701}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,6934788234}, {couch_docs_data_size,6378750070}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1359706}, {curr_items_tot,2641730}, {mem_used,6975596088}, {vb_replica_curr_items,1282024}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1202}, {memory_data,{33657663488,15230287872,{<19415.367.0>,5085816}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 17970064 kB\nBuffers: 187464 kB\nCached: 6890672 kB\nSwapCached: 0 kB\nActive: 7856572 kB\nInactive: 6755896 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 17970064 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 9212 kB\nWriteback: 1760 kB\nAnonPages: 7534608 kB\nMapped: 15528 kB\nSlab: 231828 kB\nPageTables: 19936 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 9257420 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920042,779836}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',90072056}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,138489024}, {processes,97555720}, {processes_used,93018840}, {system,40933304}, {atom,1407441}, {atom_used,1381817}, {binary,2295728}, {code,14112737}, {ets,5020944}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,926035968}, {buffered_memory,173473792}, {free_memory,23592292352}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1206877,4972}}, {context_switches,{7283962,0}}, {garbage_collection,{1405090,18916981143,0}}, {io,{{input,147080063},{output,6150552249}}}, {reductions,{1688497305,3888409}}, {run_queue,0}, {runtime,{377860,1370}}]}, {system_stats, [{cpu_utilization_rate,16.0}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,892386648}, {couch_docs_data_size,892048319}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,220458}, {curr_items_tot,358272}, {mem_used,1093698808}, {vb_replica_curr_items,137814}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1206}, {memory_data,{33657663488,10108669952,{<17795.427.0>,15568680}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 23039348 kB\nBuffers: 169408 kB\nCached: 904332 kB\nSwapCached: 0 kB\nActive: 8728948 kB\nInactive: 923600 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 23039348 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 1188 kB\nWriteback: 8 kB\nAnonPages: 8578812 kB\nMapped: 16492 kB\nSlab: 121280 kB\nPageTables: 22216 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10298132 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,920039,429492}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default", [{'ns_1@10.2.1.58',78662059},{'ns_1@10.2.1.61',29872704}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,231125840}, {processes,179902408}, {processes_used,179747464}, {system,51223432}, {atom,1338193}, {atom_used,1334188}, {binary,11590792}, {code,13493302}, {ets,6044928}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,8848392192}, {buffered_memory,206393344}, {free_memory,15278157824}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1199942,4999}}, {context_switches,{10290223,0}}, {garbage_collection,{2374351,5215124374,0}}, {io,{{input,249324401},{output,12074684348}}}, {reductions,{1593033529,4978614}}, {run_queue,0}, {runtime,{248770,860}}]}, {system_stats, [{cpu_utilization_rate,27.455919395465994}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,8742076614}, {couch_docs_data_size,7285339322}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1416938}, {curr_items_tot,3000002}, {mem_used,8085908840}, {vb_replica_curr_items,1583064}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1199}, {memory_data,{33657663488,17953644544,{<0.368.0>,5085816}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 14920076 kB\nBuffers: 201556 kB\nCached: 8641008 kB\nSwapCached: 0 kB\nActive: 9709248 kB\nInactive: 7903472 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 14920076 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 11448 kB\nWriteback: 0 kB\nAnonPages: 8769536 kB\nMapped: 16404 kB\nSlab: 279564 kB\nPageTables: 22484 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10491536 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-14T11:07:34.801,ns_1@10.2.1.63:<0.682.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,48,39}}) Stats for bucket "default": accepting_conns 1 auth_cmds 27 auth_errors 0 bucket_active_conns 2 bucket_conns 24 bytes 8088213640 bytes_read 17659570825 bytes_written 17106041852 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 82 cmd_set 2400893 conn_yields 62369 connection_structures 5000 curr_connections 25 curr_conns_on_port_11209 16 curr_conns_on_port_11210 7 curr_items 1414024 curr_items_tot 3000002 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 17 delete_misses 1 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-15 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data2/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 29825 ep_commit_time 2 ep_commit_time_total 293780 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 3 ep_data_age_highwat 150 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data2/default ep_degraded_mode 0 ep_diskqueue_drain 10826427 ep_diskqueue_fill 7147923 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 1 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 724 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 7312999 ep_io_read_bytes 0 ep_io_write_bytes 17316758737 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 292618 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 7339250270 ep_max_checkpoints 2 ep_max_data_size 12582912000 ep_max_item_size 20971520 ep_max_size 12582912000 ep_max_txn_size 10000 ep_max_vbuckets 1024 ep_mem_high_wat 9437184000 ep_mem_high_wat 9437184000 ep_mem_low_wat 7549747200 ep_mem_low_wat 7549747200 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 262968000 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1352918852 ep_storage_age 0 ep_storage_age_highwat 1129 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 3 ep_too_young 0 ep_total_cache_size 7303250246 ep_total_del_items 4 ep_total_enqueued 7147923 ep_total_new_items 3000005 ep_total_persisted 7312999 ep_uncommitted_items 0 ep_value_size 7091900551 ep_vb0 0 ep_vb_snapshot_total 2257 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_135_g40a391b ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 759 get_hits 69 get_misses 13 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 8088213640 pid 16783 pointer_size 64 rejected_conns 0 rusage_system 207.736419 rusage_user 781.005269 tap_checkpoint_end_received 858 tap_checkpoint_end_sent 1338 tap_checkpoint_start_received 2309 tap_checkpoint_start_sent 2929 tap_connect_received 945 tap_delete_received 2 tap_delete_sent 1 tap_mutation_received 4912181 tap_mutation_sent 6329608 tap_opaque_received 5062 tap_opaque_sent 4770 tap_vbucket_set_received 884 tap_vbucket_set_sent 942 threads 4 time 1352920054 total_connections 5543 uptime 1212 vb_active_curr_items 1414024 vb_active_eject 0 vb_active_expired 1 vb_active_ht_memory 12129096 vb_active_itm_memory 3442178272 vb_active_meta_data_memory 111707896 vb_active_num 483 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1414026 vb_active_ops_create 1414026 vb_active_ops_delete 3 vb_active_ops_reject 0 vb_active_ops_update 2142258 vb_active_perc_mem_resident 100 vb_active_queue_age 0 vb_active_queue_drain 5919319 vb_active_queue_fill 3428412 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 1585978 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 13585112 vb_replica_itm_memory 3861071974 vb_replica_meta_data_memory 125292262 vb_replica_num 541 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 1585978 vb_replica_ops_create 1585979 vb_replica_ops_delete 1 vb_replica_ops_reject 0 vb_replica_ops_update 2170732 vb_replica_perc_mem_resident 100 vb_replica_queue_age 0 vb_replica_queue_drain 4907108 vb_replica_queue_fill 3719511 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [stats:warn,2012-11-14T11:08:17.833,ns_1@10.2.1.63:<0.682.0>:stats_collector:latest_tick:223]Dropped 1 ticks [ns_doctor:debug,2012-11-14T11:08:24.428,ns_1@10.2.1.63:ns_doctor<0.540.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,920100,30099}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',68804196}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,213472168}, {processes,165492640}, {processes_used,165469560}, {system,47979528}, {atom,1332537}, {atom_used,1325003}, {binary,9524632}, {code,13364289}, {ets,4982032}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7442644992}, {buffered_memory,197820416}, {free_memory,17597079552}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1262498,5002}}, {context_switches,{7222759,0}}, {garbage_collection,{2046736,3818380120,0}}, {io,{{input,167627682},{output,16616333540}}}, {reductions,{891382241,4668747}}, {run_queue,0}, {runtime,{206170,910}}]}, {system_stats, [{cpu_utilization_rate,26.5}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7307952899}, {couch_docs_data_size,6729417354}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1427177}, {curr_items_tot,2785827}, {mem_used,7356527528}, {vb_replica_curr_items,1358650}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1262}, {memory_data,{33657663488,16027189248,{<19415.367.0>,5085816}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 17184648 kB\nBuffers: 193184 kB\nCached: 7268208 kB\nSwapCached: 0 kB\nActive: 8290792 kB\nInactive: 7097420 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 17184648 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 13228 kB\nWriteback: 132 kB\nAnonPages: 7926516 kB\nMapped: 15592 kB\nSlab: 241224 kB\nPageTables: 20556 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 9648304 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920102,777241}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',131185574}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,135208264}, {processes,94478000}, {processes_used,89701360}, {system,40730264}, {atom,1407441}, {atom_used,1381817}, {binary,2008344}, {code,14112737}, {ets,5139032}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,591388672}, {buffered_memory,174456832}, {free_memory,23935881216}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1266877,5000}}, {context_switches,{7379845,0}}, {garbage_collection,{1448557,19883260166,0}}, {io,{{input,152055788},{output,6167955041}}}, {reductions,{1736073030,3505198}}, {run_queue,0}, {runtime,{395130,1380}}]}, {system_stats, [{cpu_utilization_rate,16.209476309226932}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,541208880}, {couch_docs_data_size,540988944}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,143773}, {curr_items_tot,214175}, {mem_used,719392456}, {vb_replica_curr_items,70402}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1266}, {memory_data,{33657663488,9761525760,{<17795.427.0>,25159424}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 23374884 kB\nBuffers: 170368 kB\nCached: 577528 kB\nSwapCached: 0 kB\nActive: 8726812 kB\nInactive: 597388 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 23374884 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 636 kB\nWriteback: 160 kB\nAnonPages: 8576304 kB\nMapped: 16652 kB\nSlab: 113264 kB\nPageTables: 22000 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10293024 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,920099,430263}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default", [{'ns_1@10.2.1.58',70171377},{'ns_1@10.2.1.61',23164623}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,232301232}, {processes,180016160}, {processes_used,179817976}, {system,52285072}, {atom,1338193}, {atom_used,1334188}, {binary,12649528}, {code,13493302}, {ets,6052888}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,9270882304}, {buffered_memory,207441920}, {free_memory,14839939072}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1259943,5002}}, {context_switches,{10722662,0}}, {garbage_collection,{2469579,5436216079,0}}, {io,{{input,258089341},{output,12405349235}}}, {reductions,{1654851060,5302633}}, {run_queue,0}, {runtime,{258910,850}}]}, {system_stats, [{cpu_utilization_rate,25.81453634085213}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,9158293012}, {couch_docs_data_size,7285137974}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1429052}, {curr_items_tot,3000002}, {mem_used,8093651736}, {vb_replica_curr_items,1570950}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1259}, {memory_data,{33657663488,18380394496,{<0.368.0>,5085816}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 14492128 kB\nBuffers: 202580 kB\nCached: 9053596 kB\nSwapCached: 0 kB\nActive: 9946512 kB\nInactive: 8083496 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 14492128 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 3980 kB\nWriteback: 0 kB\nAnonPages: 8773728 kB\nMapped: 16404 kB\nSlab: 290424 kB\nPageTables: 22504 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10498996 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:warn,2012-11-14T11:08:35.176,ns_1@10.2.1.63:<0.682.0>:stats_collector:latest_tick:223]Dropped 1 ticks [stats:debug,2012-11-14T11:09:16.801,ns_1@10.2.1.63:<0.682.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,48,40}}) Stats for bucket "default": accepting_conns 1 auth_cmds 27 auth_errors 0 bucket_active_conns 2 bucket_conns 28 bytes 8099825304 bytes_read 18193228227 bytes_written 18061980256 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 82 cmd_set 2400893 conn_yields 64222 connection_structures 5000 curr_connections 28 curr_conns_on_port_11209 20 curr_conns_on_port_11210 7 curr_items 1426304 curr_items_tot 3000002 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 17 delete_misses 1 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-15 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data2/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 1 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 38049 ep_commit_time 2 ep_commit_time_total 319272 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 2 ep_data_age_highwat 150 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data2/default ep_degraded_mode 0 ep_diskqueue_drain 11372181 ep_diskqueue_fill 7364196 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 1 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 763 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 7533349 ep_io_read_bytes 0 ep_io_write_bytes 17838774338 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 322093 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 7339250270 ep_max_checkpoints 2 ep_max_data_size 12582912000 ep_max_item_size 20971520 ep_max_size 12582912000 ep_max_txn_size 10000 ep_max_vbuckets 1024 ep_mem_high_wat 9437184000 ep_mem_high_wat 9437184000 ep_mem_low_wat 7549747200 ep_mem_low_wat 7549747200 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 260844274 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1352918852 ep_storage_age 0 ep_storage_age_highwat 1249 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 5 ep_too_young 0 ep_total_cache_size 7303250246 ep_total_del_items 4 ep_total_enqueued 7364196 ep_total_new_items 3000005 ep_total_persisted 7533349 ep_uncommitted_items 0 ep_value_size 7091900551 ep_vb0 0 ep_vb_snapshot_total 2478 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_135_g40a391b ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 759 get_hits 69 get_misses 13 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 8099825304 pid 16783 pointer_size 64 rejected_conns 0 rusage_system 216.424098 rusage_user 872.813312 tap_checkpoint_end_received 933 tap_checkpoint_end_sent 1451 tap_checkpoint_start_received 2474 tap_checkpoint_start_sent 3133 tap_connect_received 1021 tap_delete_received 2 tap_delete_sent 1 tap_mutation_received 5132531 tap_mutation_sent 6663611 tap_opaque_received 5436 tap_opaque_sent 5078 tap_vbucket_set_received 968 tap_vbucket_set_sent 1018 threads 4 time 1352920156 total_connections 6031 uptime 1314 vb_active_curr_items 1426304 vb_active_eject 0 vb_active_expired 1 vb_active_ht_memory 12229352 vb_active_itm_memory 3471517464 vb_active_meta_data_memory 112678016 vb_active_num 487 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1426306 vb_active_ops_create 1426306 vb_active_ops_delete 3 vb_active_ops_reject 0 vb_active_ops_update 2273414 vb_active_perc_mem_resident 100 vb_active_queue_age 0 vb_active_queue_drain 6297915 vb_active_queue_fill 3568358 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 1573698 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 13484856 vb_replica_itm_memory 3831732782 vb_replica_meta_data_memory 124322142 vb_replica_num 537 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 1573698 vb_replica_ops_create 1573699 vb_replica_ops_delete 1 vb_replica_ops_reject 0 vb_replica_ops_update 2259926 vb_replica_perc_mem_resident 100 vb_replica_queue_age 0 vb_replica_queue_drain 5074266 vb_replica_queue_fill 3795838 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-14T11:09:24.428,ns_1@10.2.1.63:ns_doctor<0.540.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,920160,26842}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',53432365}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,222520416}, {processes,174573768}, {processes_used,174561192}, {system,47946648}, {atom,1332537}, {atom_used,1325003}, {binary,9371480}, {code,13364289}, {ets,5072160}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,8254103552}, {buffered_memory,201252864}, {free_memory,16377475072}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1322495,5000}}, {context_switches,{7695053,0}}, {garbage_collection,{2132462,4021040787,0}}, {io,{{input,175820952},{output,17421427719}}}, {reductions,{943435476,4650342}}, {run_queue,0}, {runtime,{217320,1130}}]}, {system_stats, [{cpu_utilization_rate,28.32080200501253}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,8057970758}, {couch_docs_data_size,7064224715}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1497557}, {curr_items_tot,2921037}, {mem_used,7712004936}, {vb_replica_curr_items,1423480}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1322}, {memory_data,{33657663488,17268252672,{<19415.367.0>,5085816}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 15993752 kB\nBuffers: 196536 kB\nCached: 8060648 kB\nSwapCached: 0 kB\nActive: 8832712 kB\nInactive: 7724660 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 15993752 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 26488 kB\nWriteback: 0 kB\nAnonPages: 8299684 kB\nMapped: 15544 kB\nSlab: 261788 kB\nPageTables: 21220 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10016768 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920162,777917}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,126502080}, {processes,86621136}, {processes_used,81603736}, {system,39880944}, {atom,1407441}, {atom_used,1381817}, {binary,1202120}, {code,14112737}, {ets,5141856}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,328609792}, {buffered_memory,175767552}, {free_memory,24218341376}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1326877,5000}}, {context_switches,{7465498,0}}, {garbage_collection,{1501152,20883339831,0}}, {io,{{input,153535656},{output,6181704268}}}, {reductions,{1782421642,3158642}}, {run_queue,0}, {runtime,{412770,1240}}]}, {system_stats, [{cpu_utilization_rate,7.231920199501247}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,206102624}, {couch_docs_data_size,205994434}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,76056}, {curr_items_tot,76056}, {mem_used,361718392}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1326}, {memory_data,{33657663488,9472090112,{<17795.427.0>,15551296}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 23650724 kB\nBuffers: 171648 kB\nCached: 320908 kB\nSwapCached: 0 kB\nActive: 8755636 kB\nInactive: 299732 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 23650724 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 3480 kB\nWriteback: 0 kB\nAnonPages: 8562548 kB\nMapped: 16492 kB\nSlab: 106312 kB\nPageTables: 22060 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10282068 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,920159,430194}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default", [{'ns_1@10.2.1.58',48161739},{'ns_1@10.2.1.61',126369609}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,226650728}, {processes,173647920}, {processes_used,173408472}, {system,53002808}, {atom,1338193}, {atom_used,1334188}, {binary,13235216}, {code,13493302}, {ets,6187368}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,9753792512}, {buffered_memory,209567744}, {free_memory,14330904576}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1319943,5000}}, {context_switches,{11156512,0}}, {garbage_collection,{2564804,5661142371,0}}, {io,{{input,270686459},{output,12723839409}}}, {reductions,{1716683816,5004426}}, {run_queue,0}, {runtime,{269530,820}}]}, {system_stats, [{cpu_utilization_rate,31.25}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,9558346600}, {couch_docs_data_size,7284957346}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1423466}, {curr_items_tot,3000002}, {mem_used,8100377944}, {vb_replica_curr_items,1576536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1319}, {memory_data,{33657663488,18821533696,{<0.368.0>,5085816}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 13995148 kB\nBuffers: 204656 kB\nCached: 9525188 kB\nSwapCached: 0 kB\nActive: 10227380 kB\nInactive: 8287208 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 13995148 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 21736 kB\nWriteback: 0 kB\nAnonPages: 8784856 kB\nMapped: 16404 kB\nSlab: 303028 kB\nPageTables: 22512 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10497080 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-14T11:10:24.428,ns_1@10.2.1.63:ns_doctor<0.540.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,920220,27776}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,246283448}, {processes,196249632}, {processes_used,196239800}, {system,50033816}, {atom,1332537}, {atom_used,1325003}, {binary,11381904}, {code,13364289}, {ets,5168808}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,8821600256}, {buffered_memory,203079680}, {free_memory,15551528960}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1382496,5000}}, {context_switches,{8112986,0}}, {garbage_collection,{2183522,4187516440,0}}, {io,{{input,182872650},{output,17975678129}}}, {reductions,{985701269,3466890}}, {run_queue,0}, {runtime,{229280,650}}]}, {system_stats, [{cpu_utilization_rate,9.226932668329177}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,8662774620}, {couch_docs_data_size,7284405678}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7921144168}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1382}, {memory_data,{33657663488,18119847936,{<19415.84.0>,11371232}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 15187040 kB\nBuffers: 198320 kB\nCached: 8614844 kB\nSwapCached: 0 kB\nActive: 9205916 kB\nInactive: 8143672 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 15187040 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 11092 kB\nWriteback: 0 kB\nAnonPages: 8536480 kB\nMapped: 15592 kB\nSlab: 275916 kB\nPageTables: 21696 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10244068 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920217,777322}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17795.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,920219,429287}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.25852.1>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,28}, {started_on,1352920189}, {total_vbuckets,1025}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1352920219}, {vbuckets_done,297}]]}, {memory, [{total,244383184}, {processes,190380240}, {processes_used,190115760}, {system,54002944}, {atom,1338193}, {atom_used,1334188}, {binary,14315544}, {code,13493302}, {ets,6161552}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,9983737856}, {buffered_memory,212709376}, {free_memory,14074167296}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1379942,5001}}, {context_switches,{11494790,0}}, {garbage_collection,{2605904,5787588398,0}}, {io,{{input,278828838},{output,12927690900}}}, {reductions,{1765100395,5593055}}, {run_queue,0}, {runtime,{276910,670}}]}, {system_stats, [{cpu_utilization_rate,27.75}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,9795935096}, {couch_docs_data_size,7284789472}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,8098645352}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1379}, {memory_data,{33657663488,19331203072,{<0.368.0>,5085816}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 13744304 kB\nBuffers: 207724 kB\nCached: 9749744 kB\nSwapCached: 0 kB\nActive: 10335808 kB\nInactive: 8422096 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 13744304 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 21772 kB\nWriteback: 108 kB\nAnonPages: 8800996 kB\nMapped: 16492 kB\nSlab: 309068 kB\nPageTables: 22740 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10505996 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-14T11:11:05.045,ns_1@10.2.1.63:<0.682.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,48,40}}) Stats for bucket "default": accepting_conns 1 auth_cmds 27 auth_errors 0 bucket_active_conns 1 bucket_conns 13 bytes 8098639256 bytes_read 18405475962 bytes_written 18307448379 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 82 cmd_set 2400893 conn_yields 64601 connection_structures 5000 curr_connections 14 curr_conns_on_port_11209 11 curr_conns_on_port_11210 2 curr_items 1499536 curr_items_tot 3000002 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 17 delete_misses 1 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-15 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data2/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 38798 ep_commit_time 0 ep_commit_time_total 325721 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 3 ep_data_age_highwat 150 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data2/default ep_degraded_mode 0 ep_diskqueue_drain 11615034 ep_diskqueue_fill 7449128 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 1 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 771 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 7621113 ep_io_read_bytes 0 ep_io_write_bytes 18046504810 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 333881 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 7339250270 ep_max_checkpoints 2 ep_max_data_size 12582912000 ep_max_item_size 20971520 ep_max_size 12582912000 ep_max_txn_size 10000 ep_max_vbuckets 1024 ep_mem_high_wat 9437184000 ep_mem_high_wat 9437184000 ep_mem_low_wat 7549747200 ep_mem_low_wat 7549747200 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 259162528 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1352918852 ep_storage_age 0 ep_storage_age_highwat 1293 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 6 ep_too_young 0 ep_total_cache_size 7303250246 ep_total_del_items 4 ep_total_enqueued 7449128 ep_total_new_items 3000005 ep_total_persisted 7621113 ep_uncommitted_items 0 ep_value_size 7091900551 ep_vb0 0 ep_vb_snapshot_total 2566 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_135_g40a391b ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 759 get_hits 69 get_misses 13 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 8098639256 pid 16783 pointer_size 64 rejected_conns 0 rusage_system 218.685754 rusage_user 918.850313 tap_checkpoint_end_received 963 tap_checkpoint_end_sent 1480 tap_checkpoint_start_received 2560 tap_checkpoint_start_sent 3171 tap_connect_received 1026 tap_delete_received 2 tap_delete_sent 1 tap_mutation_received 5220295 tap_mutation_sent 6745798 tap_opaque_received 5633 tap_opaque_sent 5122 tap_vbucket_set_received 1024 tap_vbucket_set_sent 1024 threads 4 time 1352920264 total_connections 6210 uptime 1422 vb_active_curr_items 1499536 vb_active_eject 0 vb_active_expired 1 vb_active_ht_memory 12857152 vb_active_itm_memory 3649944280 vb_active_meta_data_memory 118463344 vb_active_num 512 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1499538 vb_active_ops_create 1499538 vb_active_ops_delete 3 vb_active_ops_reject 0 vb_active_ops_update 2399244 vb_active_perc_mem_resident 100 vb_active_queue_age 0 vb_active_queue_drain 6646319 vb_active_queue_fill 3764588 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 1500466 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 12857056 vb_replica_itm_memory 3653305966 vb_replica_meta_data_memory 118536814 vb_replica_num 512 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 1500466 vb_replica_ops_create 1500467 vb_replica_ops_delete 1 vb_replica_ops_reject 0 vb_replica_ops_update 2221860 vb_replica_perc_mem_resident 100 vb_replica_queue_age 0 vb_replica_queue_drain 4968715 vb_replica_queue_fill 3684540 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-14T11:11:24.428,ns_1@10.2.1.63:ns_doctor<0.540.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,920280,27764}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,245133872}, {processes,195300280}, {processes_used,195280264}, {system,49833592}, {atom,1334153}, {atom_used,1326649}, {binary,10971448}, {code,13390578}, {ets,5351592}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,8817266688}, {buffered_memory,203427840}, {free_memory,15553007616}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1442496,5000}}, {context_switches,{8307680,0}}, {garbage_collection,{2192048,4231577887,0}}, {io,{{input,189684855},{output,17980781006}}}, {reductions,{1011111472,3353271}}, {run_queue,0}, {runtime,{235480,650}}]}, {system_stats, [{cpu_utilization_rate,6.234413965087282}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,8662772768}, {couch_docs_data_size,7284405678}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7921143592}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1442}, {memory_data,{33657663488,18108702720,{<19415.84.0>,11371232}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 15188484 kB\nBuffers: 198660 kB\nCached: 8610612 kB\nSwapCached: 0 kB\nActive: 9208060 kB\nInactive: 8140368 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 15188484 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 6108 kB\nWriteback: 0 kB\nAnonPages: 8539148 kB\nMapped: 15592 kB\nSlab: 274756 kB\nPageTables: 21776 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10248200 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920217,777322}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17795.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,920279,429515}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.25852.1>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,88}, {started_on,1352920189}, {total_vbuckets,1025}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1352920279}, {vbuckets_done,903}]]}, {memory, [{total,214798176}, {processes,163683112}, {processes_used,163589968}, {system,51115064}, {atom,1338193}, {atom_used,1334188}, {binary,11039640}, {code,13493302}, {ets,6529736}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,8548184064}, {buffered_memory,218640384}, {free_memory,15548399616}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1439942,5000}}, {context_switches,{11795298,0}}, {garbage_collection,{2636392,5875114021,0}}, {io,{{input,285778886},{output,12930812785}}}, {reductions,{1803510259,3226399}}, {run_queue,0}, {runtime,{282790,490}}]}, {system_stats, [{cpu_utilization_rate,21.804511278195488}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,8359044296}, {couch_docs_data_size,7285837870}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,8098639256}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1439}, {memory_data,{33657663488,19581845504,{<0.84.0>,11742176}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 15183984 kB\nBuffers: 213516 kB\nCached: 8347836 kB\nSwapCached: 0 kB\nActive: 9823024 kB\nInactive: 7519128 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 15183984 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 15576 kB\nWriteback: 5088 kB\nAnonPages: 8780940 kB\nMapped: 15580 kB\nSlab: 286100 kB\nPageTables: 22156 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10474652 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-14T11:12:24.428,ns_1@10.2.1.63:ns_doctor<0.540.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,920340,27663}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,242951176}, {processes,193196600}, {processes_used,193176584}, {system,49754576}, {atom,1334153}, {atom_used,1326649}, {binary,10709168}, {code,13390578}, {ets,5534704}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,8818409472}, {buffered_memory,203657216}, {free_memory,15545786368}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1502495,4999}}, {context_switches,{8502525,0}}, {garbage_collection,{2202924,4274982314,0}}, {io,{{input,193058236},{output,17983474089}}}, {reductions,{1036592378,3263771}}, {run_queue,0}, {runtime,{241800,560}}]}, {system_stats, [{cpu_utilization_rate,2.756892230576441}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,8662772768}, {couch_docs_data_size,7284405678}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7921143592}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1502}, {memory_data,{33657663488,18111750144,{<19415.84.0>,11371232}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 15181432 kB\nBuffers: 198884 kB\nCached: 8611728 kB\nSwapCached: 0 kB\nActive: 9216000 kB\nInactive: 8141520 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 15181432 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 676 kB\nWriteback: 0 kB\nAnonPages: 8546740 kB\nMapped: 15592 kB\nSlab: 274168 kB\nPageTables: 21780 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10249576 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920217,777322}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17795.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,920339,429261}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,216377112}, {processes,167034704}, {processes_used,167022728}, {system,49342408}, {atom,1338193}, {atom_used,1334188}, {binary,8953560}, {code,13493302}, {ets,6831536}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7472091136}, {buffered_memory,219930624}, {free_memory,16647680000}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1499942,5000}}, {context_switches,{12014322,0}}, {garbage_collection,{2651138,5948337968,0}}, {io,{{input,293865245},{output,12936745020}}}, {reductions,{1836252182,2613752}}, {run_queue,0}, {runtime,{289060,520}}]}, {system_stats, [{cpu_utilization_rate,4.5},{swap_total,10585923584},{swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7288618492}, {couch_docs_data_size,7286342852}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,8098639256}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1499}, {memory_data,{33657663488,18110406656,{<0.676.0>,4714984}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16257500 kB\nBuffers: 214776 kB\nCached: 7296964 kB\nSwapCached: 0 kB\nActive: 9052436 kB\nInactive: 7237528 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16257500 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 11264 kB\nWriteback: 0 kB\nAnonPages: 8778328 kB\nMapped: 15596 kB\nSlab: 264728 kB\nPageTables: 22136 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10480024 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-14T11:12:45.044,ns_1@10.2.1.63:<0.682.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,48,40}}) Stats for bucket "default": accepting_conns 1 auth_cmds 27 auth_errors 0 bucket_active_conns 1 bucket_conns 12 bytes 8098639256 bytes_read 18405486932 bytes_written 18315344245 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 82 cmd_set 2400893 conn_yields 64601 connection_structures 5000 curr_connections 13 curr_conns_on_port_11209 11 curr_conns_on_port_11210 1 curr_items 1499536 curr_items_tot 3000002 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 17 delete_misses 1 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-15 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data2/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 38798 ep_commit_time 0 ep_commit_time_total 325721 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 3 ep_data_age_highwat 150 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data2/default ep_degraded_mode 0 ep_diskqueue_drain 11615034 ep_diskqueue_fill 7449128 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 1 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 771 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 7621113 ep_io_read_bytes 0 ep_io_write_bytes 18046504810 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 333881 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 7339250270 ep_max_checkpoints 2 ep_max_data_size 12582912000 ep_max_item_size 20971520 ep_max_size 12582912000 ep_max_txn_size 10000 ep_max_vbuckets 1024 ep_mem_high_wat 9437184000 ep_mem_high_wat 9437184000 ep_mem_low_wat 7549747200 ep_mem_low_wat 7549747200 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 259162528 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1352918852 ep_storage_age 0 ep_storage_age_highwat 1293 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 6 ep_too_young 0 ep_total_cache_size 7303250246 ep_total_del_items 4 ep_total_enqueued 7449128 ep_total_new_items 3000005 ep_total_persisted 7621113 ep_uncommitted_items 0 ep_value_size 7091900551 ep_vb0 0 ep_vb_snapshot_total 2566 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_135_g40a391b ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 759 get_hits 69 get_misses 13 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 8098639256 pid 16783 pointer_size 64 rejected_conns 0 rusage_system 218.722749 rusage_user 919.518211 tap_checkpoint_end_received 963 tap_checkpoint_end_sent 1480 tap_checkpoint_start_received 2560 tap_checkpoint_start_sent 3171 tap_connect_received 1026 tap_delete_received 2 tap_delete_sent 1 tap_mutation_received 5220295 tap_mutation_sent 6745798 tap_opaque_received 5633 tap_opaque_sent 5122 tap_vbucket_set_received 1024 tap_vbucket_set_sent 1024 threads 4 time 1352920364 total_connections 6210 uptime 1522 vb_active_curr_items 1499536 vb_active_eject 0 vb_active_expired 1 vb_active_ht_memory 12857152 vb_active_itm_memory 3649944280 vb_active_meta_data_memory 118463344 vb_active_num 512 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1499538 vb_active_ops_create 1499538 vb_active_ops_delete 3 vb_active_ops_reject 0 vb_active_ops_update 2399244 vb_active_perc_mem_resident 100 vb_active_queue_age 0 vb_active_queue_drain 6646319 vb_active_queue_fill 3764588 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 1500466 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 12857056 vb_replica_itm_memory 3653305966 vb_replica_meta_data_memory 118536814 vb_replica_num 512 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 1500466 vb_replica_ops_create 1500467 vb_replica_ops_delete 1 vb_replica_ops_reject 0 vb_replica_ops_update 2221860 vb_replica_perc_mem_resident 100 vb_replica_queue_age 0 vb_replica_queue_drain 4968715 vb_replica_queue_fill 3684540 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-14T11:13:24.428,ns_1@10.2.1.63:ns_doctor<0.540.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,920400,28152}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.18653.1>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,36}, {started_on,1352920370}, {total_vbuckets,1025}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1352920412}, {vbuckets_done,371}]]}, {memory, [{total,225083752}, {processes,175345904}, {processes_used,175331440}, {system,49737848}, {atom,1334153}, {atom_used,1326649}, {binary,10628936}, {code,13390578}, {ets,5599440}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,8819912704}, {buffered_memory,208461824}, {free_memory,15540445184}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1562496,5000}}, {context_switches,{8761389,0}}, {garbage_collection,{2227355,4330123526,0}}, {io,{{input,198301234},{output,17986851023}}}, {reductions,{1066893487,3834013}}, {run_queue,0}, {runtime,{248550,650}}]}, {system_stats, [{cpu_utilization_rate,19.0}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,8669064225}, {couch_docs_data_size,7284405678}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7921143592}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1562}, {memory_data,{33657663488,18134740992,{<19415.67.0>,11000288}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 15176216 kB\nBuffers: 203576 kB\nCached: 8613196 kB\nSwapCached: 0 kB\nActive: 9216416 kB\nInactive: 8142332 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 15176216 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 4532 kB\nWriteback: 0 kB\nAnonPages: 8542132 kB\nMapped: 15596 kB\nSlab: 275876 kB\nPageTables: 21836 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10249796 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920217,777322}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17795.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,920399,429553}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,217483232}, {processes,168074144}, {processes_used,168062168}, {system,49409088}, {atom,1338193}, {atom_used,1334188}, {binary,8846096}, {code,13493302}, {ets,7005696}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7473197056}, {buffered_memory,220028928}, {free_memory,16644378624}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1559942,5000}}, {context_switches,{12212446,0}}, {garbage_collection,{2661559,6017673954,0}}, {io,{{input,297323925},{output,12938684843}}}, {reductions,{1867306890,2636871}}, {run_queue,0}, {runtime,{295570,580}}]}, {system_stats, [{cpu_utilization_rate,4.271356783919598}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7288618492}, {couch_docs_data_size,7286342852}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,8098639256}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1559}, {memory_data,{33657663488,17010245632,{<0.368.0>,4114672}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16254276 kB\nBuffers: 214872 kB\nCached: 7298044 kB\nSwapCached: 0 kB\nActive: 9054568 kB\nInactive: 7238512 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16254276 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 9700 kB\nWriteback: 0 kB\nAnonPages: 8780240 kB\nMapped: 15596 kB\nSlab: 264584 kB\nPageTables: 22136 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10472836 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-14T11:14:24.428,ns_1@10.2.1.63:ns_doctor<0.540.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,920460,27856}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.18653.1>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,88}, {started_on,1352920370}, {total_vbuckets,1025}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1352920471}, {vbuckets_done,905}]]}, {memory, [{total,210218808}, {processes,161585680}, {processes_used,161568056}, {system,48633128}, {atom,1334153}, {atom_used,1326649}, {binary,9236280}, {code,13390578}, {ets,5878248}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7626969088}, {buffered_memory,214499328}, {free_memory,16768430080}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1622496,5001}}, {context_switches,{9048807,0}}, {garbage_collection,{2257941,4390957323,0}}, {io,{{input,208445148},{output,17993721825}}}, {reductions,{1099436226,3911876}}, {run_queue,0}, {runtime,{254770,620}}]}, {system_stats, [{cpu_utilization_rate,22.693266832917704}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7461172495}, {couch_docs_data_size,7285046167}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7921143592}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1622}, {memory_data,{33657663488,16883646464,{<19415.367.0>,3143528}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16375420 kB\nBuffers: 209472 kB\nCached: 7448212 kB\nSwapCached: 0 kB\nActive: 8832100 kB\nInactive: 7356972 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16375420 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 10348 kB\nWriteback: 984 kB\nAnonPages: 8531684 kB\nMapped: 15596 kB\nSlab: 247052 kB\nPageTables: 21820 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10235348 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920217,777322}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17795.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,920459,430034}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,214316936}, {processes,164536328}, {processes_used,164524352}, {system,49780608}, {atom,1338193}, {atom_used,1334188}, {binary,9154920}, {code,13493302}, {ets,7068208}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7474343936}, {buffered_memory,220123136}, {free_memory,16644759552}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1619942,5000}}, {context_switches,{12410628,0}}, {garbage_collection,{2671475,6086916355,0}}, {io,{{input,300772499},{output,12940631170}}}, {reductions,{1898309269,2621672}}, {run_queue,0}, {runtime,{301870,550}}]}, {system_stats, [{cpu_utilization_rate,4.738154613466334}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7288618493}, {couch_docs_data_size,7286342852}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,8098639256}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1619}, {memory_data,{33657663488,17014943744,{<0.676.0>,4714984}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16254648 kB\nBuffers: 214964 kB\nCached: 7299164 kB\nSwapCached: 0 kB\nActive: 9052604 kB\nInactive: 7239524 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16254648 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 8568 kB\nWriteback: 0 kB\nAnonPages: 8778112 kB\nMapped: 15596 kB\nSlab: 264592 kB\nPageTables: 22132 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10471600 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-14T11:14:25.043,ns_1@10.2.1.63:<0.682.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,48,40}}) Stats for bucket "default": accepting_conns 1 auth_cmds 27 auth_errors 0 bucket_active_conns 1 bucket_conns 10 bytes 8098634936 bytes_read 18405497902 bytes_written 18323240687 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 82 cmd_set 2400893 conn_yields 64601 connection_structures 5000 curr_connections 13 curr_conns_on_port_11209 10 curr_conns_on_port_11210 1 curr_items 1499536 curr_items_tot 3000002 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 17 delete_misses 1 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-15 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data2/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 38798 ep_commit_time 0 ep_commit_time_total 325721 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 3 ep_data_age_highwat 150 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data2/default ep_degraded_mode 0 ep_diskqueue_drain 11615034 ep_diskqueue_fill 7449128 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 1 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 771 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 7621113 ep_io_read_bytes 0 ep_io_write_bytes 18046504810 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 333881 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 7339250270 ep_max_checkpoints 2 ep_max_data_size 12582912000 ep_max_item_size 20971520 ep_max_size 12582912000 ep_max_txn_size 10000 ep_max_vbuckets 1024 ep_mem_high_wat 9437184000 ep_mem_high_wat 9437184000 ep_mem_low_wat 7549747200 ep_mem_low_wat 7549747200 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 259162528 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1352918852 ep_storage_age 0 ep_storage_age_highwat 1293 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 6 ep_too_young 0 ep_total_cache_size 7303250246 ep_total_del_items 4 ep_total_enqueued 7449128 ep_total_new_items 3000005 ep_total_persisted 7621113 ep_uncommitted_items 0 ep_value_size 7091900551 ep_vb0 0 ep_vb_snapshot_total 2566 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_135_g40a391b ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 759 get_hits 69 get_misses 13 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 8098634936 pid 16783 pointer_size 64 rejected_conns 0 rusage_system 218.757743 rusage_user 920.217105 tap_checkpoint_end_received 963 tap_checkpoint_end_sent 1480 tap_checkpoint_start_received 2560 tap_checkpoint_start_sent 3171 tap_connect_received 1026 tap_delete_received 2 tap_delete_sent 1 tap_mutation_received 5220295 tap_mutation_sent 6745798 tap_opaque_received 5633 tap_opaque_sent 5122 tap_vbucket_set_received 1024 tap_vbucket_set_sent 1024 threads 4 time 1352920464 total_connections 6210 uptime 1622 vb_active_curr_items 1499536 vb_active_eject 0 vb_active_expired 1 vb_active_ht_memory 12857152 vb_active_itm_memory 3649944280 vb_active_meta_data_memory 118463344 vb_active_num 512 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1499538 vb_active_ops_create 1499538 vb_active_ops_delete 3 vb_active_ops_reject 0 vb_active_ops_update 2399244 vb_active_perc_mem_resident 100 vb_active_queue_age 0 vb_active_queue_drain 6646319 vb_active_queue_fill 3764588 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 1500466 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 12857056 vb_replica_itm_memory 3653305966 vb_replica_meta_data_memory 118536814 vb_replica_num 512 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 1500466 vb_replica_ops_create 1500467 vb_replica_ops_delete 1 vb_replica_ops_reject 0 vb_replica_ops_update 2221860 vb_replica_perc_mem_resident 100 vb_replica_queue_age 0 vb_replica_queue_drain 4968715 vb_replica_queue_fill 3684540 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-14T11:15:24.428,ns_1@10.2.1.63:ns_doctor<0.540.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,920520,27685}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,219663080}, {processes,170956752}, {processes_used,170932936}, {system,48706328}, {atom,1334153}, {atom_used,1326649}, {binary,9163752}, {code,13390578}, {ets,6017816}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7447588864}, {buffered_memory,215818240}, {free_memory,16947728384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1682495,4999}}, {context_switches,{9263952,0}}, {garbage_collection,{2273145,4437883384,0}}, {io,{{input,212671151},{output,17996664770}}}, {reductions,{1126395736,3241718}}, {run_queue,0}, {runtime,{261300,600}}]}, {system_stats, [{cpu_utilization_rate,3.7593984962406015}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7287798818}, {couch_docs_data_size,7285575572}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7921143592}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1682}, {memory_data,{33657663488,16709554176,{<19415.367.0>,3143528}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16550516 kB\nBuffers: 210760 kB\nCached: 7273036 kB\nSwapCached: 0 kB\nActive: 8799276 kB\nInactive: 7221956 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16550516 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 10356 kB\nWriteback: 0 kB\nAnonPages: 8537340 kB\nMapped: 15592 kB\nSlab: 241144 kB\nPageTables: 21740 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10233844 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920217,777322}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17795.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,920519,429203}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,219004728}, {processes,169338120}, {processes_used,169323744}, {system,49666608}, {atom,1338193}, {atom_used,1334188}, {binary,8864424}, {code,13493302}, {ets,7245160}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7475384320}, {buffered_memory,220336128}, {free_memory,16638664704}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1679942,5000}}, {context_switches,{12609927,0}}, {garbage_collection,{2682143,6156440217,0}}, {io,{{input,307604793},{output,12946020937}}}, {reductions,{1929456475,2602346}}, {run_queue,0}, {runtime,{308140,530}}]}, {system_stats, [{cpu_utilization_rate,4.75}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7288616215}, {couch_docs_data_size,7286342852}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,8098628776}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1679}, {memory_data,{33657663488,17014943744,{<0.368.0>,4114672}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16248696 kB\nBuffers: 215172 kB\nCached: 7300180 kB\nSwapCached: 0 kB\nActive: 9058312 kB\nInactive: 7240664 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16248696 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 6416 kB\nWriteback: 0 kB\nAnonPages: 8783816 kB\nMapped: 15596 kB\nSlab: 264556 kB\nPageTables: 22132 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10476776 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-14T11:16:05.045,ns_1@10.2.1.63:<0.682.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,48,40}}) Stats for bucket "default": accepting_conns 1 auth_cmds 27 auth_errors 0 bucket_active_conns 1 bucket_conns 10 bytes 8098628776 bytes_read 18405508872 bytes_written 18331136547 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 82 cmd_set 2400893 conn_yields 64601 connection_structures 5000 curr_connections 13 curr_conns_on_port_11209 10 curr_conns_on_port_11210 1 curr_items 1499536 curr_items_tot 3000002 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 17 delete_misses 1 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-15 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data2/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 38798 ep_commit_time 0 ep_commit_time_total 325721 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 3 ep_data_age_highwat 150 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data2/default ep_degraded_mode 0 ep_diskqueue_drain 11615034 ep_diskqueue_fill 7449128 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 1 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 771 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 7621113 ep_io_read_bytes 0 ep_io_write_bytes 18046504810 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 333881 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 7339250270 ep_max_checkpoints 2 ep_max_data_size 12582912000 ep_max_item_size 20971520 ep_max_size 12582912000 ep_max_txn_size 10000 ep_max_vbuckets 1024 ep_mem_high_wat 9437184000 ep_mem_high_wat 9437184000 ep_mem_low_wat 7549747200 ep_mem_low_wat 7549747200 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 259162528 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1352918852 ep_storage_age 0 ep_storage_age_highwat 1293 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 6 ep_too_young 0 ep_total_cache_size 7303250246 ep_total_del_items 4 ep_total_enqueued 7449128 ep_total_new_items 3000005 ep_total_persisted 7621113 ep_uncommitted_items 0 ep_value_size 7091900551 ep_vb0 0 ep_vb_snapshot_total 2566 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_135_g40a391b ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 759 get_hits 69 get_misses 13 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 8098628776 pid 16783 pointer_size 64 rejected_conns 0 rusage_system 218.783739 rusage_user 920.927997 tap_checkpoint_end_received 963 tap_checkpoint_end_sent 1480 tap_checkpoint_start_received 2560 tap_checkpoint_start_sent 3171 tap_connect_received 1026 tap_delete_received 2 tap_delete_sent 1 tap_mutation_received 5220295 tap_mutation_sent 6745798 tap_opaque_received 5633 tap_opaque_sent 5122 tap_vbucket_set_received 1024 tap_vbucket_set_sent 1024 threads 4 time 1352920564 total_connections 6210 uptime 1722 vb_active_curr_items 1499536 vb_active_eject 0 vb_active_expired 1 vb_active_ht_memory 12857152 vb_active_itm_memory 3649944280 vb_active_meta_data_memory 118463344 vb_active_num 512 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1499538 vb_active_ops_create 1499538 vb_active_ops_delete 3 vb_active_ops_reject 0 vb_active_ops_update 2399244 vb_active_perc_mem_resident 100 vb_active_queue_age 0 vb_active_queue_drain 6646319 vb_active_queue_fill 3764588 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 1500466 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 12857056 vb_replica_itm_memory 3653305966 vb_replica_meta_data_memory 118536814 vb_replica_num 512 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 1500466 vb_replica_ops_create 1500467 vb_replica_ops_delete 1 vb_replica_ops_reject 0 vb_replica_ops_update 2221860 vb_replica_perc_mem_resident 100 vb_replica_queue_age 0 vb_replica_queue_drain 4968715 vb_replica_queue_fill 3684540 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-14T11:16:24.428,ns_1@10.2.1.63:ns_doctor<0.540.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,920580,27839}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,218772168}, {processes,169948288}, {processes_used,169924472}, {system,48823880}, {atom,1334153}, {atom_used,1326649}, {binary,9107320}, {code,13390578}, {ets,6191768}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7448715264}, {buffered_memory,215973888}, {free_memory,16942903296}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1742496,5001}}, {context_switches,{9458835,0}}, {garbage_collection,{2284401,4481455759,0}}, {io,{{input,216043789},{output,17999250048}}}, {reductions,{1151858173,3276167}}, {run_queue,0}, {runtime,{267820,570}}]}, {system_stats, [{cpu_utilization_rate,3.2581453634085213}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7287798818}, {couch_docs_data_size,7285575572}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7921143592}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1742}, {memory_data,{33657663488,16714379264,{<19415.367.0>,3514472}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16545804 kB\nBuffers: 210912 kB\nCached: 7274136 kB\nSwapCached: 0 kB\nActive: 8802700 kB\nInactive: 7223076 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16545804 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 8240 kB\nWriteback: 0 kB\nAnonPages: 8540580 kB\nMapped: 15592 kB\nSlab: 241124 kB\nPageTables: 21748 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10239968 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920217,777322}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17795.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,920579,429935}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,212933272}, {processes,163378400}, {processes_used,163364024}, {system,49554872}, {atom,1338193}, {atom_used,1334188}, {binary,8690096}, {code,13493302}, {ets,7307448}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7476502528}, {buffered_memory,220422144}, {free_memory,16638791680}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1739942,5000}}, {context_switches,{12808039,0}}, {garbage_collection,{2693224,6226063082,0}}, {io,{{input,310966840},{output,12948050691}}}, {reductions,{1960491405,2623039}}, {run_queue,0}, {runtime,{314410,530}}]}, {system_stats, [{cpu_utilization_rate,4.5},{swap_total,10585923584},{swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7288613936}, {couch_docs_data_size,7286342852}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,8098628776}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1739}, {memory_data,{33657663488,17020395520,{<0.368.0>,4114672}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16248820 kB\nBuffers: 215256 kB\nCached: 7301272 kB\nSwapCached: 0 kB\nActive: 9057964 kB\nInactive: 7241648 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16248820 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 5112 kB\nWriteback: 0 kB\nAnonPages: 8782964 kB\nMapped: 15596 kB\nSlab: 264484 kB\nPageTables: 22128 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10471132 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-14T11:17:24.428,ns_1@10.2.1.63:ns_doctor<0.540.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,920640,27823}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,218289552}, {processes,169358984}, {processes_used,169334368}, {system,48930568}, {atom,1334153}, {atom_used,1326649}, {binary,9037904}, {code,13390578}, {ets,6368432}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7444328448}, {buffered_memory,216215552}, {free_memory,16944664576}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1802495,4999}}, {context_switches,{9654066,0}}, {garbage_collection,{2296392,4526038060,0}}, {io,{{input,222786537},{output,18005059360}}}, {reductions,{1177467136,3364418}}, {run_queue,0}, {runtime,{274600,660}}]}, {system_stats, [{cpu_utilization_rate,6.015037593984962}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7287798818}, {couch_docs_data_size,7285575572}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7921143592}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1802}, {memory_data,{33657663488,16716537856,{<19415.367.0>,3514472}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16547524 kB\nBuffers: 211148 kB\nCached: 7269852 kB\nSwapCached: 0 kB\nActive: 8803568 kB\nInactive: 7219648 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16547524 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 6588 kB\nWriteback: 0 kB\nAnonPages: 8542188 kB\nMapped: 15592 kB\nSlab: 241260 kB\nPageTables: 21748 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10239968 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920217,777322}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17795.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,920639,429582}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,215709568}, {processes,165768776}, {processes_used,165754400}, {system,49940792}, {atom,1338193}, {atom_used,1334188}, {binary,8901696}, {code,13493302}, {ets,7481864}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7482986496}, {buffered_memory,248209408}, {free_memory,16590471168}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1799942,5000}}, {context_switches,{13006117,0}}, {garbage_collection,{2704255,6295338882,0}}, {io,{{input,314327513},{output,12949977160}}}, {reductions,{1991526255,2638948}}, {run_queue,0}, {runtime,{320790,530}}]}, {system_stats, [{cpu_utilization_rate,4.511278195488722}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7288613936}, {couch_docs_data_size,7286342852}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,8098628776}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1799}, {memory_data,{33657663488,17019760640,{<0.676.0>,4714984}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16201632 kB\nBuffers: 242392 kB\nCached: 7307604 kB\nSwapCached: 0 kB\nActive: 9088272 kB\nInactive: 7248716 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16201632 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 3112 kB\nWriteback: 0 kB\nAnonPages: 8787032 kB\nMapped: 16664 kB\nSlab: 273368 kB\nPageTables: 22500 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10483772 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-14T11:17:45.059,ns_1@10.2.1.63:<0.682.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,48,40}}) Stats for bucket "default": accepting_conns 1 auth_cmds 27 auth_errors 0 bucket_active_conns 1 bucket_conns 10 bytes 8098628776 bytes_read 18405520563 bytes_written 18340942845 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 82 cmd_set 2400893 conn_yields 64601 connection_structures 5000 curr_connections 13 curr_conns_on_port_11209 10 curr_conns_on_port_11210 1 curr_items 1499536 curr_items_tot 3000002 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 17 delete_misses 1 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-15 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data2/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 38798 ep_commit_time 0 ep_commit_time_total 325721 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 3 ep_data_age_highwat 150 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data2/default ep_degraded_mode 0 ep_diskqueue_drain 11615034 ep_diskqueue_fill 7449128 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 1 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 771 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 7621113 ep_io_read_bytes 0 ep_io_write_bytes 18046504810 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 333881 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 7339250270 ep_max_checkpoints 2 ep_max_data_size 12582912000 ep_max_item_size 20971520 ep_max_size 12582912000 ep_max_txn_size 10000 ep_max_vbuckets 1024 ep_mem_high_wat 9437184000 ep_mem_high_wat 9437184000 ep_mem_low_wat 7549747200 ep_mem_low_wat 7549747200 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 259162528 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1352918852 ep_storage_age 0 ep_storage_age_highwat 1293 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 6 ep_too_young 0 ep_total_cache_size 7303250246 ep_total_del_items 4 ep_total_enqueued 7449128 ep_total_new_items 3000005 ep_total_persisted 7621113 ep_uncommitted_items 0 ep_value_size 7091900551 ep_vb0 0 ep_vb_snapshot_total 2566 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_135_g40a391b ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 759 get_hits 69 get_misses 13 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 8098628776 pid 16783 pointer_size 64 rejected_conns 0 rusage_system 218.818734 rusage_user 922.261794 tap_checkpoint_end_received 963 tap_checkpoint_end_sent 1480 tap_checkpoint_start_received 2560 tap_checkpoint_start_sent 3171 tap_connect_received 1026 tap_delete_received 2 tap_delete_sent 1 tap_mutation_received 5220295 tap_mutation_sent 6745798 tap_opaque_received 5633 tap_opaque_sent 5122 tap_vbucket_set_received 1024 tap_vbucket_set_sent 1024 threads 4 time 1352920665 total_connections 6219 uptime 1823 vb_active_curr_items 1499536 vb_active_eject 0 vb_active_expired 1 vb_active_ht_memory 12857152 vb_active_itm_memory 3649944280 vb_active_meta_data_memory 118463344 vb_active_num 512 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1499538 vb_active_ops_create 1499538 vb_active_ops_delete 3 vb_active_ops_reject 0 vb_active_ops_update 2399244 vb_active_perc_mem_resident 100 vb_active_queue_age 0 vb_active_queue_drain 6646319 vb_active_queue_fill 3764588 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 1500466 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 12857056 vb_replica_itm_memory 3653305966 vb_replica_meta_data_memory 118536814 vb_replica_num 512 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 1500466 vb_replica_ops_create 1500467 vb_replica_ops_delete 1 vb_replica_ops_reject 0 vb_replica_ops_update 2221860 vb_replica_perc_mem_resident 100 vb_replica_queue_age 0 vb_replica_queue_drain 4968715 vb_replica_queue_fill 3684540 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-14T11:18:24.428,ns_1@10.2.1.63:ns_doctor<0.540.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,920700,28174}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,240641736}, {processes,187567512}, {processes_used,187542896}, {system,53074224}, {atom,1334153}, {atom_used,1326813}, {binary,12995768}, {code,13399752}, {ets,6543992}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7450689536}, {buffered_memory,243257344}, {free_memory,16878153728}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1862496,5000}}, {context_switches,{9850027,0}}, {garbage_collection,{2307281,4573256122,0}}, {io,{{input,226088880},{output,18007754650}}}, {reductions,{1204484721,3247127}}, {run_queue,0}, {runtime,{281060,540}}]}, {system_stats, [{cpu_utilization_rate,3.25}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7287798818}, {couch_docs_data_size,7285575572}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7921143592}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1862}, {memory_data,{33657663488,16779382784,{<19415.11.0>,13466536}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16482572 kB\nBuffers: 237556 kB\nCached: 7276064 kB\nSwapCached: 0 kB\nActive: 8853388 kB\nInactive: 7225736 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16482572 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 9512 kB\nWriteback: 0 kB\nAnonPages: 8565572 kB\nMapped: 16644 kB\nSlab: 250620 kB\nPageTables: 22128 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10276636 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920217,777322}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17795.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,920699,429913}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,234481144}, {processes,180229088}, {processes_used,180214712}, {system,54252056}, {atom,1349617}, {atom_used,1347167}, {binary,12959792}, {code,13553851}, {ets,7664864}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7592157184}, {buffered_memory,248610816}, {free_memory,16470429696}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1859942,5000}}, {context_switches,{13227460,0}}, {garbage_collection,{2715116,6441324766,0}}, {io,{{input,326682772},{output,12976347888}}}, {reductions,{2072929899,2633722}}, {run_queue,0}, {runtime,{329690,510}}]}, {system_stats, [{cpu_utilization_rate,4.260651629072682}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7288613936}, {couch_docs_data_size,7286342852}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,8098628776}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1859}, {memory_data,{33657663488,17068589056,{<0.368.0>,4114672}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16084404 kB\nBuffers: 242784 kB\nCached: 7414216 kB\nSwapCached: 0 kB\nActive: 9264120 kB\nInactive: 7190260 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16084404 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 80556 kB\nWriteback: 0 kB\nAnonPages: 8797228 kB\nMapped: 15608 kB\nSlab: 273356 kB\nPageTables: 22168 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10490200 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-14T11:19:24.428,ns_1@10.2.1.63:ns_doctor<0.540.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,920760,27878}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,264580112}, {processes,211386192}, {processes_used,211361576}, {system,53193920}, {atom,1337385}, {atom_used,1331436}, {binary,12988800}, {code,13451143}, {ets,6614648}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7527780352}, {buffered_memory,243695616}, {free_memory,16796962816}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1922495,4955}}, {context_switches,{10067431,0}}, {garbage_collection,{2317627,4693889814,0}}, {io,{{input,234897786},{output,18030422287}}}, {reductions,{1281701458,13248960}}, {run_queue,0}, {runtime,{289740,1130}}]}, {system_stats, [{cpu_utilization_rate,5.2631578947368425}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7287798818}, {couch_docs_data_size,7285575572}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7921143592}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1922}, {memory_data,{33657663488,16862621696,{<19415.11.0>,43123400}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16403284 kB\nBuffers: 237984 kB\nCached: 7351348 kB\nSwapCached: 0 kB\nActive: 8963316 kB\nInactive: 7191700 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16403284 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 77128 kB\nWriteback: 0 kB\nAnonPages: 8565496 kB\nMapped: 15604 kB\nSlab: 253588 kB\nPageTables: 21832 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10297136 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920217,777322}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17795.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,920759,430000}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,259347024}, {processes,205007792}, {processes_used,204993416}, {system,54339232}, {atom,1349617}, {atom_used,1347167}, {binary,12872512}, {code,13553851}, {ets,7839400}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7593336832}, {buffered_memory,248717312}, {free_memory,16470048768}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1919942,5000}}, {context_switches,{13426641,0}}, {garbage_collection,{2726402,6516179881,0}}, {io,{{input,330243682},{output,12978356339}}}, {reductions,{2105485481,2633933}}, {run_queue,0}, {runtime,{336290,560}}]}, {system_stats, [{cpu_utilization_rate,4.738154613466334}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7288613936}, {couch_docs_data_size,7286342852}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,8098628776}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1919}, {memory_data,{33657663488,17187868672,{<0.11.0>,13466536}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16084032 kB\nBuffers: 242888 kB\nCached: 7415368 kB\nSwapCached: 0 kB\nActive: 9264880 kB\nInactive: 7191288 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16084032 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 44072 kB\nWriteback: 0 kB\nAnonPages: 8797648 kB\nMapped: 15608 kB\nSlab: 273312 kB\nPageTables: 22144 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10531036 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-14T11:19:25.044,ns_1@10.2.1.63:<0.682.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,48,40}}) Stats for bucket "default": accepting_conns 1 auth_cmds 27 auth_errors 0 bucket_active_conns 1 bucket_conns 10 bytes 8098628776 bytes_read 18405531594 bytes_written 18349431526 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 82 cmd_set 2400893 conn_yields 64601 connection_structures 5000 curr_connections 13 curr_conns_on_port_11209 10 curr_conns_on_port_11210 1 curr_items 1499536 curr_items_tot 3000002 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 17 delete_misses 1 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-15 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data2/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 38798 ep_commit_time 0 ep_commit_time_total 325721 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 3 ep_data_age_highwat 150 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data2/default ep_degraded_mode 0 ep_diskqueue_drain 11615034 ep_diskqueue_fill 7449128 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 1 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 771 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 7621113 ep_io_read_bytes 0 ep_io_write_bytes 18046504810 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 333881 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 7339250270 ep_max_checkpoints 2 ep_max_data_size 12582912000 ep_max_item_size 20971520 ep_max_size 12582912000 ep_max_txn_size 10000 ep_max_vbuckets 1024 ep_mem_high_wat 9437184000 ep_mem_high_wat 9437184000 ep_mem_low_wat 7549747200 ep_mem_low_wat 7549747200 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 259162528 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1352918852 ep_storage_age 0 ep_storage_age_highwat 1293 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 6 ep_too_young 0 ep_total_cache_size 7303250246 ep_total_del_items 4 ep_total_enqueued 7449128 ep_total_new_items 3000005 ep_total_persisted 7621113 ep_uncommitted_items 0 ep_value_size 7091900551 ep_vb0 0 ep_vb_snapshot_total 2566 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_135_g40a391b ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 759 get_hits 69 get_misses 13 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 8098628776 pid 16783 pointer_size 64 rejected_conns 0 rusage_system 218.842730 rusage_user 922.992683 tap_checkpoint_end_received 963 tap_checkpoint_end_sent 1480 tap_checkpoint_start_received 2560 tap_checkpoint_start_sent 3171 tap_connect_received 1026 tap_delete_received 2 tap_delete_sent 1 tap_mutation_received 5220295 tap_mutation_sent 6745798 tap_opaque_received 5633 tap_opaque_sent 5122 tap_vbucket_set_received 1024 tap_vbucket_set_sent 1024 threads 4 time 1352920764 total_connections 6219 uptime 1922 vb_active_curr_items 1499536 vb_active_eject 0 vb_active_expired 1 vb_active_ht_memory 12857152 vb_active_itm_memory 3649944280 vb_active_meta_data_memory 118463344 vb_active_num 512 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1499538 vb_active_ops_create 1499538 vb_active_ops_delete 3 vb_active_ops_reject 0 vb_active_ops_update 2399244 vb_active_perc_mem_resident 100 vb_active_queue_age 0 vb_active_queue_drain 6646319 vb_active_queue_fill 3764588 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 1500466 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 12857056 vb_replica_itm_memory 3653305966 vb_replica_meta_data_memory 118536814 vb_replica_num 512 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 1500466 vb_replica_ops_create 1500467 vb_replica_ops_delete 1 vb_replica_ops_reject 0 vb_replica_ops_update 2221860 vb_replica_perc_mem_resident 100 vb_replica_queue_age 0 vb_replica_queue_drain 4968715 vb_replica_queue_fill 3684540 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-14T11:20:24.428,ns_1@10.2.1.63:ns_doctor<0.540.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,920820,28252}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,267573808}, {processes,213989232}, {processes_used,213963016}, {system,53584576}, {atom,1337385}, {atom_used,1331436}, {binary,13194392}, {code,13451143}, {ets,6799880}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7528632320}, {buffered_memory,243998720}, {free_memory,16799518720}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1982496,5001}}, {context_switches,{10263375,0}}, {garbage_collection,{2329090,4738874764,0}}, {io,{{input,241853673},{output,18036597868}}}, {reductions,{1307524213,3397358}}, {run_queue,0}, {runtime,{296250,680}}]}, {system_stats, [{cpu_utilization_rate,6.234413965087282}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7287798818}, {couch_docs_data_size,7285575572}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7921143592}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1982}, {memory_data,{33657663488,16857890816,{<19415.11.0>,43123400}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16405780 kB\nBuffers: 238280 kB\nCached: 7352180 kB\nSwapCached: 0 kB\nActive: 8961804 kB\nInactive: 7192804 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16405780 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 19848 kB\nWriteback: 0 kB\nAnonPages: 8564100 kB\nMapped: 15604 kB\nSlab: 252032 kB\nPageTables: 21824 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10300936 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920217,777322}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17795.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,920819,430188}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,264909328}, {processes,210612968}, {processes_used,210598592}, {system,54296360}, {atom,1349617}, {atom_used,1347167}, {binary,12763976}, {code,13553851}, {ets,7904640}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7594438656}, {buffered_memory,248811520}, {free_memory,16470175744}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1979943,5000}}, {context_switches,{13624731,0}}, {garbage_collection,{2736949,6585481975,0}}, {io,{{input,333692357},{output,12980406665}}}, {reductions,{2136492632,2641977}}, {run_queue,0}, {runtime,{342630,520}}]}, {system_stats, [{cpu_utilization_rate,4.5},{swap_total,10585923584},{swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7288613936}, {couch_docs_data_size,7286342852}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,8098628776}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1979}, {memory_data,{33657663488,17189773312,{<0.11.0>,43123400}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16084156 kB\nBuffers: 242980 kB\nCached: 7416444 kB\nSwapCached: 0 kB\nActive: 9264136 kB\nInactive: 7192284 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16084156 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 15908 kB\nWriteback: 0 kB\nAnonPages: 8796744 kB\nMapped: 15608 kB\nSlab: 272748 kB\nPageTables: 22140 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10519676 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-14T11:21:05.044,ns_1@10.2.1.63:<0.682.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,48,40}}) Stats for bucket "default": accepting_conns 1 auth_cmds 27 auth_errors 0 bucket_active_conns 1 bucket_conns 10 bytes 8098628776 bytes_read 18405542564 bytes_written 18357332662 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 82 cmd_set 2400893 conn_yields 64601 connection_structures 5000 curr_connections 13 curr_conns_on_port_11209 10 curr_conns_on_port_11210 1 curr_items 1499536 curr_items_tot 3000002 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 17 delete_misses 1 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-15 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data2/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 38798 ep_commit_time 0 ep_commit_time_total 325721 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 3 ep_data_age_highwat 150 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data2/default ep_degraded_mode 0 ep_diskqueue_drain 11615034 ep_diskqueue_fill 7449128 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 1 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 771 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 7621113 ep_io_read_bytes 0 ep_io_write_bytes 18046504810 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 333881 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 7339250270 ep_max_checkpoints 2 ep_max_data_size 12582912000 ep_max_item_size 20971520 ep_max_size 12582912000 ep_max_txn_size 10000 ep_max_vbuckets 1024 ep_mem_high_wat 9437184000 ep_mem_high_wat 9437184000 ep_mem_low_wat 7549747200 ep_mem_low_wat 7549747200 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 259162528 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1352918852 ep_storage_age 0 ep_storage_age_highwat 1293 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 6 ep_too_young 0 ep_total_cache_size 7303250246 ep_total_del_items 4 ep_total_enqueued 7449128 ep_total_new_items 3000005 ep_total_persisted 7621113 ep_uncommitted_items 0 ep_value_size 7091900551 ep_vb0 0 ep_vb_snapshot_total 2566 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_135_g40a391b ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 759 get_hits 69 get_misses 13 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 8098628776 pid 16783 pointer_size 64 rejected_conns 0 rusage_system 218.875725 rusage_user 923.702575 tap_checkpoint_end_received 963 tap_checkpoint_end_sent 1480 tap_checkpoint_start_received 2560 tap_checkpoint_start_sent 3171 tap_connect_received 1026 tap_delete_received 2 tap_delete_sent 1 tap_mutation_received 5220295 tap_mutation_sent 6745798 tap_opaque_received 5633 tap_opaque_sent 5122 tap_vbucket_set_received 1024 tap_vbucket_set_sent 1024 threads 4 time 1352920864 total_connections 6219 uptime 2022 vb_active_curr_items 1499536 vb_active_eject 0 vb_active_expired 1 vb_active_ht_memory 12857152 vb_active_itm_memory 3649944280 vb_active_meta_data_memory 118463344 vb_active_num 512 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1499538 vb_active_ops_create 1499538 vb_active_ops_delete 3 vb_active_ops_reject 0 vb_active_ops_update 2399244 vb_active_perc_mem_resident 100 vb_active_queue_age 0 vb_active_queue_drain 6646319 vb_active_queue_fill 3764588 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 1500466 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 12857056 vb_replica_itm_memory 3653305966 vb_replica_meta_data_memory 118536814 vb_replica_num 512 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 1500466 vb_replica_ops_create 1500467 vb_replica_ops_delete 1 vb_replica_ops_reject 0 vb_replica_ops_update 2221860 vb_replica_perc_mem_resident 100 vb_replica_queue_age 0 vb_replica_queue_drain 4968715 vb_replica_queue_fill 3684540 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-14T11:21:24.428,ns_1@10.2.1.63:ns_doctor<0.540.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,920880,27975}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,264871528}, {processes,211442248}, {processes_used,211416032}, {system,53429280}, {atom,1337385}, {atom_used,1331436}, {binary,12975832}, {code,13451143}, {ets,6862968}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7529758720}, {buffered_memory,244154368}, {free_memory,16799772672}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2042495,4999}}, {context_switches,{10458128,0}}, {garbage_collection,{2340501,4782323659,0}}, {io,{{input,245137493},{output,18039082637}}}, {reductions,{1333059863,3265992}}, {run_queue,0}, {runtime,{302600,580}}]}, {system_stats, [{cpu_utilization_rate,3.7593984962406015}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7287798818}, {couch_docs_data_size,7285575572}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7921143592}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2042}, {memory_data,{33657663488,16858152960,{<19415.11.0>,43123400}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16406028 kB\nBuffers: 238432 kB\nCached: 7353280 kB\nSwapCached: 0 kB\nActive: 8961472 kB\nInactive: 7193872 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16406028 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 4344 kB\nWriteback: 0 kB\nAnonPages: 8563508 kB\nMapped: 15604 kB\nSlab: 251596 kB\nPageTables: 21828 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10296968 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920217,777322}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17795.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,920879,429909}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,262400728}, {processes,207666704}, {processes_used,207652328}, {system,54734024}, {atom,1349617}, {atom_used,1347167}, {binary,13024472}, {code,13553851}, {ets,8082304}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7595683840}, {buffered_memory,249008128}, {free_memory,16466771968}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2039942,4999}}, {context_switches,{13823882,0}}, {garbage_collection,{2747019,6655108234,0}}, {io,{{input,340509010},{output,12985689239}}}, {reductions,{2167614415,2624470}}, {run_queue,0}, {runtime,{349060,520}}]}, {system_stats, [{cpu_utilization_rate,4.5},{swap_total,10585923584},{swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7288613936}, {couch_docs_data_size,7286342852}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,8098628776}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2039}, {memory_data,{33657663488,17188122624,{<0.11.0>,43123400}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16080832 kB\nBuffers: 243172 kB\nCached: 7417660 kB\nSwapCached: 0 kB\nActive: 9265704 kB\nInactive: 7193412 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16080832 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 6276 kB\nWriteback: 0 kB\nAnonPages: 8798404 kB\nMapped: 15608 kB\nSlab: 272460 kB\nPageTables: 22172 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10521772 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-14T11:22:24.428,ns_1@10.2.1.63:ns_doctor<0.540.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,920940,28010}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,262763520}, {processes,208878224}, {processes_used,208852008}, {system,53885296}, {atom,1337385}, {atom_used,1331436}, {binary,13257120}, {code,13451143}, {ets,7037608}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7530885120}, {buffered_memory,244330496}, {free_memory,16796090368}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2102495,4999}}, {context_switches,{10653030,0}}, {garbage_collection,{2351208,4826162308,0}}, {io,{{input,248513547},{output,18041866735}}}, {reductions,{1358542781,3251506}}, {run_queue,0}, {runtime,{308810,540}}]}, {system_stats, [{cpu_utilization_rate,3.2581453634085213}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7287798818}, {couch_docs_data_size,7285575572}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7921143592}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2102}, {memory_data,{33657663488,16861073408,{<19415.11.0>,43123400}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16402432 kB\nBuffers: 238604 kB\nCached: 7354380 kB\nSwapCached: 0 kB\nActive: 8963596 kB\nInactive: 7194996 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16402432 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 508 kB\nWriteback: 0 kB\nAnonPages: 8565444 kB\nMapped: 15604 kB\nSlab: 251584 kB\nPageTables: 21828 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10298524 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920217,777322}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17795.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,920939,429985}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,261648360}, {processes,206995096}, {processes_used,206980720}, {system,54653264}, {atom,1349617}, {atom_used,1347167}, {binary,12871928}, {code,13553851}, {ets,8153728}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7596793856}, {buffered_memory,249102336}, {free_memory,16465756160}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2099942,5000}}, {context_switches,{14021997,0}}, {garbage_collection,{2757645,6724291945,0}}, {io,{{input,343967629},{output,12987712082}}}, {reductions,{2198561426,2622733}}, {run_queue,0}, {runtime,{355280,530}}]}, {system_stats, [{cpu_utilization_rate,4.488778054862843}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7288613936}, {couch_docs_data_size,7286342852}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,8098628776}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2099}, {memory_data,{33657663488,17192923136,{<0.11.0>,43123400}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16079840 kB\nBuffers: 243264 kB\nCached: 7418744 kB\nSwapCached: 0 kB\nActive: 9265604 kB\nInactive: 7194432 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16079840 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 396 kB\nWriteback: 0 kB\nAnonPages: 8798244 kB\nMapped: 15608 kB\nSlab: 272408 kB\nPageTables: 22172 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10525412 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-14T11:22:45.045,ns_1@10.2.1.63:<0.682.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,48,40}}) Stats for bucket "default": accepting_conns 1 auth_cmds 27 auth_errors 0 bucket_active_conns 1 bucket_conns 10 bytes 8098628776 bytes_read 18405553534 bytes_written 18365233822 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 82 cmd_set 2400893 conn_yields 64601 connection_structures 5000 curr_connections 13 curr_conns_on_port_11209 10 curr_conns_on_port_11210 1 curr_items 1499536 curr_items_tot 3000002 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 17 delete_misses 1 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-15 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data2/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 38798 ep_commit_time 0 ep_commit_time_total 325721 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 3 ep_data_age_highwat 150 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data2/default ep_degraded_mode 0 ep_diskqueue_drain 11615034 ep_diskqueue_fill 7449128 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 1 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 771 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 7621113 ep_io_read_bytes 0 ep_io_write_bytes 18046504810 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 333881 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 7339250270 ep_max_checkpoints 2 ep_max_data_size 12582912000 ep_max_item_size 20971520 ep_max_size 12582912000 ep_max_txn_size 10000 ep_max_vbuckets 1024 ep_mem_high_wat 9437184000 ep_mem_high_wat 9437184000 ep_mem_low_wat 7549747200 ep_mem_low_wat 7549747200 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 259162528 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1352918852 ep_storage_age 0 ep_storage_age_highwat 1293 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 6 ep_too_young 0 ep_total_cache_size 7303250246 ep_total_del_items 4 ep_total_enqueued 7449128 ep_total_new_items 3000005 ep_total_persisted 7621113 ep_uncommitted_items 0 ep_value_size 7091900551 ep_vb0 0 ep_vb_snapshot_total 2566 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_135_g40a391b ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 759 get_hits 69 get_misses 13 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 8098628776 pid 16783 pointer_size 64 rejected_conns 0 rusage_system 218.901721 rusage_user 924.417467 tap_checkpoint_end_received 963 tap_checkpoint_end_sent 1480 tap_checkpoint_start_received 2560 tap_checkpoint_start_sent 3171 tap_connect_received 1026 tap_delete_received 2 tap_delete_sent 1 tap_mutation_received 5220295 tap_mutation_sent 6745798 tap_opaque_received 5633 tap_opaque_sent 5122 tap_vbucket_set_received 1024 tap_vbucket_set_sent 1024 threads 4 time 1352920964 total_connections 6219 uptime 2122 vb_active_curr_items 1499536 vb_active_eject 0 vb_active_expired 1 vb_active_ht_memory 12857152 vb_active_itm_memory 3649944280 vb_active_meta_data_memory 118463344 vb_active_num 512 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1499538 vb_active_ops_create 1499538 vb_active_ops_delete 3 vb_active_ops_reject 0 vb_active_ops_update 2399244 vb_active_perc_mem_resident 100 vb_active_queue_age 0 vb_active_queue_drain 6646319 vb_active_queue_fill 3764588 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 1500466 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 12857056 vb_replica_itm_memory 3653305966 vb_replica_meta_data_memory 118536814 vb_replica_num 512 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 1500466 vb_replica_ops_create 1500467 vb_replica_ops_delete 1 vb_replica_ops_reject 0 vb_replica_ops_update 2221860 vb_replica_perc_mem_resident 100 vb_replica_queue_age 0 vb_replica_queue_drain 4968715 vb_replica_queue_fill 3684540 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-14T11:23:24.428,ns_1@10.2.1.63:ns_doctor<0.540.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,921000,28098}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,267595000}, {processes,213746104}, {processes_used,213719888}, {system,53848896}, {atom,1337385}, {atom_used,1331436}, {binary,13043848}, {code,13451143}, {ets,7214872}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7526498304}, {buffered_memory,244588544}, {free_memory,16798359552}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2162495,4999}}, {context_switches,{10848313,0}}, {garbage_collection,{2361761,4871082656,0}}, {io,{{input,255249749},{output,18048644701}}}, {reductions,{1384061140,3343713}}, {run_queue,0}, {runtime,{315300,640}}]}, {system_stats, [{cpu_utilization_rate,6.25}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7287798818}, {couch_docs_data_size,7285575572}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7921143592}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2162}, {memory_data,{33657663488,16862334976,{<19415.11.0>,43123400}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16404648 kB\nBuffers: 238856 kB\nCached: 7350096 kB\nSwapCached: 0 kB\nActive: 8964432 kB\nInactive: 7191456 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16404648 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 240 kB\nWriteback: 0 kB\nAnonPages: 8567016 kB\nMapped: 15604 kB\nSlab: 251872 kB\nPageTables: 21828 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10298196 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920217,777322}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17795.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,920999,429324}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,266679136}, {processes,211543272}, {processes_used,211528896}, {system,55135864}, {atom,1349617}, {atom_used,1347167}, {binary,13180120}, {code,13553851}, {ets,8328136}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7597903872}, {buffered_memory,249171968}, {free_memory,16463470592}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2159942,4999}}, {context_switches,{14220063,0}}, {garbage_collection,{2767033,6792930167,0}}, {io,{{input,347435846},{output,12989643897}}}, {reductions,{2229575038,2614884}}, {run_queue,0}, {runtime,{361600,530}}]}, {system_stats, [{cpu_utilization_rate,4.0},{swap_total,10585923584},{swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7288613936}, {couch_docs_data_size,7286342852}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,8098628776}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2159}, {memory_data,{33657663488,17192923136,{<0.11.0>,43123400}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16077608 kB\nBuffers: 243332 kB\nCached: 7419828 kB\nSwapCached: 0 kB\nActive: 9267032 kB\nInactive: 7195364 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16077608 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 276 kB\nWriteback: 0 kB\nAnonPages: 8799232 kB\nMapped: 15608 kB\nSlab: 272388 kB\nPageTables: 22172 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10525992 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-14T11:24:24.428,ns_1@10.2.1.63:ns_doctor<0.540.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,921060,28154}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,265435024}, {processes,211400808}, {processes_used,211374592}, {system,54034216}, {atom,1337385}, {atom_used,1331436}, {binary,13054640}, {code,13451143}, {ets,7389440}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7527616512}, {buffered_memory,244744192}, {free_memory,16796344320}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2222495,4999}}, {context_switches,{11043237,0}}, {garbage_collection,{2372140,4914760602,0}}, {io,{{input,258526018},{output,18051222063}}}, {reductions,{1409575088,3267648}}, {run_queue,0}, {runtime,{321750,560}}]}, {system_stats, [{cpu_utilization_rate,3.491271820448878}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7287798818}, {couch_docs_data_size,7285575572}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7921143592}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2222}, {memory_data,{33657663488,16861192192,{<19415.11.0>,43123400}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16402680 kB\nBuffers: 239008 kB\nCached: 7351188 kB\nSwapCached: 0 kB\nActive: 8966044 kB\nInactive: 7192552 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16402680 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 672 kB\nWriteback: 0 kB\nAnonPages: 8568272 kB\nMapped: 15604 kB\nSlab: 251552 kB\nPageTables: 21824 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10302752 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920217,777322}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17795.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,921059,429311}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,260353032}, {processes,205341008}, {processes_used,205326632}, {system,55012024}, {atom,1349617}, {atom_used,1347167}, {binary,12879320}, {code,13553851}, {ets,8505408}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7593324544}, {buffered_memory,249352192}, {free_memory,16466137088}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2219942,5000}}, {context_switches,{14418755,0}}, {garbage_collection,{2778510,6864368138,0}}, {io,{{input,354329734},{output,12996923020}}}, {reductions,{2260980924,2629723}}, {run_queue,0}, {runtime,{368210,550}}]}, {system_stats, [{cpu_utilization_rate,4.488778054862843}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7288613936}, {couch_docs_data_size,7286342852}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,8098628776}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2219}, {memory_data,{33657663488,17195843584,{<0.11.0>,43123400}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16080212 kB\nBuffers: 243508 kB\nCached: 7415356 kB\nSwapCached: 0 kB\nActive: 9267708 kB\nInactive: 7191688 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16080212 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 488 kB\nWriteback: 0 kB\nAnonPages: 8800728 kB\nMapped: 15608 kB\nSlab: 272336 kB\nPageTables: 22184 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10523296 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-14T11:24:25.044,ns_1@10.2.1.63:<0.682.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,48,41}}) Stats for bucket "default": accepting_conns 1 auth_cmds 27 auth_errors 0 bucket_active_conns 1 bucket_conns 10 bytes 8098628776 bytes_read 18405564504 bytes_written 18373134982 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 82 cmd_set 2400893 conn_yields 64601 connection_structures 5000 curr_connections 13 curr_conns_on_port_11209 10 curr_conns_on_port_11210 1 curr_items 1499536 curr_items_tot 3000002 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 17 delete_misses 1 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-15 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data2/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 38798 ep_commit_time 0 ep_commit_time_total 325721 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 3 ep_data_age_highwat 150 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data2/default ep_degraded_mode 0 ep_diskqueue_drain 11615034 ep_diskqueue_fill 7449128 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 1 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 771 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 7621113 ep_io_read_bytes 0 ep_io_write_bytes 18046504810 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 333881 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 7339250270 ep_max_checkpoints 2 ep_max_data_size 12582912000 ep_max_item_size 20971520 ep_max_size 12582912000 ep_max_txn_size 10000 ep_max_vbuckets 1024 ep_mem_high_wat 9437184000 ep_mem_high_wat 9437184000 ep_mem_low_wat 7549747200 ep_mem_low_wat 7549747200 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 259162528 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1352918852 ep_storage_age 0 ep_storage_age_highwat 1293 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 6 ep_too_young 0 ep_total_cache_size 7303250246 ep_total_del_items 4 ep_total_enqueued 7449128 ep_total_new_items 3000005 ep_total_persisted 7621113 ep_uncommitted_items 0 ep_value_size 7091900551 ep_vb0 0 ep_vb_snapshot_total 2566 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_135_g40a391b ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 759 get_hits 69 get_misses 13 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 8098628776 pid 16783 pointer_size 64 rejected_conns 0 rusage_system 218.931717 rusage_user 925.122360 tap_checkpoint_end_received 963 tap_checkpoint_end_sent 1480 tap_checkpoint_start_received 2560 tap_checkpoint_start_sent 3171 tap_connect_received 1026 tap_delete_received 2 tap_delete_sent 1 tap_mutation_received 5220295 tap_mutation_sent 6745798 tap_opaque_received 5633 tap_opaque_sent 5122 tap_vbucket_set_received 1024 tap_vbucket_set_sent 1024 threads 4 time 1352921064 total_connections 6219 uptime 2222 vb_active_curr_items 1499536 vb_active_eject 0 vb_active_expired 1 vb_active_ht_memory 12857152 vb_active_itm_memory 3649944280 vb_active_meta_data_memory 118463344 vb_active_num 512 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1499538 vb_active_ops_create 1499538 vb_active_ops_delete 3 vb_active_ops_reject 0 vb_active_ops_update 2399244 vb_active_perc_mem_resident 100 vb_active_queue_age 0 vb_active_queue_drain 6646319 vb_active_queue_fill 3764588 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 1500466 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 12857056 vb_replica_itm_memory 3653305966 vb_replica_meta_data_memory 118536814 vb_replica_num 512 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 1500466 vb_replica_ops_create 1500467 vb_replica_ops_delete 1 vb_replica_ops_reject 0 vb_replica_ops_update 2221860 vb_replica_perc_mem_resident 100 vb_replica_queue_age 0 vb_replica_queue_drain 4968715 vb_replica_queue_fill 3684540 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-14T11:25:24.428,ns_1@10.2.1.63:ns_doctor<0.540.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,921120,28319}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,266143720}, {processes,211863840}, {processes_used,211837624}, {system,54279880}, {atom,1337385}, {atom_used,1331436}, {binary,13236904}, {code,13451143}, {ets,7452616}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7528738816}, {buffered_memory,244908032}, {free_memory,16796979200}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2282496,5000}}, {context_switches,{11238103,0}}, {garbage_collection,{2383271,4958163865,0}}, {io,{{input,261941209},{output,18054205127}}}, {reductions,{1435057680,3263102}}, {run_queue,0}, {runtime,{328080,600}}]}, {system_stats, [{cpu_utilization_rate,3.9900249376558605}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7287798818}, {couch_docs_data_size,7285575572}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7921143592}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2282}, {memory_data,{33657663488,16860684288,{<19415.11.0>,43123400}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16403300 kB\nBuffers: 239168 kB\nCached: 7352284 kB\nSwapCached: 0 kB\nActive: 8964336 kB\nInactive: 7193672 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16403300 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 524 kB\nWriteback: 0 kB\nAnonPages: 8566608 kB\nMapped: 15604 kB\nSlab: 251560 kB\nPageTables: 21812 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10294020 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920217,777322}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17795.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,921119,430030}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,265236128}, {processes,209866184}, {processes_used,209851808}, {system,55369944}, {atom,1349617}, {atom_used,1347167}, {binary,13062488}, {code,13553851}, {ets,8680120}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7594459136}, {buffered_memory,249450496}, {free_memory,16464486400}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2279942,4999}}, {context_switches,{14616948,0}}, {garbage_collection,{2789418,6933717060,0}}, {io,{{input,357976632},{output,12998972132}}}, {reductions,{2292051691,2626675}}, {run_queue,0}, {runtime,{374690,540}}]}, {system_stats, [{cpu_utilization_rate,4.25}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7288613936}, {couch_docs_data_size,7286342852}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,8098628776}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2279}, {memory_data,{33657663488,17192542208,{<0.11.0>,43123400}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16078600 kB\nBuffers: 243604 kB\nCached: 7416464 kB\nSwapCached: 0 kB\nActive: 9269088 kB\nInactive: 7192696 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16078600 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 332 kB\nWriteback: 0 kB\nAnonPages: 8801676 kB\nMapped: 15608 kB\nSlab: 272312 kB\nPageTables: 22176 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10521204 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-14T11:26:05.045,ns_1@10.2.1.63:<0.682.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,48,41}}) Stats for bucket "default": accepting_conns 1 auth_cmds 27 auth_errors 0 bucket_active_conns 1 bucket_conns 10 bytes 8098628776 bytes_read 18405575474 bytes_written 18381036142 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 82 cmd_set 2400893 conn_yields 64601 connection_structures 5000 curr_connections 13 curr_conns_on_port_11209 10 curr_conns_on_port_11210 1 curr_items 1499536 curr_items_tot 3000002 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 17 delete_misses 1 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-15 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data2/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 38798 ep_commit_time 0 ep_commit_time_total 325721 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 3 ep_data_age_highwat 150 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data2/default ep_degraded_mode 0 ep_diskqueue_drain 11615034 ep_diskqueue_fill 7449128 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 1 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 771 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 7621113 ep_io_read_bytes 0 ep_io_write_bytes 18046504810 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 333881 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 7339250270 ep_max_checkpoints 2 ep_max_data_size 12582912000 ep_max_item_size 20971520 ep_max_size 12582912000 ep_max_txn_size 10000 ep_max_vbuckets 1024 ep_mem_high_wat 9437184000 ep_mem_high_wat 9437184000 ep_mem_low_wat 7549747200 ep_mem_low_wat 7549747200 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 259162528 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1352918852 ep_storage_age 0 ep_storage_age_highwat 1293 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 6 ep_too_young 0 ep_total_cache_size 7303250246 ep_total_del_items 4 ep_total_enqueued 7449128 ep_total_new_items 3000005 ep_total_persisted 7621113 ep_uncommitted_items 0 ep_value_size 7091900551 ep_vb0 0 ep_vb_snapshot_total 2566 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_135_g40a391b ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 759 get_hits 69 get_misses 13 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 8098628776 pid 16783 pointer_size 64 rejected_conns 0 rusage_system 218.969711 rusage_user 925.831252 tap_checkpoint_end_received 963 tap_checkpoint_end_sent 1480 tap_checkpoint_start_received 2560 tap_checkpoint_start_sent 3171 tap_connect_received 1026 tap_delete_received 2 tap_delete_sent 1 tap_mutation_received 5220295 tap_mutation_sent 6745798 tap_opaque_received 5633 tap_opaque_sent 5122 tap_vbucket_set_received 1024 tap_vbucket_set_sent 1024 threads 4 time 1352921164 total_connections 6219 uptime 2322 vb_active_curr_items 1499536 vb_active_eject 0 vb_active_expired 1 vb_active_ht_memory 12857152 vb_active_itm_memory 3649944280 vb_active_meta_data_memory 118463344 vb_active_num 512 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1499538 vb_active_ops_create 1499538 vb_active_ops_delete 3 vb_active_ops_reject 0 vb_active_ops_update 2399244 vb_active_perc_mem_resident 100 vb_active_queue_age 0 vb_active_queue_drain 6646319 vb_active_queue_fill 3764588 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 1500466 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 12857056 vb_replica_itm_memory 3653305966 vb_replica_meta_data_memory 118536814 vb_replica_num 512 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 1500466 vb_replica_ops_create 1500467 vb_replica_ops_delete 1 vb_replica_ops_reject 0 vb_replica_ops_update 2221860 vb_replica_perc_mem_resident 100 vb_replica_queue_age 0 vb_replica_queue_drain 4968715 vb_replica_queue_fill 3684540 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-14T11:26:24.428,ns_1@10.2.1.63:ns_doctor<0.540.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,921180,28486}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,268358344}, {processes,214078248}, {processes_used,214034552}, {system,54280096}, {atom,1337385}, {atom_used,1331436}, {binary,13081560}, {code,13451143}, {ets,7610848}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7529668608}, {buffered_memory,245178368}, {free_memory,16792408064}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2342496,5000}}, {context_switches,{11434715,0}}, {garbage_collection,{2394203,5002355904,0}}, {io,{{input,268694208},{output,18060314529}}}, {reductions,{1460865676,3408355}}, {run_queue,0}, {runtime,{334630,670}}]}, {system_stats, [{cpu_utilization_rate,6.0},{swap_total,10684628992},{swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7287819298}, {couch_docs_data_size,7285575572}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7918880328}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2342}, {memory_data,{33657663488,16864493568,{<19415.11.0>,43123400}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16398836 kB\nBuffers: 239432 kB\nCached: 7353192 kB\nSwapCached: 0 kB\nActive: 8967180 kB\nInactive: 7194704 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16398836 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 376 kB\nWriteback: 0 kB\nAnonPages: 8569028 kB\nMapped: 15604 kB\nSlab: 251764 kB\nPageTables: 21824 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10306236 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920217,777322}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17795.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,921179,429688}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,264079600}, {processes,208699912}, {processes_used,208674496}, {system,55379688}, {atom,1349617}, {atom_used,1347167}, {binary,13021784}, {code,13553851}, {ets,8731552}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7595601920}, {buffered_memory,249548800}, {free_memory,16463978496}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2339942,5000}}, {context_switches,{14815520,0}}, {garbage_collection,{2799477,7002600785,0}}, {io,{{input,361435432},{output,13000922776}}}, {reductions,{2323119059,2633164}}, {run_queue,0}, {runtime,{381220,580}}]}, {system_stats, [{cpu_utilization_rate,4.25}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7288663088}, {couch_docs_data_size,7286342850}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,8098635688}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2339}, {memory_data,{33657663488,17193938944,{<0.11.0>,43123400}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16078104 kB\nBuffers: 243700 kB\nCached: 7417580 kB\nSwapCached: 0 kB\nActive: 9267936 kB\nInactive: 7193692 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16078104 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 280 kB\nWriteback: 0 kB\nAnonPages: 8800372 kB\nMapped: 15608 kB\nSlab: 272376 kB\nPageTables: 22172 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10523752 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-14T11:27:24.428,ns_1@10.2.1.63:ns_doctor<0.540.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,921240,28335}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,264900840}, {processes,210342488}, {processes_used,210161712}, {system,54558352}, {atom,1337385}, {atom_used,1331436}, {binary,13463440}, {code,13451143}, {ets,7525096}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7531401216}, {buffered_memory,245366784}, {free_memory,16793423872}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2402496,5001}}, {context_switches,{11634478,0}}, {garbage_collection,{2405732,5047097988,0}}, {io,{{input,272291887},{output,18062935852}}}, {reductions,{1486540032,3279638}}, {run_queue,0}, {runtime,{341250,600}}]}, {system_stats, [{cpu_utilization_rate,3.75}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7288441892}, {couch_docs_data_size,7285575571}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7849899464}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2402}, {memory_data,{33657663488,16864239616,{<19415.11.0>,43123400}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16399828 kB\nBuffers: 239616 kB\nCached: 7354884 kB\nSwapCached: 0 kB\nActive: 8966284 kB\nInactive: 7195352 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16399828 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 752 kB\nWriteback: 0 kB\nAnonPages: 8567000 kB\nMapped: 15604 kB\nSlab: 251732 kB\nPageTables: 21824 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10309228 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920217,777322}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17795.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,921239,429443}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,260530144}, {processes,204946688}, {processes_used,204782352}, {system,55583456}, {atom,1349617}, {atom_used,1347167}, {binary,13219648}, {code,13553851}, {ets,8756336}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7597281280}, {buffered_memory,249688064}, {free_memory,16460931072}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2399942,5000}}, {context_switches,{15019730,0}}, {garbage_collection,{2811424,7072995151,0}}, {io,{{input,368290198},{output,13006423605}}}, {reductions,{2354444894,2636185}}, {run_queue,0}, {runtime,{388020,630}}]}, {system_stats, [{cpu_utilization_rate,4.25}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7289281586}, {couch_docs_data_size,7286342846}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,8030137976}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2399}, {memory_data,{33657663488,17195081728,{<0.11.0>,43123400}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16075128 kB\nBuffers: 243836 kB\nCached: 7419220 kB\nSwapCached: 0 kB\nActive: 9270304 kB\nInactive: 7194428 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16075128 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 736 kB\nWriteback: 0 kB\nAnonPages: 8801596 kB\nMapped: 15608 kB\nSlab: 272584 kB\nPageTables: 22180 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10521312 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-14T11:27:45.045,ns_1@10.2.1.63:<0.682.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,48,41}}) Stats for bucket "default": accepting_conns 1 auth_cmds 27 auth_errors 0 bucket_active_conns 1 bucket_conns 10 bytes 8000645704 bytes_read 18405606990 bytes_written 18388957295 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 82 cmd_set 2400893 conn_yields 64601 connection_structures 5000 curr_connections 13 curr_conns_on_port_11209 10 curr_conns_on_port_11210 1 curr_items 1499536 curr_items_tot 3000002 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 17 delete_misses 1 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-15 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data2/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 38798 ep_commit_time 0 ep_commit_time_total 325721 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 3 ep_data_age_highwat 150 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data2/default ep_degraded_mode 0 ep_diskqueue_drain 11615034 ep_diskqueue_fill 7449128 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 1 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 771 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 7621113 ep_io_read_bytes 0 ep_io_write_bytes 18046504810 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 965695 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 7339250270 ep_max_checkpoints 2 ep_max_data_size 12582912000 ep_max_item_size 20971520 ep_max_size 12582912000 ep_max_txn_size 10000 ep_max_vbuckets 1024 ep_mem_high_wat 9437184000 ep_mem_high_wat 9437184000 ep_mem_low_wat 7549747200 ep_mem_low_wat 7549747200 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 208538728 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1352918852 ep_storage_age 0 ep_storage_age_highwat 1293 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 6 ep_too_young 0 ep_total_cache_size 7303250246 ep_total_del_items 4 ep_total_enqueued 7449128 ep_total_new_items 3000005 ep_total_persisted 7621113 ep_uncommitted_items 0 ep_value_size 7091900551 ep_vb0 0 ep_vb_snapshot_total 2606 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_135_g40a391b ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 759 get_hits 69 get_misses 13 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 8000645704 pid 16783 pointer_size 64 rejected_conns 0 rusage_system 219.220673 rusage_user 927.869942 tap_checkpoint_end_received 1078 tap_checkpoint_end_sent 1587 tap_checkpoint_start_received 2675 tap_checkpoint_start_sent 3278 tap_connect_received 1026 tap_delete_received 2 tap_delete_sent 1 tap_mutation_received 5220295 tap_mutation_sent 6745798 tap_opaque_received 5633 tap_opaque_sent 5122 tap_vbucket_set_received 1024 tap_vbucket_set_sent 1024 threads 4 time 1352921264 total_connections 6219 uptime 2422 vb_active_curr_items 1499536 vb_active_eject 0 vb_active_expired 1 vb_active_ht_memory 12857152 vb_active_itm_memory 3649944280 vb_active_meta_data_memory 118463344 vb_active_num 512 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1499538 vb_active_ops_create 1499538 vb_active_ops_delete 3 vb_active_ops_reject 0 vb_active_ops_update 2399244 vb_active_perc_mem_resident 100 vb_active_queue_age 0 vb_active_queue_drain 6646319 vb_active_queue_fill 3764588 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 1500466 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 12857056 vb_replica_itm_memory 3653305966 vb_replica_meta_data_memory 118536814 vb_replica_num 512 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 1500466 vb_replica_ops_create 1500467 vb_replica_ops_delete 1 vb_replica_ops_reject 0 vb_replica_ops_update 2221860 vb_replica_perc_mem_resident 100 vb_replica_queue_age 0 vb_replica_queue_drain 4968715 vb_replica_queue_fill 3684540 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-14T11:28:24.428,ns_1@10.2.1.63:ns_doctor<0.540.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,921300,28661}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,263211072}, {processes,209058464}, {processes_used,208772808}, {system,54152608}, {atom,1337385}, {atom_used,1331436}, {binary,13013224}, {code,13451143}, {ets,7583616}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7532974080}, {buffered_memory,245510144}, {free_memory,16795455488}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2462496,5000}}, {context_switches,{11833032,0}}, {garbage_collection,{2417465,5091061201,0}}, {io,{{input,275593248},{output,18065578758}}}, {reductions,{1512181176,3279210}}, {run_queue,0}, {runtime,{347960,590}}]}, {system_stats, [{cpu_utilization_rate,3.5},{swap_total,10684628992},{swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7288921129}, {couch_docs_data_size,7285575570}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7796680088}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2462}, {memory_data,{33657663488,16861954048,{<19415.11.0>,43123400}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16401812 kB\nBuffers: 239756 kB\nCached: 7356420 kB\nSwapCached: 0 kB\nActive: 8962984 kB\nInactive: 7196184 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16401812 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 784 kB\nWriteback: 0 kB\nAnonPages: 8563020 kB\nMapped: 15604 kB\nSlab: 251712 kB\nPageTables: 21824 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10299268 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920217,777322}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17795.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,921299,429768}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,265439336}, {processes,209787288}, {processes_used,209517152}, {system,55652048}, {atom,1349617}, {atom_used,1347167}, {binary,13356296}, {code,13553851}, {ets,8702336}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7598874624}, {buffered_memory,249757696}, {free_memory,16459280384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2459942,5000}}, {context_switches,{15221559,0}}, {garbage_collection,{2822118,7142640267,0}}, {io,{{input,371662548},{output,13008380745}}}, {reductions,{2385603212,2629630}}, {run_queue,0}, {runtime,{394740,560}}]}, {system_stats, [{cpu_utilization_rate,4.260651629072682}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7289752631}, {couch_docs_data_size,7286342844}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,7975981960}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2459}, {memory_data,{33657663488,17197875200,{<0.11.0>,43123400}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16073516 kB\nBuffers: 243904 kB\nCached: 7420776 kB\nSwapCached: 0 kB\nActive: 9271692 kB\nInactive: 7195160 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16073516 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 412 kB\nWriteback: 0 kB\nAnonPages: 8802168 kB\nMapped: 15608 kB\nSlab: 272516 kB\nPageTables: 22184 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10519972 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-14T11:29:24.428,ns_1@10.2.1.63:ns_doctor<0.540.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,921360,28373}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,262317136}, {processes,208124576}, {processes_used,207761640}, {system,54192560}, {atom,1337385}, {atom_used,1331436}, {binary,12972176}, {code,13451143}, {ets,7675528}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7528935424}, {buffered_memory,245739520}, {free_memory,16797089792}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2522495,4999}}, {context_switches,{12031033,0}}, {garbage_collection,{2429201,5135865524,0}}, {io,{{input,282354306},{output,18073206634}}}, {reductions,{1537882798,3374894}}, {run_queue,0}, {runtime,{354460,670}}]}, {system_stats, [{cpu_utilization_rate,6.483790523690773}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7289281580}, {couch_docs_data_size,7285575567}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7756650456}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2522}, {memory_data,{33657663488,16862715904,{<19415.11.0>,43123400}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16403408 kB\nBuffers: 239980 kB\nCached: 7352476 kB\nSwapCached: 0 kB\nActive: 8964152 kB\nInactive: 7192356 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16403408 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 352 kB\nWriteback: 0 kB\nAnonPages: 8564260 kB\nMapped: 15604 kB\nSlab: 251996 kB\nPageTables: 21824 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10299496 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920217,777322}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17795.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,921359,430509}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,261189528}, {processes,205553992}, {processes_used,205202896}, {system,55635536}, {atom,1349617}, {atom_used,1347167}, {binary,13265088}, {code,13553851}, {ets,8787800}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7600336896}, {buffered_memory,249847808}, {free_memory,16457756672}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2519943,5001}}, {context_switches,{15422487,0}}, {garbage_collection,{2833237,7212107066,0}}, {io,{{input,375231511},{output,13010340050}}}, {reductions,{2416684479,2627050}}, {run_queue,0}, {runtime,{401310,550}}]}, {system_stats, [{cpu_utilization_rate,4.25}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7290113082}, {couch_docs_data_size,7286342842}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,7935473496}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2519}, {memory_data,{33657663488,17200541696,{<0.11.0>,43123400}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16072028 kB\nBuffers: 243992 kB\nCached: 7422204 kB\nSwapCached: 0 kB\nActive: 9272152 kB\nInactive: 7195888 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16072028 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 92 kB\nWriteback: 32 kB\nAnonPages: 8801636 kB\nMapped: 15608 kB\nSlab: 272460 kB\nPageTables: 22176 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10520776 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-14T11:29:25.045,ns_1@10.2.1.63:<0.682.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,48,41}}) Stats for bucket "default": accepting_conns 1 auth_cmds 27 auth_errors 0 bucket_active_conns 1 bucket_conns 10 bytes 7928643672 bytes_read 18405632232 bytes_written 18396872700 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 82 cmd_set 2400893 conn_yields 64601 connection_structures 5000 curr_connections 13 curr_conns_on_port_11209 10 curr_conns_on_port_11210 1 curr_items 1499536 curr_items_tot 3000002 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 17 delete_misses 1 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-15 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data2/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 38798 ep_commit_time 0 ep_commit_time_total 325721 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 3 ep_data_age_highwat 150 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data2/default ep_degraded_mode 0 ep_diskqueue_drain 11615034 ep_diskqueue_fill 7449128 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 1 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 771 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 7621113 ep_io_read_bytes 0 ep_io_write_bytes 18046504810 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 1430108 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 7339250270 ep_max_checkpoints 2 ep_max_data_size 12582912000 ep_max_item_size 20971520 ep_max_size 12582912000 ep_max_txn_size 10000 ep_max_vbuckets 1024 ep_mem_high_wat 9437184000 ep_mem_high_wat 9437184000 ep_mem_low_wat 7549747200 ep_mem_low_wat 7549747200 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 171245012 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1352918852 ep_storage_age 0 ep_storage_age_highwat 1293 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 6 ep_too_young 0 ep_total_cache_size 7303250246 ep_total_del_items 4 ep_total_enqueued 7449128 ep_total_new_items 3000005 ep_total_persisted 7621113 ep_uncommitted_items 0 ep_value_size 7091900551 ep_vb0 0 ep_vb_snapshot_total 2648 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_135_g40a391b ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 759 get_hits 69 get_misses 13 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 7928643672 pid 16783 pointer_size 64 rejected_conns 0 rusage_system 219.404645 rusage_user 929.685666 tap_checkpoint_end_received 1158 tap_checkpoint_end_sent 1661 tap_checkpoint_start_received 2755 tap_checkpoint_start_sent 3352 tap_connect_received 1026 tap_delete_received 2 tap_delete_sent 1 tap_mutation_received 5220295 tap_mutation_sent 6745798 tap_opaque_received 5633 tap_opaque_sent 5122 tap_vbucket_set_received 1024 tap_vbucket_set_sent 1024 threads 4 time 1352921364 total_connections 6219 uptime 2522 vb_active_curr_items 1499536 vb_active_eject 0 vb_active_expired 1 vb_active_ht_memory 12857152 vb_active_itm_memory 3649944280 vb_active_meta_data_memory 118463344 vb_active_num 512 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1499538 vb_active_ops_create 1499538 vb_active_ops_delete 3 vb_active_ops_reject 0 vb_active_ops_update 2399244 vb_active_perc_mem_resident 100 vb_active_queue_age 0 vb_active_queue_drain 6646319 vb_active_queue_fill 3764588 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 1500466 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 12857056 vb_replica_itm_memory 3653305966 vb_replica_meta_data_memory 118536814 vb_replica_num 512 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 1500466 vb_replica_ops_create 1500467 vb_replica_ops_delete 1 vb_replica_ops_reject 0 vb_replica_ops_update 2221860 vb_replica_perc_mem_resident 100 vb_replica_queue_age 0 vb_replica_queue_drain 4968715 vb_replica_queue_fill 3684540 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-14T11:30:24.428,ns_1@10.2.1.63:ns_doctor<0.540.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,921420,28663}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,260132176}, {processes,206056240}, {processes_used,205620624}, {system,54075936}, {atom,1337385}, {atom_used,1331436}, {binary,12762128}, {code,13451143}, {ets,7778976}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7530401792}, {buffered_memory,245886976}, {free_memory,16797995008}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2582496,5000}}, {context_switches,{12228497,0}}, {garbage_collection,{2441519,5179924374,0}}, {io,{{input,285745948},{output,18075812416}}}, {reductions,{1563480578,3256705}}, {run_queue,0}, {runtime,{360960,590}}]}, {system_stats, [{cpu_utilization_rate,3.491271820448878}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7289601068}, {couch_docs_data_size,7285575564}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7721214312}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2582}, {memory_data,{33657663488,16860176384,{<19415.11.0>,43123400}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16404292 kB\nBuffers: 240124 kB\nCached: 7353908 kB\nSwapCached: 0 kB\nActive: 8963508 kB\nInactive: 7193196 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16404292 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 692 kB\nWriteback: 0 kB\nAnonPages: 8562540 kB\nMapped: 15604 kB\nSlab: 251748 kB\nPageTables: 21812 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10298584 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920217,777322}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17795.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,921419,429466}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,262702304}, {processes,206868104}, {processes_used,206447088}, {system,55834200}, {atom,1349617}, {atom_used,1347167}, {binary,13372288}, {code,13553851}, {ets,8888696}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7596077056}, {buffered_memory,250044416}, {free_memory,16460296192}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2579942,5000}}, {context_switches,{15623690,0}}, {garbage_collection,{2844047,7282819257,0}}, {io,{{input,382043631},{output,13018570830}}}, {reductions,{2448184226,2635177}}, {run_queue,0}, {runtime,{407940,510}}]}, {system_stats, [{cpu_utilization_rate,4.25}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7290424378}, {couch_docs_data_size,7286342841}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,7900071336}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2579}, {memory_data,{33657663488,17201938432,{<0.11.0>,43123400}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16074508 kB\nBuffers: 244184 kB\nCached: 7418044 kB\nSwapCached: 0 kB\nActive: 9273000 kB\nInactive: 7192008 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16074508 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 816 kB\nWriteback: 0 kB\nAnonPages: 8802768 kB\nMapped: 15608 kB\nSlab: 272476 kB\nPageTables: 22184 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10521456 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-14T11:31:05.046,ns_1@10.2.1.63:<0.682.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,48,41}}) Stats for bucket "default": accepting_conns 1 auth_cmds 27 auth_errors 0 bucket_active_conns 1 bucket_conns 10 bytes 7875452088 bytes_read 18405654064 bytes_written 18404784792 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 82 cmd_set 2400893 conn_yields 64601 connection_structures 5000 curr_connections 13 curr_conns_on_port_11209 10 curr_conns_on_port_11210 1 curr_items 1499536 curr_items_tot 3000002 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 17 delete_misses 1 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-15 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data2/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 38798 ep_commit_time 0 ep_commit_time_total 325721 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 3 ep_data_age_highwat 150 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data2/default ep_degraded_mode 0 ep_diskqueue_drain 11615034 ep_diskqueue_fill 7449128 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 1 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 771 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 7621113 ep_io_read_bytes 0 ep_io_write_bytes 18046504810 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 1773143 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 7339250270 ep_max_checkpoints 2 ep_max_data_size 12582912000 ep_max_item_size 20971520 ep_max_size 12582912000 ep_max_txn_size 10000 ep_max_vbuckets 1024 ep_mem_high_wat 9437184000 ep_mem_high_wat 9437184000 ep_mem_low_wat 7549747200 ep_mem_low_wat 7549747200 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 143588120 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1352918852 ep_storage_age 0 ep_storage_age_highwat 1293 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 6 ep_too_young 0 ep_total_cache_size 7303250246 ep_total_del_items 4 ep_total_enqueued 7449128 ep_total_new_items 3000005 ep_total_persisted 7621113 ep_uncommitted_items 0 ep_value_size 7091900551 ep_vb0 0 ep_vb_snapshot_total 2689 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_135_g40a391b ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 759 get_hits 69 get_misses 13 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 7875452088 pid 16783 pointer_size 64 rejected_conns 0 rusage_system 219.537625 rusage_user 931.333415 tap_checkpoint_end_received 1219 tap_checkpoint_end_sent 1717 tap_checkpoint_start_received 2816 tap_checkpoint_start_sent 3408 tap_connect_received 1026 tap_delete_received 2 tap_delete_sent 1 tap_mutation_received 5220295 tap_mutation_sent 6745798 tap_opaque_received 5633 tap_opaque_sent 5122 tap_vbucket_set_received 1024 tap_vbucket_set_sent 1024 threads 4 time 1352921464 total_connections 6219 uptime 2622 vb_active_curr_items 1499536 vb_active_eject 0 vb_active_expired 1 vb_active_ht_memory 12857152 vb_active_itm_memory 3649944280 vb_active_meta_data_memory 118463344 vb_active_num 512 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1499538 vb_active_ops_create 1499538 vb_active_ops_delete 3 vb_active_ops_reject 0 vb_active_ops_update 2399244 vb_active_perc_mem_resident 100 vb_active_queue_age 0 vb_active_queue_drain 6646319 vb_active_queue_fill 3764588 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 1500466 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 12857056 vb_replica_itm_memory 3653305966 vb_replica_meta_data_memory 118536814 vb_replica_num 512 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 1500466 vb_replica_ops_create 1500467 vb_replica_ops_delete 1 vb_replica_ops_reject 0 vb_replica_ops_update 2221860 vb_replica_perc_mem_resident 100 vb_replica_queue_age 0 vb_replica_queue_drain 4968715 vb_replica_queue_fill 3684540 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-14T11:31:24.428,ns_1@10.2.1.63:ns_doctor<0.540.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,921480,28653}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,265178528}, {processes,211185368}, {processes_used,210693632}, {system,53993160}, {atom,1337385}, {atom_used,1331436}, {binary,12684672}, {code,13451143}, {ets,7780920}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7531749376}, {buffered_memory,246038528}, {free_memory,16795963392}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2642496,5001}}, {context_switches,{12425430,0}}, {garbage_collection,{2453875,5224042873,0}}, {io,{{input,289264390},{output,18078793085}}}, {reductions,{1589102315,3271734}}, {run_queue,0}, {runtime,{367570,610}}]}, {system_stats, [{cpu_utilization_rate,4.0},{swap_total,10684628992},{swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7289863212}, {couch_docs_data_size,7285575562}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7692085384}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2642}, {memory_data,{33657663488,16862081024,{<19415.11.0>,43123400}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16402308 kB\nBuffers: 240272 kB\nCached: 7355224 kB\nSwapCached: 0 kB\nActive: 8963940 kB\nInactive: 7194052 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16402308 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 652 kB\nWriteback: 0 kB\nAnonPages: 8562524 kB\nMapped: 15604 kB\nSlab: 251732 kB\nPageTables: 21812 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10288040 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920217,777322}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17795.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,921479,429417}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,266202864}, {processes,210227408}, {processes_used,209747512}, {system,55975456}, {atom,1349617}, {atom_used,1347167}, {binary,13411888}, {code,13553851}, {ets,8998472}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7597445120}, {buffered_memory,250134528}, {free_memory,16459534336}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2639942,5000}}, {context_switches,{15823847,0}}, {garbage_collection,{2854104,7352125276,0}}, {io,{{input,385605127},{output,13020731906}}}, {reductions,{2479266426,2632813}}, {run_queue,0}, {runtime,{414380,560}}]}, {system_stats, [{cpu_utilization_rate,4.488778054862843}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7290686522}, {couch_docs_data_size,7286342841}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,7870933192}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2639}, {memory_data,{33657663488,17198637056,{<0.11.0>,43123400}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16073764 kB\nBuffers: 244272 kB\nCached: 7419380 kB\nSwapCached: 0 kB\nActive: 9273748 kB\nInactive: 7192804 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16073764 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 416 kB\nWriteback: 0 kB\nAnonPages: 8802784 kB\nMapped: 15608 kB\nSlab: 272500 kB\nPageTables: 22180 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10520616 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-14T11:32:24.428,ns_1@10.2.1.63:ns_doctor<0.540.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,921540,28704}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,262619832}, {processes,208550120}, {processes_used,208000424}, {system,54069712}, {atom,1337385}, {atom_used,1331436}, {binary,12647240}, {code,13451143}, {ets,7902976}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7532892160}, {buffered_memory,246296576}, {free_memory,16794693632}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2702496,5000}}, {context_switches,{12623796,0}}, {garbage_collection,{2465215,5268776976,0}}, {io,{{input,296037101},{output,18084985350}}}, {reductions,{1615116970,3416565}}, {run_queue,0}, {runtime,{374400,710}}]}, {system_stats, [{cpu_utilization_rate,6.25}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7290108972}, {couch_docs_data_size,7285575562}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7664763496}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2702}, {memory_data,{33657663488,16861954048,{<19415.11.0>,43123400}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16401068 kB\nBuffers: 240524 kB\nCached: 7356340 kB\nSwapCached: 0 kB\nActive: 8964740 kB\nInactive: 7194920 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16401068 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 376 kB\nWriteback: 0 kB\nAnonPages: 8562632 kB\nMapped: 15604 kB\nSlab: 251892 kB\nPageTables: 21832 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10301844 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920217,777322}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17795.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,921539,429543}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,262593208}, {processes,206922848}, {processes_used,206386832}, {system,55670360}, {atom,1349617}, {atom_used,1347167}, {binary,13104240}, {code,13553851}, {ets,9008032}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7598809088}, {buffered_memory,250224640}, {free_memory,16459407360}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2699942,5000}}, {context_switches,{16023962,0}}, {garbage_collection,{2864950,7422021024,0}}, {io,{{input,389168768},{output,13022685279}}}, {reductions,{2510353479,2631468}}, {run_queue,0}, {runtime,{420880,540}}]}, {system_stats, [{cpu_utilization_rate,4.260651629072682}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7290936378}, {couch_docs_data_size,7286342841}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,7843161704}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2699}, {memory_data,{33657663488,17200287744,{<0.11.0>,43123400}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16073640 kB\nBuffers: 244360 kB\nCached: 7420712 kB\nSwapCached: 0 kB\nActive: 9273480 kB\nInactive: 7193624 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16073640 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 160 kB\nWriteback: 0 kB\nAnonPages: 8801708 kB\nMapped: 15608 kB\nSlab: 272492 kB\nPageTables: 22180 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10522484 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-14T11:32:45.045,ns_1@10.2.1.63:<0.682.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,48,41}}) Stats for bucket "default": accepting_conns 1 auth_cmds 27 auth_errors 0 bucket_active_conns 1 bucket_conns 10 bytes 7829456840 bytes_read 18405674230 bytes_written 18412695820 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 82 cmd_set 2400893 conn_yields 64601 connection_structures 5000 curr_connections 13 curr_conns_on_port_11209 10 curr_conns_on_port_11210 1 curr_items 1499536 curr_items_tot 3000002 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 17 delete_misses 1 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-15 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data2/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 38798 ep_commit_time 0 ep_commit_time_total 325721 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 3 ep_data_age_highwat 150 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data2/default ep_degraded_mode 0 ep_diskqueue_drain 11615034 ep_diskqueue_fill 7449128 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 1 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 771 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 7621113 ep_io_read_bytes 0 ep_io_write_bytes 18046504810 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 2069811 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 7339250270 ep_max_checkpoints 2 ep_max_data_size 12582912000 ep_max_item_size 20971520 ep_max_size 12582912000 ep_max_txn_size 10000 ep_max_vbuckets 1024 ep_mem_high_wat 9437184000 ep_mem_high_wat 9437184000 ep_mem_low_wat 7549747200 ep_mem_low_wat 7549747200 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 119639166 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1352918852 ep_storage_age 0 ep_storage_age_highwat 1293 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 6 ep_too_young 0 ep_total_cache_size 7303250246 ep_total_del_items 4 ep_total_enqueued 7449128 ep_total_new_items 3000005 ep_total_persisted 7621113 ep_uncommitted_items 0 ep_value_size 7091900551 ep_vb0 0 ep_vb_snapshot_total 2729 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_135_g40a391b ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 759 get_hits 69 get_misses 13 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 7829456840 pid 16783 pointer_size 64 rejected_conns 0 rusage_system 219.676604 rusage_user 932.877181 tap_checkpoint_end_received 1269 tap_checkpoint_end_sent 1769 tap_checkpoint_start_received 2866 tap_checkpoint_start_sent 3460 tap_connect_received 1026 tap_delete_received 2 tap_delete_sent 1 tap_mutation_received 5220295 tap_mutation_sent 6745798 tap_opaque_received 5633 tap_opaque_sent 5122 tap_vbucket_set_received 1024 tap_vbucket_set_sent 1024 threads 4 time 1352921564 total_connections 6219 uptime 2722 vb_active_curr_items 1499536 vb_active_eject 0 vb_active_expired 1 vb_active_ht_memory 12857152 vb_active_itm_memory 3649944280 vb_active_meta_data_memory 118463344 vb_active_num 512 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1499538 vb_active_ops_create 1499538 vb_active_ops_delete 3 vb_active_ops_reject 0 vb_active_ops_update 2399244 vb_active_perc_mem_resident 100 vb_active_queue_age 0 vb_active_queue_drain 6646319 vb_active_queue_fill 3764588 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 1500466 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 12857056 vb_replica_itm_memory 3653305966 vb_replica_meta_data_memory 118536814 vb_replica_num 512 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 1500466 vb_replica_ops_create 1500467 vb_replica_ops_delete 1 vb_replica_ops_reject 0 vb_replica_ops_update 2221860 vb_replica_perc_mem_resident 100 vb_replica_queue_age 0 vb_replica_queue_drain 4968715 vb_replica_queue_fill 3684540 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-14T11:33:24.428,ns_1@10.2.1.63:ns_doctor<0.540.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,921600,28489}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,262521064}, {processes,208300584}, {processes_used,207698448}, {system,54220480}, {atom,1337385}, {atom_used,1331436}, {binary,12798912}, {code,13451143}, {ets,7908856}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7534239744}, {buffered_memory,246464512}, {free_memory,16792408064}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2762495,4999}}, {context_switches,{12820511,0}}, {garbage_collection,{2477290,5312749823,0}}, {io,{{input,299610674},{output,18087783477}}}, {reductions,{1640727758,3249802}}, {run_queue,0}, {runtime,{380840,550}}]}, {system_stats, [{cpu_utilization_rate,4.0},{swap_total,10684628992},{swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7290350636}, {couch_docs_data_size,7285575559}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7637922888}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2762}, {memory_data,{33657663488,16864620544,{<19415.11.0>,43123400}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16398836 kB\nBuffers: 240688 kB\nCached: 7357656 kB\nSwapCached: 0 kB\nActive: 8966476 kB\nInactive: 7195784 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16398836 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 764 kB\nWriteback: 0 kB\nAnonPages: 8563728 kB\nMapped: 15604 kB\nSlab: 251740 kB\nPageTables: 21840 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10306116 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920217,777322}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17795.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,921599,430928}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,267753248}, {processes,211759416}, {processes_used,211170960}, {system,55993832}, {atom,1349617}, {atom_used,1347167}, {binary,13316000}, {code,13553851}, {ets,9127560}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7600123904}, {buffered_memory,250417152}, {free_memory,16456994816}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2759942,5000}}, {context_switches,{16224965,0}}, {garbage_collection,{2875493,7491761927,0}}, {io,{{input,395996308},{output,13028157707}}}, {reductions,{2541564039,2636865}}, {run_queue,0}, {runtime,{427260,580}}]}, {system_stats, [{cpu_utilization_rate,4.010025062656641}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7291169850}, {couch_docs_data_size,7286342839}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,7816765368}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2759}, {memory_data,{33657663488,17200414720,{<0.11.0>,43123400}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16071284 kB\nBuffers: 244548 kB\nCached: 7421996 kB\nSwapCached: 0 kB\nActive: 9274640 kB\nInactive: 7194648 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16071284 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 684 kB\nWriteback: 0 kB\nAnonPages: 8802712 kB\nMapped: 15608 kB\nSlab: 272484 kB\nPageTables: 22180 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10522424 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-14T11:34:24.428,ns_1@10.2.1.63:ns_doctor<0.540.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,921660,28865}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,265529312}, {processes,211017848}, {processes_used,210369712}, {system,54511464}, {atom,1337385}, {atom_used,1331436}, {binary,12972504}, {code,13451143}, {ets,8032504}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7535558656}, {buffered_memory,246616064}, {free_memory,16786186240}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2822496,5000}}, {context_switches,{13017055,0}}, {garbage_collection,{2487949,5356252749,0}}, {io,{{input,303090194},{output,18090510249}}}, {reductions,{1666299748,3270746}}, {run_queue,0}, {runtime,{387320,600}}]}, {system_stats, [{cpu_utilization_rate,3.25}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7290559531}, {couch_docs_data_size,7285575557}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7614739848}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2822}, {memory_data,{33657663488,16871604224,{<19415.11.0>,43123400}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16392760 kB\nBuffers: 240836 kB\nCached: 7358944 kB\nSwapCached: 0 kB\nActive: 8970904 kB\nInactive: 7196696 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16392760 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 476 kB\nWriteback: 0 kB\nAnonPages: 8568004 kB\nMapped: 15604 kB\nSlab: 251792 kB\nPageTables: 21844 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10296236 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920217,777322}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17795.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,921659,429252}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,264765936}, {processes,208698464}, {processes_used,208062168}, {system,56067472}, {atom,1349617}, {atom_used,1347167}, {binary,13385456}, {code,13553851}, {ets,9137720}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7601459200}, {buffered_memory,250511360}, {free_memory,16459661312}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2819942,5000}}, {context_switches,{16424766,0}}, {garbage_collection,{2884799,7561027453,0}}, {io,{{input,399391854},{output,13030285749}}}, {reductions,{2572567759,2622429}}, {run_queue,0}, {runtime,{433560,530}}]}, {system_stats, [{cpu_utilization_rate,4.25}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7291382842}, {couch_docs_data_size,7286342835}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,7793135464}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2819}, {memory_data,{33657663488,17201430528,{<0.11.0>,43123400}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16073888 kB\nBuffers: 244640 kB\nCached: 7423300 kB\nSwapCached: 0 kB\nActive: 9270556 kB\nInactive: 7195496 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16073888 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 400 kB\nWriteback: 0 kB\nAnonPages: 8798032 kB\nMapped: 15608 kB\nSlab: 272476 kB\nPageTables: 22180 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10516708 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-14T11:34:25.045,ns_1@10.2.1.63:<0.682.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,48,41}}) Stats for bucket "default": accepting_conns 1 auth_cmds 27 auth_errors 0 bucket_active_conns 1 bucket_conns 10 bytes 7789481208 bytes_read 18405693160 bytes_written 18420605348 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 82 cmd_set 2400893 conn_yields 64601 connection_structures 5000 curr_connections 13 curr_conns_on_port_11209 10 curr_conns_on_port_11210 1 curr_items 1499536 curr_items_tot 3000002 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 17 delete_misses 1 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-15 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data2/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 38798 ep_commit_time 0 ep_commit_time_total 325721 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 3 ep_data_age_highwat 150 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data2/default ep_degraded_mode 0 ep_diskqueue_drain 11615034 ep_diskqueue_fill 7449128 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 1 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 771 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 7621113 ep_io_read_bytes 0 ep_io_write_bytes 18046504810 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 2327596 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 7339250270 ep_max_checkpoints 2 ep_max_data_size 12582912000 ep_max_item_size 20971520 ep_max_size 12582912000 ep_max_txn_size 10000 ep_max_vbuckets 1024 ep_mem_high_wat 9437184000 ep_mem_high_wat 9437184000 ep_mem_low_wat 7549747200 ep_mem_low_wat 7549747200 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 98795148 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1352918852 ep_storage_age 0 ep_storage_age_highwat 1293 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 6 ep_too_young 0 ep_total_cache_size 7303250246 ep_total_del_items 4 ep_total_enqueued 7449128 ep_total_new_items 3000005 ep_total_persisted 7621113 ep_uncommitted_items 0 ep_value_size 7091900551 ep_vb0 0 ep_vb_snapshot_total 2769 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_135_g40a391b ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 759 get_hits 69 get_misses 13 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 7789481208 pid 16783 pointer_size 64 rejected_conns 0 rusage_system 219.811583 rusage_user 934.367954 tap_checkpoint_end_received 1313 tap_checkpoint_end_sent 1812 tap_checkpoint_start_received 2910 tap_checkpoint_start_sent 3503 tap_connect_received 1026 tap_delete_received 2 tap_delete_sent 1 tap_mutation_received 5220295 tap_mutation_sent 6745798 tap_opaque_received 5633 tap_opaque_sent 5122 tap_vbucket_set_received 1024 tap_vbucket_set_sent 1024 threads 4 time 1352921664 total_connections 6219 uptime 2822 vb_active_curr_items 1499536 vb_active_eject 0 vb_active_expired 1 vb_active_ht_memory 12857152 vb_active_itm_memory 3649944280 vb_active_meta_data_memory 118463344 vb_active_num 512 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1499538 vb_active_ops_create 1499538 vb_active_ops_delete 3 vb_active_ops_reject 0 vb_active_ops_update 2399244 vb_active_perc_mem_resident 100 vb_active_queue_age 0 vb_active_queue_drain 6646319 vb_active_queue_fill 3764588 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 1500466 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 12857056 vb_replica_itm_memory 3653305966 vb_replica_meta_data_memory 118536814 vb_replica_num 512 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 1500466 vb_replica_ops_create 1500467 vb_replica_ops_delete 1 vb_replica_ops_reject 0 vb_replica_ops_update 2221860 vb_replica_perc_mem_resident 100 vb_replica_queue_age 0 vb_replica_queue_drain 4968715 vb_replica_queue_fill 3684540 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-14T11:35:24.428,ns_1@10.2.1.63:ns_doctor<0.540.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,921720,28843}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,262005304}, {processes,207463528}, {processes_used,206768472}, {system,54541776}, {atom,1337385}, {atom_used,1331436}, {binary,12885672}, {code,13451143}, {ets,8157264}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7531376640}, {buffered_memory,246853632}, {free_memory,16786669568}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2882496,5001}}, {context_switches,{13214022,0}}, {garbage_collection,{2500306,5401990307,0}}, {io,{{input,309857040},{output,18098900541}}}, {reductions,{1691993913,3367755}}, {run_queue,0}, {runtime,{394120,700}}]}, {system_stats, [{cpu_utilization_rate,7.268170426065163}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7290764330}, {couch_docs_data_size,7285575554}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7592021816}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2882}, {memory_data,{33657663488,16873000960,{<19415.11.0>,43123400}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16393232 kB\nBuffers: 241068 kB\nCached: 7354860 kB\nSwapCached: 0 kB\nActive: 8973312 kB\nInactive: 7193000 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16393232 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 412 kB\nWriteback: 0 kB\nAnonPages: 8570444 kB\nMapped: 15604 kB\nSlab: 252200 kB\nPageTables: 21844 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10307000 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920217,777322}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17795.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,921719,429259}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,262490584}, {processes,206279216}, {processes_used,205598760}, {system,56211368}, {atom,1349617}, {atom_used,1347167}, {binary,13409424}, {code,13553851}, {ets,9263856}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7602769920}, {buffered_memory,250609664}, {free_memory,16458518528}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2879942,5000}}, {context_switches,{16624465,0}}, {garbage_collection,{2896109,7630734180,0}}, {io,{{input,402942848},{output,13032245161}}}, {reductions,{2603688585,2642757}}, {run_queue,0}, {runtime,{440090,530}}]}, {system_stats, [{cpu_utilization_rate,4.25}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7291579449}, {couch_docs_data_size,7286342835}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,7770866600}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2879}, {memory_data,{33657663488,17198764032,{<0.11.0>,43123400}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16072772 kB\nBuffers: 244736 kB\nCached: 7424580 kB\nSwapCached: 0 kB\nActive: 9271556 kB\nInactive: 7196320 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16072772 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 156 kB\nWriteback: 0 kB\nAnonPages: 8798756 kB\nMapped: 15608 kB\nSlab: 272452 kB\nPageTables: 22180 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10515804 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-14T11:36:05.045,ns_1@10.2.1.63:<0.682.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,48,41}}) Stats for bucket "default": accepting_conns 1 auth_cmds 27 auth_errors 0 bucket_active_conns 1 bucket_conns 10 bytes 7753672504 bytes_read 18405711018 bytes_written 18428514388 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 82 cmd_set 2400893 conn_yields 64601 connection_structures 5000 curr_connections 13 curr_conns_on_port_11209 10 curr_conns_on_port_11210 1 curr_items 1499536 curr_items_tot 3000002 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 17 delete_misses 1 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-15 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data2/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 38798 ep_commit_time 0 ep_commit_time_total 325721 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 3 ep_data_age_highwat 150 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data2/default ep_degraded_mode 0 ep_diskqueue_drain 11615034 ep_diskqueue_fill 7449128 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 1 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 771 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 7621113 ep_io_read_bytes 0 ep_io_write_bytes 18046504810 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 2558475 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 7339250270 ep_max_checkpoints 2 ep_max_data_size 12582912000 ep_max_item_size 20971520 ep_max_size 12582912000 ep_max_txn_size 10000 ep_max_vbuckets 1024 ep_mem_high_wat 9437184000 ep_mem_high_wat 9437184000 ep_mem_low_wat 7549747200 ep_mem_low_wat 7549747200 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 80059584 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1352918852 ep_storage_age 0 ep_storage_age_highwat 1293 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 6 ep_too_young 0 ep_total_cache_size 7303250246 ep_total_del_items 4 ep_total_enqueued 7449128 ep_total_new_items 3000005 ep_total_persisted 7621113 ep_uncommitted_items 0 ep_value_size 7091900551 ep_vb0 0 ep_vb_snapshot_total 2807 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_135_g40a391b ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 759 get_hits 69 get_misses 13 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 7753672504 pid 16783 pointer_size 64 rejected_conns 0 rusage_system 219.931565 rusage_user 935.707750 tap_checkpoint_end_received 1349 tap_checkpoint_end_sent 1855 tap_checkpoint_start_received 2946 tap_checkpoint_start_sent 3546 tap_connect_received 1026 tap_delete_received 2 tap_delete_sent 1 tap_mutation_received 5220295 tap_mutation_sent 6745798 tap_opaque_received 5633 tap_opaque_sent 5122 tap_vbucket_set_received 1024 tap_vbucket_set_sent 1024 threads 4 time 1352921764 total_connections 6219 uptime 2922 vb_active_curr_items 1499536 vb_active_eject 0 vb_active_expired 1 vb_active_ht_memory 12857152 vb_active_itm_memory 3649944280 vb_active_meta_data_memory 118463344 vb_active_num 512 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1499538 vb_active_ops_create 1499538 vb_active_ops_delete 3 vb_active_ops_reject 0 vb_active_ops_update 2399244 vb_active_perc_mem_resident 100 vb_active_queue_age 0 vb_active_queue_drain 6646319 vb_active_queue_fill 3764588 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 1500466 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 12857056 vb_replica_itm_memory 3653305966 vb_replica_meta_data_memory 118536814 vb_replica_num 512 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 1500466 vb_replica_ops_create 1500467 vb_replica_ops_delete 1 vb_replica_ops_reject 0 vb_replica_ops_update 2221860 vb_replica_perc_mem_resident 100 vb_replica_queue_age 0 vb_replica_queue_drain 4968715 vb_replica_queue_fill 3684540 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-14T11:36:24.428,ns_1@10.2.1.63:ns_doctor<0.540.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,921780,28647}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,264625240}, {processes,210077400}, {processes_used,209339104}, {system,54547840}, {atom,1337385}, {atom_used,1331436}, {binary,12769912}, {code,13451143}, {ets,8284576}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7532683264}, {buffered_memory,246984704}, {free_memory,16789741568}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2942495,4999}}, {context_switches,{13410381,0}}, {garbage_collection,{2513009,5445460044,0}}, {io,{{input,313134987},{output,18101415945}}}, {reductions,{1717616209,3260365}}, {run_queue,0}, {runtime,{400510,540}}]}, {system_stats, [{cpu_utilization_rate,3.007518796992481}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7290944554}, {couch_docs_data_size,7285575553}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7572093976}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2942}, {memory_data,{33657663488,16867414016,{<19415.11.0>,43123400}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16396232 kB\nBuffers: 241196 kB\nCached: 7356136 kB\nSwapCached: 0 kB\nActive: 8970040 kB\nInactive: 7193852 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16396232 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 784 kB\nWriteback: 0 kB\nAnonPages: 8566336 kB\nMapped: 15604 kB\nSlab: 251936 kB\nPageTables: 21832 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10304380 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920217,777322}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17795.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,921779,429550}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,268650440}, {processes,212402336}, {processes_used,211678640}, {system,56248104}, {atom,1349617}, {atom_used,1347167}, {binary,13313352}, {code,13553851}, {ets,9402784}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7598379008}, {buffered_memory,250773504}, {free_memory,16462200832}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2939942,5000}}, {context_switches,{16824760,0}}, {garbage_collection,{2906400,7701548128,0}}, {io,{{input,409746122},{output,13041188247}}}, {reductions,{2635211079,2623965}}, {run_queue,0}, {runtime,{446820,550}}]}, {system_stats, [{cpu_utilization_rate,4.25}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7291771960}, {couch_docs_data_size,7286342835}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,7750482104}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2939}, {memory_data,{33657663488,17200033792,{<0.11.0>,43123400}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16076368 kB\nBuffers: 244896 kB\nCached: 7420292 kB\nSwapCached: 0 kB\nActive: 9271224 kB\nInactive: 7192452 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16076368 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 376 kB\nWriteback: 112 kB\nAnonPages: 8798416 kB\nMapped: 15608 kB\nSlab: 272500 kB\nPageTables: 22184 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10516720 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-14T11:37:24.428,ns_1@10.2.1.63:ns_doctor<0.540.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,921840,28923}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,262981160}, {processes,208472744}, {processes_used,207695808}, {system,54508416}, {atom,1337385}, {atom_used,1331436}, {binary,12713856}, {code,13451143}, {ets,8305936}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7533985792}, {buffered_memory,247156736}, {free_memory,16789106688}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{3002496,5000}}, {context_switches,{13606648,0}}, {garbage_collection,{2524758,5489051917,0}}, {io,{{input,316543168},{output,18104217807}}}, {reductions,{1743132703,3241185}}, {run_queue,0}, {runtime,{406790,580}}]}, {system_stats, [{cpu_utilization_rate,3.75}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7291124778}, {couch_docs_data_size,7285575553}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7552069512}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3002}, {memory_data,{33657663488,16868302848,{<19415.11.0>,43123400}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16395612 kB\nBuffers: 241364 kB\nCached: 7357408 kB\nSwapCached: 0 kB\nActive: 8970588 kB\nInactive: 7194772 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16395612 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 576 kB\nWriteback: 16 kB\nAnonPages: 8566452 kB\nMapped: 15604 kB\nSlab: 251860 kB\nPageTables: 21832 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10302716 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920217,777322}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17795.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,921839,429263}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,261902584}, {processes,205890640}, {processes_used,205127384}, {system,56011944}, {atom,1349617}, {atom_used,1347167}, {binary,12951616}, {code,13553851}, {ets,9533688}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7599661056}, {buffered_memory,250859520}, {free_memory,16461312000}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2999942,5000}}, {context_switches,{17024259,0}}, {garbage_collection,{2916825,7770998181,0}}, {io,{{input,413311613},{output,13043233642}}}, {reductions,{2666262346,2621809}}, {run_queue,0}, {runtime,{453310,560}}]}, {system_stats, [{cpu_utilization_rate,4.260651629072682}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7291948088}, {couch_docs_data_size,7286342834}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,7730457208}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2999}, {memory_data,{33657663488,17196478464,{<0.11.0>,43123400}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16075500 kB\nBuffers: 244980 kB\nCached: 7421544 kB\nSwapCached: 0 kB\nActive: 9270828 kB\nInactive: 7193316 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16075500 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 424 kB\nWriteback: 0 kB\nAnonPages: 8797816 kB\nMapped: 15608 kB\nSlab: 272500 kB\nPageTables: 22176 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10514944 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-14T11:37:45.046,ns_1@10.2.1.63:<0.682.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,48,41}}) Stats for bucket "default": accepting_conns 1 auth_cmds 27 auth_errors 0 bucket_active_conns 1 bucket_conns 10 bytes 7721329128 bytes_read 18405728234 bytes_written 18436422662 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 82 cmd_set 2400893 conn_yields 64601 connection_structures 5000 curr_connections 13 curr_conns_on_port_11209 10 curr_conns_on_port_11210 1 curr_items 1499536 curr_items_tot 3000002 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 17 delete_misses 1 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-15 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data2/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 38798 ep_commit_time 0 ep_commit_time_total 325721 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 3 ep_data_age_highwat 150 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data2/default ep_degraded_mode 0 ep_diskqueue_drain 11615034 ep_diskqueue_fill 7449128 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 1 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 771 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 7621113 ep_io_read_bytes 0 ep_io_write_bytes 18046504810 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 2767091 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 7339250270 ep_max_checkpoints 2 ep_max_data_size 12582912000 ep_max_item_size 20971520 ep_max_size 12582912000 ep_max_txn_size 10000 ep_max_vbuckets 1024 ep_mem_high_wat 9437184000 ep_mem_high_wat 9437184000 ep_mem_low_wat 7549747200 ep_mem_low_wat 7549747200 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 63123544 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1352918852 ep_storage_age 0 ep_storage_age_highwat 1293 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 6 ep_too_young 0 ep_total_cache_size 7303250246 ep_total_del_items 4 ep_total_enqueued 7449128 ep_total_new_items 3000005 ep_total_persisted 7621113 ep_uncommitted_items 0 ep_value_size 7091900551 ep_vb0 0 ep_vb_snapshot_total 2844 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_135_g40a391b ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 759 get_hits 69 get_misses 13 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 7721329128 pid 16783 pointer_size 64 rejected_conns 0 rusage_system 220.048547 rusage_user 937.061544 tap_checkpoint_end_received 1382 tap_checkpoint_end_sent 1893 tap_checkpoint_start_received 2979 tap_checkpoint_start_sent 3584 tap_connect_received 1026 tap_delete_received 2 tap_delete_sent 1 tap_mutation_received 5220295 tap_mutation_sent 6745798 tap_opaque_received 5633 tap_opaque_sent 5122 tap_vbucket_set_received 1024 tap_vbucket_set_sent 1024 threads 4 time 1352921864 total_connections 6219 uptime 3022 vb_active_curr_items 1499536 vb_active_eject 0 vb_active_expired 1 vb_active_ht_memory 12857152 vb_active_itm_memory 3649944280 vb_active_meta_data_memory 118463344 vb_active_num 512 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1499538 vb_active_ops_create 1499538 vb_active_ops_delete 3 vb_active_ops_reject 0 vb_active_ops_update 2399244 vb_active_perc_mem_resident 100 vb_active_queue_age 0 vb_active_queue_drain 6646319 vb_active_queue_fill 3764588 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 1500466 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 12857056 vb_replica_itm_memory 3653305966 vb_replica_meta_data_memory 118536814 vb_replica_num 512 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 1500466 vb_replica_ops_create 1500467 vb_replica_ops_delete 1 vb_replica_ops_reject 0 vb_replica_ops_update 2221860 vb_replica_perc_mem_resident 100 vb_replica_queue_age 0 vb_replica_queue_drain 4968715 vb_replica_queue_fill 3684540 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-14T11:38:24.428,ns_1@10.2.1.63:ns_doctor<0.540.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,921900,28675}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,262957752}, {processes,208177848}, {processes_used,207361352}, {system,54779904}, {atom,1337385}, {atom_used,1331436}, {binary,12857472}, {code,13451143}, {ets,8439792}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7535058944}, {buffered_memory,247377920}, {free_memory,16786567168}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{3062495,4999}}, {context_switches,{13804266,0}}, {garbage_collection,{2536768,5534247934,0}}, {io,{{input,323409649},{output,18110429773}}}, {reductions,{1769105262,3398643}}, {run_queue,0}, {runtime,{413310,670}}]}, {system_stats, [{cpu_utilization_rate,5.970149253731344}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7291300906}, {couch_docs_data_size,7285575552}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7532408392}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3062}, {memory_data,{33657663488,16870461440,{<19415.11.0>,43123400}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16393132 kB\nBuffers: 241580 kB\nCached: 7358456 kB\nSwapCached: 0 kB\nActive: 8971376 kB\nInactive: 7195588 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16393132 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 452 kB\nWriteback: 0 kB\nAnonPages: 8566756 kB\nMapped: 15604 kB\nSlab: 252088 kB\nPageTables: 21836 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10305460 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920217,777322}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17795.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,921899,429269}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,263145256}, {processes,207085320}, {processes_used,206281584}, {system,56059936}, {atom,1349617}, {atom_used,1347167}, {binary,12985752}, {code,13553851}, {ets,9552256}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7600967680}, {buffered_memory,250961920}, {free_memory,16459788288}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{3059942,5000}}, {context_switches,{17223774,0}}, {garbage_collection,{2927916,7840254497,0}}, {io,{{input,416790566},{output,13045286393}}}, {reductions,{2697358726,2635663}}, {run_queue,0}, {runtime,{459850,540}}]}, {system_stats, [{cpu_utilization_rate,4.25}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7292128312}, {couch_docs_data_size,7286342832}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,7711259656}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3059}, {memory_data,{33657663488,17197875200,{<0.11.0>,43123400}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16074012 kB\nBuffers: 245080 kB\nCached: 7422820 kB\nSwapCached: 0 kB\nActive: 9272156 kB\nInactive: 7194196 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16074012 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 220 kB\nWriteback: 0 kB\nAnonPages: 8798344 kB\nMapped: 15608 kB\nSlab: 272520 kB\nPageTables: 22176 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10516032 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-14T11:39:24.428,ns_1@10.2.1.63:ns_doctor<0.540.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,921960,29371}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,265967032}, {processes,211260472}, {processes_used,210409016}, {system,54706560}, {atom,1337385}, {atom_used,1331436}, {binary,12765480}, {code,13451143}, {ets,8462096}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7536304128}, {buffered_memory,247521280}, {free_memory,16783900672}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{3122495,4999}}, {context_switches,{14000381,0}}, {garbage_collection,{2547961,5577796144,0}}, {io,{{input,326897917},{output,18113126764}}}, {reductions,{1794673102,3273443}}, {run_queue,0}, {runtime,{419980,610}}]}, {system_stats, [{cpu_utilization_rate,4.010025062656641}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7291472938}, {couch_docs_data_size,7285575551}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7513297960}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3122}, {memory_data,{33657663488,16873000960,{<19415.11.0>,43123400}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16390528 kB\nBuffers: 241720 kB\nCached: 7359672 kB\nSwapCached: 0 kB\nActive: 8973636 kB\nInactive: 7196464 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16390528 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 724 kB\nWriteback: 0 kB\nAnonPages: 8568516 kB\nMapped: 15604 kB\nSlab: 251920 kB\nPageTables: 21840 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10304776 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920217,777322}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17795.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,921959,429285}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,266243424}, {processes,210018664}, {processes_used,209179048}, {system,56224760}, {atom,1349617}, {atom_used,1347167}, {binary,13018688}, {code,13553851}, {ets,9688984}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7602204672}, {buffered_memory,251154432}, {free_memory,16457629696}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{3119942,5000}}, {context_switches,{17424401,0}}, {garbage_collection,{2938966,7910039791,0}}, {io,{{input,423712604},{output,13050682466}}}, {reductions,{2728569015,2628752}}, {run_queue,0}, {runtime,{466580,550}}]}, {system_stats, [{cpu_utilization_rate,4.488778054862843}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7292288056}, {couch_docs_data_size,7286342832}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,7691721944}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3119}, {memory_data,{33657663488,17199271936,{<0.11.0>,43123400}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16071904 kB\nBuffers: 245268 kB\nCached: 7424028 kB\nSwapCached: 0 kB\nActive: 9273032 kB\nInactive: 7195248 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16071904 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 536 kB\nWriteback: 0 kB\nAnonPages: 8799076 kB\nMapped: 15608 kB\nSlab: 272496 kB\nPageTables: 22180 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10518316 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-14T11:39:25.045,ns_1@10.2.1.63:<0.682.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,48,41}}) Stats for bucket "default": accepting_conns 1 auth_cmds 27 auth_errors 0 bucket_active_conns 1 bucket_conns 10 bytes 7688954648 bytes_read 18405745364 bytes_written 18444331074 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 82 cmd_set 2400893 conn_yields 64601 connection_structures 5000 curr_connections 13 curr_conns_on_port_11209 10 curr_conns_on_port_11210 1 curr_items 1499536 curr_items_tot 3000002 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 17 delete_misses 1 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-15 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data2/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 38798 ep_commit_time 0 ep_commit_time_total 325721 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 3 ep_data_age_highwat 150 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data2/default ep_degraded_mode 0 ep_diskqueue_drain 11615034 ep_diskqueue_fill 7449128 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 1 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 771 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 7621113 ep_io_read_bytes 0 ep_io_write_bytes 18046504810 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 2975923 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 7339250270 ep_max_checkpoints 2 ep_max_data_size 12582912000 ep_max_item_size 20971520 ep_max_size 12582912000 ep_max_txn_size 10000 ep_max_vbuckets 1024 ep_mem_high_wat 9437184000 ep_mem_high_wat 9437184000 ep_mem_low_wat 7549747200 ep_mem_low_wat 7549747200 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 46263320 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1352918852 ep_storage_age 0 ep_storage_age_highwat 1293 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 6 ep_too_young 0 ep_total_cache_size 7303250246 ep_total_del_items 4 ep_total_enqueued 7449128 ep_total_new_items 3000005 ep_total_persisted 7621113 ep_uncommitted_items 0 ep_value_size 7091900551 ep_vb0 0 ep_vb_snapshot_total 2882 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_135_g40a391b ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 759 get_hits 69 get_misses 13 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 7688954648 pid 16783 pointer_size 64 rejected_conns 0 rusage_system 220.157530 rusage_user 938.440335 tap_checkpoint_end_received 1414 tap_checkpoint_end_sent 1932 tap_checkpoint_start_received 3011 tap_checkpoint_start_sent 3623 tap_connect_received 1026 tap_delete_received 2 tap_delete_sent 1 tap_mutation_received 5220295 tap_mutation_sent 6745798 tap_opaque_received 5633 tap_opaque_sent 5122 tap_vbucket_set_received 1024 tap_vbucket_set_sent 1024 threads 4 time 1352921964 total_connections 6219 uptime 3122 vb_active_curr_items 1499536 vb_active_eject 0 vb_active_expired 1 vb_active_ht_memory 12857152 vb_active_itm_memory 3649944280 vb_active_meta_data_memory 118463344 vb_active_num 512 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1499538 vb_active_ops_create 1499538 vb_active_ops_delete 3 vb_active_ops_reject 0 vb_active_ops_update 2399244 vb_active_perc_mem_resident 100 vb_active_queue_age 0 vb_active_queue_drain 6646319 vb_active_queue_fill 3764588 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 1500466 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 12857056 vb_replica_itm_memory 3653305966 vb_replica_meta_data_memory 118536814 vb_replica_num 512 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 1500466 vb_replica_ops_create 1500467 vb_replica_ops_delete 1 vb_replica_ops_reject 0 vb_replica_ops_update 2221860 vb_replica_perc_mem_resident 100 vb_replica_queue_age 0 vb_replica_queue_drain 4968715 vb_replica_queue_fill 3684540 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-14T11:40:24.428,ns_1@10.2.1.63:ns_doctor<0.540.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,922020,28928}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,264213176}, {processes,209233384}, {processes_used,208358008}, {system,54979792}, {atom,1337385}, {atom_used,1331436}, {binary,12884960}, {code,13451143}, {ets,8619064}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7537549312}, {buffered_memory,247676928}, {free_memory,16784916480}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{3182496,5000}}, {context_switches,{14196157,0}}, {garbage_collection,{2559086,5621806138,0}}, {io,{{input,330280054},{output,18115952056}}}, {reductions,{1820250106,3247683}}, {run_queue,0}, {runtime,{426440,570}}]}, {system_stats, [{cpu_utilization_rate,3.25}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7291579434}, {couch_docs_data_size,7285575551}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7501491432}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3182}, {memory_data,{33657663488,16872620032,{<19415.11.0>,43123400}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16391520 kB\nBuffers: 241872 kB\nCached: 7360888 kB\nSwapCached: 0 kB\nActive: 8972032 kB\nInactive: 7197460 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16391520 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 540 kB\nWriteback: 0 kB\nAnonPages: 8566568 kB\nMapped: 15604 kB\nSlab: 251868 kB\nPageTables: 21840 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10291184 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920217,777322}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17795.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,922019,430064}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,262254984}, {processes,206044600}, {processes_used,205181064}, {system,56210384}, {atom,1349617}, {atom_used,1347167}, {binary,12971592}, {code,13553851}, {ets,9724576}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7603429376}, {buffered_memory,251252736}, {free_memory,16454963200}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{3179943,5001}}, {context_switches,{17623302,0}}, {garbage_collection,{2950315,7979947447,0}}, {io,{{input,427199191},{output,13052716715}}}, {reductions,{2759648426,2633675}}, {run_queue,0}, {runtime,{472970,530}}]}, {system_stats, [{cpu_utilization_rate,4.0},{swap_total,10585923584},{swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7292394552}, {couch_docs_data_size,7286342831}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,7678974744}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3179}, {memory_data,{33657663488,17201049600,{<0.11.0>,43123400}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16069300 kB\nBuffers: 245364 kB\nCached: 7425224 kB\nSwapCached: 0 kB\nActive: 9274228 kB\nInactive: 7196204 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16069300 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 520 kB\nWriteback: 0 kB\nAnonPages: 8800040 kB\nMapped: 15608 kB\nSlab: 272412 kB\nPageTables: 22180 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10529524 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-14T11:41:05.046,ns_1@10.2.1.63:<0.682.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,48,42}}) Stats for bucket "default": accepting_conns 1 auth_cmds 27 auth_errors 0 bucket_active_conns 1 bucket_conns 10 bytes 7678974744 bytes_read 18405757294 bytes_written 18452235436 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 82 cmd_set 2400893 conn_yields 64601 connection_structures 5000 curr_connections 13 curr_conns_on_port_11209 10 curr_conns_on_port_11210 1 curr_items 1499536 curr_items_tot 3000002 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 17 delete_misses 1 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-15 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data2/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 38798 ep_commit_time 0 ep_commit_time_total 325721 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 3 ep_data_age_highwat 150 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data2/default ep_degraded_mode 0 ep_diskqueue_drain 11615034 ep_diskqueue_fill 7449128 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 1 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 771 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 7621113 ep_io_read_bytes 0 ep_io_write_bytes 18046504810 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 3040265 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 7339250270 ep_max_checkpoints 2 ep_max_data_size 12582912000 ep_max_item_size 20971520 ep_max_size 12582912000 ep_max_txn_size 10000 ep_max_vbuckets 1024 ep_mem_high_wat 9437184000 ep_mem_high_wat 9437184000 ep_mem_low_wat 7549747200 ep_mem_low_wat 7549747200 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 41118924 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1352918852 ep_storage_age 0 ep_storage_age_highwat 1293 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 6 ep_too_young 0 ep_total_cache_size 7303250246 ep_total_del_items 4 ep_total_enqueued 7449128 ep_total_new_items 3000005 ep_total_persisted 7621113 ep_uncommitted_items 0 ep_value_size 7091900551 ep_vb0 0 ep_vb_snapshot_total 2891 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_135_g40a391b ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 759 get_hits 69 get_misses 13 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 7678974744 pid 16783 pointer_size 64 rejected_conns 0 rusage_system 220.207523 rusage_user 939.390190 tap_checkpoint_end_received 1414 tap_checkpoint_end_sent 1952 tap_checkpoint_start_received 3011 tap_checkpoint_start_sent 3643 tap_connect_received 1026 tap_delete_received 2 tap_delete_sent 1 tap_mutation_received 5220295 tap_mutation_sent 6745798 tap_opaque_received 5633 tap_opaque_sent 5122 tap_vbucket_set_received 1024 tap_vbucket_set_sent 1024 threads 4 time 1352922064 total_connections 6219 uptime 3222 vb_active_curr_items 1499536 vb_active_eject 0 vb_active_expired 1 vb_active_ht_memory 12857152 vb_active_itm_memory 3649944280 vb_active_meta_data_memory 118463344 vb_active_num 512 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1499538 vb_active_ops_create 1499538 vb_active_ops_delete 3 vb_active_ops_reject 0 vb_active_ops_update 2399244 vb_active_perc_mem_resident 100 vb_active_queue_age 0 vb_active_queue_drain 6646319 vb_active_queue_fill 3764588 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 1500466 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 12857056 vb_replica_itm_memory 3653305966 vb_replica_meta_data_memory 118536814 vb_replica_num 512 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 1500466 vb_replica_ops_create 1500467 vb_replica_ops_delete 1 vb_replica_ops_reject 0 vb_replica_ops_update 2221860 vb_replica_perc_mem_resident 100 vb_replica_queue_age 0 vb_replica_queue_drain 4968715 vb_replica_queue_fill 3684540 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-14T11:41:24.428,ns_1@10.2.1.63:ns_doctor<0.540.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,922080,29149}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,264330312}, {processes,209086552}, {processes_used,208211176}, {system,55243760}, {atom,1337385}, {atom_used,1331436}, {binary,12972928}, {code,13451143}, {ets,8795768}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7533133824}, {buffered_memory,247898112}, {free_memory,16780075008}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{3242496,5001}}, {context_switches,{14391438,0}}, {garbage_collection,{2571087,5666831855,0}}, {io,{{input,337039655},{output,18125041377}}}, {reductions,{1845898788,3388505}}, {run_queue,0}, {runtime,{433060,690}}]}, {system_stats, [{cpu_utilization_rate,7.5},{swap_total,10684628992},{swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7291579434}, {couch_docs_data_size,7285575551}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7501491432}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3242}, {memory_data,{33657663488,16874905600,{<19415.11.0>,43123400}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16386792 kB\nBuffers: 242088 kB\nCached: 7356576 kB\nSwapCached: 0 kB\nActive: 8978292 kB\nInactive: 7193884 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16386792 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 236 kB\nWriteback: 0 kB\nAnonPages: 8573340 kB\nMapped: 15604 kB\nSlab: 252220 kB\nPageTables: 21840 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10302656 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920217,777322}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17795.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,922079,429297}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,268797056}, {processes,212133296}, {processes_used,211272048}, {system,56663760}, {atom,1353657}, {atom_used,1352629}, {binary,13172848}, {code,13603146}, {ets,9920312}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7606054912}, {buffered_memory,251383808}, {free_memory,16453746688}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{3239942,5000}}, {context_switches,{17823466,0}}, {garbage_collection,{2961164,8056703955,0}}, {io,{{input,432095016},{output,13056226535}}}, {reductions,{2792640565,2620587}}, {run_queue,0}, {runtime,{479560,560}}]}, {system_stats, [{cpu_utilization_rate,4.488778054862843}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7292394552}, {couch_docs_data_size,7286342831}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,7678974744}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3239}, {memory_data,{33657663488,17203462144,{<0.11.0>,43123400}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16068112 kB\nBuffers: 245492 kB\nCached: 7427788 kB\nSwapCached: 0 kB\nActive: 9273740 kB\nInactive: 7198708 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16068112 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 144 kB\nWriteback: 0 kB\nAnonPages: 8799336 kB\nMapped: 15608 kB\nSlab: 272372 kB\nPageTables: 22184 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10521044 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-14T11:42:24.428,ns_1@10.2.1.63:ns_doctor<0.540.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,922140,29092}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,267243208}, {processes,211993096}, {processes_used,211117720}, {system,55250112}, {atom,1337385}, {atom_used,1331436}, {binary,12804072}, {code,13451143}, {ets,8970616}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7534239744}, {buffered_memory,248053760}, {free_memory,16787075072}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{3302496,5000}}, {context_switches,{14586407,0}}, {garbage_collection,{2582451,5710605874,0}}, {io,{{input,340412459},{output,18127759956}}}, {reductions,{1871454841,3276690}}, {run_queue,0}, {runtime,{439680,590}}]}, {system_stats, [{cpu_utilization_rate,3.25}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7291579434}, {couch_docs_data_size,7285575551}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7501491432}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3302}, {memory_data,{33657663488,16870715392,{<19415.11.0>,43123400}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16393628 kB\nBuffers: 242240 kB\nCached: 7357656 kB\nSwapCached: 0 kB\nActive: 8972036 kB\nInactive: 7194956 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16393628 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 696 kB\nWriteback: 0 kB\nAnonPages: 8567136 kB\nMapped: 15604 kB\nSlab: 251848 kB\nPageTables: 21820 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10294436 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920217,777322}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17795.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,922139,429343}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,265137336}, {processes,208397080}, {processes_used,207535832}, {system,56740256}, {atom,1353657}, {atom_used,1352629}, {binary,13063360}, {code,13603146}, {ets,10106632}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7601467392}, {buffered_memory,251564032}, {free_memory,16459206656}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{3299942,4999}}, {context_switches,{18022886,0}}, {garbage_collection,{2970387,8130530527,0}}, {io,{{input,438897437},{output,13066104336}}}, {reductions,{2824634856,2621579}}, {run_queue,0}, {runtime,{486400,540}}]}, {system_stats, [{cpu_utilization_rate,4.239401496259352}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7292394552}, {couch_docs_data_size,7286342831}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,7678974744}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3299}, {memory_data,{33657663488,17206202368,{<0.11.0>,43123400}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16073444 kB\nBuffers: 245668 kB\nCached: 7423308 kB\nSwapCached: 0 kB\nActive: 9272428 kB\nInactive: 7194948 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16073444 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 556 kB\nWriteback: 0 kB\nAnonPages: 8798424 kB\nMapped: 15608 kB\nSlab: 272356 kB\nPageTables: 22176 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10516892 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-14T11:42:45.045,ns_1@10.2.1.63:<0.682.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,48,42}}) Stats for bucket "default": accepting_conns 1 auth_cmds 27 auth_errors 0 bucket_active_conns 1 bucket_conns 10 bytes 7678974744 bytes_read 18405768264 bytes_written 18460137306 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 82 cmd_set 2400893 conn_yields 64601 connection_structures 5000 curr_connections 13 curr_conns_on_port_11209 10 curr_conns_on_port_11210 1 curr_items 1499536 curr_items_tot 3000002 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 17 delete_misses 1 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-15 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data2/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 38798 ep_commit_time 0 ep_commit_time_total 325721 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 3 ep_data_age_highwat 150 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data2/default ep_degraded_mode 0 ep_diskqueue_drain 11615034 ep_diskqueue_fill 7449128 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 1 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 771 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 7621113 ep_io_read_bytes 0 ep_io_write_bytes 18046504810 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 3040265 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 7339250270 ep_max_checkpoints 2 ep_max_data_size 12582912000 ep_max_item_size 20971520 ep_max_size 12582912000 ep_max_txn_size 10000 ep_max_vbuckets 1024 ep_mem_high_wat 9437184000 ep_mem_high_wat 9437184000 ep_mem_low_wat 7549747200 ep_mem_low_wat 7549747200 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 41118924 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1352918852 ep_storage_age 0 ep_storage_age_highwat 1293 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 6 ep_too_young 0 ep_total_cache_size 7303250246 ep_total_del_items 4 ep_total_enqueued 7449128 ep_total_new_items 3000005 ep_total_persisted 7621113 ep_uncommitted_items 0 ep_value_size 7091900551 ep_vb0 0 ep_vb_snapshot_total 2891 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_135_g40a391b ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 759 get_hits 69 get_misses 13 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 7678974744 pid 16783 pointer_size 64 rejected_conns 0 rusage_system 220.231519 rusage_user 940.092084 tap_checkpoint_end_received 1414 tap_checkpoint_end_sent 1952 tap_checkpoint_start_received 3011 tap_checkpoint_start_sent 3643 tap_connect_received 1026 tap_delete_received 2 tap_delete_sent 1 tap_mutation_received 5220295 tap_mutation_sent 6745798 tap_opaque_received 5633 tap_opaque_sent 5122 tap_vbucket_set_received 1024 tap_vbucket_set_sent 1024 threads 4 time 1352922164 total_connections 6219 uptime 3322 vb_active_curr_items 1499536 vb_active_eject 0 vb_active_expired 1 vb_active_ht_memory 12857152 vb_active_itm_memory 3649944280 vb_active_meta_data_memory 118463344 vb_active_num 512 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1499538 vb_active_ops_create 1499538 vb_active_ops_delete 3 vb_active_ops_reject 0 vb_active_ops_update 2399244 vb_active_perc_mem_resident 100 vb_active_queue_age 0 vb_active_queue_drain 6646319 vb_active_queue_fill 3764588 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 1500466 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 12857056 vb_replica_itm_memory 3653305966 vb_replica_meta_data_memory 118536814 vb_replica_num 512 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 1500466 vb_replica_ops_create 1500467 vb_replica_ops_delete 1 vb_replica_ops_reject 0 vb_replica_ops_update 2221860 vb_replica_perc_mem_resident 100 vb_replica_queue_age 0 vb_replica_queue_drain 4968715 vb_replica_queue_fill 3684540 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-14T11:43:24.427,ns_1@10.2.1.63:ns_doctor<0.540.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,922200,29171}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,264560040}, {processes,209071184}, {processes_used,208195808}, {system,55488856}, {atom,1337385}, {atom_used,1331436}, {binary,12979000}, {code,13451143}, {ets,9034088}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7580667904}, {buffered_memory,248360960}, {free_memory,16721465344}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{3362496,5000}}, {context_switches,{14781290,0}}, {garbage_collection,{2593641,5754371192,0}}, {io,{{input,343805882},{output,18130346477}}}, {reductions,{1896951421,3255418}}, {run_queue,0}, {runtime,{446170,550}}]}, {system_stats, [{cpu_utilization_rate,10.275689223057643}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7291579434}, {couch_docs_data_size,7285575551}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7501491432}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3362}, {memory_data,{33657663488,16881532928,{<19415.11.0>,43123400}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16329556 kB\nBuffers: 242540 kB\nCached: 7402996 kB\nSwapCached: 0 kB\nActive: 8992016 kB\nInactive: 7236284 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16329556 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 44312 kB\nWriteback: 0 kB\nAnonPages: 8582716 kB\nMapped: 16676 kB\nSlab: 254064 kB\nPageTables: 22284 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10383700 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920217,777322}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17795.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,922199,429304}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,262666688}, {processes,205570960}, {processes_used,204709712}, {system,57095728}, {atom,1353657}, {atom_used,1352629}, {binary,13244376}, {code,13603146}, {ets,10281136}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7602573312}, {buffered_memory,251654144}, {free_memory,16456159232}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{3359942,5000}}, {context_switches,{18221648,0}}, {garbage_collection,{2981847,8203570583,0}}, {io,{{input,442263253},{output,13068174858}}}, {reductions,{2856209955,2628548}}, {run_queue,0}, {runtime,{493060,580}}]}, {system_stats, [{cpu_utilization_rate,4.5},{swap_total,10585923584},{swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7292394552}, {couch_docs_data_size,7286342831}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,7678974744}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3359}, {memory_data,{33657663488,17199345664,{<0.11.0>,43123400}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16070468 kB\nBuffers: 245756 kB\nCached: 7424388 kB\nSwapCached: 0 kB\nActive: 9273600 kB\nInactive: 7195952 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16070468 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 404 kB\nWriteback: 0 kB\nAnonPages: 8799404 kB\nMapped: 15608 kB\nSlab: 272328 kB\nPageTables: 22180 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10520800 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-14T11:44:24.428,ns_1@10.2.1.63:ns_doctor<0.540.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,922260,28946}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,268075048}, {processes,212934712}, {processes_used,212059336}, {system,55140336}, {atom,1337385}, {atom_used,1331436}, {binary,12453464}, {code,13451143}, {ets,9211736}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7608786944}, {buffered_memory,248659968}, {free_memory,16701493248}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{3422496,5000}}, {context_switches,{14997443,0}}, {garbage_collection,{2604381,5867175342,0}}, {io,{{input,355341429},{output,18154023630}}}, {reductions,{1966563915,3400056}}, {run_queue,0}, {runtime,{454840,610}}]}, {system_stats, [{cpu_utilization_rate,5.513784461152882}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7291579434}, {couch_docs_data_size,7285575551}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7501491432}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3422}, {memory_data,{33657663488,16954265600,{<19415.11.0>,43123400}}}, {disk_data, [{"/",49781912,7}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16310052 kB\nBuffers: 242832 kB\nCached: 7430456 kB\nSwapCached: 0 kB\nActive: 9054664 kB\nInactive: 7193656 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16310052 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 32748 kB\nWriteback: 0 kB\nAnonPages: 8575044 kB\nMapped: 15604 kB\nSlab: 253972 kB\nPageTables: 21860 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10306484 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920217,777322}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17795.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,922259,429322}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,269009120}, {processes,212533120}, {processes_used,211671872}, {system,56476000}, {atom,1353657}, {atom_used,1352629}, {binary,12561800}, {code,13603146}, {ets,10343792}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7604334592}, {buffered_memory,251805696}, {free_memory,16451944448}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{3419942,5000}}, {context_switches,{18421441,0}}, {garbage_collection,{2993556,8280733708,0}}, {io,{{input,445724041},{output,13070142928}}}, {reductions,{2889021218,2625382}}, {run_queue,0}, {runtime,{499770,550}}]}, {system_stats, [{cpu_utilization_rate,4.25}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7292394552}, {couch_docs_data_size,7286342831}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,7678974744}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3419}, {memory_data,{33657663488,17202900992,{<0.11.0>,43123400}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16066352 kB\nBuffers: 245904 kB\nCached: 7426108 kB\nSwapCached: 0 kB\nActive: 9277292 kB\nInactive: 7195776 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16066352 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 336 kB\nWriteback: 0 kB\nAnonPages: 8801240 kB\nMapped: 16720 kB\nSlab: 272588 kB\nPageTables: 22532 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10527884 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-14T11:44:25.045,ns_1@10.2.1.63:<0.682.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,48,42}}) Stats for bucket "default": accepting_conns 1 auth_cmds 27 auth_errors 0 bucket_active_conns 1 bucket_conns 10 bytes 7678974744 bytes_read 18405779295 bytes_written 18468621163 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 82 cmd_set 2400893 conn_yields 64601 connection_structures 5000 curr_connections 13 curr_conns_on_port_11209 10 curr_conns_on_port_11210 1 curr_items 1499536 curr_items_tot 3000002 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 17 delete_misses 1 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-15 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data2/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 38798 ep_commit_time 0 ep_commit_time_total 325721 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 3 ep_data_age_highwat 150 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data2/default ep_degraded_mode 0 ep_diskqueue_drain 11615034 ep_diskqueue_fill 7449128 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 1 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 771 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 7621113 ep_io_read_bytes 0 ep_io_write_bytes 18046504810 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 3040265 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 7339250270 ep_max_checkpoints 2 ep_max_data_size 12582912000 ep_max_item_size 20971520 ep_max_size 12582912000 ep_max_txn_size 10000 ep_max_vbuckets 1024 ep_mem_high_wat 9437184000 ep_mem_high_wat 9437184000 ep_mem_low_wat 7549747200 ep_mem_low_wat 7549747200 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 41118924 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1352918852 ep_storage_age 0 ep_storage_age_highwat 1293 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 6 ep_too_young 0 ep_total_cache_size 7303250246 ep_total_del_items 4 ep_total_enqueued 7449128 ep_total_new_items 3000005 ep_total_persisted 7621113 ep_uncommitted_items 0 ep_value_size 7091900551 ep_vb0 0 ep_vb_snapshot_total 2891 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_135_g40a391b ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 759 get_hits 69 get_misses 13 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 7678974744 pid 16783 pointer_size 64 rejected_conns 0 rusage_system 220.267514 rusage_user 940.812974 tap_checkpoint_end_received 1414 tap_checkpoint_end_sent 1952 tap_checkpoint_start_received 3011 tap_checkpoint_start_sent 3643 tap_connect_received 1026 tap_delete_received 2 tap_delete_sent 1 tap_mutation_received 5220295 tap_mutation_sent 6745798 tap_opaque_received 5633 tap_opaque_sent 5122 tap_vbucket_set_received 1024 tap_vbucket_set_sent 1024 threads 4 time 1352922264 total_connections 6219 uptime 3422 vb_active_curr_items 1499536 vb_active_eject 0 vb_active_expired 1 vb_active_ht_memory 12857152 vb_active_itm_memory 3649944280 vb_active_meta_data_memory 118463344 vb_active_num 512 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1499538 vb_active_ops_create 1499538 vb_active_ops_delete 3 vb_active_ops_reject 0 vb_active_ops_update 2399244 vb_active_perc_mem_resident 100 vb_active_queue_age 0 vb_active_queue_drain 6646319 vb_active_queue_fill 3764588 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 1500466 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 12857056 vb_replica_itm_memory 3653305966 vb_replica_meta_data_memory 118536814 vb_replica_num 512 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 1500466 vb_replica_ops_create 1500467 vb_replica_ops_delete 1 vb_replica_ops_reject 0 vb_replica_ops_update 2221860 vb_replica_perc_mem_resident 100 vb_replica_queue_age 0 vb_replica_queue_drain 4968715 vb_replica_queue_fill 3684540 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a