============================================================================== couchbase logs (stats) cbbrowse_logs stats ============================================================================== [ns_doctor:debug,2012-11-02T18:23:06.774,ns_1@127.0.0.1:ns_doctor<0.376.0>:ns_doctor:handle_info:133]Got initial status [{'ns_1@127.0.0.1', [{last_heard,{1351,905786,774551}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,39895736}, {processes,9098576}, {processes_used,9087808}, {system,30797160}, {atom,1171065}, {atom_used,1138453}, {binary,146016}, {code,11344932}, {ets,875056}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,597585920}, {buffered_memory,297488384}, {free_memory,32388177920}, {total_memory,33657663488}]}, {node_storage_conf, [{db_path, "/opt/couchbase/var/lib/couchbase/data/"}, {index_path, "/opt/couchbase/var/lib/couchbase/data/"}]}, {statistics, [{wall_clock,{1902,3}}, {context_switches,{26960,0}}, {garbage_collection,{7655,35141922,0}}, {io,{{input,6769003},{output,663490}}}, {reductions,{20816104,18988}}, {run_queue,0}, {runtime,{1240,10}}]}, {system_stats, [{cpu_utilization_rate,0}, {swap_total,0}, {swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,1}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1}, {memory_data, {33657663488,1249058816,{<0.6.0>,601272}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 31629080 kB\nBuffers: 290516 kB\nCached: 583580 kB\nSwapCached: 12 kB\nActive: 418208 kB\nInactive: 581868 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 31629080 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 419740 kB\nWriteback: 0 kB\nAnonPages: 125320 kB\nMapped: 13824 kB\nSlab: 201164 kB\nPageTables: 4656 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 1572084 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-02T18:23:16.063,ns_1@127.0.0.1:ns_doctor<0.736.0>:ns_doctor:handle_info:133]Got initial status [{'ns_1@127.0.0.1', [{last_heard,{1351,905796,63650}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,42736336}, {processes,10249840}, {processes_used,10176432}, {system,32486496}, {atom,1242281}, {atom_used,1240746}, {binary,231104}, {code,12711989}, {ets,1000120}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,598093824}, {buffered_memory,297865216}, {free_memory,32382713856}, {total_memory,33657663488}]}, {node_storage_conf, [{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{11191,2}}, {context_switches,{34737,0}}, {garbage_collection,{9717,38785780,0}}, {io,{{input,7662144},{output,1096865}}}, {reductions,{22191838,58206}}, {run_queue,0}, {runtime,{1930,10}}]}, {system_stats, [{cpu_utilization_rate,0}, {swap_total,0}, {swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,11}, {memory_data, {33657663488,1249058816,{<0.6.0>,601272}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 31623744 kB\nBuffers: 290884 kB\nCached: 584076 kB\nSwapCached: 12 kB\nActive: 425200 kB\nInactive: 578640 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 31623744 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 419860 kB\nWriteback: 48 kB\nAnonPages: 128696 kB\nMapped: 16092 kB\nSlab: 201508 kB\nPageTables: 4804 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 1595712 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-02T18:24:16.058,ns_1@127.0.0.1:ns_doctor<0.736.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@127.0.0.1', [{last_heard,{1351,905851,59381}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,46707240}, {processes,14084080}, {processes_used,14062664}, {system,32623160}, {atom,1242281}, {atom_used,1240746}, {binary,283688}, {code,12711989}, {ets,1070040}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,598405120}, {buffered_memory,298037248}, {free_memory,32320770048}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{66187,5000}}, {context_switches,{42327,0}}, {garbage_collection,{10963,41409561,0}}, {io,{{input,7715569},{output,1242640}}}, {reductions,{23072989,70847}}, {run_queue,0}, {runtime,{4390,230}}]}, {system_stats, [{cpu_utilization_rate,1.4962593516209477}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,66}, {memory_data,{33657663488,1337020416,{<0.6.0>,601480}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 31563252 kB\nBuffers: 291052 kB\nCached: 584380 kB\nSwapCached: 12 kB\nActive: 488084 kB\nInactive: 578852 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 31563252 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 162544 kB\nWriteback: 0 kB\nAnonPages: 191532 kB\nMapped: 16096 kB\nSlab: 197948 kB\nPageTables: 5140 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 1835200 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-02T18:25:16.058,ns_1@127.0.0.1:ns_doctor<0.736.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@127.0.0.1', [{last_heard,{1351,905911,59433}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,47378656}, {processes,14696352}, {processes_used,14674936}, {system,32682304}, {atom,1243089}, {atom_used,1242343}, {binary,283752}, {code,12736234}, {ets,1100976}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,598495232}, {buffered_memory,298119168}, {free_memory,32321150976}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{126187,4999}}, {context_switches,{49942,0}}, {garbage_collection,{11991,43932466,0}}, {io,{{input,7768110},{output,1331635}}}, {reductions,{23981674,79601}}, {run_queue,0}, {runtime,{7110,250}}]}, {system_stats, [{cpu_utilization_rate,1.5}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,126}, {memory_data,{33657663488,1337147392,{<0.6.0>,601480}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 31563624 kB\nBuffers: 291132 kB\nCached: 584468 kB\nSwapCached: 12 kB\nActive: 488852 kB\nInactive: 578952 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 31563624 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 112932 kB\nWriteback: 0 kB\nAnonPages: 192180 kB\nMapped: 16108 kB\nSlab: 197436 kB\nPageTables: 5140 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 1835200 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-02T18:26:10.412,ns_1@10.2.1.68:ns_doctor<0.1369.0>:ns_doctor:handle_info:133]Got initial status [{'ns_1@10.2.1.68', [{last_heard,{1351,905970,412802}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,47603312}, {processes,14896648}, {processes_used,14877848}, {system,32706664}, {atom,1244705}, {atom_used,1244375}, {binary,290104}, {code,12784915}, {ets,1051264}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,598327296}, {buffered_memory,298414080}, {free_memory,32318865408}, {total_memory,33657663488}]}, {node_storage_conf, [{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{185540,59}}, {context_switches,{60016,0}}, {garbage_collection,{13692,47787824,0}}, {io,{{input,8035258},{output,1731408}}}, {reductions,{25305492,311198}}, {run_queue,0}, {runtime,{9920,80}}]}, {system_stats, [{cpu_utilization_rate,0}, {swap_total,0}, {swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,185}, {memory_data, {33657663488,1337147392,{<0.6.0>,601480}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 31561516 kB\nBuffers: 291420 kB\nCached: 584304 kB\nSwapCached: 12 kB\nActive: 491580 kB\nInactive: 579160 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 31561516 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 81600 kB\nWriteback: 0 kB\nAnonPages: 194516 kB\nMapped: 16108 kB\nSlab: 197396 kB\nPageTables: 5144 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 1838536 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-02T18:26:11.307,ns_1@10.2.1.68:ns_doctor<0.1479.0>:ns_doctor:handle_info:133]Got initial status [{'ns_1@10.2.1.65', [{last_heard,{1351,905971,306993}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,61655256}, {processes,26164496}, {processes_used,24037944}, {system,35490760}, {atom,1400977}, {atom_used,1372242}, {binary,555128}, {code,14085234}, {ets,1835976}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,609570816}, {buffered_memory,84189184}, {free_memory,32584884224}, {total_memory,33657663488}]}, {node_storage_conf, [{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{187159,18}}, {context_switches,{282458,0}}, {garbage_collection,{43990,158293541,0}}, {io,{{input,16125796},{output,8262088}}}, {reductions,{81571478,50141}}, {run_queue,0}, {runtime,{15450,10}}]}, {system_stats, [{cpu_utilization_rate,3.5}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,187}, {memory_data, {33657663488,1071636480,{<18077.695.0>,972728}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 31821300 kB\nBuffers: 82216 kB\nCached: 595284 kB\nSwapCached: 0 kB\nActive: 451876 kB\nInactive: 441636 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 31821300 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 82228 kB\nWriteback: 0 kB\nAnonPages: 215960 kB\nMapped: 15228 kB\nSlab: 117864 kB\nPageTables: 4568 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 1886916 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,905971,306981}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,46035376}, {processes,13144032}, {processes_used,13075232}, {system,32891344}, {atom,1277481}, {atom_used,1244595}, {binary,309984}, {code,12784915}, {ets,1121624}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,598355968}, {buffered_memory,298487808}, {free_memory,32378707968}, {total_memory,33657663488}]}, {node_storage_conf, [{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{186434,1}}, {context_switches,{62775,0}}, {garbage_collection,{14369,49262895,0}}, {io,{{input,8097576},{output,1917956}}}, {reductions,{25724009,29435}}, {run_queue,0}, {runtime,{10030,0}}]}, {system_stats, [{cpu_utilization_rate,0}, {swap_total,0}, {swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,186}, {memory_data, {33657663488,1337147392,{<0.6.0>,601480}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 31619956 kB\nBuffers: 291492 kB\nCached: 584332 kB\nSwapCached: 12 kB\nActive: 433188 kB\nInactive: 579304 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 31619956 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 81632 kB\nWriteback: 88 kB\nAnonPages: 136296 kB\nMapped: 15384 kB\nSlab: 197448 kB\nPageTables: 4724 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 1604284 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-02T18:26:24.830,ns_1@10.2.1.68:<0.1669.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,31,46}}) Stats for bucket "default": accepting_conns 1 auth_cmds 0 auth_errors 0 bucket_active_conns 1 bucket_conns 5 bytes 245008 bytes_read 196 bytes_written 120 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 0 conn_yields 0 connection_structures 5000 curr_connections 9 curr_conns_on_port_11209 6 curr_conns_on_port_11210 1 curr_items 0 curr_items_tot 0 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-03 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 0 ep_commit_time 0 ep_commit_time_total 0 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 0 ep_data_age_highwat 0 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data/default ep_degraded_mode 1 ep_diskqueue_drain 0 ep_diskqueue_fill 0 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 0 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 0 ep_io_read_bytes 0 ep_io_write_bytes 0 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 0 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 0 ep_max_checkpoints 2 ep_max_data_size 5242880000 ep_max_item_size 20971520 ep_max_size 5242880000 ep_max_txn_size 10000 ep_max_vbuckets 512 ep_mem_high_wat 3932160000 ep_mem_high_wat 3932160000 ep_mem_low_wat 3145728000 ep_mem_low_wat 3145728000 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 1152 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1351905984 ep_storage_age 0 ep_storage_age_highwat 0 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 0 ep_total_del_items 0 ep_total_enqueued 0 ep_total_new_items 0 ep_total_persisted 0 ep_uncommitted_items 0 ep_value_size 0 ep_vb0 0 ep_vb_snapshot_total 1 ep_vb_total 0 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_130_g8b09894 ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 741 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 245008 pid 17027 pointer_size 64 rejected_conns 0 rusage_system 0.011998 rusage_user 0.035994 threads 4 time 1351905984 total_connections 10 uptime 15 vb_active_curr_items 0 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 0 vb_active_itm_memory 0 vb_active_meta_data_memory 0 vb_active_num 0 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 0 vb_active_ops_create 0 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 0 vb_active_queue_age 0 vb_active_queue_drain 0 vb_active_queue_fill 0 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_meta_data_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-02T18:27:11.301,ns_1@10.2.1.68:ns_doctor<0.1479.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,906026,337978}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,74257936}, {processes,37832144}, {processes_used,36255088}, {system,36425792}, {atom,1400977}, {atom_used,1372242}, {binary,1140488}, {code,14085234}, {ets,2071384}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,612155392}, {buffered_memory,88670208}, {free_memory,32574922752}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{242193,5000}}, {context_switches,{333391,0}}, {garbage_collection,{51641,198958736,0}}, {io,{{input,16965826},{output,9972498}}}, {reductions,{90983469,603717}}, {run_queue,0}, {runtime,{20520,400}}]}, {system_stats, [{cpu_utilization_rate,2.25}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,1061412}, {couch_docs_data_size,1054464}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,0}, {curr_items_tot,0}, {mem_used,4169944}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,242}, {memory_data,{33657663488,1082613760,{<18077.695.0>,1572896}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 31811448 kB\nBuffers: 86592 kB\nCached: 597808 kB\nSwapCached: 0 kB\nActive: 455248 kB\nInactive: 446716 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 31811448 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 60 kB\nWriteback: 0 kB\nAnonPages: 217424 kB\nMapped: 15252 kB\nSlab: 117968 kB\nPageTables: 4596 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 1912844 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,906028,975487}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,69545472}, {processes,34221992}, {processes_used,34208120}, {system,35323480}, {atom,1336577}, {atom_used,1329113}, {binary,1259024}, {code,13449841}, {ets,1715488}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,609120256}, {buffered_memory,99512320}, {free_memory,32565166080}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{244439,5000}}, {context_switches,{109275,0}}, {garbage_collection,{21885,66579724,0}}, {io,{{input,9319432},{output,3396573}}}, {reductions,{31189203,299608}}, {run_queue,0}, {runtime,{14630,410}}]}, {system_stats, [{cpu_utilization_rate,2.493765586034913}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,1061412}, {couch_docs_data_size,1054464}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,0}, {curr_items_tot,0}, {mem_used,4174248}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,244}, {memory_data,{33657663488,1091227648,{<18118.693.0>,1943840}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 31802044 kB\nBuffers: 97180 kB\nCached: 594844 kB\nSwapCached: 140 kB\nActive: 473436 kB\nInactive: 439568 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 31802044 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 57196 kB\nWriteback: 0 kB\nAnonPages: 220824 kB\nMapped: 17020 kB\nSlab: 114512 kB\nPageTables: 5208 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 1926540 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,906027,949396}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,70996344}, {processes,35666304}, {processes_used,35652432}, {system,35330040}, {atom,1336577}, {atom_used,1329113}, {binary,1265280}, {code,13459097}, {ets,1706368}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,647421952}, {buffered_memory,401833984}, {free_memory,32149688320}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{244397,4997}}, {context_switches,{108906,0}}, {garbage_collection,{22039,64627839,0}}, {io,{{input,9332764},{output,3403170}}}, {reductions,{30464823,361013}}, {run_queue,0}, {runtime,{14040,430}}]}, {system_stats, [{cpu_utilization_rate,3.5},{swap_total,10701078528},{swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,1061412}, {couch_docs_data_size,1054464}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,0}, {curr_items_tot,0}, {mem_used,4174248}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,244}, {memory_data,{33657663488,1507975168,{<18117.701.0>,1943840}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 31396180 kB\nBuffers: 392416 kB\nCached: 632248 kB\nSwapCached: 0 kB\nActive: 724072 kB\nInactive: 518260 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 31396180 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 69800 kB\nWriteback: 0 kB\nAnonPages: 217712 kB\nMapped: 17020 kB\nSlab: 193460 kB\nPageTables: 5224 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 1920576 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,906026,303034}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,71465656}, {processes,36114600}, {processes_used,36100728}, {system,35351056}, {atom,1336577}, {atom_used,1329113}, {binary,1281400}, {code,13487081}, {ets,1686808}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,601063424}, {buffered_memory,298717184}, {free_memory,32289730560}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{241430,5000}}, {context_switches,{108749,0}}, {garbage_collection,{22008,66460406,0}}, {io,{{input,9312924},{output,3369223}}}, {reductions,{32070934,310365}}, {run_queue,0}, {runtime,{14580,430}}]}, {system_stats, [{cpu_utilization_rate,4.0}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,1061412}, {couch_docs_data_size,1054464}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,0}, {curr_items_tot,0}, {mem_used,4174248}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,241}, {memory_data,{33657663488,1367298048,{<0.694.0>,1572896}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 31533064 kB\nBuffers: 291716 kB\nCached: 586976 kB\nSwapCached: 12 kB\nActive: 523132 kB\nInactive: 574852 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 31533064 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 57160 kB\nWriteback: 0 kB\nAnonPages: 219304 kB\nMapped: 17164 kB\nSlab: 198160 kB\nPageTables: 5160 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 1922404 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-02T18:28:04.828,ns_1@10.2.1.68:<0.1669.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,31,46}}) Stats for bucket "default": accepting_conns 1 auth_cmds 1 auth_errors 0 bucket_active_conns 1 bucket_conns 5 bytes 4174248 bytes_read 14436 bytes_written 1472851 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 0 conn_yields 0 connection_structures 5000 curr_connections 9 curr_conns_on_port_11209 6 curr_conns_on_port_11210 1 curr_items 0 curr_items_tot 0 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-03 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 0 ep_commit_time 0 ep_commit_time_total 0 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 0 ep_data_age_highwat 0 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data/default ep_degraded_mode 0 ep_diskqueue_drain 0 ep_diskqueue_fill 0 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 0 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 0 ep_io_read_bytes 0 ep_io_write_bytes 0 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 0 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 0 ep_max_checkpoints 2 ep_max_data_size 5242880000 ep_max_item_size 20971520 ep_max_size 5242880000 ep_max_txn_size 10000 ep_max_vbuckets 512 ep_mem_high_wat 3932160000 ep_mem_high_wat 3932160000 ep_mem_low_wat 3145728000 ep_mem_low_wat 3145728000 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 3404160 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1351905984 ep_storage_age 0 ep_storage_age_highwat 0 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 0 ep_total_del_items 0 ep_total_enqueued 0 ep_total_new_items 0 ep_total_persisted 0 ep_uncommitted_items 0 ep_value_size 0 ep_vb0 0 ep_vb_snapshot_total 6 ep_vb_total 128 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_130_g8b09894 ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 741 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 4174248 pid 17027 pointer_size 64 rejected_conns 0 rusage_system 0.068989 rusage_user 0.201969 threads 4 time 1351906084 total_connections 11 uptime 115 vb_active_curr_items 0 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 3214336 vb_active_itm_memory 0 vb_active_meta_data_memory 0 vb_active_num 128 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 0 vb_active_ops_create 0 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 0 vb_active_queue_age 0 vb_active_queue_drain 0 vb_active_queue_fill 0 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_meta_data_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-02T18:28:11.301,ns_1@10.2.1.68:ns_doctor<0.1479.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,906086,337331}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,89436584}, {processes,51347256}, {processes_used,49769064}, {system,38089328}, {atom,1404209}, {atom_used,1376964}, {binary,2585760}, {code,14117455}, {ets,2247752}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,613122048}, {buffered_memory,88891392}, {free_memory,32555114496}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{302193,5000}}, {context_switches,{376891,0}}, {garbage_collection,{56906,249743428,0}}, {io,{{input,30963191},{output,12099459}}}, {reductions,{102202202,993542}}, {run_queue,0}, {runtime,{27360,520}}]}, {system_stats, [{cpu_utilization_rate,3.25}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,1068341}, {couch_docs_data_size,1054464}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,0}, {curr_items_tot,0}, {mem_used,4169944}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,302}, {memory_data,{33657663488,1102422016,{<18077.5273.0>,7628384}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 31792104 kB\nBuffers: 86808 kB\nCached: 598752 kB\nSwapCached: 0 kB\nActive: 475048 kB\nInactive: 447816 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 31792104 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 408 kB\nWriteback: 0 kB\nAnonPages: 237132 kB\nMapped: 15252 kB\nSlab: 117980 kB\nPageTables: 4648 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 1946992 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,906088,976192}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,74344952}, {processes,38617592}, {processes_used,38603720}, {system,35727360}, {atom,1352041}, {atom_used,1349085}, {binary,1156640}, {code,13622682}, {ets,2029904}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,610246656}, {buffered_memory,99692544}, {free_memory,32556404736}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{304439,4999}}, {context_switches,{146175,0}}, {garbage_collection,{25661,83257294,0}}, {io,{{input,10062848},{output,4723568}}}, {reductions,{35219840,328442}}, {run_queue,0}, {runtime,{19720,410}}]}, {system_stats, [{cpu_utilization_rate,2.0}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,1068341}, {couch_docs_data_size,1054464}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,0}, {curr_items_tot,0}, {mem_used,4174248}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,304}, {memory_data,{33657663488,1101258752,{<18118.693.0>,1943840}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 31793364 kB\nBuffers: 97356 kB\nCached: 595944 kB\nSwapCached: 140 kB\nActive: 480904 kB\nInactive: 440636 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 31793364 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 45236 kB\nWriteback: 0 kB\nAnonPages: 228136 kB\nMapped: 17020 kB\nSlab: 114664 kB\nPageTables: 5232 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 1928152 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,906087,950633}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,77233192}, {processes,41513392}, {processes_used,41499520}, {system,35719800}, {atom,1352041}, {atom_used,1349085}, {binary,1148504}, {code,13631938}, {ets,2020960}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,648544256}, {buffered_memory,401911808}, {free_memory,32142704640}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{304398,4999}}, {context_switches,{145849,0}}, {garbage_collection,{26124,81522325,0}}, {io,{{input,10122322},{output,4774593}}}, {reductions,{34610622,408532}}, {run_queue,0}, {runtime,{19100,450}}]}, {system_stats, [{cpu_utilization_rate,3.5},{swap_total,10701078528},{swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,1068341}, {couch_docs_data_size,1054464}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,0}, {curr_items_tot,0}, {mem_used,4174248}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,304}, {memory_data,{33657663488,1514831872,{<18117.1691.0>,3143528}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 31389360 kB\nBuffers: 392492 kB\nCached: 633344 kB\nSwapCached: 0 kB\nActive: 728844 kB\nInactive: 519160 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 31389360 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 48964 kB\nWriteback: 0 kB\nAnonPages: 222188 kB\nMapped: 17020 kB\nSlab: 193712 kB\nPageTables: 5256 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 1929580 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,906086,302997}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,77023872}, {processes,41096032}, {processes_used,41082160}, {system,35927840}, {atom,1352041}, {atom_used,1349085}, {binary,1312552}, {code,13659922}, {ets,2041256}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,602230784}, {buffered_memory,298831872}, {free_memory,32281477120}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{301430,5000}}, {context_switches,{145689,0}}, {garbage_collection,{25869,83218007,0}}, {io,{{input,10097295},{output,4775968}}}, {reductions,{36106851,335012}}, {run_queue,0}, {runtime,{19770,430}}]}, {system_stats, [{cpu_utilization_rate,4.0}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,1068341}, {couch_docs_data_size,1054464}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,0}, {curr_items_tot,0}, {mem_used,4174248}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,301}, {memory_data,{33657663488,1376567296,{<0.694.0>,1572896}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 31524880 kB\nBuffers: 291828 kB\nCached: 588116 kB\nSwapCached: 12 kB\nActive: 529692 kB\nInactive: 575756 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 31524880 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 48816 kB\nWriteback: 104 kB\nAnonPages: 225088 kB\nMapped: 17164 kB\nSlab: 198008 kB\nPageTables: 5176 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 1927408 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-02T18:29:11.300,ns_1@10.2.1.68:ns_doctor<0.1479.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,906146,336732}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,88186736}, {processes,49850528}, {processes_used,48272336}, {system,38336208}, {atom,1404209}, {atom_used,1376964}, {binary,2641088}, {code,14117455}, {ets,2433568}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,614178816}, {buffered_memory,89198592}, {free_memory,32548765696}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{362193,5000}}, {context_switches,{420633,0}}, {garbage_collection,{62465,301071040,0}}, {io,{{input,49032539},{output,16778978}}}, {reductions,{113017393,981120}}, {run_queue,0}, {runtime,{34470,570}}]}, {system_stats, [{cpu_utilization_rate,3.25}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,1068356}, {couch_docs_data_size,1054464}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,0}, {curr_items_tot,0}, {mem_used,4169944}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,362}, {memory_data,{33657663488,1108262912,{<18077.5273.0>,7628384}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 31786028 kB\nBuffers: 87108 kB\nCached: 599784 kB\nSwapCached: 0 kB\nActive: 479168 kB\nInactive: 449228 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 31786028 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 188 kB\nWriteback: 0 kB\nAnonPages: 241224 kB\nMapped: 15252 kB\nSlab: 118224 kB\nPageTables: 4608 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 1947580 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,906148,976924}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,74744488}, {processes,38727520}, {processes_used,38712848}, {system,36016968}, {atom,1352041}, {atom_used,1349085}, {binary,1256840}, {code,13622682}, {ets,2214784}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,611328000}, {buffered_memory,99979264}, {free_memory,32551198720}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{364439,4999}}, {context_switches,{184006,0}}, {garbage_collection,{29782,100463052,0}}, {io,{{input,13671678},{output,8710323}}}, {reductions,{39307927,319472}}, {run_queue,0}, {runtime,{24890,400}}]}, {system_stats, [{cpu_utilization_rate,2.493765586034913}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,1068356}, {couch_docs_data_size,1054464}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,0}, {curr_items_tot,0}, {mem_used,4174248}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,364}, {memory_data,{33657663488,1106210816,{<18118.693.0>,1943840}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 31788404 kB\nBuffers: 97636 kB\nCached: 597000 kB\nSwapCached: 140 kB\nActive: 483784 kB\nInactive: 441816 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 31788404 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 32492 kB\nWriteback: 0 kB\nAnonPages: 231172 kB\nMapped: 17020 kB\nSlab: 114800 kB\nPageTables: 5244 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 1931976 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,906147,950430}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,76260072}, {processes,40135152}, {processes_used,40120480}, {system,36124920}, {atom,1352041}, {atom_used,1349085}, {binary,1363856}, {code,13631938}, {ets,2206456}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,649613312}, {buffered_memory,402055168}, {free_memory,32138895360}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{364397,4999}}, {context_switches,{183906,0}}, {garbage_collection,{30434,98774380,0}}, {io,{{input,13841244},{output,8939942}}}, {reductions,{38802187,398541}}, {run_queue,0}, {runtime,{24190,460}}]}, {system_stats, [{cpu_utilization_rate,3.9800995024875623}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,1068356}, {couch_docs_data_size,1054464}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,0}, {curr_items_tot,0}, {mem_used,4174248}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,364}, {memory_data,{33657663488,1519149056,{<18117.1691.0>,3143528}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 31385640 kB\nBuffers: 392632 kB\nCached: 634388 kB\nSwapCached: 0 kB\nActive: 731668 kB\nInactive: 520232 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 31385640 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 40452 kB\nWriteback: 0 kB\nAnonPages: 224936 kB\nMapped: 17020 kB\nSlab: 193868 kB\nPageTables: 5256 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 1933456 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,906146,302502}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,75810864}, {processes,39709304}, {processes_used,39695432}, {system,36101560}, {atom,1352041}, {atom_used,1349085}, {binary,1300264}, {code,13659922}, {ets,2224016}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,603295744}, {buffered_memory,298938368}, {free_memory,32279445504}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{361430,5000}}, {context_switches,{182584,0}}, {garbage_collection,{29722,99933953,0}}, {io,{{input,10770151},{output,6111113}}}, {reductions,{40103305,326947}}, {run_queue,0}, {runtime,{25240,460}}]}, {system_stats, [{cpu_utilization_rate,4.0}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,1068356}, {couch_docs_data_size,1054464}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,0}, {curr_items_tot,0}, {mem_used,4174248}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,361}, {memory_data,{33657663488,1378598912,{<0.694.0>,1572896}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 31523020 kB\nBuffers: 291932 kB\nCached: 589156 kB\nSwapCached: 12 kB\nActive: 531120 kB\nInactive: 576728 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 31523020 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 36468 kB\nWriteback: 104 kB\nAnonPages: 226636 kB\nMapped: 17164 kB\nSlab: 197948 kB\nPageTables: 5184 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 1930104 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-02T18:29:44.828,ns_1@10.2.1.68:<0.1669.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,31,46}}) Stats for bucket "default": accepting_conns 1 auth_cmds 2 auth_errors 0 bucket_active_conns 1 bucket_conns 5 bytes 4174248 bytes_read 24964 bytes_written 2954648 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 0 conn_yields 0 connection_structures 5000 curr_connections 9 curr_conns_on_port_11209 6 curr_conns_on_port_11210 1 curr_items 0 curr_items_tot 0 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-03 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 0 ep_commit_time 0 ep_commit_time_total 0 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 0 ep_data_age_highwat 0 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data/default ep_degraded_mode 0 ep_diskqueue_drain 0 ep_diskqueue_fill 0 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 0 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 0 ep_io_read_bytes 0 ep_io_write_bytes 0 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 0 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 0 ep_max_checkpoints 2 ep_max_data_size 5242880000 ep_max_item_size 20971520 ep_max_size 5242880000 ep_max_txn_size 10000 ep_max_vbuckets 512 ep_mem_high_wat 3932160000 ep_mem_high_wat 3932160000 ep_mem_low_wat 3145728000 ep_mem_low_wat 3145728000 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 3404160 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1351905984 ep_storage_age 0 ep_storage_age_highwat 0 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 0 ep_total_del_items 0 ep_total_enqueued 0 ep_total_new_items 0 ep_total_persisted 0 ep_uncommitted_items 0 ep_value_size 0 ep_vb0 0 ep_vb_snapshot_total 6 ep_vb_total 128 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_130_g8b09894 ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 741 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 4174248 pid 17027 pointer_size 64 rejected_conns 0 rusage_system 0.085986 rusage_user 0.352946 threads 4 time 1351906184 total_connections 12 uptime 215 vb_active_curr_items 0 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 3214336 vb_active_itm_memory 0 vb_active_meta_data_memory 0 vb_active_num 128 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 0 vb_active_ops_create 0 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 0 vb_active_queue_age 0 vb_active_queue_drain 0 vb_active_queue_fill 0 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_meta_data_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-02T18:30:11.301,ns_1@10.2.1.68:ns_doctor<0.1479.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,906206,336564}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,96121464}, {processes,57482064}, {processes_used,55904352}, {system,38639400}, {atom,1405017}, {atom_used,1377299}, {binary,2468840}, {code,14117455}, {ets,2608936}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,706899968}, {buffered_memory,173453312}, {free_memory,32306495488}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{422193,5001}}, {context_switches,{705291,0}}, {garbage_collection,{93783,374902576,0}}, {io,{{input,65273000},{output,19522819}}}, {reductions,{133985868,7345299}}, {run_queue,0}, {runtime,{46370,3450}}]}, {system_stats, [{cpu_utilization_rate,40.399002493765586}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,73495492}, {couch_docs_data_size,41798220}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,21085}, {curr_items_tot,21085}, {mem_used,60024376}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,422}, {memory_data,{33657663488,1311170560,{<18077.5273.0>,7628384}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 31549312 kB\nBuffers: 169388 kB\nCached: 690004 kB\nSwapCached: 0 kB\nActive: 672512 kB\nInactive: 492060 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 31549312 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 660 kB\nWriteback: 0 kB\nAnonPages: 304880 kB\nMapped: 15284 kB\nSlab: 118728 kB\nPageTables: 4796 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 2012164 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,906208,976697}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,85539480}, {processes,48917888}, {processes_used,48904416}, {system,36621592}, {atom,1352041}, {atom_used,1349412}, {binary,1238248}, {code,13622682}, {ets,2389880}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,746065920}, {buffered_memory,213938176}, {free_memory,32201248768}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{424439,4999}}, {context_switches,{585939,0}}, {garbage_collection,{82710,153361218,0}}, {io,{{input,16075049},{output,11399985}}}, {reductions,{59597622,8131634}}, {run_queue,0}, {runtime,{36730,3490}}]}, {system_stats, [{cpu_utilization_rate,38.75}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,75211716}, {couch_docs_data_size,42874575}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,29882}, {curr_items_tot,29882}, {mem_used,83319840}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,424}, {memory_data,{33657663488,1342517248,{<18118.1696.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 31446656 kB\nBuffers: 208924 kB\nCached: 728580 kB\nSwapCached: 140 kB\nActive: 779492 kB\nInactive: 488244 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 31446656 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 24640 kB\nWriteback: 0 kB\nAnonPages: 329952 kB\nMapped: 17056 kB\nSlab: 115440 kB\nPageTables: 5320 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 2038544 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,906207,950938}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,84937680}, {processes,47343504}, {processes_used,47312312}, {system,37594176}, {atom,1352041}, {atom_used,1349412}, {binary,2204552}, {code,13631938}, {ets,2381104}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,732807168}, {buffered_memory,403181568}, {free_memory,31956430848}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{424397,5000}}, {context_switches,{519413,0}}, {garbage_collection,{74668,145801107,0}}, {io,{{input,17960942},{output,11548698}}}, {reductions,{56176409,7330694}}, {run_queue,0}, {runtime,{34480,2980}}]}, {system_stats, [{cpu_utilization_rate,38.59649122807018}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,53285919}, {couch_docs_data_size,40781093}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,26755}, {curr_items_tot,26755}, {mem_used,74966336}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,424}, {memory_data,{33657663488,1636364288,{<18117.1691.0>,3143528}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 31207452 kB\nBuffers: 393732 kB\nCached: 715632 kB\nSwapCached: 0 kB\nActive: 864272 kB\nInactive: 566100 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 31207452 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 33776 kB\nWriteback: 0 kB\nAnonPages: 320832 kB\nMapped: 17056 kB\nSlab: 194640 kB\nPageTables: 5440 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 2027536 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,906206,302272}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,84634616}, {processes,47896632}, {processes_used,47883224}, {system,36737984}, {atom,1352041}, {atom_used,1349412}, {binary,1308520}, {code,13659922}, {ets,2401176}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,693288960}, {buffered_memory,299667456}, {free_memory,32109973504}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{421430,5001}}, {context_switches,{440447,0}}, {garbage_collection,{63735,138507403,0}}, {io,{{input,15668870},{output,11107415}}}, {reductions,{53811640,6258416}}, {run_queue,0}, {runtime,{35190,3170}}]}, {system_stats, [{cpu_utilization_rate,43.1077694235589}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,70931396}, {couch_docs_data_size,42005987}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,21322}, {curr_items_tot,21322}, {mem_used,60764504}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,421}, {memory_data,{33657663488,1525088256,{<0.1662.0>,1801552}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 31357396 kB\nBuffers: 292644 kB\nCached: 677040 kB\nSwapCached: 12 kB\nActive: 774008 kB\nInactive: 499876 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 31357396 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 28592 kB\nWriteback: 0 kB\nAnonPages: 303496 kB\nMapped: 17248 kB\nSlab: 198396 kB\nPageTables: 5372 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 2009460 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-02T18:31:11.301,ns_1@10.2.1.68:ns_doctor<0.1479.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,906266,336504}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,105275824}, {processes,65437440}, {processes_used,63859584}, {system,39838384}, {atom,1405017}, {atom_used,1377299}, {binary,3438712}, {code,14117455}, {ets,2790712}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,1163640832}, {buffered_memory,203055104}, {free_memory,31325478912}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{482193,5000}}, {context_switches,{1905835,0}}, {garbage_collection,{220690,559865977,0}}, {io,{{input,93203315},{output,26307466}}}, {reductions,{206571371,5697355}}, {run_queue,0}, {runtime,{80170,2680}}]}, {system_stats, [{cpu_utilization_rate,40.5}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,526517324}, {couch_docs_data_size,442932681}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,186643}, {curr_items_tot,186643}, {mem_used,497134488}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,482}, {memory_data,{33657663488,2292187136,{<18077.5273.0>,7628384}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 30591288 kB\nBuffers: 198296 kB\nCached: 1136368 kB\nSwapCached: 0 kB\nActive: 1326456 kB\nInactive: 792908 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 30591288 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 2224 kB\nWriteback: 0 kB\nAnonPages: 784376 kB\nMapped: 15308 kB\nSlab: 120420 kB\nPageTables: 5748 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 2497656 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,906268,978671}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,87573384}, {processes,49743448}, {processes_used,49711456}, {system,37829936}, {atom,1352041}, {atom_used,1349412}, {binary,2256552}, {code,13622682}, {ets,2572352}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,1134317568}, {buffered_memory,232042496}, {free_memory,31315718144}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{484440,4999}}, {context_switches,{1813438,0}}, {garbage_collection,{227129,304674961,0}}, {io,{{input,30653854},{output,17863287}}}, {reductions,{126805898,4978521}}, {run_queue,0}, {runtime,{66480,2520}}]}, {system_stats, [{cpu_utilization_rate,38.095238095238095}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,468218201}, {couch_docs_data_size,447197867}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,196903}, {curr_items_tot,196903}, {mem_used,523654024}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,484}, {memory_data,{33657663488,2259791872,{<18118.1696.0>,2914384}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 30582004 kB\nBuffers: 226604 kB\nCached: 1107732 kB\nSwapCached: 140 kB\nActive: 1262232 kB\nInactive: 866060 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 30582004 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 22656 kB\nWriteback: 0 kB\nAnonPages: 793440 kB\nMapped: 17016 kB\nSlab: 118496 kB\nPageTables: 6184 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 2498068 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,906267,952551}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,92620360}, {processes,54790600}, {processes_used,54758608}, {system,37829760}, {atom,1352041}, {atom_used,1349412}, {binary,2256632}, {code,13631938}, {ets,2564088}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,1307541504}, {buffered_memory,404893696}, {free_memory,30895165440}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{484398,5000}}, {context_switches,{1669360,0}}, {garbage_collection,{196919,288146655,0}}, {io,{{input,27054859},{output,17158092}}}, {reductions,{119555037,4747693}}, {run_queue,0}, {runtime,{64430,2590}}]}, {system_stats, [{cpu_utilization_rate,35.57213930348259}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,612049991}, {couch_docs_data_size,447401356}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,193255}, {curr_items_tot,193255}, {mem_used,514472232}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,484}, {memory_data,{33657663488,2678312960,{<18117.1681.0>,4714984}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 30171060 kB\nBuffers: 395404 kB\nCached: 1276896 kB\nSwapCached: 0 kB\nActive: 1627992 kB\nInactive: 837456 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 30171060 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 24824 kB\nWriteback: 164 kB\nAnonPages: 792984 kB\nMapped: 17056 kB\nSlab: 194784 kB\nPageTables: 6388 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 2501840 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,906266,303025}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,90399872}, {processes,52346712}, {processes_used,52314640}, {system,38053160}, {atom,1352041}, {atom_used,1349412}, {binary,2435744}, {code,13659922}, {ets,2583736}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,1190948864}, {buffered_memory,302379008}, {free_memory,31116767232}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{481430,5000}}, {context_switches,{1677519,0}}, {garbage_collection,{202818,290237332,0}}, {io,{{input,28490140},{output,17439558}}}, {reductions,{121192023,4778458}}, {run_queue,0}, {runtime,{63790,2080}}]}, {system_stats, [{cpu_utilization_rate,35.25}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,564446279}, {couch_docs_data_size,445796474}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,188042}, {curr_items_tot,188042}, {mem_used,500604072}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,481}, {memory_data,{33657663488,2513596416,{<0.1663.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 30387592 kB\nBuffers: 295292 kB\nCached: 1163036 kB\nSwapCached: 12 kB\nActive: 1445660 kB\nInactive: 795980 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 30387592 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 21080 kB\nWriteback: 0 kB\nAnonPages: 783384 kB\nMapped: 17256 kB\nSlab: 199112 kB\nPageTables: 6312 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 2494828 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-02T18:31:24.827,ns_1@10.2.1.68:<0.1669.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,31,46}}) Stats for bucket "default": accepting_conns 1 auth_cmds 3 auth_errors 0 bucket_active_conns 1 bucket_conns 6 bytes 640066408 bytes_read 576402554 bytes_written 14281830 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 240267 conn_yields 63 connection_structures 5000 curr_connections 10 curr_conns_on_port_11209 6 curr_conns_on_port_11210 2 curr_items 240267 curr_items_tot 240267 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-03 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 484 ep_commit_time 2 ep_commit_time_total 438 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 7 ep_data_age_highwat 7 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data/default ep_degraded_mode 0 ep_diskqueue_drain 222557 ep_diskqueue_fill 240267 ep_diskqueue_items 17711 ep_diskqueue_memory 566720 ep_diskqueue_pending 850080 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 80 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 175 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 222551 ep_io_read_bytes 0 ep_io_write_bytes 526675294 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 0 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 587505809 ep_max_checkpoints 2 ep_max_data_size 5242880000 ep_max_item_size 20971520 ep_max_size 5242880000 ep_max_txn_size 10000 ep_max_vbuckets 512 ep_mem_high_wat 3932160000 ep_mem_high_wat 3932160000 ep_mem_low_wat 3145728000 ep_mem_low_wat 3145728000 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 24549128 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 17536 ep_startup_time 1351905984 ep_storage_age 7 ep_storage_age_highwat 7 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 584622605 ep_total_del_items 0 ep_total_enqueued 240267 ep_total_new_items 222551 ep_total_persisted 222551 ep_uncommitted_items 476 ep_value_size 567717450 ep_vb0 0 ep_vb_snapshot_total 6 ep_vb_total 128 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_130_g8b09894 ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 741 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 640066408 pid 17027 pointer_size 64 rejected_conns 0 rusage_system 13.661923 rusage_user 32.601043 threads 4 time 1351906284 total_connections 13 uptime 315 vb_active_curr_items 240267 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 3214336 vb_active_itm_memory 584622605 vb_active_meta_data_memory 18981093 vb_active_num 128 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 240267 vb_active_ops_create 222556 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 100 vb_active_queue_age 61854000 vb_active_queue_drain 222557 vb_active_queue_fill 240267 vb_active_queue_memory 566720 vb_active_queue_pending 850080 vb_active_queue_size 17710 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_meta_data_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-02T18:32:11.301,ns_1@10.2.1.68:ns_doctor<0.1479.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,906326,336883}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,106059176}, {processes,65455408}, {processes_used,63877392}, {system,40603768}, {atom,1405017}, {atom_used,1377299}, {binary,4029608}, {code,14117455}, {ets,2969096}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,1825570816}, {buffered_memory,204267520}, {free_memory,30169489408}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{542194,5001}}, {context_switches,{2445820,0}}, {garbage_collection,{273946,679812144,0}}, {io,{{input,114446485},{output,31228687}}}, {reductions,{252910109,3511856}}, {run_queue,0}, {runtime,{97300,930}}]}, {system_stats, [{cpu_utilization_rate,31.920199501246884}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,1185596638}, {couch_docs_data_size,847789728}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,352756}, {curr_items_tot,352756}, {mem_used,935866024}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,542}, {memory_data,{33657663488,3439034368,{<18077.5273.0>,7628384}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29462516 kB\nBuffers: 199480 kB\nCached: 1782784 kB\nSwapCached: 0 kB\nActive: 2187832 kB\nInactive: 1044016 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29462516 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 1800 kB\nWriteback: 0 kB\nAnonPages: 1249708 kB\nMapped: 15308 kB\nSlab: 136584 kB\nPageTables: 6676 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 2959240 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,906328,981015}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,92097016}, {processes,54041016}, {processes_used,54008864}, {system,38056000}, {atom,1352041}, {atom_used,1349412}, {binary,2304168}, {code,13622682}, {ets,2750224}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,1813581824}, {buffered_memory,233742336}, {free_memory,30140559360}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{544441,5001}}, {context_switches,{2417123,0}}, {garbage_collection,{290009,397203012,0}}, {io,{{input,37399485},{output,22314624}}}, {reductions,{169154320,2506934}}, {run_queue,0}, {runtime,{84010,690}}]}, {system_stats, [{cpu_utilization_rate,29.64824120603015}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,1136452829}, {couch_docs_data_size,838761910}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,363675}, {curr_items_tot,363675}, {mem_used,964413832}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,544}, {memory_data,{33657663488,3417178112,{<18118.1697.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29434388 kB\nBuffers: 228264 kB\nCached: 1771076 kB\nSwapCached: 140 kB\nActive: 2158376 kB\nInactive: 1100300 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29434388 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 12724 kB\nWriteback: 0 kB\nAnonPages: 1258820 kB\nMapped: 17056 kB\nSlab: 134912 kB\nPageTables: 7160 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 2970624 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,906327,957203}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.7032.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,97}, {started_on,1351906300}, {total_vbuckets,129}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351906335}, {vbuckets_done,126}]]}, {memory, [{total,89279104}, {processes,50979024}, {processes_used,50927064}, {system,38300080}, {atom,1352041}, {atom_used,1349412}, {binary,2534904}, {code,13631938}, {ets,2754080}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,1423003648}, {buffered_memory,406638592}, {free_memory,30307508224}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{544402,5004}}, {context_switches,{1924377,0}}, {garbage_collection,{225523,352278047,0}}, {io,{{input,35548421},{output,21032247}}}, {reductions,{150296828,2245519}}, {run_queue,0}, {runtime,{73520,520}}]}, {system_stats, [{cpu_utilization_rate,15.960099750623442}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,786546006}, {couch_docs_data_size,739147417}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,359922}, {curr_items_tot,359922}, {mem_used,955342264}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,544}, {memory_data,{33657663488,3337601024,{<18117.1691.0>,3143528}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29597424 kB\nBuffers: 397108 kB\nCached: 1389324 kB\nSwapCached: 0 kB\nActive: 1784644 kB\nInactive: 1251644 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29597424 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 17628 kB\nWriteback: 0 kB\nAnonPages: 1250008 kB\nMapped: 17056 kB\nSlab: 195408 kB\nPageTables: 7280 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 2962912 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,906326,303218}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,91805400}, {processes,53454664}, {processes_used,53422192}, {system,38350736}, {atom,1352041}, {atom_used,1349412}, {binary,2546624}, {code,13659922}, {ets,2758040}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,1808289792}, {buffered_memory,303034368}, {free_memory,30027182080}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{541430,5000}}, {context_switches,{2073981,0}}, {garbage_collection,{242510,364004181,0}}, {io,{{input,30837000},{output,19956841}}}, {reductions,{155926835,2844168}}, {run_queue,0}, {runtime,{75410,990}}]}, {system_stats, [{cpu_utilization_rate,29.42643391521197}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,1188455065}, {couch_docs_data_size,853770390}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,354230}, {curr_items_tot,354230}, {mem_used,939123000}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,541}, {memory_data,{33657663488,3615244288,{<0.1662.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29323544 kB\nBuffers: 295932 kB\nCached: 1765908 kB\nSwapCached: 12 kB\nActive: 2226088 kB\nInactive: 1079244 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29323544 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 16876 kB\nWriteback: 200 kB\nAnonPages: 1243612 kB\nMapped: 17256 kB\nSlab: 199020 kB\nPageTables: 7100 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 2955928 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-02T18:33:04.826,ns_1@10.2.1.68:<0.1669.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,31,46}}) Stats for bucket "default": accepting_conns 1 auth_cmds 3 auth_errors 0 bucket_active_conns 1 bucket_conns 6 bytes 1378842584 bytes_read 1246350255 bytes_written 27316188 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 519326 conn_yields 181 connection_structures 5000 curr_connections 10 curr_conns_on_port_11209 6 curr_conns_on_port_11210 2 curr_items 519326 curr_items_tot 519326 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-03 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 611 ep_commit_time 10 ep_commit_time_total 1033 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 15 ep_data_age_highwat 19 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data/default ep_degraded_mode 0 ep_diskqueue_drain 482133 ep_diskqueue_fill 519326 ep_diskqueue_items 37194 ep_diskqueue_memory 1190176 ep_diskqueue_pending 1785264 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 181 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 21282 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 481873 ep_io_read_bytes 0 ep_io_write_bytes 1140401217 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 0 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 1270416458 ep_max_checkpoints 2 ep_max_data_size 5242880000 ep_max_item_size 20971520 ep_max_size 5242880000 ep_max_txn_size 10000 ep_max_vbuckets 512 ep_mem_high_wat 3932160000 ep_mem_high_wat 3932160000 ep_mem_low_wat 3145728000 ep_mem_low_wat 3145728000 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 49262888 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 15912 ep_startup_time 1351905984 ep_storage_age 13 ep_storage_age_highwat 17 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 1264184546 ep_total_del_items 0 ep_total_enqueued 519326 ep_total_new_items 481873 ep_total_persisted 481873 ep_uncommitted_items 9348 ep_value_size 1227608537 ep_vb0 0 ep_vb_snapshot_total 6 ep_vb_total 128 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_130_g8b09894 ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 741 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 1378842584 pid 17027 pointer_size 64 rejected_conns 0 rusage_system 24.752237 rusage_user 60.859747 threads 4 time 1351906384 total_connections 13 uptime 415 vb_active_curr_items 519326 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 3202048 vb_active_itm_memory 1264184546 vb_active_meta_data_memory 41026754 vb_active_num 128 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 519326 vb_active_ops_create 482132 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 100 vb_active_queue_age 295552000 vb_active_queue_drain 482133 vb_active_queue_fill 519326 vb_active_queue_memory 1190176 vb_active_queue_pending 1785264 vb_active_queue_size 37193 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_meta_data_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-02T18:33:11.301,ns_1@10.2.1.68:ns_doctor<0.1479.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,906386,335031}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.12892.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,44}, {started_on,1351906406}, {total_vbuckets,129}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351906425}, {vbuckets_done,58}]]}, {memory, [{total,103440168}, {processes,62131272}, {processes_used,60566528}, {system,41308896}, {atom,1405017}, {atom_used,1377299}, {binary,4544680}, {code,14117455}, {ets,3160240}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,2099789824}, {buffered_memory,205864960}, {free_memory,29420449792}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{602193,5001}}, {context_switches,{2672326,0}}, {garbage_collection,{297533,772773548,0}}, {io,{{input,133605445},{output,34110639}}}, {reductions,{288406163,2718614}}, {run_queue,0}, {runtime,{106800,560}}]}, {system_stats, [{cpu_utilization_rate,34.1645885286783}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,1504748749}, {couch_docs_data_size,1180697327}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,520748}, {curr_items_tot,520748}, {mem_used,1381342888}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,602}, {memory_data,{33657663488,4250546176,{<18077.5273.0>,7628384}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 28730908 kB\nBuffers: 201040 kB\nCached: 2050576 kB\nSwapCached: 0 kB\nActive: 2513160 kB\nInactive: 1440324 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 28730908 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 2548 kB\nWriteback: 3280 kB\nAnonPages: 1702212 kB\nMapped: 15312 kB\nSlab: 144912 kB\nPageTables: 7636 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 3414760 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,906388,979724}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,88823984}, {processes,50438792}, {processes_used,50406320}, {system,38385192}, {atom,1352041}, {atom_used,1349412}, {binary,2435832}, {code,13622682}, {ets,2933176}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,2442096640}, {buffered_memory,234389504}, {free_memory,29016059904}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{604439,4999}}, {context_switches,{2695119,0}}, {garbage_collection,{318221,460513587,0}}, {io,{{input,39189838},{output,24448213}}}, {reductions,{199550802,2712048}}, {run_queue,0}, {runtime,{92480,690}}]}, {system_stats, [{cpu_utilization_rate,24.12060301507538}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,1750332673}, {couch_docs_data_size,1243308660}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,531404}, {curr_items_tot,531404}, {mem_used,1409121192}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,604}, {memory_data,{33657663488,4526817280,{<18118.1696.0>,2914384}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 28335996 kB\nBuffers: 228896 kB\nCached: 2384860 kB\nSwapCached: 140 kB\nActive: 2923532 kB\nInactive: 1417392 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 28335996 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 1828 kB\nWriteback: 0 kB\nAnonPages: 1726836 kB\nMapped: 17056 kB\nSlab: 149844 kB\nPageTables: 8176 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 3437740 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,906387,953227}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,91417688}, {processes,52983976}, {processes_used,52915968}, {system,38433712}, {atom,1352041}, {atom_used,1349412}, {binary,2488304}, {code,13631938}, {ets,2924520}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,2297012224}, {buffered_memory,407732224}, {free_memory,28952174592}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{604398,5000}}, {context_switches,{2630470,0}}, {garbage_collection,{299237,453427360,0}}, {io,{{input,39336455},{output,24647610}}}, {reductions,{196488002,2394123}}, {run_queue,0}, {runtime,{91760,670}}]}, {system_stats, [{cpu_utilization_rate,29.25}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,1563104514}, {couch_docs_data_size,1223151488}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,527532}, {curr_items_tot,527532}, {mem_used,1398470152}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,604}, {memory_data,{33657663488,4586385408,{<18117.1691.0>,3143528}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 28273608 kB\nBuffers: 398176 kB\nCached: 2243176 kB\nSwapCached: 0 kB\nActive: 2791716 kB\nInactive: 1569348 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 28273608 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 13396 kB\nWriteback: 0 kB\nAnonPages: 1719264 kB\nMapped: 17056 kB\nSlab: 195044 kB\nPageTables: 8204 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 3428856 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,906386,303176}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.7411.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,54}, {started_on,1351906361}, {total_vbuckets,129}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351906385}, {vbuckets_done,70}]]}, {memory, [{total,89364176}, {processes,50780704}, {processes_used,50758944}, {system,38583472}, {atom,1352041}, {atom_used,1349412}, {binary,2536920}, {code,13659922}, {ets,2968936}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,2029617152}, {buffered_memory,304832512}, {free_memory,29341503488}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{601430,5000}}, {context_switches,{2376273,0}}, {garbage_collection,{274721,431781238,0}}, {io,{{input,40225211},{output,23936895}}}, {reductions,{188082670,2030805}}, {run_queue,0}, {runtime,{85230,410}}]}, {system_stats, [{cpu_utilization_rate,15.538847117794486}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,1430930306}, {couch_docs_data_size,1169392956}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,522212}, {curr_items_tot,522212}, {mem_used,1385343512}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,601}, {memory_data,{33657663488,4310065152,{<0.1662.0>,2914384}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 28653936 kB\nBuffers: 297688 kB\nCached: 1982048 kB\nSwapCached: 12 kB\nActive: 2507292 kB\nInactive: 1465644 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 28653936 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 17288 kB\nWriteback: 1516 kB\nAnonPages: 1693452 kB\nMapped: 17260 kB\nSlab: 199872 kB\nPageTables: 7884 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 3401360 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-02T18:34:11.301,ns_1@10.2.1.68:ns_doctor<0.1479.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,906446,336353}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,105274344}, {processes,62637440}, {processes_used,61059504}, {system,42636904}, {atom,1405017}, {atom_used,1377299}, {binary,5698712}, {code,14117455}, {ets,3326224}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,2525442048}, {buffered_memory,207634432}, {free_memory,28551180288}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{662195,5000}}, {context_switches,{3209904,0}}, {garbage_collection,{351729,894776851,0}}, {io,{{input,154956246},{output,38407091}}}, {reductions,{335384090,5268096}}, {run_queue,0}, {runtime,{123000,1820}}]}, {system_stats, [{cpu_utilization_rate,31.155778894472363}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,1887704357}, {couch_docs_data_size,1660261228}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,688585}, {curr_items_tot,688585}, {mem_used,1722363224}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,662}, {memory_data,{33657663488,5070548992,{<18077.5273.0>,7628384}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 27882136 kB\nBuffers: 202768 kB\nCached: 2466252 kB\nSwapCached: 0 kB\nActive: 2882264 kB\nInactive: 1910436 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 27882136 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 1112 kB\nWriteback: 316 kB\nAnonPages: 2123456 kB\nMapped: 15308 kB\nSlab: 154168 kB\nPageTables: 8420 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 3836936 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,906448,979260}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.7807.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,32}, {started_on,1351906407}, {total_vbuckets,129}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351906421}, {vbuckets_done,42}]]}, {memory, [{total,89319288}, {processes,50779960}, {processes_used,50764840}, {system,38539328}, {atom,1352041}, {atom_used,1349412}, {binary,2397872}, {code,13622682}, {ets,3120912}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,2788012032}, {buffered_memory,235872256}, {free_memory,28219531264}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{664439,4999}}, {context_switches,{2972349,0}}, {garbage_collection,{347377,524684024,0}}, {io,{{input,45277559},{output,27146509}}}, {reductions,{230452775,2350474}}, {run_queue,0}, {runtime,{101130,490}}]}, {system_stats, [{cpu_utilization_rate,36.75}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,2233066969}, {couch_docs_data_size,1638142908}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,699014}, {curr_items_tot,699014}, {mem_used,1750457240}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,664}, {memory_data,{33657663488,5473685504,{<18118.1696.0>,2914384}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 27558260 kB\nBuffers: 230344 kB\nCached: 2722668 kB\nSwapCached: 140 kB\nActive: 3291336 kB\nInactive: 1816212 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 27558260 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 13056 kB\nWriteback: 0 kB\nAnonPages: 2154204 kB\nMapped: 17060 kB\nSlab: 160260 kB\nPageTables: 9108 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 3865280 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,906447,953113}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,92815400}, {processes,54222168}, {processes_used,54153696}, {system,38593232}, {atom,1352041}, {atom_used,1349412}, {binary,2461184}, {code,13631938}, {ets,3098824}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,2939936768}, {buffered_memory,408408064}, {free_memory,27872247808}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{664397,4999}}, {context_switches,{2895943,0}}, {garbage_collection,{326383,515797278,0}}, {io,{{input,41090443},{output,26738491}}}, {reductions,{226575917,2904735}}, {run_queue,0}, {runtime,{100040,900}}]}, {system_stats, [{cpu_utilization_rate,7.5},{swap_total,10701078528},{swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,2240009485}, {couch_docs_data_size,1662030474}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,695837}, {curr_items_tot,695837}, {mem_used,1740613320}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,664}, {memory_data,{33657663488,5682946048,{<18117.1691.0>,3143528}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 27218992 kB\nBuffers: 398836 kB\nCached: 2871032 kB\nSwapCached: 0 kB\nActive: 3527772 kB\nInactive: 1886672 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 27218992 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 1160 kB\nWriteback: 4 kB\nAnonPages: 2144632 kB\nMapped: 17056 kB\nSlab: 194812 kB\nPageTables: 9040 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 3855224 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,906446,303035}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,93575904}, {processes,54580992}, {processes_used,54514760}, {system,38994912}, {atom,1352041}, {atom_used,1349412}, {binary,2798088}, {code,13659922}, {ets,3118744}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,2536103936}, {buffered_memory,306479104}, {free_memory,28396294144}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{661430,4999}}, {context_switches,{2978530,0}}, {garbage_collection,{337710,524740456,0}}, {io,{{input,47301148},{output,27895919}}}, {reductions,{230962393,3299602}}, {run_queue,0}, {runtime,{100910,1160}}]}, {system_stats, [{cpu_utilization_rate,8.0}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,1932835129}, {couch_docs_data_size,1659780348}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,690079}, {curr_items_tot,690079}, {mem_used,1725606136}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,661}, {memory_data,{33657663488,5251465216,{<0.1662.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 27730880 kB\nBuffers: 299296 kB\nCached: 2476664 kB\nSwapCached: 12 kB\nActive: 2994096 kB\nInactive: 1901764 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 27730880 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 1364 kB\nWriteback: 220 kB\nAnonPages: 2119484 kB\nMapped: 17256 kB\nSlab: 199468 kB\nPageTables: 8828 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 3824468 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-02T18:34:44.826,ns_1@10.2.1.68:<0.1669.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,31,46}}) Stats for bucket "default": accepting_conns 1 auth_cmds 3 auth_errors 0 bucket_active_conns 1 bucket_conns 6 bytes 2013864616 bytes_read 1917333723 bytes_written 40426030 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 799359 conn_yields 258 connection_structures 5000 curr_connections 10 curr_conns_on_port_11209 6 curr_conns_on_port_11210 2 curr_items 799359 curr_items_tot 799359 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-03 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 863 ep_commit_time 2 ep_commit_time_total 1971 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 3 ep_data_age_highwat 30 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data/default ep_degraded_mode 0 ep_diskqueue_drain 798686 ep_diskqueue_fill 799359 ep_diskqueue_items 674 ep_diskqueue_memory 21536 ep_diskqueue_pending 32304 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 284 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 218 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 798677 ep_io_read_bytes 0 ep_io_write_bytes 1890130505 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 640256 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 1954409253 ep_max_checkpoints 2 ep_max_data_size 5242880000 ep_max_item_size 20971520 ep_max_size 5242880000 ep_max_txn_size 10000 ep_max_vbuckets 512 ep_mem_high_wat 3932160000 ep_mem_high_wat 3932160000 ep_mem_low_wat 3145728000 ep_mem_low_wat 3145728000 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 17638752 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 456 ep_startup_time 1351905984 ep_storage_age 1 ep_storage_age_highwat 28 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 1944816945 ep_total_del_items 0 ep_total_enqueued 799359 ep_total_new_items 798677 ep_total_persisted 798677 ep_uncommitted_items 447 ep_value_size 1888500327 ep_vb0 0 ep_vb_snapshot_total 52 ep_vb_total 128 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_130_g8b09894 ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 741 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 2013864616 pid 17027 pointer_size 64 rejected_conns 0 rusage_system 38.630127 rusage_user 99.492874 threads 4 time 1351906484 total_connections 13 uptime 515 vb_active_curr_items 799359 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 3423520 vb_active_itm_memory 1944816945 vb_active_meta_data_memory 63149361 vb_active_num 128 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 799359 vb_active_ops_create 798685 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 100 vb_active_queue_age 457000 vb_active_queue_drain 798686 vb_active_queue_fill 799359 vb_active_queue_memory 21536 vb_active_queue_pending 32304 vb_active_queue_size 673 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_meta_data_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-02T18:35:11.301,ns_1@10.2.1.68:ns_doctor<0.1479.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,906506,334465}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,103708200}, {processes,60747976}, {processes_used,59169264}, {system,42960224}, {atom,1405017}, {atom_used,1377299}, {binary,5845552}, {code,14117455}, {ets,3503736}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,3216293888}, {buffered_memory,208416768}, {free_memory,27412332544}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{722193,5000}}, {context_switches,{3597370,0}}, {garbage_collection,{390554,1002149773,0}}, {io,{{input,175555409},{output,44727055}}}, {reductions,{376289563,3365903}}, {run_queue,0}, {runtime,{136470,900}}]}, {system_stats, [{cpu_utilization_rate,29.072681704260653}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,2551811257}, {couch_docs_data_size,2054261670}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,855792}, {curr_items_tot,855792}, {mem_used,2164403368}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,722}, {memory_data,{33657663488,6174351360,{<18077.5273.0>,7628384}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 26769860 kB\nBuffers: 203532 kB\nCached: 3140912 kB\nSwapCached: 0 kB\nActive: 3686204 kB\nInactive: 2200768 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 26769860 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 1368 kB\nWriteback: 0 kB\nAnonPages: 2542368 kB\nMapped: 15312 kB\nSlab: 170800 kB\nPageTables: 9228 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 4256308 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,906508,981430}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,92015920}, {processes,53268184}, {processes_used,53202992}, {system,38747736}, {atom,1352041}, {atom_used,1349412}, {binary,2437768}, {code,13622682}, {ets,3285168}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,2924797952}, {buffered_memory,238051328}, {free_memory,27654115328}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{724440,5000}}, {context_switches,{3436217,0}}, {garbage_collection,{397560,606299394,0}}, {io,{{input,56183656},{output,33928810}}}, {reductions,{268523327,6103913}}, {run_queue,0}, {runtime,{114210,2510}}]}, {system_stats, [{cpu_utilization_rate,38.847117794486216}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,2246382884}, {couch_docs_data_size,2072616446}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,867047}, {curr_items_tot,867047}, {mem_used,2194530512}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,724}, {memory_data,{33657663488,5915680768,{<18118.1697.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 27006096 kB\nBuffers: 232472 kB\nCached: 2856248 kB\nSwapCached: 140 kB\nActive: 3300652 kB\nInactive: 2356064 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 27006096 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 2092 kB\nWriteback: 0 kB\nAnonPages: 2567792 kB\nMapped: 17056 kB\nSlab: 162416 kB\nPageTables: 9852 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 4270536 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,906507,955286}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,93668048}, {processes,54774272}, {processes_used,54705800}, {system,38893776}, {atom,1352041}, {atom_used,1349412}, {binary,2584696}, {code,13631938}, {ets,3276456}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,3651858432}, {buffered_memory,409120768}, {free_memory,26732765184}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{724399,5000}}, {context_switches,{3253950,0}}, {garbage_collection,{363313,586969820,0}}, {io,{{input,46651094},{output,32292536}}}, {reductions,{260098447,2778891}}, {run_queue,0}, {runtime,{110130,770}}]}, {system_stats, [{cpu_utilization_rate,8.229426433915211}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,2939548953}, {couch_docs_data_size,2068538567}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,863882}, {curr_items_tot,863882}, {mem_used,2185355800}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,724}, {memory_data,{33657663488,6813667328,{<18117.1681.0>,4714984}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 26106340 kB\nBuffers: 399532 kB\nCached: 3566268 kB\nSwapCached: 0 kB\nActive: 4338528 kB\nInactive: 2186824 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 26106340 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 1000 kB\nWriteback: 0 kB\nAnonPages: 2559404 kB\nMapped: 17056 kB\nSlab: 196388 kB\nPageTables: 9860 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 4274164 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,906506,303243}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,93228424}, {processes,54130920}, {processes_used,54065008}, {system,39097504}, {atom,1352041}, {atom_used,1349412}, {binary,2724512}, {code,13659922}, {ets,3293784}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,3316211712}, {buffered_memory,307511296}, {free_memory,27189895168}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{721431,5002}}, {context_switches,{3484528,0}}, {garbage_collection,{389236,608227645,0}}, {io,{{input,50126318},{output,30783782}}}, {reductions,{269717939,3842107}}, {run_queue,0}, {runtime,{113340,1160}}]}, {system_stats, [{cpu_utilization_rate,30.92269326683292}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,2687010075}, {couch_docs_data_size,2078633507}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,858288}, {curr_items_tot,858288}, {mem_used,2169692936}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,721}, {memory_data,{33657663488,6442881024,{<0.1662.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 26552756 kB\nBuffers: 300304 kB\nCached: 3238488 kB\nSwapCached: 12 kB\nActive: 3899212 kB\nInactive: 2174116 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 26552756 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 1344 kB\nWriteback: 0 kB\nAnonPages: 2534680 kB\nMapped: 17256 kB\nSlab: 199436 kB\nPageTables: 9948 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 4242088 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-02T18:36:11.301,ns_1@10.2.1.68:ns_doctor<0.1479.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,906566,333657}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,107287424}, {processes,64519088}, {processes_used,62940912}, {system,42768336}, {atom,1405017}, {atom_used,1377299}, {binary,5578112}, {code,14117455}, {ets,3578296}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,3903700992}, {buffered_memory,209543168}, {free_memory,26280845312}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{782193,5000}}, {context_switches,{3888931,0}}, {garbage_collection,{419433,1100409251,0}}, {io,{{input,192332119},{output,47485452}}}, {reductions,{413686900,2856316}}, {run_queue,0}, {runtime,{147000,600}}]}, {system_stats, [{cpu_utilization_rate,26.683291770573565}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,3235507130}, {couch_docs_data_size,2461537180}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1023403}, {curr_items_tot,1023403}, {mem_used,2610677832}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,782}, {memory_data,{33657663488,7312441344,{<18077.5273.0>,7628384}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 25664888 kB\nBuffers: 204632 kB\nCached: 3812208 kB\nSwapCached: 0 kB\nActive: 4453908 kB\nInactive: 2521228 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 25664888 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 1580 kB\nWriteback: 0 kB\nAnonPages: 2957740 kB\nMapped: 15312 kB\nSlab: 187248 kB\nPageTables: 10052 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 4668888 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,906568,981743}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,90628104}, {processes,51665952}, {processes_used,51600120}, {system,38962152}, {atom,1352041}, {atom_used,1349412}, {binary,2582792}, {code,13622682}, {ets,3358416}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,3598045184}, {buffered_memory,238694400}, {free_memory,26541293568}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{784440,5000}}, {context_switches,{3854122,0}}, {garbage_collection,{441156,682059191,0}}, {io,{{input,58586875},{output,36495078}}}, {reductions,{304074617,2662524}}, {run_queue,0}, {runtime,{126430,830}}]}, {system_stats, [{cpu_utilization_rate,31.75}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,2918057266}, {couch_docs_data_size,2469685410}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1034836}, {curr_items_tot,1034836}, {mem_used,2639138856}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,784}, {memory_data,{33657663488,7018852352,{<18118.1696.0>,2914384}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 25919356 kB\nBuffers: 233100 kB\nCached: 3513716 kB\nSwapCached: 140 kB\nActive: 4086512 kB\nInactive: 2639920 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 25919356 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 848 kB\nWriteback: 100 kB\nAnonPages: 2979420 kB\nMapped: 17056 kB\nSlab: 178496 kB\nPageTables: 10648 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 4688008 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,906567,954216}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.9566.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,59}, {started_on,1351906515}, {total_vbuckets,129}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351906574}, {vbuckets_done,77}]]}, {memory, [{total,93544624}, {processes,54554352}, {processes_used,54515112}, {system,38990272}, {atom,1352041}, {atom_used,1349412}, {binary,2578416}, {code,13631938}, {ets,3372216}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,3290853376}, {buffered_memory,410836992}, {free_memory,26657079296}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{784397,4996}}, {context_switches,{3328525,0}}, {garbage_collection,{372086,634320725,0}}, {io,{{input,57681398},{output,35461063}}}, {reductions,{284418782,1935211}}, {run_queue,0}, {runtime,{115330,450}}]}, {system_stats, [{cpu_utilization_rate,6.982543640897756}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,2653288194}, {couch_docs_data_size,2224206412}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1031983}, {curr_items_tot,1031983}, {mem_used,2631779144}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,784}, {memory_data,{33657663488,7006171136,{<18117.1681.0>,4714984}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 26032304 kB\nBuffers: 401208 kB\nCached: 3213724 kB\nSwapCached: 0 kB\nActive: 4046484 kB\nInactive: 2549144 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 26032304 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 1976 kB\nWriteback: 16068 kB\nAnonPages: 2980792 kB\nMapped: 17060 kB\nSlab: 198468 kB\nPageTables: 10752 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 4696020 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,906566,303061}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,94997872}, {processes,55666280}, {processes_used,55600368}, {system,39331592}, {atom,1352041}, {atom_used,1349412}, {binary,2782232}, {code,13659922}, {ets,3470480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,4094836736}, {buffered_memory,308416512}, {free_memory,25972830208}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{781430,4999}}, {context_switches,{3942996,0}}, {garbage_collection,{436080,687985725,0}}, {io,{{input,56190303},{output,36744597}}}, {reductions,{306837533,2980265}}, {run_queue,0}, {runtime,{123970,810}}]}, {system_stats, [{cpu_utilization_rate,29.75}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,3459724328}, {couch_docs_data_size,2491337756}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1025969}, {curr_items_tot,1025969}, {mem_used,2614336536}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,781}, {memory_data,{33657663488,7659057152,{<0.1662.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 25364340 kB\nBuffers: 301188 kB\nCached: 3998864 kB\nSwapCached: 12 kB\nActive: 4807184 kB\nInactive: 2442964 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 25364340 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 2060 kB\nWriteback: 0 kB\nAnonPages: 2949920 kB\nMapped: 17256 kB\nSlab: 209904 kB\nPageTables: 10608 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 4657828 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-02T18:36:24.826,ns_1@10.2.1.68:<0.1669.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,31,46}}) Stats for bucket "default": accepting_conns 1 auth_cmds 3 auth_errors 0 bucket_active_conns 1 bucket_conns 6 bytes 2753720808 bytes_read 2588491224 bytes_written 53532043 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 1078888 conn_yields 347 connection_structures 5000 curr_connections 10 curr_conns_on_port_11209 6 curr_conns_on_port_11210 2 curr_items 1078888 curr_items_tot 1078888 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-03 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 1065 ep_commit_time 1 ep_commit_time_total 2525 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 2 ep_data_age_highwat 30 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data/default ep_degraded_mode 0 ep_diskqueue_drain 1076498 ep_diskqueue_fill 1078888 ep_diskqueue_items 2391 ep_diskqueue_memory 76480 ep_diskqueue_pending 114720 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 384 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 149 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 1076494 ep_io_read_bytes 0 ep_io_write_bytes 2548315937 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 640256 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 2638553808 ep_max_checkpoints 2 ep_max_data_size 5242880000 ep_max_item_size 20971520 ep_max_size 5242880000 ep_max_txn_size 10000 ep_max_vbuckets 512 ep_mem_high_wat 3932160000 ep_mem_high_wat 3932160000 ep_mem_low_wat 3145728000 ep_mem_low_wat 3145728000 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 45165104 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 2242 ep_startup_time 1351905984 ep_storage_age 0 ep_storage_age_highwat 28 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 2625607152 ep_total_del_items 0 ep_total_enqueued 1078888 ep_total_new_items 1076494 ep_total_persisted 1076494 ep_uncommitted_items 520 ep_value_size 2549584294 ep_vb0 0 ep_vb_snapshot_total 52 ep_vb_total 128 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_130_g8b09894 ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 741 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 2753720808 pid 17027 pointer_size 64 rejected_conns 0 rusage_system 52.712986 rusage_user 140.084703 threads 4 time 1351906584 total_connections 13 uptime 615 vb_active_curr_items 1078888 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 6351872 vb_active_itm_memory 2625607152 vb_active_meta_data_memory 85232152 vb_active_num 128 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1078888 vb_active_ops_create 1076497 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 100 vb_active_queue_age 2375000 vb_active_queue_drain 1076498 vb_active_queue_fill 1078888 vb_active_queue_memory 76480 vb_active_queue_pending 114720 vb_active_queue_size 2390 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_meta_data_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-02T18:37:11.301,ns_1@10.2.1.68:ns_doctor<0.1479.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,906626,333306}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,109269640}, {processes,65690088}, {processes_used,64111752}, {system,43579552}, {atom,1405017}, {atom_used,1377299}, {binary,6214608}, {code,14117455}, {ets,3752768}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,4568653824}, {buffered_memory,210264064}, {free_memory,25138573312}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{842193,5000}}, {context_switches,{4105658,0}}, {garbage_collection,{440887,1192363292,0}}, {io,{{input,208742033},{output,49949928}}}, {reductions,{448368730,3277387}}, {run_queue,0}, {runtime,{156140,760}}]}, {system_stats, [{cpu_utilization_rate,21.35678391959799}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,3933305590}, {couch_docs_data_size,2897466930}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1190955}, {curr_items_tot,1190955}, {mem_used,3053554408}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,842}, {memory_data,{33657663488,8482521088,{<18077.5273.0>,7628384}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 24549388 kB\nBuffers: 205336 kB\nCached: 4461576 kB\nSwapCached: 0 kB\nActive: 5224360 kB\nInactive: 2848568 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 24549388 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 1584 kB\nWriteback: 0 kB\nAnonPages: 3405556 kB\nMapped: 15312 kB\nSlab: 203732 kB\nPageTables: 10880 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 5124372 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,906628,984759}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,89530520}, {processes,50565264}, {processes_used,50499432}, {system,38965256}, {atom,1352041}, {atom_used,1349412}, {binary,2411272}, {code,13622682}, {ets,3533312}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,4292997120}, {buffered_memory,239374336}, {free_memory,25368162304}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{844441,5001}}, {context_switches,{4154480,0}}, {garbage_collection,{471883,747516671,0}}, {io,{{input,60484139},{output,38719012}}}, {reductions,{335300785,2633771}}, {run_queue,0}, {runtime,{135080,670}}]}, {system_stats, [{cpu_utilization_rate,28.64321608040201}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,3605378357}, {couch_docs_data_size,2881273141}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1202202}, {curr_items_tot,1202202}, {mem_used,3083249704}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,844}, {memory_data,{33657663488,8186269696,{<18118.1696.0>,2914384}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 24773596 kB\nBuffers: 233764 kB\nCached: 4192380 kB\nSwapCached: 140 kB\nActive: 4900768 kB\nInactive: 2953496 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 24773596 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 1264 kB\nWriteback: 0 kB\nAnonPages: 3427572 kB\nMapped: 17056 kB\nSlab: 195504 kB\nPageTables: 11476 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 5131556 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,906627,955277}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,91605224}, {processes,52683912}, {processes_used,52597040}, {system,38921312}, {atom,1352041}, {atom_used,1349412}, {binary,2363296}, {code,13631938}, {ets,3520704}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,3752460288}, {buffered_memory,412409856}, {free_memory,25741336576}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{844398,4997}}, {context_switches,{3562472,0}}, {garbage_collection,{397046,694999192,0}}, {io,{{input,59783319},{output,37450440}}}, {reductions,{313757988,4615684}}, {run_queue,0}, {runtime,{123190,1680}}]}, {system_stats, [{cpu_utilization_rate,13.965087281795512}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,3061675346}, {couch_docs_data_size,2886744828}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1200603}, {curr_items_tot,1200603}, {mem_used,3078037352}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,844}, {memory_data,{33657663488,7834808320,{<18117.1681.0>,4714984}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 25138024 kB\nBuffers: 402744 kB\nCached: 3664512 kB\nSwapCached: 0 kB\nActive: 4230044 kB\nInactive: 3258860 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 25138024 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 1128 kB\nWriteback: 0 kB\nAnonPages: 3421740 kB\nMapped: 17056 kB\nSlab: 199312 kB\nPageTables: 11552 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 5133700 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,906626,302238}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.10121.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,8}, {started_on,1351906614}, {total_vbuckets,129}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351906626}, {vbuckets_done,11}]]}, {memory, [{total,93452608}, {processes,54071496}, {processes_used,54017016}, {system,39381112}, {atom,1352041}, {atom_used,1349412}, {binary,2754208}, {code,13659922}, {ets,3546208}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,4580909056}, {buffered_memory,309342208}, {free_memory,25006415872}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{841430,4999}}, {context_switches,{4196411,0}}, {garbage_collection,{461714,749827654,0}}, {io,{{input,60601013},{output,39162374}}}, {reductions,{336689518,1931497}}, {run_queue,0}, {runtime,{132960,450}}]}, {system_stats, [{cpu_utilization_rate,21.608040201005025}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,4033068597}, {couch_docs_data_size,2833660289}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1193565}, {curr_items_tot,1193565}, {mem_used,3060261480}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,841}, {memory_data,{33657663488,8701911040,{<0.1662.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 24420452 kB\nBuffers: 302092 kB\nCached: 4473544 kB\nSwapCached: 12 kB\nActive: 5440868 kB\nInactive: 2737692 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 24420452 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 7348 kB\nWriteback: 0 kB\nAnonPages: 3402632 kB\nMapped: 17256 kB\nSlab: 224620 kB\nPageTables: 11400 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 5110700 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-02T18:38:04.824,ns_1@10.2.1.68:<0.1669.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,31,46}}) Stats for bucket "default": accepting_conns 1 auth_cmds 3 auth_errors 0 bucket_active_conns 1 bucket_conns 6 bytes 3496135288 bytes_read 3260367389 bytes_written 66652618 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 1358539 conn_yields 447 connection_structures 5000 curr_connections 10 curr_conns_on_port_11209 6 curr_conns_on_port_11210 2 curr_items 1358539 curr_items_tot 1358539 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-03 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 1107 ep_commit_time 642 ep_commit_time_total 5085 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 30 ep_data_age_highwat 62 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data/default ep_degraded_mode 0 ep_diskqueue_drain 1243465 ep_diskqueue_fill 1358539 ep_diskqueue_items 115319 ep_diskqueue_memory 3682368 ep_diskqueue_pending 5523552 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 480 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 105114 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 1242619 ep_io_read_bytes 0 ep_io_write_bytes 2942173080 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 640256 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 3323421193 ep_max_checkpoints 2 ep_max_data_size 5242880000 ep_max_item_size 20971520 ep_max_size 5242880000 ep_max_txn_size 10000 ep_max_vbuckets 512 ep_mem_high_wat 3932160000 ep_mem_high_wat 3932160000 ep_mem_low_wat 3145728000 ep_mem_low_wat 3145728000 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 70666080 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 10205 ep_startup_time 1351905984 ep_storage_age 27 ep_storage_age_highwat 61 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 3307118725 ep_total_del_items 0 ep_total_enqueued 1358539 ep_total_new_items 1242619 ep_total_persisted 1242619 ep_uncommitted_items 5098 ep_value_size 3211382533 ep_vb0 0 ep_vb_snapshot_total 52 ep_vb_total 128 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_130_g8b09894 ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 741 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 3496135288 pid 17027 pointer_size 64 rejected_conns 0 rusage_system 61.054718 rusage_user 159.308781 threads 4 time 1351906683 total_connections 13 uptime 714 vb_active_curr_items 1358539 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 6351872 vb_active_itm_memory 3307118725 vb_active_meta_data_memory 107324581 vb_active_num 128 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1358539 vb_active_ops_create 1243464 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 100 vb_active_queue_age 2407894000 vb_active_queue_drain 1243465 vb_active_queue_fill 1358539 vb_active_queue_memory 3682368 vb_active_queue_pending 5523552 vb_active_queue_size 115074 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_meta_data_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-02T18:38:11.301,ns_1@10.2.1.68:ns_doctor<0.1479.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,906686,332461}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,107457680}, {processes,63366600}, {processes_used,61788264}, {system,44091080}, {atom,1405017}, {atom_used,1377299}, {binary,6549160}, {code,14117455}, {ets,3930456}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,5299970048}, {buffered_memory,211062784}, {free_memory,23950835712}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{902193,5000}}, {context_switches,{4455831,0}}, {garbage_collection,{476080,1296526064,0}}, {io,{{input,229246060},{output,54999258}}}, {reductions,{487957381,3689911}}, {run_queue,0}, {runtime,{168260,1210}}]}, {system_stats, [{cpu_utilization_rate,6.7669172932330826}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,4687021369}, {couch_docs_data_size,3314724417}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1359270}, {curr_items_tot,1359270}, {mem_used,3396090648}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,902}, {memory_data,{33657663488,9692225536,{<18077.14785.0>,7028184}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 23389488 kB\nBuffers: 206116 kB\nCached: 5175752 kB\nSwapCached: 0 kB\nActive: 6072388 kB\nInactive: 3141444 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 23389488 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 900 kB\nWriteback: 0 kB\nAnonPages: 3831532 kB\nMapped: 15312 kB\nSlab: 221924 kB\nPageTables: 11700 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 5555648 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,906688,983406}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,91619824}, {processes,52585464}, {processes_used,52519472}, {system,39034360}, {atom,1352041}, {atom_used,1349412}, {binary,2303016}, {code,13622682}, {ets,3710312}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,4971290624}, {buffered_memory,240558080}, {free_memory,24239603712}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{904440,5000}}, {context_switches,{4380254,0}}, {garbage_collection,{495199,806988730,0}}, {io,{{input,65467697},{output,42622455}}}, {reductions,{363906434,2193473}}, {run_queue,0}, {runtime,{143400,520}}]}, {system_stats, [{cpu_utilization_rate,26.5}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,4286797131}, {couch_docs_data_size,3291117083}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1369919}, {curr_items_tot,1369919}, {mem_used,3424929304}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,904}, {memory_data,{33657663488,9314197504,{<18118.1697.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 23671736 kB\nBuffers: 234920 kB\nCached: 4854776 kB\nSwapCached: 140 kB\nActive: 5655912 kB\nInactive: 3283136 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 23671736 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 1968 kB\nWriteback: 168 kB\nAnonPages: 3848528 kB\nMapped: 17056 kB\nSlab: 211936 kB\nPageTables: 12328 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 5552412 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,906687,955093}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,94958296}, {processes,55634176}, {processes_used,55547624}, {system,39324120}, {atom,1352041}, {atom_used,1349412}, {binary,2588512}, {code,13631938}, {ets,3698488}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,4452605952}, {buffered_memory,413515776}, {free_memory,24588775424}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{904397,4998}}, {context_switches,{3973083,0}}, {garbage_collection,{439655,770861630,0}}, {io,{{input,65623464},{output,42051054}}}, {reductions,{349264881,2204656}}, {run_queue,0}, {runtime,{134520,590}}]}, {system_stats, [{cpu_utilization_rate,27.860696517412936}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,3721082197}, {couch_docs_data_size,3261907165}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1368729}, {curr_items_tot,1368729}, {mem_used,3420656728}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,904}, {memory_data,{33657663488,8953602048,{<18117.1678.0>,4714984}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 24012476 kB\nBuffers: 403824 kB\nCached: 4348248 kB\nSwapCached: 0 kB\nActive: 5045276 kB\nInactive: 3551672 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 24012476 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 1632 kB\nWriteback: 0 kB\nAnonPages: 3844800 kB\nMapped: 17056 kB\nSlab: 216036 kB\nPageTables: 12388 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 5551184 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,906686,304497}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.10121.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,86}, {started_on,1351906614}, {total_vbuckets,129}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351906684}, {vbuckets_done,111}]]}, {memory, [{total,90449488}, {processes,51269056}, {processes_used,51257416}, {system,39180432}, {atom,1352041}, {atom_used,1349412}, {binary,2324024}, {code,13659922}, {ets,3769008}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,3841998848}, {buffered_memory,311988224}, {free_memory,25311285248}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{901432,5003}}, {context_switches,{4272885,0}}, {garbage_collection,{470923,797319555,0}}, {io,{{input,70485701},{output,42219488}}}, {reductions,{360993551,1911366}}, {run_queue,0}, {runtime,{138030,420}}]}, {system_stats, [{cpu_utilization_rate,24.93765586034913}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,3234360161}, {couch_docs_data_size,3017105956}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1361303}, {curr_items_tot,1361303}, {mem_used,3406166488}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,901}, {memory_data,{33657663488,8345362432,{<0.1662.0>,2914384}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 24718176 kB\nBuffers: 304676 kB\nCached: 3751952 kB\nSwapCached: 12 kB\nActive: 4573172 kB\nInactive: 3322780 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 24718176 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 15280 kB\nWriteback: 0 kB\nAnonPages: 3839088 kB\nMapped: 17184 kB\nSlab: 208004 kB\nPageTables: 12264 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 5544252 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-02T18:39:11.301,ns_1@10.2.1.68:ns_doctor<0.1479.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,906746,330578}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.17519.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,20}, {started_on,1351906753}, {total_vbuckets,129}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351906786}, {vbuckets_done,26}]]}, {memory, [{total,114937208}, {processes,70087600}, {processes_used,68527456}, {system,44849608}, {atom,1405017}, {atom_used,1377299}, {binary,7114712}, {code,14117455}, {ets,4118952}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,5390905344}, {buffered_memory,212557824}, {free_memory,23425404928}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{962192,4999}}, {context_switches,{4621521,0}}, {garbage_collection,{493093,1384297575,0}}, {io,{{input,250677200},{output,58168334}}}, {reductions,{521470567,2589854}}, {run_queue,0}, {runtime,{176710,530}}]}, {system_stats, [{cpu_utilization_rate,14.25}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,4822486347}, {couch_docs_data_size,3534952316}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1526750}, {curr_items_tot,1526750}, {mem_used,3840830088}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,962}, {memory_data,{33657663488,10263494656,{<18077.14785.0>,7028184}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 22876372 kB\nBuffers: 207576 kB\nCached: 5264556 kB\nSwapCached: 0 kB\nActive: 6286816 kB\nInactive: 3435560 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 22876372 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 13016 kB\nWriteback: 0 kB\nAnonPages: 4250396 kB\nMapped: 15320 kB\nSlab: 226028 kB\nPageTables: 12728 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 5967552 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,906748,985115}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,92396608}, {processes,53012080}, {processes_used,52945848}, {system,39384528}, {atom,1352041}, {atom_used,1349412}, {binary,2477568}, {code,13622682}, {ets,3885424}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,5685858304}, {buffered_memory,241295360}, {free_memory,23078535168}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{964440,5000}}, {context_switches,{4686954,0}}, {garbage_collection,{526737,872743729,0}}, {io,{{input,67376241},{output,44776786}}}, {reductions,{395419491,2623193}}, {run_queue,0}, {runtime,{153170,800}}]}, {system_stats, [{cpu_utilization_rate,5.985037406483791}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,5033784640}, {couch_docs_data_size,3716870134}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1537443}, {curr_items_tot,1537443}, {mem_used,3868057048}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,964}, {memory_data,{33657663488,10520338432,{<18118.1697.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 22537632 kB\nBuffers: 235640 kB\nCached: 5552596 kB\nSwapCached: 140 kB\nActive: 6478052 kB\nInactive: 3576004 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 22537632 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 772 kB\nWriteback: 20 kB\nAnonPages: 4265912 kB\nMapped: 17056 kB\nSlab: 229800 kB\nPageTables: 13064 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 5970344 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,906747,955755}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,91351928}, {processes,51898360}, {processes_used,51811488}, {system,39453568}, {atom,1352041}, {atom_used,1349412}, {binary,2544240}, {code,13631938}, {ets,3872672}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,5102358528}, {buffered_memory,414273536}, {free_memory,23495004160}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{964397,4998}}, {context_switches,{4179819,0}}, {garbage_collection,{460550,827978443,0}}, {io,{{input,67055974},{output,43909906}}}, {reductions,{377329438,2341354}}, {run_queue,0}, {runtime,{141870,540}}]}, {system_stats, [{cpu_utilization_rate,7.5},{swap_total,10701078528},{swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,4424709891}, {couch_docs_data_size,3705134768}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1537069}, {curr_items_tot,1537069}, {mem_used,3865906712}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,964}, {memory_data,{33657663488,10081521664,{<18117.1678.0>,2914384}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 22944340 kB\nBuffers: 404564 kB\nCached: 4982772 kB\nSwapCached: 0 kB\nActive: 5774612 kB\nInactive: 3873412 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 22944340 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 2116 kB\nWriteback: 0 kB\nAnonPages: 4260612 kB\nMapped: 17056 kB\nSlab: 231768 kB\nPageTables: 13176 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 5965128 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,906746,303442}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,93645768}, {processes,54137824}, {processes_used,54050952}, {system,39507944}, {atom,1352041}, {atom_used,1349412}, {binary,2535560}, {code,13659922}, {ets,3891072}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,4582064128}, {buffered_memory,313597952}, {free_memory,24131944448}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{961431,5001}}, {context_switches,{4572982,0}}, {garbage_collection,{502991,863424743,0}}, {io,{{input,75983487},{output,46439596}}}, {reductions,{392581637,3155883}}, {run_queue,0}, {runtime,{146720,920}}]}, {system_stats, [{cpu_utilization_rate,25.5}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,3965069536}, {couch_docs_data_size,3697123117}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1529382}, {curr_items_tot,1529382}, {mem_used,3845768344}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,961}, {memory_data,{33657663488,9511370752,{<0.1662.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 23566352 kB\nBuffers: 306248 kB\nCached: 4474672 kB\nSwapCached: 12 kB\nActive: 5105288 kB\nInactive: 3927376 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 23566352 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 5412 kB\nWriteback: 0 kB\nAnonPages: 4250840 kB\nMapped: 17256 kB\nSlab: 222412 kB\nPageTables: 13044 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 5955644 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-02T18:39:44.823,ns_1@10.2.1.68:<0.1669.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,31,46}}) Stats for bucket "default": accepting_conns 1 auth_cmds 3 auth_errors 0 bucket_active_conns 1 bucket_conns 6 bytes 3625477032 bytes_read 3932522688 bytes_written 79814765 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 1638725 conn_yields 545 connection_structures 5000 curr_connections 10 curr_conns_on_port_11209 6 curr_conns_on_port_11210 2 curr_items 1638725 curr_items_tot 1638725 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-03 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 1251 ep_commit_time 1 ep_commit_time_total 7072 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 2 ep_data_age_highwat 62 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data/default ep_degraded_mode 0 ep_diskqueue_drain 1634983 ep_diskqueue_fill 1638725 ep_diskqueue_items 3743 ep_diskqueue_memory 119744 ep_diskqueue_pending 179616 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 580 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 146 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 1634977 ep_io_read_bytes 0 ep_io_write_bytes 3871127272 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 1583660 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 3548674127 ep_max_checkpoints 2 ep_max_data_size 5242880000 ep_max_item_size 20971520 ep_max_size 5242880000 ep_max_txn_size 10000 ep_max_vbuckets 512 ep_mem_high_wat 3932160000 ep_mem_high_wat 3932160000 ep_mem_low_wat 3145728000 ep_mem_low_wat 3145728000 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 420 ep_num_expiry_pager_runs 0 ep_num_non_resident 195656 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 2 ep_num_value_ejects 195656 ep_oom_errors 0 ep_overhead 17771040 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 3597 ep_startup_time 1351905984 ep_storage_age 1 ep_storage_age_highwat 62 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 3529009427 ep_total_del_items 0 ep_total_enqueued 1638725 ep_total_new_items 1634977 ep_total_persisted 1634977 ep_uncommitted_items 444 ep_value_size 3413418846 ep_vb0 0 ep_vb_snapshot_total 55 ep_vb_total 128 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_130_g8b09894 ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 741 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 3625477032 pid 17027 pointer_size 64 rejected_conns 0 rusage_system 72.928913 rusage_user 195.387296 threads 4 time 1351906783 total_connections 13 uptime 814 vb_active_curr_items 1638725 vb_active_eject 195656 vb_active_expired 0 vb_active_ht_memory 12645376 vb_active_itm_memory 3529009427 vb_active_meta_data_memory 129459275 vb_active_num 128 vb_active_num_non_resident 195656 vb_active_num_ref_ejects 195656 vb_active_num_ref_items 1638725 vb_active_ops_create 1634982 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 88 vb_active_queue_age 1287000 vb_active_queue_drain 1634983 vb_active_queue_fill 1638725 vb_active_queue_memory 119744 vb_active_queue_pending 179616 vb_active_queue_size 3742 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_meta_data_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [stats:warn,2012-11-02T18:39:49.689,ns_1@10.2.1.68:system_stats_collector<0.1557.0>:system_stats_collector:handle_info:133]lost 1 ticks [stats:warn,2012-11-02T18:39:49.695,ns_1@10.2.1.68:<0.1669.0>:stats_collector:latest_tick:223]Dropped 2 ticks [ns_doctor:debug,2012-11-02T18:40:11.301,ns_1@10.2.1.68:ns_doctor<0.1479.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,906806,331332}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.17519.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,59}, {started_on,1351906753}, {total_vbuckets,129}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351906844}, {vbuckets_done,77}]]}, {memory, [{total,114164688}, {processes,69332664}, {processes_used,67778040}, {system,44832024}, {atom,1405017}, {atom_used,1377299}, {binary,7026016}, {code,14117455}, {ets,4187128}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,4986740736}, {buffered_memory,214376448}, {free_memory,23665262592}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1022193,4997}}, {context_switches,{4685556,0}}, {garbage_collection,{499939,1460815323,0}}, {io,{{input,266195918},{output,60220070}}}, {reductions,{550630871,2567803}}, {run_queue,0}, {runtime,{183140,530}}]}, {system_stats, [{cpu_utilization_rate,14.035087719298245}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,4384372578}, {couch_docs_data_size,3695307440}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1694307}, {curr_items_tot,1694307}, {mem_used,3795150472}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1022}, {memory_data,{33657663488,9982496768,{<18077.16820.0>,7028184}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 23110732 kB\nBuffers: 209352 kB\nCached: 4869864 kB\nSwapCached: 0 kB\nActive: 5679308 kB\nInactive: 3816804 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 23110732 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 22500 kB\nWriteback: 0 kB\nAnonPages: 4416876 kB\nMapped: 15316 kB\nSlab: 217064 kB\nPageTables: 12924 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6134324 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,906808,984202}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,93591176}, {processes,54325592}, {processes_used,54259760}, {system,39265584}, {atom,1352041}, {atom_used,1349412}, {binary,2294832}, {code,13622682}, {ets,3948744}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,6428884992}, {buffered_memory,241987584}, {free_memory,22159228928}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1024439,4999}}, {context_switches,{4934288,0}}, {garbage_collection,{552143,933322482,0}}, {io,{{input,69001864},{output,46767264}}}, {reductions,{424674211,2510055}}, {run_queue,0}, {runtime,{161080,660}}]}, {system_stats, [{cpu_utilization_rate,26.25}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,5762987356}, {couch_docs_data_size,4137168402}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1704748}, {curr_items_tot,1704748}, {mem_used,3803613800}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1024}, {memory_data,{33657663488,11444596736,{<18118.1696.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 21639996 kB\nBuffers: 236316 kB\nCached: 6278208 kB\nSwapCached: 140 kB\nActive: 7013864 kB\nInactive: 3919640 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 21639996 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 1860 kB\nWriteback: 92 kB\nAnonPages: 4418460 kB\nMapped: 17060 kB\nSlab: 247300 kB\nPageTables: 13484 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6123376 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,906807,956747}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,90858864}, {processes,51233224}, {processes_used,51146752}, {system,39625640}, {atom,1352041}, {atom_used,1349412}, {binary,2652848}, {code,13631938}, {ets,3936240}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,5865684992}, {buffered_memory,414892032}, {free_memory,22552334336}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1024397,4998}}, {context_switches,{4498500,0}}, {garbage_collection,{493339,894887636,0}}, {io,{{input,69085666},{output,46129948}}}, {reductions,{409517902,4260875}}, {run_queue,0}, {runtime,{150640,1300}}]}, {system_stats, [{cpu_utilization_rate,30.423940149625935}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,5162526047}, {couch_docs_data_size,4131352861}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1705630}, {curr_items_tot,1705630}, {mem_used,3801210584}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1024}, {memory_data,{33657663488,11039174656,{<18117.1678.0>,2914384}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 22023764 kB\nBuffers: 405168 kB\nCached: 5728208 kB\nSwapCached: 0 kB\nActive: 6348804 kB\nInactive: 4201444 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 22023764 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 1068 kB\nWriteback: 108 kB\nAnonPages: 4416772 kB\nMapped: 17060 kB\nSlab: 250468 kB\nPageTables: 13492 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 6125584 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,906806,303854}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,94097040}, {processes,54455824}, {processes_used,54369352}, {system,39641216}, {atom,1352041}, {atom_used,1349412}, {binary,2494344}, {code,13659922}, {ets,4065504}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,5255139328}, {buffered_memory,314228736}, {free_memory,23269265408}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1021431,5001}}, {context_switches,{4874364,0}}, {garbage_collection,{534424,928805844,0}}, {io,{{input,77896091},{output,48616811}}}, {reductions,{423831943,2541482}}, {run_queue,0}, {runtime,{156130,1050}}]}, {system_stats, [{cpu_utilization_rate,32.66832917705736}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,4603954529}, {couch_docs_data_size,4089879296}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1697942}, {curr_items_tot,1697942}, {mem_used,3782315384}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1021}, {memory_data,{33657663488,10364272640,{<0.1663.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 22724140 kB\nBuffers: 306864 kB\nCached: 5131972 kB\nSwapCached: 12 kB\nActive: 5609556 kB\nInactive: 4249372 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 22724140 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 2096 kB\nWriteback: 0 kB\nAnonPages: 4419420 kB\nMapped: 17260 kB\nSlab: 238572 kB\nPageTables: 13496 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6125460 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-02T18:41:11.300,ns_1@10.2.1.68:ns_doctor<0.1479.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,906866,330427}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.17519.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,93}, {started_on,1351906753}, {total_vbuckets,129}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351906903}, {vbuckets_done,120}]]}, {memory, [{total,110781144}, {processes,65439120}, {processes_used,63871616}, {system,45342024}, {atom,1405017}, {atom_used,1377299}, {binary,7364472}, {code,14117455}, {ets,4359776}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,4623765504}, {buffered_memory,216170496}, {free_memory,24040349696}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1082193,5000}}, {context_switches,{4747817,0}}, {garbage_collection,{507039,1538936225,0}}, {io,{{input,285524272},{output,65420908}}}, {reductions,{579855047,2570120}}, {run_queue,0}, {runtime,{189800,520}}]}, {system_stats, [{cpu_utilization_rate,6.78391959798995}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,4007707121}, {couch_docs_data_size,3824253518}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1862217}, {curr_items_tot,1862217}, {mem_used,3803883256}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1082}, {memory_data,{33657663488,9617186816,{<18077.16820.0>,7028184}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 23476904 kB\nBuffers: 211104 kB\nCached: 4515396 kB\nSwapCached: 0 kB\nActive: 5028312 kB\nInactive: 4111612 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 23476904 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 24124 kB\nWriteback: 4616 kB\nAnonPages: 4413468 kB\nMapped: 15320 kB\nSlab: 207272 kB\nPageTables: 13004 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6128032 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,906868,985312}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,91833192}, {processes,51925320}, {processes_used,51859088}, {system,39907872}, {atom,1352041}, {atom_used,1349412}, {binary,2754656}, {code,13622682}, {ets,4133864}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,7103922176}, {buffered_memory,243122176}, {free_memory,21462511616}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1084440,5001}}, {context_switches,{5138272,0}}, {garbage_collection,{572036,990525145,0}}, {io,{{input,73894020},{output,51903254}}}, {reductions,{452495749,2409410}}, {run_queue,0}, {runtime,{167960,550}}]}, {system_stats, [{cpu_utilization_rate,6.7669172932330826}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,6446609792}, {couch_docs_data_size,4538530972}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1871472}, {curr_items_tot,1871472}, {mem_used,3746189224}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1084}, {memory_data,{33657663488,12163153920,{<18118.1696.0>,2914384}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 20959608 kB\nBuffers: 237424 kB\nCached: 6937424 kB\nSwapCached: 140 kB\nActive: 7355832 kB\nInactive: 4241240 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 20959608 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 1672 kB\nWriteback: 288 kB\nAnonPages: 4422436 kB\nMapped: 17060 kB\nSlab: 264392 kB\nPageTables: 13476 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6132448 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,906867,956770}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,93508296}, {processes,53817008}, {processes_used,53730136}, {system,39691288}, {atom,1352041}, {atom_used,1349412}, {binary,2535480}, {code,13631938}, {ets,4121008}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,6634319872}, {buffered_memory,416014336}, {free_memory,21760004096}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1084397,4997}}, {context_switches,{4909044,0}}, {garbage_collection,{535604,970676820,0}}, {io,{{input,74909531},{output,51954025}}}, {reductions,{445051427,2465694}}, {run_queue,0}, {runtime,{161390,650}}]}, {system_stats, [{cpu_utilization_rate,7.25}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,5982123378}, {couch_docs_data_size,4549459033}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1874315}, {curr_items_tot,1874315}, {mem_used,3747226744}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1084}, {memory_data,{33657663488,11886485504,{<18117.1681.0>,4714984}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 21250004 kB\nBuffers: 406264 kB\nCached: 6478828 kB\nSwapCached: 0 kB\nActive: 6838360 kB\nInactive: 4465604 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 21250004 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 1736 kB\nWriteback: 4 kB\nAnonPages: 4418952 kB\nMapped: 17060 kB\nSlab: 269380 kB\nPageTables: 13504 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 6125092 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,906866,302755}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,94447768}, {processes,54771744}, {processes_used,54685192}, {system,39676024}, {atom,1352041}, {atom_used,1349412}, {binary,2457304}, {code,13659922}, {ets,4137800}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,5961416704}, {buffered_memory,315072512}, {free_memory,22546010112}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1081430,5000}}, {context_switches,{5165808,0}}, {garbage_collection,{564796,993290927,0}}, {io,{{input,79731992},{output,50698297}}}, {reductions,{454771174,2274756}}, {run_queue,0}, {runtime,{165740,570}}]}, {system_stats, [{cpu_utilization_rate,26.75}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,5290065217}, {couch_docs_data_size,4488525765}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1866018}, {curr_items_tot,1866018}, {mem_used,3733022968}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1081}, {memory_data,{33657663488,11071782912,{<0.1662.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 22017588 kB\nBuffers: 307688 kB\nCached: 5821696 kB\nSwapCached: 12 kB\nActive: 5979420 kB\nInactive: 4568512 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 22017588 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 708 kB\nWriteback: 56 kB\nAnonPages: 4418536 kB\nMapped: 17260 kB\nSlab: 255736 kB\nPageTables: 13500 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6120736 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-02T18:41:26.823,ns_1@10.2.1.68:<0.1669.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,31,46}}) Stats for bucket "default": accepting_conns 1 auth_cmds 3 auth_errors 0 bucket_active_conns 1 bucket_conns 6 bytes 3888622568 bytes_read 4618840056 bytes_written 93372796 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 1924917 conn_yields 649 connection_structures 5000 curr_connections 10 curr_conns_on_port_11209 6 curr_conns_on_port_11210 2 curr_items 1924917 curr_items_tot 1924917 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-03 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 1368 ep_commit_time 7 ep_commit_time_total 8572 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 1 ep_data_age_highwat 62 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data/default ep_degraded_mode 0 ep_diskqueue_drain 1923516 ep_diskqueue_fill 1924917 ep_diskqueue_items 1402 ep_diskqueue_memory 44832 ep_diskqueue_pending 67248 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 683 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 317 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 1923493 ep_io_read_bytes 0 ep_io_write_bytes 4553748303 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 1766740 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 3792621703 ep_max_checkpoints 2 ep_max_data_size 5242880000 ep_max_item_size 20971520 ep_max_size 5242880000 ep_max_txn_size 10000 ep_max_vbuckets 512 ep_mem_high_wat 3932160000 ep_mem_high_wat 3932160000 ep_mem_low_wat 3145728000 ep_mem_low_wat 3145728000 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 27432 ep_num_expiry_pager_runs 0 ep_num_non_resident 389847 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 4 ep_num_value_ejects 389847 ep_oom_errors 0 ep_overhead 26868792 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 1085 ep_startup_time 1351905984 ep_storage_age 1 ep_storage_age_highwat 62 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 3769522699 ep_total_del_items 0 ep_total_enqueued 1924917 ep_total_new_items 1923493 ep_total_persisted 1923493 ep_uncommitted_items 2989 ep_value_size 3633657702 ep_vb0 0 ep_vb_snapshot_total 57 ep_vb_total 128 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_130_g8b09894 ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 741 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 3888622568 pid 17027 pointer_size 64 rejected_conns 0 rusage_system 84.592140 rusage_user 228.634242 threads 4 time 1351906886 total_connections 13 uptime 917 vb_active_curr_items 1924917 vb_active_eject 389847 vb_active_expired 0 vb_active_ht_memory 12645376 vb_active_itm_memory 3769522699 vb_active_meta_data_memory 152068443 vb_active_num 128 vb_active_num_non_resident 389847 vb_active_num_ref_ejects 389847 vb_active_num_ref_items 1924917 vb_active_ops_create 1923515 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 79 vb_active_queue_age 211000 vb_active_queue_drain 1923516 vb_active_queue_fill 1924917 vb_active_queue_memory 44832 vb_active_queue_pending 67248 vb_active_queue_size 1401 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_meta_data_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-02T18:42:11.301,ns_1@10.2.1.68:ns_doctor<0.1479.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,906926,330759}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,109658928}, {processes,64742560}, {processes_used,63164224}, {system,44916368}, {atom,1405017}, {atom_used,1377299}, {binary,6881808}, {code,14117455}, {ets,4417040}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,5423075328}, {buffered_memory,217227264}, {free_memory,23201681408}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1142194,5001}}, {context_switches,{4813599,0}}, {garbage_collection,{513881,1617567129,0}}, {io,{{input,301436722},{output,67427758}}}, {reductions,{609265657,2686276}}, {run_queue,0}, {runtime,{196450,590}}]}, {system_stats, [{cpu_utilization_rate,13.216957605985037}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,4794655856}, {couch_docs_data_size,4659067803}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2029937}, {curr_items_tot,2029937}, {mem_used,3800093256}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1142}, {memory_data,{33657663488,10443284480,{<18077.16820.0>,7028184}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 22657892 kB\nBuffers: 212136 kB\nCached: 5295972 kB\nSwapCached: 0 kB\nActive: 4967988 kB\nInactive: 4972140 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 22657892 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 4580 kB\nWriteback: 2048 kB\nAnonPages: 4431752 kB\nMapped: 15308 kB\nSlab: 226316 kB\nPageTables: 13132 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6146844 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,906928,987618}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.11448.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,25}, {started_on,1351906845}, {total_vbuckets,129}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351906901}, {vbuckets_done,33}]]}, {memory, [{total,92351192}, {processes,52425672}, {processes_used,52373952}, {system,39925520}, {atom,1352041}, {atom_used,1349412}, {binary,2695912}, {code,13622682}, {ets,4206832}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,6854590464}, {buffered_memory,244781056}, {free_memory,21711757312}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1144441,5001}}, {context_switches,{5234480,0}}, {garbage_collection,{581164,1039702378,0}}, {io,{{input,88344416},{output,55563252}}}, {reductions,{477583518,1904333}}, {run_queue,0}, {runtime,{173720,390}}]}, {system_stats, [{cpu_utilization_rate,14.285714285714286}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,6296552914}, {couch_docs_data_size,4696626709}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2039358}, {curr_items_tot,2039358}, {mem_used,3715749656}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1144}, {memory_data,{33657663488,12003680256,{<18118.1696.0>,2914384}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 21202888 kB\nBuffers: 239044 kB\nCached: 6693936 kB\nSwapCached: 140 kB\nActive: 6814556 kB\nInactive: 4542836 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 21202888 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 39744 kB\nWriteback: 108 kB\nAnonPages: 4424644 kB\nMapped: 17068 kB\nSlab: 260436 kB\nPageTables: 13648 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6138768 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,906927,958921}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,92648736}, {processes,52920784}, {processes_used,52834152}, {system,39727952}, {atom,1352041}, {atom_used,1349412}, {binary,2508096}, {code,13631938}, {ets,4183264}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,7414345728}, {buffered_memory,416735232}, {free_memory,20958531584}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1144399,4996}}, {context_switches,{5193357,0}}, {garbage_collection,{564741,1034782217,0}}, {io,{{input,76678540},{output,54001777}}}, {reductions,{475973709,2412850}}, {run_queue,0}, {runtime,{170090,680}}]}, {system_stats, [{cpu_utilization_rate,28.82205513784461}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,6719421664}, {couch_docs_data_size,4961693141}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2041865}, {curr_items_tot,2041865}, {mem_used,3700006200}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1144}, {memory_data,{33657663488,12656214016,{<18117.1681.0>,4714984}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 20467316 kB\nBuffers: 406968 kB\nCached: 7240572 kB\nSwapCached: 0 kB\nActive: 7258768 kB\nInactive: 4809784 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 20467316 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 1680 kB\nWriteback: 0 kB\nAnonPages: 4421004 kB\nMapped: 17060 kB\nSlab: 288288 kB\nPageTables: 13500 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 6128008 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,906926,303018}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,95626696}, {processes,55498416}, {processes_used,55411944}, {system,40128280}, {atom,1352041}, {atom_used,1349412}, {binary,2731184}, {code,13659922}, {ets,4314576}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,6725799936}, {buffered_memory,316178432}, {free_memory,21760405504}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1141430,4999}}, {context_switches,{5496742,0}}, {garbage_collection,{599517,1061931965,0}}, {io,{{input,85162448},{output,56235191}}}, {reductions,{487360524,2554305}}, {run_queue,0}, {runtime,{174940,670}}]}, {system_stats, [{cpu_utilization_rate,28.25}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,6070404463}, {couch_docs_data_size,4921924926}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2034512}, {curr_items_tot,2034512}, {mem_used,3686364968}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1141}, {memory_data,{33657663488,11872116736,{<0.1662.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 21250520 kB\nBuffers: 308768 kB\nCached: 6568164 kB\nSwapCached: 12 kB\nActive: 6421200 kB\nInactive: 4874544 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 21250520 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 1360 kB\nWriteback: 0 kB\nAnonPages: 4418596 kB\nMapped: 17260 kB\nSlab: 274396 kB\nPageTables: 13500 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6125028 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-02T18:43:06.822,ns_1@10.2.1.68:<0.1669.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,31,47}}) Stats for bucket "default": accepting_conns 1 auth_cmds 3 auth_errors 0 bucket_active_conns 1 bucket_conns 6 bytes 3653542280 bytes_read 5292537656 bytes_written 106556378 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 2205493 conn_yields 726 connection_structures 5000 curr_connections 10 curr_conns_on_port_11209 6 curr_conns_on_port_11210 2 curr_items 2205493 curr_items_tot 2205493 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-03 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 1491 ep_commit_time 1 ep_commit_time_total 9314 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 5 ep_data_age_highwat 62 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data/default ep_degraded_mode 0 ep_diskqueue_drain 2191816 ep_diskqueue_fill 2205493 ep_diskqueue_items 13678 ep_diskqueue_memory 437664 ep_diskqueue_pending 656496 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 777 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 388 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 2191808 ep_io_read_bytes 0 ep_io_write_bytes 5189423176 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 2132256 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 3576476399 ep_max_checkpoints 2 ep_max_data_size 5242880000 ep_max_item_size 20971520 ep_max_size 5242880000 ep_max_txn_size 10000 ep_max_vbuckets 512 ep_mem_high_wat 3932160000 ep_mem_high_wat 3932160000 ep_mem_low_wat 3145728000 ep_mem_low_wat 3145728000 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 152762 ep_num_expiry_pager_runs 0 ep_num_non_resident 774443 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 8 ep_num_value_ejects 774443 ep_oom_errors 0 ep_overhead 19439696 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 13290 ep_startup_time 1351905984 ep_storage_age 5 ep_storage_age_highwat 62 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 3550010483 ep_total_del_items 0 ep_total_enqueued 2205493 ep_total_new_items 2191808 ep_total_persisted 2191808 ep_uncommitted_items 311 ep_value_size 3394166349 ep_vb0 0 ep_vb_snapshot_total 59 ep_vb_total 128 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_130_g8b09894 ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 741 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 3653542280 pid 17027 pointer_size 64 rejected_conns 0 rusage_system 96.254367 rusage_user 263.729906 threads 4 time 1351906986 total_connections 13 uptime 1017 vb_active_curr_items 2205493 vb_active_eject 774443 vb_active_expired 0 vb_active_ht_memory 12645376 vb_active_itm_memory 3550010483 vb_active_meta_data_memory 174233947 vb_active_num 128 vb_active_num_non_resident 774443 vb_active_num_ref_ejects 774443 vb_active_num_ref_items 2205493 vb_active_ops_create 2191815 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 64 vb_active_queue_age 30599000 vb_active_queue_drain 2191816 vb_active_queue_fill 2205493 vb_active_queue_memory 437664 vb_active_queue_pending 656496 vb_active_queue_size 13677 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_meta_data_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-02T18:43:11.301,ns_1@10.2.1.68:ns_doctor<0.1479.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,906986,329318}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,110678016}, {processes,66203176}, {processes_used,64625160}, {system,44474840}, {atom,1405017}, {atom_used,1377299}, {binary,6263432}, {code,14117455}, {ets,4592016}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,6321897472}, {buffered_memory,218284032}, {free_memory,22281994240}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1202193,5000}}, {context_switches,{4929351,0}}, {garbage_collection,{525528,1699152817,0}}, {io,{{input,316812609},{output,69592183}}}, {reductions,{640265782,2977797}}, {run_queue,0}, {runtime,{203730,720}}]}, {system_stats, [{cpu_utilization_rate,26.616915422885572}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,5633398183}, {couch_docs_data_size,5289338920}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2198008}, {curr_items_tot,2198008}, {mem_used,3747799400}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1202}, {memory_data,{33657663488,11302404096,{<18077.16820.0>,7028184}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 21759760 kB\nBuffers: 213168 kB\nCached: 6173728 kB\nSwapCached: 0 kB\nActive: 5299624 kB\nInactive: 5516616 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 21759760 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 5660 kB\nWriteback: 0 kB\nAnonPages: 4429020 kB\nMapped: 15316 kB\nSlab: 247624 kB\nPageTables: 12960 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6142856 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,906988,986940}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.11448.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,55}, {started_on,1351906845}, {total_vbuckets,129}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351906960}, {vbuckets_done,71}]]}, {memory, [{total,95645984}, {processes,55862328}, {processes_used,55845568}, {system,39783656}, {atom,1352041}, {atom_used,1349412}, {binary,2336424}, {code,13622682}, {ets,4418520}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,6314491904}, {buffered_memory,246288384}, {free_memory,22234517504}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1204440,5000}}, {context_switches,{5288678,0}}, {garbage_collection,{586829,1083032645,0}}, {io,{{input,89418288},{output,56858607}}}, {reductions,{500035659,1892474}}, {run_queue,0}, {runtime,{178770,400}}]}, {system_stats, [{cpu_utilization_rate,5.764411027568922}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,5770003856}, {couch_docs_data_size,4767709739}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2206709}, {curr_items_tot,2206709}, {mem_used,3733801576}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1204}, {memory_data,{33657663488,11460222976,{<18118.1696.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 21713396 kB\nBuffers: 240516 kB\nCached: 6166496 kB\nSwapCached: 140 kB\nActive: 6051904 kB\nInactive: 4807692 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 21713396 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 9924 kB\nWriteback: 84 kB\nAnonPages: 4452528 kB\nMapped: 17064 kB\nSlab: 247796 kB\nPageTables: 13636 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6164776 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,906987,958912}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,93383800}, {processes,53536232}, {processes_used,53449680}, {system,39847568}, {atom,1352041}, {atom_used,1349412}, {binary,2453160}, {code,13631938}, {ets,4357808}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,8112070656}, {buffered_memory,417460224}, {free_memory,20234641408}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1204399,4997}}, {context_switches,{5403782,0}}, {garbage_collection,{586482,1092382575,0}}, {io,{{input,78123865},{output,55817930}}}, {reductions,{504194756,2450629}}, {run_queue,0}, {runtime,{177170,640}}]}, {system_stats, [{cpu_utilization_rate,7.25}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7455825886}, {couch_docs_data_size,5381747270}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2209914}, {curr_items_tot,2209914}, {mem_used,3664707896}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1204}, {memory_data,{33657663488,13409816576,{<18117.1681.0>,4714984}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 19760516 kB\nBuffers: 407676 kB\nCached: 7921944 kB\nSwapCached: 0 kB\nActive: 7634516 kB\nInactive: 5123256 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 19760516 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 1844 kB\nWriteback: 0 kB\nAnonPages: 4428036 kB\nMapped: 17060 kB\nSlab: 305368 kB\nPageTables: 13516 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 6136684 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,906986,303116}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,93677736}, {processes,53378544}, {processes_used,53291672}, {system,40299192}, {atom,1352041}, {atom_used,1349412}, {binary,2840800}, {code,13659922}, {ets,4377144}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,7422046208}, {buffered_memory,316837888}, {free_memory,21045784576}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1201430,5000}}, {context_switches,{5742765,0}}, {garbage_collection,{624955,1122373666,0}}, {io,{{input,86768143},{output,58188658}}}, {reductions,{516772608,2252823}}, {run_queue,0}, {runtime,{182660,680}}]}, {system_stats, [{cpu_utilization_rate,7.73067331670823}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,6814271572}, {couch_docs_data_size,5338139594}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2202758}, {curr_items_tot,2202758}, {mem_used,3646290856}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1201}, {memory_data,{33657663488,12611624960,{<0.1662.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 20552524 kB\nBuffers: 309412 kB\nCached: 7248092 kB\nSwapCached: 12 kB\nActive: 6800484 kB\nInactive: 5176240 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 20552524 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 1316 kB\nWriteback: 0 kB\nAnonPages: 4418936 kB\nMapped: 17260 kB\nSlab: 291672 kB\nPageTables: 13504 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6123840 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-02T18:44:11.301,ns_1@10.2.1.68:ns_doctor<0.1479.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,907046,331040}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,112957048}, {processes,68246992}, {processes_used,66668976}, {system,44710056}, {atom,1405017}, {atom_used,1377299}, {binary,6322952}, {code,14117455}, {ets,4769048}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,6994608128}, {buffered_memory,219099136}, {free_memory,21587562496}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1262195,5001}}, {context_switches,{5114314,0}}, {garbage_collection,{543867,1789144297,0}}, {io,{{input,336569504},{output,74952088}}}, {reductions,{673958997,3084681}}, {run_queue,0}, {runtime,{212260,710}}]}, {system_stats, [{cpu_utilization_rate,27.1356783919598}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,6325234963}, {couch_docs_data_size,5716376071}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2366260}, {curr_items_tot,2366260}, {mem_used,3698984840}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1262}, {memory_data,{33657663488,12008517632,{<18077.16820.0>,7028184}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 21081728 kB\nBuffers: 213964 kB\nCached: 6830672 kB\nSwapCached: 0 kB\nActive: 5619080 kB\nInactive: 5860092 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 21081728 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 2040 kB\nWriteback: 0 kB\nAnonPages: 4434852 kB\nMapped: 15316 kB\nSlab: 263600 kB\nPageTables: 12932 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6154696 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,907048,988203}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.11448.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,76}, {started_on,1351906845}, {total_vbuckets,129}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351907020}, {vbuckets_done,99}]]}, {memory, [{total,91191112}, {processes,51205448}, {processes_used,51164368}, {system,39985664}, {atom,1352041}, {atom_used,1349412}, {binary,2387328}, {code,13622682}, {ets,4572336}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,6088273920}, {buffered_memory,247824384}, {free_memory,22439837696}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1264440,4999}}, {context_switches,{5344892,0}}, {garbage_collection,{592602,1127733265,0}}, {io,{{input,93725384},{output,61011782}}}, {reductions,{522683683,1849331}}, {run_queue,0}, {runtime,{184160,450}}]}, {system_stats, [{cpu_utilization_rate,9.5}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,5480195068}, {couch_docs_data_size,4925027912}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2374307}, {curr_items_tot,2374307}, {mem_used,3746074264}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1264}, {memory_data,{33657663488,11220246528,{<18118.1696.0>,2914384}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 21914028 kB\nBuffers: 242016 kB\nCached: 5945580 kB\nSwapCached: 140 kB\nActive: 5550432 kB\nInactive: 5113228 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 21914028 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 9896 kB\nWriteback: 14688 kB\nAnonPages: 4476316 kB\nMapped: 17064 kB\nSlab: 243052 kB\nPageTables: 13668 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6189560 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,907047,958971}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.13654.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,2}, {started_on,1351907050}, {total_vbuckets,129}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351907054}, {vbuckets_done,3}]]}, {memory, [{total,95663648}, {processes,55025920}, {processes_used,54945280}, {system,40637728}, {atom,1352041}, {atom_used,1349412}, {binary,3063824}, {code,13631938}, {ets,4536672}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,8876556288}, {buffered_memory,418734080}, {free_memory,19428700160}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1264398,5000}}, {context_switches,{5741825,0}}, {garbage_collection,{616237,1162623256,0}}, {io,{{input,86849226},{output,61370622}}}, {reductions,{537406863,2391925}}, {run_queue,0}, {runtime,{186460,500}}]}, {system_stats, [{cpu_utilization_rate,15.0}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,8261694720}, {couch_docs_data_size,5774812618}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2377528}, {curr_items_tot,2377528}, {mem_used,3625871752}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1264}, {memory_data,{33657663488,14225281024,{<18117.1678.0>,4714984}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 18973340 kB\nBuffers: 408920 kB\nCached: 8668512 kB\nSwapCached: 0 kB\nActive: 8101028 kB\nInactive: 5422544 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 18973340 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 45660 kB\nWriteback: 0 kB\nAnonPages: 4446196 kB\nMapped: 17064 kB\nSlab: 326232 kB\nPageTables: 13644 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 6162664 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,907046,303737}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,95898648}, {processes,55682024}, {processes_used,55595152}, {system,40216624}, {atom,1352041}, {atom_used,1349412}, {binary,2582792}, {code,13659922}, {ets,4551704}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,8209305600}, {buffered_memory,317927424}, {free_memory,20228186112}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1261431,5001}}, {context_switches,{5993928,0}}, {garbage_collection,{651005,1183500823,0}}, {io,{{input,88394013},{output,60150819}}}, {reductions,{546375585,2385777}}, {run_queue,0}, {runtime,{190830,560}}]}, {system_stats, [{cpu_utilization_rate,15.538847117794486}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,7589446936}, {couch_docs_data_size,5770554460}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2370844}, {curr_items_tot,2370844}, {mem_used,3612217576}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1261}, {memory_data,{33657663488,13428334592,{<0.1662.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 19754088 kB\nBuffers: 310476 kB\nCached: 8016900 kB\nSwapCached: 12 kB\nActive: 7230788 kB\nInactive: 5525452 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 19754088 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 1132 kB\nWriteback: 300 kB\nAnonPages: 4428648 kB\nMapped: 17260 kB\nSlab: 310680 kB\nPageTables: 13516 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6133948 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-02T18:44:46.820,ns_1@10.2.1.68:<0.1669.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,31,47}}) Stats for bucket "default": accepting_conns 1 auth_cmds 3 auth_errors 0 bucket_active_conns 1 bucket_conns 6 bytes 3913612200 bytes_read 5963837343 bytes_written 119721421 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 2485148 conn_yields 850 connection_structures 5000 curr_connections 10 curr_conns_on_port_11209 6 curr_conns_on_port_11210 2 curr_items 2485148 curr_items_tot 2485148 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-03 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 1592 ep_commit_time 7 ep_commit_time_total 10402 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 1 ep_data_age_highwat 62 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data/default ep_degraded_mode 0 ep_diskqueue_drain 2483443 ep_diskqueue_fill 2485148 ep_diskqueue_items 1706 ep_diskqueue_memory 54560 ep_diskqueue_pending 81840 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 882 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 371 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 2483417 ep_io_read_bytes 0 ep_io_write_bytes 5880070345 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 2314361 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 3817894780 ep_max_checkpoints 2 ep_max_data_size 5242880000 ep_max_item_size 20971520 ep_max_size 5242880000 ep_max_txn_size 10000 ep_max_vbuckets 512 ep_mem_high_wat 3932160000 ep_mem_high_wat 3932160000 ep_mem_low_wat 3145728000 ep_mem_low_wat 3145728000 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 252063 ep_num_expiry_pager_runs 0 ep_num_non_resident 963254 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 10 ep_num_value_ejects 963254 ep_oom_errors 0 ep_overhead 28046488 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 1335 ep_startup_time 1351905984 ep_storage_age 1 ep_storage_age_highwat 62 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 3788073004 ep_total_del_items 0 ep_total_enqueued 2485148 ep_total_new_items 2483417 ep_total_persisted 2483417 ep_uncommitted_items 3072 ep_value_size 3612419268 ep_vb0 0 ep_vb_snapshot_total 60 ep_vb_total 128 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_130_g8b09894 ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 741 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 3913612200 pid 17027 pointer_size 64 rejected_conns 0 rusage_system 107.741620 rusage_user 298.278654 threads 4 time 1351907086 total_connections 13 uptime 1117 vb_active_curr_items 2485148 vb_active_eject 963254 vb_active_expired 0 vb_active_ht_memory 12645376 vb_active_itm_memory 3788073004 vb_active_meta_data_memory 196326692 vb_active_num 128 vb_active_num_non_resident 963254 vb_active_num_ref_ejects 963254 vb_active_num_ref_items 2485148 vb_active_ops_create 2483442 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 61 vb_active_queue_age 330000 vb_active_queue_drain 2483443 vb_active_queue_fill 2485148 vb_active_queue_memory 54560 vb_active_queue_pending 81840 vb_active_queue_size 1705 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_meta_data_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-02T18:45:11.301,ns_1@10.2.1.68:ns_doctor<0.1479.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,907106,328811}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,112079192}, {processes,66327480}, {processes_used,64749144}, {system,45751712}, {atom,1405017}, {atom_used,1377299}, {binary,7189976}, {code,14117455}, {ets,4943800}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,7562485760}, {buffered_memory,219865088}, {free_memory,20983914496}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1322193,5000}}, {context_switches,{5315635,0}}, {garbage_collection,{563774,1880252282,0}}, {io,{{input,353112930},{output,77450658}}}, {reductions,{708239053,2524088}}, {run_queue,0}, {runtime,{221180,620}}]}, {system_stats, [{cpu_utilization_rate,3.5}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,6947489170}, {couch_docs_data_size,6073317216}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2497553}, {curr_items_tot,2497553}, {mem_used,3561592760}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1322}, {memory_data,{33657663488,12672860160,{<18077.16820.0>,7028184}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 20492104 kB\nBuffers: 214712 kB\nCached: 7385240 kB\nSwapCached: 0 kB\nActive: 5925756 kB\nInactive: 6127020 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 20492104 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 428 kB\nWriteback: 0 kB\nAnonPages: 4452364 kB\nMapped: 15316 kB\nSlab: 278712 kB\nPageTables: 12972 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6174900 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,907108,988028}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,90101152}, {processes,50035864}, {processes_used,50006032}, {system,40065288}, {atom,1352041}, {atom_used,1349412}, {binary,2271960}, {code,13622682}, {ets,4765112}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,5886697472}, {buffered_memory,249368576}, {free_memory,22622945280}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1324439,4999}}, {context_switches,{5398231,0}}, {garbage_collection,{598049,1172044371,0}}, {io,{{input,94608708},{output,62346128}}}, {reductions,{545186883,1849067}}, {run_queue,0}, {runtime,{189420,390}}]}, {system_stats, [{cpu_utilization_rate,4.761904761904762}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,5162693991}, {couch_docs_data_size,5110398745}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2497102}, {curr_items_tot,2497102}, {mem_used,3684981960}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1324}, {memory_data,{33657663488,10935422976,{<18118.1696.0>,2914384}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 22092844 kB\nBuffers: 243524 kB\nCached: 5748728 kB\nSwapCached: 140 kB\nActive: 4977824 kB\nInactive: 5513288 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 22092844 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 7204 kB\nWriteback: 1184 kB\nAnonPages: 4498744 kB\nMapped: 17060 kB\nSlab: 236452 kB\nPageTables: 13628 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6206056 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,907107,958654}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.13654.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,27}, {started_on,1351907050}, {total_vbuckets,129}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351907114}, {vbuckets_done,35}]]}, {memory, [{total,93694960}, {processes,52897504}, {processes_used,52828824}, {system,40797456}, {atom,1352041}, {atom_used,1349412}, {binary,3034056}, {code,13631938}, {ets,4724144}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,8357584896}, {buffered_memory,420409344}, {free_memory,19938525184}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1324397,4996}}, {context_switches,{5807076,0}}, {garbage_collection,{622671,1212722857,0}}, {io,{{input,97503149},{output,64445176}}}, {reductions,{564162424,2677089}}, {run_queue,0}, {runtime,{191660,490}}]}, {system_stats, [{cpu_utilization_rate,16.25}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7741788614}, {couch_docs_data_size,5846431956}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2503544}, {curr_items_tot,2503544}, {mem_used,3535708696}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1324}, {memory_data,{33657663488,13759135744,{<18117.1678.0>,2914384}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 19471216 kB\nBuffers: 410556 kB\nCached: 8161704 kB\nSwapCached: 0 kB\nActive: 7351668 kB\nInactive: 5685852 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 19471216 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 6808 kB\nWriteback: 0 kB\nAnonPages: 4465272 kB\nMapped: 17064 kB\nSlab: 314028 kB\nPageTables: 13672 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 6177592 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,907106,302447}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,94564288}, {processes,54196304}, {processes_used,54109432}, {system,40367984}, {atom,1352041}, {atom_used,1349412}, {binary,2556536}, {code,13659922}, {ets,4729400}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,8804958208}, {buffered_memory,318734336}, {free_memory,19601813504}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1321430,5000}}, {context_switches,{6181255,0}}, {garbage_collection,{670390,1240311057,0}}, {io,{{input,93169370},{output,64808563}}}, {reductions,{573898474,1842924}}, {run_queue,0}, {runtime,{197840,440}}]}, {system_stats, [{cpu_utilization_rate,4.0}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,8200766813}, {couch_docs_data_size,6107702164}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2501801}, {curr_items_tot,2501801}, {mem_used,3488892616}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1321}, {memory_data,{33657663488,14055596032,{<0.1662.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 19142520 kB\nBuffers: 311264 kB\nCached: 8598592 kB\nSwapCached: 12 kB\nActive: 7563604 kB\nInactive: 5790144 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 19142520 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 1988 kB\nWriteback: 0 kB\nAnonPages: 4443872 kB\nMapped: 17260 kB\nSlab: 325296 kB\nPageTables: 13544 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6145756 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-02T18:46:11.301,ns_1@10.2.1.68:ns_doctor<0.1479.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,907166,328042}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,113846728}, {processes,68124664}, {processes_used,66546328}, {system,45722064}, {atom,1405017}, {atom_used,1377299}, {binary,7097688}, {code,14117455}, {ets,5006104}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,7563505664}, {buffered_memory,219987968}, {free_memory,20990902272}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1382193,5000}}, {context_switches,{5368272,0}}, {garbage_collection,{568865,1958544668,0}}, {io,{{input,368868073},{output,79357947}}}, {reductions,{737010494,2554498}}, {run_queue,0}, {runtime,{228600,610}}]}, {system_stats, [{cpu_utilization_rate,3.2581453634085213}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,6947489135}, {couch_docs_data_size,6073317216}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2497553}, {curr_items_tot,2497553}, {mem_used,3561592760}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1382}, {memory_data,{33657663488,12672856064,{<18077.16820.0>,7028184}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 20498928 kB\nBuffers: 214832 kB\nCached: 7386236 kB\nSwapCached: 0 kB\nActive: 5918696 kB\nInactive: 6128036 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 20498928 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 464 kB\nWriteback: 0 kB\nAnonPages: 4445388 kB\nMapped: 15316 kB\nSlab: 278264 kB\nPageTables: 13028 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6160804 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,907168,988658}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,90969552}, {processes,50748616}, {processes_used,50680144}, {system,40220936}, {atom,1352041}, {atom_used,1349412}, {binary,2411648}, {code,13622682}, {ets,4785696}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,6845014016}, {buffered_memory,250339328}, {free_memory,21640024064}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1384439,5000}}, {context_switches,{5456845,0}}, {garbage_collection,{603881,1217009882,0}}, {io,{{input,95344270},{output,63652163}}}, {reductions,{567729850,1842037}}, {run_queue,0}, {runtime,{194540,450}}]}, {system_stats, [{cpu_utilization_rate,2.5}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,6231451068}, {couch_docs_data_size,6054380609}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2497102}, {curr_items_tot,2497102}, {mem_used,3652502088}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1384}, {memory_data,{33657663488,12018282496,{<18118.1696.0>,2914384}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 21132836 kB\nBuffers: 244472 kB\nCached: 6684584 kB\nSwapCached: 140 kB\nActive: 5104936 kB\nInactive: 6323332 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 21132836 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 7620 kB\nWriteback: 0 kB\nAnonPages: 4499040 kB\nMapped: 17060 kB\nSlab: 259248 kB\nPageTables: 13628 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6206380 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,907167,959787}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.13654.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,55}, {started_on,1351907050}, {total_vbuckets,129}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351907173}, {vbuckets_done,71}]]}, {memory, [{total,93549072}, {processes,52009344}, {processes_used,51973784}, {system,41539728}, {atom,1352041}, {atom_used,1349412}, {binary,3673088}, {code,13631938}, {ets,4822048}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,7745216512}, {buffered_memory,422154240}, {free_memory,20562104320}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1384398,4999}}, {context_switches,{5866185,0}}, {garbage_collection,{628827,1269174916,0}}, {io,{{input,101608136},{output,66388499}}}, {reductions,{596203296,2680607}}, {run_queue,0}, {runtime,{197080,500}}]}, {system_stats, [{cpu_utilization_rate,3.5175879396984926}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7148452735}, {couch_docs_data_size,5912198655}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2503544}, {curr_items_tot,2503544}, {mem_used,3535610472}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1384}, {memory_data,{33657663488,13152317440,{<18117.1678.0>,2914384}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 20080180 kB\nBuffers: 412260 kB\nCached: 7563688 kB\nSwapCached: 0 kB\nActive: 6479356 kB\nInactive: 5964672 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 20080180 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 40280 kB\nWriteback: 0 kB\nAnonPages: 4468076 kB\nMapped: 17064 kB\nSlab: 298732 kB\nPageTables: 13676 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 6180340 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,907166,302507}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,92021528}, {processes,51388464}, {processes_used,51301592}, {system,40633064}, {atom,1352041}, {atom_used,1349412}, {binary,2646912}, {code,13659922}, {ets,4904112}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,8806055936}, {buffered_memory,318844928}, {free_memory,19603083264}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1381430,5000}}, {context_switches,{6228081,0}}, {garbage_collection,{674951,1284428015,0}}, {io,{{input,93887805},{output,66127844}}}, {reductions,{596057478,1850056}}, {run_queue,0}, {runtime,{203370,460}}]}, {system_stats, [{cpu_utilization_rate,4.0}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,8200766792}, {couch_docs_data_size,6107702164}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2501801}, {curr_items_tot,2501801}, {mem_used,3488892616}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,14054326272,{<0.1662.0>,2914384}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 19143636 kB\nBuffers: 311372 kB\nCached: 8599664 kB\nSwapCached: 12 kB\nActive: 7561056 kB\nInactive: 5791128 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 19143636 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 544 kB\nWriteback: 0 kB\nAnonPages: 4441144 kB\nMapped: 17260 kB\nSlab: 324968 kB\nPageTables: 13540 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6145336 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-02T18:46:26.820,ns_1@10.2.1.68:<0.1669.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,31,47}}) Stats for bucket "default": accepting_conns 1 auth_cmds 3 auth_errors 0 bucket_active_conns 1 bucket_conns 5 bytes 3488892616 bytes_read 6003862304 bytes_written 126598507 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 2501801 conn_yields 851 connection_structures 5000 curr_connections 9 curr_conns_on_port_11209 6 curr_conns_on_port_11210 1 curr_items 2501801 curr_items_tot 2501801 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-03 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 1600 ep_commit_time 5 ep_commit_time_total 10441 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 4 ep_data_age_highwat 62 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data/default ep_degraded_mode 0 ep_diskqueue_drain 2501801 ep_diskqueue_fill 2501801 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 888 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 2501801 ep_io_read_bytes 0 ep_io_write_bytes 5923674684 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 2496590 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 3426507387 ep_max_checkpoints 2 ep_max_data_size 5242880000 ep_max_item_size 20971520 ep_max_size 5242880000 ep_max_txn_size 10000 ep_max_vbuckets 512 ep_mem_high_wat 3932160000 ep_mem_high_wat 3932160000 ep_mem_low_wat 3145728000 ep_mem_low_wat 3145728000 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 373351 ep_num_expiry_pager_runs 0 ep_num_non_resident 1147214 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 12 ep_num_value_ejects 1147214 ep_oom_errors 0 ep_overhead 13495288 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1351905984 ep_storage_age 1 ep_storage_age_highwat 62 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 3396485775 ep_total_del_items 0 ep_total_enqueued 2501801 ep_total_new_items 2501801 ep_total_persisted 2501801 ep_uncommitted_items 0 ep_value_size 3219561761 ep_vb0 0 ep_vb_snapshot_total 61 ep_vb_total 128 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_130_g8b09894 ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 741 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 3488892616 pid 17027 pointer_size 64 rejected_conns 0 rusage_system 108.500505 rusage_user 301.955095 threads 4 time 1351907186 total_connections 13 uptime 1217 vb_active_curr_items 2501801 vb_active_eject 1147214 vb_active_expired 0 vb_active_ht_memory 12645376 vb_active_itm_memory 3396485775 vb_active_meta_data_memory 197642279 vb_active_num 128 vb_active_num_non_resident 1147214 vb_active_num_ref_ejects 1147214 vb_active_num_ref_items 2501801 vb_active_ops_create 2501801 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 54 vb_active_queue_age 0 vb_active_queue_drain 2501801 vb_active_queue_fill 2501801 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_meta_data_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-02T18:47:11.301,ns_1@10.2.1.68:ns_doctor<0.1479.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,907226,327798}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,114586224}, {processes,68433488}, {processes_used,66855152}, {system,46152736}, {atom,1405017}, {atom_used,1377299}, {binary,7351520}, {code,14117455}, {ets,5183456}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,7564632064}, {buffered_memory,220209152}, {free_memory,20982775808}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1442193,5000}}, {context_switches,{5422704,0}}, {garbage_collection,{574284,2038832741,0}}, {io,{{input,388262841},{output,84601563}}}, {reductions,{766810165,2593695}}, {run_queue,0}, {runtime,{236120,630}}]}, {system_stats, [{cpu_utilization_rate,3.2581453634085213}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,6947489102}, {couch_docs_data_size,6073317216}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2497553}, {curr_items_tot,2497553}, {mem_used,3561592760}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1442}, {memory_data,{33657663488,12675395584,{<18077.16820.0>,7028184}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 20490992 kB\nBuffers: 215048 kB\nCached: 7387336 kB\nSwapCached: 0 kB\nActive: 5925320 kB\nInactive: 6129296 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 20490992 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 328 kB\nWriteback: 0 kB\nAnonPages: 4452368 kB\nMapped: 15316 kB\nSlab: 278216 kB\nPageTables: 12920 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6178272 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,907228,990220}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,93276824}, {processes,52699160}, {processes_used,52630688}, {system,40577664}, {atom,1352041}, {atom_used,1349412}, {binary,2590848}, {code,13622682}, {ets,4963464}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,6845992960}, {buffered_memory,250556416}, {free_memory,21642055680}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1444440,5000}}, {context_switches,{5505400,0}}, {garbage_collection,{608625,1263136864,0}}, {io,{{input,99784265},{output,68286850}}}, {reductions,{590869077,1855100}}, {run_queue,0}, {runtime,{200320,460}}]}, {system_stats, [{cpu_utilization_rate,2.506265664160401}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,6231451007}, {couch_docs_data_size,6054380609}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2497102}, {curr_items_tot,2497102}, {mem_used,3652502072}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1444}, {memory_data,{33657663488,12017901568,{<18118.1696.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 21134944 kB\nBuffers: 244684 kB\nCached: 6685540 kB\nSwapCached: 140 kB\nActive: 5101936 kB\nInactive: 6324632 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 21134944 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 6400 kB\nWriteback: 0 kB\nAnonPages: 4496184 kB\nMapped: 17060 kB\nSlab: 259176 kB\nPageTables: 13636 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6209048 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,907227,959695}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.13654.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,79}, {started_on,1351907050}, {total_vbuckets,129}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351907232}, {vbuckets_done,103}]]}, {memory, [{total,92840168}, {processes,51372096}, {processes_used,51328256}, {system,41468072}, {atom,1352041}, {atom_used,1349412}, {binary,3433712}, {code,13631938}, {ets,4990144}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,7227478016}, {buffered_memory,423780352}, {free_memory,21094133760}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1444397,4997}}, {context_switches,{5922027,0}}, {garbage_collection,{635106,1317107846,0}}, {io,{{input,106654191},{output,71107593}}}, {reductions,{620633760,1921464}}, {run_queue,0}, {runtime,{202320,470}}]}, {system_stats, [{cpu_utilization_rate,3.75}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,6608204741}, {couch_docs_data_size,5987692593}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2503544}, {curr_items_tot,2503544}, {mem_used,3510927608}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1444}, {memory_data,{33657663488,12600225792,{<18117.1678.0>,2914384}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 20599740 kB\nBuffers: 413848 kB\nCached: 7058084 kB\nSwapCached: 0 kB\nActive: 5735940 kB\nInactive: 6201304 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 20599740 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 35080 kB\nWriteback: 0 kB\nAnonPages: 4465364 kB\nMapped: 17064 kB\nSlab: 285856 kB\nPageTables: 13672 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 6176404 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,907226,302920}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,92914416}, {processes,51672552}, {processes_used,51585680}, {system,41241864}, {atom,1352041}, {atom_used,1349412}, {binary,3191720}, {code,13659922}, {ets,4967784}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,8807190528}, {buffered_memory,318951424}, {free_memory,19596099584}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1441430,5000}}, {context_switches,{6279453,0}}, {garbage_collection,{679727,1339019846,0}}, {io,{{input,97370263},{output,67988740}}}, {reductions,{626849180,2635298}}, {run_queue,0}, {runtime,{209700,540}}]}, {system_stats, [{cpu_utilization_rate,4.010025062656641}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,8200766792}, {couch_docs_data_size,6107702164}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2501801}, {curr_items_tot,2501801}, {mem_used,3488892616}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1441}, {memory_data,{33657663488,14063849472,{<0.1662.0>,2914384}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 19136816 kB\nBuffers: 311476 kB\nCached: 8600772 kB\nSwapCached: 12 kB\nActive: 7566140 kB\nInactive: 5792164 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 19136816 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 68 kB\nWriteback: 0 kB\nAnonPages: 4446084 kB\nMapped: 17260 kB\nSlab: 324816 kB\nPageTables: 13564 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6146924 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-02T18:48:06.820,ns_1@10.2.1.68:<0.1669.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,31,47}}) Stats for bucket "default": accepting_conns 1 auth_cmds 3 auth_errors 0 bucket_active_conns 1 bucket_conns 5 bytes 3488892616 bytes_read 6003872794 bytes_written 133110907 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 2501801 conn_yields 851 connection_structures 5000 curr_connections 9 curr_conns_on_port_11209 6 curr_conns_on_port_11210 1 curr_items 2501801 curr_items_tot 2501801 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-03 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 1600 ep_commit_time 5 ep_commit_time_total 10441 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 4 ep_data_age_highwat 62 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data/default ep_degraded_mode 0 ep_diskqueue_drain 2501801 ep_diskqueue_fill 2501801 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 888 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 2501801 ep_io_read_bytes 0 ep_io_write_bytes 5923674684 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 2496590 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 3426507387 ep_max_checkpoints 2 ep_max_data_size 5242880000 ep_max_item_size 20971520 ep_max_size 5242880000 ep_max_txn_size 10000 ep_max_vbuckets 512 ep_mem_high_wat 3932160000 ep_mem_high_wat 3932160000 ep_mem_low_wat 3145728000 ep_mem_low_wat 3145728000 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 373351 ep_num_expiry_pager_runs 0 ep_num_non_resident 1147214 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 12 ep_num_value_ejects 1147214 ep_oom_errors 0 ep_overhead 13495288 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1351905984 ep_storage_age 1 ep_storage_age_highwat 62 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 3396485775 ep_total_del_items 0 ep_total_enqueued 2501801 ep_total_new_items 2501801 ep_total_persisted 2501801 ep_uncommitted_items 0 ep_value_size 3219561761 ep_vb0 0 ep_vb_snapshot_total 61 ep_vb_total 128 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_130_g8b09894 ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 741 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 3488892616 pid 17027 pointer_size 64 rejected_conns 0 rusage_system 108.518502 rusage_user 302.179061 threads 4 time 1351907286 total_connections 13 uptime 1317 vb_active_curr_items 2501801 vb_active_eject 1147214 vb_active_expired 0 vb_active_ht_memory 12645376 vb_active_itm_memory 3396485775 vb_active_meta_data_memory 197642279 vb_active_num 128 vb_active_num_non_resident 1147214 vb_active_num_ref_ejects 1147214 vb_active_num_ref_items 2501801 vb_active_ops_create 2501801 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 54 vb_active_queue_age 0 vb_active_queue_drain 2501801 vb_active_queue_fill 2501801 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_meta_data_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-02T18:48:11.301,ns_1@10.2.1.68:ns_doctor<0.1479.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,907286,326904}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.24220.0>">>}, {changes_done,0}, {design_documents,[<<"_design/A">>]}, {indexer_type,main}, {initial_build,true}, {progress,0}, {set,<<"default">>}, {signature,<<"afeb9408e40add49ae0c9bf58f0c5f26">>}, {started_on,1351907326}, {total_changes,2497553}, {type,indexer}, {updated_on,1351907326}]]}, {memory, [{total,129998272}, {processes,79995912}, {processes_used,78466384}, {system,50002360}, {atom,1490553}, {atom_used,1476320}, {binary,10209992}, {code,14915198}, {ets,5283616}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,7565516800}, {buffered_memory,220364800}, {free_memory,20974362624}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1502193,5000}}, {context_switches,{5494063,0}}, {garbage_collection,{581079,2130655938,0}}, {io,{{input,445071266},{output,86919119}}}, {reductions,{803697066,10513532}}, {run_queue,0}, {runtime,{244250,1310}}]}, {system_stats, [{cpu_utilization_rate,6.234413965087282}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,6947489102}, {couch_docs_data_size,6073317216}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2497553}, {curr_items_tot,2497553}, {mem_used,3561592760}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1502}, {memory_data,{33657663488,12678443008,{<18077.16820.0>,7028184}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 20482776 kB\nBuffers: 215200 kB\nCached: 7388200 kB\nSwapCached: 0 kB\nActive: 5947104 kB\nInactive: 6115632 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 20482776 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 0 kB\nWriteback: 0 kB\nAnonPages: 4458948 kB\nMapped: 15644 kB\nSlab: 278132 kB\nPageTables: 12948 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6184940 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,907288,990595}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.14982.0>">>}, {changes_done,6807}, {design_documents,[<<"_design/A">>]}, {indexer_type,main}, {initial_build,true}, {progress,0}, {set,<<"default">>}, {signature,<<"afeb9408e40add49ae0c9bf58f0c5f26">>}, {started_on,1351907259}, {total_changes,2497102}, {type,indexer}, {updated_on,1351907259}]]}, {memory, [{total,152608920}, {processes,106537192}, {processes_used,106489344}, {system,46071728}, {atom,1406633}, {atom_used,1382237}, {binary,7255144}, {code,14299859}, {ets,5053216}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,6849789952}, {buffered_memory,250699776}, {free_memory,21582868480}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1504440,5000}}, {context_switches,{5722249,0}}, {garbage_collection,{628975,1421713665,0}}, {io,{{input,594721809},{output,74633247}}}, {reductions,{673934477,60248743}}, {run_queue,0}, {runtime,{210810,5190}}]}, {system_stats, [{cpu_utilization_rate,49.3734335839599}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,6231450962}, {couch_docs_data_size,6054380609}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2497102}, {curr_items_tot,2497102}, {mem_used,3652502072}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1504}, {memory_data,{33657663488,12026408960,{<18118.1696.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 21077144 kB\nBuffers: 244824 kB\nCached: 6689248 kB\nSwapCached: 140 kB\nActive: 5331256 kB\nInactive: 6154232 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 21077144 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 7580 kB\nWriteback: 0 kB\nAnonPages: 4550944 kB\nMapped: 17964 kB\nSlab: 258756 kB\nPageTables: 13800 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6293632 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,907287,959236}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.16400.0>">>}, {changes_done,6495}, {design_documents,[<<"_design/A">>]}, {indexer_type,main}, {initial_build,true}, {progress,0}, {set,<<"default">>}, {signature,<<"afeb9408e40add49ae0c9bf58f0c5f26">>}, {started_on,1351907293}, {total_changes,2503544}, {type,indexer}, {updated_on,1351907294}]]}, {memory, [{total,112693736}, {processes,65310808}, {processes_used,65247440}, {system,47382928}, {atom,1406633}, {atom_used,1382237}, {binary,8563400}, {code,14309115}, {ets,5039624}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,6784684032}, {buffered_memory,425107456}, {free_memory,21497909248}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1504397,4995}}, {context_switches,{6088381,0}}, {garbage_collection,{651725,1436041836,0}}, {io,{{input,424034107},{output,76600609}}}, {reductions,{684090941,41620717}}, {run_queue,0}, {runtime,{210850,3660}}]}, {system_stats, [{cpu_utilization_rate,45.86466165413534}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,6128133555}, {couch_docs_data_size,6075992082}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2503544}, {curr_items_tot,2503544}, {mem_used,3506636824}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1504}, {memory_data,{33657663488,12107931648,{<18117.1678.0>,2914384}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 20994052 kB\nBuffers: 415144 kB\nCached: 6625668 kB\nSwapCached: 0 kB\nActive: 5279032 kB\nInactive: 6277176 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 20994052 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 4672 kB\nWriteback: 0 kB\nAnonPages: 4515332 kB\nMapped: 17964 kB\nSlab: 273772 kB\nPageTables: 13740 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 6234876 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,907286,302329}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.15038.0>">>}, {changes_done,0}, {design_documents,[<<"_design/A">>]}, {indexer_type,main}, {initial_build,true}, {progress,0}, {set,<<"default">>}, {signature,<<"afeb9408e40add49ae0c9bf58f0c5f26">>}, {started_on,1351907286}, {total_changes,2501801}, {type,indexer}, {updated_on,1351907286}]]}, {memory, [{total,99455448}, {processes,55219576}, {processes_used,55156208}, {system,44235872}, {atom,1406633}, {atom_used,1381764}, {binary,5250272}, {code,14331472}, {ets,5171120}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,8808124416}, {buffered_memory,319188992}, {free_memory,19588718592}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1501430,5000}}, {context_switches,{6345999,0}}, {garbage_collection,{686071,1402933527,0}}, {io,{{input,135899158},{output,73381778}}}, {reductions,{663082832,7997581}}, {run_queue,0}, {runtime,{216480,950}}]}, {system_stats, [{cpu_utilization_rate,5.5}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,8200766792}, {couch_docs_data_size,6107702164}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2501801}, {curr_items_tot,2501801}, {mem_used,3488892616}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1501}, {memory_data,{33657663488,14069309440,{<0.1662.0>,2914384}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 19129732 kB\nBuffers: 311708 kB\nCached: 8601684 kB\nSwapCached: 12 kB\nActive: 7584464 kB\nInactive: 5782008 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 19129732 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 372 kB\nWriteback: 0 kB\nAnonPages: 4452960 kB\nMapped: 17588 kB\nSlab: 324788 kB\nPageTables: 13556 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6161292 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-02T18:49:11.301,ns_1@10.2.1.68:ns_doctor<0.1479.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,907346,325874}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.24220.0>">>}, {changes_done,1320848}, {design_documents,[<<"_design/A">>]}, {indexer_type,main}, {initial_build,true}, {progress,52}, {set,<<"default">>}, {signature,<<"afeb9408e40add49ae0c9bf58f0c5f26">>}, {started_on,1351907326}, {total_changes,2497553}, {type,indexer}, {updated_on,1351907383}]]}, {memory, [{total,187904880}, {processes,134851816}, {processes_used,133325288}, {system,53053064}, {atom,1492169}, {atom_used,1478827}, {binary,12937248}, {code,15028654}, {ets,5489288}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,7666941952}, {buffered_memory,220950528}, {free_memory,20817666048}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1562193,5001}}, {context_switches,{8707479,0}}, {garbage_collection,{875562,4668276035,0}}, {io,{{input,9696494081},{output,308544851}}}, {reductions,{2047858402,102999075}}, {run_queue,0}, {runtime,{350510,8780}}]}, {system_stats, [{cpu_utilization_rate,50.37593984962406}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,6947493243}, {couch_docs_data_size,6073317216}, {couch_views_actual_disk_size,95799725}, {couch_views_data_size,8775}, {curr_items,2497553}, {curr_items_tot,2497553}, {mem_used,3561592760}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1562}, {memory_data,{33657663488,12853837824,{<18077.24220.0>,36982592}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 20329752 kB\nBuffers: 215772 kB\nCached: 7487248 kB\nSwapCached: 0 kB\nActive: 9172240 kB\nInactive: 3039472 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 20329752 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 94040 kB\nWriteback: 0 kB\nAnonPages: 4508836 kB\nMapped: 16276 kB\nSlab: 281380 kB\nPageTables: 13116 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6271444 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,907348,990261}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.14982.0>">>}, {changes_done,1347257}, {design_documents,[<<"_design/A">>]}, {indexer_type,main}, {initial_build,true}, {progress,53}, {set,<<"default">>}, {signature,<<"afeb9408e40add49ae0c9bf58f0c5f26">>}, {started_on,1351907259}, {total_changes,2497102}, {type,indexer}, {updated_on,1351907317}]]}, {memory, [{total,141577112}, {processes,94360864}, {processes_used,94311576}, {system,47216248}, {atom,1408249}, {atom_used,1384271}, {binary,8076000}, {code,14415880}, {ets,5258232}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,6950764544}, {buffered_memory,251248640}, {free_memory,21465034752}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1564439,4997}}, {context_switches,{8917363,0}}, {garbage_collection,{923035,3937554002,0}}, {io,{{input,9839689083},{output,295808933}}}, {reductions,{1909875623,96604515}}, {run_queue,0}, {runtime,{316760,8330}}]}, {system_stats, [{cpu_utilization_rate,46.13466334164588}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,6231455103}, {couch_docs_data_size,6054380609}, {couch_views_actual_disk_size,95749206}, {couch_views_data_size,8776}, {curr_items,2497102}, {curr_items_tot,2497102}, {mem_used,3652502072}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1564}, {memory_data,{33657663488,12237205504,{<18118.14982.0>,74966016}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 20961948 kB\nBuffers: 245360 kB\nCached: 6787856 kB\nSwapCached: 140 kB\nActive: 8569668 kB\nInactive: 3028300 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 20961948 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 96252 kB\nWriteback: 0 kB\nAnonPages: 4564176 kB\nMapped: 17964 kB\nSlab: 261628 kB\nPageTables: 13852 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6294096 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,907347,960866}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.16400.0>">>}, {changes_done,1435075}, {design_documents,[<<"_design/A">>]}, {indexer_type,main}, {initial_build,true}, {progress,57}, {set,<<"default">>}, {signature,<<"afeb9408e40add49ae0c9bf58f0c5f26">>}, {started_on,1351907293}, {total_changes,2503544}, {type,indexer}, {updated_on,1351907354}]]}, {memory, [{total,128270016}, {processes,81371168}, {processes_used,81306216}, {system,46898848}, {atom,1408249}, {atom_used,1384271}, {binary,7755584}, {code,14425136}, {ets,5244680}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,6887120896}, {buffered_memory,425611264}, {free_memory,21366489088}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1564398,4999}}, {context_switches,{9350923,0}}, {garbage_collection,{956659,3987546897,0}}, {io,{{input,9922006794},{output,297373121}}}, {reductions,{1944133698,96504104}}, {run_queue,1}, {runtime,{316280,8180}}]}, {system_stats, [{cpu_utilization_rate,43.25}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,6128137651}, {couch_docs_data_size,6075992082}, {couch_views_actual_disk_size,98638846}, {couch_views_data_size,8782}, {curr_items,2503544}, {curr_items_tot,2503544}, {mem_used,3506636808}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1564}, {memory_data,{33657663488,12314157056,{<18117.16400.0>,74965952}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 20865712 kB\nBuffers: 415636 kB\nCached: 6725704 kB\nSwapCached: 0 kB\nActive: 8609556 kB\nInactive: 3072848 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 20865712 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 93976 kB\nWriteback: 0 kB\nAnonPages: 4541160 kB\nMapped: 17964 kB\nSlab: 275492 kB\nPageTables: 13852 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 6264132 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,907346,301599}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.15038.0>">>}, {changes_done,1369479}, {design_documents,[<<"_design/A">>]}, {indexer_type,main}, {initial_build,true}, {progress,54}, {set,<<"default">>}, {signature,<<"afeb9408e40add49ae0c9bf58f0c5f26">>}, {started_on,1351907286}, {total_changes,2501801}, {type,indexer}, {updated_on,1351907346}]]}, {memory, [{total,155817464}, {processes,104974848}, {processes_used,104910040}, {system,50842616}, {atom,1408249}, {atom_used,1384271}, {binary,11639776}, {code,14453120}, {ets,5263976}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,8907649024}, {buffered_memory,319762432}, {free_memory,19407659008}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1561429,4998}}, {context_switches,{9507311,0}}, {garbage_collection,{975937,3880284530,0}}, {io,{{input,9274145300},{output,289825683}}}, {reductions,{1887131649,93179919}}, {run_queue,0}, {runtime,{322880,8420}}]}, {system_stats, [{cpu_utilization_rate,48.37092731829574}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,8200770933}, {couch_docs_data_size,6107702164}, {couch_views_actual_disk_size,94808562}, {couch_views_data_size,8774}, {curr_items,2501801}, {curr_items_tot,2501801}, {mem_used,3488892616}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1561}, {memory_data,{33657663488,14244925440,{<0.15038.0>,51855896}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 18952792 kB\nBuffers: 312268 kB\nCached: 8698876 kB\nSwapCached: 12 kB\nActive: 10595200 kB\nInactive: 2945504 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 18952792 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 92240 kB\nWriteback: 0 kB\nAnonPages: 4529332 kB\nMapped: 18164 kB\nSlab: 328128 kB\nPageTables: 13600 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6239576 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-02T18:49:46.816,ns_1@10.2.1.68:<0.1669.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,31,47}}) Stats for bucket "default": accepting_conns 1 auth_cmds 3 auth_errors 0 bucket_active_conns 1 bucket_conns 5 bytes 3488892616 bytes_read 6003883284 bytes_written 139625050 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 2501801 conn_yields 851 connection_structures 5000 curr_connections 9 curr_conns_on_port_11209 6 curr_conns_on_port_11210 1 curr_items 2501801 curr_items_tot 2501801 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-03 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 1600 ep_commit_time 5 ep_commit_time_total 10441 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 4 ep_data_age_highwat 62 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data/default ep_degraded_mode 0 ep_diskqueue_drain 2501801 ep_diskqueue_fill 2501801 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 888 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 2501801 ep_io_read_bytes 0 ep_io_write_bytes 5923674684 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 2496590 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 3426507387 ep_max_checkpoints 2 ep_max_data_size 5242880000 ep_max_item_size 20971520 ep_max_size 5242880000 ep_max_txn_size 10000 ep_max_vbuckets 512 ep_mem_high_wat 3932160000 ep_mem_high_wat 3932160000 ep_mem_low_wat 3145728000 ep_mem_low_wat 3145728000 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 373351 ep_num_expiry_pager_runs 0 ep_num_non_resident 1147214 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 12 ep_num_value_ejects 1147214 ep_oom_errors 0 ep_overhead 13495288 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1351905984 ep_storage_age 1 ep_storage_age_highwat 62 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 3396485775 ep_total_del_items 0 ep_total_enqueued 2501801 ep_total_new_items 2501801 ep_total_persisted 2501801 ep_uncommitted_items 0 ep_value_size 3219561761 ep_vb0 0 ep_vb_snapshot_total 61 ep_vb_total 128 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_130_g8b09894 ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 741 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 3488892616 pid 17027 pointer_size 64 rejected_conns 0 rusage_system 108.523501 rusage_user 302.281046 threads 4 time 1351907386 total_connections 13 uptime 1417 vb_active_curr_items 2501801 vb_active_eject 1147214 vb_active_expired 0 vb_active_ht_memory 12645376 vb_active_itm_memory 3396485775 vb_active_meta_data_memory 197642279 vb_active_num 128 vb_active_num_non_resident 1147214 vb_active_num_ref_ejects 1147214 vb_active_num_ref_items 2501801 vb_active_ops_create 2501801 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 54 vb_active_queue_age 0 vb_active_queue_drain 2501801 vb_active_queue_fill 2501801 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_meta_data_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-02T18:50:11.301,ns_1@10.2.1.68:ns_doctor<0.1479.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,907406,325330}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.24220.0>">>}, {changes_done,2397729}, {design_documents,[<<"_design/A">>]}, {indexer_type,main}, {initial_build,true}, {progress,96}, {set,<<"default">>}, {signature,<<"afeb9408e40add49ae0c9bf58f0c5f26">>}, {started_on,1351907326}, {total_changes,2497553}, {type,indexer}, {updated_on,1351907429}]]}, {memory, [{total,219087400}, {processes,163063856}, {processes_used,161530552}, {system,56023544}, {atom,1495401}, {atom_used,1483293}, {binary,15652784}, {code,15096052}, {ets,5678264}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,7726153728}, {buffered_memory,221806592}, {free_memory,20706390016}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1622193,5001}}, {context_switches,{11373561,0}}, {garbage_collection,{1122178,7344095249,0}}, {io,{{input,17236731448},{output,705505962}}}, {reductions,{3286477224,131545420}}, {run_queue,0}, {runtime,{449340,4950}}]}, {system_stats, [{cpu_utilization_rate,25.12562814070352}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,6947493243}, {couch_docs_data_size,6073317216}, {couch_views_actual_disk_size,271013781}, {couch_views_data_size,8775}, {curr_items,2497553}, {curr_items_tot,2497553}, {mem_used,3561592760}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1622}, {memory_data,{33657663488,13047967744,{<18077.24220.0>,91014736}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 20220772 kB\nBuffers: 216608 kB\nCached: 7545400 kB\nSwapCached: 0 kB\nActive: 11731444 kB\nInactive: 584116 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 20220772 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 102632 kB\nWriteback: 0 kB\nAnonPages: 4552964 kB\nMapped: 16304 kB\nSlab: 286832 kB\nPageTables: 13544 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6295628 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,907408,993092}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.14982.0>">>}, {changes_done,2437524}, {design_documents,[<<"_design/A">>]}, {indexer_type,main}, {initial_build,true}, {progress,97}, {set,<<"default">>}, {signature,<<"afeb9408e40add49ae0c9bf58f0c5f26">>}, {started_on,1351907259}, {total_changes,2497102}, {type,indexer}, {updated_on,1351907364}]]}, {memory, [{total,169672688}, {processes,111043968}, {processes_used,110990288}, {system,58628720}, {atom,1411481}, {atom_used,1388780}, {binary,19229968}, {code,14483278}, {ets,5446696}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,6978482176}, {buffered_memory,252059648}, {free_memory,21412462592}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1624441,5001}}, {context_switches,{11518172,0}}, {garbage_collection,{1156261,6601202423,0}}, {io,{{input,16996180611},{output,714017585}}}, {reductions,{3248067178,249100522}}, {run_queue,0}, {runtime,{412690,4620}}]}, {system_stats, [{cpu_utilization_rate,24.25}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,6231455103}, {couch_docs_data_size,6054380609}, {couch_views_actual_disk_size,273767660}, {couch_views_data_size,8776}, {curr_items,2497102}, {curr_items_tot,2497102}, {mem_used,3652502072}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1624}, {memory_data,{33657663488,12306178048,{<18118.14982.0>,58013208}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 20910608 kB\nBuffers: 246152 kB\nCached: 6814924 kB\nSwapCached: 140 kB\nActive: 11006656 kB\nInactive: 638892 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 20910608 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 118912 kB\nWriteback: 0 kB\nAnonPages: 4584264 kB\nMapped: 17964 kB\nSlab: 264232 kB\nPageTables: 13908 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6311120 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,907407,962036}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.16400.0>">>}, {changes_done,2478111}, {design_documents,[<<"_design/A">>]}, {indexer_type,main}, {initial_build,true}, {progress,98}, {set,<<"default">>}, {signature,<<"afeb9408e40add49ae0c9bf58f0c5f26">>}, {started_on,1351907293}, {total_changes,2503544}, {type,indexer}, {updated_on,1351907399}]]}, {memory, [{total,157607376}, {processes,89265440}, {processes_used,89196240}, {system,68341936}, {atom,1411481}, {atom_used,1388780}, {binary,28940592}, {code,14492534}, {ets,5432856}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,6905298944}, {buffered_memory,426364928}, {free_memory,21358399488}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1624398,4998}}, {context_switches,{11980683,0}}, {garbage_collection,{1191472,6704592786,0}}, {io,{{input,17113702048},{output,719298654}}}, {reductions,{3335002807,287084679}}, {run_queue,0}, {runtime,{410800,4370}}]}, {system_stats, [{cpu_utilization_rate,29.75}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,6128137605}, {couch_docs_data_size,6075992082}, {couch_views_actual_disk_size,186335467}, {couch_views_data_size,0}, {curr_items,2503544}, {curr_items_tot,2503544}, {mem_used,3506636808}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1624}, {memory_data,{33657663488,12343693312,{<18117.16400.0>,36982640}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 20857812 kB\nBuffers: 416372 kB\nCached: 6743456 kB\nSwapCached: 0 kB\nActive: 11007416 kB\nInactive: 678544 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 20857812 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 120812 kB\nWriteback: 0 kB\nAnonPages: 4525964 kB\nMapped: 17964 kB\nSlab: 278172 kB\nPageTables: 13804 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 6249440 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,907406,301658}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.15038.0>">>}, {changes_done,2445274}, {design_documents,[<<"_design/A">>]}, {indexer_type,main}, {initial_build,true}, {progress,97}, {set,<<"default">>}, {signature,<<"afeb9408e40add49ae0c9bf58f0c5f26">>}, {started_on,1351907286}, {total_changes,2501801}, {type,indexer}, {updated_on,1351907393}]]}, {memory, [{total,177220416}, {processes,129005288}, {processes_used,128937968}, {system,48215128}, {atom,1408249}, {atom_used,1384271}, {binary,8657928}, {code,14453120}, {ets,5447624}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,9078046720}, {buffered_memory,320446464}, {free_memory,19230797824}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1621429,4999}}, {context_switches,{12175997,0}}, {garbage_collection,{1221874,6474817869,0}}, {io,{{input,16942586149},{output,659646858}}}, {reductions,{3050594082,65619124}}, {run_queue,0}, {runtime,{421840,4910}}]}, {system_stats, [{cpu_utilization_rate,26.75}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,8200770933}, {couch_docs_data_size,6107702164}, {couch_views_actual_disk_size,248656360}, {couch_views_data_size,8774}, {curr_items,2501801}, {curr_items_tot,2501801}, {mem_used,3488892616}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1621}, {memory_data,{33657663488,14413533184,{<0.15038.0>,74966056}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 18780324 kB\nBuffers: 312936 kB\nCached: 8865280 kB\nSwapCached: 12 kB\nActive: 13076852 kB\nInactive: 630444 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 18780324 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 228396 kB\nWriteback: 0 kB\nAnonPages: 4528520 kB\nMapped: 18164 kB\nSlab: 332252 kB\nPageTables: 13820 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6250048 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-02T18:51:11.300,ns_1@10.2.1.68:ns_doctor<0.1479.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,907466,325423}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,119082104}, {processes,69774176}, {processes_used,68238016}, {system,49307928}, {atom,1495401}, {atom_used,1483293}, {binary,8747104}, {code,15096052}, {ets,5867920}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,7665856512}, {buffered_memory,222113792}, {free_memory,20840632320}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1682193,5000}}, {context_switches,{12028558,0}}, {garbage_collection,{1132920,8285645090,0}}, {io,{{input,17505933735},{output,885610636}}}, {reductions,{4420059485,2560168}}, {run_queue,0}, {runtime,{472960,610}}]}, {system_stats, [{cpu_utilization_rate,3.25}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,6947493243}, {couch_docs_data_size,6073317216}, {couch_views_actual_disk_size,102294458}, {couch_views_data_size,102278150}, {curr_items,2497553}, {curr_items_tot,2497553}, {mem_used,3561592760}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1682}, {memory_data,{33657663488,12818935808,{<18077.24182.0>,10771072}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 20352180 kB\nBuffers: 216908 kB\nCached: 7486188 kB\nSwapCached: 0 kB\nActive: 11668812 kB\nInactive: 519204 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 20352180 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 91868 kB\nWriteback: 0 kB\nAnonPages: 4485040 kB\nMapped: 16276 kB\nSlab: 283508 kB\nPageTables: 13232 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6207920 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,907468,992160}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,96318152}, {processes,53578520}, {processes_used,53522912}, {system,42739632}, {atom,1411481}, {atom_used,1388780}, {binary,3127768}, {code,14483278}, {ets,5660176}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,6947336192}, {buffered_memory,252407808}, {free_memory,21509697536}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1684439,5000}}, {context_switches,{12084444,0}}, {garbage_collection,{1166036,7415713001,0}}, {io,{{input,17213832263},{output,867653494}}}, {reductions,{4219396308,2251997}}, {run_queue,0}, {runtime,{433050,530}}]}, {system_stats, [{cpu_utilization_rate,3.491271820448878}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,6231455103}, {couch_docs_data_size,6054380609}, {couch_views_actual_disk_size,102421439}, {couch_views_data_size,102406871}, {curr_items,2497102}, {curr_items_tot,2497102}, {mem_used,3652502072}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1684}, {memory_data,{33657663488,12147331072,{<18118.1696.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 21005564 kB\nBuffers: 246492 kB\nCached: 6784508 kB\nSwapCached: 140 kB\nActive: 10948644 kB\nInactive: 603296 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 21005564 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 93600 kB\nWriteback: 0 kB\nAnonPages: 4520716 kB\nMapped: 17964 kB\nSlab: 262888 kB\nPageTables: 13752 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6229580 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,907467,963522}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,99413128}, {processes,56199872}, {processes_used,56125224}, {system,43213256}, {atom,1413905}, {atom_used,1392521}, {binary,3505280}, {code,14569021}, {ets,5648832}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,6842019840}, {buffered_memory,426614784}, {free_memory,21468479488}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1684398,4998}}, {context_switches,{12542461,0}}, {garbage_collection,{1208103,7545468534,0}}, {io,{{input,17332541787},{output,874370611}}}, {reductions,{4276961956,12635385}}, {run_queue,0}, {runtime,{429530,1060}}]}, {system_stats, [{cpu_utilization_rate,46.5}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,6128137605}, {couch_docs_data_size,6075992082}, {couch_views_actual_disk_size,102691789}, {couch_views_data_size,102676813}, {curr_items,2483952}, {curr_items_tot,2483952}, {mem_used,3485302880}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1684}, {memory_data,{33657663488,12230569984,{<18117.1678.0>,4714984}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 20965312 kB\nBuffers: 416616 kB\nCached: 6681660 kB\nSwapCached: 0 kB\nActive: 10928636 kB\nInactive: 650920 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 20965312 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 8668 kB\nWriteback: 0 kB\nAnonPages: 4481068 kB\nMapped: 17972 kB\nSlab: 278328 kB\nPageTables: 13684 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 6185688 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,907466,302319}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,97130912}, {processes,53784960}, {processes_used,53713760}, {system,43345952}, {atom,1411481}, {atom_used,1388780}, {binary,3693424}, {code,14520518}, {ets,5643984}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,8910467072}, {buffered_memory,320872448}, {free_memory,19460952064}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1681430,5000}}, {context_switches,{12875697,0}}, {garbage_collection,{1233470,7538873439,0}}, {io,{{input,17224674800},{output,865026547}}}, {reductions,{4271339220,1855637}}, {run_queue,0}, {runtime,{446410,470}}]}, {system_stats, [{cpu_utilization_rate,4.25}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,8200770933}, {couch_docs_data_size,6107702164}, {couch_views_actual_disk_size,102585286}, {couch_views_data_size,102570159}, {curr_items,2501801}, {curr_items_tot,2501801}, {mem_used,3488892616}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1681}, {memory_data,{33657663488,14197092352,{<0.1663.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 19004836 kB\nBuffers: 313352 kB\nCached: 8701628 kB\nSwapCached: 12 kB\nActive: 12936536 kB\nInactive: 549336 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 19004836 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 92452 kB\nWriteback: 0 kB\nAnonPages: 4470996 kB\nMapped: 18164 kB\nSlab: 329512 kB\nPageTables: 13672 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6167408 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-02T18:51:26.827,ns_1@10.2.1.68:<0.1669.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,31,47}}) Stats for bucket "default": accepting_conns 1 auth_cmds 3 auth_errors 0 bucket_active_conns 1 bucket_conns 5 bytes 3488892616 bytes_read 6003893743 bytes_written 146134618 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 2501801 conn_yields 851 connection_structures 5000 curr_connections 9 curr_conns_on_port_11209 6 curr_conns_on_port_11210 1 curr_items 2501801 curr_items_tot 2501801 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-03 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 1600 ep_commit_time 5 ep_commit_time_total 10441 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 4 ep_data_age_highwat 62 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data/default ep_degraded_mode 0 ep_diskqueue_drain 2501801 ep_diskqueue_fill 2501801 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 888 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 2501801 ep_io_read_bytes 0 ep_io_write_bytes 5923674684 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 2496590 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 3426507387 ep_max_checkpoints 2 ep_max_data_size 5242880000 ep_max_item_size 20971520 ep_max_size 5242880000 ep_max_txn_size 10000 ep_max_vbuckets 512 ep_mem_high_wat 3932160000 ep_mem_high_wat 3932160000 ep_mem_low_wat 3145728000 ep_mem_low_wat 3145728000 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 373351 ep_num_expiry_pager_runs 0 ep_num_non_resident 1147214 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 12 ep_num_value_ejects 1147214 ep_oom_errors 0 ep_overhead 13495288 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1351905984 ep_storage_age 1 ep_storage_age_highwat 62 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 3396485775 ep_total_del_items 0 ep_total_enqueued 2501801 ep_total_new_items 2501801 ep_total_persisted 2501801 ep_uncommitted_items 0 ep_value_size 3219561761 ep_vb0 0 ep_vb_snapshot_total 61 ep_vb_total 128 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_130_g8b09894 ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 741 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 3488892616 pid 17027 pointer_size 64 rejected_conns 0 rusage_system 108.542499 rusage_user 302.459019 threads 4 time 1351907486 total_connections 13 uptime 1517 vb_active_curr_items 2501801 vb_active_eject 1147214 vb_active_expired 0 vb_active_ht_memory 12645376 vb_active_itm_memory 3396485775 vb_active_meta_data_memory 197642279 vb_active_num 128 vb_active_num_non_resident 1147214 vb_active_num_ref_ejects 1147214 vb_active_num_ref_items 2501801 vb_active_ops_create 2501801 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 54 vb_active_queue_age 0 vb_active_queue_drain 2501801 vb_active_queue_fill 2501801 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_meta_data_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-02T18:52:11.301,ns_1@10.2.1.68:ns_doctor<0.1479.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,907526,325755}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.27766.0>">>}, {changes_done,40215}, {design_documents,[<<"_design/A">>]}, {indexer_type,main}, {initial_build,false}, {progress,5}, {set,<<"default">>}, {signature,<<"afeb9408e40add49ae0c9bf58f0c5f26">>}, {started_on,1351907547}, {total_changes,749784}, {type,indexer}, {updated_on,1351907560}]]}, {memory, [{total,154563944}, {processes,101305928}, {processes_used,99968984}, {system,53258016}, {atom,1502673}, {atom_used,1494165}, {binary,12234080}, {code,15250635}, {ets,5984320}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,10430492672}, {buffered_memory,226189312}, {free_memory,17366343680}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1742194,5002}}, {context_switches,{15433568,0}}, {garbage_collection,{1464757,12056057409,0}}, {io,{{input,23500587251},{output,4619410633}}}, {reductions,{5688944879,87053458}}, {run_queue,0}, {runtime,{571480,6520}}]}, {system_stats, [{cpu_utilization_rate,32.25}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,9001244801}, {couch_docs_data_size,8110492466}, {couch_views_actual_disk_size,785576937}, {couch_views_data_size,110012293}, {curr_items,3339073}, {curr_items_tot,3339073}, {mem_used,3742703352}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1742}, {memory_data,{33657663488,16270241792,{<18077.27766.0>,12342552}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16959196 kB\nBuffers: 220888 kB\nCached: 10186028 kB\nSwapCached: 0 kB\nActive: 13048124 kB\nInactive: 2457728 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16959196 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 776728 kB\nWriteback: 0 kB\nAnonPages: 5099184 kB\nMapped: 16312 kB\nSlab: 357600 kB\nPageTables: 14456 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6845144 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,907529,12362}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.16630.0>">>}, {changes_done,0}, {design_documents,[<<"_design/A">>]}, {indexer_type,main}, {initial_build,false}, {progress,0}, {set,<<"default">>}, {signature,<<"afeb9408e40add49ae0c9bf58f0c5f26">>}, {started_on,1351907501}, {total_changes,383760}, {type,indexer}, {updated_on,1351907501}]]}, {memory, [{total,98870064}, {processes,53574424}, {processes_used,53558224}, {system,45295640}, {atom,1413905}, {atom_used,1392343}, {binary,5484824}, {code,14559765}, {ets,5741344}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,8128471040}, {buffered_memory,254631936}, {free_memory,20151496704}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1744458,5003}}, {context_switches,{13357259,0}}, {garbage_collection,{1297101,8468488808,0}}, {io,{{input,18748919014},{output,2455340366}}}, {reductions,{4587506160,105029950}}, {run_queue,5}, {runtime,{464730,7910}}]}, {system_stats, [{cpu_utilization_rate,74.56359102244389}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,6996735373}, {couch_docs_data_size,6764622340}, {couch_views_actual_disk_size,237577232}, {couch_views_data_size,104860630}, {curr_items,2868417}, {curr_items_tot,2870633}, {mem_used,4224229672}, {vb_replica_curr_items,2216}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1744}, {memory_data,{33657663488,13234946048,{<18118.695.0>,3143528}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 19679320 kB\nBuffers: 248664 kB\nCached: 7937960 kB\nSwapCached: 140 kB\nActive: 11319088 kB\nInactive: 1527524 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 19679320 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 276472 kB\nWriteback: 1640 kB\nAnonPages: 4659696 kB\nMapped: 18016 kB\nSlab: 294120 kB\nPageTables: 14056 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6384716 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,907527,965272}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,91835216}, {processes,49826672}, {processes_used,49489464}, {system,42008544}, {atom,1413905}, {atom_used,1392521}, {binary,2281592}, {code,14569021}, {ets,5701680}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,4721934336}, {buffered_memory,428290048}, {free_memory,23605334016}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1744399,5001}}, {context_switches,{14099524,0}}, {garbage_collection,{2125125,11260831470,0}}, {io,{{input,19915706425},{output,1653103625}}}, {reductions,{5208636186,76265603}}, {run_queue,0}, {runtime,{492250,5390}}]}, {system_stats, [{cpu_utilization_rate,44.97487437185929}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,3347401779}, {couch_docs_data_size,3322269502}, {couch_views_actual_disk_size,823259147}, {couch_views_data_size,85649871}, {curr_items,1310090}, {curr_items_tot,1310090}, {mem_used,2034876984}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1744}, {memory_data,{33657663488,10151116800,{<18117.703.0>,3143528}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 23052084 kB\nBuffers: 418252 kB\nCached: 4610936 kB\nSwapCached: 0 kB\nActive: 8896028 kB\nInactive: 610996 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 23052084 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 7328 kB\nWriteback: 60 kB\nAnonPages: 4477892 kB\nMapped: 18052 kB\nSlab: 265076 kB\nPageTables: 13672 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 6183564 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,907526,303135}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,97783640}, {processes,54502528}, {processes_used,54431400}, {system,43281112}, {atom,1411481}, {atom_used,1388780}, {binary,3423216}, {code,14520518}, {ets,5848624}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,8911962112}, {buffered_memory,321732608}, {free_memory,19465908224}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1741430,5000}}, {context_switches,{12927895,0}}, {garbage_collection,{1238585,7596188683,0}}, {io,{{input,17226972087},{output,870065075}}}, {reductions,{4296550483,2029435}}, {run_queue,0}, {runtime,{452410,470}}]}, {system_stats, [{cpu_utilization_rate,3.7593984962406015}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,8200770933}, {couch_docs_data_size,6107702164}, {couch_views_actual_disk_size,102585286}, {couch_views_data_size,102570159}, {curr_items,2501801}, {curr_items_tot,2501801}, {mem_used,3488892616}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1741}, {memory_data,{33657663488,14198231040,{<0.1662.0>,2914384}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 19009676 kB\nBuffers: 314192 kB\nCached: 8703088 kB\nSwapCached: 12 kB\nActive: 12930696 kB\nInactive: 551100 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 19009676 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 88232 kB\nWriteback: 0 kB\nAnonPages: 4464456 kB\nMapped: 18164 kB\nSlab: 329144 kB\nPageTables: 13652 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6169520 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-02T18:53:06.814,ns_1@10.2.1.68:<0.1669.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,31,47}}) Stats for bucket "default": accepting_conns 1 auth_cmds 3 auth_errors 0 bucket_active_conns 1 bucket_conns 7 bytes 4224483592 bytes_read 7970344716 bytes_written 154487721 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 2501801 conn_yields 5982 connection_structures 5000 curr_connections 11 curr_conns_on_port_11209 8 curr_conns_on_port_11210 1 curr_items 2991494 curr_items_tot 3010978 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-03 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 1639 ep_commit_time 411 ep_commit_time_total 24530 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 1 ep_data_age_highwat 62 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data/default ep_degraded_mode 0 ep_diskqueue_drain 3005871 ep_diskqueue_fill 3020487 ep_diskqueue_items 14741 ep_diskqueue_memory 472096 ep_diskqueue_pending 708144 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 913 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 7392 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 2999467 ep_io_read_bytes 0 ep_io_write_bytes 7103391173 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 2496590 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 4147099752 ep_max_checkpoints 2 ep_max_data_size 5242880000 ep_max_item_size 20971520 ep_max_size 5242880000 ep_max_txn_size 10000 ep_max_vbuckets 512 ep_mem_high_wat 3932160000 ep_mem_high_wat 3932160000 ep_mem_low_wat 3145728000 ep_mem_low_wat 3145728000 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 530791 ep_num_expiry_pager_runs 0 ep_num_non_resident 1371857 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 14 ep_num_value_ejects 1371857 ep_oom_errors 0 ep_overhead 15436804 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 7349 ep_startup_time 1351905984 ep_storage_age 1 ep_storage_age_highwat 62 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 4109367566 ep_total_del_items 0 ep_total_enqueued 3020487 ep_total_new_items 2991494 ep_total_persisted 2999467 ep_uncommitted_items 6098 ep_value_size 3898041741 ep_vb0 0 ep_vb_snapshot_total 105 ep_vb_total 154 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_130_g8b09894 ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 741 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 4224483592 pid 17027 pointer_size 64 rejected_conns 0 rusage_system 113.666720 rusage_user 325.617498 tap_checkpoint_start_received 50 tap_mutation_received 812862 tap_opaque_received 178 tap_vbucket_set_received 50 threads 4 time 1351907586 total_connections 115 uptime 1617 vb_active_curr_items 2991494 vb_active_eject 1371857 vb_active_expired 0 vb_active_ht_memory 13273176 vb_active_itm_memory 4061776682 vb_active_meta_data_memory 236328026 vb_active_num 153 vb_active_num_non_resident 1371857 vb_active_num_ref_ejects 1371857 vb_active_num_ref_items 2991494 vb_active_ops_create 2991494 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 7973 vb_active_perc_mem_resident 54 vb_active_queue_age 0 vb_active_queue_drain 3001140 vb_active_queue_fill 3001003 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 19484 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 25112 vb_replica_itm_memory 47590884 vb_replica_meta_data_memory 1539236 vb_replica_num 1 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 19484 vb_replica_ops_create 4730 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 100 vb_replica_queue_age 8364000 vb_replica_queue_drain 4731 vb_replica_queue_fill 19484 vb_replica_queue_memory 472096 vb_replica_queue_pending 708144 vb_replica_queue_size 14753 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-02T18:53:11.302,ns_1@10.2.1.68:ns_doctor<0.1479.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,907586,324253}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.27766.0>">>}, {changes_done,236143}, {design_documents,[<<"_design/A">>]}, {indexer_type,main}, {initial_build,false}, {progress,31}, {set,<<"default">>}, {signature,<<"afeb9408e40add49ae0c9bf58f0c5f26">>}, {started_on,1351907547}, {total_changes,749784}, {type,indexer}, {updated_on,1351907620}]]}, {memory, [{total,150714424}, {processes,97961424}, {processes_used,96624480}, {system,52753000}, {atom,1502673}, {atom_used,1494165}, {binary,11548544}, {code,15250635}, {ets,6164520}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,11209662464}, {buffered_memory,227885056}, {free_memory,16559427584}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1802193,5001}}, {context_switches,{17268869,0}}, {garbage_collection,{1564805,15783810030,0}}, {io,{{input,26958399841},{output,5421296990}}}, {reductions,{6761721842,89132826}}, {run_queue,0}, {runtime,{653150,6620}}]}, {system_stats, [{cpu_utilization_rate,41.14713216957606}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,9001244807}, {couch_docs_data_size,8110492466}, {couch_views_actual_disk_size,1562814086}, {couch_views_data_size,117813275}, {curr_items,3339073}, {curr_items_tot,3339073}, {mem_used,3744910056}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1802}, {memory_data,{33657663488,17070444544,{<18077.27766.0>,12342464}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16171316 kB\nBuffers: 222544 kB\nCached: 10946936 kB\nSwapCached: 0 kB\nActive: 14260980 kB\nInactive: 2011564 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16171316 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 1526476 kB\nWriteback: 0 kB\nAnonPages: 5102704 kB\nMapped: 16312 kB\nSlab: 378324 kB\nPageTables: 14484 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6851476 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,907588,992697}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.17170.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,6}, {started_on,1351907527}, {total_vbuckets,166}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351907560}, {vbuckets_done,10}], [{pid,<<"<0.17372.0>">>}, {changes_done,75663}, {design_documents,[<<"_design/A">>]}, {indexer_type,main}, {initial_build,false}, {progress,9}, {set,<<"default">>}, {signature,<<"afeb9408e40add49ae0c9bf58f0c5f26">>}, {started_on,1351907537}, {total_changes,826720}, {type,indexer}, {updated_on,1351907560}]]}, {memory, [{total,122154712}, {processes,74609144}, {processes_used,74595232}, {system,47545568}, {atom,1413905}, {atom_used,1392343}, {binary,7433640}, {code,14559765}, {ets,5935208}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,10360348672}, {buffered_memory,258383872}, {free_memory,17548529664}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1804439,5000}}, {context_switches,{16473656,0}}, {garbage_collection,{1539068,12180367987,0}}, {io,{{input,23635160572},{output,5000454918}}}, {reductions,{5748730651,92117323}}, {run_queue,1}, {runtime,{550480,6350}}]}, {system_stats, [{cpu_utilization_rate,47.25}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,8641551416}, {couch_docs_data_size,7946521202}, {couch_views_actual_disk_size,996603643}, {couch_views_data_size,111800119}, {curr_items,3336681}, {curr_items_tot,3336681}, {mem_used,3697737544}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1804}, {memory_data,{33657663488,16005402624,{<18118.17372.0>,12342520}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 17137236 kB\nBuffers: 252328 kB\nCached: 10117528 kB\nSwapCached: 140 kB\nActive: 12090260 kB\nInactive: 3235408 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 17137236 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 1086180 kB\nWriteback: 14432 kB\nAnonPages: 4955692 kB\nMapped: 18020 kB\nSlab: 357388 kB\nPageTables: 14668 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6683748 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,907587,963272}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,86715944}, {processes,45517656}, {processes_used,44957416}, {system,41198288}, {atom,1413905}, {atom_used,1392521}, {binary,1315088}, {code,14569021}, {ets,5872560}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,2922139648}, {buffered_memory,429793280}, {free_memory,25428701184}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1804397,4996}}, {context_switches,{16106138,0}}, {garbage_collection,{3000217,14564704823,0}}, {io,{{input,23893204065},{output,2321532128}}}, {reductions,{6082448744,64717606}}, {run_queue,0}, {runtime,{559280,5610}}]}, {system_stats, [{cpu_utilization_rate,55.0}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,910878039}, {couch_docs_data_size,904072046}, {couch_views_actual_disk_size,1470983693}, {couch_views_data_size,44928024}, {curr_items,313268}, {curr_items_tot,313268}, {mem_used,517400552}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1804}, {memory_data,{33657663488,8392777728,{<18117.703.0>,3143528}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 24832716 kB\nBuffers: 419720 kB\nCached: 2853652 kB\nSwapCached: 0 kB\nActive: 7173776 kB\nInactive: 567136 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 24832716 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 7728 kB\nWriteback: 0 kB\nAnonPages: 4467816 kB\nMapped: 18052 kB\nSlab: 249636 kB\nPageTables: 13648 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 6178672 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,907586,323911}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.17216.0>">>}, {changes_done,0}, {design_documents,[<<"_design/A">>]}, {indexer_type,main}, {initial_build,false}, {progress,0}, {set,<<"default">>}, {signature,<<"afeb9408e40add49ae0c9bf58f0c5f26">>}, {started_on,1351907586}, {total_changes,458865}, {type,indexer}, {updated_on,1351907586}]]}, {memory, [{total,107613264}, {processes,60208560}, {processes_used,60196912}, {system,47404704}, {atom,1413905}, {atom_used,1392343}, {binary,7320848}, {code,14597005}, {ets,5932792}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,10434977792}, {buffered_memory,324112384}, {free_memory,17503993856}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1801451,5021}}, {context_switches,{14767377,0}}, {garbage_collection,{1440219,9444922092,0}}, {io,{{input,20407637926},{output,3113073772}}}, {reductions,{4929982518,128721407}}, {run_queue,0}, {runtime,{503570,9890}}]}, {system_stats, [{cpu_utilization_rate,89.4088669950739}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,9317341300}, {couch_docs_data_size,7198827767}, {couch_views_actual_disk_size,387579066}, {couch_views_data_size,106586394}, {curr_items,2971884}, {curr_items_tot,2991494}, {mem_used,4172680904}, {vb_replica_curr_items,19610}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1801}, {memory_data,{33657663488,16063262720,{<0.1662.0>,2914384}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 17093868 kB\nBuffers: 316516 kB\nCached: 10190408 kB\nSwapCached: 12 kB\nActive: 13626272 kB\nInactive: 1732708 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 17093868 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 418828 kB\nWriteback: 8 kB\nAnonPages: 4851756 kB\nMapped: 18204 kB\nSlab: 367200 kB\nPageTables: 14364 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6578260 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-02T18:54:11.301,ns_1@10.2.1.68:ns_doctor<0.1479.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,907646,323344}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.27766.0>">>}, {changes_done,431836}, {design_documents,[<<"_design/A">>]}, {indexer_type,main}, {initial_build,false}, {progress,57}, {set,<<"default">>}, {signature,<<"afeb9408e40add49ae0c9bf58f0c5f26">>}, {started_on,1351907547}, {total_changes,749784}, {type,indexer}, {updated_on,1351907681}]]}, {memory, [{total,158368952}, {processes,105880792}, {processes_used,104544536}, {system,52488160}, {atom,1502673}, {atom_used,1494165}, {binary,11254064}, {code,15250635}, {ets,6225672}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,12006735872}, {buffered_memory,229113856}, {free_memory,15729815552}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1862192,5000}}, {context_switches,{19096465,0}}, {garbage_collection,{1669724,19531828528,0}}, {io,{{input,30366537304},{output,6235090739}}}, {reductions,{7850471535,87357913}}, {run_queue,0}, {runtime,{734050,6770}}]}, {system_stats, [{cpu_utilization_rate,39.80099502487562}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,9001243856}, {couch_docs_data_size,8110492466}, {couch_views_actual_disk_size,2359849149}, {couch_views_data_size,126731661}, {curr_items,3339073}, {curr_items_tot,3339073}, {mem_used,3744910056}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1862}, {memory_data,{33657663488,17901834240,{<18077.27766.0>,12342464}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,2}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 15361148 kB\nBuffers: 223744 kB\nCached: 11725328 kB\nSwapCached: 0 kB\nActive: 15501208 kB\nInactive: 1561040 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 15361148 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 2293540 kB\nWriteback: 0 kB\nAnonPages: 5112944 kB\nMapped: 16312 kB\nSlab: 398960 kB\nPageTables: 14484 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6857212 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,907648,993391}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.17170.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,20}, {started_on,1351907527}, {total_vbuckets,166}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351907619}, {vbuckets_done,34}], [{pid,<<"<0.17372.0>">>}, {changes_done,262467}, {design_documents,[<<"_design/A">>]}, {indexer_type,main}, {initial_build,false}, {progress,31}, {set,<<"default">>}, {signature,<<"afeb9408e40add49ae0c9bf58f0c5f26">>}, {started_on,1351907537}, {total_changes,826720}, {type,indexer}, {updated_on,1351907618}]]}, {memory, [{total,129487464}, {processes,81918912}, {processes_used,81910208}, {system,47568552}, {atom,1413905}, {atom_used,1392343}, {binary,7410328}, {code,14559765}, {ets,6010816}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,11189800960}, {buffered_memory,260689920}, {free_memory,16695877632}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1864439,5000}}, {context_switches,{18568932,0}}, {garbage_collection,{1648327,15991367639,0}}, {io,{{input,27125072251},{output,5833352359}}}, {reductions,{6843997063,85858195}}, {run_queue,0}, {runtime,{624730,5950}}]}, {system_stats, [{cpu_utilization_rate,39.75}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,8702230988}, {couch_docs_data_size,8089088865}, {couch_views_actual_disk_size,1811450244}, {couch_views_data_size,119402134}, {curr_items,3336681}, {curr_items_tot,3336681}, {mem_used,3688429320}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1864}, {memory_data,{33657663488,16906059776,{<18118.17372.0>,12342552}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16304692 kB\nBuffers: 254580 kB\nCached: 10927540 kB\nSwapCached: 140 kB\nActive: 12252792 kB\nInactive: 3884144 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16304692 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 1792932 kB\nWriteback: 7524 kB\nAnonPages: 4954332 kB\nMapped: 18020 kB\nSlab: 378536 kB\nPageTables: 14708 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6684668 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,907615,66454}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,71829656}, {processes,30887696}, {processes_used,30214280}, {system,40941960}, {atom,1414713}, {atom_used,1393896}, {binary,1037296}, {code,14594040}, {ets,5914304}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,659300352}, {buffered_memory,430133248}, {free_memory,27750006784}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1831230,1832}}, {context_switches,{16949803,0}}, {garbage_collection,{3182437,15221525271,0}}, {io,{{input,25799730626},{output,2418607685}}}, {reductions,{6295465588,1693467}}, {run_queue,0}, {runtime,{578140,230}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1831}, {memory_data,{33657663488,8392777728,{<18117.703.0>,3143528}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 27099616 kB\nBuffers: 420052 kB\nCached: 643848 kB\nSwapCached: 0 kB\nActive: 4986024 kB\nInactive: 530132 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 27099616 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 636 kB\nWriteback: 0 kB\nAnonPages: 4452376 kB\nMapped: 18052 kB\nSlab: 206568 kB\nPageTables: 13644 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 6139444 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,907646,301609}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.17722.0>">>}, {changes_done,118312}, {design_documents,[<<"_design/A">>]}, {indexer_type,main}, {initial_build,false}, {progress,15}, {set,<<"default">>}, {signature,<<"afeb9408e40add49ae0c9bf58f0c5f26">>}, {started_on,1351907604}, {total_changes,787082}, {type,indexer}, {updated_on,1351907642}]]}, {memory, [{total,133463728}, {processes,83172312}, {processes_used,83159576}, {system,50291416}, {atom,1413905}, {atom_used,1392343}, {binary,9834912}, {code,14597005}, {ets,6097696}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,12055625728}, {buffered_memory,326348800}, {free_memory,15635255296}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1861429,5000}}, {context_switches,{17388561,0}}, {garbage_collection,{1650425,13504408524,0}}, {io,{{input,25512709453},{output,5229758063}}}, {reductions,{6119928663,82052950}}, {run_queue,0}, {runtime,{593380,5880}}]}, {system_stats, [{cpu_utilization_rate,33.05084745762712}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,10260119311}, {couch_docs_data_size,8101061148}, {couch_views_actual_disk_size,1166979752}, {couch_views_data_size,113402707}, {curr_items,3324246}, {curr_items_tot,3324246}, {mem_used,3672423048}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1861}, {memory_data,{33657663488,18009186304,{<0.17722.0>,12342552}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,3}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 15267936 kB\nBuffers: 318700 kB\nCached: 11773072 kB\nSwapCached: 12 kB\nActive: 14804944 kB\nInactive: 2340556 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 15267936 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 1122304 kB\nWriteback: 0 kB\nAnonPages: 5053484 kB\nMapped: 18216 kB\nSlab: 406592 kB\nPageTables: 14888 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6776784 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-02T18:54:46.822,ns_1@10.2.1.68:<0.1669.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,31,47}}) Stats for bucket "default": accepting_conns 1 auth_cmds 3 auth_errors 0 bucket_active_conns 1 bucket_conns 5 bytes 3672423032 bytes_read 9242850309 bytes_written 162770376 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 2501801 conn_yields 9629 connection_structures 5000 curr_connections 9 curr_conns_on_port_11209 6 curr_conns_on_port_11210 1 curr_items 3324246 curr_items_tot 3324246 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-03 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 1664 ep_commit_time 144 ep_commit_time_total 34843 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 3 ep_data_age_highwat 62 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data/default ep_degraded_mode 0 ep_diskqueue_drain 3345332 ep_diskqueue_fill 3345135 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 932 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 3342679 ep_io_read_bytes 0 ep_io_write_bytes 7917325188 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 2496590 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 3601291738 ep_max_checkpoints 2 ep_max_data_size 5242880000 ep_max_item_size 20971520 ep_max_size 5242880000 ep_max_txn_size 10000 ep_max_vbuckets 512 ep_mem_high_wat 3932160000 ep_mem_high_wat 3932160000 ep_mem_low_wat 3145728000 ep_mem_low_wat 3145728000 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 1032541 ep_num_expiry_pager_runs 0 ep_num_non_resident 1930273 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 18 ep_num_value_ejects 1930273 ep_oom_errors 0 ep_overhead 22049956 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1351905984 ep_storage_age 1 ep_storage_age_highwat 62 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 3561400786 ep_total_del_items 0 ep_total_enqueued 3345135 ep_total_new_items 3324246 ep_total_persisted 3342679 ep_uncommitted_items 0 ep_value_size 3326102306 ep_vb0 0 ep_vb_snapshot_total 139 ep_vb_total 170 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_130_g8b09894 ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 741 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 3672423032 pid 17027 pointer_size 64 rejected_conns 0 rusage_system 116.498289 rusage_user 343.268815 tap_checkpoint_start_received 84 tap_mutation_received 1338733 tap_opaque_received 294 tap_vbucket_set_received 84 threads 4 time 1351907686 total_connections 181 uptime 1717 vb_active_curr_items 3324246 vb_active_eject 1930273 vb_active_expired 0 vb_active_ht_memory 20921392 vb_active_itm_memory 3561400786 vb_active_meta_data_memory 262615434 vb_active_num 170 vb_active_num_non_resident 1930273 vb_active_num_ref_ejects 1930273 vb_active_num_ref_items 3324246 vb_active_ops_create 3324246 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 18433 vb_active_perc_mem_resident 41 vb_active_queue_age 0 vb_active_queue_drain 3345332 vb_active_queue_fill 3345135 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_meta_data_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-02T18:55:11.300,ns_1@10.2.1.68:ns_doctor<0.1479.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,907706,323113}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.27766.0>">>}, {changes_done,627618}, {design_documents,[<<"_design/A">>]}, {indexer_type,main}, {initial_build,false}, {progress,83}, {set,<<"default">>}, {signature,<<"afeb9408e40add49ae0c9bf58f0c5f26">>}, {started_on,1351907547}, {total_changes,749784}, {type,indexer}, {updated_on,1351907742}]]}, {memory, [{total,144632256}, {processes,93214376}, {processes_used,91874696}, {system,51417880}, {atom,1502673}, {atom_used,1494165}, {binary,10004712}, {code,15250635}, {ets,6406472}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,12749791232}, {buffered_memory,230076416}, {free_memory,14982832128}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1922193,5000}}, {context_switches,{21020228,0}}, {garbage_collection,{1765225,22998579639,0}}, {io,{{input,33953218425},{output,6993578849}}}, {reductions,{8887625416,79682704}}, {run_queue,0}, {runtime,{816130,6600}}]}, {system_stats, [{cpu_utilization_rate,44.27860696517413}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,9001243856}, {couch_docs_data_size,8110492466}, {couch_views_actual_disk_size,3100990764}, {couch_views_data_size,138289456}, {curr_items,3339073}, {curr_items_tot,3339073}, {mem_used,3744910056}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1922}, {memory_data,{33657663488,18653261824,{<18077.27766.0>,12342464}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,2}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 14631672 kB\nBuffers: 224684 kB\nCached: 12450968 kB\nSwapCached: 0 kB\nActive: 16660100 kB\nInactive: 1111720 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 14631672 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 3014016 kB\nWriteback: 0 kB\nAnonPages: 5096284 kB\nMapped: 16312 kB\nSlab: 418416 kB\nPageTables: 14468 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6832972 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,907708,994972}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.17170.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,41}, {started_on,1351907527}, {total_vbuckets,166}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351907680}, {vbuckets_done,69}], [{pid,<<"<0.17372.0>">>}, {changes_done,411705}, {design_documents,[<<"_design/A">>]}, {indexer_type,main}, {initial_build,false}, {progress,49}, {set,<<"default">>}, {signature,<<"afeb9408e40add49ae0c9bf58f0c5f26">>}, {started_on,1351907537}, {total_changes,826720}, {type,indexer}, {updated_on,1351907675}]]}, {memory, [{total,133413632}, {processes,86317448}, {processes_used,86305600}, {system,47096184}, {atom,1413905}, {atom_used,1392343}, {binary,6571248}, {code,14559765}, {ets,6226240}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,11944415232}, {buffered_memory,263106560}, {free_memory,15920414720}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1924440,5001}}, {context_switches,{20509556,0}}, {garbage_collection,{1737172,19657847503,0}}, {io,{{input,30305923932},{output,6647869583}}}, {reductions,{7878033089,84785465}}, {run_queue,0}, {runtime,{697150,6000}}]}, {system_stats, [{cpu_utilization_rate,30.075187969924812}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,8657909498}, {couch_docs_data_size,8089919362}, {couch_views_actual_disk_size,2611956923}, {couch_views_data_size,126659958}, {curr_items,3336681}, {curr_items_tot,3336681}, {mem_used,3688429320}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1924}, {memory_data,{33657663488,17662443520,{<18118.17370.0>,13313552}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,2}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 15546412 kB\nBuffers: 256940 kB\nCached: 11664468 kB\nSwapCached: 140 kB\nActive: 11869496 kB\nInactive: 5004112 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 15546412 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 2591312 kB\nWriteback: 16588 kB\nAnonPages: 4952172 kB\nMapped: 18012 kB\nSlab: 399196 kB\nPageTables: 14624 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6698712 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,907615,66454}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,71829656}, {processes,30887696}, {processes_used,30214280}, {system,40941960}, {atom,1414713}, {atom_used,1393896}, {binary,1037296}, {code,14594040}, {ets,5914304}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,659300352}, {buffered_memory,430133248}, {free_memory,27750006784}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1831230,1832}}, {context_switches,{16949803,0}}, {garbage_collection,{3182437,15221525271,0}}, {io,{{input,25799730626},{output,2418607685}}}, {reductions,{6295465588,1693467}}, {run_queue,0}, {runtime,{578140,230}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1831}, {memory_data,{33657663488,8392777728,{<18117.703.0>,3143528}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 27099616 kB\nBuffers: 420052 kB\nCached: 643848 kB\nSwapCached: 0 kB\nActive: 4986024 kB\nInactive: 530132 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 27099616 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 636 kB\nWriteback: 0 kB\nAnonPages: 4452376 kB\nMapped: 18052 kB\nSlab: 206568 kB\nPageTables: 13644 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 6139444 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,907706,301541}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.17722.0>">>}, {changes_done,313775}, {design_documents,[<<"_design/A">>]}, {indexer_type,main}, {initial_build,false}, {progress,39}, {set,<<"default">>}, {signature,<<"afeb9408e40add49ae0c9bf58f0c5f26">>}, {started_on,1351907604}, {total_changes,787082}, {type,indexer}, {updated_on,1351907700}]]}, {memory, [{total,129581328}, {processes,80006944}, {processes_used,79994352}, {system,49574384}, {atom,1413905}, {atom_used,1392343}, {binary,9048504}, {code,14597005}, {ets,6166760}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,12859772928}, {buffered_memory,327323648}, {free_memory,14807764992}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1921429,4998}}, {context_switches,{19230036,0}}, {garbage_collection,{1749705,17553983799,0}}, {io,{{input,29008251944},{output,6048627352}}}, {reductions,{7217751712,90683291}}, {run_queue,0}, {runtime,{671530,6200}}]}, {system_stats, [{cpu_utilization_rate,36.25}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,10260116595}, {couch_docs_data_size,8101061148}, {couch_views_actual_disk_size,1959944816}, {couch_views_data_size,121162595}, {curr_items,3324246}, {curr_items_tot,3324246}, {mem_used,3672423048}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1921}, {memory_data,{33657663488,18837327872,{<0.17722.0>,12342520}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,3}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 14460708 kB\nBuffers: 319652 kB\nCached: 12558372 kB\nSwapCached: 12 kB\nActive: 16052248 kB\nInactive: 1878544 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 14460708 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 1901124 kB\nWriteback: 0 kB\nAnonPages: 5052556 kB\nMapped: 18216 kB\nSlab: 427608 kB\nPageTables: 14876 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6767312 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-02T18:56:11.301,ns_1@10.2.1.68:ns_doctor<0.1479.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,907766,322397}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,118731512}, {processes,70566592}, {processes_used,69219168}, {system,48164920}, {atom,1502673}, {atom_used,1494165}, {binary,6568744}, {code,15250635}, {ets,6590056}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,13356056576}, {buffered_memory,231006208}, {free_memory,14386569216}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1982193,5000}}, {context_switches,{22732863,0}}, {garbage_collection,{1855029,25803599246,0}}, {io,{{input,37363388199},{output,7625345348}}}, {reductions,{9756457727,2216489}}, {run_queue,0}, {runtime,{886280,510}}]}, {system_stats, [{cpu_utilization_rate,2.743142144638404}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,9001243856}, {couch_docs_data_size,8110492466}, {couch_views_actual_disk_size,3734983855}, {couch_views_data_size,148053429}, {curr_items,3339073}, {curr_items_tot,3339073}, {mem_used,3744910056}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1982}, {memory_data,{33657663488,19271094272,{<18077.5546.0>,4714984}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,2}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 14049384 kB\nBuffers: 225592 kB\nCached: 13043024 kB\nSwapCached: 0 kB\nActive: 17602324 kB\nInactive: 735968 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 14049384 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 283764 kB\nWriteback: 0 kB\nAnonPages: 5069608 kB\nMapped: 16312 kB\nSlab: 434488 kB\nPageTables: 14440 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6789248 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,907768,995807}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.17170.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,61}, {started_on,1351907527}, {total_vbuckets,166}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351907741}, {vbuckets_done,102}], [{pid,<<"<0.17372.0>">>}, {changes_done,560917}, {design_documents,[<<"_design/A">>]}, {indexer_type,main}, {initial_build,false}, {progress,67}, {set,<<"default">>}, {signature,<<"afeb9408e40add49ae0c9bf58f0c5f26">>}, {started_on,1351907537}, {total_changes,826720}, {type,indexer}, {updated_on,1351907735}]]}, {memory, [{total,132553160}, {processes,86234096}, {processes_used,86218048}, {system,46319064}, {atom,1413905}, {atom_used,1392343}, {binary,5574320}, {code,14559765}, {ets,6441896}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,12586831872}, {buffered_memory,265453568}, {free_memory,15257350144}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1984440,5001}}, {context_switches,{22474113,0}}, {garbage_collection,{1830086,23011400800,0}}, {io,{{input,33574027565},{output,7391957936}}}, {reductions,{8837603619,78642091}}, {run_queue,0}, {runtime,{769400,6100}}]}, {system_stats, [{cpu_utilization_rate,39.30348258706468}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,8615413104}, {couch_docs_data_size,8090694752}, {couch_views_actual_disk_size,3342600414}, {couch_views_data_size,135747741}, {curr_items,3336681}, {curr_items_tot,3336681}, {mem_used,3688429320}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1984}, {memory_data,{33657663488,18379149312,{<18118.17370.0>,13313480}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,2}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 14899012 kB\nBuffers: 259228 kB\nCached: 12291504 kB\nSwapCached: 140 kB\nActive: 11508376 kB\nInactive: 5996256 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 14899012 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 3268108 kB\nWriteback: 0 kB\nAnonPages: 4953308 kB\nMapped: 18020 kB\nSlab: 416572 kB\nPageTables: 14740 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6690520 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,907615,66454}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,71829656}, {processes,30887696}, {processes_used,30214280}, {system,40941960}, {atom,1414713}, {atom_used,1393896}, {binary,1037296}, {code,14594040}, {ets,5914304}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,659300352}, {buffered_memory,430133248}, {free_memory,27750006784}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1831230,1832}}, {context_switches,{16949803,0}}, {garbage_collection,{3182437,15221525271,0}}, {io,{{input,25799730626},{output,2418607685}}}, {reductions,{6295465588,1693467}}, {run_queue,0}, {runtime,{578140,230}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1831}, {memory_data,{33657663488,8392777728,{<18117.703.0>,3143528}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 27099616 kB\nBuffers: 420052 kB\nCached: 643848 kB\nSwapCached: 0 kB\nActive: 4986024 kB\nInactive: 530132 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 27099616 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 636 kB\nWriteback: 0 kB\nAnonPages: 4452376 kB\nMapped: 18052 kB\nSlab: 206568 kB\nPageTables: 13644 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 6139444 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,907766,302168}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.17722.0>">>}, {changes_done,509399}, {design_documents,[<<"_design/A">>]}, {indexer_type,main}, {initial_build,false}, {progress,64}, {set,<<"default">>}, {signature,<<"afeb9408e40add49ae0c9bf58f0c5f26">>}, {started_on,1351907604}, {total_changes,787082}, {type,indexer}, {updated_on,1351907765}]]}, {memory, [{total,133294720}, {processes,84139112}, {processes_used,84126376}, {system,49155608}, {atom,1413905}, {atom_used,1392343}, {binary,8448984}, {code,14597005}, {ets,6347560}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,13637804032}, {buffered_memory,328269824}, {free_memory,13998305280}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1981430,5000}}, {context_switches,{21052700,0}}, {garbage_collection,{1853620,21439591242,0}}, {io,{{input,32362925058},{output,6840744983}}}, {reductions,{8267513880,88482361}}, {run_queue,0}, {runtime,{749740,6960}}]}, {system_stats, [{cpu_utilization_rate,36.75}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,10260116595}, {couch_docs_data_size,8101061148}, {couch_views_actual_disk_size,2739675176}, {couch_views_data_size,130628247}, {curr_items,3324246}, {curr_items_tot,3324246}, {mem_used,3672423048}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1981}, {memory_data,{33657663488,19644129280,{<0.17722.0>,12342552}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,3}, {"/data2",246132896,2}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 13665260 kB\nBuffers: 320576 kB\nCached: 13318168 kB\nSwapCached: 12 kB\nActive: 17268084 kB\nInactive: 1438024 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 13665260 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 2651192 kB\nWriteback: 0 kB\nAnonPages: 5067400 kB\nMapped: 18216 kB\nSlab: 448308 kB\nPageTables: 14884 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6786888 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-02T18:56:26.814,ns_1@10.2.1.68:<0.1669.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,31,47}}) Stats for bucket "default": accepting_conns 1 auth_cmds 3 auth_errors 0 bucket_active_conns 1 bucket_conns 5 bytes 3672423032 bytes_read 9242860799 bytes_written 169533446 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 2501801 conn_yields 9629 connection_structures 5000 curr_connections 9 curr_conns_on_port_11209 6 curr_conns_on_port_11210 1 curr_items 3324246 curr_items_tot 3324246 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-03 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 1664 ep_commit_time 144 ep_commit_time_total 34843 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 3 ep_data_age_highwat 62 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data/default ep_degraded_mode 0 ep_diskqueue_drain 3345332 ep_diskqueue_fill 3345135 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 932 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 3342679 ep_io_read_bytes 0 ep_io_write_bytes 7917325188 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 2496590 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 3601291738 ep_max_checkpoints 2 ep_max_data_size 5242880000 ep_max_item_size 20971520 ep_max_size 5242880000 ep_max_txn_size 10000 ep_max_vbuckets 512 ep_mem_high_wat 3932160000 ep_mem_high_wat 3932160000 ep_mem_low_wat 3145728000 ep_mem_low_wat 3145728000 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 1032541 ep_num_expiry_pager_runs 0 ep_num_non_resident 1930273 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 18 ep_num_value_ejects 1930273 ep_oom_errors 0 ep_overhead 22049956 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1351905984 ep_storage_age 1 ep_storage_age_highwat 62 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 3561400786 ep_total_del_items 0 ep_total_enqueued 3345135 ep_total_new_items 3324246 ep_total_persisted 3342679 ep_uncommitted_items 0 ep_value_size 3326102306 ep_vb0 0 ep_vb_snapshot_total 139 ep_vb_total 170 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_130_g8b09894 ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 741 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 3672423032 pid 17027 pointer_size 64 rejected_conns 0 rusage_system 116.511287 rusage_user 343.379798 tap_checkpoint_start_received 84 tap_mutation_received 1338733 tap_opaque_received 294 tap_vbucket_set_received 84 threads 4 time 1351907786 total_connections 181 uptime 1817 vb_active_curr_items 3324246 vb_active_eject 1930273 vb_active_expired 0 vb_active_ht_memory 20921392 vb_active_itm_memory 3561400786 vb_active_meta_data_memory 262615434 vb_active_num 170 vb_active_num_non_resident 1930273 vb_active_num_ref_ejects 1930273 vb_active_num_ref_items 3324246 vb_active_ops_create 3324246 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 18433 vb_active_perc_mem_resident 41 vb_active_queue_age 0 vb_active_queue_drain 3345332 vb_active_queue_fill 3345135 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_meta_data_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-02T18:57:11.301,ns_1@10.2.1.68:ns_doctor<0.1479.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,907826,322143}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,83135184}, {processes,37394168}, {processes_used,35265288}, {system,45741016}, {atom,1502673}, {atom_used,1494165}, {binary,4367224}, {code,15250635}, {ets,6638864}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,620724224}, {buffered_memory,231202816}, {free_memory,27461566464}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2042193,3196}}, {context_switches,{22794276,0}}, {garbage_collection,{1861682,25857014763,0}}, {io,{{input,37364378721},{output,7627172885}}}, {reductions,{9782470626,295352}}, {run_queue,0}, {runtime,{892020,180}}]}, {system_stats, [{cpu_utilization_rate,1.4962593516209477}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2042}, {memory_data,{33657663488,6196224000,{<18077.697.0>,3143528}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 26817936 kB\nBuffers: 225784 kB\nCached: 606176 kB\nSwapCached: 0 kB\nActive: 5430504 kB\nInactive: 425572 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 26817936 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 560 kB\nWriteback: 96 kB\nAnonPages: 5024176 kB\nMapped: 16288 kB\nSlab: 148080 kB\nPageTables: 14344 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6728452 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,907828,995982}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,74690080}, {processes,33278488}, {processes_used,32341128}, {system,41411592}, {atom,1413905}, {atom_used,1392343}, {binary,1068704}, {code,14559765}, {ets,6420368}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,617340928}, {buffered_memory,268029952}, {free_memory,27550367744}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2044439,5000}}, {context_switches,{24439112,0}}, {garbage_collection,{1927547,25988839439,0}}, {io,{{input,37018427322},{output,8045959093}}}, {reductions,{9725606655,99311}}, {run_queue,0}, {runtime,{836200,190}}]}, {system_stats, [{cpu_utilization_rate,1.5}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2044}, {memory_data,{33657663488,6107348992,{<18118.695.0>,3143528}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 26904656 kB\nBuffers: 261748 kB\nCached: 602872 kB\nSwapCached: 140 kB\nActive: 5328500 kB\nInactive: 432924 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 26904656 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 9304 kB\nWriteback: 0 kB\nAnonPages: 4896748 kB\nMapped: 17996 kB\nSlab: 153112 kB\nPageTables: 14528 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6594840 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,907615,66454}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,71829656}, {processes,30887696}, {processes_used,30214280}, {system,40941960}, {atom,1414713}, {atom_used,1393896}, {binary,1037296}, {code,14594040}, {ets,5914304}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,659300352}, {buffered_memory,430133248}, {free_memory,27750006784}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1831230,1832}}, {context_switches,{16949803,0}}, {garbage_collection,{3182437,15221525271,0}}, {io,{{input,25799730626},{output,2418607685}}}, {reductions,{6295465588,1693467}}, {run_queue,0}, {runtime,{578140,230}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1831}, {memory_data,{33657663488,8392777728,{<18117.703.0>,3143528}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 27099616 kB\nBuffers: 420052 kB\nCached: 643848 kB\nSwapCached: 0 kB\nActive: 4986024 kB\nInactive: 530132 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 27099616 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 636 kB\nWriteback: 0 kB\nAnonPages: 4452376 kB\nMapped: 18052 kB\nSlab: 206568 kB\nPageTables: 13644 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 6139444 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,907826,302752}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,76729488}, {processes,35126104}, {processes_used,34331464}, {system,41603384}, {atom,1413905}, {atom_used,1392343}, {binary,1213448}, {code,14597005}, {ets,6408944}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,611127296}, {buffered_memory,329236480}, {free_memory,27313061888}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2041430,2575}}, {context_switches,{22869279,0}}, {garbage_collection,{1949965,24815804309,0}}, {io,{{input,35800407791},{output,7533409215}}}, {reductions,{9214144470,91195}}, {run_queue,0}, {runtime,{824210,130}}]}, {system_stats, [{cpu_utilization_rate,3.0}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2041}, {memory_data,{33657663488,6344609792,{<0.694.0>,2173056}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 26672912 kB\nBuffers: 321520 kB\nCached: 596804 kB\nSwapCached: 12 kB\nActive: 5470320 kB\nInactive: 458900 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 26672912 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 1212 kB\nWriteback: 0 kB\nAnonPages: 5011172 kB\nMapped: 18192 kB\nSlab: 216844 kB\nPageTables: 14756 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6702564 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-02T18:57:32.813,ns_1@10.2.1.68:<0.19063.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,31,47}}) Stats for bucket "default": accepting_conns 1 auth_cmds 0 auth_errors 0 bucket_active_conns 1 bucket_conns 5 bytes 240752 bytes_read 226 bytes_written 800 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 0 conn_yields 0 connection_structures 5000 curr_connections 9 curr_conns_on_port_11209 6 curr_conns_on_port_11210 1 curr_items 0 curr_items_tot 0 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-03 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 0 ep_commit_time 0 ep_commit_time_total 0 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 0 ep_data_age_highwat 0 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data/default ep_degraded_mode 1 ep_diskqueue_drain 0 ep_diskqueue_fill 0 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 0 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 0 ep_io_read_bytes 0 ep_io_write_bytes 0 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 0 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 0 ep_max_checkpoints 2 ep_max_data_size 5242880000 ep_max_item_size 20971520 ep_max_size 5242880000 ep_max_txn_size 10000 ep_max_vbuckets 512 ep_mem_high_wat 3932160000 ep_mem_high_wat 3932160000 ep_mem_low_wat 3145728000 ep_mem_low_wat 3145728000 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 1152 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1351907851 ep_storage_age 0 ep_storage_age_highwat 0 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 0 ep_total_del_items 0 ep_total_enqueued 0 ep_total_new_items 0 ep_total_persisted 0 ep_uncommitted_items 0 ep_value_size 0 ep_vb0 0 ep_vb_snapshot_total 1 ep_vb_total 0 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_130_g8b09894 ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 555 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 240752 pid 17027 pointer_size 64 rejected_conns 0 rusage_system 116.523285 rusage_user 346.245362 tap_checkpoint_start_received 84 tap_mutation_received 1338733 tap_opaque_received 294 tap_vbucket_set_received 84 threads 4 time 1351907852 total_connections 188 uptime 1883 vb_active_curr_items 0 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 0 vb_active_itm_memory 0 vb_active_meta_data_memory 0 vb_active_num 0 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 0 vb_active_ops_create 0 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 0 vb_active_queue_age 0 vb_active_queue_drain 0 vb_active_queue_fill 0 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_meta_data_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-02T18:58:11.301,ns_1@10.2.1.68:ns_doctor<0.1479.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,907886,321641}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,97224848}, {processes,51956304}, {processes_used,50379392}, {system,45268544}, {atom,1502673}, {atom_used,1494429}, {binary,3689704}, {code,15256817}, {ets,6708792}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,651956224}, {buffered_memory,232001536}, {free_memory,27425972224}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2102193,5000}}, {context_switches,{22949020,0}}, {garbage_collection,{1880595,25909700934,0}}, {io,{{input,37365697949},{output,7629549133}}}, {reductions,{9795699721,4938464}}, {run_queue,1}, {runtime,{898290,1920}}]}, {system_stats, [{cpu_utilization_rate,38.5}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,1061412}, {couch_docs_data_size,1054464}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,5553}, {curr_items_tot,5553}, {mem_used,18975096}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2102}, {memory_data,{33657663488,6208962560,{<18077.697.0>,3143528}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 26783176 kB\nBuffers: 226564 kB\nCached: 636676 kB\nSwapCached: 0 kB\nActive: 5464684 kB\nInactive: 427904 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 26783176 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 984 kB\nWriteback: 0 kB\nAnonPages: 5029232 kB\nMapped: 16324 kB\nSlab: 147420 kB\nPageTables: 14340 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6752804 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,907888,996424}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,91938824}, {processes,49620008}, {processes_used,49236248}, {system,42318816}, {atom,1413905}, {atom_used,1392509}, {binary,1786800}, {code,14575828}, {ets,6470296}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,682463232}, {buffered_memory,268742656}, {free_memory,27485077504}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2104439,4999}}, {context_switches,{24681185,0}}, {garbage_collection,{1960166,26025221106,0}}, {io,{{input,37020106559},{output,8048075523}}}, {reductions,{9739981221,8261847}}, {run_queue,0}, {runtime,{843390,3240}}]}, {system_stats, [{cpu_utilization_rate,38.59649122807018}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,37316772}, {couch_docs_data_size,20867009}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,14025}, {curr_items_tot,14025}, {mem_used,41258280}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2104}, {memory_data,{33657663488,6121803776,{<18118.695.0>,3143528}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 26841020 kB\nBuffers: 262444 kB\nCached: 666468 kB\nSwapCached: 140 kB\nActive: 5391140 kB\nInactive: 437256 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 26841020 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 1236 kB\nWriteback: 120 kB\nAnonPages: 4899168 kB\nMapped: 18020 kB\nSlab: 151344 kB\nPageTables: 14492 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6616960 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,907891,92096}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,88120672}, {processes,50719736}, {processes_used,50567360}, {system,37400936}, {atom,1414713}, {atom_used,1394174}, {binary,1532328}, {code,14610103}, {ets,1772928}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,744017920}, {buffered_memory,431136768}, {free_memory,31901065216}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2107524,4999}}, {context_switches,{17298049,0}}, {garbage_collection,{3232093,15278109066,0}}, {io,{{input,25802340876},{output,2422087529}}}, {reductions,{6317801413,8918722}}, {run_queue,0}, {runtime,{596340,3130}}]}, {system_stats, [{cpu_utilization_rate,39.25}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,35241216}, {couch_docs_data_size,20484328}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,19488}, {curr_items_tot,19488}, {mem_used,55781280}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2107}, {memory_data,{33657663488,1606639616,{<18117.703.0>,3143528}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 31153508 kB\nBuffers: 421032 kB\nCached: 726580 kB\nSwapCached: 0 kB\nActive: 944776 kB\nInactive: 528564 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 31153508 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 1104 kB\nWriteback: 0 kB\nAnonPages: 325648 kB\nMapped: 17932 kB\nSlab: 205276 kB\nPageTables: 5424 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 2032988 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,907886,301525}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,91807312}, {processes,49327112}, {processes_used,49081192}, {system,42480200}, {atom,1413905}, {atom_used,1392509}, {binary,1769936}, {code,14613068}, {ets,6591952}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,640282624}, {buffered_memory,329940992}, {free_memory,27286863872}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2101429,4999}}, {context_switches,{22998324,0}}, {garbage_collection,{1968252,24840915262,0}}, {io,{{input,35801539562},{output,7535102976}}}, {reductions,{9223335919,3669274}}, {run_queue,0}, {runtime,{830470,2220}}]}, {system_stats, [{cpu_utilization_rate,45.38653366583541}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,1061412}, {couch_docs_data_size,1054464}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,5549}, {curr_items_tot,5549}, {mem_used,19019512}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2101}, {memory_data,{33657663488,6359126016,{<0.694.0>,2173096}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 26647452 kB\nBuffers: 322208 kB\nCached: 625276 kB\nSwapCached: 12 kB\nActive: 5495476 kB\nInactive: 461820 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 26647452 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 1056 kB\nWriteback: 68 kB\nAnonPages: 5009756 kB\nMapped: 18216 kB\nSlab: 215796 kB\nPageTables: 14696 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6717756 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-02T18:59:11.301,ns_1@10.2.1.68:ns_doctor<0.1479.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,907946,323290}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,119442280}, {processes,72941504}, {processes_used,71364744}, {system,46500776}, {atom,1502673}, {atom_used,1494429}, {binary,4760624}, {code,15256817}, {ets,6874808}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,1118117888}, {buffered_memory,233648128}, {free_memory,26939375616}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2162195,5002}}, {context_switches,{24254850,0}}, {garbage_collection,{2013857,26102288966,0}}, {io,{{input,37395021521},{output,7639486477}}}, {reductions,{9872314073,6998512}}, {run_queue,0}, {runtime,{928300,2810}}]}, {system_stats, [{cpu_utilization_rate,38.25}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,436950073}, {couch_docs_data_size,389304125}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,171552}, {curr_items_tot,171552}, {mem_used,457266888}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2162}, {memory_data,{33657663488,6695432192,{<18077.178.1>,7628384}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 26307984 kB\nBuffers: 228172 kB\nCached: 1091912 kB\nSwapCached: 0 kB\nActive: 5596384 kB\nInactive: 770868 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 26307984 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 1556 kB\nWriteback: 0 kB\nAnonPages: 5047344 kB\nMapped: 16312 kB\nSlab: 148152 kB\nPageTables: 14308 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6782072 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,907948,997906}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,98823096}, {processes,55326944}, {processes_used,54941464}, {system,43496152}, {atom,1413905}, {atom_used,1392509}, {binary,2801656}, {code,14575828}, {ets,6638560}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,1267929088}, {buffered_memory,270696448}, {free_memory,26888921088}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2164440,5000}}, {context_switches,{25853547,0}}, {garbage_collection,{2101192,26169814121,0}}, {io,{{input,37032175508},{output,8056410293}}}, {reductions,{9804311900,4546179}}, {run_queue,0}, {runtime,{868740,2100}}]}, {system_stats, [{cpu_utilization_rate,36.15960099750623}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,621462289}, {couch_docs_data_size,429602196}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,181597}, {curr_items_tot,181597}, {mem_used,483589416}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2164}, {memory_data,{33657663488,6718976000,{<18118.695.0>,3143528}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 26258836 kB\nBuffers: 264352 kB\nCached: 1238212 kB\nSwapCached: 140 kB\nActive: 5694896 kB\nInactive: 715672 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 26258836 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 2060 kB\nWriteback: 196 kB\nAnonPages: 4907720 kB\nMapped: 18020 kB\nSlab: 151756 kB\nPageTables: 14500 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6624000 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,907951,94057}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,91725704}, {processes,53173968}, {processes_used,53019792}, {system,38551736}, {atom,1414713}, {atom_used,1394174}, {binary,2343952}, {code,14610103}, {ets,2115304}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,1256542208}, {buffered_memory,433516544}, {free_memory,30914355200}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2167525,5000}}, {context_switches,{18753063,0}}, {garbage_collection,{3394411,15448803763,0}}, {io,{{input,25815310733},{output,2429114570}}}, {reductions,{6393107607,4785091}}, {run_queue,0}, {runtime,{626640,1980}}]}, {system_stats, [{cpu_utilization_rate,33.0}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,549725245}, {couch_docs_data_size,424637290}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,186315}, {curr_items_tot,186315}, {mem_used,495947272}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2167}, {memory_data,{33657663488,2586361856,{<18117.703.0>,3143528}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 30189800 kB\nBuffers: 423356 kB\nCached: 1227092 kB\nSwapCached: 0 kB\nActive: 1585596 kB\nInactive: 850096 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 30189800 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 2064 kB\nWriteback: 216 kB\nAnonPages: 784924 kB\nMapped: 17960 kB\nSlab: 205812 kB\nPageTables: 6228 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 2490564 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,907946,302727}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,100817848}, {processes,57245656}, {processes_used,56998176}, {system,43572192}, {atom,1413905}, {atom_used,1392509}, {binary,2808600}, {code,14613068}, {ets,6642352}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,1101246464}, {buffered_memory,331350016}, {free_memory,26818199552}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2161430,5000}}, {context_switches,{24417797,0}}, {garbage_collection,{2135092,25008066218,0}}, {io,{{input,35815521821},{output,7542192493}}}, {reductions,{9297519250,6458772}}, {run_queue,0}, {runtime,{860370,2760}}]}, {system_stats, [{cpu_utilization_rate,36.75}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,426913918}, {couch_docs_data_size,390797827}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,172617}, {curr_items_tot,172617}, {mem_used,460085128}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2161}, {memory_data,{33657663488,6824996864,{<0.19057.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 26189896 kB\nBuffers: 323584 kB\nCached: 1075436 kB\nSwapCached: 12 kB\nActive: 5593776 kB\nInactive: 820516 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 26189896 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 1416 kB\nWriteback: 0 kB\nAnonPages: 5014792 kB\nMapped: 18216 kB\nSlab: 216660 kB\nPageTables: 14708 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6720312 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-02T18:59:12.816,ns_1@10.2.1.68:<0.19063.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,31,47}}) Stats for bucket "default": accepting_conns 1 auth_cmds 1 auth_errors 0 bucket_active_conns 2 bucket_conns 6 bytes 511450840 bytes_read 460943003 bytes_written 9473403 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 192160 conn_yields 33 connection_structures 5000 curr_connections 10 curr_conns_on_port_11209 6 curr_conns_on_port_11210 2 curr_items 192133 curr_items_tot 192133 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-03 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 508 ep_commit_time 2 ep_commit_time_total 280 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 3 ep_data_age_highwat 11 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data/default ep_degraded_mode 0 ep_diskqueue_drain 191549 ep_diskqueue_fill 192133 ep_diskqueue_items 596 ep_diskqueue_memory 18688 ep_diskqueue_pending 28032 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 68 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 560 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 191540 ep_io_read_bytes 0 ep_io_write_bytes 453284605 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 0 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 469817021 ep_max_checkpoints 2 ep_max_data_size 5242880000 ep_max_item_size 20971520 ep_max_size 5242880000 ep_max_txn_size 10000 ep_max_vbuckets 512 ep_mem_high_wat 3932160000 ep_mem_high_wat 3932160000 ep_mem_low_wat 3145728000 ep_mem_low_wat 3145728000 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 20318000 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 36 ep_startup_time 1351907851 ep_storage_age 1 ep_storage_age_highwat 9 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 467457611 ep_total_del_items 0 ep_total_enqueued 192143 ep_total_new_items 191540 ep_total_persisted 191540 ep_uncommitted_items 44 ep_value_size 453990180 ep_vb0 0 ep_vb_snapshot_total 6 ep_vb_total 128 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_130_g8b09894 ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 555 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 511450840 pid 17027 pointer_size 64 rejected_conns 0 rusage_system 126.449776 rusage_user 369.443836 tap_checkpoint_start_received 84 tap_mutation_received 1338733 tap_opaque_received 294 tap_vbucket_set_received 84 threads 4 time 1351907952 total_connections 189 uptime 1983 vb_active_curr_items 192133 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 3214336 vb_active_itm_memory 467457611 vb_active_meta_data_memory 15178507 vb_active_num 128 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 192133 vb_active_ops_create 191548 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 100 vb_active_queue_age 559000 vb_active_queue_drain 191549 vb_active_queue_fill 192133 vb_active_queue_memory 18688 vb_active_queue_pending 28032 vb_active_queue_size 584 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_meta_data_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [stats:warn,2012-11-02T18:59:38.834,ns_1@10.2.1.68:system_stats_collector<0.1557.0>:system_stats_collector:handle_info:133]lost 1 ticks [stats:warn,2012-11-02T18:59:38.837,ns_1@10.2.1.68:<0.19063.0>:stats_collector:latest_tick:223]Dropped 1 ticks [ns_doctor:debug,2012-11-02T19:00:11.301,ns_1@10.2.1.68:ns_doctor<0.1479.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,908006,321345}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,118393368}, {processes,71765784}, {processes_used,70188544}, {system,46627584}, {atom,1502673}, {atom_used,1494429}, {binary,4821608}, {code,15256817}, {ets,6946272}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,1757061120}, {buffered_memory,235376640}, {free_memory,26302361600}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2222194,5001}}, {context_switches,{24730672,0}}, {garbage_collection,{2059972,26216472298,0}}, {io,{{input,37412224704},{output,7643024873}}}, {reductions,{9916371326,2773404}}, {run_queue,0}, {runtime,{943890,670}}]}, {system_stats, [{cpu_utilization_rate,26.25}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,1076235263}, {couch_docs_data_size,782673405}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,337801}, {curr_items_tot,337801}, {mem_used,897489592}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2222}, {memory_data,{33657663488,7301210112,{<18077.178.1>,7628384}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 25686024 kB\nBuffers: 229860 kB\nCached: 1715880 kB\nSwapCached: 0 kB\nActive: 5947920 kB\nInactive: 1041248 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 25686024 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 1564 kB\nWriteback: 80 kB\nAnonPages: 5043336 kB\nMapped: 16312 kB\nSlab: 147940 kB\nPageTables: 14324 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6772324 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,908008,999497}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,101871176}, {processes,58282048}, {processes_used,57896352}, {system,43589128}, {atom,1413905}, {atom_used,1392509}, {binary,2821752}, {code,14575828}, {ets,6709232}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,1560993792}, {buffered_memory,272646144}, {free_memory,26586976256}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2224441,5001}}, {context_switches,{26237683,0}}, {garbage_collection,{2142309,26244461738,0}}, {io,{{input,37039162177},{output,8059775224}}}, {reductions,{9839721563,4453057}}, {run_queue,0}, {runtime,{880760,2160}}]}, {system_stats, [{cpu_utilization_rate,34.83709273182957}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,915080439}, {couch_docs_data_size,831649119}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,348719}, {curr_items_tot,348719}, {mem_used,924919608}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2224}, {memory_data,{33657663488,7019905024,{<18118.695.0>,3143528}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 25963844 kB\nBuffers: 266256 kB\nCached: 1524408 kB\nSwapCached: 140 kB\nActive: 5504944 kB\nInactive: 1199820 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 25963844 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 2040 kB\nWriteback: 0 kB\nAnonPages: 4913732 kB\nMapped: 18000 kB\nSlab: 151996 kB\nPageTables: 14644 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6621704 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,908011,93951}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,95728488}, {processes,56967832}, {processes_used,56812240}, {system,38760656}, {atom,1414713}, {atom_used,1394174}, {binary,2370608}, {code,14610103}, {ets,2300832}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,1899991040}, {buffered_memory,434196480}, {free_memory,29807898624}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2227525,5000}}, {context_switches,{19177665,0}}, {garbage_collection,{3437836,15525516874,0}}, {io,{{input,25820231113},{output,2433842992}}}, {reductions,{6428955288,3039034}}, {run_queue,0}, {runtime,{638890,860}}]}, {system_stats, [{cpu_utilization_rate,22.25}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,1178756026}, {couch_docs_data_size,817284217}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,353167}, {curr_items_tot,353167}, {mem_used,936715832}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2227}, {memory_data,{33657663488,3722280960,{<18117.703.0>,3143528}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29109276 kB\nBuffers: 424020 kB\nCached: 1855460 kB\nSwapCached: 0 kB\nActive: 2388992 kB\nInactive: 1125924 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29109276 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 680 kB\nWriteback: 0 kB\nAnonPages: 1235384 kB\nMapped: 17960 kB\nSlab: 205744 kB\nPageTables: 7288 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 2941816 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,908006,301898}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,99018280}, {processes,55097152}, {processes_used,54848336}, {system,43921128}, {atom,1413905}, {atom_used,1392509}, {binary,2975680}, {code,14613068}, {ets,6827472}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,1732792320}, {buffered_memory,333074432}, {free_memory,26184839168}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2221429,4999}}, {context_switches,{24970978,0}}, {garbage_collection,{2192650,25096241078,0}}, {io,{{input,35821486142},{output,7547973428}}}, {reductions,{9338055658,1903135}}, {run_queue,0}, {runtime,{876410,510}}]}, {system_stats, [{cpu_utilization_rate,8.771929824561404}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,1102888236}, {couch_docs_data_size,782836388}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,339220}, {curr_items_tot,339220}, {mem_used,899740888}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2221}, {memory_data,{33657663488,7459246080,{<0.19057.0>,2914384}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 25571132 kB\nBuffers: 325268 kB\nCached: 1692180 kB\nSwapCached: 12 kB\nActive: 5993444 kB\nInactive: 1039380 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 25571132 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 2408 kB\nWriteback: 188 kB\nAnonPages: 5015288 kB\nMapped: 18216 kB\nSlab: 216448 kB\nPageTables: 14700 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6719416 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-02T19:00:53.812,ns_1@10.2.1.68:<0.19063.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,31,48}}) Stats for bucket "default": accepting_conns 1 auth_cmds 1 auth_errors 0 bucket_active_conns 1 bucket_conns 6 bytes 1254310472 bytes_read 1135445468 bytes_written 22609161 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 473331 conn_yields 108 connection_structures 5000 curr_connections 10 curr_conns_on_port_11209 6 curr_conns_on_port_11210 2 curr_items 473331 curr_items_tot 473331 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-03 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 646 ep_commit_time 27 ep_commit_time_total 1136 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 19 ep_data_age_highwat 24 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data/default ep_degraded_mode 0 ep_diskqueue_drain 430963 ep_diskqueue_fill 473331 ep_diskqueue_items 42369 ep_diskqueue_memory 1355776 ep_diskqueue_pending 2033664 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 162 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 22586 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 430643 ep_io_read_bytes 0 ep_io_write_bytes 1019151998 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 0 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 1157394169 ep_max_checkpoints 2 ep_max_data_size 5242880000 ep_max_item_size 20971520 ep_max_size 5242880000 ep_max_txn_size 10000 ep_max_vbuckets 512 ep_mem_high_wat 3932160000 ep_mem_high_wat 3932160000 ep_mem_low_wat 3145728000 ep_mem_low_wat 3145728000 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 45225760 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 19783 ep_startup_time 1351907851 ep_storage_age 18 ep_storage_age_highwat 22 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 1151714197 ep_total_del_items 0 ep_total_enqueued 473331 ep_total_new_items 430643 ep_total_persisted 430643 ep_uncommitted_items 9074 ep_value_size 1118380571 ep_vb0 0 ep_vb_snapshot_total 6 ep_vb_total 128 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_130_g8b09894 ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 555 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 1254310472 pid 17027 pointer_size 64 rejected_conns 0 rusage_system 137.258133 rusage_user 397.915507 tap_checkpoint_start_received 84 tap_mutation_received 1338733 tap_opaque_received 294 tap_vbucket_set_received 84 threads 4 time 1351908052 total_connections 189 uptime 2083 vb_active_curr_items 473331 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 3202048 vb_active_itm_memory 1151714197 vb_active_meta_data_memory 37393149 vb_active_num 128 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 473331 vb_active_ops_create 430962 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 100 vb_active_queue_age 378291000 vb_active_queue_drain 430963 vb_active_queue_fill 473331 vb_active_queue_memory 1355776 vb_active_queue_pending 2033664 vb_active_queue_size 42368 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_meta_data_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-02T19:01:11.301,ns_1@10.2.1.68:ns_doctor<0.1479.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,908066,320083}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,119240840}, {processes,71514592}, {processes_used,69937112}, {system,47726248}, {atom,1502673}, {atom_used,1494429}, {binary,5745352}, {code,15256817}, {ets,7120328}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,2378321920}, {buffered_memory,236052480}, {free_memory,25664688128}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2282193,5000}}, {context_switches,{25034365,0}}, {garbage_collection,{2089390,26315961012,0}}, {io,{{input,37428815977},{output,7645865934}}}, {reductions,{9954298541,3500813}}, {run_queue,0}, {runtime,{955530,940}}]}, {system_stats, [{cpu_utilization_rate,26.433915211970074}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,1675753107}, {couch_docs_data_size,1180974371}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,505576}, {curr_items_tot,505576}, {mem_used,1341128352}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2282}, {memory_data,{33657663488,7976976384,{<18077.178.1>,7628384}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 25063172 kB\nBuffers: 230520 kB\nCached: 2322580 kB\nSwapCached: 0 kB\nActive: 6251492 kB\nInactive: 1349860 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 25063172 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 2184 kB\nWriteback: 0 kB\nAnonPages: 5048204 kB\nMapped: 16312 kB\nSlab: 158040 kB\nPageTables: 14356 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6774044 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,908068,999531}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,102625000}, {processes,58793008}, {processes_used,58406192}, {system,43831992}, {atom,1413905}, {atom_used,1392509}, {binary,2892104}, {code,14575828}, {ets,6883976}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,2218876928}, {buffered_memory,273440768}, {free_memory,25925177344}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2284440,5000}}, {context_switches,{26656920,0}}, {garbage_collection,{2185761,26320934397,0}}, {io,{{input,37041598616},{output,8062381270}}}, {reductions,{9875411887,2774333}}, {run_queue,0}, {runtime,{892910,830}}]}, {system_stats, [{cpu_utilization_rate,7.0}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,1590936843}, {couch_docs_data_size,1244913925}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,516335}, {curr_items_tot,516335}, {mem_used,1369612680}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2284}, {memory_data,{33657663488,7673315328,{<18118.20373.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 25317556 kB\nBuffers: 267032 kB\nCached: 2166872 kB\nSwapCached: 140 kB\nActive: 5877196 kB\nInactive: 1473024 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 25317556 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 820 kB\nWriteback: 44 kB\nAnonPages: 4915976 kB\nMapped: 18020 kB\nSlab: 152680 kB\nPageTables: 14516 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6630360 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,908071,94794}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,97404704}, {processes,58359232}, {processes_used,58204120}, {system,39045472}, {atom,1414713}, {atom_used,1394174}, {binary,2478120}, {code,14610103}, {ets,2475416}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,2081361920}, {buffered_memory,436363264}, {free_memory,29157273600}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2287525,4999}}, {context_switches,{19641875,0}}, {garbage_collection,{3488205,15607721759,0}}, {io,{{input,25830392530},{output,2437733543}}}, {reductions,{6467467866,5642816}}, {run_queue,0}, {runtime,{651040,2120}}]}, {system_stats, [{cpu_utilization_rate,32.83208020050125}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,1367532998}, {couch_docs_data_size,1235268167}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,520845}, {curr_items_tot,520845}, {mem_used,1380799080}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2287}, {memory_data,{33657663488,4346748928,{<18117.19799.0>,4714984}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 28473900 kB\nBuffers: 426136 kB\nCached: 2032580 kB\nSwapCached: 0 kB\nActive: 2496732 kB\nInactive: 1652680 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 28473900 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 1100 kB\nWriteback: 0 kB\nAnonPages: 1690168 kB\nMapped: 17960 kB\nSlab: 206280 kB\nPageTables: 8172 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 3402160 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,908066,303384}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,99396256}, {processes,55927736}, {processes_used,55679320}, {system,43468520}, {atom,1413905}, {atom_used,1392509}, {binary,2457408}, {code,14613068}, {ets,6890224}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,1940733952}, {buffered_memory,335417344}, {free_memory,25972408320}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2281431,5001}}, {context_switches,{25236614,0}}, {garbage_collection,{2222433,25161218560,0}}, {io,{{input,35830267246},{output,7551285884}}}, {reductions,{9369313097,4735342}}, {run_queue,0}, {runtime,{885130,2140}}]}, {system_stats, [{cpu_utilization_rate,35.5}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,1268749591}, {couch_docs_data_size,1201014874}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,506934}, {curr_items_tot,506934}, {mem_used,1344614440}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2281}, {memory_data,{33657663488,7671042048,{<0.19057.0>,2914384}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 25363804 kB\nBuffers: 327556 kB\nCached: 1895248 kB\nSwapCached: 12 kB\nActive: 5635020 kB\nInactive: 1604220 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 25363804 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 3056 kB\nWriteback: 0 kB\nAnonPages: 5016056 kB\nMapped: 18340 kB\nSlab: 216784 kB\nPageTables: 14708 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6719628 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-02T19:02:11.301,ns_1@10.2.1.68:ns_doctor<0.1479.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,908126,319288}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.5319.1>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,91}, {started_on,1351908119}, {total_vbuckets,129}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351908164}, {vbuckets_done,118}]]}, {memory, [{total,115475392}, {processes,68462744}, {processes_used,66946696}, {system,47012648}, {atom,1502673}, {atom_used,1494429}, {binary,4781704}, {code,15256817}, {ets,7357584}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,2176913408}, {buffered_memory,238239744}, {free_memory,25868857344}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2342193,5000}}, {context_switches,{25183622,0}}, {garbage_collection,{2105802,26406623988,0}}, {io,{{input,37455425830},{output,7654794583}}}, {reductions,{9987930238,2787547}}, {run_queue,0}, {runtime,{963750,570}}]}, {system_stats, [{cpu_utilization_rate,7.0}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,1579459875}, {couch_docs_data_size,1459444850}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,674294}, {curr_items_tot,674294}, {mem_used,1735721272}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2342}, {memory_data,{33657663488,7789060096,{<18077.178.1>,7628384}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 25262556 kB\nBuffers: 232656 kB\nCached: 2125892 kB\nSwapCached: 0 kB\nActive: 5562420 kB\nInactive: 1842520 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 25262556 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 14112 kB\nWriteback: 260 kB\nAnonPages: 5046048 kB\nMapped: 16316 kB\nSlab: 154556 kB\nPageTables: 14392 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6769052 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,908128,999845}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,101545776}, {processes,57626656}, {processes_used,57239840}, {system,43919120}, {atom,1413905}, {atom_used,1392509}, {binary,2801664}, {code,14575828}, {ets,7062568}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,2891853824}, {buffered_memory,274583552}, {free_memory,25234173952}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2344440,4999}}, {context_switches,{26963151,0}}, {garbage_collection,{2217662,26388396918,0}}, {io,{{input,37047030852},{output,8069906876}}}, {reductions,{9907206025,2947621}}, {run_queue,0}, {runtime,{901880,820}}]}, {system_stats, [{cpu_utilization_rate,14.75}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,2255455507}, {couch_docs_data_size,1659486901}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,684463}, {curr_items_tot,684463}, {mem_used,1711940504}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2344}, {memory_data,{33657663488,8351494144,{<18118.20374.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 24642748 kB\nBuffers: 268148 kB\nCached: 2824076 kB\nSwapCached: 140 kB\nActive: 6220292 kB\nInactive: 1788188 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 24642748 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 2084 kB\nWriteback: 0 kB\nAnonPages: 4915800 kB\nMapped: 18020 kB\nSlab: 169168 kB\nPageTables: 14520 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6627860 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,908131,94887}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,98679752}, {processes,59486128}, {processes_used,59330536}, {system,39193624}, {atom,1414713}, {atom_used,1394174}, {binary,2452632}, {code,14610103}, {ets,2651488}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,2877030400}, {buffered_memory,437473280}, {free_memory,27916460032}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2347525,5000}}, {context_switches,{20381211,0}}, {garbage_collection,{3564735,15712007111,0}}, {io,{{input,25834376854},{output,2441412801}}}, {reductions,{6514676358,2662966}}, {run_queue,0}, {runtime,{669140,730}}]}, {system_stats, [{cpu_utilization_rate,28.0}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,2135494618}, {couch_docs_data_size,1622644075}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,689661}, {curr_items_tot,689661}, {mem_used,1725024856}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2347}, {memory_data,{33657663488,5604831232,{<18117.19799.0>,4714984}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 27262168 kB\nBuffers: 427220 kB\nCached: 2809600 kB\nSwapCached: 0 kB\nActive: 3507760 kB\nInactive: 1852816 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 27262168 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 1888 kB\nWriteback: 24 kB\nAnonPages: 2123596 kB\nMapped: 17960 kB\nSlab: 205988 kB\nPageTables: 9032 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 3837368 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,908126,303559}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,101943848}, {processes,58223408}, {processes_used,57974832}, {system,43720440}, {atom,1413905}, {atom_used,1392509}, {binary,2537776}, {code,14613068}, {ets,7064936}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,2646884352}, {buffered_memory,336060416}, {free_memory,25266548736}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2341431,5001}}, {context_switches,{25796337,0}}, {garbage_collection,{2278599,25249627549,0}}, {io,{{input,35833342137},{output,7554552323}}}, {reductions,{9410010256,2346627}}, {run_queue,0}, {runtime,{900500,910}}]}, {system_stats, [{cpu_utilization_rate,30.827067669172934}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,1952965930}, {couch_docs_data_size,1589391958}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,675325}, {curr_items_tot,675325}, {mem_used,1690209528}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2341}, {memory_data,{33657663488,8365727744,{<0.19057.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 24674488 kB\nBuffers: 328184 kB\nCached: 2584848 kB\nSwapCached: 12 kB\nActive: 6073552 kB\nInactive: 1856144 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 24674488 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 756 kB\nWriteback: 0 kB\nAnonPages: 5016144 kB\nMapped: 18340 kB\nSlab: 216176 kB\nPageTables: 14608 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6721708 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-02T19:02:33.810,ns_1@10.2.1.68:<0.19063.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,31,48}}) Stats for bucket "default": accepting_conns 1 auth_cmds 1 auth_errors 0 bucket_active_conns 1 bucket_conns 6 bytes 1895655048 bytes_read 1809742013 bytes_written 35770238 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 754375 conn_yields 212 connection_structures 5000 curr_connections 10 curr_conns_on_port_11209 6 curr_conns_on_port_11210 2 curr_items 754375 curr_items_tot 754375 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-03 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 874 ep_commit_time 10 ep_commit_time_total 1870 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 7 ep_data_age_highwat 25 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data/default ep_degraded_mode 0 ep_diskqueue_drain 747952 ep_diskqueue_fill 754375 ep_diskqueue_items 6423 ep_diskqueue_memory 205536 ep_diskqueue_pending 308304 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 269 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 4077 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 747824 ep_io_read_bytes 0 ep_io_write_bytes 1770082444 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 640256 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 1844747605 ep_max_checkpoints 2 ep_max_data_size 5242880000 ep_max_item_size 20971520 ep_max_size 5242880000 ep_max_txn_size 10000 ep_max_vbuckets 512 ep_mem_high_wat 3932160000 ep_mem_high_wat 3932160000 ep_mem_low_wat 3145728000 ep_mem_low_wat 3145728000 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 13489560 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 2346 ep_startup_time 1351907851 ep_storage_age 5 ep_storage_age_highwat 23 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 1835695105 ep_total_del_items 0 ep_total_enqueued 754375 ep_total_new_items 747824 ep_total_persisted 747824 ep_uncommitted_items 127 ep_value_size 1782549042 ep_vb0 0 ep_vb_snapshot_total 15 ep_vb_total 128 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_130_g8b09894 ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 555 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 1895655048 pid 17027 pointer_size 64 rejected_conns 0 rusage_system 150.608104 rusage_user 435.600778 tap_checkpoint_start_received 84 tap_mutation_received 1338733 tap_opaque_received 294 tap_vbucket_set_received 84 threads 4 time 1351908152 total_connections 189 uptime 2183 vb_active_curr_items 754375 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 3202048 vb_active_itm_memory 1835695105 vb_active_meta_data_memory 59595625 vb_active_num 128 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 754375 vb_active_ops_create 747951 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 100 vb_active_queue_age 12114000 vb_active_queue_drain 747952 vb_active_queue_fill 754375 vb_active_queue_memory 205536 vb_active_queue_pending 308304 vb_active_queue_size 6423 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_meta_data_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-02T19:03:11.302,ns_1@10.2.1.68:ns_doctor<0.1479.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,908186,322076}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,117175720}, {processes,70021488}, {processes_used,68444248}, {system,47154232}, {atom,1502673}, {atom_used,1494429}, {binary,4817872}, {code,15256817}, {ets,7472984}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,2972524544}, {buffered_memory,239570944}, {free_memory,25049997312}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2402196,5003}}, {context_switches,{25645555,0}}, {garbage_collection,{2152047,26520193200,0}}, {io,{{input,37473064619},{output,7658266686}}}, {reductions,{10031716710,3780077}}, {run_queue,0}, {runtime,{977150,1010}}]}, {system_stats, [{cpu_utilization_rate,26.06516290726817}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,2298656037}, {couch_docs_data_size,2014521353}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,841816}, {curr_items_tot,841816}, {mem_used,2127326072}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2402}, {memory_data,{33657663488,8569827328,{<18077.178.1>,7628384}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 24462888 kB\nBuffers: 233956 kB\nCached: 2902856 kB\nSwapCached: 0 kB\nActive: 5916228 kB\nInactive: 2270372 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 24462888 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 1928 kB\nWriteback: 0 kB\nAnonPages: 5049396 kB\nMapped: 16312 kB\nSlab: 173044 kB\nPageTables: 14300 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6770568 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,908189,444}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.25886.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,11}, {started_on,1351908152}, {total_vbuckets,129}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351908161}, {vbuckets_done,15}]]}, {memory, [{total,105409472}, {processes,61694640}, {processes_used,61324176}, {system,43714832}, {atom,1413905}, {atom_used,1392509}, {binary,2408664}, {code,14575828}, {ets,7249688}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,3427790848}, {buffered_memory,275542016}, {free_memory,24706326528}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2404440,4999}}, {context_switches,{27258179,0}}, {garbage_collection,{2247772,26454043956,0}}, {io,{{input,37051243463},{output,8072464470}}}, {reductions,{9938565613,1982334}}, {run_queue,0}, {runtime,{910390,430}}]}, {system_stats, [{cpu_utilization_rate,36.5}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,2834424779}, {couch_docs_data_size,2023913026}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,853120}, {curr_items_tot,853120}, {mem_used,2159011592}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2404}, {memory_data,{33657663488,8987144192,{<18118.20373.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 24127272 kB\nBuffers: 269084 kB\nCached: 3347452 kB\nSwapCached: 140 kB\nActive: 6401628 kB\nInactive: 2108252 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 24127272 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 16840 kB\nWriteback: 0 kB\nAnonPages: 4893160 kB\nMapped: 18032 kB\nSlab: 183028 kB\nPageTables: 14416 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6622660 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,908191,95565}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,101386352}, {processes,61932856}, {processes_used,61777264}, {system,39453496}, {atom,1414713}, {atom_used,1394174}, {binary,2534864}, {code,14610103}, {ets,2828496}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,3589099520}, {buffered_memory,438165504}, {free_memory,26777743360}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2407525,5000}}, {context_switches,{20738093,0}}, {garbage_collection,{3601034,15783421767,0}}, {io,{{input,25840017523},{output,2445133840}}}, {reductions,{6548044560,3141866}}, {run_queue,0}, {runtime,{679020,940}}]}, {system_stats, [{cpu_utilization_rate,7.268170426065163}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,2893137804}, {couch_docs_data_size,2066073640}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,858322}, {curr_items_tot,858322}, {mem_used,2170498184}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2407}, {memory_data,{33657663488,6750023680,{<18117.19799.0>,4714984}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 26150140 kB\nBuffers: 427896 kB\nCached: 3504980 kB\nSwapCached: 0 kB\nActive: 4329408 kB\nInactive: 2142000 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 26150140 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 740 kB\nWriteback: 0 kB\nAnonPages: 2538248 kB\nMapped: 17960 kB\nSlab: 205980 kB\nPageTables: 9836 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 4246968 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,908186,302892}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,102940296}, {processes,58862656}, {processes_used,58614000}, {system,44077640}, {atom,1413905}, {atom_used,1392509}, {binary,2718144}, {code,14613068}, {ets,7243112}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,3342942208}, {buffered_memory,337264640}, {free_memory,24570343424}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2401430,5000}}, {context_switches,{26141888,0}}, {garbage_collection,{2314205,25320482521,0}}, {io,{{input,35838900608},{output,7562183019}}}, {reductions,{9443172693,2192308}}, {run_queue,0}, {runtime,{911490,550}}]}, {system_stats, [{cpu_utilization_rate,27.930174563591024}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,2675410197}, {couch_docs_data_size,2022845228}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,844242}, {curr_items_tot,844242}, {mem_used,2134004584}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2401}, {memory_data,{33657663488,9050124288,{<0.19055.0>,2914384}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 23994476 kB\nBuffers: 329360 kB\nCached: 3264592 kB\nSwapCached: 12 kB\nActive: 6451588 kB\nInactive: 2157836 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 23994476 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 2168 kB\nWriteback: 0 kB\nAnonPages: 5015352 kB\nMapped: 18216 kB\nSlab: 216380 kB\nPageTables: 14704 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6720736 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-02T19:04:11.301,ns_1@10.2.1.68:ns_doctor<0.1479.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,908246,318014}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,116781128}, {processes,69544016}, {processes_used,67966616}, {system,47237112}, {atom,1502673}, {atom_used,1494429}, {binary,4830304}, {code,15256817}, {ets,7543008}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,3625054208}, {buffered_memory,240226304}, {free_memory,24380579840}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2462192,4998}}, {context_switches,{26017071,0}}, {garbage_collection,{2188826,26626386499,0}}, {io,{{input,37490240001},{output,7661307501}}}, {reductions,{10072291888,2730775}}, {run_queue,0}, {runtime,{990170,670}}]}, {system_stats, [{cpu_utilization_rate,6.982543640897756}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,2988135693}, {couch_docs_data_size,2412336632}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1010140}, {curr_items_tot,1010140}, {mem_used,2574316904}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2462}, {memory_data,{33657663488,9262862336,{<18077.178.1>,7628384}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 23809160 kB\nBuffers: 234596 kB\nCached: 3540092 kB\nSwapCached: 0 kB\nActive: 6291300 kB\nInactive: 2533524 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 23809160 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 2028 kB\nWriteback: 0 kB\nAnonPages: 5049756 kB\nMapped: 16312 kB\nSlab: 188916 kB\nPageTables: 14320 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6772284 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,908249,100}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.25886.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,82}, {started_on,1351908152}, {total_vbuckets,129}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351908221}, {vbuckets_done,106}]]}, {memory, [{total,104087168}, {processes,60482848}, {processes_used,60141824}, {system,43604320}, {atom,1413905}, {atom_used,1392509}, {binary,2191536}, {code,14575828}, {ets,7351440}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,3003723776}, {buffered_memory,277528576}, {free_memory,25137283072}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2464439,4998}}, {context_switches,{27330553,0}}, {garbage_collection,{2255963,26501255485,0}}, {io,{{input,37057426029},{output,8074770431}}}, {reductions,{9962605675,1933293}}, {run_queue,0}, {runtime,{915600,450}}]}, {system_stats, [{cpu_utilization_rate,30.174563591022444}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,2420602636}, {couch_docs_data_size,2188416438}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1021400}, {curr_items_tot,1021400}, {mem_used,2604368648}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2464}, {memory_data,{33657663488,8559742976,{<18118.20373.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 24548252 kB\nBuffers: 271024 kB\nCached: 2933324 kB\nSwapCached: 140 kB\nActive: 5582152 kB\nInactive: 2516312 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 24548252 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 7400 kB\nWriteback: 0 kB\nAnonPages: 4893904 kB\nMapped: 18024 kB\nSlab: 173304 kB\nPageTables: 14592 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6625260 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,908251,94861}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.26937.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,93}, {started_on,1351908206}, {total_vbuckets,129}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351908258}, {vbuckets_done,120}]]}, {memory, [{total,98300184}, {processes,58765656}, {processes_used,58634696}, {system,39534528}, {atom,1414713}, {atom_used,1394174}, {binary,2462944}, {code,14610103}, {ets,2976760}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,3078119424}, {buffered_memory,440791040}, {free_memory,26853793792}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2467524,4999}}, {context_switches,{20870348,0}}, {garbage_collection,{3615753,15836895255,0}}, {io,{{input,25853149515},{output,2448633838}}}, {reductions,{6574696469,1914890}}, {run_queue,0}, {runtime,{684830,420}}]}, {system_stats, [{cpu_utilization_rate,6.75}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,2473379605}, {couch_docs_data_size,2309289800}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1027032}, {curr_items_tot,1027032}, {mem_used,2619399752}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2467}, {memory_data,{33657663488,6816694272,{<18117.19800.0>,4714984}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 26224408 kB\nBuffers: 430460 kB\nCached: 3005976 kB\nSwapCached: 0 kB\nActive: 3650732 kB\nInactive: 2744520 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 26224408 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 13288 kB\nWriteback: 0 kB\nAnonPages: 2958868 kB\nMapped: 17964 kB\nSlab: 206948 kB\nPageTables: 10728 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 4669764 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,908246,302520}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,104960712}, {processes,60546232}, {processes_used,60297336}, {system,44414480}, {atom,1413905}, {atom_used,1392509}, {binary,2872720}, {code,14613068}, {ets,7425936}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,4024520704}, {buffered_memory,338010112}, {free_memory,23886323712}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2461430,4999}}, {context_switches,{26409533,0}}, {garbage_collection,{2340995,25383354914,0}}, {io,{{input,35840659645},{output,7564364283}}}, {reductions,{9473450391,2555125}}, {run_queue,0}, {runtime,{920010,640}}]}, {system_stats, [{cpu_utilization_rate,28.75}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,3316776763}, {couch_docs_data_size,2411413358}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1012563}, {curr_items_tot,1012563}, {mem_used,2579244264}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2461}, {memory_data,{33657663488,9762205696,{<0.19057.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 23326488 kB\nBuffers: 330088 kB\nCached: 3930196 kB\nSwapCached: 12 kB\nActive: 6803628 kB\nInactive: 2474104 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 23326488 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 2072 kB\nWriteback: 0 kB\nAnonPages: 5017180 kB\nMapped: 18216 kB\nSlab: 216192 kB\nPageTables: 14704 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6723580 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-02T19:04:13.808,ns_1@10.2.1.68:<0.19063.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,31,48}}) Stats for bucket "default": accepting_conns 1 auth_cmds 1 auth_errors 0 bucket_active_conns 1 bucket_conns 6 bytes 2639421400 bytes_read 2482753839 bytes_written 48937786 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 1034925 conn_yields 291 connection_structures 5000 curr_connections 10 curr_conns_on_port_11209 6 curr_conns_on_port_11210 2 curr_items 1034925 curr_items_tot 1034925 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-03 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 1005 ep_commit_time 13 ep_commit_time_total 2992 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 7 ep_data_age_highwat 25 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data/default ep_degraded_mode 0 ep_diskqueue_drain 1029843 ep_diskqueue_fill 1034925 ep_diskqueue_items 5083 ep_diskqueue_memory 162624 ep_diskqueue_pending 243936 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 367 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 1343 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 1029671 ep_io_read_bytes 0 ep_io_write_bytes 2437162739 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 640256 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 2530793127 ep_max_checkpoints 2 ep_max_data_size 5242880000 ep_max_item_size 20971520 ep_max_size 5242880000 ep_max_txn_size 10000 ep_max_vbuckets 512 ep_mem_high_wat 3932160000 ep_mem_high_wat 3932160000 ep_mem_low_wat 3145728000 ep_mem_low_wat 3145728000 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 41305912 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 3740 ep_startup_time 1351907851 ep_storage_age 4 ep_storage_age_highwat 23 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 2518374027 ep_total_del_items 0 ep_total_enqueued 1034925 ep_total_new_items 1029671 ep_total_persisted 1029671 ep_uncommitted_items 7925 ep_value_size 2445450578 ep_vb0 0 ep_vb_snapshot_total 15 ep_vb_total 128 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_130_g8b09894 ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 555 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 2639421400 pid 17027 pointer_size 64 rejected_conns 0 rusage_system 162.393312 rusage_user 468.717744 tap_checkpoint_start_received 84 tap_mutation_received 1338733 tap_opaque_received 294 tap_vbucket_set_received 84 threads 4 time 1351908252 total_connections 189 uptime 2283 vb_active_curr_items 1034925 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 6351872 vb_active_itm_memory 2518374027 vb_active_meta_data_memory 81759075 vb_active_num 128 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1034925 vb_active_ops_create 1029842 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 100 vb_active_queue_age 7490000 vb_active_queue_drain 1029843 vb_active_queue_fill 1034925 vb_active_queue_memory 162624 vb_active_queue_pending 243936 vb_active_queue_size 5082 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_meta_data_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-02T19:05:11.301,ns_1@10.2.1.68:ns_doctor<0.1479.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,908306,317865}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,119019816}, {processes,71856880}, {processes_used,70278544}, {system,47162936}, {atom,1502673}, {atom_used,1494429}, {binary,4580288}, {code,15256817}, {ets,7720432}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,4364050432}, {buffered_memory,241160192}, {free_memory,23617323008}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2522193,5000}}, {context_switches,{26293897,0}}, {garbage_collection,{2215953,26723679855,0}}, {io,{{input,37510442433},{output,7667481914}}}, {reductions,{10109334284,3573192}}, {run_queue,0}, {runtime,{1000670,1070}}]}, {system_stats, [{cpu_utilization_rate,17.705735660847882}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,3664037161}, {couch_docs_data_size,2826437826}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1178298}, {curr_items_tot,1178298}, {mem_used,3020350664}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2522}, {memory_data,{33657663488,10026373120,{<18077.178.1>,7628384}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 23063792 kB\nBuffers: 235508 kB\nCached: 4261768 kB\nSwapCached: 0 kB\nActive: 6670032 kB\nInactive: 2882336 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 23063792 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 1624 kB\nWriteback: 0 kB\nAnonPages: 5054696 kB\nMapped: 16312 kB\nSlab: 207004 kB\nPageTables: 14340 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6780604 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,908309,2413}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,105316104}, {processes,61378192}, {processes_used,60991376}, {system,43937912}, {atom,1413905}, {atom_used,1392509}, {binary,2398920}, {code,14575828}, {ets,7485288}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,3704139776}, {buffered_memory,278913024}, {free_memory,24419876864}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2524440,5000}}, {context_switches,{27503888,0}}, {garbage_collection,{2273903,26556801999,0}}, {io,{{input,37062399034},{output,8079738099}}}, {reductions,{9989674937,3075916}}, {run_queue,0}, {runtime,{922530,990}}]}, {system_stats, [{cpu_utilization_rate,13.5}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,3067786557}, {couch_docs_data_size,2871338041}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1189787}, {curr_items_tot,1189787}, {mem_used,3050620408}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2524}, {memory_data,{33657663488,9173790720,{<18118.20373.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 23847536 kB\nBuffers: 272376 kB\nCached: 3617324 kB\nSwapCached: 140 kB\nActive: 5625872 kB\nInactive: 3157456 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 23847536 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 2628 kB\nWriteback: 144 kB\nAnonPages: 4893680 kB\nMapped: 18020 kB\nSlab: 189504 kB\nPageTables: 14524 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6625192 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,908311,96594}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,101798792}, {processes,62111400}, {processes_used,61956128}, {system,39687392}, {atom,1414713}, {atom_used,1394174}, {binary,2465936}, {code,14610103}, {ets,3130640}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,3875540992}, {buffered_memory,441782272}, {free_memory,25589121024}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2527525,5000}}, {context_switches,{21317276,0}}, {garbage_collection,{3660858,15915590366,0}}, {io,{{input,25855894200},{output,2451370755}}}, {reductions,{6611334179,2837679}}, {run_queue,1}, {runtime,{696570,810}}]}, {system_stats, [{cpu_utilization_rate,31.0}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,3158508857}, {couch_docs_data_size,2863878808}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1196159}, {curr_items_tot,1196159}, {mem_used,3066481416}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2527}, {memory_data,{33657663488,7927472128,{<18117.19799.0>,4714984}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 24989376 kB\nBuffers: 431428 kB\nCached: 3784708 kB\nSwapCached: 0 kB\nActive: 4437348 kB\nInactive: 3193520 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 24989376 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 1264 kB\nWriteback: 0 kB\nAnonPages: 3414464 kB\nMapped: 17960 kB\nSlab: 206116 kB\nPageTables: 11544 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 5120244 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,908306,303146}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,105384864}, {processes,61292552}, {processes_used,61043656}, {system,44092312}, {atom,1413905}, {atom_used,1392509}, {binary,2486760}, {code,14613068}, {ets,7487808}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,4700844032}, {buffered_memory,338653184}, {free_memory,23200399360}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2521430,5000}}, {context_switches,{26648894,0}}, {garbage_collection,{2365151,25443746849,0}}, {io,{{input,35842289212},{output,7566281498}}}, {reductions,{9502592409,2783262}}, {run_queue,0}, {runtime,{927820,780}}]}, {system_stats, [{cpu_utilization_rate,19.402985074626866}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,4004588697}, {couch_docs_data_size,2835014927}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1180647}, {curr_items_tot,1180647}, {mem_used,3025811000}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2521}, {memory_data,{33657663488,10456891392,{<0.19055.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 22656640 kB\nBuffers: 330716 kB\nCached: 4590668 kB\nSwapCached: 12 kB\nActive: 7133080 kB\nInactive: 2802512 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 22656640 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 1420 kB\nWriteback: 0 kB\nAnonPages: 5013424 kB\nMapped: 18216 kB\nSlab: 228148 kB\nPageTables: 14688 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6719644 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-02T19:05:53.811,ns_1@10.2.1.68:<0.19063.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,31,48}}) Stats for bucket "default": accepting_conns 1 auth_cmds 1 auth_errors 0 bucket_active_conns 2 bucket_conns 6 bytes 3333464312 bytes_read 3156472161 bytes_written 62104477 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 1315333 conn_yields 378 connection_structures 5000 curr_connections 10 curr_conns_on_port_11209 6 curr_conns_on_port_11210 2 curr_items 1315312 curr_items_tot 1315312 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-03 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 1087 ep_commit_time 43 ep_commit_time_total 3559 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 14 ep_data_age_highwat 25 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data/default ep_degraded_mode 0 ep_diskqueue_drain 1264279 ep_diskqueue_fill 1315311 ep_diskqueue_items 51110 ep_diskqueue_memory 1633024 ep_diskqueue_pending 2449536 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 457 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 15840 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 1264022 ep_io_read_bytes 0 ep_io_write_bytes 2992568245 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 945378 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 3217516379 ep_max_checkpoints 2 ep_max_data_size 5242880000 ep_max_item_size 20971520 ep_max_size 5242880000 ep_max_txn_size 10000 ep_max_vbuckets 512 ep_mem_high_wat 3932160000 ep_mem_high_wat 3932160000 ep_mem_low_wat 3145728000 ep_mem_low_wat 3145728000 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 39566946 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 35270 ep_startup_time 1351907851 ep_storage_age 13 ep_storage_age_highwat 23 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 3201694520 ep_total_del_items 0 ep_total_enqueued 1315323 ep_total_new_items 1264022 ep_total_persisted 1264022 ep_uncommitted_items 4378 ep_value_size 3109042937 ep_vb0 0 ep_vb_snapshot_total 16 ep_vb_total 128 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_130_g8b09894 ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 555 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 3333464312 pid 17027 pointer_size 64 rejected_conns 0 rusage_system 172.305805 rusage_user 495.121730 tap_checkpoint_start_received 84 tap_mutation_received 1338733 tap_opaque_received 294 tap_vbucket_set_received 84 threads 4 time 1351908353 total_connections 189 uptime 2384 vb_active_curr_items 1315312 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 6351872 vb_active_itm_memory 3201694520 vb_active_meta_data_memory 103909648 vb_active_num 128 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1315312 vb_active_ops_create 1264278 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 100 vb_active_queue_age 493180000 vb_active_queue_drain 1264279 vb_active_queue_fill 1315311 vb_active_queue_memory 1633024 vb_active_queue_pending 2449536 vb_active_queue_size 51032 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_meta_data_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-02T19:06:11.301,ns_1@10.2.1.68:ns_doctor<0.1479.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,908366,318786}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,119900800}, {processes,72355904}, {processes_used,70778744}, {system,47544896}, {atom,1502673}, {atom_used,1494429}, {binary,4886416}, {code,15256817}, {ets,7794024}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,5118189568}, {buffered_memory,242204672}, {free_memory,22863978496}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2582194,5001}}, {context_switches,{26631376,0}}, {garbage_collection,{2249335,26826453425,0}}, {io,{{input,37527521510},{output,7670551473}}}, {reductions,{10148722782,3786604}}, {run_queue,0}, {runtime,{1011560,1080}}]}, {system_stats, [{cpu_utilization_rate,31.07769423558897}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,4431877437}, {couch_docs_data_size,3262594912}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1347263}, {curr_items_tot,1347263}, {mem_used,3364330568}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2582}, {memory_data,{33657663488,10757750784,{<18077.178.1>,7628384}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 22328104 kB\nBuffers: 236528 kB\nCached: 4998232 kB\nSwapCached: 0 kB\nActive: 7071820 kB\nInactive: 3197864 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 22328104 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 1592 kB\nWriteback: 0 kB\nAnonPages: 5034876 kB\nMapped: 16312 kB\nSlab: 225404 kB\nPageTables: 14332 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6778848 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,908369,2075}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,105049680}, {processes,60945256}, {processes_used,60558600}, {system,44104424}, {atom,1413905}, {atom_used,1392509}, {binary,2492560}, {code,14575828}, {ets,7556952}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,4374339584}, {buffered_memory,279777280}, {free_memory,23731539968}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2584439,4999}}, {context_switches,{27783203,0}}, {garbage_collection,{2302086,26620463112,0}}, {io,{{input,37064176559},{output,8081865342}}}, {reductions,{10020385291,2073994}}, {run_queue,0}, {runtime,{931350,500}}]}, {system_stats, [{cpu_utilization_rate,25.870646766169155}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,3696804629}, {couch_docs_data_size,3252808597}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1357671}, {curr_items_tot,1357671}, {mem_used,3392330728}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2584}, {memory_data,{33657663488,9870000128,{<18118.20373.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 23175332 kB\nBuffers: 273220 kB\nCached: 4271816 kB\nSwapCached: 140 kB\nActive: 5950008 kB\nInactive: 3490052 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 23175332 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 1676 kB\nWriteback: 0 kB\nAnonPages: 4894668 kB\nMapped: 18020 kB\nSlab: 205356 kB\nPageTables: 14520 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6623696 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,908371,98063}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,103078608}, {processes,62962264}, {processes_used,62807072}, {system,40116344}, {atom,1414713}, {atom_used,1394174}, {binary,2718520}, {code,14610103}, {ets,3308704}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,4686090240}, {buffered_memory,442789888}, {free_memory,24334336000}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2587526,5001}}, {context_switches,{22018346,0}}, {garbage_collection,{3733544,16017045224,0}}, {io,{{input,25863162820},{output,2458111730}}}, {reductions,{6657296376,3535649}}, {run_queue,0}, {runtime,{714240,1080}}]}, {system_stats, [{cpu_utilization_rate,31.0}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,3970991404}, {couch_docs_data_size,3284820362}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1364946}, {curr_items_tot,1364946}, {mem_used,3410170008}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2587}, {memory_data,{33657663488,9169682432,{<18117.19799.0>,4714984}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 23764124 kB\nBuffers: 432412 kB\nCached: 4576260 kB\nSwapCached: 0 kB\nActive: 5429348 kB\nInactive: 3409020 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 23764124 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 1032 kB\nWriteback: 0 kB\nAnonPages: 3829284 kB\nMapped: 17960 kB\nSlab: 222200 kB\nPageTables: 12244 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 5547340 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,908366,303017}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.26755.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,35}, {started_on,1351908324}, {total_vbuckets,129}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351908365}, {vbuckets_done,46}]]}, {memory, [{total,101637936}, {processes,57335552}, {processes_used,57112208}, {system,44302384}, {atom,1413905}, {atom_used,1392509}, {binary,2494384}, {code,14613068}, {ets,7687432}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,4609183744}, {buffered_memory,340586496}, {free_memory,23292829696}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2581430,5000}}, {context_switches,{26774853,0}}, {garbage_collection,{2378394,25496160886,0}}, {io,{{input,35855011539},{output,7572443626}}}, {reductions,{9528588367,1949225}}, {run_queue,0}, {runtime,{934100,440}}]}, {system_stats, [{cpu_utilization_rate,23.383084577114428}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,4036518399}, {couch_docs_data_size,3103588710}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1349193}, {curr_items_tot,1349193}, {mem_used,3422566216}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2581}, {memory_data,{33657663488,10378682368,{<0.19055.0>,2914384}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 22746904 kB\nBuffers: 332604 kB\nCached: 4501156 kB\nSwapCached: 12 kB\nActive: 6659492 kB\nInactive: 3183256 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 22746904 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 4264 kB\nWriteback: 800 kB\nAnonPages: 5008912 kB\nMapped: 18220 kB\nSlab: 230428 kB\nPageTables: 14748 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6717128 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-02T19:07:11.301,ns_1@10.2.1.68:ns_doctor<0.1479.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,908426,318115}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,120762848}, {processes,73347288}, {processes_used,71770288}, {system,47415560}, {atom,1502673}, {atom_used,1494429}, {binary,4582568}, {code,15256817}, {ets,7969128}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,5922803712}, {buffered_memory,242909184}, {free_memory,22034821120}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2642194,5002}}, {context_switches,{27086239,0}}, {garbage_collection,{2294200,26939928594,0}}, {io,{{input,37545169917},{output,7673868358}}}, {reductions,{10192117755,4563849}}, {run_queue,0}, {runtime,{1025130,1700}}]}, {system_stats, [{cpu_utilization_rate,35.08771929824562}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,5228402666}, {couch_docs_data_size,3678349582}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1514970}, {curr_items_tot,1514970}, {mem_used,3808083448}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2642}, {memory_data,{33657663488,11591479296,{<18077.178.1>,7628384}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 21518380 kB\nBuffers: 237216 kB\nCached: 5783660 kB\nSwapCached: 0 kB\nActive: 7589032 kB\nInactive: 3470492 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 21518380 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 2492 kB\nWriteback: 0 kB\nAnonPages: 5038372 kB\nMapped: 16312 kB\nSlab: 244564 kB\nPageTables: 14272 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6779036 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,908429,3571}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,106381792}, {processes,62017936}, {processes_used,61631200}, {system,44363856}, {atom,1413905}, {atom_used,1392509}, {binary,2577720}, {code,14575828}, {ets,7731832}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,5011087360}, {buffered_memory,280707072}, {free_memory,23077105664}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2644440,5001}}, {context_switches,{27948029,0}}, {garbage_collection,{2318271,26674351737,0}}, {io,{{input,37065395912},{output,8083517075}}}, {reductions,{10046941304,2251938}}, {run_queue,0}, {runtime,{938280,620}}]}, {system_stats, [{cpu_utilization_rate,7.75}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,4386432330}, {couch_docs_data_size,3691193244}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1526086}, {curr_items_tot,1526086}, {mem_used,3838160744}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2644}, {memory_data,{33657663488,10558717952,{<18118.20373.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 22536236 kB\nBuffers: 274128 kB\nCached: 4893640 kB\nSwapCached: 140 kB\nActive: 6246180 kB\nInactive: 3816788 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 22536236 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 2080 kB\nWriteback: 96 kB\nAnonPages: 4894932 kB\nMapped: 18020 kB\nSlab: 221076 kB\nPageTables: 14520 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6622520 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,908431,97422}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,99561488}, {processes,59254272}, {processes_used,59099000}, {system,40307216}, {atom,1414713}, {atom_used,1394174}, {binary,2737072}, {code,14610103}, {ets,3483304}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,5462351872}, {buffered_memory,443584512}, {free_memory,23103057920}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2647525,5000}}, {context_switches,{22424126,0}}, {garbage_collection,{3774620,16092106661,0}}, {io,{{input,25865511795},{output,2460736974}}}, {reductions,{6692330364,3292347}}, {run_queue,0}, {runtime,{725270,1030}}]}, {system_stats, [{cpu_utilization_rate,29.925187032418954}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,4742503030}, {couch_docs_data_size,3701378027}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1533817}, {curr_items_tot,1533817}, {mem_used,3857449064}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2647}, {memory_data,{33657663488,10380648448,{<18117.703.0>,3143528}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 22561580 kB\nBuffers: 433188 kB\nCached: 5334328 kB\nSwapCached: 0 kB\nActive: 6328048 kB\nInactive: 3693196 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 22561580 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 1428 kB\nWriteback: 96 kB\nAnonPages: 4253748 kB\nMapped: 17960 kB\nSlab: 240924 kB\nPageTables: 13048 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 5964432 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,908426,303170}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.26755.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,75}, {started_on,1351908324}, {total_vbuckets,129}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351908421}, {vbuckets_done,97}]]}, {memory, [{total,101849784}, {processes,57720776}, {processes_used,57513992}, {system,44129008}, {atom,1413905}, {atom_used,1392509}, {binary,2227472}, {code,14613068}, {ets,7777192}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,4207181824}, {buffered_memory,342114304}, {free_memory,23716040704}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2641431,5001}}, {context_switches,{26833742,0}}, {garbage_collection,{2384710,25540882329,0}}, {io,{{input,35856182067},{output,7573821841}}}, {reductions,{9551419577,1905883}}, {run_queue,0}, {runtime,{939440,450}}]}, {system_stats, [{cpu_utilization_rate,7.980049875311721}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,3565516114}, {couch_docs_data_size,3212893148}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1517679}, {curr_items_tot,1517679}, {mem_used,3815637816}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2641}, {memory_data,{33657663488,9942011904,{<0.19055.0>,2914384}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 23160196 kB\nBuffers: 334096 kB\nCached: 4108576 kB\nSwapCached: 12 kB\nActive: 5958736 kB\nInactive: 3476528 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 23160196 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 3044 kB\nWriteback: 168 kB\nAnonPages: 4992508 kB\nMapped: 18228 kB\nSlab: 224068 kB\nPageTables: 14576 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6713928 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:warn,2012-11-02T19:07:17.304,ns_1@10.2.1.68:<0.19063.0>:stats_collector:latest_tick:223]Dropped 1 ticks [stats:debug,2012-11-02T19:07:34.808,ns_1@10.2.1.68:<0.19063.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,31,48}}) Stats for bucket "default": accepting_conns 1 auth_cmds 1 auth_errors 0 bucket_active_conns 1 bucket_conns 6 bytes 3619977320 bytes_read 3837899771 bytes_written 75394860 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 1599244 conn_yields 464 connection_structures 5000 curr_connections 10 curr_conns_on_port_11209 6 curr_conns_on_port_11210 2 curr_items 1599244 curr_items_tot 1599244 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-03 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 1095 ep_commit_time 279 ep_commit_time_total 9456 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 104 ep_data_age_highwat 112 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data/default ep_degraded_mode 0 ep_diskqueue_drain 1344846 ep_diskqueue_fill 1599244 ep_diskqueue_items 254439 ep_diskqueue_memory 8140736 ep_diskqueue_pending 12211104 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 491 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 30865 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 1344117 ep_io_read_bytes 0 ep_io_write_bytes 3182857860 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 1280512 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 3508352988 ep_max_checkpoints 2 ep_max_data_size 5242880000 ep_max_item_size 20971520 ep_max_size 5242880000 ep_max_txn_size 10000 ep_max_vbuckets 512 ep_mem_high_wat 3932160000 ep_mem_high_wat 3932160000 ep_mem_low_wat 3145728000 ep_mem_low_wat 3145728000 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 29750 ep_num_expiry_pager_runs 0 ep_num_non_resident 172216 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 2 ep_num_value_ejects 172216 ep_oom_errors 0 ep_overhead 35013864 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 223574 ep_startup_time 1351907851 ep_storage_age 101 ep_storage_age_highwat 112 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 3489162060 ep_total_del_items 0 ep_total_enqueued 1599244 ep_total_new_items 1344117 ep_total_persisted 1344117 ep_uncommitted_items 2187 ep_value_size 3376366968 ep_vb0 0 ep_vb_snapshot_total 17 ep_vb_total 128 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_130_g8b09894 ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 555 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 3619977320 pid 17027 pointer_size 64 rejected_conns 0 rusage_system 179.018785 rusage_user 507.638827 tap_checkpoint_start_received 84 tap_mutation_received 1338733 tap_opaque_received 294 tap_vbucket_set_received 84 threads 4 time 1351908454 total_connections 189 uptime 2485 vb_active_curr_items 1599244 vb_active_eject 172216 vb_active_expired 0 vb_active_ht_memory 6351872 vb_active_itm_memory 3489162060 vb_active_meta_data_memory 126340276 vb_active_num 128 vb_active_num_non_resident 172216 vb_active_num_ref_ejects 172216 vb_active_num_ref_items 1599244 vb_active_ops_create 1344845 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 89 vb_active_queue_age 11814588000 vb_active_queue_drain 1344846 vb_active_queue_fill 1599244 vb_active_queue_memory 8140736 vb_active_queue_pending 12211104 vb_active_queue_size 254398 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_meta_data_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-02T19:08:11.301,ns_1@10.2.1.68:ns_doctor<0.1479.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,908486,317953}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.9970.1>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,41}, {started_on,1351908470}, {total_vbuckets,129}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351908526}, {vbuckets_done,54}]]}, {memory, [{total,117672792}, {processes,69256624}, {processes_used,67716656}, {system,48416168}, {atom,1502673}, {atom_used,1494429}, {binary,5366648}, {code,15256817}, {ets,8180728}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,5375868928}, {buffered_memory,245325824}, {free_memory,22590840832}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2702194,5001}}, {context_switches,{27163145,0}}, {garbage_collection,{2302954,27022882875,0}}, {io,{{input,37570430338},{output,7683150255}}}, {reductions,{10222753716,2651181}}, {run_queue,0}, {runtime,{1032190,520}}]}, {system_stats, [{cpu_utilization_rate,17.206982543640898}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,4815519931}, {couch_docs_data_size,3797086708}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1683293}, {curr_items_tot,1683293}, {mem_used,3772547272}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2702}, {memory_data,{33657663488,11140849664,{<18077.178.1>,7628384}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 22061616 kB\nBuffers: 239576 kB\nCached: 5249544 kB\nSwapCached: 0 kB\nActive: 6719976 kB\nInactive: 3806216 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 22061616 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 5580 kB\nWriteback: 0 kB\nAnonPages: 5037468 kB\nMapped: 16352 kB\nSlab: 234120 kB\nPageTables: 14476 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6775448 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,908489,5445}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,102190184}, {processes,57466696}, {processes_used,57079880}, {system,44723488}, {atom,1413905}, {atom_used,1392509}, {binary,2760504}, {code,14575828}, {ets,7909736}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,5752107008}, {buffered_memory,281501696}, {free_memory,22317154304}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2704441,5001}}, {context_switches,{28226039,0}}, {garbage_collection,{2346313,26739559270,0}}, {io,{{input,37070613324},{output,8091803372}}}, {reductions,{10077856225,2731473}}, {run_queue,0}, {runtime,{947200,720}}]}, {system_stats, [{cpu_utilization_rate,7.105263157894737}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,5118289794}, {couch_docs_data_size,4114442677}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1693686}, {curr_items_tot,1693686}, {mem_used,3785065752}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2704}, {memory_data,{33657663488,11262959616,{<18118.695.0>,3143528}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 21794096 kB\nBuffers: 274904 kB\nCached: 5617292 kB\nSwapCached: 140 kB\nActive: 6642484 kB\nInactive: 4144552 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 21794096 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 932 kB\nWriteback: 0 kB\nAnonPages: 4894620 kB\nMapped: 18020 kB\nSlab: 239168 kB\nPageTables: 14504 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6615224 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,908491,97992}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,101448832}, {processes,61270912}, {processes_used,61115320}, {system,40177920}, {atom,1414713}, {atom_used,1394174}, {binary,2526792}, {code,14610103}, {ets,3563680}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,6189436928}, {buffered_memory,444551168}, {free_memory,22233907200}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2707525,5000}}, {context_switches,{22711625,0}}, {garbage_collection,{3803729,16156692284,0}}, {io,{{input,25867374301},{output,2462870929}}}, {reductions,{6723118367,2404673}}, {run_queue,0}, {runtime,{733910,530}}]}, {system_stats, [{cpu_utilization_rate,7.0},{swap_total,10701078528},{swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,5459686520}, {couch_docs_data_size,4095853890}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1702975}, {curr_items_tot,1702975}, {mem_used,3810012312}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2707}, {memory_data,{33657663488,11321413632,{<18117.19800.0>,4714984}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 21712800 kB\nBuffers: 434132 kB\nCached: 6044372 kB\nSwapCached: 0 kB\nActive: 6850904 kB\nInactive: 4000736 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 21712800 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 1660 kB\nWriteback: 4 kB\nAnonPages: 4373132 kB\nMapped: 17964 kB\nSlab: 259120 kB\nPageTables: 13440 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 6085196 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,908486,302721}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,102560288}, {processes,58001328}, {processes_used,57752432}, {system,44558960}, {atom,1413905}, {atom_used,1392509}, {binary,2527016}, {code,14613068}, {ets,7911872}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,4346245120}, {buffered_memory,343482368}, {free_memory,23578406912}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2701430,5000}}, {context_switches,{26891676,0}}, {garbage_collection,{2391433,25585535444,0}}, {io,{{input,35857294077},{output,7575205058}}}, {reductions,{9574207593,1882199}}, {run_queue,0}, {runtime,{944520,460}}]}, {system_stats, [{cpu_utilization_rate,10.447761194029852}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,3679763826}, {couch_docs_data_size,3602657832}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1686820}, {curr_items_tot,1686820}, {mem_used,3822712088}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2701}, {memory_data,{33657663488,10057551872,{<0.19055.0>,2914384}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 23025788 kB\nBuffers: 335432 kB\nCached: 4244380 kB\nSwapCached: 12 kB\nActive: 5561308 kB\nInactive: 4012076 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 23025788 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 1724 kB\nWriteback: 0 kB\nAnonPages: 4993588 kB\nMapped: 18216 kB\nSlab: 220328 kB\nPageTables: 14672 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6718388 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-02T19:09:11.301,ns_1@10.2.1.68:ns_doctor<0.1479.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,908546,316901}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.9970.1>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,95}, {started_on,1351908470}, {total_vbuckets,129}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351908585}, {vbuckets_done,123}]]}, {memory, [{total,119755568}, {processes,71091800}, {processes_used,69556432}, {system,48663768}, {atom,1502673}, {atom_used,1494429}, {binary,5433168}, {code,15256817}, {ets,8361016}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,4801921024}, {buffered_memory,247816192}, {free_memory,23176962048}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2762194,5002}}, {context_switches,{27229941,0}}, {garbage_collection,{2310296,27103025362,0}}, {io,{{input,37587004285},{output,7685334095}}}, {reductions,{10252333651,2646213}}, {run_queue,0}, {runtime,{1038580,460}}]}, {system_stats, [{cpu_utilization_rate,6.733167082294265}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,4197881195}, {couch_docs_data_size,4008557397}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1851786}, {curr_items_tot,1851786}, {mem_used,3736063032}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2762}, {memory_data,{33657663488,10492383232,{<18077.178.1>,7628384}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 22633752 kB\nBuffers: 242008 kB\nCached: 4689376 kB\nSwapCached: 0 kB\nActive: 5639140 kB\nInactive: 4328656 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 22633752 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 19780 kB\nWriteback: 0 kB\nAnonPages: 5036652 kB\nMapped: 16232 kB\nSlab: 220076 kB\nPageTables: 14160 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6778668 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,908549,4699}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,104249264}, {processes,59619880}, {processes_used,59233064}, {system,44629384}, {atom,1413905}, {atom_used,1392509}, {binary,2491536}, {code,14575828}, {ets,8084328}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,6454521856}, {buffered_memory,282185728}, {free_memory,21595803648}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2764440,5000}}, {context_switches,{28455527,0}}, {garbage_collection,{2369165,26799295011,0}}, {io,{{input,37072232546},{output,8093707542}}}, {reductions,{10106750528,2693486}}, {run_queue,0}, {runtime,{954880,730}}]}, {system_stats, [{cpu_utilization_rate,7.25}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,5832074612}, {couch_docs_data_size,4529765212}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1861175}, {curr_items_tot,1861175}, {mem_used,3717005560}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2764}, {memory_data,{33657663488,12011839488,{<18118.20373.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 21089652 kB\nBuffers: 275572 kB\nCached: 6303244 kB\nSwapCached: 140 kB\nActive: 7001740 kB\nInactive: 4472728 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 21089652 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 1468 kB\nWriteback: 44 kB\nAnonPages: 4895424 kB\nMapped: 18020 kB\nSlab: 256292 kB\nPageTables: 14516 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6621712 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,908551,98526}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,102821856}, {processes,61802264}, {processes_used,61646672}, {system,41019592}, {atom,1414713}, {atom_used,1394174}, {binary,3191688}, {code,14610103}, {ets,3740744}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,6970507264}, {buffered_memory,445308928}, {free_memory,21386088448}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2767525,4999}}, {context_switches,{23032464,0}}, {garbage_collection,{3833124,16225028380,0}}, {io,{{input,25872806257},{output,2467150741}}}, {reductions,{6755186732,2930130}}, {run_queue,0}, {runtime,{742940,790}}]}, {system_stats, [{cpu_utilization_rate,6.965174129353234}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,6252344705}, {couch_docs_data_size,4531258116}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1872384}, {curr_items_tot,1872384}, {mem_used,3737878360}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2767}, {memory_data,{33657663488,12187262976,{<18117.19800.0>,4714984}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 20884976 kB\nBuffers: 434872 kB\nCached: 6807136 kB\nSwapCached: 0 kB\nActive: 7346440 kB\nInactive: 4314640 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 20884976 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 776 kB\nWriteback: 0 kB\nAnonPages: 4418976 kB\nMapped: 17964 kB\nSlab: 278156 kB\nPageTables: 13536 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 6130064 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,908546,302495}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,106760432}, {processes,62018720}, {processes_used,61769824}, {system,44741712}, {atom,1413905}, {atom_used,1392509}, {binary,2532064}, {code,14613068}, {ets,8089304}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,5383430144}, {buffered_memory,344526848}, {free_memory,22512697344}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2761430,5000}}, {context_switches,{26992214,0}}, {garbage_collection,{2401627,25635209271,0}}, {io,{{input,35861693440},{output,7582886987}}}, {reductions,{9598654747,2696425}}, {run_queue,0}, {runtime,{950310,810}}]}, {system_stats, [{cpu_utilization_rate,18.0}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,4682794413}, {couch_docs_data_size,4439151068}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1855546}, {curr_items_tot,1855546}, {mem_used,3775693352}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2761}, {memory_data,{33657663488,11145101312,{<0.19055.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 21985180 kB\nBuffers: 336452 kB\nCached: 5257256 kB\nSwapCached: 12 kB\nActive: 5842928 kB\nInactive: 4747116 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 21985180 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 2068 kB\nWriteback: 0 kB\nAnonPages: 4995748 kB\nMapped: 18216 kB\nSlab: 245268 kB\nPageTables: 14676 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6717532 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-02T19:09:14.807,ns_1@10.2.1.68:<0.19063.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,31,48}}) Stats for bucket "default": accepting_conns 1 auth_cmds 1 auth_errors 0 bucket_active_conns 1 bucket_conns 6 bytes 3842442840 bytes_read 4513316718 bytes_written 88663261 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 1880898 conn_yields 530 connection_structures 5000 curr_connections 10 curr_conns_on_port_11209 6 curr_conns_on_port_11210 2 curr_items 1880898 curr_items_tot 1880898 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-03 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 1162 ep_commit_time 2 ep_commit_time_total 19965 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 1 ep_data_age_highwat 130 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data/default ep_degraded_mode 0 ep_diskqueue_drain 1879984 ep_diskqueue_fill 1880898 ep_diskqueue_items 915 ep_diskqueue_memory 29248 ep_diskqueue_pending 43872 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 670 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 564 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 1879975 ep_io_read_bytes 0 ep_io_write_bytes 4450885851 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 1747586 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 3750555526 ep_max_checkpoints 2 ep_max_data_size 5242880000 ep_max_item_size 20971520 ep_max_size 5242880000 ep_max_txn_size 10000 ep_max_vbuckets 512 ep_mem_high_wat 3932160000 ep_mem_high_wat 3932160000 ep_mem_low_wat 3145728000 ep_mem_low_wat 3145728000 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 61744 ep_num_expiry_pager_runs 0 ep_num_non_resident 362829 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 4 ep_num_value_ejects 362829 ep_oom_errors 0 ep_overhead 24681768 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 351 ep_startup_time 1351907851 ep_storage_age 0 ep_storage_age_highwat 128 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 3727984750 ep_total_del_items 0 ep_total_enqueued 1880898 ep_total_new_items 1879975 ep_total_persisted 1879975 ep_uncommitted_items 577 ep_value_size 3595233886 ep_vb0 0 ep_vb_snapshot_total 21 ep_vb_total 128 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_130_g8b09894 ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 555 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 3842442840 pid 17027 pointer_size 64 rejected_conns 0 rusage_system 188.542337 rusage_user 541.447687 tap_checkpoint_start_received 84 tap_mutation_received 1338733 tap_opaque_received 294 tap_vbucket_set_received 84 threads 4 time 1351908554 total_connections 189 uptime 2585 vb_active_curr_items 1880898 vb_active_eject 362829 vb_active_expired 0 vb_active_ht_memory 12645376 vb_active_itm_memory 3727984750 vb_active_meta_data_memory 148590942 vb_active_num 128 vb_active_num_non_resident 362829 vb_active_num_ref_ejects 362829 vb_active_num_ref_items 1880898 vb_active_ops_create 1879983 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 80 vb_active_queue_age 0 vb_active_queue_drain 1879984 vb_active_queue_fill 1880898 vb_active_queue_memory 29248 vb_active_queue_pending 43872 vb_active_queue_size 914 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_meta_data_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-02T19:10:11.301,ns_1@10.2.1.68:ns_doctor<0.1479.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,908606,314440}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,118317120}, {processes,70296560}, {processes_used,68718544}, {system,48020560}, {atom,1502673}, {atom_used,1494429}, {binary,4762608}, {code,15256817}, {ets,8392288}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,5796257792}, {buffered_memory,249188352}, {free_memory,22155702272}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2822192,4999}}, {context_switches,{27402093,0}}, {garbage_collection,{2326787,27191124368,0}}, {io,{{input,37603366536},{output,7687834878}}}, {reductions,{10285536041,2700432}}, {run_queue,0}, {runtime,{1046690,620}}]}, {system_stats, [{cpu_utilization_rate,20.25}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,5121416264}, {couch_docs_data_size,4859930311}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2020103}, {curr_items_tot,2020103}, {mem_used,3647247928}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2822}, {memory_data,{33657663488,11451678720,{<18077.178.1>,7628384}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 21636428 kB\nBuffers: 243348 kB\nCached: 5660408 kB\nSwapCached: 0 kB\nActive: 5847400 kB\nInactive: 5096740 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 21636428 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 7472 kB\nWriteback: 0 kB\nAnonPages: 5040172 kB\nMapped: 16312 kB\nSlab: 242440 kB\nPageTables: 14340 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6776256 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,908609,5902}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,102400632}, {processes,57784824}, {processes_used,57398168}, {system,44615808}, {atom,1413905}, {atom_used,1392509}, {binary,2405072}, {code,14575828}, {ets,8155824}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,7202086912}, {buffered_memory,283316224}, {free_memory,20830134272}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2824440,5000}}, {context_switches,{28731324,0}}, {garbage_collection,{2396914,26862861121,0}}, {io,{{input,37074072290},{output,8095825296}}}, {reductions,{10137472725,2848383}}, {run_queue,0}, {runtime,{963030,720}}]}, {system_stats, [{cpu_utilization_rate,7.4812967581047385}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,6576685857}, {couch_docs_data_size,4944802786}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2029500}, {curr_items_tot,2029500}, {mem_used,3660549160}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2824}, {memory_data,{33657663488,12787023872,{<18118.695.0>,3143528}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 20341928 kB\nBuffers: 276676 kB\nCached: 7033288 kB\nSwapCached: 140 kB\nActive: 7408992 kB\nInactive: 4795528 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 20341928 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 2228 kB\nWriteback: 0 kB\nAnonPages: 4893824 kB\nMapped: 18020 kB\nSlab: 274564 kB\nPageTables: 14512 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6619548 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,908611,98264}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,103237368}, {processes,62275328}, {processes_used,62119896}, {system,40962040}, {atom,1414713}, {atom_used,1394174}, {binary,3063440}, {code,14610103}, {ets,3811664}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,7798198272}, {buffered_memory,446377984}, {free_memory,20540047360}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2827524,4998}}, {context_switches,{23415092,0}}, {garbage_collection,{3864325,16298006911,0}}, {io,{{input,25875071564},{output,2469685717}}}, {reductions,{6789604294,2747808}}, {run_queue,0}, {runtime,{752550,740}}]}, {system_stats, [{cpu_utilization_rate,16.209476309226932}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7090983290}, {couch_docs_data_size,4960041118}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2040399}, {curr_items_tot,2040399}, {mem_used,3683062584}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2827}, {memory_data,{33657663488,13009813504,{<18117.703.0>,3143528}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 20058640 kB\nBuffers: 435916 kB\nCached: 7615428 kB\nSwapCached: 0 kB\nActive: 7846916 kB\nInactive: 4620324 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 20058640 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 1292 kB\nWriteback: 48 kB\nAnonPages: 4415940 kB\nMapped: 17964 kB\nSlab: 298104 kB\nPageTables: 13532 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 6129804 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,908606,302511}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,105941352}, {processes,60910264}, {processes_used,60661368}, {system,45031088}, {atom,1413905}, {atom_used,1392509}, {binary,2639344}, {code,14613068}, {ets,8272352}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,6107430912}, {buffered_memory,345665536}, {free_memory,21769760768}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2821430,5000}}, {context_switches,{27275515,0}}, {garbage_collection,{2430329,25699633376,0}}, {io,{{input,35863649632},{output,7585049317}}}, {reductions,{9629627879,2616678}}, {run_queue,0}, {runtime,{959730,690}}]}, {system_stats, [{cpu_utilization_rate,26.93266832917706}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,5401539990}, {couch_docs_data_size,4857560003}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2024501}, {curr_items_tot,2024501}, {mem_used,3726848312}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2821}, {memory_data,{33657663488,11882070016,{<0.19054.0>,2914384}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 21259532 kB\nBuffers: 337564 kB\nCached: 5964288 kB\nSwapCached: 12 kB\nActive: 6225584 kB\nInactive: 5072084 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 21259532 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 2024 kB\nWriteback: 0 kB\nAnonPages: 4996064 kB\nMapped: 18216 kB\nSlab: 262560 kB\nPageTables: 14680 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6718256 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-02T19:10:54.805,ns_1@10.2.1.68:<0.19063.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,31,48}}) Stats for bucket "default": accepting_conns 1 auth_cmds 1 auth_errors 0 bucket_active_conns 1 bucket_conns 6 bytes 3594108440 bytes_read 5189251951 bytes_written 101928212 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 2162464 conn_yields 638 connection_structures 5000 curr_connections 10 curr_conns_on_port_11209 6 curr_conns_on_port_11210 2 curr_items 2162464 curr_items_tot 2162464 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-03 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 1253 ep_commit_time 3 ep_commit_time_total 21027 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 5 ep_data_age_highwat 130 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data/default ep_degraded_mode 0 ep_diskqueue_drain 2146933 ep_diskqueue_fill 2162464 ep_diskqueue_items 15532 ep_diskqueue_memory 496992 ep_diskqueue_pending 745488 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 769 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 14303 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 2146760 ep_io_read_bytes 0 ep_io_write_bytes 5082535221 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 2115338 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 3520273760 ep_max_checkpoints 2 ep_max_data_size 5242880000 ep_max_item_size 20971520 ep_max_size 5242880000 ep_max_txn_size 10000 ep_max_vbuckets 512 ep_mem_high_wat 3932160000 ep_mem_high_wat 3932160000 ep_mem_low_wat 3145728000 ep_mem_low_wat 3145728000 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 185568 ep_num_expiry_pager_runs 0 ep_num_non_resident 754687 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 8 ep_num_value_ejects 754687 ep_oom_errors 0 ep_overhead 17272784 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 1229 ep_startup_time 1351907851 ep_storage_age 3 ep_storage_age_highwat 128 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 3494324192 ep_total_del_items 0 ep_total_enqueued 2162464 ep_total_new_items 2146760 ep_total_persisted 2146760 ep_uncommitted_items 5391 ep_value_size 3341519397 ep_vb0 0 ep_vb_snapshot_total 25 ep_vb_total 128 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_130_g8b09894 ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 555 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 3594108440 pid 17027 pointer_size 64 rejected_conns 0 rusage_system 199.082734 rusage_user 572.492967 tap_checkpoint_start_received 84 tap_mutation_received 1338733 tap_opaque_received 294 tap_vbucket_set_received 84 threads 4 time 1351908654 total_connections 189 uptime 2685 vb_active_curr_items 2162464 vb_active_eject 754687 vb_active_expired 0 vb_active_ht_memory 12645376 vb_active_itm_memory 3494324192 vb_active_meta_data_memory 170834656 vb_active_num 128 vb_active_num_non_resident 754687 vb_active_num_ref_ejects 754687 vb_active_num_ref_items 2162464 vb_active_ops_create 2146932 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 65 vb_active_queue_age 56474000 vb_active_queue_drain 2146933 vb_active_queue_fill 2162464 vb_active_queue_memory 496992 vb_active_queue_pending 745488 vb_active_queue_size 15531 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_meta_data_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-02T19:11:11.301,ns_1@10.2.1.68:ns_doctor<0.1479.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,908666,315279}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,118597608}, {processes,70271624}, {processes_used,68694144}, {system,48325984}, {atom,1502673}, {atom_used,1494429}, {binary,4893360}, {code,15256817}, {ets,8569488}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,6478667776}, {buffered_memory,250044416}, {free_memory,21454671872}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2882193,5000}}, {context_switches,{27757290,0}}, {garbage_collection,{2362113,27297382662,0}}, {io,{{input,37624119266},{output,7694356523}}}, {reductions,{10325402923,3487629}}, {run_queue,0}, {runtime,{1059010,980}}]}, {system_stats, [{cpu_utilization_rate,7.017543859649122}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,5826161261}, {couch_docs_data_size,5278467358}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2188629}, {curr_items_tot,2188629}, {mem_used,3600011816}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2882}, {memory_data,{33657663488,12201975808,{<18077.178.1>,7628384}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 20951828 kB\nBuffers: 244184 kB\nCached: 6326824 kB\nSwapCached: 0 kB\nActive: 6231504 kB\nInactive: 5379776 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 20951828 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 3996 kB\nWriteback: 0 kB\nAnonPages: 5040100 kB\nMapped: 16312 kB\nSlab: 259008 kB\nPageTables: 14324 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6778784 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,908669,7083}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,103455160}, {processes,58516304}, {processes_used,58129488}, {system,44938856}, {atom,1413905}, {atom_used,1392509}, {binary,2551680}, {code,14575828}, {ets,8333080}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,7978807296}, {buffered_memory,284094464}, {free_memory,20034248704}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2884441,5001}}, {context_switches,{29052849,0}}, {garbage_collection,{2429515,26931325466,0}}, {io,{{input,37079562418},{output,8101389233}}}, {reductions,{10169745942,2697480}}, {run_queue,0}, {runtime,{972730,810}}]}, {system_stats, [{cpu_utilization_rate,6.78391959798995}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,7351584378}, {couch_docs_data_size,5359283751}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2197370}, {curr_items_tot,2197370}, {mem_used,3616077656}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2884}, {memory_data,{33657663488,13572243456,{<18118.695.0>,3143528}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 19564696 kB\nBuffers: 277436 kB\nCached: 7791804 kB\nSwapCached: 140 kB\nActive: 7857636 kB\nInactive: 5105060 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 19564696 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 1068 kB\nWriteback: 72 kB\nAnonPages: 4893036 kB\nMapped: 18020 kB\nSlab: 293532 kB\nPageTables: 14504 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6622600 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,908671,102339}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.30462.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,30}, {started_on,1351908622}, {total_vbuckets,129}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351908678}, {vbuckets_done,39}]]}, {memory, [{total,102983272}, {processes,62148320}, {processes_used,62006096}, {system,40834952}, {atom,1414713}, {atom_used,1394174}, {binary,2751080}, {code,14610103}, {ets,3992512}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,7309975552}, {buffered_memory,448028672}, {free_memory,21003620352}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2887528,5003}}, {context_switches,{23508343,0}}, {garbage_collection,{3872948,16346556541,0}}, {io,{{input,25881556753},{output,2471925087}}}, {reductions,{6813971476,1854303}}, {run_queue,0}, {runtime,{758090,450}}]}, {system_stats, [{cpu_utilization_rate,7.0},{swap_total,10701078528},{swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,6718368771}, {couch_docs_data_size,5090794734}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2208968}, {curr_items_tot,2208968}, {mem_used,3679390936}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2887}, {memory_data,{33657663488,12760322048,{<18117.19799.0>,4714984}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 20508248 kB\nBuffers: 437528 kB\nCached: 7141600 kB\nSwapCached: 0 kB\nActive: 7085324 kB\nInactive: 4939704 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 20508248 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 2796 kB\nWriteback: 0 kB\nAnonPages: 4445716 kB\nMapped: 17968 kB\nSlab: 289728 kB\nPageTables: 13664 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 6160868 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,908666,303011}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,108156440}, {processes,63111968}, {processes_used,62863664}, {system,45044472}, {atom,1413905}, {atom_used,1392509}, {binary,2588880}, {code,14613068}, {ets,8335408}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,6791499776}, {buffered_memory,346320896}, {free_memory,21060341760}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2881431,5000}}, {context_switches,{27437829,0}}, {garbage_collection,{2446456,25753399923,0}}, {io,{{input,35864924042},{output,7586788803}}}, {reductions,{9656052685,2202859}}, {run_queue,0}, {runtime,{966460,510}}]}, {system_stats, [{cpu_utilization_rate,25.06265664160401}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,6080812811}, {couch_docs_data_size,5271780957}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2193500}, {curr_items_tot,2193500}, {mem_used,3675564616}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2881}, {memory_data,{33657663488,12554403840,{<0.19055.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 20566864 kB\nBuffers: 338204 kB\nCached: 6632324 kB\nSwapCached: 12 kB\nActive: 6537424 kB\nInactive: 5436888 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 20566864 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 1752 kB\nWriteback: 0 kB\nAnonPages: 5003572 kB\nMapped: 18216 kB\nSlab: 279164 kB\nPageTables: 14688 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6725320 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-02T19:12:11.301,ns_1@10.2.1.68:ns_doctor<0.1479.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,908726,314816}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,120195472}, {processes,71600088}, {processes_used,70022608}, {system,48595384}, {atom,1502673}, {atom_used,1494429}, {binary,5099800}, {code,15256817}, {ets,8633256}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,7251193856}, {buffered_memory,251162624}, {free_memory,20658528256}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2942193,5001}}, {context_switches,{28157961,0}}, {garbage_collection,{2402179,27406487519,0}}, {io,{{input,37641516517},{output,7697507824}}}, {reductions,{10366947290,3329247}}, {run_queue,0}, {runtime,{1071890,840}}]}, {system_stats, [{cpu_utilization_rate,7.017543859649122}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,6560118135}, {couch_docs_data_size,5672227028}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2357564}, {curr_items_tot,2357564}, {mem_used,3561537224}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2942}, {memory_data,{33657663488,12996341760,{<18077.178.1>,7628384}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 20174344 kB\nBuffers: 245276 kB\nCached: 7081244 kB\nSwapCached: 0 kB\nActive: 6696968 kB\nInactive: 5673708 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 20174344 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 3832 kB\nWriteback: 0 kB\nAnonPages: 5044096 kB\nMapped: 16312 kB\nSlab: 277880 kB\nPageTables: 14304 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6789448 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,908729,6991}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,107599992}, {processes,62595200}, {processes_used,62208384}, {system,45004792}, {atom,1413905}, {atom_used,1392509}, {binary,2556632}, {code,14575828}, {ets,8394848}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,8743354368}, {buffered_memory,285175808}, {free_memory,19246993408}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2944440,5000}}, {context_switches,{29316439,0}}, {garbage_collection,{2455978,26993677640,0}}, {io,{{input,37081316980},{output,8103521985}}}, {reductions,{10199726970,2651582}}, {run_queue,0}, {runtime,{980980,640}}]}, {system_stats, [{cpu_utilization_rate,7.0}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,8117766719}, {couch_docs_data_size,5774825519}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2365621}, {curr_items_tot,2365621}, {mem_used,3582507304}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2944}, {memory_data,{33657663488,14360768512,{<18118.20373.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 18795892 kB\nBuffers: 278492 kB\nCached: 8538432 kB\nSwapCached: 140 kB\nActive: 8284696 kB\nInactive: 5427356 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 18795892 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 1640 kB\nWriteback: 0 kB\nAnonPages: 4894792 kB\nMapped: 18020 kB\nSlab: 312720 kB\nPageTables: 14508 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6625396 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,908731,99913}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.30462.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,67}, {started_on,1351908622}, {total_vbuckets,129}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351908738}, {vbuckets_done,87}]]}, {memory, [{total,102127440}, {processes,61256464}, {processes_used,61151328}, {system,40870976}, {atom,1414713}, {atom_used,1394174}, {binary,2563720}, {code,14610103}, {ets,4211432}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,6626123776}, {buffered_memory,450293760}, {free_memory,21656911872}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2947525,5000}}, {context_switches,{23573696,0}}, {garbage_collection,{3879956,16393341979,0}}, {io,{{input,25894734346},{output,2478020498}}}, {reductions,{6837795407,1892071}}, {run_queue,0}, {runtime,{763180,370}}]}, {system_stats, [{cpu_utilization_rate,7.75}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,6044713747}, {couch_docs_data_size,5188163764}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2377346}, {curr_items_tot,2377346}, {mem_used,3665823448}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2947}, {memory_data,{33657663488,12111220736,{<18117.19799.0>,4714984}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 21149328 kB\nBuffers: 439740 kB\nCached: 6470824 kB\nSwapCached: 0 kB\nActive: 6047380 kB\nInactive: 5353332 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 21149328 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 12984 kB\nWriteback: 3604 kB\nAnonPages: 4490312 kB\nMapped: 17952 kB\nSlab: 273192 kB\nPageTables: 13752 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 6203920 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,908726,303052}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,106430144}, {processes,61122744}, {processes_used,60873848}, {system,45307400}, {atom,1413905}, {atom_used,1392509}, {binary,2677232}, {code,14613068}, {ets,8512256}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,7481520128}, {buffered_memory,347095040}, {free_memory,20353593344}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2941430,5000}}, {context_switches,{27615651,0}}, {garbage_collection,{2464434,25808942905,0}}, {io,{{input,35869744776},{output,7591819743}}}, {reductions,{9683073959,2732810}}, {run_queue,0}, {runtime,{973140,750}}]}, {system_stats, [{cpu_utilization_rate,7.980049875311721}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,6784397181}, {couch_docs_data_size,5698136145}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2362214}, {curr_items_tot,2362214}, {mem_used,3639219432}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2941}, {memory_data,{33657663488,13303181312,{<0.19054.0>,2914384}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 19876556 kB\nBuffers: 338960 kB\nCached: 7306172 kB\nSwapCached: 12 kB\nActive: 6875824 kB\nInactive: 5771240 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 19876556 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 2020 kB\nWriteback: 0 kB\nAnonPages: 5001856 kB\nMapped: 18216 kB\nSlab: 296088 kB\nPageTables: 14680 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6725824 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-02T19:12:34.807,ns_1@10.2.1.68:<0.19063.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,31,48}}) Stats for bucket "default": accepting_conns 1 auth_cmds 1 auth_errors 0 bucket_active_conns 1 bucket_conns 6 bytes 3853306104 bytes_read 5863806144 bytes_written 115178440 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 2443389 conn_yields 706 connection_structures 5000 curr_connections 10 curr_conns_on_port_11209 6 curr_conns_on_port_11210 2 curr_items 2443389 curr_items_tot 2443389 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-03 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 1331 ep_commit_time 3 ep_commit_time_total 21887 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 1 ep_data_age_highwat 130 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data/default ep_degraded_mode 0 ep_diskqueue_drain 2442245 ep_diskqueue_fill 2443389 ep_diskqueue_items 1145 ep_diskqueue_memory 36608 ep_diskqueue_pending 54912 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 869 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 571 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 2442236 ep_io_read_bytes 0 ep_io_write_bytes 5782746789 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 2298309 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 3762153495 ep_max_checkpoints 2 ep_max_data_size 5242880000 ep_max_item_size 20971520 ep_max_size 5242880000 ep_max_txn_size 10000 ep_max_vbuckets 512 ep_mem_high_wat 3932160000 ep_mem_high_wat 3932160000 ep_mem_low_wat 3145728000 ep_mem_low_wat 3145728000 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 284969 ep_num_expiry_pager_runs 0 ep_num_non_resident 944413 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 10 ep_num_value_ejects 944413 ep_oom_errors 0 ep_overhead 25805408 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 574 ep_startup_time 1351907851 ep_storage_age 0 ep_storage_age_highwat 128 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 3732832827 ep_total_del_items 0 ep_total_enqueued 2443389 ep_total_new_items 2442236 ep_total_persisted 2442236 ep_uncommitted_items 361 ep_value_size 3560129514 ep_vb0 0 ep_vb_snapshot_total 26 ep_vb_total 128 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_130_g8b09894 ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 555 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 3853306104 pid 17027 pointer_size 64 rejected_conns 0 rusage_system 209.695121 rusage_user 603.607237 tap_checkpoint_start_received 84 tap_mutation_received 1338733 tap_opaque_received 294 tap_vbucket_set_received 84 threads 4 time 1351908754 total_connections 189 uptime 2785 vb_active_curr_items 2443389 vb_active_eject 944413 vb_active_expired 0 vb_active_ht_memory 12645376 vb_active_itm_memory 3732832827 vb_active_meta_data_memory 193027731 vb_active_num 128 vb_active_num_non_resident 944413 vb_active_num_ref_ejects 944413 vb_active_num_ref_items 2443389 vb_active_ops_create 2442244 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 61 vb_active_queue_age 98000 vb_active_queue_drain 2442245 vb_active_queue_fill 2443389 vb_active_queue_memory 36608 vb_active_queue_pending 54912 vb_active_queue_size 1144 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_meta_data_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-02T19:13:11.301,ns_1@10.2.1.68:ns_doctor<0.1479.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,908786,315642}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,126134536}, {processes,77125672}, {processes_used,75551464}, {system,49008864}, {atom,1502673}, {atom_used,1494429}, {binary,5253880}, {code,15256817}, {ets,8808104}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,7909134336}, {buffered_memory,251846656}, {free_memory,19990253568}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{3002195,5002}}, {context_switches,{28435573,0}}, {garbage_collection,{2429520,27508097204,0}}, {io,{{input,37659335788},{output,7700548663}}}, {reductions,{10406324817,2623438}}, {run_queue,0}, {runtime,{1082430,600}}]}, {system_stats, [{cpu_utilization_rate,3.25}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,7281055106}, {couch_docs_data_size,6068913143}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2497553}, {curr_items_tot,2497553}, {mem_used,3930588616}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3002}, {memory_data,{33657663488,13674139648,{<18077.8103.1>,7628384}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 19522352 kB\nBuffers: 245944 kB\nCached: 7723764 kB\nSwapCached: 0 kB\nActive: 7060812 kB\nInactive: 5946076 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 19522352 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 3688 kB\nWriteback: 0 kB\nAnonPages: 5037116 kB\nMapped: 16312 kB\nSlab: 293788 kB\nPageTables: 14300 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6774276 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,908789,6456}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.30141.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,19}, {started_on,1351908719}, {total_vbuckets,129}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351908761}, {vbuckets_done,25}]]}, {memory, [{total,105454488}, {processes,60360504}, {processes_used,59988200}, {system,45093984}, {atom,1413905}, {atom_used,1392509}, {binary,2457424}, {code,14575828}, {ets,8579576}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,8664031232}, {buffered_memory,286715904}, {free_memory,19322544128}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{3004439,4999}}, {context_switches,{29452967,0}}, {garbage_collection,{2469992,27047005452,0}}, {io,{{input,37093717356},{output,8106823556}}}, {reductions,{10226051423,1910354}}, {run_queue,0}, {runtime,{987070,400}}]}, {system_stats, [{cpu_utilization_rate,8.728179551122194}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,8056849664}, {couch_docs_data_size,5970666189}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2497102}, {curr_items_tot,2497102}, {mem_used,3931514824}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3004}, {memory_data,{33657663488,14401527808,{<18118.20373.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 18869672 kB\nBuffers: 279996 kB\nCached: 8460968 kB\nSwapCached: 140 kB\nActive: 7907064 kB\nInactive: 5729148 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 18869672 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 49540 kB\nWriteback: 0 kB\nAnonPages: 4894660 kB\nMapped: 18024 kB\nSlab: 313544 kB\nPageTables: 14592 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6624824 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,908791,100468}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,101155632}, {processes,60284144}, {processes_used,60128552}, {system,40871488}, {atom,1414713}, {atom_used,1394174}, {binary,2443032}, {code,14610103}, {ets,4338712}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,6551425024}, {buffered_memory,452739072}, {free_memory,21729169408}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{3007525,5000}}, {context_switches,{23635022,0}}, {garbage_collection,{3886187,16438565414,0}}, {io,{{input,25895952646},{output,2479521912}}}, {reductions,{6860615449,1985739}}, {run_queue,0}, {runtime,{767930,490}}]}, {system_stats, [{cpu_utilization_rate,6.467661691542289}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,5808215475}, {couch_docs_data_size,5717570991}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2503544}, {curr_items_tot,2503544}, {mem_used,3600165864}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3007}, {memory_data,{33657663488,11790852096,{<18117.19799.0>,4714984}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 21219892 kB\nBuffers: 442128 kB\nCached: 6397876 kB\nSwapCached: 0 kB\nActive: 5184196 kB\nInactive: 6149440 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 21219892 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 5808 kB\nWriteback: 0 kB\nAnonPages: 4493400 kB\nMapped: 17968 kB\nSlab: 269448 kB\nPageTables: 13680 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 6204208 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,908786,302982}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,107770152}, {processes,62382536}, {processes_used,62133640}, {system,45387616}, {atom,1413905}, {atom_used,1392509}, {binary,2694208}, {code,14613068}, {ets,8575032}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,8099127296}, {buffered_memory,347865088}, {free_memory,19720110080}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{3001430,5000}}, {context_switches,{27789323,0}}, {garbage_collection,{2481775,25863881033,0}}, {io,{{input,35871047256},{output,7593491181}}}, {reductions,{9709817301,1855833}}, {run_queue,0}, {runtime,{979940,490}}]}, {system_stats, [{cpu_utilization_rate,4.010025062656641}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,7480853905}, {couch_docs_data_size,6098523648}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2501801}, {curr_items_tot,2501801}, {mem_used,3522567176}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3001}, {memory_data,{33657663488,13937172480,{<0.19054.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 19257920 kB\nBuffers: 339712 kB\nCached: 7909304 kB\nSwapCached: 12 kB\nActive: 7182720 kB\nInactive: 6068156 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 19257920 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 776 kB\nWriteback: 48 kB\nAnonPages: 5001320 kB\nMapped: 18216 kB\nSlab: 311056 kB\nPageTables: 14692 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6728140 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-02T19:14:11.301,ns_1@10.2.1.68:ns_doctor<0.1479.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,908846,313264}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,121722632}, {processes,72301320}, {processes_used,70723840}, {system,49421312}, {atom,1502673}, {atom_used,1494429}, {binary,5574896}, {code,15256817}, {ets,8985560}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,7904366592}, {buffered_memory,252104704}, {free_memory,19993948160}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{3062193,5000}}, {context_switches,{28493641,0}}, {garbage_collection,{2434938,27599854404,0}}, {io,{{input,37681328957},{output,7710299201}}}, {reductions,{10444772034,3356999}}, {run_queue,0}, {runtime,{1090740,720}}]}, {system_stats, [{cpu_utilization_rate,3.2663316582914574}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,7281055082}, {couch_docs_data_size,6068913143}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2497553}, {curr_items_tot,2497553}, {mem_used,3930588600}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3062}, {memory_data,{33657663488,13662699520,{<18077.8103.1>,7628384}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 19525340 kB\nBuffers: 246196 kB\nCached: 7719108 kB\nSwapCached: 0 kB\nActive: 7061292 kB\nInactive: 5942520 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 19525340 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 372 kB\nWriteback: 0 kB\nAnonPages: 5038408 kB\nMapped: 16312 kB\nSlab: 293284 kB\nPageTables: 14356 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6781744 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,908849,8837}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.30141.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,47}, {started_on,1351908719}, {total_vbuckets,129}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351908819}, {vbuckets_done,61}]]}, {memory, [{total,104698416}, {processes,59452088}, {processes_used,59112904}, {system,45246328}, {atom,1413905}, {atom_used,1392509}, {binary,2391976}, {code,14575828}, {ets,8792816}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,8036331520}, {buffered_memory,288878592}, {free_memory,19962097664}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{3064440,5000}}, {context_switches,{29508021,0}}, {garbage_collection,{2475770,27093674615,0}}, {io,{{input,37098272370},{output,8115125093}}}, {reductions,{10248968057,1869137}}, {run_queue,0}, {runtime,{992240,430}}]}, {system_stats, [{cpu_utilization_rate,3.7406483790523692}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,7415967096}, {couch_docs_data_size,6031774261}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2497102}, {curr_items_tot,2497102}, {mem_used,3931271784}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3064}, {memory_data,{33657663488,13730230272,{<18118.20373.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 19494236 kB\nBuffers: 282108 kB\nCached: 7847980 kB\nSwapCached: 140 kB\nActive: 7046844 kB\nInactive: 5979524 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 19494236 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 7004 kB\nWriteback: 28 kB\nAnonPages: 4896160 kB\nMapped: 18020 kB\nSlab: 298716 kB\nPageTables: 14512 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6624684 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,908851,100829}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,100274640}, {processes,59436296}, {processes_used,59280704}, {system,40838344}, {atom,1414713}, {atom_used,1394174}, {binary,2347728}, {code,14610103}, {ets,4400448}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,6897516544}, {buffered_memory,453263360}, {free_memory,21376303104}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{3067525,5000}}, {context_switches,{23688367,0}}, {garbage_collection,{3890867,16483619398,0}}, {io,{{input,25896712529},{output,2480808495}}}, {reductions,{6883007320,1839038}}, {run_queue,0}, {runtime,{773270,490}}]}, {system_stats, [{cpu_utilization_rate,3.0},{swap_total,10701078528},{swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,6236677511}, {couch_docs_data_size,6073140266}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2503544}, {curr_items_tot,2503544}, {mem_used,3545759256}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3067}, {memory_data,{33657663488,12281233408,{<18117.703.0>,3143528}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 20875296 kB\nBuffers: 442640 kB\nCached: 6735856 kB\nSwapCached: 0 kB\nActive: 5258536 kB\nInactive: 6412880 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 20875296 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 292 kB\nWriteback: 0 kB\nAnonPages: 4492888 kB\nMapped: 17968 kB\nSlab: 276340 kB\nPageTables: 13684 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 6203916 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,908846,302401}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,106357240}, {processes,60973192}, {processes_used,60724296}, {system,45384048}, {atom,1413905}, {atom_used,1392509}, {binary,2516296}, {code,14613068}, {ets,8749488}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,8100540416}, {buffered_memory,347971584}, {free_memory,19717701632}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{3061430,5000}}, {context_switches,{27836269,0}}, {garbage_collection,{2486035,25908480612,0}}, {io,{{input,35871754994},{output,7594815579}}}, {reductions,{9732040499,1868240}}, {run_queue,0}, {runtime,{985600,480}}]}, {system_stats, [{cpu_utilization_rate,4.239401496259352}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,7480853871}, {couch_docs_data_size,6098523648}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2501801}, {curr_items_tot,2501801}, {mem_used,3522567160}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3061}, {memory_data,{33657663488,13939707904,{<0.19054.0>,2914384}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 19255568 kB\nBuffers: 339816 kB\nCached: 7910684 kB\nSwapCached: 12 kB\nActive: 7185068 kB\nInactive: 6069176 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 19255568 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 116 kB\nWriteback: 0 kB\nAnonPages: 5003720 kB\nMapped: 18216 kB\nSlab: 310796 kB\nPageTables: 14692 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6724872 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-02T19:14:14.806,ns_1@10.2.1.68:<0.19063.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,31,48}}) Stats for bucket "default": accepting_conns 1 auth_cmds 1 auth_errors 0 bucket_active_conns 1 bucket_conns 5 bytes 3522567160 bytes_read 6003841217 bytes_written 123107857 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 2501801 conn_yields 726 connection_structures 5000 curr_connections 9 curr_conns_on_port_11209 6 curr_conns_on_port_11210 1 curr_items 2501801 curr_items_tot 2501801 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-03 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 1345 ep_commit_time 3 ep_commit_time_total 22175 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 2 ep_data_age_highwat 130 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data/default ep_degraded_mode 0 ep_diskqueue_drain 2501801 ep_diskqueue_fill 2501801 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 890 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 2501801 ep_io_read_bytes 0 ep_io_write_bytes 5923674684 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 2481131 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 3457043179 ep_max_checkpoints 2 ep_max_data_size 5242880000 ep_max_item_size 20971520 ep_max_size 5242880000 ep_max_txn_size 10000 ep_max_vbuckets 512 ep_mem_high_wat 3932160000 ep_mem_high_wat 3932160000 ep_mem_low_wat 3145728000 ep_mem_low_wat 3145728000 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 407049 ep_num_expiry_pager_runs 0 ep_num_non_resident 1135077 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 12 ep_num_value_ejects 1135077 ep_oom_errors 0 ep_overhead 14875216 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1351907851 ep_storage_age 1 ep_storage_age_highwat 128 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 3427021567 ep_total_del_items 0 ep_total_enqueued 2501801 ep_total_new_items 2501801 ep_total_persisted 2501801 ep_uncommitted_items 0 ep_value_size 3250100059 ep_vb0 0 ep_vb_snapshot_total 27 ep_vb_total 128 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_130_g8b09894 ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 555 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 3522567160 pid 17027 pointer_size 64 rejected_conns 0 rusage_system 211.834796 rusage_user 610.833139 tap_checkpoint_start_received 84 tap_mutation_received 1338733 tap_opaque_received 294 tap_vbucket_set_received 84 threads 4 time 1351908854 total_connections 189 uptime 2885 vb_active_curr_items 2501801 vb_active_eject 1135077 vb_active_expired 0 vb_active_ht_memory 12645376 vb_active_itm_memory 3427021567 vb_active_meta_data_memory 197642279 vb_active_num 128 vb_active_num_non_resident 1135077 vb_active_num_ref_ejects 1135077 vb_active_num_ref_items 2501801 vb_active_ops_create 2501801 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 54 vb_active_queue_age 0 vb_active_queue_drain 2501801 vb_active_queue_fill 2501801 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_meta_data_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-02T19:15:11.301,ns_1@10.2.1.68:ns_doctor<0.1479.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,908906,312920}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,126808872}, {processes,76635160}, {processes_used,75102648}, {system,50173712}, {atom,1502673}, {atom_used,1494429}, {binary,6135264}, {code,15256817}, {ets,9173632}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,7905681408}, {buffered_memory,252235776}, {free_memory,19989377024}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{3122193,5000}}, {context_switches,{28552915,0}}, {garbage_collection,{2440885,27690981191,0}}, {io,{{input,37698725298},{output,7712752343}}}, {reductions,{10479909222,6982136}}, {run_queue,0}, {runtime,{1098720,950}}]}, {system_stats, [{cpu_utilization_rate,5.513784461152882}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,7281055042}, {couch_docs_data_size,6068913143}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2497553}, {curr_items_tot,2497553}, {mem_used,3930588600}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3122}, {memory_data,{33657663488,13668794368,{<18077.8103.1>,7628384}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 19520876 kB\nBuffers: 246324 kB\nCached: 7720392 kB\nSwapCached: 0 kB\nActive: 7065572 kB\nInactive: 5943572 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 19520876 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 164 kB\nWriteback: 0 kB\nAnonPages: 5042260 kB\nMapped: 16312 kB\nSlab: 293232 kB\nPageTables: 14320 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6779612 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,908909,9646}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.30141.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,69}, {started_on,1351908719}, {total_vbuckets,129}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351908880}, {vbuckets_done,90}], [{pid,<<"<0.31992.0>">>}, {changes_done,0}, {design_documents,[<<"_design/A">>]}, {indexer_type,main}, {initial_build,true}, {progress,0}, {set,<<"default">>}, {signature,<<"afeb9408e40add49ae0c9bf58f0c5f26">>}, {started_on,1351908881}, {total_changes,2497102}, {type,indexer}, {updated_on,1351908881}]]}, {memory, [{total,112790416}, {processes,63858200}, {processes_used,63544520}, {system,48932216}, {atom,1413905}, {atom_used,1392509}, {binary,5913512}, {code,14575828}, {ets,8956896}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,7528931328}, {buffered_memory,290439168}, {free_memory,20471660544}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{3124441,5002}}, {context_switches,{29584186,0}}, {garbage_collection,{2483198,27152851022,0}}, {io,{{input,37142599188},{output,8117525648}}}, {reductions,{10280391562,7049497}}, {run_queue,0}, {runtime,{998170,830}}]}, {system_stats, [{cpu_utilization_rate,12.5}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,6927524208}, {couch_docs_data_size,6074616267},