============================================================================== couchbase logs (stats) cbbrowse_logs stats ============================================================================== [ns_doctor:debug,2012-11-02T18:23:13.092,ns_1@127.0.0.1:ns_doctor<0.374.0>:ns_doctor:handle_info:133]Got initial status [{'ns_1@127.0.0.1', [{last_heard,{1351,905793,92549}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,39998280}, {processes,9231192}, {processes_used,9220424}, {system,30767088}, {atom,1171065}, {atom_used,1138453}, {binary,137736}, {code,11316948}, {ets,875160}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,3175759872}, {buffered_memory,401043456}, {free_memory,29653458944}, {total_memory,33657663488}]}, {node_storage_conf, [{db_path, "/opt/couchbase/var/lib/couchbase/data/"}, {index_path, "/opt/couchbase/var/lib/couchbase/data/"}]}, {statistics, [{wall_clock,{1832,4}}, {context_switches,{25956,0}}, {garbage_collection,{6860,32025282,0}}, {io,{{input,6768908},{output,664133}}}, {reductions,{18788614,18926}}, {run_queue,0}, {runtime,{1060,10}}]}, {system_stats, [{cpu_utilization_rate,0}, {swap_total,0}, {swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,1}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1}, {memory_data, {33657663488,3986419712,{<0.6.0>,601272}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,2}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 28958456 kB\nBuffers: 391644 kB\nCached: 3101328 kB\nSwapCached: 0 kB\nActive: 3093200 kB\nInactive: 523640 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 28958456 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 419620 kB\nWriteback: 0 kB\nAnonPages: 123680 kB\nMapped: 13960 kB\nSlab: 257416 kB\nPageTables: 4668 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 1574376 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-02T18:23:24.418,ns_1@127.0.0.1:ns_doctor<0.743.0>:ns_doctor:handle_info:133]Got initial status [{'ns_1@127.0.0.1', [{last_heard,{1351,905804,418389}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,43475656}, {processes,11008168}, {processes_used,10956712}, {system,32467488}, {atom,1242281}, {atom_used,1240746}, {binary,223944}, {code,12684005}, {ets,1002184}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,644358144}, {buffered_memory,401219584}, {free_memory,32237559808}, {total_memory,33657663488}]}, {node_storage_conf, [{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{13156,2}}, {context_switches,{34357,0}}, {garbage_collection,{9087,35993877,0}}, {io,{{input,7669311},{output,1107290}}}, {reductions,{20260320,39054}}, {run_queue,0}, {runtime,{1850,10}}]}, {system_stats, [{cpu_utilization_rate,0}, {swap_total,0}, {swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,13}, {memory_data, {33657663488,3986419712,{<0.6.0>,601272}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,2}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 31482124 kB\nBuffers: 391816 kB\nCached: 629256 kB\nSwapCached: 0 kB\nActive: 634592 kB\nInactive: 518292 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 31482124 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 420200 kB\nWriteback: 0 kB\nAnonPages: 131608 kB\nMapped: 15868 kB\nSlab: 197764 kB\nPageTables: 4840 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 1605312 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-02T18:24:24.412,ns_1@127.0.0.1:ns_doctor<0.743.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@127.0.0.1', [{last_heard,{1351,905859,411638}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,46581440}, {processes,13981312}, {processes_used,13960696}, {system,32600128}, {atom,1242281}, {atom_used,1240746}, {binary,279032}, {code,12684005}, {ets,1071088}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,644567040}, {buffered_memory,401297408}, {free_memory,32178552832}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{68150,5000}}, {context_switches,{41738,0}}, {garbage_collection,{10358,38546707,0}}, {io,{{input,7722705},{output,1255335}}}, {reductions,{21165228,90075}}, {run_queue,0}, {runtime,{4180,210}}]}, {system_stats, [{cpu_utilization_rate,1.2468827930174564}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,68}, {memory_data,{33657663488,1479237632,{<0.6.0>,601480}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 31424368 kB\nBuffers: 391892 kB\nCached: 629460 kB\nSwapCached: 0 kB\nActive: 695020 kB\nInactive: 518384 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 31424368 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 162536 kB\nWriteback: 0 kB\nAnonPages: 192088 kB\nMapped: 15896 kB\nSlab: 193336 kB\nPageTables: 5152 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 1838960 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-02T18:25:24.410,ns_1@127.0.0.1:ns_doctor<0.743.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@127.0.0.1', [{last_heard,{1351,905919,411640}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,47059216}, {processes,14376656}, {processes_used,14356040}, {system,32682560}, {atom,1243089}, {atom_used,1242343}, {binary,305264}, {code,12708250}, {ets,1100984}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,644657152}, {buffered_memory,401367040}, {free_memory,32177029120}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{128150,5000}}, {context_switches,{49410,0}}, {garbage_collection,{11498,41076939,0}}, {io,{{input,7775255},{output,1347751}}}, {reductions,{22112764,88576}}, {run_queue,0}, {runtime,{6810,230}}]}, {system_stats, [{cpu_utilization_rate,1.4962593516209477}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,128}, {memory_data,{33657663488,1480634368,{<0.6.0>,601480}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 31422880 kB\nBuffers: 391960 kB\nCached: 629548 kB\nSwapCached: 0 kB\nActive: 697192 kB\nInactive: 518448 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 31422880 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 112900 kB\nWriteback: 0 kB\nAnonPages: 194164 kB\nMapped: 15908 kB\nSlab: 193144 kB\nPageTables: 5156 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 1838960 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-02T18:26:10.141,ns_1@10.2.1.67:ns_doctor<0.1359.0>:ns_doctor:handle_info:133]Got initial status [{'ns_1@10.2.1.67', [{last_heard,{1351,905970,141662}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,47883672}, {processes,15163656}, {processes_used,15148160}, {system,32720016}, {atom,1244705}, {atom_used,1244375}, {binary,323840}, {code,12756931}, {ets,1051784}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,644567040}, {buffered_memory,401571840}, {free_memory,32174862336}, {total_memory,33657663488}]}, {node_storage_conf, [{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{178881,52}}, {context_switches,{58514,0}}, {garbage_collection,{13244,44708483,0}}, {io,{{input,8037353},{output,1732835}}}, {reductions,{23357135,294767}}, {run_queue,1}, {runtime,{9110,70}}]}, {system_stats, [{cpu_utilization_rate,0}, {swap_total,0}, {swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,178}, {memory_data, {33657663488,1480634368,{<0.6.0>,601480}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 31420764 kB\nBuffers: 392160 kB\nCached: 629460 kB\nSwapCached: 0 kB\nActive: 699600 kB\nInactive: 518560 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 31420764 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 96372 kB\nWriteback: 0 kB\nAnonPages: 196568 kB\nMapped: 15908 kB\nSlab: 193076 kB\nPageTables: 5156 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 1841584 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-02T18:26:10.661,ns_1@10.2.1.67:ns_doctor<0.1461.0>:ns_doctor:handle_info:133]Got initial status [{'ns_1@10.2.1.65', [{last_heard,{1351,905970,661060}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,59623664}, {processes,24198248}, {processes_used,22070616}, {system,35425416}, {atom,1400977}, {atom_used,1372188}, {binary,551088}, {code,14085234}, {ets,1811504}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,609517568}, {buffered_memory,84148224}, {free_memory,32585138176}, {total_memory,33657663488}]}, {node_storage_conf, [{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{178804,16}}, {context_switches,{280357,0}}, {garbage_collection,{43661,155613008,0}}, {io,{{input,16044435},{output,8150091}}}, {reductions,{81159423,52422}}, {run_queue,0}, {runtime,{14990,10}}]}, {system_stats, [{cpu_utilization_rate,2.756892230576441}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,178}, {memory_data, {33657663488,1077022720,{<18077.169.0>,972344}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 31821424 kB\nBuffers: 82176 kB\nCached: 595232 kB\nSwapCached: 0 kB\nActive: 451224 kB\nInactive: 441548 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 31821424 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 82176 kB\nWriteback: 0 kB\nAnonPages: 214948 kB\nMapped: 15228 kB\nSlab: 117960 kB\nPageTables: 4564 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 1885004 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,905970,661056}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,44853432}, {processes,11990160}, {processes_used,11918520}, {system,32863272}, {atom,1277481}, {atom_used,1244541}, {binary,331336}, {code,12756931}, {ets,1096656}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,644517888}, {buffered_memory,401620992}, {free_memory,32235720704}, {total_memory,33657663488}]}, {node_storage_conf, [{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{179399,1}}, {context_switches,{61170,0}}, {garbage_collection,{13922,46097341,0}}, {io,{{input,8097956},{output,1901592}}}, {reductions,{23775761,39102}}, {run_queue,0}, {runtime,{9210,0}}]}, {system_stats, [{cpu_utilization_rate,0}, {swap_total,0}, {swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,179}, {memory_data, {33657663488,1480634368,{<0.6.0>,601480}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 31480196 kB\nBuffers: 392208 kB\nCached: 629412 kB\nSwapCached: 0 kB\nActive: 640648 kB\nInactive: 518624 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 31480196 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 96372 kB\nWriteback: 0 kB\nAnonPages: 137528 kB\nMapped: 14924 kB\nSlab: 193076 kB\nPageTables: 4828 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 1608824 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-02T18:26:32.541,ns_1@10.2.1.67:<0.1687.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,31,46}}) Stats for bucket "default": accepting_conns 1 auth_cmds 0 auth_errors 0 bucket_active_conns 1 bucket_conns 5 bytes 245008 bytes_read 196 bytes_written 120 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 0 conn_yields 0 connection_structures 5000 curr_connections 9 curr_conns_on_port_11209 6 curr_conns_on_port_11210 1 curr_items 0 curr_items_tot 0 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-03 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 0 ep_commit_time 0 ep_commit_time_total 0 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 0 ep_data_age_highwat 0 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data/default ep_degraded_mode 1 ep_diskqueue_drain 0 ep_diskqueue_fill 0 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 0 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 0 ep_io_read_bytes 0 ep_io_write_bytes 0 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 0 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 0 ep_max_checkpoints 2 ep_max_data_size 5242880000 ep_max_item_size 20971520 ep_max_size 5242880000 ep_max_txn_size 10000 ep_max_vbuckets 512 ep_mem_high_wat 3932160000 ep_mem_high_wat 3932160000 ep_mem_low_wat 3145728000 ep_mem_low_wat 3145728000 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 1152 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1351905991 ep_storage_age 0 ep_storage_age_highwat 0 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 0 ep_total_del_items 0 ep_total_enqueued 0 ep_total_new_items 0 ep_total_persisted 0 ep_uncommitted_items 0 ep_value_size 0 ep_vb0 0 ep_vb_snapshot_total 1 ep_vb_total 0 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_130_g8b09894 ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 725 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 245008 pid 16451 pointer_size 64 rejected_conns 0 rusage_system 0.012998 rusage_user 0.027995 threads 4 time 1351905991 total_connections 10 uptime 23 vb_active_curr_items 0 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 0 vb_active_itm_memory 0 vb_active_meta_data_memory 0 vb_active_num 0 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 0 vb_active_ops_create 0 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 0 vb_active_queue_age 0 vb_active_queue_drain 0 vb_active_queue_fill 0 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_meta_data_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-02T18:27:10.658,ns_1@10.2.1.67:ns_doctor<0.1461.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,906029,48359}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,73735168}, {processes,37370488}, {processes_used,35790008}, {system,36364680}, {atom,1400977}, {atom_used,1372242}, {binary,1128152}, {code,14085234}, {ets,2025840}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,612077568}, {buffered_memory,88662016}, {free_memory,32575430656}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{237193,5000}}, {context_switches,{330179,0}}, {garbage_collection,{51196,197707773,0}}, {io,{{input,16908684},{output,9857752}}}, {reductions,{90379752,297536}}, {run_queue,0}, {runtime,{20120,460}}]}, {system_stats, [{cpu_utilization_rate,2.9925187032418954}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,1061412}, {couch_docs_data_size,1054464}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,0}, {curr_items_tot,0}, {mem_used,4169944}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,237}, {memory_data,{33657663488,1071636480,{<18077.695.0>,972728}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 31811944 kB\nBuffers: 86584 kB\nCached: 597732 kB\nSwapCached: 0 kB\nActive: 455200 kB\nInactive: 446624 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 31811944 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 57420 kB\nWriteback: 0 kB\nAnonPages: 217456 kB\nMapped: 15252 kB\nSlab: 117928 kB\nPageTables: 4592 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 1915004 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,906026,685730}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,68506512}, {processes,33288568}, {processes_used,33274696}, {system,35217944}, {atom,1336577}, {atom_used,1329113}, {binary,1243488}, {code,13449841}, {ets,1625864}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,608935936}, {buffered_memory,99479552}, {free_memory,32566435840}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{234439,4999}}, {context_switches,{103325,0}}, {garbage_collection,{21187,64418703,0}}, {io,{{input,9192516},{output,3201346}}}, {reductions,{30593074,346609}}, {run_queue,0}, {runtime,{13820,440}}]}, {system_stats, [{cpu_utilization_rate,3.0}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,1061412}, {couch_docs_data_size,1054464}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,0}, {curr_items_tot,0}, {mem_used,4174248}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,234}, {memory_data,{33657663488,1062219776,{<18117.693.0>,1343600}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 31803160 kB\nBuffers: 97148 kB\nCached: 594664 kB\nSwapCached: 140 kB\nActive: 472096 kB\nInactive: 439392 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 31803160 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 57608 kB\nWriteback: 0 kB\nAnonPages: 219444 kB\nMapped: 17020 kB\nSlab: 114524 kB\nPageTables: 5208 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 1920660 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,906025,659917}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,69861224}, {processes,34601320}, {processes_used,34587448}, {system,35259904}, {atom,1336577}, {atom_used,1329113}, {binary,1287680}, {code,13459097}, {ets,1613440}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,647200768}, {buffered_memory,401821696}, {free_memory,32150577152}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{234398,5001}}, {context_switches,{102690,0}}, {garbage_collection,{21306,62219902,0}}, {io,{{input,9237774},{output,3169480}}}, {reductions,{29779314,286747}}, {run_queue,0}, {runtime,{13200,390}}]}, {system_stats, [{cpu_utilization_rate,3.0},{swap_total,10701078528},{swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,1061412}, {couch_docs_data_size,1054464}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,0}, {curr_items_tot,0}, {mem_used,4174248}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,234}, {memory_data,{33657663488,1481523200,{<0.1512.0>,830416}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 31397048 kB\nBuffers: 392404 kB\nCached: 632032 kB\nSwapCached: 0 kB\nActive: 723168 kB\nInactive: 518100 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 31397048 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 69720 kB\nWriteback: 0 kB\nAnonPages: 216832 kB\nMapped: 17020 kB\nSlab: 193464 kB\nPageTables: 5220 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 1920720 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,906029,13266}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,71141736}, {processes,35883608}, {processes_used,35869736}, {system,35258128}, {atom,1336577}, {atom_used,1329113}, {binary,1241504}, {code,13487081}, {ets,1634000}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,601055232}, {buffered_memory,298708992}, {free_memory,32291000320}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{236430,5000}}, {context_switches,{105745,0}}, {garbage_collection,{21593,65360333,0}}, {io,{{input,9258595},{output,3267408}}}, {reductions,{31760569,283791}}, {run_queue,0}, {runtime,{14150,390}}]}, {system_stats, [{cpu_utilization_rate,2.5}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,1061412}, {couch_docs_data_size,1054464}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,0}, {curr_items_tot,0}, {mem_used,4174248}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,236}, {memory_data,{33657663488,1337147392,{<18118.6.0>,601480}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 31534304 kB\nBuffers: 291708 kB\nCached: 586968 kB\nSwapCached: 12 kB\nActive: 522052 kB\nInactive: 574736 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 31534304 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 69568 kB\nWriteback: 0 kB\nAnonPages: 218136 kB\nMapped: 17164 kB\nSlab: 197876 kB\nPageTables: 5160 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 1920208 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-02T18:28:10.658,ns_1@10.2.1.67:ns_doctor<0.1461.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,906089,47529}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,88977768}, {processes,50993704}, {processes_used,49415512}, {system,37984064}, {atom,1404209}, {atom_used,1376964}, {binary,2526216}, {code,14117455}, {ets,2202336}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,613068800}, {buffered_memory,88875008}, {free_memory,32553082880}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{297193,5000}}, {context_switches,{373356,0}}, {garbage_collection,{56479,246262478,0}}, {io,{{input,29704909},{output,11915039}}}, {reductions,{101208660,689131}}, {run_queue,0}, {runtime,{26840,610}}]}, {system_stats, [{cpu_utilization_rate,3.7406483790523692}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,1068341}, {couch_docs_data_size,1054464}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,0}, {curr_items_tot,0}, {mem_used,4169944}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,297}, {memory_data,{33657663488,1082613760,{<18077.695.0>,1572896}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 31790120 kB\nBuffers: 86792 kB\nCached: 598700 kB\nSwapCached: 0 kB\nActive: 477032 kB\nInactive: 447700 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 31790120 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 376 kB\nWriteback: 0 kB\nAnonPages: 239120 kB\nMapped: 15252 kB\nSlab: 117976 kB\nPageTables: 4664 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 1953596 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,906086,685953}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,74009376}, {processes,38334184}, {processes_used,38320312}, {system,35675192}, {atom,1352041}, {atom_used,1349085}, {binary,1113600}, {code,13622682}, {ets,2021184}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,610045952}, {buffered_memory,99659776}, {free_memory,32556277760}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{294439,4998}}, {context_switches,{140173,0}}, {garbage_collection,{25067,80655964,0}}, {io,{{input,9954281},{output,4501319}}}, {reductions,{34558975,370280}}, {run_queue,0}, {runtime,{18870,450}}]}, {system_stats, [{cpu_utilization_rate,3.0}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,1068341}, {couch_docs_data_size,1054464}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,0}, {curr_items_tot,0}, {mem_used,4174248}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,294}, {memory_data,{33657663488,1091227648,{<18117.693.0>,1943840}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 31793240 kB\nBuffers: 97324 kB\nCached: 595748 kB\nSwapCached: 140 kB\nActive: 480628 kB\nInactive: 440448 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 31793240 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 45024 kB\nWriteback: 0 kB\nAnonPages: 227876 kB\nMapped: 17020 kB\nSlab: 114692 kB\nPageTables: 5232 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 1928308 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,906085,660065}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,76841528}, {processes,41188672}, {processes_used,41174800}, {system,35652856}, {atom,1352041}, {atom_used,1349085}, {binary,1094152}, {code,13631938}, {ets,2008832}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,648310784}, {buffered_memory,401899520}, {free_memory,32143085568}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{294398,5000}}, {context_switches,{139499,0}}, {garbage_collection,{25540,78709977,0}}, {io,{{input,10014550},{output,4527746}}}, {reductions,{33868814,320106}}, {run_queue,0}, {runtime,{18260,430}}]}, {system_stats, [{cpu_utilization_rate,3.0},{swap_total,10701078528},{swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,1068341}, {couch_docs_data_size,1054464}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,0}, {curr_items_tot,0}, {mem_used,4174248}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,294}, {memory_data,{33657663488,1507975168,{<0.701.0>,1943840}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 31389732 kB\nBuffers: 392480 kB\nCached: 633116 kB\nSwapCached: 0 kB\nActive: 728728 kB\nInactive: 518964 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 31389732 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 48720 kB\nWriteback: 0 kB\nAnonPages: 222152 kB\nMapped: 17020 kB\nSlab: 193740 kB\nPageTables: 5252 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 1929912 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,906089,13178}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,76350384}, {processes,40554584}, {processes_used,40540712}, {system,35795800}, {atom,1352041}, {atom_used,1349085}, {binary,1233320}, {code,13659922}, {ets,1988704}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,602198016}, {buffered_memory,298823680}, {free_memory,32281604096}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{296430,5000}}, {context_switches,{142652,0}}, {garbage_collection,{25619,81844249,0}}, {io,{{input,10011558},{output,4690271}}}, {reductions,{35771839,309230}}, {run_queue,0}, {runtime,{19340,420}}]}, {system_stats, [{cpu_utilization_rate,2.743142144638404}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,1068341}, {couch_docs_data_size,1054464}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,0}, {curr_items_tot,0}, {mem_used,4174248}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,296}, {memory_data,{33657663488,1367298048,{<18118.694.0>,1572896}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 31525004 kB\nBuffers: 291820 kB\nCached: 588084 kB\nSwapCached: 12 kB\nActive: 529396 kB\nInactive: 575664 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 31525004 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 48880 kB\nWriteback: 104 kB\nAnonPages: 225148 kB\nMapped: 17164 kB\nSlab: 198016 kB\nPageTables: 5176 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 1928052 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-02T18:28:12.538,ns_1@10.2.1.67:<0.1687.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,31,46}}) Stats for bucket "default": accepting_conns 1 auth_cmds 1 auth_errors 0 bucket_active_conns 1 bucket_conns 5 bytes 4174248 bytes_read 14436 bytes_written 1476820 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 0 conn_yields 0 connection_structures 5000 curr_connections 9 curr_conns_on_port_11209 6 curr_conns_on_port_11210 1 curr_items 0 curr_items_tot 0 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-03 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 0 ep_commit_time 0 ep_commit_time_total 0 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 0 ep_data_age_highwat 0 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data/default ep_degraded_mode 0 ep_diskqueue_drain 0 ep_diskqueue_fill 0 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 0 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 0 ep_io_read_bytes 0 ep_io_write_bytes 0 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 0 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 0 ep_max_checkpoints 2 ep_max_data_size 5242880000 ep_max_item_size 20971520 ep_max_size 5242880000 ep_max_txn_size 10000 ep_max_vbuckets 512 ep_mem_high_wat 3932160000 ep_mem_high_wat 3932160000 ep_mem_low_wat 3145728000 ep_mem_low_wat 3145728000 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 3404160 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1351905991 ep_storage_age 0 ep_storage_age_highwat 0 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 0 ep_total_del_items 0 ep_total_enqueued 0 ep_total_new_items 0 ep_total_persisted 0 ep_uncommitted_items 0 ep_value_size 0 ep_vb0 0 ep_vb_snapshot_total 6 ep_vb_total 128 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_130_g8b09894 ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 725 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 4174248 pid 16451 pointer_size 64 rejected_conns 0 rusage_system 0.063990 rusage_user 0.202969 threads 4 time 1351906091 total_connections 11 uptime 123 vb_active_curr_items 0 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 3214336 vb_active_itm_memory 0 vb_active_meta_data_memory 0 vb_active_num 128 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 0 vb_active_ops_create 0 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 0 vb_active_queue_age 0 vb_active_queue_drain 0 vb_active_queue_fill 0 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_meta_data_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-02T18:29:10.658,ns_1@10.2.1.67:ns_doctor<0.1461.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,906149,46394}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,88286648}, {processes,49857216}, {processes_used,48279024}, {system,38429432}, {atom,1404209}, {atom_used,1376964}, {binary,2780064}, {code,14117455}, {ets,2388048}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,614121472}, {buffered_memory,89190400}, {free_memory,32546607104}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{357193,5000}}, {context_switches,{417134,0}}, {garbage_collection,{61996,297598984,0}}, {io,{{input,47777194},{output,16630463}}}, {reductions,{112036273,690275}}, {run_queue,0}, {runtime,{33900,570}}]}, {system_stats, [{cpu_utilization_rate,3.5}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,1068356}, {couch_docs_data_size,1054464}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,0}, {curr_items_tot,0}, {mem_used,4169944}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,357}, {memory_data,{33657663488,1102422016,{<18077.5273.0>,7628384}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 31783796 kB\nBuffers: 87100 kB\nCached: 599728 kB\nSwapCached: 0 kB\nActive: 481332 kB\nInactive: 449156 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 31783796 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 132 kB\nWriteback: 0 kB\nAnonPages: 243676 kB\nMapped: 15252 kB\nSlab: 118228 kB\nPageTables: 4600 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 1957376 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,906146,686177}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,74805648}, {processes,38843376}, {processes_used,38828704}, {system,35962272}, {atom,1352041}, {atom_used,1349085}, {binary,1211200}, {code,13622682}, {ets,2206080}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,611123200}, {buffered_memory,99938304}, {free_memory,32551452672}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{354439,4997}}, {context_switches,{177991,0}}, {garbage_collection,{29144,97748259,0}}, {io,{{input,13542846},{output,8503577}}}, {reductions,{38657756,358650}}, {run_queue,0}, {runtime,{24070,460}}]}, {system_stats, [{cpu_utilization_rate,3.7313432835820897}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,1068356}, {couch_docs_data_size,1054464}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,0}, {curr_items_tot,0}, {mem_used,4174248}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,354}, {memory_data,{33657663488,1101258752,{<18117.693.0>,1943840}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 31788528 kB\nBuffers: 97596 kB\nCached: 596800 kB\nSwapCached: 140 kB\nActive: 483572 kB\nInactive: 441672 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 31788528 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 36492 kB\nWriteback: 0 kB\nAnonPages: 230688 kB\nMapped: 17020 kB\nSlab: 114848 kB\nPageTables: 5244 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 1931664 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,906145,659608}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,76143128}, {processes,40095344}, {processes_used,40081472}, {system,36047784}, {atom,1352041}, {atom_used,1349085}, {binary,1303536}, {code,13631938}, {ets,2191472}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,649449472}, {buffered_memory,401985536}, {free_memory,32140165120}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{354397,4999}}, {context_switches,{176363,0}}, {garbage_collection,{29558,95332347,0}}, {io,{{input,10697514},{output,5869524}}}, {reductions,{37960017,319216}}, {run_queue,0}, {runtime,{23230,390}}]}, {system_stats, [{cpu_utilization_rate,3.0},{swap_total,10701078528},{swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,1068356}, {couch_docs_data_size,1054464}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,0}, {curr_items_tot,0}, {mem_used,4174248}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,354}, {memory_data,{33657663488,1514831872,{<0.1691.0>,3143528}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 31386260 kB\nBuffers: 392564 kB\nCached: 634228 kB\nSwapCached: 0 kB\nActive: 731504 kB\nInactive: 519964 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 31386260 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 44380 kB\nWriteback: 0 kB\nAnonPages: 224740 kB\nMapped: 17020 kB\nSlab: 193708 kB\nPageTables: 5256 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 1932920 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,906149,12398}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,75829328}, {processes,39788944}, {processes_used,39775072}, {system,36040384}, {atom,1352041}, {atom_used,1349085}, {binary,1292072}, {code,13659922}, {ets,2171200}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,603303936}, {buffered_memory,298930176}, {free_memory,32278810624}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{356430,5000}}, {context_switches,{179551,0}}, {garbage_collection,{29354,98559057,0}}, {io,{{input,10716297},{output,6018453}}}, {reductions,{39776358,305701}}, {run_queue,0}, {runtime,{24780,430}}]}, {system_stats, [{cpu_utilization_rate,2.506265664160401}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,1068356}, {couch_docs_data_size,1054464}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,0}, {curr_items_tot,0}, {mem_used,4174248}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,356}, {memory_data,{33657663488,1376567296,{<18118.694.0>,1572896}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 31522276 kB\nBuffers: 291924 kB\nCached: 589164 kB\nSwapCached: 12 kB\nActive: 532024 kB\nInactive: 576628 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 31522276 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 36468 kB\nWriteback: 104 kB\nAnonPages: 227620 kB\nMapped: 17164 kB\nSlab: 197992 kB\nPageTables: 5184 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 1930104 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-02T18:29:52.537,ns_1@10.2.1.67:<0.1687.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,31,46}}) Stats for bucket "default": accepting_conns 1 auth_cmds 2 auth_errors 0 bucket_active_conns 1 bucket_conns 5 bytes 4174248 bytes_read 24964 bytes_written 2962617 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 0 conn_yields 0 connection_structures 5000 curr_connections 9 curr_conns_on_port_11209 6 curr_conns_on_port_11210 1 curr_items 0 curr_items_tot 0 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-03 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 0 ep_commit_time 0 ep_commit_time_total 0 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 0 ep_data_age_highwat 0 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data/default ep_degraded_mode 0 ep_diskqueue_drain 0 ep_diskqueue_fill 0 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 0 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 0 ep_io_read_bytes 0 ep_io_write_bytes 0 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 0 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 0 ep_max_checkpoints 2 ep_max_data_size 5242880000 ep_max_item_size 20971520 ep_max_size 5242880000 ep_max_txn_size 10000 ep_max_vbuckets 512 ep_mem_high_wat 3932160000 ep_mem_high_wat 3932160000 ep_mem_low_wat 3145728000 ep_mem_low_wat 3145728000 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 3404160 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1351905991 ep_storage_age 0 ep_storage_age_highwat 0 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 0 ep_total_del_items 0 ep_total_enqueued 0 ep_total_new_items 0 ep_total_persisted 0 ep_uncommitted_items 0 ep_value_size 0 ep_vb0 0 ep_vb_snapshot_total 6 ep_vb_total 128 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_130_g8b09894 ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 725 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 4174248 pid 16451 pointer_size 64 rejected_conns 0 rusage_system 0.077988 rusage_user 0.345947 threads 4 time 1351906191 total_connections 12 uptime 223 vb_active_curr_items 0 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 3214336 vb_active_itm_memory 0 vb_active_meta_data_memory 0 vb_active_num 128 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 0 vb_active_ops_create 0 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 0 vb_active_queue_age 0 vb_active_queue_drain 0 vb_active_queue_fill 0 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_meta_data_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-02T18:30:10.658,ns_1@10.2.1.67:ns_doctor<0.1461.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,906209,44641}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,92762480}, {processes,54077048}, {processes_used,52499576}, {system,38685432}, {atom,1404209}, {atom_used,1376964}, {binary,2630848}, {code,14117455}, {ets,2563272}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,650100736}, {buffered_memory,140853248}, {free_memory,32436518912}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{417192,4999}}, {context_switches,{567744,0}}, {garbage_collection,{79273,357486887,0}}, {io,{{input,63376930},{output,18906988}}}, {reductions,{126640569,5107041}}, {run_queue,0}, {runtime,{42920,2660}}]}, {system_stats, [{cpu_utilization_rate,42.0}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,15412847}, {couch_docs_data_size,15379840}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,7315}, {curr_items_tot,7315}, {mem_used,23600592}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,417}, {memory_data,{33657663488,1108262912,{<18077.5273.0>,7628384}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 31676288 kB\nBuffers: 137548 kB\nCached: 634868 kB\nSwapCached: 0 kB\nActive: 549836 kB\nInactive: 488332 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 31676288 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 1244 kB\nWriteback: 104 kB\nAnonPages: 265920 kB\nMapped: 15284 kB\nSlab: 118548 kB\nPageTables: 4724 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 1984532 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,906206,688776}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,77369632}, {processes,41189856}, {processes_used,41176384}, {system,36179776}, {atom,1352041}, {atom_used,1349085}, {binary,1187664}, {code,13622682}, {ets,2381144}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,620130304}, {buffered_memory,118071296}, {free_memory,32514629632}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{414442,4999}}, {context_switches,{243568,0}}, {garbage_collection,{37147,117048602,0}}, {io,{{input,14391120},{output,10020891}}}, {reductions,{43631520,1563981}}, {run_queue,0}, {runtime,{29820,1050}}]}, {system_stats, [{cpu_utilization_rate,29.925187032418954}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,1068356}, {couch_docs_data_size,1054464}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1837}, {curr_items_tot,1837}, {mem_used,9152088}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,414}, {memory_data,{33657663488,1106210816,{<18117.693.0>,1943840}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 31752692 kB\nBuffers: 115208 kB\nCached: 605364 kB\nSwapCached: 140 kB\nActive: 501668 kB\nInactive: 460544 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 31752692 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 28948 kB\nWriteback: 0 kB\nAnonPages: 241240 kB\nMapped: 17020 kB\nSlab: 114940 kB\nPageTables: 5240 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 1946280 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,906205,659868}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,77538472}, {processes,41291896}, {processes_used,41278368}, {system,36246576}, {atom,1352041}, {atom_used,1349085}, {binary,1320224}, {code,13631938}, {ets,2368864}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,650534912}, {buffered_memory,402128896}, {free_memory,32136482816}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{414398,5001}}, {context_switches,{214173,0}}, {garbage_collection,{33878,112300106,0}}, {io,{{input,14438961},{output,9998200}}}, {reductions,{42045964,328737}}, {run_queue,0}, {runtime,{28420,410}}]}, {system_stats, [{cpu_utilization_rate,3.0},{swap_total,10701078528},{swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,1068356}, {couch_docs_data_size,1054464}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,0}, {curr_items_tot,0}, {mem_used,4174248}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,414}, {memory_data,{33657663488,1519149056,{<0.1691.0>,3143528}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 31383408 kB\nBuffers: 392704 kB\nCached: 635288 kB\nSwapCached: 0 kB\nActive: 733404 kB\nInactive: 521028 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 31383408 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 32364 kB\nWriteback: 0 kB\nAnonPages: 226432 kB\nMapped: 17020 kB\nSlab: 193780 kB\nPageTables: 5260 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 1933208 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,906209,10887}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,80387696}, {processes,43900920}, {processes_used,43887288}, {system,36486776}, {atom,1352041}, {atom_used,1349085}, {binary,1348736}, {code,13659922}, {ets,2348544}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,637845504}, {buffered_memory,299659264}, {free_memory,32208506880}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{416429,4999}}, {context_switches,{316473,0}}, {garbage_collection,{47434,124462090,0}}, {io,{{input,15048858},{output,10581442}}}, {reductions,{47553224,4084480}}, {run_queue,0}, {runtime,{32020,2390}}]}, {system_stats, [{cpu_utilization_rate,37.25}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,13108164}, {couch_docs_data_size,13012352}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,7386}, {curr_items_tot,7386}, {mem_used,23936568}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,416}, {memory_data,{33657663488,1378598912,{<18118.694.0>,1572896}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 31453744 kB\nBuffers: 292636 kB\nCached: 622896 kB\nSwapCached: 12 kB\nActive: 648832 kB\nInactive: 528764 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 31453744 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 28488 kB\nWriteback: 0 kB\nAnonPages: 261792 kB\nMapped: 17228 kB\nSlab: 198424 kB\nPageTables: 5268 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 1963400 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-02T18:31:10.658,ns_1@10.2.1.67:ns_doctor<0.1461.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,906269,44743}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,105751416}, {processes,65434776}, {processes_used,63857064}, {system,40316640}, {atom,1405017}, {atom_used,1377299}, {binary,3966832}, {code,14117455}, {ets,2745120}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,1102639104}, {buffered_memory,202309632}, {free_memory,31425028096}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{477193,5001}}, {context_switches,{1815949,0}}, {garbage_collection,{212098,545660154,0}}, {io,{{input,91482454},{output,25814612}}}, {reductions,{200874016,6415587}}, {run_queue,0}, {runtime,{77490,3010}}]}, {system_stats, [{cpu_utilization_rate,39.75}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,466027596}, {couch_docs_data_size,410022641}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,172940}, {curr_items_tot,172940}, {mem_used,460913720}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,477}, {memory_data,{33657663488,1311170560,{<18077.5273.0>,7628384}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 30688504 kB\nBuffers: 197568 kB\nCached: 1076796 kB\nSwapCached: 0 kB\nActive: 1241332 kB\nInactive: 782900 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 30688504 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 1552 kB\nWriteback: 0 kB\nAnonPages: 749924 kB\nMapped: 15308 kB\nSlab: 119372 kB\nPageTables: 5692 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 2476768 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,906266,686902}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.5397.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,65}, {started_on,1351906227}, {total_vbuckets,129}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351906232}, {vbuckets_done,85}]]}, {memory, [{total,87080408}, {processes,49274616}, {processes_used,49253376}, {system,37805792}, {atom,1352041}, {atom_used,1349412}, {binary,2234944}, {code,13622682}, {ets,2569712}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,1093259264}, {buffered_memory,231403520}, {free_memory,31432908800}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{474439,4998}}, {context_switches,{1690925,0}}, {garbage_collection,{213219,286599599,0}}, {io,{{input,28085228},{output,16971560}}}, {reductions,{118528869,4712202}}, {run_queue,0}, {runtime,{63060,1390}}]}, {system_stats, [{cpu_utilization_rate,40.25}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,561833035}, {couch_docs_data_size,381877194}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,169229}, {curr_items_tot,169229}, {mem_used,451281320}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,474}, {memory_data,{33657663488,1342517248,{<18117.1696.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 30696200 kB\nBuffers: 225980 kB\nCached: 1067636 kB\nSwapCached: 140 kB\nActive: 1231480 kB\nInactive: 781920 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 30696200 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 22624 kB\nWriteback: 0 kB\nAnonPages: 719708 kB\nMapped: 17000 kB\nSlab: 118740 kB\nPageTables: 6248 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 2427620 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,906265,660260}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,92238096}, {processes,54223832}, {processes_used,54191920}, {system,38014264}, {atom,1352041}, {atom_used,1349412}, {binary,2452328}, {code,13631938}, {ets,2552016}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,1176350720}, {buffered_memory,404004864}, {free_memory,31103533056}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{474398,5001}}, {context_switches,{1503292,0}}, {garbage_collection,{180016,266780910,0}}, {io,{{input,26209265},{output,16373462}}}, {reductions,{110001041,4866676}}, {run_queue,0}, {runtime,{59780,2410}}]}, {system_stats, [{cpu_utilization_rate,34.75}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,481915975}, {couch_docs_data_size,378491738}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,165495}, {curr_items_tot,165495}, {mem_used,441396424}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,474}, {memory_data,{33657663488,1636364288,{<0.1691.0>,3143528}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 30374544 kB\nBuffers: 394536 kB\nCached: 1148780 kB\nSwapCached: 0 kB\nActive: 1447816 kB\nInactive: 813924 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 30374544 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 25276 kB\nWriteback: 72 kB\nAnonPages: 718240 kB\nMapped: 17056 kB\nSlab: 194780 kB\nPageTables: 6248 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 2430480 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,906269,11201}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,89976872}, {processes,51969456}, {processes_used,51938024}, {system,38007416}, {atom,1352041}, {atom_used,1349412}, {binary,2439592}, {code,13659922}, {ets,2530968}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,1128804352}, {buffered_memory,301473792}, {free_memory,31223173120}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{476430,5000}}, {context_switches,{1593136,0}}, {garbage_collection,{194393,279557405,0}}, {io,{{input,28056650},{output,17059857}}}, {reductions,{116413565,6968910}}, {run_queue,0}, {runtime,{61710,2860}}]}, {system_stats, [{cpu_utilization_rate,39.900249376558605}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,497345607}, {couch_docs_data_size,412614868}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,173997}, {curr_items_tot,173997}, {mem_used,463767736}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,476}, {memory_data,{33657663488,1525088256,{<18118.1662.0>,1801552}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 30491504 kB\nBuffers: 294408 kB\nCached: 1102348 kB\nSwapCached: 12 kB\nActive: 1354612 kB\nInactive: 783704 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 30491504 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 21288 kB\nWriteback: 0 kB\nAnonPages: 741456 kB\nMapped: 17400 kB\nSlab: 199120 kB\nPageTables: 6368 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 2447824 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-02T18:31:32.534,ns_1@10.2.1.67:<0.1687.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,31,46}}) Stats for bucket "default": accepting_conns 1 auth_cmds 3 auth_errors 0 bucket_active_conns 1 bucket_conns 6 bytes 640522832 bytes_read 577031250 bytes_written 14300900 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 1 cmd_set 240544 conn_yields 83 connection_structures 5000 curr_connections 10 curr_conns_on_port_11209 6 curr_conns_on_port_11210 2 curr_items 240544 curr_items_tot 240544 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-03 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 469 ep_commit_time 2 ep_commit_time_total 461 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 4 ep_data_age_highwat 7 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data/default ep_degraded_mode 0 ep_diskqueue_drain 232425 ep_diskqueue_fill 240544 ep_diskqueue_items 8120 ep_diskqueue_memory 259808 ep_diskqueue_pending 389712 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 85 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 185 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 232423 ep_io_read_bytes 0 ep_io_write_bytes 550183384 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 0 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 588147416 ep_max_checkpoints 2 ep_max_data_size 5242880000 ep_max_item_size 20971520 ep_max_size 5242880000 ep_max_txn_size 10000 ep_max_vbuckets 512 ep_mem_high_wat 3932160000 ep_mem_high_wat 3932160000 ep_mem_low_wat 3145728000 ep_mem_low_wat 3145728000 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 24573584 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 7935 ep_startup_time 1351905991 ep_storage_age 3 ep_storage_age_highwat 5 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 585260888 ep_total_del_items 0 ep_total_enqueued 240544 ep_total_new_items 232423 ep_total_persisted 232423 ep_uncommitted_items 425 ep_value_size 568336134 ep_vb0 0 ep_vb_snapshot_total 6 ep_vb_total 128 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_130_g8b09894 ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 725 get_hits 0 get_misses 1 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 640522832 pid 16451 pointer_size 64 rejected_conns 0 rusage_system 13.735911 rusage_user 33.960837 threads 4 time 1351906292 total_connections 13 uptime 324 vb_active_curr_items 240544 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 3214336 vb_active_itm_memory 585260888 vb_active_meta_data_memory 19002976 vb_active_num 128 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 240544 vb_active_ops_create 232424 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 100 vb_active_queue_age 11596000 vb_active_queue_drain 232425 vb_active_queue_fill 240544 vb_active_queue_memory 259808 vb_active_queue_pending 389712 vb_active_queue_size 8119 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_meta_data_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-02T18:32:10.658,ns_1@10.2.1.67:ns_doctor<0.1461.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,906329,43333}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,105353064}, {processes,64486968}, {processes_used,62908632}, {system,40866096}, {atom,1405017}, {atom_used,1377299}, {binary,4339832}, {code,14117455}, {ets,2923504}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,1755873280}, {buffered_memory,204251136}, {free_memory,30276403200}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{537193,5000}}, {context_switches,{2414314,0}}, {garbage_collection,{270818,671492085,0}}, {io,{{input,113020709},{output,30973143}}}, {reductions,{249398253,3105541}}, {run_queue,0}, {runtime,{96370,1020}}]}, {system_stats, [{cpu_utilization_rate,7.5}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,1144195806}, {couch_docs_data_size,816936711}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,338839}, {curr_items_tot,338839}, {mem_used,899636424}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,537}, {memory_data,{33657663488,2292187136,{<18077.5273.0>,7628384}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29566800 kB\nBuffers: 199464 kB\nCached: 1714720 kB\nSwapCached: 0 kB\nActive: 2120684 kB\nInactive: 1008208 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29566800 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 1764 kB\nWriteback: 76 kB\nAnonPages: 1214484 kB\nMapped: 15308 kB\nSlab: 134928 kB\nPageTables: 6620 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 2928036 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,906326,688246}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,88836224}, {processes,50919192}, {processes_used,50886720}, {system,37917032}, {atom,1352041}, {atom_used,1349412}, {binary,2175792}, {code,13622682}, {ets,2741728}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,1668222976}, {buffered_memory,233324544}, {free_memory,30365294592}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{534441,4999}}, {context_switches,{2364827,0}}, {garbage_collection,{284659,386346226,0}}, {io,{{input,37080381},{output,21954716}}}, {reductions,{163872739,2026244}}, {run_queue,0}, {runtime,{82550,580}}]}, {system_stats, [{cpu_utilization_rate,7.5}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,1057199325}, {couch_docs_data_size,785779314}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,335840}, {curr_items_tot,335840}, {mem_used,891047608}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,534}, {memory_data,{33657663488,2259791872,{<18117.1696.0>,2914384}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29653732 kB\nBuffers: 227856 kB\nCached: 1629124 kB\nSwapCached: 140 kB\nActive: 2021472 kB\nInactive: 1021020 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29653732 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 12556 kB\nWriteback: 0 kB\nAnonPages: 1184900 kB\nMapped: 17056 kB\nSlab: 131388 kB\nPageTables: 7148 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 2895624 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,906325,659568}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.7032.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,62}, {started_on,1351906300}, {total_vbuckets,129}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351906323}, {vbuckets_done,81}]]}, {memory, [{total,90213056}, {processes,52141088}, {processes_used,52117488}, {system,38071968}, {atom,1352041}, {atom_used,1349412}, {binary,2288848}, {code,13631938}, {ets,2768608}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,1469829120}, {buffered_memory,405995520}, {free_memory,30341410816}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{534397,4999}}, {context_switches,{1906277,0}}, {garbage_collection,{222647,342806045,0}}, {io,{{input,31595781},{output,19507317}}}, {reductions,{145832209,2107511}}, {run_queue,0}, {runtime,{72460,500}}]}, {system_stats, [{cpu_utilization_rate,6.5},{swap_total,10701078528},{swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,829411976}, {couch_docs_data_size,689605170}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,331795}, {curr_items_tot,331795}, {mem_used,880997160}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,534}, {memory_data,{33657663488,2678312960,{<0.1681.0>,4714984}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29630284 kB\nBuffers: 396480 kB\nCached: 1435380 kB\nSwapCached: 0 kB\nActive: 1866492 kB\nInactive: 1137900 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29630284 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 21208 kB\nWriteback: 4 kB\nAnonPages: 1172568 kB\nMapped: 17060 kB\nSlab: 195364 kB\nPageTables: 7208 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 2884176 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,906329,10663}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,90036352}, {processes,51857200}, {processes_used,51824728}, {system,38179152}, {atom,1352041}, {atom_used,1349412}, {binary,2430784}, {code,13659922}, {ets,2705352}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,1731526656}, {buffered_memory,302997504}, {free_memory,30143492096}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{536430,4999}}, {context_switches,{2042914,0}}, {garbage_collection,{239367,358058195,0}}, {io,{{input,30620038},{output,19756925}}}, {reductions,{153082667,2576855}}, {run_queue,0}, {runtime,{74420,890}}]}, {system_stats, [{cpu_utilization_rate,6.75}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,1125647595}, {couch_docs_data_size,808118206}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,340250}, {curr_items_tot,340250}, {mem_used,902285512}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,536}, {memory_data,{33657663488,2513596416,{<18118.1663.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29437128 kB\nBuffers: 295896 kB\nCached: 1690944 kB\nSwapCached: 12 kB\nActive: 2153488 kB\nInactive: 1037484 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29437128 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 17356 kB\nWriteback: 0 kB\nAnonPages: 1203720 kB\nMapped: 17256 kB\nSlab: 199016 kB\nPageTables: 7084 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 2914840 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-02T18:33:10.658,ns_1@10.2.1.67:ns_doctor<0.1461.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,906389,41099}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.12892.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,33}, {started_on,1351906406}, {total_vbuckets,129}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351906421}, {vbuckets_done,43}]]}, {memory, [{total,103085488}, {processes,61737712}, {processes_used,60190448}, {system,41347776}, {atom,1405017}, {atom_used,1377299}, {binary,4609288}, {code,14117455}, {ets,3132560}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,2133979136}, {buffered_memory,205606912}, {free_memory,29423497216}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{597192,4999}}, {context_switches,{2664197,0}}, {garbage_collection,{296458,766332018,0}}, {io,{{input,132146813},{output,33927272}}}, {reductions,{285687549,2591041}}, {run_queue,0}, {runtime,{106240,570}}]}, {system_stats, [{cpu_utilization_rate,19.548872180451127}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,1545518832}, {couch_docs_data_size,1166369458}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,506723}, {curr_items_tot,506723}, {mem_used,1344509192}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,597}, {memory_data,{33657663488,3439034368,{<18077.5273.0>,7628384}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 28733884 kB\nBuffers: 200788 kB\nCached: 2083964 kB\nSwapCached: 0 kB\nActive: 2558728 kB\nInactive: 1391736 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 28733884 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 10248 kB\nWriteback: 0 kB\nAnonPages: 1665752 kB\nMapped: 15244 kB\nSlab: 145432 kB\nPageTables: 7776 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 3386912 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,906386,687800}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,89799912}, {processes,51453624}, {processes_used,51421472}, {system,38346288}, {atom,1352041}, {atom_used,1349412}, {binary,2407768}, {code,13622682}, {ets,2924216}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,2328473600}, {buffered_memory,234364928}, {free_memory,29208932352}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{594440,4995}}, {context_switches,{2643330,0}}, {garbage_collection,{312880,449423802,0}}, {io,{{input,38847274},{output,24046625}}}, {reductions,{194280679,2024971}}, {run_queue,0}, {runtime,{91090,500}}]}, {system_stats, [{cpu_utilization_rate,25.935162094763093}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,1666672411}, {couch_docs_data_size,1187250032}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,503544}, {curr_items_tot,503544}, {mem_used,1336327368}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,594}, {memory_data,{33657663488,3417178112,{<18117.1697.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 28524348 kB\nBuffers: 228872 kB\nCached: 2273900 kB\nSwapCached: 140 kB\nActive: 2784360 kB\nInactive: 1371128 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 28524348 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 5812 kB\nWriteback: 0 kB\nAnonPages: 1652688 kB\nMapped: 17056 kB\nSlab: 147376 kB\nPageTables: 8028 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 3362304 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,906385,660426}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,92006040}, {processes,53609504}, {processes_used,53541352}, {system,38396536}, {atom,1352041}, {atom_used,1349412}, {binary,2470240}, {code,13631938}, {ets,2912992}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,2196099072}, {buffered_memory,407273472}, {free_memory,29138067456}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{594398,4999}}, {context_switches,{2595480,0}}, {garbage_collection,{295620,443787884,0}}, {io,{{input,39077148},{output,24321741}}}, {reductions,{191770542,2388004}}, {run_queue,0}, {runtime,{90340,590}}]}, {system_stats, [{cpu_utilization_rate,28.678304239401495}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,1486083330}, {couch_docs_data_size,1174595976}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,499681}, {curr_items_tot,499681}, {mem_used,1325304584}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,594}, {memory_data,{33657663488,3337601024,{<0.1691.0>,3143528}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 28455144 kB\nBuffers: 397728 kB\nCached: 2144628 kB\nSwapCached: 0 kB\nActive: 2666016 kB\nInactive: 1513700 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 28455144 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 12268 kB\nWriteback: 0 kB\nAnonPages: 1637324 kB\nMapped: 17056 kB\nSlab: 195076 kB\nPageTables: 8044 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 3347464 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,906389,10530}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.7411.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,44}, {started_on,1351906361}, {total_vbuckets,129}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351906379}, {vbuckets_done,57}]]}, {memory, [{total,89328928}, {processes,50949544}, {processes_used,50933704}, {system,38379384}, {atom,1352041}, {atom_used,1349412}, {binary,2384736}, {code,13659922}, {ets,2920408}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,2065514496}, {buffered_memory,304640000}, {free_memory,29343408128}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{596430,5000}}, {context_switches,{2369484,0}}, {garbage_collection,{273787,427696317,0}}, {io,{{input,40039600},{output,23820356}}}, {reductions,{186051865,2002505}}, {run_queue,0}, {runtime,{84820,450}}]}, {system_stats, [{cpu_utilization_rate,10.0}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,1463140219}, {couch_docs_data_size,1158400791}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,508054}, {curr_items_tot,508054}, {mem_used,1348462888}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,596}, {memory_data,{33657663488,3615244288,{<18118.1662.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 28655796 kB\nBuffers: 297500 kB\nCached: 2017104 kB\nSwapCached: 12 kB\nActive: 2545520 kB\nInactive: 1424752 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 28655796 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 9488 kB\nWriteback: 5564 kB\nAnonPages: 1655716 kB\nMapped: 17260 kB\nSlab: 200096 kB\nPageTables: 8156 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 3365664 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-02T18:33:12.538,ns_1@10.2.1.67:<0.1687.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,31,46}}) Stats for bucket "default": accepting_conns 1 auth_cmds 3 auth_errors 0 bucket_active_conns 1 bucket_conns 6 bytes 1376174248 bytes_read 1245364367 bytes_written 27363063 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 1 cmd_set 519080 conn_yields 134 connection_structures 5000 curr_connections 10 curr_conns_on_port_11209 6 curr_conns_on_port_11210 2 curr_items 519080 curr_items_tot 519080 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-03 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 680 ep_commit_time 1 ep_commit_time_total 1092 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 8 ep_data_age_highwat 32 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data/default ep_degraded_mode 0 ep_diskqueue_drain 502177 ep_diskqueue_fill 519080 ep_diskqueue_items 16904 ep_diskqueue_memory 540896 ep_diskqueue_pending 811344 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 182 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 245 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 502170 ep_io_read_bytes 0 ep_io_write_bytes 1188806841 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 0 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 1269418560 ep_max_checkpoints 2 ep_max_data_size 5242880000 ep_max_item_size 20971520 ep_max_size 5242880000 ep_max_txn_size 10000 ep_max_vbuckets 512 ep_mem_high_wat 3932160000 ep_mem_high_wat 3932160000 ep_mem_low_wat 3145728000 ep_mem_low_wat 3145728000 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 49072944 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 16659 ep_startup_time 1351905991 ep_storage_age 6 ep_storage_age_highwat 31 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 1263189600 ep_total_del_items 0 ep_total_enqueued 519080 ep_total_new_items 502170 ep_total_persisted 502170 ep_uncommitted_items 495 ep_value_size 1226631465 ep_vb0 0 ep_vb_snapshot_total 6 ep_vb_total 128 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_130_g8b09894 ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 725 get_hits 0 get_misses 1 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 1376174248 pid 16451 pointer_size 64 rejected_conns 0 rusage_system 25.960053 rusage_user 65.997966 threads 4 time 1351906392 total_connections 13 uptime 424 vb_active_curr_items 519080 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 3202048 vb_active_itm_memory 1263189600 vb_active_meta_data_memory 41007320 vb_active_num 128 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 519080 vb_active_ops_create 502176 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 100 vb_active_queue_age 52869000 vb_active_queue_drain 502177 vb_active_queue_fill 519080 vb_active_queue_memory 540896 vb_active_queue_pending 811344 vb_active_queue_size 16903 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_meta_data_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-02T18:34:10.658,ns_1@10.2.1.67:ns_doctor<0.1461.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,906449,43581}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,104897424}, {processes,62473736}, {processes_used,60895720}, {system,42423688}, {atom,1405017}, {atom_used,1377299}, {binary,5530064}, {code,14117455}, {ets,3280680}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,2458312704}, {buffered_memory,207618048}, {free_memory,28648316928}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{657195,5002}}, {context_switches,{3130336,0}}, {garbage_collection,{343630,882017202,0}}, {io,{{input,153281266},{output,37970307}}}, {reductions,{330115994,5032202}}, {run_queue,0}, {runtime,{121180,2350}}]}, {system_stats, [{cpu_utilization_rate,35.57213930348259}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,1820467652}, {couch_docs_data_size,1626110148}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,674874}, {curr_items_tot,674874}, {mem_used,1686918456}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,657}, {memory_data,{33657663488,4250546176,{<18077.5273.0>,7628384}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 27976872 kB\nBuffers: 202752 kB\nCached: 2400696 kB\nSwapCached: 0 kB\nActive: 2800992 kB\nInactive: 1898008 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 27976872 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 1032 kB\nWriteback: 0 kB\nAnonPages: 2095192 kB\nMapped: 15308 kB\nSlab: 152592 kB\nPageTables: 8356 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 3813820 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,906446,687906}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.7807.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,7}, {started_on,1351906407}, {total_vbuckets,129}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351906411}, {vbuckets_done,10}]]}, {memory, [{total,91094136}, {processes,52448152}, {processes_used,52424672}, {system,38645984}, {atom,1352041}, {atom_used,1349412}, {binary,2524904}, {code,13622682}, {ets,3103200}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,2915680256}, {buffered_memory,235208704}, {free_memory,28148424704}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{654440,4995}}, {context_switches,{2951468,0}}, {garbage_collection,{344718,515745211,0}}, {io,{{input,42529310},{output,26461877}}}, {reductions,{225918360,2285029}}, {run_queue,0}, {runtime,{100120,520}}]}, {system_stats, [{cpu_utilization_rate,18.75}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,2347738667}, {couch_docs_data_size,1607783971}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,670941}, {curr_items_tot,670941}, {mem_used,1680565352}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,654}, {memory_data,{33657663488,4526817280,{<18117.1696.0>,2914384}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 27488696 kB\nBuffers: 229696 kB\nCached: 2847344 kB\nSwapCached: 140 kB\nActive: 3490764 kB\nInactive: 1683340 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 27488696 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 2216 kB\nWriteback: 1932 kB\nAnonPages: 2096896 kB\nMapped: 17060 kB\nSlab: 163032 kB\nPageTables: 8992 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 3805556 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,906445,659011}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,90840080}, {processes,52253536}, {processes_used,52185384}, {system,38586544}, {atom,1352041}, {atom_used,1349412}, {binary,2469176}, {code,13631938}, {ets,3086552}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,2828476416}, {buffered_memory,408285184}, {free_memory,28050903040}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{654397,4999}}, {context_switches,{2833332,0}}, {garbage_collection,{320023,503833265,0}}, {io,{{input,40697024},{output,26293859}}}, {reductions,{220835912,2500048}}, {run_queue,0}, {runtime,{98330,700}}]}, {system_stats, [{cpu_utilization_rate,21.608040201005025}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,2143941901}, {couch_docs_data_size,1602776868}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,667837}, {curr_items_tot,667837}, {mem_used,1671173800}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,654}, {memory_data,{33657663488,4586385408,{<0.1691.0>,3143528}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 27393584 kB\nBuffers: 398716 kB\nCached: 2762184 kB\nSwapCached: 0 kB\nActive: 3392228 kB\nInactive: 1848384 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 27393584 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 1644 kB\nWriteback: 64 kB\nAnonPages: 2079876 kB\nMapped: 17056 kB\nSlab: 194872 kB\nPageTables: 8916 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 3790852 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,906449,10528}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,93189288}, {processes,54350016}, {processes_used,54284184}, {system,38839272}, {atom,1352041}, {atom_used,1349412}, {binary,2694112}, {code,13659922}, {ets,3066008}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,2497044480}, {buffered_memory,306470912}, {free_memory,28470321152}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{656431,5001}}, {context_switches,{2934951,0}}, {garbage_collection,{333220,517679229,0}}, {io,{{input,47024751},{output,27639627}}}, {reductions,{227662791,3363414}}, {run_queue,0}, {runtime,{99750,1130}}]}, {system_stats, [{cpu_utilization_rate,31.09452736318408}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,1871561995}, {couch_docs_data_size,1628435600}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,675867}, {curr_items_tot,675867}, {mem_used,1688171912}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,656}, {memory_data,{33657663488,4310065152,{<18118.1662.0>,2914384}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 27803172 kB\nBuffers: 299288 kB\nCached: 2438520 kB\nSwapCached: 12 kB\nActive: 2929596 kB\nInactive: 1894060 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 27803172 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 1412 kB\nWriteback: 0 kB\nAnonPages: 2085752 kB\nMapped: 17256 kB\nSlab: 199492 kB\nPageTables: 8828 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 3788380 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-02T18:34:52.532,ns_1@10.2.1.67:<0.1687.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,31,46}}) Stats for bucket "default": accepting_conns 1 auth_cmds 3 auth_errors 0 bucket_active_conns 1 bucket_conns 6 bytes 2015173672 bytes_read 1918882959 bytes_written 40509560 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 1 cmd_set 799631 conn_yields 210 connection_structures 5000 curr_connections 10 curr_conns_on_port_11209 6 curr_conns_on_port_11210 2 curr_items 799631 curr_items_tot 799631 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-03 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 823 ep_commit_time 2 ep_commit_time_total 2045 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 4 ep_data_age_highwat 32 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data/default ep_degraded_mode 0 ep_diskqueue_drain 798471 ep_diskqueue_fill 799631 ep_diskqueue_items 1161 ep_diskqueue_memory 37120 ep_diskqueue_pending 55680 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 288 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 385 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 798460 ep_io_read_bytes 0 ep_io_write_bytes 1890425999 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 640256 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 1955968821 ep_max_checkpoints 2 ep_max_data_size 5242880000 ep_max_item_size 20971520 ep_max_size 5242880000 ep_max_txn_size 10000 ep_max_vbuckets 512 ep_mem_high_wat 3932160000 ep_mem_high_wat 3932160000 ep_mem_low_wat 3145728000 ep_mem_low_wat 3145728000 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 17688632 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 776 ep_startup_time 1351905991 ep_storage_age 1 ep_storage_age_highwat 31 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 1946373249 ep_total_del_items 0 ep_total_enqueued 799631 ep_total_new_items 798460 ep_total_persisted 798460 ep_uncommitted_items 462 ep_value_size 1890039731 ep_vb0 0 ep_vb_snapshot_total 23 ep_vb_total 128 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_130_g8b09894 ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 725 get_hits 0 get_misses 1 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 2015173672 pid 16451 pointer_size 64 rejected_conns 0 rusage_system 37.657275 rusage_user 98.722991 threads 4 time 1351906492 total_connections 13 uptime 524 vb_active_curr_items 799631 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 3448128 vb_active_itm_memory 1946373249 vb_active_meta_data_memory 63170849 vb_active_num 128 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 799631 vb_active_ops_create 798470 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 100 vb_active_queue_age 268000 vb_active_queue_drain 798471 vb_active_queue_fill 799631 vb_active_queue_memory 37120 vb_active_queue_pending 55680 vb_active_queue_size 1160 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_meta_data_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-02T18:35:10.658,ns_1@10.2.1.67:ns_doctor<0.1461.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,906509,40021}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,105668104}, {processes,62663600}, {processes_used,61085264}, {system,43004504}, {atom,1405017}, {atom_used,1377299}, {binary,5936168}, {code,14117455}, {ets,3458192}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,3157831680}, {buffered_memory,208404480}, {free_memory,27503882240}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{717193,4999}}, {context_switches,{3571931,0}}, {garbage_collection,{388012,993615038,0}}, {io,{{input,174166055},{output,44503687}}}, {reductions,{372923660,3297983}}, {run_queue,0}, {runtime,{135570,1100}}]}, {system_stats, [{cpu_utilization_rate,33.0}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,2517116209}, {couch_docs_data_size,2036319277}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,841835}, {curr_items_tot,841835}, {mem_used,2126929448}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,717}, {memory_data,{33657663488,5070548992,{<18077.5273.0>,7628384}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 26859260 kB\nBuffers: 203520 kB\nCached: 3083820 kB\nSwapCached: 0 kB\nActive: 3625316 kB\nInactive: 2174072 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 26859260 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 1424 kB\nWriteback: 0 kB\nAnonPages: 2511892 kB\nMapped: 15312 kB\nSlab: 169528 kB\nPageTables: 9168 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 4228532 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,906506,688048}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,91277384}, {processes,52517832}, {processes_used,52452000}, {system,38759552}, {atom,1352041}, {atom_used,1349412}, {binary,2460432}, {code,13622682}, {ets,3276224}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,2783997952}, {buffered_memory,237871104}, {free_memory,27867308032}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{714440,4999}}, {context_switches,{3225871,0}}, {garbage_collection,{375325,581446691,0}}, {io,{{input,55130645},{output,33019980}}}, {reductions,{257459475,4513275}}, {run_queue,0}, {runtime,{109390,2070}}]}, {system_stats, [{cpu_utilization_rate,34.66334164588529}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,2115622180}, {couch_docs_data_size,2005383166}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,839068}, {curr_items_tot,839068}, {mem_used,2120107720}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,714}, {memory_data,{33657663488,5473685504,{<18117.1696.0>,2914384}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 27214168 kB\nBuffers: 232296 kB\nCached: 2718748 kB\nSwapCached: 140 kB\nActive: 3115276 kB\nInactive: 2336912 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 27214168 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 1644 kB\nWriteback: 0 kB\nAnonPages: 2500548 kB\nMapped: 17056 kB\nSlab: 159060 kB\nPageTables: 9716 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 4208604 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,906505,658751}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,96227000}, {processes,57285720}, {processes_used,57217568}, {system,38941280}, {atom,1352041}, {atom_used,1349412}, {binary,2645936}, {code,13631938}, {ets,3261768}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,3546804224}, {buffered_memory,409018368}, {free_memory,26907484160}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{714397,5000}}, {context_switches,{3200399,0}}, {garbage_collection,{357775,575021618,0}}, {io,{{input,42878727},{output,28678712}}}, {reductions,{254633162,2164502}}, {run_queue,0}, {runtime,{108410,500}}]}, {system_stats, [{cpu_utilization_rate,25.81453634085213}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,2841899550}, {couch_docs_data_size,2008213149}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,835926}, {curr_items_tot,835926}, {mem_used,2111176424}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,714}, {memory_data,{33657663488,5682946048,{<0.1691.0>,3143528}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 26276840 kB\nBuffers: 399432 kB\nCached: 3463348 kB\nSwapCached: 0 kB\nActive: 4212652 kB\nInactive: 2143772 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 26276840 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 856 kB\nWriteback: 0 kB\nAnonPages: 2493528 kB\nMapped: 17056 kB\nSlab: 194880 kB\nPageTables: 9728 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 4206644 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,906509,8641}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,95526992}, {processes,56331768}, {processes_used,56265616}, {system,39195224}, {atom,1352041}, {atom_used,1349412}, {binary,2875792}, {code,13659922}, {ets,3240992}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,3228778496}, {buffered_memory,307167232}, {free_memory,27311157248}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{716429,4997}}, {context_switches,{3425854,0}}, {garbage_collection,{383378,599728390,0}}, {io,{{input,49813731},{output,30485787}}}, {reductions,{265875832,2220614}}, {run_queue,0}, {runtime,{112180,540}}]}, {system_stats, [{cpu_utilization_rate,11.779448621553884}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,2595550491}, {couch_docs_data_size,2025072893}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,844299}, {curr_items_tot,844299}, {mem_used,2132778712}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,716}, {memory_data,{33657663488,5251465216,{<18118.1662.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 26671176 kB\nBuffers: 299968 kB\nCached: 3153104 kB\nSwapCached: 12 kB\nActive: 3800472 kB\nInactive: 2153832 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 26671176 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 1432 kB\nWriteback: 0 kB\nAnonPages: 2501236 kB\nMapped: 17256 kB\nSlab: 199424 kB\nPageTables: 9620 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 4209896 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-02T18:36:10.658,ns_1@10.2.1.67:ns_doctor<0.1461.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,906569,39207}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,105609488}, {processes,63273504}, {processes_used,61695168}, {system,42335984}, {atom,1405017}, {atom_used,1377299}, {binary,5191864}, {code,14117455}, {ets,3532584}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,3851108352}, {buffered_memory,209367040}, {free_memory,26367315968}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{777193,5000}}, {context_switches,{3876159,0}}, {garbage_collection,{418284,1093684064,0}}, {io,{{input,190994349},{output,47294295}}}, {reductions,{410830584,3221020}}, {run_queue,0}, {runtime,{146400,1050}}]}, {system_stats, [{cpu_utilization_rate,32.16957605985037}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,3216099385}, {couch_docs_data_size,2452306579}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1009588}, {curr_items_tot,1009588}, {mem_used,2572889480}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,777}, {memory_data,{33657663488,6174351360,{<18077.5273.0>,7628384}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 25749332 kB\nBuffers: 204460 kB\nCached: 3760848 kB\nSwapCached: 0 kB\nActive: 4406736 kB\nInactive: 2485244 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 25749332 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 1824 kB\nWriteback: 0 kB\nAnonPages: 2926580 kB\nMapped: 15312 kB\nSlab: 186328 kB\nPageTables: 9808 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 4644576 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,906566,687416}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,91562600}, {processes,52636632}, {processes_used,52570400}, {system,38925968}, {atom,1352041}, {atom_used,1349412}, {binary,2556776}, {code,13622682}, {ets,3349448}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,3448061952}, {buffered_memory,238682112}, {free_memory,26765406208}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{774439,4998}}, {context_switches,{3799479,0}}, {garbage_collection,{435509,670779253,0}}, {io,{{input,58265271},{output,36134494}}}, {reductions,{298688873,2151839}}, {run_queue,0}, {runtime,{124810,550}}]}, {system_stats, [{cpu_utilization_rate,6.75}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,2821369049}, {couch_docs_data_size,2408329164}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1006752}, {curr_items_tot,1006752}, {mem_used,2565407912}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,774}, {memory_data,{33657663488,5915680768,{<18117.1697.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 26138092 kB\nBuffers: 233088 kB\nCached: 3367248 kB\nSwapCached: 140 kB\nActive: 3945560 kB\nInactive: 2565244 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 26138092 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 2104 kB\nWriteback: 0 kB\nAnonPages: 2909976 kB\nMapped: 17056 kB\nSlab: 175048 kB\nPageTables: 10520 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 4617640 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,906565,660419}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.9566.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,48}, {started_on,1351906515}, {total_vbuckets,129}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351906564}, {vbuckets_done,63}]]}, {memory, [{total,95268888}, {processes,56120600}, {processes_used,56068480}, {system,39148288}, {atom,1352041}, {atom_used,1349412}, {binary,2659984}, {code,13631938}, {ets,3451216}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,3363885056}, {buffered_memory,410542080}, {free_memory,26653777920}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{774398,5000}}, {context_switches,{3317752,0}}, {garbage_collection,{370765,626618863,0}}, {io,{{input,57419219},{output,35204999}}}, {reductions,{280500718,1910089}}, {run_queue,0}, {runtime,{114430,400}}]}, {system_stats, [{cpu_utilization_rate,10.275689223057643}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,2752544355}, {couch_docs_data_size,2202462458}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1003668}, {curr_items_tot,1003668}, {mem_used,2557588968}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,774}, {memory_data,{33657663488,6813667328,{<0.1681.0>,4714984}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 26029080 kB\nBuffers: 400920 kB\nCached: 3285044 kB\nSwapCached: 0 kB\nActive: 4117040 kB\nInactive: 2482480 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 26029080 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 2704 kB\nWriteback: 0 kB\nAnonPages: 2913408 kB\nMapped: 17060 kB\nSlab: 197544 kB\nPageTables: 10620 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 4631024 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,906569,9722}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,93353400}, {processes,54166200}, {processes_used,54100448}, {system,39187200}, {atom,1352041}, {atom_used,1349412}, {binary,2690232}, {code,13659922}, {ets,3417704}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,4030382080}, {buffered_memory,308408320}, {free_memory,26074411008}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{776431,5001}}, {context_switches,{3908065,0}}, {garbage_collection,{432550,681715369,0}}, {io,{{input,55989216},{output,36550056}}}, {reductions,{303857268,2514700}}, {run_queue,0}, {runtime,{123160,630}}]}, {system_stats, [{cpu_utilization_rate,26.75}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,3381298450}, {couch_docs_data_size,2446134273}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1011942}, {curr_items_tot,1011942}, {mem_used,2577690248}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,776}, {memory_data,{33657663488,6442881024,{<18118.1662.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 25463292 kB\nBuffers: 301180 kB\nCached: 3935920 kB\nSwapCached: 12 kB\nActive: 4732084 kB\nInactive: 2420276 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 25463292 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 2248 kB\nWriteback: 0 kB\nAnonPages: 2915080 kB\nMapped: 17256 kB\nSlab: 208220 kB\nPageTables: 10612 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 4620940 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-02T18:36:32.531,ns_1@10.2.1.67:<0.1687.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,31,46}}) Stats for bucket "default": accepting_conns 1 auth_cmds 3 auth_errors 0 bucket_active_conns 1 bucket_conns 6 bytes 2757927864 bytes_read 2591582805 bytes_written 53665889 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 1 cmd_set 1079708 conn_yields 287 connection_structures 5000 curr_connections 10 curr_conns_on_port_11209 6 curr_conns_on_port_11210 2 curr_items 1079708 curr_items_tot 1079708 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-03 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 858 ep_commit_time 188 ep_commit_time_total 3132 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 30 ep_data_age_highwat 71 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data/default ep_degraded_mode 0 ep_diskqueue_drain 945496 ep_diskqueue_fill 1079708 ep_diskqueue_items 134213 ep_diskqueue_memory 4294784 ep_diskqueue_pending 6442176 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 381 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 121595 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 944501 ep_io_read_bytes 0 ep_io_write_bytes 2237140281 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 640256 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 2641680132 ep_max_checkpoints 2 ep_max_data_size 5242880000 ep_max_item_size 20971520 ep_max_size 5242880000 ep_max_txn_size 10000 ep_max_vbuckets 512 ep_mem_high_wat 3932160000 ep_mem_high_wat 3932160000 ep_mem_low_wat 3145728000 ep_mem_low_wat 3145728000 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 46208832 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 12618 ep_startup_time 1351905991 ep_storage_age 28 ep_storage_age_highwat 68 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 2628723636 ep_total_del_items 0 ep_total_enqueued 1079708 ep_total_new_items 944501 ep_total_persisted 944501 ep_uncommitted_items 2065 ep_value_size 2552646315 ep_vb0 0 ep_vb_snapshot_total 23 ep_vb_total 128 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_130_g8b09894 ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 725 get_hits 0 get_misses 1 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 2757927864 pid 16451 pointer_size 64 rejected_conns 0 rusage_system 45.427094 rusage_user 115.108500 threads 4 time 1351906592 total_connections 13 uptime 624 vb_active_curr_items 1079708 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 6351872 vb_active_itm_memory 2628723636 vb_active_meta_data_memory 85296932 vb_active_num 128 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1079708 vb_active_ops_create 945495 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 100 vb_active_queue_age 3509426000 vb_active_queue_drain 945496 vb_active_queue_fill 1079708 vb_active_queue_memory 4294784 vb_active_queue_pending 6442176 vb_active_queue_size 134212 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_meta_data_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-02T18:37:10.658,ns_1@10.2.1.67:ns_doctor<0.1461.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,906629,38365}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,108895856}, {processes,65298344}, {processes_used,63720008}, {system,43597512}, {atom,1405017}, {atom_used,1377299}, {binary,6278048}, {code,14117455}, {ets,3707272}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,4475764736}, {buffered_memory,210239488}, {free_memory,25268850688}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{837193,4999}}, {context_switches,{4081931,0}}, {garbage_collection,{438473,1184544055,0}}, {io,{{input,207347229},{output,49699268}}}, {reductions,{445091343,2575625}}, {run_queue,0}, {runtime,{155380,780}}]}, {system_stats, [{cpu_utilization_rate,7.75}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,3858224808}, {couch_docs_data_size,2847483067}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1177008}, {curr_items_tot,1177008}, {mem_used,3016925432}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,837}, {memory_data,{33657663488,7312441344,{<18077.5273.0>,7628384}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 24676612 kB\nBuffers: 205312 kB\nCached: 4370864 kB\nSwapCached: 0 kB\nActive: 5149416 kB\nInactive: 2799104 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 24676612 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 1640 kB\nWriteback: 0 kB\nAnonPages: 3372368 kB\nMapped: 15312 kB\nSlab: 201640 kB\nPageTables: 10884 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 5098640 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,906626,687746}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,91050184}, {processes,52012936}, {processes_used,51946704}, {system,39037248}, {atom,1352041}, {atom_used,1349412}, {binary,2493408}, {code,13622682}, {ets,3524360}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,4130795520}, {buffered_memory,239333376}, {free_memory,25609797632}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{834439,4998}}, {context_switches,{4103388,0}}, {garbage_collection,{466566,736474886,0}}, {io,{{input,60146050},{output,38347756}}}, {reductions,{330063643,2089479}}, {run_queue,0}, {runtime,{133830,630}}]}, {system_stats, [{cpu_utilization_rate,8.0}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,3514503498}, {couch_docs_data_size,2825678733}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1174628}, {curr_items_tot,1174628}, {mem_used,3010725416}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,834}, {memory_data,{33657663488,7018852352,{<18117.1696.0>,2914384}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 25009568 kB\nBuffers: 233724 kB\nCached: 4033980 kB\nSwapCached: 140 kB\nActive: 4746388 kB\nInactive: 2875340 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 25009568 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 876 kB\nWriteback: 340 kB\nAnonPages: 3353340 kB\nMapped: 17056 kB\nSlab: 191660 kB\nPageTables: 11276 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 5060200 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,906625,659160}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,94194320}, {processes,55115768}, {processes_used,55029056}, {system,39078552}, {atom,1352041}, {atom_used,1349412}, {binary,2531840}, {code,13631938}, {ets,3508536}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,3615842304}, {buffered_memory,412389376}, {free_memory,25956433920}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{834397,5000}}, {context_switches,{3429825,0}}, {garbage_collection,{383264,676399699,0}}, {io,{{input,59096942},{output,36801652}}}, {reductions,{305458940,3116793}}, {run_queue,0}, {runtime,{120210,1140}}]}, {system_stats, [{cpu_utilization_rate,11.027568922305765}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,2940159542}, {couch_docs_data_size,2818839411}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1172602}, {curr_items_tot,1172602}, {mem_used,3004404792}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,834}, {memory_data,{33657663488,7006171136,{<0.1681.0>,4714984}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 25348080 kB\nBuffers: 402724 kB\nCached: 3531096 kB\nSwapCached: 0 kB\nActive: 4061244 kB\nInactive: 3221076 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 25348080 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 2376 kB\nWriteback: 0 kB\nAnonPages: 3348612 kB\nMapped: 17056 kB\nSlab: 195984 kB\nPageTables: 11416 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 5060872 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,906629,9135}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.10121.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,4}, {started_on,1351906614}, {total_vbuckets,129}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351906619}, {vbuckets_done,6}]]}, {memory, [{total,95520704}, {processes,56120728}, {processes_used,56061648}, {system,39399976}, {atom,1352041}, {atom_used,1349412}, {binary,2831496}, {code,13659922}, {ets,3488504}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,4639662080}, {buffered_memory,309231616}, {free_memory,24982798336}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{836431,5001}}, {context_switches,{4191158,0}}, {garbage_collection,{461154,745940398,0}}, {io,{{input,60458461},{output,39046816}}}, {reductions,{334758021,2275342}}, {run_queue,0}, {runtime,{132510,470}}]}, {system_stats, [{cpu_utilization_rate,6.265664160401003}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,4031593191}, {couch_docs_data_size,2833660289}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1179419}, {curr_items_tot,1179419}, {mem_used,3022992360}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,836}, {memory_data,{33657663488,7659057152,{<18118.1662.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 24397264 kB\nBuffers: 301984 kB\nCached: 4530920 kB\nSwapCached: 12 kB\nActive: 5486084 kB\nInactive: 2714596 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 24397264 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 18184 kB\nWriteback: 0 kB\nAnonPages: 3367668 kB\nMapped: 17252 kB\nSlab: 225052 kB\nPageTables: 11324 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 5069312 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-02T18:38:10.659,ns_1@10.2.1.67:ns_doctor<0.1461.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,906689,37252}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,105316432}, {processes,61820184}, {processes_used,60242168}, {system,43496248}, {atom,1405017}, {atom_used,1377299}, {binary,5998752}, {code,14117455}, {ets,3884864}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,5230149632}, {buffered_memory,211046400}, {free_memory,24062574592}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{897193,5001}}, {context_switches,{4420594,0}}, {garbage_collection,{472522,1286896070,0}}, {io,{{input,227766125},{output,54698407}}}, {reductions,{484267470,3143159}}, {run_queue,0}, {runtime,{167050,1100}}]}, {system_stats, [{cpu_utilization_rate,7.769423558897243}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,4616803385}, {couch_docs_data_size,3277019439}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1345417}, {curr_items_tot,1345417}, {mem_used,3359502584}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,897}, {memory_data,{33657663488,8482521088,{<18077.5273.0>,7628384}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 23498608 kB\nBuffers: 206100 kB\nCached: 5107568 kB\nSwapCached: 0 kB\nActive: 5991824 kB\nInactive: 3115348 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 23498608 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 1836 kB\nWriteback: 0 kB\nAnonPages: 3792848 kB\nMapped: 15312 kB\nSlab: 220212 kB\nPageTables: 11652 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 5509332 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,906686,688767}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,92167008}, {processes,53049192}, {processes_used,52982960}, {system,39117816}, {atom,1352041}, {atom_used,1349412}, {binary,2396072}, {code,13622682}, {ets,3701448}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,4800794624}, {buffered_memory,240099328}, {free_memory,24486813696}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{894440,4997}}, {context_switches,{4344725,0}}, {garbage_collection,{491471,797205699,0}}, {io,{{input,65203983},{output,42294740}}}, {reductions,{359215147,1909161}}, {run_queue,0}, {runtime,{142180,540}}]}, {system_stats, [{cpu_utilization_rate,7.035175879396985}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,4190409223}, {couch_docs_data_size,3230577277}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1341627}, {curr_items_tot,1341627}, {mem_used,3349953608}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,894}, {memory_data,{33657663488,8186269696,{<18117.1696.0>,2914384}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 23912904 kB\nBuffers: 234472 kB\nCached: 4688276 kB\nSwapCached: 140 kB\nActive: 5513348 kB\nInactive: 3187996 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 23912904 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 704 kB\nWriteback: 0 kB\nAnonPages: 3778076 kB\nMapped: 17056 kB\nSlab: 208228 kB\nPageTables: 12200 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 5489220 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,906685,660127}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,93729920}, {processes,54406512}, {processes_used,54319880}, {system,39323408}, {atom,1352041}, {atom_used,1349412}, {binary,2603216}, {code,13631938}, {ets,3683464}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,4351385600}, {buffered_memory,413081600}, {free_memory,24767303680}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{894398,5001}}, {context_switches,{3939441,0}}, {garbage_collection,{436126,760481335,0}}, {io,{{input,61952736},{output,39686227}}}, {reductions,{344418338,2084967}}, {run_queue,0}, {runtime,{133150,470}}]}, {system_stats, [{cpu_utilization_rate,25.6857855361596}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,3636785285}, {couch_docs_data_size,3209253425}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1340758}, {curr_items_tot,1340758}, {mem_used,3346597192}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,894}, {memory_data,{33657663488,7834808320,{<0.1681.0>,4714984}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 24186944 kB\nBuffers: 403400 kB\nCached: 4249400 kB\nSwapCached: 0 kB\nActive: 4925452 kB\nInactive: 3499544 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 24186944 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 904 kB\nWriteback: 0 kB\nAnonPages: 3772452 kB\nMapped: 17056 kB\nSlab: 213540 kB\nPageTables: 12252 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 5482180 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,906689,6785}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.10121.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,82}, {started_on,1351906614}, {total_vbuckets,129}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351906681}, {vbuckets_done,106}]]}, {memory, [{total,91945560}, {processes,52831584}, {processes_used,52817584}, {system,39113976}, {atom,1352041}, {atom_used,1349412}, {binary,2315800}, {code,13659922}, {ets,3711456}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,3864657920}, {buffered_memory,311836672}, {free_memory,25315221504}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{896429,5000}}, {context_switches,{4268029,0}}, {garbage_collection,{470326,793537362,0}}, {io,{{input,70374751},{output,42122761}}}, {reductions,{359082185,1951151}}, {run_queue,0}, {runtime,{137610,410}}]}, {system_stats, [{cpu_utilization_rate,24.87437185929648}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,3313963455}, {couch_docs_data_size,3002205889}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1347399}, {curr_items_tot,1347399}, {mem_used,3465372248}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,896}, {memory_data,{33657663488,8701911040,{<18118.1662.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 24721896 kB\nBuffers: 304528 kB\nCached: 3774080 kB\nSwapCached: 12 kB\nActive: 4602108 kB\nInactive: 3289620 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 24721896 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 12464 kB\nWriteback: 2052 kB\nAnonPages: 3813404 kB\nMapped: 17236 kB\nSlab: 208612 kB\nPageTables: 12488 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 5516412 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-02T18:38:12.529,ns_1@10.2.1.67:<0.1687.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,31,46}}) Stats for bucket "default": accepting_conns 1 auth_cmds 3 auth_errors 0 bucket_active_conns 1 bucket_conns 6 bytes 3397712040 bytes_read 3264389366 bytes_written 66860922 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 1 cmd_set 1360210 conn_yields 344 connection_structures 5000 curr_connections 10 curr_conns_on_port_11209 6 curr_conns_on_port_11210 2 curr_items 1360210 curr_items_tot 1360210 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-03 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 1033 ep_commit_time 2 ep_commit_time_total 4995 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 8 ep_data_age_highwat 71 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data/default ep_degraded_mode 0 ep_diskqueue_drain 1344390 ep_diskqueue_fill 1360210 ep_diskqueue_items 15820 ep_diskqueue_memory 506240 ep_diskqueue_pending 759360 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 479 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 34 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 1344385 ep_io_read_bytes 0 ep_io_write_bytes 3183305013 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 1280512 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 3327518374 ep_max_checkpoints 2 ep_max_data_size 5242880000 ep_max_item_size 20971520 ep_max_size 5242880000 ep_max_txn_size 10000 ep_max_vbuckets 512 ep_mem_high_wat 3932160000 ep_mem_high_wat 3932160000 ep_mem_low_wat 3145728000 ep_mem_low_wat 3145728000 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 13600520 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 15786 ep_startup_time 1351905991 ep_storage_age 6 ep_storage_age_highwat 68 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 3311195854 ep_total_del_items 0 ep_total_enqueued 1360210 ep_total_new_items 1344385 ep_total_persisted 1344385 ep_uncommitted_items 639 ep_value_size 3215344314 ep_vb0 0 ep_vb_snapshot_total 54 ep_vb_total 128 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_130_g8b09894 ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 725 get_hits 0 get_misses 1 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 3397712040 pid 16451 pointer_size 64 rejected_conns 0 rusage_system 57.561249 rusage_user 150.419132 threads 4 time 1351906692 total_connections 13 uptime 724 vb_active_curr_items 1360210 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 6351872 vb_active_itm_memory 3311195854 vb_active_meta_data_memory 107456590 vb_active_num 128 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1360210 vb_active_ops_create 1344389 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 100 vb_active_queue_age 50370000 vb_active_queue_drain 1344390 vb_active_queue_fill 1360210 vb_active_queue_memory 506240 vb_active_queue_pending 759360 vb_active_queue_size 15820 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_meta_data_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [stats:warn,2012-11-02T18:38:59.866,ns_1@10.2.1.67:system_stats_collector<0.1530.0>:system_stats_collector:handle_info:133]lost 2 ticks [stats:warn,2012-11-02T18:38:59.870,ns_1@10.2.1.67:<0.1687.0>:stats_collector:latest_tick:223]Dropped 2 ticks [ns_doctor:debug,2012-11-02T18:39:10.658,ns_1@10.2.1.67:ns_doctor<0.1461.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,906749,36799}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.17519.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,17}, {started_on,1351906753}, {total_vbuckets,129}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351906780}, {vbuckets_done,22}]]}, {memory, [{total,114886424}, {processes,69872856}, {processes_used,68309032}, {system,45013568}, {atom,1405017}, {atom_used,1377299}, {binary,7328560}, {code,14117455}, {ets,4069520}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,5447905280}, {buffered_memory,212451328}, {free_memory,23402930176}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{957193,5000}}, {context_switches,{4616104,0}}, {garbage_collection,{492508,1378225195,0}}, {io,{{input,249339112},{output,58010556}}}, {reductions,{518880713,2352898}}, {run_queue,0}, {runtime,{176180,570}}]}, {system_stats, [{cpu_utilization_rate,28.82205513784461}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,4880687966}, {couch_docs_data_size,3532730429}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1512645}, {curr_items_tot,1512645}, {mem_used,3803190376}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,957}, {memory_data,{33657663488,9692225536,{<18077.14785.0>,7028184}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 22854424 kB\nBuffers: 207472 kB\nCached: 5320220 kB\nSwapCached: 0 kB\nActive: 6327208 kB\nInactive: 3415852 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 22854424 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 16872 kB\nWriteback: 9328 kB\nAnonPages: 4215388 kB\nMapped: 15316 kB\nSlab: 227856 kB\nPageTables: 12452 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 5938436 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,906746,688214}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,92569016}, {processes,53207896}, {processes_used,53142496}, {system,39361120}, {atom,1352041}, {atom_used,1349412}, {binary,2461752}, {code,13622682}, {ets,3876552}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,5592358912}, {buffered_memory,241270784}, {free_memory,23247032320}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{954440,4999}}, {context_switches,{4643075,0}}, {garbage_collection,{522234,862402913,0}}, {io,{{input,67021429},{output,44444571}}}, {reductions,{390428828,2371582}}, {run_queue,0}, {runtime,{151600,660}}]}, {system_stats, [{cpu_utilization_rate,27.68079800498753}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,4910250299}, {couch_docs_data_size,3644737069}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1509623}, {curr_items_tot,1509623}, {mem_used,3794316840}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,954}, {memory_data,{33657663488,9314197504,{<18117.1697.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 22702180 kB\nBuffers: 235616 kB\nCached: 5461288 kB\nSwapCached: 140 kB\nActive: 6353724 kB\nInactive: 3538684 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 22702180 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 2012 kB\nWriteback: 0 kB\nAnonPages: 4195584 kB\nMapped: 17056 kB\nSlab: 227328 kB\nPageTables: 13008 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 5902228 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,906745,659500}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,91625736}, {processes,52086176}, {processes_used,51999304}, {system,39539560}, {atom,1352041}, {atom_used,1349412}, {binary,2642224}, {code,13631938}, {ets,3860680}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,4997361664}, {buffered_memory,414130176}, {free_memory,23674802176}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{954397,5000}}, {context_switches,{4154263,0}}, {garbage_collection,{458038,818908285,0}}, {io,{{input,66868902},{output,43599400}}}, {reductions,{372878042,2506518}}, {run_queue,0}, {runtime,{140840,690}}]}, {system_stats, [{cpu_utilization_rate,6.5},{swap_total,10701078528},{swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,4311299405}, {couch_docs_data_size,3634159074}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1509187}, {curr_items_tot,1509187}, {mem_used,3791952344}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,954}, {memory_data,{33657663488,8953602048,{<0.1678.0>,4714984}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 23119924 kB\nBuffers: 404424 kB\nCached: 4880236 kB\nSwapCached: 0 kB\nActive: 5658768 kB\nInactive: 3816724 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 23119924 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 1632 kB\nWriteback: 0 kB\nAnonPages: 4190784 kB\nMapped: 17056 kB\nSlab: 229228 kB\nPageTables: 13044 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 5897708 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,906749,7748}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,92438560}, {processes,53010096}, {processes_used,52923224}, {system,39428464}, {atom,1352041}, {atom_used,1349412}, {binary,2508752}, {code,13659922}, {ets,3838352}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,4520988672}, {buffered_memory,313589760}, {free_memory,24229842944}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{956430,5000}}, {context_switches,{4533237,0}}, {garbage_collection,{498880,856747680,0}}, {io,{{input,75749609},{output,46214007}}}, {reductions,{389425754,3030565}}, {run_queue,0}, {runtime,{145800,850}}]}, {system_stats, [{cpu_utilization_rate,20.5}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,3906782544}, {couch_docs_data_size,3663194596}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1515347}, {curr_items_tot,1515347}, {mem_used,3808504984}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,956}, {memory_data,{33657663488,8345362432,{<18118.1662.0>,2914384}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 23661956 kB\nBuffers: 306240 kB\nCached: 4415028 kB\nSwapCached: 12 kB\nActive: 5034620 kB\nInactive: 3903760 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 23661956 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 5264 kB\nWriteback: 0 kB\nAnonPages: 4216700 kB\nMapped: 17256 kB\nSlab: 220984 kB\nPageTables: 13044 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 5915876 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:warn,2012-11-02T18:39:57.397,ns_1@10.2.1.67:system_stats_collector<0.1530.0>:system_stats_collector:handle_info:133]lost 2 ticks [stats:warn,2012-11-02T18:39:57.401,ns_1@10.2.1.67:<0.1687.0>:stats_collector:latest_tick:223]Dropped 2 ticks [stats:debug,2012-11-02T18:39:57.402,ns_1@10.2.1.67:<0.1687.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,31,46}}) Stats for bucket "default": accepting_conns 1 auth_cmds 3 auth_errors 0 bucket_active_conns 1 bucket_conns 6 bytes 3666186168 bytes_read 3971968297 bytes_written 80649108 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 1 cmd_set 1654768 conn_yields 403 connection_structures 5000 curr_connections 10 curr_conns_on_port_11209 6 curr_conns_on_port_11210 2 curr_items 1654768 curr_items_tot 1654768 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-03 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 1127 ep_commit_time 3 ep_commit_time_total 5836 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 0 ep_data_age_highwat 71 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data/default ep_degraded_mode 0 ep_diskqueue_drain 1653996 ep_diskqueue_fill 1654768 ep_diskqueue_items 773 ep_diskqueue_memory 24704 ep_diskqueue_pending 37056 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 589 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 724 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 1653985 ep_io_read_bytes 0 ep_io_write_bytes 3917059592 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 1585333 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 3587045176 ep_max_checkpoints 2 ep_max_data_size 5242880000 ep_max_item_size 20971520 ep_max_size 5242880000 ep_max_txn_size 10000 ep_max_vbuckets 512 ep_mem_high_wat 3932160000 ep_mem_high_wat 3932160000 ep_mem_low_wat 3145728000 ep_mem_low_wat 3145728000 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 299 ep_num_expiry_pager_runs 0 ep_num_non_resident 196869 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 2 ep_num_value_ejects 196869 ep_oom_errors 0 ep_overhead 19039168 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 49 ep_startup_time 1351905991 ep_storage_age 0 ep_storage_age_highwat 68 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 3567187960 ep_total_del_items 0 ep_total_enqueued 1654768 ep_total_new_items 1653985 ep_total_persisted 1653985 ep_uncommitted_items 71 ep_value_size 3450469481 ep_vb0 0 ep_vb_snapshot_total 56 ep_vb_total 128 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_130_g8b09894 ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 725 get_hits 0 get_misses 1 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 3666186168 pid 16451 pointer_size 64 rejected_conns 0 rusage_system 68.903525 rusage_user 182.253293 threads 4 time 1351906796 total_connections 13 uptime 828 vb_active_curr_items 1654768 vb_active_eject 196869 vb_active_expired 0 vb_active_ht_memory 12645376 vb_active_itm_memory 3567187960 vb_active_meta_data_memory 130726672 vb_active_num 128 vb_active_num_non_resident 196869 vb_active_num_ref_ejects 196869 vb_active_num_ref_items 1654768 vb_active_ops_create 1653995 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 88 vb_active_queue_age 0 vb_active_queue_drain 1653996 vb_active_queue_fill 1654768 vb_active_queue_memory 24704 vb_active_queue_pending 37056 vb_active_queue_size 772 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_meta_data_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-02T18:40:10.658,ns_1@10.2.1.67:ns_doctor<0.1461.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,906809,38286}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.17519.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,55}, {started_on,1351906753}, {total_vbuckets,129}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351906841}, {vbuckets_done,72}]]}, {memory, [{total,113039152}, {processes,68353344}, {processes_used,66803256}, {system,44685808}, {atom,1405017}, {atom_used,1377299}, {binary,6921360}, {code,14117455}, {ets,4143472}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,5010784256}, {buffered_memory,214183936}, {free_memory,23639732224}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1017196,5003}}, {context_switches,{4680061,0}}, {garbage_collection,{499327,1454690964,0}}, {io,{{input,264862921},{output,60044548}}}, {reductions,{548063068,2361863}}, {run_queue,0}, {runtime,{182610,550}}]}, {system_stats, [{cpu_utilization_rate,12.219451371571072}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,4435449515}, {couch_docs_data_size,3673066024}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1680327}, {curr_items_tot,1680327}, {mem_used,3758885160}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1017}, {memory_data,{33657663488,10263494656,{<18077.14785.0>,7028184}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 23082328 kB\nBuffers: 209164 kB\nCached: 4896624 kB\nSwapCached: 0 kB\nActive: 5748100 kB\nInactive: 3775980 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 23082328 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 3476 kB\nWriteback: 0 kB\nAnonPages: 4418164 kB\nMapped: 15300 kB\nSlab: 217128 kB\nPageTables: 12836 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6145164 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,906806,689134}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,94154744}, {processes,54808936}, {processes_used,54742704}, {system,39345808}, {atom,1352041}, {atom_used,1349412}, {binary,2392760}, {code,13622682}, {ets,3932056}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,6290292736}, {buffered_memory,241930240}, {free_memory,22303346688}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1014440,4999}}, {context_switches,{4890103,0}}, {garbage_collection,{547528,922842288,0}}, {io,{{input,68730794},{output,46451881}}}, {reductions,{419668508,2453360}}, {run_queue,0}, {runtime,{159800,640}}]}, {system_stats, [{cpu_utilization_rate,18.75}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,5581823660}, {couch_docs_data_size,4029838054}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1674124}, {curr_items_tot,1674124}, {mem_used,3721970712}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1014}, {memory_data,{33657663488,10520338432,{<18117.1697.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 21780612 kB\nBuffers: 236260 kB\nCached: 6142864 kB\nSwapCached: 140 kB\nActive: 6939524 kB\nInactive: 3856724 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 21780612 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 1332 kB\nWriteback: 4 kB\nAnonPages: 4416824 kB\nMapped: 17060 kB\nSlab: 244500 kB\nPageTables: 13480 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6123376 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,906805,658424}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,91201408}, {processes,51440832}, {processes_used,51354200}, {system,39760576}, {atom,1352041}, {atom_used,1349412}, {binary,2689016}, {code,13631938}, {ets,4034648}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,5705949184}, {buffered_memory,414875648}, {free_memory,22717276160}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1014397,5000}}, {context_switches,{4386679,0}}, {garbage_collection,{481888,878394272,0}}, {io,{{input,68466246},{output,45534554}}}, {reductions,{401929663,2903894}}, {run_queue,0}, {runtime,{148250,860}}]}, {system_stats, [{cpu_utilization_rate,6.75}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,5015165582}, {couch_docs_data_size,4057888084}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1677557}, {curr_items_tot,1677557}, {mem_used,3726175576}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1014}, {memory_data,{33657663488,10081521664,{<0.1678.0>,2914384}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 22184840 kB\nBuffers: 405152 kB\nCached: 5572216 kB\nSwapCached: 0 kB\nActive: 6235572 kB\nInactive: 4157452 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 22184840 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 2012 kB\nWriteback: 0 kB\nAnonPages: 4415768 kB\nMapped: 17060 kB\nSlab: 246520 kB\nPageTables: 13492 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 6126572 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,906809,7010}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,93386376}, {processes,53815224}, {processes_used,53728352}, {system,39571152}, {atom,1352041}, {atom_used,1349412}, {binary,2478224}, {code,13659922}, {ets,4012720}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,5182926848}, {buffered_memory,314220544}, {free_memory,23345197056}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1016430,4998}}, {context_switches,{4850939,0}}, {garbage_collection,{532001,923565186,0}}, {io,{{input,77747382},{output,48453873}}}, {reductions,{421290461,2832563}}, {run_queue,0}, {runtime,{155080,880}}]}, {system_stats, [{cpu_utilization_rate,7.518796992481203}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,4574119265}, {couch_docs_data_size,4071943922}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1683847}, {curr_items_tot,1683847}, {mem_used,3744082712}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1016}, {memory_data,{33657663488,9511370752,{<18118.1662.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 22798044 kB\nBuffers: 306856 kB\nCached: 5061452 kB\nSwapCached: 12 kB\nActive: 5572352 kB\nInactive: 4213420 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 22798044 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 1952 kB\nWriteback: 0 kB\nAnonPages: 4417536 kB\nMapped: 17260 kB\nSlab: 237068 kB\nPageTables: 13496 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6123172 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-02T18:41:10.658,ns_1@10.2.1.67:ns_doctor<0.1461.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,906869,34233}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.17519.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,90}, {started_on,1351906753}, {total_vbuckets,129}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351906901}, {vbuckets_done,117}]]}, {memory, [{total,110069240}, {processes,64810640}, {processes_used,63244056}, {system,45258600}, {atom,1405017}, {atom_used,1377299}, {binary,7325600}, {code,14117455}, {ets,4315120}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,4648673280}, {buffered_memory,216035328}, {free_memory,24008097792}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1077193,5000}}, {context_switches,{4742731,0}}, {garbage_collection,{506470,1532857462,0}}, {io,{{input,284146577},{output,65211278}}}, {reductions,{577284927,2319076}}, {run_queue,0}, {runtime,{189280,630}}]}, {system_stats, [{cpu_utilization_rate,8.75}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,4045665638}, {couch_docs_data_size,3806027493}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1848172}, {curr_items_tot,1848172}, {mem_used,3767129688}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1077}, {memory_data,{33657663488,9982496768,{<18077.16820.0>,7028184}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 23445408 kB\nBuffers: 210972 kB\nCached: 4539720 kB\nSwapCached: 0 kB\nActive: 5078684 kB\nInactive: 4090528 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 23445408 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 19336 kB\nWriteback: 160 kB\nAnonPages: 4418824 kB\nMapped: 15324 kB\nSlab: 208840 kB\nPageTables: 13072 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6141236 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,906866,688598}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,92572032}, {processes,52873480}, {processes_used,52807248}, {system,39698552}, {atom,1352041}, {atom_used,1349412}, {binary,2554168}, {code,13622682}, {ets,4125064}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,7009316864}, {buffered_memory,242806784}, {free_memory,21561298944}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1074440,4999}}, {context_switches,{5110864,0}}, {garbage_collection,{569663,981622263,0}}, {io,{{input,73678839},{output,51598925}}}, {reductions,{448072993,2584418}}, {run_queue,0}, {runtime,{166960,620}}]}, {system_stats, [{cpu_utilization_rate,18.045112781954888}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,6304142684}, {couch_docs_data_size,4452630804}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1843899}, {curr_items_tot,1843899}, {mem_used,3672471256}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1074}, {memory_data,{33657663488,11444596736,{<18117.1696.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 21055956 kB\nBuffers: 237116 kB\nCached: 6845036 kB\nSwapCached: 140 kB\nActive: 7308060 kB\nInactive: 4194940 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 21055956 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 1020 kB\nWriteback: 0 kB\nAnonPages: 4420760 kB\nMapped: 17060 kB\nSlab: 261888 kB\nPageTables: 13484 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6126676 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,906865,660063}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,92888064}, {processes,53299272}, {processes_used,53212400}, {system,39588792}, {atom,1352041}, {atom_used,1349412}, {binary,2446248}, {code,13631938}, {ets,4106760}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,6549069824}, {buffered_memory,415703040}, {free_memory,21851553792}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1074398,5000}}, {context_switches,{4874776,0}}, {garbage_collection,{532077,960315388,0}}, {io,{{input,71242468},{output,48363841}}}, {reductions,{440213734,2834983}}, {run_queue,0}, {runtime,{160150,790}}]}, {system_stats, [{cpu_utilization_rate,26.25}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,5822767349}, {couch_docs_data_size,4461254437}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1846046}, {curr_items_tot,1846046}, {mem_used,3672173336}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1074}, {memory_data,{33657663488,11039174656,{<0.1678.0>,2914384}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 21339408 kB\nBuffers: 405960 kB\nCached: 6395576 kB\nSwapCached: 0 kB\nActive: 6788768 kB\nInactive: 4428584 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 21339408 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 1104 kB\nWriteback: 12 kB\nAnonPages: 4415928 kB\nMapped: 17060 kB\nSlab: 267288 kB\nPageTables: 13508 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 6124072 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,906869,6360}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,95008608}, {processes,55357336}, {processes_used,55270464}, {system,39651272}, {atom,1352041}, {atom_used,1349412}, {binary,2486272}, {code,13659922}, {ets,4084984}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,5876953088}, {buffered_memory,314900480}, {free_memory,22632988672}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1076430,5000}}, {context_switches,{5150434,0}}, {garbage_collection,{563138,988514878,0}}, {io,{{input,79589355},{output,50564211}}}, {reductions,{452496418,2602409}}, {run_queue,0}, {runtime,{165170,810}}]}, {system_stats, [{cpu_utilization_rate,6.265664160401003}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,5271369246}, {couch_docs_data_size,4476966844}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1852008}, {curr_items_tot,1852008}, {mem_used,3695696824}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1076}, {memory_data,{33657663488,10364272640,{<18118.1663.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 22102528 kB\nBuffers: 307520 kB\nCached: 5739212 kB\nSwapCached: 12 kB\nActive: 5945428 kB\nInactive: 4519408 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 22102528 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 1232 kB\nWriteback: 0 kB\nAnonPages: 4418188 kB\nMapped: 17260 kB\nSlab: 253756 kB\nPageTables: 13500 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6122480 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-02T18:41:36.526,ns_1@10.2.1.67:<0.1687.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,31,46}}) Stats for bucket "default": accepting_conns 1 auth_cmds 3 auth_errors 0 bucket_active_conns 1 bucket_conns 6 bytes 3902155320 bytes_read 4640823892 bytes_written 93800731 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 1 cmd_set 1932921 conn_yields 472 connection_structures 5000 curr_connections 10 curr_conns_on_port_11209 6 curr_conns_on_port_11210 2 curr_items 1932921 curr_items_tot 1932921 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-03 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 1309 ep_commit_time 2 ep_commit_time_total 6534 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 3 ep_data_age_highwat 71 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data/default ep_degraded_mode 0 ep_diskqueue_drain 1929309 ep_diskqueue_fill 1932921 ep_diskqueue_items 3613 ep_diskqueue_memory 115584 ep_diskqueue_pending 173376 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 687 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 243 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 1929303 ep_io_read_bytes 0 ep_io_write_bytes 4570243169 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 1769176 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 3805689947 ep_max_checkpoints 2 ep_max_data_size 5242880000 ep_max_item_size 20971520 ep_max_size 5242880000 ep_max_txn_size 10000 ep_max_vbuckets 512 ep_mem_high_wat 3932160000 ep_mem_high_wat 3932160000 ep_mem_low_wat 3145728000 ep_mem_low_wat 3145728000 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 27484 ep_num_expiry_pager_runs 0 ep_num_non_resident 393936 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 4 ep_num_value_ejects 393936 ep_oom_errors 0 ep_overhead 27357128 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 3370 ep_startup_time 1351905991 ep_storage_age 1 ep_storage_age_highwat 68 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 3782494895 ep_total_del_items 0 ep_total_enqueued 1932921 ep_total_new_items 1929303 ep_total_persisted 1929303 ep_uncommitted_items 564 ep_value_size 3646065042 ep_vb0 0 ep_vb_snapshot_total 57 ep_vb_total 128 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_130_g8b09894 ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 725 get_hits 0 get_misses 1 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 3902155320 pid 16451 pointer_size 64 rejected_conns 0 rusage_system 82.170508 rusage_user 221.973254 threads 4 time 1351906895 total_connections 13 uptime 927 vb_active_curr_items 1932921 vb_active_eject 393936 vb_active_expired 0 vb_active_ht_memory 12645376 vb_active_itm_memory 3782494895 vb_active_meta_data_memory 152700759 vb_active_num 128 vb_active_num_non_resident 393936 vb_active_num_ref_ejects 393936 vb_active_num_ref_items 1932921 vb_active_ops_create 1929308 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 79 vb_active_queue_age 952000 vb_active_queue_drain 1929309 vb_active_queue_fill 1932921 vb_active_queue_memory 115584 vb_active_queue_pending 173376 vb_active_queue_size 3612 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_meta_data_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-02T18:42:10.658,ns_1@10.2.1.67:ns_doctor<0.1461.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,906929,32890}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,109615400}, {processes,64508504}, {processes_used,62930168}, {system,45106896}, {atom,1405017}, {atom_used,1377299}, {binary,7118096}, {code,14117455}, {ets,4371496}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,5305380864}, {buffered_memory,217083904}, {free_memory,23320784896}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1137193,5000}}, {context_switches,{4804808,0}}, {garbage_collection,{512971,1611195219,0}}, {io,{{input,300069802},{output,67234588}}}, {reductions,{606579381,2310511}}, {run_queue,0}, {runtime,{195860,540}}]}, {system_stats, [{cpu_utilization_rate,11.528822055137844}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,4675407917}, {couch_docs_data_size,4552134281}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2016037}, {curr_items_tot,2016037}, {mem_used,3782357912}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1137}, {memory_data,{33657663488,9617186816,{<18077.16820.0>,7028184}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 22774204 kB\nBuffers: 211996 kB\nCached: 5181036 kB\nSwapCached: 0 kB\nActive: 4948680 kB\nInactive: 4877984 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 22774204 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 4892 kB\nWriteback: 1112 kB\nAnonPages: 4433328 kB\nMapped: 15308 kB\nSlab: 223532 kB\nPageTables: 13140 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6153316 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,906926,689401}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.11448.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,20}, {started_on,1351906845}, {total_vbuckets,129}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351906888}, {vbuckets_done,26}]]}, {memory, [{total,93962616}, {processes,53943312}, {processes_used,53885152}, {system,40019304}, {atom,1352041}, {atom_used,1349412}, {binary,2806808}, {code,13622682}, {ets,4190880}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,6945828864}, {buffered_memory,244498432}, {free_memory,21617668096}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1134440,4999}}, {context_switches,{5225198,0}}, {garbage_collection,{580201,1032362499,0}}, {io,{{input,88128526},{output,55349981}}}, {reductions,{473789596,2950745}}, {run_queue,0}, {runtime,{172880,540}}]}, {system_stats, [{cpu_utilization_rate,6.75}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,6346626597}, {couch_docs_data_size,4667006430}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2011410}, {curr_items_tot,2011410}, {mem_used,3643448648}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1134}, {memory_data,{33657663488,12163153920,{<18117.1696.0>,2914384}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 21111004 kB\nBuffers: 238768 kB\nCached: 6783036 kB\nSwapCached: 140 kB\nActive: 6955440 kB\nInactive: 4491672 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 21111004 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 23524 kB\nWriteback: 0 kB\nAnonPages: 4425248 kB\nMapped: 17064 kB\nSlab: 262648 kB\nPageTables: 13568 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6136004 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,906925,660168}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,93358184}, {processes,53321824}, {processes_used,53234952}, {system,40036360}, {atom,1352041}, {atom_used,1349412}, {binary,2717136}, {code,13631938}, {ets,4283672}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,7273582592}, {buffered_memory,416710656}, {free_memory,21102268416}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1134398,5001}}, {context_switches,{5161920,0}}, {garbage_collection,{561532,1025238823,0}}, {io,{{input,76458823},{output,53712526}}}, {reductions,{471311697,2507773}}, {run_queue,0}, {runtime,{168920,700}}]}, {system_stats, [{cpu_utilization_rate,6.75}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,6607292092}, {couch_docs_data_size,4896537875}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2013957}, {curr_items_tot,2013957}, {mem_used,3626897880}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1134}, {memory_data,{33657663488,11886485504,{<0.1681.0>,4714984}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 20607684 kB\nBuffers: 406944 kB\nCached: 7103108 kB\nSwapCached: 0 kB\nActive: 7197872 kB\nInactive: 4733164 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 20607684 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 1336 kB\nWriteback: 0 kB\nAnonPages: 4421012 kB\nMapped: 17060 kB\nSlab: 284972 kB\nPageTables: 13496 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 6130760 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,906929,6460}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,95516304}, {processes,55463240}, {processes_used,55376368}, {system,40053064}, {atom,1352041}, {atom_used,1349412}, {binary,2709728}, {code,13659922}, {ets,4261848}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,6659166208}, {buffered_memory,316162048}, {free_memory,21830242304}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1136431,5001}}, {context_switches,{5473657,0}}, {garbage_collection,{597043,1056620704,0}}, {io,{{input,85013962},{output,56039663}}}, {reductions,{484806219,3013387}}, {run_queue,0}, {runtime,{174270,860}}]}, {system_stats, [{cpu_utilization_rate,30.25}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,6029403773}, {couch_docs_data_size,4901515280}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2020383}, {curr_items_tot,2020383}, {mem_used,3649152376}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1136}, {memory_data,{33657663488,11071782912,{<18118.1662.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 21318596 kB\nBuffers: 308752 kB\nCached: 6503092 kB\nSwapCached: 12 kB\nActive: 6384132 kB\nInactive: 4845652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 21318596 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 1204 kB\nWriteback: 0 kB\nAnonPages: 4417340 kB\nMapped: 17260 kB\nSlab: 272800 kB\nPageTables: 13500 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6123820 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-02T18:43:10.658,ns_1@10.2.1.67:ns_doctor<0.1461.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,906989,32601}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,111051432}, {processes,66506168}, {processes_used,64927832}, {system,44545264}, {atom,1405017}, {atom_used,1377299}, {binary,6380528}, {code,14117455}, {ets,4546392}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,6247870464}, {buffered_memory,218275840}, {free_memory,22352592896}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1197193,5000}}, {context_switches,{4912082,0}}, {garbage_collection,{523814,1692118245,0}}, {io,{{input,315460821},{output,69401062}}}, {reductions,{637287985,2963422}}, {run_queue,0}, {runtime,{203010,880}}]}, {system_stats, [{cpu_utilization_rate,7.0}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,5628837900}, {couch_docs_data_size,5287141105}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2184025}, {curr_items_tot,2184025}, {mem_used,3710800920}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1197}, {memory_data,{33657663488,10443284480,{<18077.16820.0>,7028184}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 21828704 kB\nBuffers: 213160 kB\nCached: 6101436 kB\nSwapCached: 0 kB\nActive: 5274696 kB\nInactive: 5475224 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 21828704 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 5672 kB\nWriteback: 0 kB\nAnonPages: 4435324 kB\nMapped: 15316 kB\nSlab: 245988 kB\nPageTables: 12948 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6156328 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,906986,689696}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.11448.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,51}, {started_on,1351906845}, {total_vbuckets,129}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351906950}, {vbuckets_done,67}]]}, {memory, [{total,95124424}, {processes,55282872}, {processes_used,55262432}, {system,39841552}, {atom,1352041}, {atom_used,1349412}, {binary,2407976}, {code,13622682}, {ets,4405760}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,6400094208}, {buffered_memory,246145024}, {free_memory,22155919360}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1194440,4998}}, {context_switches,{5280118,0}}, {garbage_collection,{585969,1075807031,0}}, {io,{{input,89274704},{output,56647450}}}, {reductions,{496301534,1903116}}, {run_queue,0}, {runtime,{177880,420}}]}, {system_stats, [{cpu_utilization_rate,15.288220551378446}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,5841796336}, {couch_docs_data_size,4761695639}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2178650}, {curr_items_tot,2178650}, {mem_used,3659416056}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1194}, {memory_data,{33657663488,12003680256,{<18117.1696.0>,2914384}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 21636640 kB\nBuffers: 240376 kB\nCached: 6250092 kB\nSwapCached: 140 kB\nActive: 6149812 kB\nInactive: 4784620 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 21636640 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 26604 kB\nWriteback: 0 kB\nAnonPages: 4443392 kB\nMapped: 17064 kB\nSlab: 249456 kB\nPageTables: 13608 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6159520 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,906985,659548}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,91850840}, {processes,51936496}, {processes_used,51849624}, {system,39914344}, {atom,1352041}, {atom_used,1349412}, {binary,2532152}, {code,13631938}, {ets,4345912}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,8022835200}, {buffered_memory,417357824}, {free_memory,20326318080}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1194397,4999}}, {context_switches,{5370663,0}}, {garbage_collection,{583096,1082628013,0}}, {io,{{input,77906050},{output,55510259}}}, {reductions,{499518972,2607147}}, {run_queue,0}, {runtime,{176040,680}}]}, {system_stats, [{cpu_utilization_rate,28.75}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7278321984}, {couch_docs_data_size,5282524976}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2182013}, {curr_items_tot,2182013}, {mem_used,3590810632}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1194}, {memory_data,{33657663488,12656214016,{<0.1681.0>,4714984}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 19849920 kB\nBuffers: 407576 kB\nCached: 7834800 kB\nSwapCached: 0 kB\nActive: 7584416 kB\nInactive: 5085784 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 19849920 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 1372 kB\nWriteback: 0 kB\nAnonPages: 4427824 kB\nMapped: 17060 kB\nSlab: 303236 kB\nPageTables: 13504 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 6135508 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,906989,5202}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,95788424}, {processes,55576360}, {processes_used,55489888}, {system,40212064}, {atom,1352041}, {atom_used,1349412}, {binary,2805000}, {code,13659922}, {ets,4324528}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,7407759360}, {buffered_memory,316829696}, {free_memory,21057085440}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1196430,5000}}, {context_switches,{5728063,0}}, {garbage_collection,{623388,1117705436,0}}, {io,{{input,86661356},{output,58057862}}}, {reductions,{514519785,2301584}}, {run_queue,0}, {runtime,{181980,510}}]}, {system_stats, [{cpu_utilization_rate,25.25}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,6741387030}, {couch_docs_data_size,5297693092}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2188860}, {curr_items_tot,2188860}, {mem_used,3610004280}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1196}, {memory_data,{33657663488,11872116736,{<18118.1662.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 20563684 kB\nBuffers: 309404 kB\nCached: 7234140 kB\nSwapCached: 12 kB\nActive: 6791436 kB\nInactive: 5175244 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 20563684 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 1392 kB\nWriteback: 124 kB\nAnonPages: 4422576 kB\nMapped: 17260 kB\nSlab: 291052 kB\nPageTables: 13504 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6125756 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-02T18:43:16.523,ns_1@10.2.1.67:<0.1687.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,31,47}}) Stats for bucket "default": accepting_conns 1 auth_cmds 3 auth_errors 0 bucket_active_conns 1 bucket_conns 6 bytes 3672909192 bytes_read 5312895526 bytes_written 106991058 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 1 cmd_set 2212651 conn_yields 538 connection_structures 5000 curr_connections 10 curr_conns_on_port_11209 6 curr_conns_on_port_11210 2 curr_items 2212651 curr_items_tot 2212651 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-03 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 1400 ep_commit_time 2 ep_commit_time_total 7181 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 3 ep_data_age_highwat 71 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data/default ep_degraded_mode 0 ep_diskqueue_drain 2208056 ep_diskqueue_fill 2212651 ep_diskqueue_items 4596 ep_diskqueue_memory 147040 ep_diskqueue_pending 220560 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 788 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 2476 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 2207976 ep_io_read_bytes 0 ep_io_write_bytes 5230904940 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 2133612 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 3593626241 ep_max_checkpoints 2 ep_max_data_size 5242880000 ep_max_item_size 20971520 ep_max_size 5242880000 ep_max_txn_size 10000 ep_max_vbuckets 512 ep_mem_high_wat 3932160000 ep_mem_high_wat 3932160000 ep_mem_low_wat 3145728000 ep_mem_low_wat 3145728000 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 154694 ep_num_expiry_pager_runs 0 ep_num_non_resident 775562 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 8 ep_num_value_ejects 775562 ep_oom_errors 0 ep_overhead 19965920 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 2120 ep_startup_time 1351905991 ep_storage_age 2 ep_storage_age_highwat 68 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 3567074429 ep_total_del_items 0 ep_total_enqueued 2212651 ep_total_new_items 2207976 ep_total_persisted 2207976 ep_uncommitted_items 7454 ep_value_size 3410722576 ep_vb0 0 ep_vb_snapshot_total 60 ep_vb_total 128 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_130_g8b09894 ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 725 get_hits 0 get_misses 1 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 3672909192 pid 16451 pointer_size 64 rejected_conns 0 rusage_system 92.856883 rusage_user 253.704431 threads 4 time 1351906995 total_connections 13 uptime 1027 vb_active_curr_items 2212651 vb_active_eject 775562 vb_active_expired 0 vb_active_ht_memory 12645376 vb_active_itm_memory 3567074429 vb_active_meta_data_memory 174799429 vb_active_num 128 vb_active_num_non_resident 775562 vb_active_num_ref_ejects 775562 vb_active_num_ref_items 2212651 vb_active_ops_create 2208055 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 64 vb_active_queue_age 5564000 vb_active_queue_drain 2208056 vb_active_queue_fill 2212651 vb_active_queue_memory 147040 vb_active_queue_pending 220560 vb_active_queue_size 4595 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_meta_data_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-02T18:44:10.658,ns_1@10.2.1.67:ns_doctor<0.1461.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,907049,32207}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,111476672}, {processes,66512584}, {processes_used,64934248}, {system,44964088}, {atom,1405017}, {atom_used,1377299}, {binary,6623504}, {code,14117455}, {ets,4723560}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,6911963136}, {buffered_memory,219049984}, {free_memory,21673525248}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1257194,5001}}, {context_switches,{5094747,0}}, {garbage_collection,{541957,1781182167,0}}, {io,{{input,335186714},{output,74737851}}}, {reductions,{670874316,3209451}}, {run_queue,0}, {runtime,{211550,950}}]}, {system_stats, [{cpu_utilization_rate,7.017543859649122}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,6297855386}, {couch_docs_data_size,5699291522}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2352279}, {curr_items_tot,2352279}, {mem_used,3662065240}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1257}, {memory_data,{33657663488,11302404096,{<18077.16820.0>,7028184}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 21165552 kB\nBuffers: 213916 kB\nCached: 6749964 kB\nSwapCached: 0 kB\nActive: 5580116 kB\nInactive: 5817052 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 21165552 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 2032 kB\nWriteback: 0 kB\nAnonPages: 4433304 kB\nMapped: 15316 kB\nSlab: 261880 kB\nPageTables: 12936 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6144416 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,907046,688984}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.11448.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,73}, {started_on,1351906845}, {total_vbuckets,129}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351907010}, {vbuckets_done,95}]]}, {memory, [{total,91152208}, {processes,51064688}, {processes_used,51019928}, {system,40087520}, {atom,1352041}, {atom_used,1349412}, {binary,2502592}, {code,13622682}, {ets,4559352}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,6122774528}, {buffered_memory,247603200}, {free_memory,22412537856}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1254440,5000}}, {context_switches,{5336360,0}}, {garbage_collection,{591738,1120560730,0}}, {io,{{input,93596749},{output,60790966}}}, {reductions,{518959308,1944866}}, {run_queue,0}, {runtime,{183260,380}}]}, {system_stats, [{cpu_utilization_rate,8.771929824561404}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,5514680410}, {couch_docs_data_size,4896792792}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2346331}, {curr_items_tot,2346331}, {mem_used,3672797640}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1254}, {memory_data,{33657663488,11460222976,{<18117.1696.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 21887244 kB\nBuffers: 241800 kB\nCached: 5979272 kB\nSwapCached: 140 kB\nActive: 5620100 kB\nInactive: 5070476 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 21887244 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 36672 kB\nWriteback: 0 kB\nAnonPages: 4469508 kB\nMapped: 17064 kB\nSlab: 243072 kB\nPageTables: 13660 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6178948 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,907045,660023}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,97157480}, {processes,56384240}, {processes_used,56297448}, {system,40773240}, {atom,1352041}, {atom_used,1349412}, {binary,3216472}, {code,13631938}, {ets,4520496}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,8799240192}, {buffered_memory,418418688}, {free_memory,19510624256}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1254398,5000}}, {context_switches,{5697983,0}}, {garbage_collection,{612038,1150394672,0}}, {io,{{input,79867867},{output,57714682}}}, {reductions,{531893978,2818550}}, {run_queue,0}, {runtime,{185070,840}}]}, {system_stats, [{cpu_utilization_rate,6.297229219143577}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,8115428854}, {couch_docs_data_size,5726176686}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2349579}, {curr_items_tot,2349579}, {mem_used,3552032888}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1254}, {memory_data,{33657663488,13409816576,{<0.1681.0>,4714984}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 19053344 kB\nBuffers: 408612 kB\nCached: 8593008 kB\nSwapCached: 0 kB\nActive: 8064864 kB\nInactive: 5383180 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 19053344 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 1468 kB\nWriteback: 0 kB\nAnonPages: 4446492 kB\nMapped: 17060 kB\nSlab: 322296 kB\nPageTables: 13556 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 6159960 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,907049,4808}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,95355128}, {processes,55193728}, {processes_used,55106856}, {system,40161400}, {atom,1352041}, {atom_used,1349412}, {binary,2580240}, {code,13659922}, {ets,4499016}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,8106012672}, {buffered_memory,317644800}, {free_memory,20335861760}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1256430,5000}}, {context_switches,{5975648,0}}, {garbage_collection,{649062,1178586682,0}}, {io,{{input,88270989},{output,60022996}}}, {reductions,{543989808,1850472}}, {run_queue,0}, {runtime,{190270,460}}]}, {system_stats, [{cpu_utilization_rate,6.25}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,7497442671}, {couch_docs_data_size,5714391568}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2356812}, {curr_items_tot,2356812}, {mem_used,3574972664}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1256}, {memory_data,{33657663488,12611624960,{<18118.1662.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 19859240 kB\nBuffers: 310200 kB\nCached: 7916028 kB\nSwapCached: 12 kB\nActive: 7182448 kB\nInactive: 5470376 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 19859240 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 1036 kB\nWriteback: 0 kB\nAnonPages: 4426440 kB\nMapped: 17260 kB\nSlab: 308228 kB\nPageTables: 13512 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6130692 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-02T18:44:56.522,ns_1@10.2.1.67:<0.1687.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,31,47}}) Stats for bucket "default": accepting_conns 1 auth_cmds 3 auth_errors 0 bucket_active_conns 1 bucket_conns 6 bytes 3930125304 bytes_read 5985066716 bytes_written 120183148 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 1 cmd_set 2492415 conn_yields 592 connection_structures 5000 curr_connections 10 curr_conns_on_port_11209 6 curr_conns_on_port_11210 2 curr_items 2492415 curr_items_tot 2492415 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-03 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 1486 ep_commit_time 56 ep_commit_time_total 7760 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 41 ep_data_age_highwat 71 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data/default ep_degraded_mode 0 ep_diskqueue_drain 2387300 ep_diskqueue_fill 2492415 ep_diskqueue_items 105116 ep_diskqueue_memory 3363680 ep_diskqueue_pending 5045520 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 866 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 42624 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 2386822 ep_io_read_bytes 0 ep_io_write_bytes 5654673243 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 2316119 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 3833034405 ep_max_checkpoints 2 ep_max_data_size 5242880000 ep_max_item_size 20971520 ep_max_size 5242880000 ep_max_txn_size 10000 ep_max_vbuckets 512 ep_mem_high_wat 3932160000 ep_mem_high_wat 3932160000 ep_mem_low_wat 3145728000 ep_mem_low_wat 3145728000 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 254501 ep_num_expiry_pager_runs 0 ep_num_non_resident 965510 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 10 ep_num_value_ejects 965510 ep_oom_errors 0 ep_overhead 28868248 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 62492 ep_startup_time 1351905991 ep_storage_age 40 ep_storage_age_highwat 68 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 3803125425 ep_total_del_items 0 ep_total_enqueued 2492415 ep_total_new_items 2386822 ep_total_persisted 2386822 ep_uncommitted_items 2628 ep_value_size 3626954375 ep_vb0 0 ep_vb_snapshot_total 61 ep_vb_total 128 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_130_g8b09894 ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 725 get_hits 0 get_misses 1 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 3930125304 pid 16451 pointer_size 64 rejected_conns 0 rusage_system 102.658393 rusage_user 279.663484 threads 4 time 1351907095 total_connections 13 uptime 1127 vb_active_curr_items 2492415 vb_active_eject 965510 vb_active_expired 0 vb_active_ht_memory 12645376 vb_active_itm_memory 3803125425 vb_active_meta_data_memory 196900785 vb_active_num 128 vb_active_num_non_resident 965510 vb_active_num_ref_ejects 965510 vb_active_num_ref_items 2492415 vb_active_ops_create 2387299 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 61 vb_active_queue_age 2039951000 vb_active_queue_drain 2387300 vb_active_queue_fill 2492415 vb_active_queue_memory 3363680 vb_active_queue_pending 5045520 vb_active_queue_size 105115 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_meta_data_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-02T18:45:10.658,ns_1@10.2.1.67:ns_doctor<0.1461.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,907109,29629}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,111970936}, {processes,66286848}, {processes_used,64708512}, {system,45684088}, {atom,1405017}, {atom_used,1377299}, {binary,7167840}, {code,14117455}, {ets,4898304}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,7562448896}, {buffered_memory,219856896}, {free_memory,20986200064}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1317193,5000}}, {context_switches,{5311222,0}}, {garbage_collection,{563364,1873605610,0}}, {io,{{input,351801417},{output,77303269}}}, {reductions,{705714965,2253371}}, {run_queue,0}, {runtime,{220560,680}}]}, {system_stats, [{cpu_utilization_rate,3.5}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,6947489170}, {couch_docs_data_size,6073317216}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2497553}, {curr_items_tot,2497553}, {mem_used,3561592760}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1317}, {memory_data,{33657663488,12008517632,{<18077.16820.0>,7028184}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 20494336 kB\nBuffers: 214704 kB\nCached: 7385204 kB\nSwapCached: 0 kB\nActive: 5923340 kB\nInactive: 6126940 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 20494336 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 3136 kB\nWriteback: 0 kB\nAnonPages: 4450480 kB\nMapped: 15316 kB\nSlab: 278808 kB\nPageTables: 12992 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6162504 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,907106,689119}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.11448.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,96}, {started_on,1351906845}, {total_vbuckets,129}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351907071}, {vbuckets_done,125}]]}, {memory, [{total,91169056}, {processes,51025520}, {processes_used,51008360}, {system,40143536}, {atom,1352041}, {atom_used,1349412}, {binary,2348728}, {code,13622682}, {ets,4764808}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,5826437120}, {buffered_memory,249049088}, {free_memory,22681346048}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1314439,4998}}, {context_switches,{5388764,0}}, {garbage_collection,{597118,1164340564,0}}, {io,{{input,94465985},{output,62090398}}}, {reductions,{541402185,1909231}}, {run_queue,0}, {runtime,{188540,420}}]}, {system_stats, [{cpu_utilization_rate,19.09547738693467}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,5246193945}, {couch_docs_data_size,5050867485}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2497102}, {curr_items_tot,2497102}, {mem_used,3689105016}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1314}, {memory_data,{33657663488,11220246528,{<18117.1696.0>,2914384}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 22149752 kB\nBuffers: 243212 kB\nCached: 5689880 kB\nSwapCached: 140 kB\nActive: 5070032 kB\nInactive: 5364140 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 22149752 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 15760 kB\nWriteback: 20 kB\nAnonPages: 4500956 kB\nMapped: 17064 kB\nSlab: 235472 kB\nPageTables: 13712 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6210712 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,907105,659357}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.13654.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,23}, {started_on,1351907050}, {total_vbuckets,129}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351907104}, {vbuckets_done,30}]]}, {memory, [{total,93889544}, {processes,53119984}, {processes_used,53046704}, {system,40769560}, {atom,1352041}, {atom_used,1349412}, {binary,3023704}, {code,13631938}, {ets,4707304}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,8447946752}, {buffered_memory,420143104}, {free_memory,19845832704}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1314397,4999}}, {context_switches,{5797404,0}}, {garbage_collection,{621700,1203107920,0}}, {io,{{input,96803776},{output,64086596}}}, {reductions,{558766470,2671439}}, {run_queue,0}, {runtime,{190700,420}}]}, {system_stats, [{cpu_utilization_rate,4.25}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7834714254}, {couch_docs_data_size,5841999958}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2503544}, {curr_items_tot,2503544}, {mem_used,3536581784}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1314}, {memory_data,{33657663488,14225281024,{<0.1678.0>,4714984}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 19380696 kB\nBuffers: 410296 kB\nCached: 8249948 kB\nSwapCached: 0 kB\nActive: 7475260 kB\nInactive: 5649476 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 19380696 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 24324 kB\nWriteback: 0 kB\nAnonPages: 4464488 kB\nMapped: 17064 kB\nSlab: 316380 kB\nPageTables: 13676 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 6176220 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,907109,4177}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,94530272}, {processes,54194408}, {processes_used,54107536}, {system,40335864}, {atom,1352041}, {atom_used,1349412}, {binary,2577168}, {code,13659922}, {ets,4676648}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,8804884480}, {buffered_memory,318726144}, {free_memory,19604099072}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1316430,5000}}, {context_switches,{6177365,0}}, {garbage_collection,{670012,1236604524,0}}, {io,{{input,93113205},{output,64715468}}}, {reductions,{572055550,1833143}}, {run_queue,0}, {runtime,{197400,450}}]}, {system_stats, [{cpu_utilization_rate,2.5}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,8200766813}, {couch_docs_data_size,6107702164}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2501801}, {curr_items_tot,2501801}, {mem_used,3488892632}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1316}, {memory_data,{33657663488,13428334592,{<18118.1662.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 19144628 kB\nBuffers: 311256 kB\nCached: 8598520 kB\nSwapCached: 12 kB\nActive: 7561248 kB\nInactive: 5790060 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 19144628 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 1908 kB\nWriteback: 0 kB\nAnonPages: 4441392 kB\nMapped: 17260 kB\nSlab: 325376 kB\nPageTables: 13536 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6145684 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-02T18:46:10.658,ns_1@10.2.1.67:ns_doctor<0.1461.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,907169,29000}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,112723976}, {processes,66982112}, {processes_used,65403776}, {system,45741864}, {atom,1405017}, {atom_used,1377299}, {binary,7163160}, {code,14117455}, {ets,4960424}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,7563472896}, {buffered_memory,219979776}, {free_memory,20987346944}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1377193,5000}}, {context_switches,{5363847,0}}, {garbage_collection,{568379,1951633976,0}}, {io,{{input,367561213},{output,79212042}}}, {reductions,{734455996,2267710}}, {run_queue,0}, {runtime,{227990,630}}]}, {system_stats, [{cpu_utilization_rate,3.7406483790523692}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,6947489135}, {couch_docs_data_size,6073317216}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2497553}, {curr_items_tot,2497553}, {mem_used,3561592760}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1377}, {memory_data,{33657663488,12672860160,{<18077.16820.0>,7028184}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 20495456 kB\nBuffers: 214824 kB\nCached: 7386204 kB\nSwapCached: 0 kB\nActive: 5922920 kB\nInactive: 6127964 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 20495456 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 432 kB\nWriteback: 0 kB\nAnonPages: 4449760 kB\nMapped: 15316 kB\nSlab: 278280 kB\nPageTables: 13016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6162928 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,907166,689256}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,90688280}, {processes,50489880}, {processes_used,50421408}, {system,40198400}, {atom,1352041}, {atom_used,1349412}, {binary,2397872}, {code,13622682}, {ets,4776920}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,6844850176}, {buffered_memory,250314752}, {free_memory,21642182656}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1374439,4998}}, {context_switches,{5448815,0}}, {garbage_collection,{603085,1209557402,0}}, {io,{{input,95241818},{output,63422969}}}, {reductions,{563977209,1972516}}, {run_queue,0}, {runtime,{193620,560}}]}, {system_stats, [{cpu_utilization_rate,3.25}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,6208521660}, {couch_docs_data_size,6029914038}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2497102}, {curr_items_tot,2497102}, {mem_used,3652502104}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1374}, {memory_data,{33657663488,10935422976,{<18117.1696.0>,2914384}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 21134944 kB\nBuffers: 244448 kB\nCached: 6684424 kB\nSwapCached: 140 kB\nActive: 5103936 kB\nInactive: 6323176 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 21134944 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 7748 kB\nWriteback: 0 kB\nAnonPages: 4497904 kB\nMapped: 17060 kB\nSlab: 259336 kB\nPageTables: 13628 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6203772 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,907165,659409}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.13654.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,50}, {started_on,1351907050}, {total_vbuckets,129}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351907165}, {vbuckets_done,65}]]}, {memory, [{total,93409096}, {processes,52015728}, {processes_used,51974648}, {system,41393368}, {atom,1352041}, {atom_used,1349412}, {binary,3433376}, {code,13631938}, {ets,4916448}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,7854022656}, {buffered_memory,421883904}, {free_memory,20452270080}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1374397,5000}}, {context_switches,{5856421,0}}, {garbage_collection,{627823,1259756506,0}}, {io,{{input,100935725},{output,66056815}}}, {reductions,{590843248,2675860}}, {run_queue,0}, {runtime,{196140,420}}]}, {system_stats, [{cpu_utilization_rate,25.87939698492462}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7249595422}, {couch_docs_data_size,5902013054}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2503544}, {curr_items_tot,2503544}, {mem_used,3536576936}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1374}, {memory_data,{33657663488,13759135744,{<0.1678.0>,2914384}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 19972920 kB\nBuffers: 411996 kB\nCached: 7669944 kB\nSwapCached: 0 kB\nActive: 6623840 kB\nInactive: 5924576 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 19972920 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 20680 kB\nWriteback: 0 kB\nAnonPages: 4466472 kB\nMapped: 17064 kB\nSlab: 301268 kB\nPageTables: 13680 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 6178704 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,907169,3714}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,91693552}, {processes,51070968}, {processes_used,50984096}, {system,40622584}, {atom,1352041}, {atom_used,1349412}, {binary,2689248}, {code,13659922}, {ets,4851312}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,8805949440}, {buffered_memory,318832640}, {free_memory,19602702336}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1376430,5000}}, {context_switches,{6224191,0}}, {garbage_collection,{674542,1280604408,0}}, {io,{{input,93832380},{output,66041445}}}, {reductions,{594207422,1827383}}, {run_queue,0}, {runtime,{202910,430}}]}, {system_stats, [{cpu_utilization_rate,2.506265664160401}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,8200766792}, {couch_docs_data_size,6107702164}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2501801}, {curr_items_tot,2501801}, {mem_used,3488892632}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1376}, {memory_data,{33657663488,14055596032,{<18118.1662.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 19143264 kB\nBuffers: 311360 kB\nCached: 8599560 kB\nSwapCached: 12 kB\nActive: 7560856 kB\nInactive: 5791060 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 19143264 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 392 kB\nWriteback: 0 kB\nAnonPages: 4441140 kB\nMapped: 17260 kB\nSlab: 324956 kB\nPageTables: 13540 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6145308 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-02T18:46:36.518,ns_1@10.2.1.67:<0.1687.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,31,47}}) Stats for bucket "default": accepting_conns 1 auth_cmds 3 auth_errors 0 bucket_active_conns 1 bucket_conns 5 bytes 3513624408 bytes_read 6011815487 bytes_written 126972590 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 1 cmd_set 2503544 conn_yields 595 connection_structures 5000 curr_connections 9 curr_conns_on_port_11209 6 curr_conns_on_port_11210 1 curr_items 2503544 curr_items_tot 2503544 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-03 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 1491 ep_commit_time 878 ep_commit_time_total 12636 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 111 ep_data_age_highwat 124 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data/default ep_degraded_mode 0 ep_diskqueue_drain 2436323 ep_diskqueue_fill 2503544 ep_diskqueue_items 67456 ep_diskqueue_memory 2151072 ep_diskqueue_pending 3226608 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 970 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 67456 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 2435772 ep_io_read_bytes 0 ep_io_write_bytes 5770780069 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 2498272 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 3443869792 ep_max_checkpoints 2 ep_max_data_size 5242880000 ep_max_item_size 20971520 ep_max_size 5242880000 ep_max_txn_size 10000 ep_max_vbuckets 512 ep_mem_high_wat 3932160000 ep_mem_high_wat 3932160000 ep_mem_low_wat 3145728000 ep_mem_low_wat 3145728000 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 393411 ep_num_expiry_pager_runs 0 ep_num_non_resident 1142582 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 12 ep_num_value_ejects 1142582 ep_oom_errors 0 ep_overhead 17216110 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1351905991 ep_storage_age 111 ep_storage_age_highwat 123 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 3413827264 ep_total_del_items 0 ep_total_enqueued 2503544 ep_total_new_items 2435772 ep_total_persisted 2435772 ep_uncommitted_items 6399 ep_value_size 3236781315 ep_vb0 0 ep_vb_snapshot_total 61 ep_vb_total 128 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_130_g8b09894 ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 725 get_hits 0 get_misses 1 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 3513624408 pid 16451 pointer_size 64 rejected_conns 0 rusage_system 103.448273 rusage_user 284.370769 threads 4 time 1351907195 total_connections 13 uptime 1227 vb_active_curr_items 2503544 vb_active_eject 1142582 vb_active_expired 0 vb_active_ht_memory 12645376 vb_active_itm_memory 3413827264 vb_active_meta_data_memory 197779976 vb_active_num 128 vb_active_num_non_resident 1142582 vb_active_num_ref_ejects 1142582 vb_active_num_ref_items 2503544 vb_active_ops_create 2436322 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 54 vb_active_queue_age 7318964000 vb_active_queue_drain 2436323 vb_active_queue_fill 2503544 vb_active_queue_memory 2151072 vb_active_queue_pending 3226608 vb_active_queue_size 67221 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_meta_data_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [stats:warn,2012-11-02T18:47:09.551,ns_1@10.2.1.67:<0.1687.0>:stats_collector:latest_tick:223]Dropped 1 ticks [ns_doctor:debug,2012-11-02T18:47:10.658,ns_1@10.2.1.67:ns_doctor<0.1461.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,907229,548009}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,115240304}, {processes,69156048}, {processes_used,67579128}, {system,46084256}, {atom,1405017}, {atom_used,1377299}, {binary,7328608}, {code,14117455}, {ets,5137888}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,7564627968}, {buffered_memory,220200960}, {free_memory,20988108800}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1437193,5000}}, {context_switches,{5418267,0}}, {garbage_collection,{573876,2032155885,0}}, {io,{{input,386886206},{output,84434382}}}, {reductions,{764216470,2241401}}, {run_queue,0}, {runtime,{235490,660}}]}, {system_stats, [{cpu_utilization_rate,4.0}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,6947489102}, {couch_docs_data_size,6073317216}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2497553}, {curr_items_tot,2497553}, {mem_used,3561592760}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1437}, {memory_data,{33657663488,12672856064,{<18077.16820.0>,7028184}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 20496200 kB\nBuffers: 215040 kB\nCached: 7387332 kB\nSwapCached: 0 kB\nActive: 5920812 kB\nInactive: 6129200 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 20496200 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 312 kB\nWriteback: 0 kB\nAnonPages: 4447696 kB\nMapped: 15316 kB\nSlab: 278240 kB\nPageTables: 12928 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6161684 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,907226,690416}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,93754896}, {processes,53016000}, {processes_used,52947528}, {system,40738896}, {atom,1352041}, {atom_used,1349412}, {binary,2760816}, {code,13622682}, {ets,4954728}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,6845845504}, {buffered_memory,250540032}, {free_memory,21637357568}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1434440,5000}}, {context_switches,{5497386,0}}, {garbage_collection,{607828,1255511473,0}}, {io,{{input,99674084},{output,68094717}}}, {reductions,{587096582,1882947}}, {run_queue,0}, {runtime,{199390,510}}]}, {system_stats, [{cpu_utilization_rate,3.007518796992481}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,6231451007}, {couch_docs_data_size,6054380609}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2497102}, {curr_items_tot,2497102}, {mem_used,3652502072}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1434}, {memory_data,{33657663488,12018282496,{<18117.1696.0>,2914384}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 21130232 kB\nBuffers: 244668 kB\nCached: 6685396 kB\nSwapCached: 140 kB\nActive: 5107468 kB\nInactive: 6324460 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 21130232 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 6240 kB\nWriteback: 0 kB\nAnonPages: 4501664 kB\nMapped: 17060 kB\nSlab: 259252 kB\nPageTables: 13640 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6208080 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,907225,659378}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.13654.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,76}, {started_on,1351907050}, {total_vbuckets,129}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351907223}, {vbuckets_done,99}]]}, {memory, [{total,93006384}, {processes,51417584}, {processes_used,51376504}, {system,41588800}, {atom,1352041}, {atom_used,1349412}, {binary,3565912}, {code,13631938}, {ets,4978384}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,7295176704}, {buffered_memory,423518208}, {free_memory,21024804864}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1434397,5000}}, {context_switches,{5912044,0}}, {garbage_collection,{633873,1308826754,0}}, {io,{{input,103045359},{output,67607368}}}, {reductions,{616668830,1866129}}, {run_queue,0}, {runtime,{201370,490}}]}, {system_stats, [{cpu_utilization_rate,2.75}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,6660727654}, {couch_docs_data_size,5983445172}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2503544}, {curr_items_tot,2503544}, {mem_used,3510454248}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1434}, {memory_data,{33657663488,13152317440,{<0.1678.0>,2914384}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 20532036 kB\nBuffers: 413592 kB\nCached: 7124196 kB\nSwapCached: 0 kB\nActive: 5829512 kB\nInactive: 6174452 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 20532036 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 46876 kB\nWriteback: 0 kB\nAnonPages: 4466196 kB\nMapped: 17064 kB\nSlab: 287492 kB\nPageTables: 13672 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 6175636 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,907229,548699}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,92756352}, {processes,51599648}, {processes_used,51512776}, {system,41156704}, {atom,1352041}, {atom_used,1349412}, {binary,3159384}, {code,13659922}, {ets,4915040}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,8807120896}, {buffered_memory,318943232}, {free_memory,19595493376}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1436430,5000}}, {context_switches,{6275157,0}}, {garbage_collection,{679335,1334358083,0}}, {io,{{input,97034182},{output,67831598}}}, {reductions,{624213882,2627970}}, {run_queue,0}, {runtime,{209160,480}}]}, {system_stats, [{cpu_utilization_rate,2.75}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,8200766792}, {couch_docs_data_size,6107702164}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2501801}, {curr_items_tot,2501801}, {mem_used,3488892632}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1436}, {memory_data,{33657663488,14054326272,{<18118.1662.0>,2914384}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 19134584 kB\nBuffers: 311468 kB\nCached: 8600704 kB\nSwapCached: 12 kB\nActive: 7568648 kB\nInactive: 5792072 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 19134584 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 652 kB\nWriteback: 0 kB\nAnonPages: 4448592 kB\nMapped: 17260 kB\nSlab: 324792 kB\nPageTables: 13556 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6150708 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-02T18:48:10.658,ns_1@10.2.1.67:ns_doctor<0.1461.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,907289,26483}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,121178672}, {processes,75011528}, {processes_used,73433192}, {system,46167144}, {atom,1405017}, {atom_used,1377299}, {binary,7347536}, {code,14117455}, {ets,5201392}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,7565529088}, {buffered_memory,220327936}, {free_memory,20979855360}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1497193,5000}}, {context_switches,{5471030,0}}, {garbage_collection,{578614,2109670444,0}}, {io,{{input,402572985},{output,86291446}}}, {reductions,{793183534,2301624}}, {run_queue,0}, {runtime,{242940,630}}]}, {system_stats, [{cpu_utilization_rate,3.2581453634085213}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,6947489102}, {couch_docs_data_size,6073317216}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2497553}, {curr_items_tot,2497553}, {mem_used,3561592760}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1497}, {memory_data,{33657663488,12675395584,{<18077.16820.0>,7028184}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 20488140 kB\nBuffers: 215164 kB\nCached: 7388212 kB\nSwapCached: 0 kB\nActive: 5927072 kB\nInactive: 6130236 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 20488140 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 508 kB\nWriteback: 0 kB\nAnonPages: 4453816 kB\nMapped: 15316 kB\nSlab: 278128 kB\nPageTables: 12944 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6171428 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,907286,689871}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,93397128}, {processes,52604552}, {processes_used,52536080}, {system,40792576}, {atom,1352041}, {atom_used,1349412}, {binary,2752040}, {code,13622682}, {ets,5016688}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,6846967808}, {buffered_memory,250638336}, {free_memory,21641166848}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1494440,4999}}, {context_switches,{5544337,0}}, {garbage_collection,{612191,1300040075,0}}, {io,{{input,100405587},{output,69430883}}}, {reductions,{609397518,1872811}}, {run_queue,0}, {runtime,{204990,500}}]}, {system_stats, [{cpu_utilization_rate,3.25}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,6231450962}, {couch_docs_data_size,6054380609}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2497102}, {curr_items_tot,2497102}, {mem_used,3652502072}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1494}, {memory_data,{33657663488,12017901568,{<18117.1696.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 21133952 kB\nBuffers: 244764 kB\nCached: 6686492 kB\nSwapCached: 140 kB\nActive: 5103476 kB\nInactive: 6325432 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 21133952 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 5376 kB\nWriteback: 0 kB\nAnonPages: 4497592 kB\nMapped: 17060 kB\nSlab: 258720 kB\nPageTables: 13636 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6208988 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,907285,659476}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,91449064}, {processes,49955920}, {processes_used,49869048}, {system,41493144}, {atom,1352041}, {atom_used,1349412}, {binary,3341184}, {code,13631938}, {ets,5112072}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,6781882368}, {buffered_memory,425050112}, {free_memory,21552652288}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1494397,5000}}, {context_switches,{5970768,0}}, {garbage_collection,{640538,1354964299,0}}, {io,{{input,107545951},{output,72223962}}}, {reductions,{639752462,2084053}}, {run_queue,0}, {runtime,{206670,590}}]}, {system_stats, [{cpu_utilization_rate,3.0},{swap_total,10701078528},{swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,6125934003}, {couch_docs_data_size,6074547599}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2503544}, {curr_items_tot,2503544}, {mem_used,3506636856}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1494}, {memory_data,{33657663488,12600225792,{<0.1678.0>,2914384}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 21047512 kB\nBuffers: 415088 kB\nCached: 6622932 kB\nSwapCached: 0 kB\nActive: 5115412 kB\nInactive: 6386168 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 21047512 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 1848 kB\nWriteback: 0 kB\nAnonPages: 4463632 kB\nMapped: 17060 kB\nSlab: 274284 kB\nPageTables: 13580 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 6170616 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,907289,3125}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,92415432}, {processes,50763008}, {processes_used,50676136}, {system,41652424}, {atom,1352041}, {atom_used,1349412}, {binary,3477872}, {code,13659922}, {ets,5092576}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,8808116224}, {buffered_memory,319152128}, {free_memory,19595718656}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1496430,5000}}, {context_switches,{6327557,0}}, {garbage_collection,{684285,1389331296,0}}, {io,{{input,103951552},{output,72920392}}}, {reductions,{655085251,2630974}}, {run_queue,0}, {runtime,{215530,540}}]}, {system_stats, [{cpu_utilization_rate,2.75}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,8200766792}, {couch_docs_data_size,6107702164}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2501801}, {curr_items_tot,2501801}, {mem_used,3488892632}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1496}, {memory_data,{33657663488,14063849472,{<18118.1662.0>,2914384}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 19136444 kB\nBuffers: 311672 kB\nCached: 8601676 kB\nSwapCached: 12 kB\nActive: 7566664 kB\nInactive: 5793336 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 19136444 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 328 kB\nWriteback: 0 kB\nAnonPages: 4446844 kB\nMapped: 17260 kB\nSlab: 324796 kB\nPageTables: 13544 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6147340 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-02T18:48:17.517,ns_1@10.2.1.67:<0.1687.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,31,47}}) Stats for bucket "default": accepting_conns 1 auth_cmds 3 auth_errors 0 bucket_active_conns 1 bucket_conns 5 bytes 3506636824 bytes_read 6011825977 bytes_written 133554692 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 1 cmd_set 2503544 conn_yields 595 connection_structures 5000 curr_connections 9 curr_conns_on_port_11209 6 curr_conns_on_port_11210 1 curr_items 2503544 curr_items_tot 2503544 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-03 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 1499 ep_commit_time 27 ep_commit_time_total 18606 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 193 ep_data_age_highwat 209 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data/default ep_degraded_mode 0 ep_diskqueue_drain 2503544 ep_diskqueue_fill 2503544 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 1074 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 2503544 ep_io_read_bytes 0 ep_io_write_bytes 5931572051 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 2498272 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 3443869792 ep_max_checkpoints 2 ep_max_data_size 5242880000 ep_max_item_size 20971520 ep_max_size 5242880000 ep_max_txn_size 10000 ep_max_vbuckets 512 ep_mem_high_wat 3932160000 ep_mem_high_wat 3932160000 ep_mem_low_wat 3145728000 ep_mem_low_wat 3145728000 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 393411 ep_num_expiry_pager_runs 0 ep_num_non_resident 1142582 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 12 ep_num_value_ejects 1142582 ep_oom_errors 0 ep_overhead 13499600 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1351905991 ep_storage_age 190 ep_storage_age_highwat 208 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 3413827264 ep_total_del_items 0 ep_total_enqueued 2503544 ep_total_new_items 2503544 ep_total_persisted 2503544 ep_uncommitted_items 0 ep_value_size 3236781315 ep_vb0 0 ep_vb_snapshot_total 62 ep_vb_total 128 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_130_g8b09894 ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 725 get_hits 0 get_misses 1 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 3506636824 pid 16451 pointer_size 64 rejected_conns 0 rusage_system 104.065179 rusage_user 287.976220 threads 4 time 1351907296 total_connections 13 uptime 1328 vb_active_curr_items 2503544 vb_active_eject 1142582 vb_active_expired 0 vb_active_ht_memory 12645376 vb_active_itm_memory 3413827264 vb_active_meta_data_memory 197779976 vb_active_num 128 vb_active_num_non_resident 1142582 vb_active_num_ref_ejects 1142582 vb_active_num_ref_items 2503544 vb_active_ops_create 2503544 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 54 vb_active_queue_age 0 vb_active_queue_drain 2503544 vb_active_queue_fill 2503544 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_meta_data_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-02T18:49:10.658,ns_1@10.2.1.67:ns_doctor<0.1461.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,907349,24997}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.24220.0>">>}, {changes_done,1212367}, {design_documents,[<<"_design/A">>]}, {indexer_type,main}, {initial_build,true}, {progress,48}, {set,<<"default">>}, {signature,<<"afeb9408e40add49ae0c9bf58f0c5f26">>}, {started_on,1351907326}, {total_changes,2497553}, {type,indexer}, {updated_on,1351907378}]]}, {memory, [{total,156430688}, {processes,105043200}, {processes_used,103518728}, {system,51387488}, {atom,1492169}, {atom_used,1478827}, {binary,11104432}, {code,15028654}, {ets,5441704}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,7671975936}, {buffered_memory,220909568}, {free_memory,20823126016}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1557192,4999}}, {context_switches,{8443336,0}}, {garbage_collection,{851219,4461573706,0}}, {io,{{input,8932023294},{output,289826225}}}, {reductions,{1944859327,106216775}}, {run_queue,0}, {runtime,{341730,8910}}]}, {system_stats, [{cpu_utilization_rate,50.62344139650873}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,6947493243}, {couch_docs_data_size,6073317216}, {couch_views_actual_disk_size,87783975}, {couch_views_data_size,8775}, {curr_items,2497553}, {curr_items_tot,2497553}, {mem_used,3561592760}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1557}, {memory_data,{33657663488,12678443008,{<18077.16820.0>,7028184}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 20335084 kB\nBuffers: 215732 kB\nCached: 7491836 kB\nSwapCached: 0 kB\nActive: 8914280 kB\nInactive: 3292544 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 20335084 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 103068 kB\nWriteback: 0 kB\nAnonPages: 4499220 kB\nMapped: 16276 kB\nSlab: 281736 kB\nPageTables: 13576 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6241424 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,907346,706124}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.14982.0>">>}, {changes_done,1103743}, {design_documents,[<<"_design/A">>]}, {indexer_type,main}, {initial_build,true}, {progress,44}, {set,<<"default">>}, {signature,<<"afeb9408e40add49ae0c9bf58f0c5f26">>}, {started_on,1351907259}, {total_changes,2497102}, {type,indexer}, {updated_on,1351907306}]]}, {memory, [{total,167257312}, {processes,116945064}, {processes_used,116895856}, {system,50312248}, {atom,1408249}, {atom_used,1384271}, {binary,11184752}, {code,14415880}, {ets,5245224}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,6933258240}, {buffered_memory,251187200}, {free_memory,21441290240}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1554455,5014}}, {context_switches,{8372411,0}}, {garbage_collection,{872891,3526362154,0}}, {io,{{input,8254460065},{output,252556207}}}, {reductions,{1704984138,97538104}}, {run_queue,1}, {runtime,{299300,8370}}]}, {system_stats, [{cpu_utilization_rate,44.25}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,6231455103}, {couch_docs_data_size,6054380609}, {couch_views_actual_disk_size,78478854}, {couch_views_data_size,8776}, {curr_items,2497102}, {curr_items_tot,2497102}, {mem_used,3652502072}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1554}, {memory_data,{33657663488,12026408960,{<18117.1696.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 20938760 kB\nBuffers: 245300 kB\nCached: 6770760 kB\nSwapCached: 140 kB\nActive: 8058820 kB\nInactive: 3561540 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 20938760 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 87416 kB\nWriteback: 0 kB\nAnonPages: 4604312 kB\nMapped: 17964 kB\nSlab: 261228 kB\nPageTables: 13868 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6335440 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,907345,658620}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.16400.0>">>}, {changes_done,1173880}, {design_documents,[<<"_design/A">>]}, {indexer_type,main}, {initial_build,true}, {progress,46}, {set,<<"default">>}, {signature,<<"afeb9408e40add49ae0c9bf58f0c5f26">>}, {started_on,1351907293}, {total_changes,2503544}, {type,indexer}, {updated_on,1351907343}]]}, {memory, [{total,130167320}, {processes,80627816}, {processes_used,80565064}, {system,49539504}, {atom,1408249}, {atom_used,1384271}, {binary,10197432}, {code,14425136}, {ets,5228432}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,6877495296}, {buffered_memory,425545728}, {free_memory,21397471232}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1554397,4997}}, {context_switches,{8802774,0}}, {garbage_collection,{905917,3583062610,0}}, {io,{{input,8321381795},{output,261407407}}}, {reductions,{1739378157,100658289}}, {run_queue,0}, {runtime,{299150,8450}}]}, {system_stats, [{cpu_utilization_rate,52.63157894736842}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,6128137651}, {couch_docs_data_size,6075992082}, {couch_views_actual_disk_size,80717204}, {couch_views_data_size,8782}, {curr_items,2503544}, {curr_items_tot,2503544}, {mem_used,3506636808}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1554}, {memory_data,{33657663488,12107931648,{<0.1678.0>,2914384}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 20895968 kB\nBuffers: 415572 kB\nCached: 6716304 kB\nSwapCached: 0 kB\nActive: 8038892 kB\nInactive: 3612632 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 20895968 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 93500 kB\nWriteback: 0 kB\nAnonPages: 4519840 kB\nMapped: 17964 kB\nSlab: 275292 kB\nPageTables: 13932 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 6251196 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,907349,2637}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.15038.0>">>}, {changes_done,1239235}, {design_documents,[<<"_design/A">>]}, {indexer_type,main}, {initial_build,true}, {progress,49}, {set,<<"default">>}, {signature,<<"afeb9408e40add49ae0c9bf58f0c5f26">>}, {started_on,1351907286}, {total_changes,2501801}, {type,indexer}, {updated_on,1351907340}]]}, {memory, [{total,151927984}, {processes,101612496}, {processes_used,101547688}, {system,50315488}, {atom,1408249}, {atom_used,1384271}, {binary,11167904}, {code,14453120}, {ets,5208776}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,8899092480}, {buffered_memory,319717376}, {free_memory,19407151104}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1556431,5001}}, {context_switches,{9261677,0}}, {garbage_collection,{953335,3696893921,0}}, {io,{{input,8568724348},{output,277930901}}}, {reductions,{1793951730,105594272}}, {run_queue,2}, {runtime,{314460,9060}}]}, {system_stats, [{cpu_utilization_rate,52.5}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,8200770933}, {couch_docs_data_size,6107702164}, {couch_views_actual_disk_size,95533626}, {couch_views_data_size,8774}, {curr_items,2501801}, {curr_items_tot,2501801}, {mem_used,3488892632}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1556}, {memory_data,{33657663488,14069309440,{<18118.1662.0>,2914384}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 18952420 kB\nBuffers: 312224 kB\nCached: 8690520 kB\nSwapCached: 12 kB\nActive: 10374704 kB\nInactive: 3166416 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 18952420 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 88000 kB\nWriteback: 0 kB\nAnonPages: 4538468 kB\nMapped: 18164 kB\nSlab: 327824 kB\nPageTables: 13656 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6256848 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-02T18:49:57.514,ns_1@10.2.1.67:<0.1687.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,31,47}}) Stats for bucket "default": accepting_conns 1 auth_cmds 3 auth_errors 0 bucket_active_conns 1 bucket_conns 5 bytes 3506636808 bytes_read 6011836467 bytes_written 140145372 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 1 cmd_set 2503544 conn_yields 595 connection_structures 5000 curr_connections 9 curr_conns_on_port_11209 6 curr_conns_on_port_11210 1 curr_items 2503544 curr_items_tot 2503544 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-03 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 1499 ep_commit_time 27 ep_commit_time_total 18606 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 193 ep_data_age_highwat 209 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data/default ep_degraded_mode 0 ep_diskqueue_drain 2503544 ep_diskqueue_fill 2503544 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 1074 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 2503544 ep_io_read_bytes 0 ep_io_write_bytes 5931572051 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 2498272 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 3443869792 ep_max_checkpoints 2 ep_max_data_size 5242880000 ep_max_item_size 20971520 ep_max_size 5242880000 ep_max_txn_size 10000 ep_max_vbuckets 512 ep_mem_high_wat 3932160000 ep_mem_high_wat 3932160000 ep_mem_low_wat 3145728000 ep_mem_low_wat 3145728000 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 393411 ep_num_expiry_pager_runs 0 ep_num_non_resident 1142582 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 12 ep_num_value_ejects 1142582 ep_oom_errors 0 ep_overhead 13499600 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1351905991 ep_storage_age 190 ep_storage_age_highwat 208 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 3413827264 ep_total_del_items 0 ep_total_enqueued 2503544 ep_total_new_items 2503544 ep_total_persisted 2503544 ep_uncommitted_items 0 ep_value_size 3236781315 ep_vb0 0 ep_vb_snapshot_total 62 ep_vb_total 128 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_130_g8b09894 ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 725 get_hits 0 get_misses 1 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 3506636808 pid 16451 pointer_size 64 rejected_conns 0 rusage_system 104.073178 rusage_user 288.086204 threads 4 time 1351907397 total_connections 13 uptime 1429 vb_active_curr_items 2503544 vb_active_eject 1142582 vb_active_expired 0 vb_active_ht_memory 12645376 vb_active_itm_memory 3413827264 vb_active_meta_data_memory 197779976 vb_active_num 128 vb_active_num_non_resident 1142582 vb_active_num_ref_ejects 1142582 vb_active_num_ref_items 2503544 vb_active_ops_create 2503544 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 54 vb_active_queue_age 0 vb_active_queue_drain 2503544 vb_active_queue_fill 2503544 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_meta_data_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-02T18:50:10.658,ns_1@10.2.1.67:ns_doctor<0.1461.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,907409,23406}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.24220.0>">>}, {changes_done,2397729}, {design_documents,[<<"_design/A">>]}, {indexer_type,main}, {initial_build,true}, {progress,96}, {set,<<"default">>}, {signature,<<"afeb9408e40add49ae0c9bf58f0c5f26">>}, {started_on,1351907326}, {total_changes,2497553}, {type,indexer}, {updated_on,1351907429}]]}, {memory, [{total,218000760}, {processes,164873768}, {processes_used,163341248}, {system,53126992}, {atom,1492169}, {atom_used,1478827}, {binary,12748800}, {code,15028654}, {ets,5628136}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,7795040256}, {buffered_memory,221736960}, {free_memory,20649566208}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1617192,5000}}, {context_switches,{11302747,0}}, {garbage_collection,{1117562,7117111561,0}}, {io,{{input,17175009908},{output,649502743}}}, {reductions,{3154931804,69986467}}, {run_queue,0}, {runtime,{444390,5040}}]}, {system_stats, [{cpu_utilization_rate,27.0}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,6947493243}, {couch_docs_data_size,6073317216}, {couch_views_actual_disk_size,210864569}, {couch_views_data_size,8775}, {curr_items,2497553}, {curr_items_tot,2497553}, {mem_used,3561592760}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1617}, {memory_data,{33657663488,12853837824,{<18077.24220.0>,36982592}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 20165592 kB\nBuffers: 216540 kB\nCached: 7612344 kB\nSwapCached: 0 kB\nActive: 11760748 kB\nInactive: 610460 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 20165592 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 207096 kB\nWriteback: 0 kB\nAnonPages: 4542032 kB\nMapped: 16276 kB\nSlab: 285664 kB\nPageTables: 13356 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6287788 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,907406,691374}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.14982.0>">>}, {changes_done,2437524}, {design_documents,[<<"_design/A">>]}, {indexer_type,main}, {initial_build,true}, {progress,97}, {set,<<"default">>}, {signature,<<"afeb9408e40add49ae0c9bf58f0c5f26">>}, {started_on,1351907259}, {total_changes,2497102}, {type,indexer}, {updated_on,1351907364}]]}, {memory, [{total,161781664}, {processes,115234504}, {processes_used,115182464}, {system,46547160}, {atom,1408249}, {atom_used,1384271}, {binary,7102336}, {code,14415880}, {ets,5435232}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,7050788864}, {buffered_memory,251936768}, {free_memory,21354201088}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1614441,5000}}, {context_switches,{11346589,0}}, {garbage_collection,{1148717,6155764336,0}}, {io,{{input,16875509477},{output,605952688}}}, {reductions,{2931592485,70822695}}, {run_queue,0}, {runtime,{403060,4930}}]}, {system_stats, [{cpu_utilization_rate,27.0}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,6231455103}, {couch_docs_data_size,6054380609}, {couch_views_actual_disk_size,188476104}, {couch_views_data_size,8776}, {curr_items,2497102}, {curr_items_tot,2497102}, {mem_used,3652502088}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1614}, {memory_data,{33657663488,12237205504,{<18117.14982.0>,74966016}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 20853960 kB\nBuffers: 246032 kB\nCached: 6885536 kB\nSwapCached: 140 kB\nActive: 11008076 kB\nInactive: 694572 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 20853960 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 184776 kB\nWriteback: 0 kB\nAnonPages: 4570736 kB\nMapped: 17964 kB\nSlab: 264156 kB\nPageTables: 13864 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6295420 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,907405,660266}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.16400.0>">>}, {changes_done,2478111}, {design_documents,[<<"_design/A">>]}, {indexer_type,main}, {initial_build,true}, {progress,98}, {set,<<"default">>}, {signature,<<"afeb9408e40add49ae0c9bf58f0c5f26">>}, {started_on,1351907293}, {total_changes,2503544}, {type,indexer}, {updated_on,1351907399}]]}, {memory, [{total,139819936}, {processes,95471864}, {processes_used,95404544}, {system,44348072}, {atom,1408249}, {atom_used,1384271}, {binary,4864568}, {code,14425136}, {ets,5414136}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,6999027712}, {buffered_memory,426168320}, {free_memory,21279526912}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1614398,5001}}, {context_switches,{11787534,0}}, {garbage_collection,{1185260,6235285371,0}}, {io,{{input,16986715384},{output,606882092}}}, {reductions,{2978512914,71098452}}, {run_queue,0}, {runtime,{401450,4800}}]}, {system_stats, [{cpu_utilization_rate,26.31578947368421}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,6128137605}, {couch_docs_data_size,6075992082}, {couch_views_actual_disk_size,253368506}, {couch_views_data_size,8782}, {curr_items,2503544}, {curr_items_tot,2503544}, {mem_used,3506636808}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1614}, {memory_data,{33657663488,12314157056,{<0.16400.0>,74965952}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 20780912 kB\nBuffers: 416180 kB\nCached: 6834988 kB\nSwapCached: 0 kB\nActive: 11026100 kB\nInactive: 736396 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 20780912 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 189388 kB\nWriteback: 0 kB\nAnonPages: 4511372 kB\nMapped: 17964 kB\nSlab: 279872 kB\nPageTables: 13760 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 6233652 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,907409,1394}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.15038.0>">>}, {changes_done,2445274}, {design_documents,[<<"_design/A">>]}, {indexer_type,main}, {initial_build,true}, {progress,97}, {set,<<"default">>}, {signature,<<"afeb9408e40add49ae0c9bf58f0c5f26">>}, {started_on,1351907286}, {total_changes,2501801}, {type,indexer}, {updated_on,1351907393}]]}, {memory, [{total,180014112}, {processes,131949240}, {processes_used,131881920}, {system,48064872}, {atom,1408249}, {atom_used,1384271}, {binary,8562712}, {code,14453120}, {ets,5392520}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,9023385600}, {buffered_memory,320380928}, {free_memory,19289075712}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1616430,5000}}, {context_switches,{12142648,0}}, {garbage_collection,{1218140,6248382322,0}}, {io,{{input,16887642569},{output,604697649}}}, {reductions,{2984974958,71139184}}, {run_queue,0}, {runtime,{416930,4790}}]}, {system_stats, [{cpu_utilization_rate,26.31578947368421}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,8200770933}, {couch_docs_data_size,6107702164}, {couch_views_actual_disk_size,193120462}, {couch_views_data_size,8774}, {curr_items,2501801}, {curr_items_tot,2501801}, {mem_used,3488892632}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1616}, {memory_data,{33657663488,14244925440,{<18118.15038.0>,51855896}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 18837112 kB\nBuffers: 312872 kB\nCached: 8811900 kB\nSwapCached: 12 kB\nActive: 13021424 kB\nInactive: 630920 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 18837112 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 187776 kB\nWriteback: 0 kB\nAnonPages: 4527600 kB\nMapped: 18164 kB\nSlab: 331188 kB\nPageTables: 13820 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6253464 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-02T18:51:10.658,ns_1@10.2.1.67:ns_doctor<0.1461.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,907469,23757}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,119745424}, {processes,70403016}, {processes_used,68866856}, {system,49342408}, {atom,1495401}, {atom_used,1483293}, {binary,8829288}, {code,15096052}, {ets,5820224}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,7665864704}, {buffered_memory,222097408}, {free_memory,20839997440}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1677193,5000}}, {context_switches,{12024169,0}}, {garbage_collection,{1132436,8278787916,0}}, {io,{{input,17504464070},{output,885446021}}}, {reductions,{4417499317,2229775}}, {run_queue,0}, {runtime,{472350,620}}]}, {system_stats, [{cpu_utilization_rate,3.508771929824561}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,6947493243}, {couch_docs_data_size,6073317216}, {couch_views_actual_disk_size,102294458}, {couch_views_data_size,102278150}, {curr_items,2497553}, {curr_items_tot,2497553}, {mem_used,3561592760}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1677}, {memory_data,{33657663488,13047967744,{<18077.24220.0>,91014736}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 20351560 kB\nBuffers: 216892 kB\nCached: 7486196 kB\nSwapCached: 0 kB\nActive: 11669588 kB\nInactive: 519116 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 20351560 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 96252 kB\nWriteback: 0 kB\nAnonPages: 4485832 kB\nMapped: 16276 kB\nSlab: 283380 kB\nPageTables: 13292 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6202228 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,907466,691048}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,92841048}, {processes,50240040}, {processes_used,50184360}, {system,42601008}, {atom,1411481}, {atom_used,1388780}, {binary,3022632}, {code,14483278}, {ets,5626880}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,6947127296}, {buffered_memory,252309504}, {free_memory,21510586368}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1674440,4999}}, {context_switches,{12075709,0}}, {garbage_collection,{1165186,7406865828,0}}, {io,{{input,17213551371},{output,867063780}}}, {reductions,{4215286948,1851677}}, {run_queue,0}, {runtime,{432030,520}}]}, {system_stats, [{cpu_utilization_rate,3.0}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,6231455103}, {couch_docs_data_size,6054380609}, {couch_views_actual_disk_size,102421439}, {couch_views_data_size,102406871}, {curr_items,2497102}, {curr_items_tot,2497102}, {mem_used,3652502088}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1674}, {memory_data,{33657663488,12306178048,{<18117.14982.0>,58013208}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 21006432 kB\nBuffers: 246396 kB\nCached: 6784304 kB\nSwapCached: 140 kB\nActive: 10947324 kB\nInactive: 603060 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 21006432 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 98648 kB\nWriteback: 0 kB\nAnonPages: 4519424 kB\nMapped: 17964 kB\nSlab: 262956 kB\nPageTables: 13748 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6226900 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,907465,660385}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,98180576}, {processes,54793784}, {processes_used,54722584}, {system,43386792}, {atom,1411481}, {atom_used,1388780}, {binary,3796992}, {code,14492534}, {ets,5610872}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,6882168832}, {buffered_memory,426565632}, {free_memory,21428871168}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1674398,5000}}, {context_switches,{12517002,0}}, {garbage_collection,{1198531,7496078299,0}}, {io,{{input,17322177054},{output,865491723}}}, {reductions,{4262450711,1898093}}, {run_queue,0}, {runtime,{428040,470}}]}, {system_stats, [{cpu_utilization_rate,3.2418952618453867}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,6128137605}, {couch_docs_data_size,6075992082}, {couch_views_actual_disk_size,102691789}, {couch_views_data_size,102676813}, {curr_items,2503544}, {curr_items_tot,2503544}, {mem_used,3506636808}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1674}, {memory_data,{33657663488,12343693312,{<0.16400.0>,36982640}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 20926632 kB\nBuffers: 416568 kB\nCached: 6720868 kB\nSwapCached: 0 kB\nActive: 10968620 kB\nInactive: 651868 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 20926632 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 96468 kB\nWriteback: 0 kB\nAnonPages: 4482748 kB\nMapped: 17964 kB\nSlab: 276936 kB\nPageTables: 13680 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 6185108 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,907469,1034}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,96907280}, {processes,53604648}, {processes_used,53533448}, {system,43302632}, {atom,1411481}, {atom_used,1388780}, {binary,3705312}, {code,14520518}, {ets,5588744}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,8910393344}, {buffered_memory,320856064}, {free_memory,19462094848}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1676430,5000}}, {context_switches,{12871802,0}}, {garbage_collection,{1233122,7535304521,0}}, {io,{{input,17224617154},{output,864914659}}}, {reductions,{4269483583,1844373}}, {run_queue,0}, {runtime,{445940,480}}]}, {system_stats, [{cpu_utilization_rate,2.75}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,8200770933}, {couch_docs_data_size,6107702164}, {couch_views_actual_disk_size,102585286}, {couch_views_data_size,102570159}, {curr_items,2501801}, {curr_items_tot,2501801}, {mem_used,3488892632}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1676}, {memory_data,{33657663488,14413533184,{<18118.15038.0>,74966056}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 19005952 kB\nBuffers: 313336 kB\nCached: 8701556 kB\nSwapCached: 12 kB\nActive: 12936580 kB\nInactive: 549284 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 19005952 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 96632 kB\nWriteback: 0 kB\nAnonPages: 4470888 kB\nMapped: 18164 kB\nSlab: 329536 kB\nPageTables: 13672 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6169900 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-02T18:51:37.534,ns_1@10.2.1.67:<0.1687.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,31,47}}) Stats for bucket "default": accepting_conns 1 auth_cmds 3 auth_errors 0 bucket_active_conns 1 bucket_conns 8 bytes 2933093592 bytes_read 6012099496 bytes_written 1921776867 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 1 cmd_set 2503544 conn_yields 595 connection_structures 5000 curr_connections 11 curr_conns_on_port_11209 8 curr_conns_on_port_11210 1 curr_items 2014449 curr_items_tot 2014449 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-03 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 1499 ep_commit_time 27 ep_commit_time_total 18606 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 193 ep_data_age_highwat 209 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data/default ep_degraded_mode 0 ep_diskqueue_drain 2014449 ep_diskqueue_fill 2014449 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 1074 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 2503544 ep_io_read_bytes 0 ep_io_write_bytes 5931572051 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 2498272 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 2879356447 ep_max_checkpoints 2 ep_max_data_size 5242880000 ep_max_item_size 20971520 ep_max_size 5242880000 ep_max_txn_size 10000 ep_max_vbuckets 512 ep_mem_high_wat 3932160000 ep_mem_high_wat 3932160000 ep_mem_low_wat 3145728000 ep_mem_low_wat 3145728000 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 393411 ep_num_expiry_pager_runs 0 ep_num_non_resident 874616 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 12 ep_num_value_ejects 1142582 ep_oom_errors 0 ep_overhead 11600910 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1351905991 ep_storage_age 190 ep_storage_age_highwat 208 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 507331 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 2851656247 ep_total_del_items 0 ep_total_enqueued 2503544 ep_total_new_items 2503544 ep_total_persisted 2503544 ep_uncommitted_items 0 ep_value_size 2712748422 ep_vb0 0 ep_vb_snapshot_total 112 ep_vb_total 103 ep_vbucket_del 25 ep_vbucket_del_avg_walltime 55927 ep_vbucket_del_fail 0 ep_vbucket_del_max_walltime 82294 ep_version 2.0.0r_130_g8b09894 ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 725 get_hits 0 get_misses 1 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 2933093592 pid 16451 pointer_size 64 rejected_conns 0 rusage_system 109.874296 rusage_user 297.718739 tap_checkpoint_start_sent 50 tap_connect_received 51 tap_mutation_sent 731646 tap_opaque_sent 153 tap_vbucket_set_sent 50 threads 4 time 1351907497 total_connections 140 uptime 1529 vb_active_curr_items 2014449 vb_active_eject 874616 vb_active_expired 0 vb_active_ht_memory 10175576 vb_active_itm_memory 2851656247 vb_active_meta_data_memory 159141471 vb_active_num 103 vb_active_num_non_resident 874616 vb_active_num_ref_ejects 874616 vb_active_num_ref_items 2014449 vb_active_ops_create 2014449 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 56 vb_active_queue_age 0 vb_active_queue_drain 2014449 vb_active_queue_fill 2014449 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_meta_data_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-02T18:52:10.658,ns_1@10.2.1.67:ns_doctor<0.1461.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,907529,21578}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.27766.0>">>}, {changes_done,40215}, {design_documents,[<<"_design/A">>]}, {indexer_type,main}, {initial_build,false}, {progress,5}, {set,<<"default">>}, {signature,<<"afeb9408e40add49ae0c9bf58f0c5f26">>}, {started_on,1351907547}, {total_changes,749784}, {type,indexer}, {updated_on,1351907560}]]}, {memory, [{total,156386168}, {processes,102239336}, {processes_used,100902536}, {system,54146832}, {atom,1502673}, {atom_used,1494165}, {binary,13168472}, {code,15250635}, {ets,5938488}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,10368225280}, {buffered_memory,226066432}, {free_memory,17428942848}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1737192,5000}}, {context_switches,{15286785,0}}, {garbage_collection,{1458906,11749886634,0}}, {io,{{input,23233051643},{output,4555828850}}}, {reductions,{5601891421,90781454}}, {run_queue,0}, {runtime,{564960,6850}}]}, {system_stats, [{cpu_utilization_rate,39.097744360902254}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,9001244801}, {couch_docs_data_size,8110492466}, {couch_views_actual_disk_size,722497560}, {couch_views_data_size,109463066}, {curr_items,3339073}, {curr_items_tot,3339073}, {mem_used,3742703352}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1737}, {memory_data,{33657663488,12818935808,{<18077.24182.0>,10771072}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 17020576 kB\nBuffers: 220768 kB\nCached: 10125220 kB\nSwapCached: 0 kB\nActive: 12968544 kB\nInactive: 2477176 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 17020576 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 716208 kB\nWriteback: 0 kB\nAnonPages: 5099412 kB\nMapped: 16312 kB\nSlab: 356212 kB\nPageTables: 14396 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6845972 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,907526,690574}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.16391.0>">>}, {changes_done,0}, {design_documents,[<<"_design/A">>]}, {indexer_type,main}, {initial_build,false}, {progress,0}, {set,<<"default">>}, {signature,<<"afeb9408e40add49ae0c9bf58f0c5f26">>}, {started_on,1351907491}, {total_changes,178672}, {type,indexer}, {updated_on,1351907491}]]}, {memory, [{total,102264208}, {processes,56473768}, {processes_used,56457704}, {system,45790440}, {atom,1413905}, {atom_used,1392343}, {binary,5999584}, {code,14559765}, {ets,5728536}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,7498792960}, {buffered_memory,253796352}, {free_memory,20833738752}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1734440,4995}}, {context_switches,{12673394,0}}, {garbage_collection,{1231078,7853523764,0}}, {io,{{input,17786362344},{output,1769186316}}}, {reductions,{4375379926,109896156}}, {run_queue,0}, {runtime,{448820,8200}}]}, {system_stats, [{cpu_utilization_rate,68.1592039800995}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,6568935396}, {couch_docs_data_size,6338834156}, {couch_views_actual_disk_size,122914957}, {couch_views_data_size,102916851}, {curr_items,2712005}, {curr_items_tot,2722478}, {mem_used,4188986744}, {vb_replica_curr_items,10473}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1734}, {memory_data,{33657663488,12147331072,{<18117.1696.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 20345448 kB\nBuffers: 247848 kB\nCached: 7323040 kB\nSwapCached: 140 kB\nActive: 11182228 kB\nInactive: 1015592 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 20345448 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 187720 kB\nWriteback: 0 kB\nAnonPages: 4626216 kB\nMapped: 18016 kB\nSlab: 277592 kB\nPageTables: 13924 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6351760 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,907525,659124}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,93837400}, {processes,51693056}, {processes_used,51390856}, {system,42144344}, {atom,1413905}, {atom_used,1392521}, {binary,2311024}, {code,14569021}, {ets,5804192}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,4983726080}, {buffered_memory,428072960}, {free_memory,23337168896}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1734397,5000}}, {context_switches,{13840265,0}}, {garbage_collection,{1971868,10657725569,0}}, {io,{{input,19417474184},{output,1528601324}}}, {reductions,{5056320762,74949607}}, {run_queue,0}, {runtime,{481600,5360}}]}, {system_stats, [{cpu_utilization_rate,49.75124378109453}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,3730345739}, {couch_docs_data_size,3702376996}, {couch_views_actual_disk_size,703039097}, {couch_views_data_size,91422121}, {curr_items,1466511}, {curr_items_tot,1466511}, {mem_used,2222857048}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1734}, {memory_data,{33657663488,12230569984,{<0.1678.0>,4714984}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 22790328 kB\nBuffers: 418040 kB\nCached: 4866920 kB\nSwapCached: 0 kB\nActive: 9148164 kB\nInactive: 618472 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 22790328 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 12276 kB\nWriteback: 0 kB\nAnonPages: 4481452 kB\nMapped: 18052 kB\nSlab: 266820 kB\nPageTables: 13672 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 6187760 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,907529,223}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,98057368}, {processes,54699528}, {processes_used,54628400}, {system,43357840}, {atom,1411481}, {atom_used,1388780}, {binary,3555072}, {code,14520518}, {ets,5793512}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,8911548416}, {buffered_memory,321671168}, {free_memory,19460702208}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1736430,5000}}, {context_switches,{12923714,0}}, {garbage_collection,{1238151,7591831595,0}}, {io,{{input,17226813295},{output,869744273}}}, {reductions,{4294521048,2032981}}, {run_queue,0}, {runtime,{451940,470}}]}, {system_stats, [{cpu_utilization_rate,2.743142144638404}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,8200770933}, {couch_docs_data_size,6107702164}, {couch_views_actual_disk_size,102585286}, {couch_views_data_size,102570159}, {curr_items,2501801}, {curr_items_tot,2501801}, {mem_used,3488892632}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1736}, {memory_data,{33657663488,14197092352,{<18118.1663.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 19004592 kB\nBuffers: 314132 kB\nCached: 8702684 kB\nSwapCached: 12 kB\nActive: 12936396 kB\nInactive: 550972 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 19004592 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 88160 kB\nWriteback: 32 kB\nAnonPages: 4470220 kB\nMapped: 18164 kB\nSlab: 329144 kB\nPageTables: 13656 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6174096 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-02T18:53:10.659,ns_1@10.2.1.67:ns_doctor<0.1461.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,907589,20387}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.27766.0>">>}, {changes_done,236143}, {design_documents,[<<"_design/A">>]}, {indexer_type,main}, {initial_build,false}, {progress,31}, {set,<<"default">>}, {signature,<<"afeb9408e40add49ae0c9bf58f0c5f26">>}, {started_on,1351907547}, {total_changes,749784}, {type,indexer}, {updated_on,1351907620}]]}, {memory, [{total,153640952}, {processes,98966400}, {processes_used,97629456}, {system,54674552}, {atom,1502673}, {atom_used,1494165}, {binary,13516976}, {code,15250635}, {ets,6117664}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,11143548928}, {buffered_memory,227749888}, {free_memory,16624668672}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1797192,4999}}, {context_switches,{17124058,0}}, {garbage_collection,{1558084,15467532426,0}}, {io,{{input,26694546551},{output,5354005845}}}, {reductions,{6672589016,88507214}}, {run_queue,1}, {runtime,{646530,7030}}]}, {system_stats, [{cpu_utilization_rate,31.67082294264339}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,9001244807}, {couch_docs_data_size,8110492466}, {couch_views_actual_disk_size,1500073518}, {couch_views_data_size,117287539}, {curr_items,3339073}, {curr_items_tot,3339073}, {mem_used,3744910056}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1797}, {memory_data,{33657663488,16270241792,{<18077.27766.0>,12342552}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16235028 kB\nBuffers: 222412 kB\nCached: 10882372 kB\nSwapCached: 0 kB\nActive: 14170008 kB\nInactive: 2041288 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16235028 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 1461844 kB\nWriteback: 0 kB\nAnonPages: 5105848 kB\nMapped: 16312 kB\nSlab: 376452 kB\nPageTables: 14492 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6851740 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,907586,689990}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.17170.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,4}, {started_on,1351907527}, {total_vbuckets,166}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351907549}, {vbuckets_done,8}], [{pid,<<"<0.17372.0>">>}, {changes_done,38349}, {design_documents,[<<"_design/A">>]}, {indexer_type,main}, {initial_build,false}, {progress,4}, {set,<<"default">>}, {signature,<<"afeb9408e40add49ae0c9bf58f0c5f26">>}, {started_on,1351907537}, {total_changes,826720}, {type,indexer}, {updated_on,1351907549}]]}, {memory, [{total,127021968}, {processes,75411184}, {processes_used,75395432}, {system,51610784}, {atom,1413905}, {atom_used,1392343}, {binary,11508800}, {code,14559765}, {ets,5925792}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,10140286976}, {buffered_memory,257961984}, {free_memory,17783164928}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1794439,4998}}, {context_switches,{16128932,0}}, {garbage_collection,{1524180,11565056607,0}}, {io,{{input,23066638494},{output,4868467330}}}, {reductions,{5572029749,92455446}}, {run_queue,0}, {runtime,{538070,6370}}]}, {system_stats, [{cpu_utilization_rate,37.905236907730675}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,8551439041}, {couch_docs_data_size,7899622768}, {couch_views_actual_disk_size,863749895}, {couch_views_data_size,110556474}, {curr_items,3336681}, {curr_items_tot,3336681}, {mem_used,3701118984}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1794}, {memory_data,{33657663488,13234946048,{<18117.695.0>,3143528}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 17366372 kB\nBuffers: 251916 kB\nCached: 9902624 kB\nSwapCached: 140 kB\nActive: 11984892 kB\nInactive: 3118224 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 17366372 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 968084 kB\nWriteback: 5676 kB\nAnonPages: 4948380 kB\nMapped: 18020 kB\nSlab: 350784 kB\nPageTables: 14680 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6671616 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,907585,659292}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,88354312}, {processes,46702952}, {processes_used,46181952}, {system,41651360}, {atom,1413905}, {atom_used,1392521}, {binary,1776080}, {code,14569021}, {ets,5859424}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,3268882432}, {buffered_memory,429469696}, {free_memory,25079136256}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1794397,4999}}, {context_switches,{15682762,0}}, {garbage_collection,{2867719,14079612970,0}}, {io,{{input,23021383484},{output,2228853966}}}, {reductions,{5944452747,72474543}}, {run_queue,0}, {runtime,{547690,5860}}]}, {system_stats, [{cpu_utilization_rate,51.12219451371571}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,1342335114}, {couch_docs_data_size,1332265693}, {couch_views_actual_disk_size,1383268350}, {couch_views_data_size,57877253}, {curr_items,489292}, {curr_items_tot,489292}, {mem_used,816188424}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1794}, {memory_data,{33657663488,10151116800,{<0.703.0>,3143528}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 24491468 kB\nBuffers: 419404 kB\nCached: 3192268 kB\nSwapCached: 0 kB\nActive: 7501572 kB\nInactive: 578208 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 24491468 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 4568 kB\nWriteback: 0 kB\nAnonPages: 4467948 kB\nMapped: 18052 kB\nSlab: 252588 kB\nPageTables: 13644 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 6181424 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,907588,999744}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.17075.0>">>}, {changes_done,0}, {design_documents,[<<"_design/A">>]}, {indexer_type,main}, {initial_build,false}, {progress,0}, {set,<<"default">>}, {signature,<<"afeb9408e40add49ae0c9bf58f0c5f26">>}, {started_on,1351907580}, {total_changes,359681}, {type,indexer}, {updated_on,1351907580}]]}, {memory, [{total,119523616}, {processes,61425088}, {processes_used,61408032}, {system,58098528}, {atom,1413905}, {atom_used,1392343}, {binary,18076064}, {code,14597005}, {ets,5875416}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,10130825216}, {buffered_memory,323706880}, {free_memory,17812557824}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1796430,4999}}, {context_switches,{14397144,0}}, {garbage_collection,{1400224,9068049176,0}}, {io,{{input,19699440336},{output,2680157456}}}, {reductions,{4801261111,128850087}}, {run_queue,0}, {runtime,{493680,9960}}]}, {system_stats, [{cpu_utilization_rate,97.73869346733669}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,9077599555}, {couch_docs_data_size,6956006936}, {couch_views_actual_disk_size,325645347}, {couch_views_data_size,106305422}, {curr_items,2873766}, {curr_items_tot,2893176}, {mem_used,3935305176}, {vb_replica_curr_items,19410}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1796}, {memory_data,{33657663488,14198231040,{<18118.1662.0>,2914384}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 17394952 kB\nBuffers: 316120 kB\nCached: 9893384 kB\nSwapCached: 12 kB\nActive: 13592576 kB\nInactive: 1469532 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 17394952 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 321580 kB\nWriteback: 25340 kB\nAnonPages: 4852016 kB\nMapped: 18204 kB\nSlab: 362752 kB\nPageTables: 14388 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6589956 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-02T18:53:17.512,ns_1@10.2.1.67:<0.1687.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,31,47}}) Stats for bucket "default": accepting_conns 1 auth_cmds 3 auth_errors 0 bucket_active_conns 1 bucket_conns 8 bytes 454982920 bytes_read 6012292225 bytes_written 8508660520 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 1 cmd_set 2503544 conn_yields 595 connection_structures 5000 curr_connections 11 curr_conns_on_port_11209 8 curr_conns_on_port_11210 1 curr_items 274175 curr_items_tot 274175 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-03 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 1499 ep_commit_time 27 ep_commit_time_total 18606 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 193 ep_data_age_highwat 209 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data/default ep_degraded_mode 0 ep_diskqueue_drain 274175 ep_diskqueue_fill 274175 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 1074 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 2503544 ep_io_read_bytes 0 ep_io_write_bytes 5931572051 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 2498272 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 444999893 ep_max_checkpoints 2 ep_max_data_size 5242880000 ep_max_item_size 20971520 ep_max_size 5242880000 ep_max_txn_size 10000 ep_max_vbuckets 512 ep_mem_high_wat 3932160000 ep_mem_high_wat 3932160000 ep_mem_low_wat 3145728000 ep_mem_low_wat 3145728000 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 393411 ep_num_expiry_pager_runs 0 ep_num_non_resident 96227 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 12 ep_num_value_ejects 1142582 ep_oom_errors 0 ep_overhead 2346292 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1351905991 ep_storage_age 190 ep_storage_age_highwat 208 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 2249075 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 441709793 ep_total_del_items 0 ep_total_enqueued 2503544 ep_total_new_items 2503544 ep_total_persisted 2503544 ep_uncommitted_items 0 ep_value_size 422335221 ep_vb0 0 ep_vb_snapshot_total 290 ep_vb_total 14 ep_vbucket_del 114 ep_vbucket_del_avg_walltime 63430 ep_vbucket_del_fail 0 ep_vbucket_del_max_walltime 355816 ep_version 2.0.0r_130_g8b09894 ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 725 get_hits 0 get_misses 1 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 454982920 pid 16451 pointer_size 64 rejected_conns 0 rusage_system 127.079680 rusage_user 331.167654 tap_checkpoint_start_sent 228 tap_connect_received 229 tap_mutation_sent 3448063 tap_opaque_sent 687 tap_vbucket_set_sent 228 threads 4 time 1351907597 total_connections 585 uptime 1629 vb_active_curr_items 274175 vb_active_eject 96227 vb_active_expired 0 vb_active_ht_memory 1383088 vb_active_itm_memory 441709793 vb_active_meta_data_memory 21659825 vb_active_num 14 vb_active_num_non_resident 96227 vb_active_num_ref_ejects 96227 vb_active_num_ref_items 274175 vb_active_ops_create 274175 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 64 vb_active_queue_age 0 vb_active_queue_drain 274175 vb_active_queue_fill 274175 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_meta_data_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-02T18:53:44.490,ns_1@10.2.1.67:ns_doctor<0.18838.0>:ns_doctor:handle_info:133]Got initial status [{'ns_1@10.2.1.67', [{last_heard,{1351,907624,490386}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,54065208}, {processes,18309848}, {processes_used,17596464}, {system,35755360}, {atom,1414713}, {atom_used,1393952}, {binary,878136}, {code,14594040}, {ets,991520}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,651608064}, {buffered_memory,430194688}, {free_memory,32132993024}, {total_memory,33657663488}]}, {node_storage_conf, [{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1833228,2}}, {context_switches,{16952461,0}}, {garbage_collection,{3182943,15223095616,0}}, {io,{{input,25799773087},{output,2418778797}}}, {reductions,{6295963036,26465}}, {run_queue,0}, {runtime,{578230,0}}]}, {system_stats, [{cpu_utilization_rate,0}, {swap_total,0}, {swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,1}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1833}, {memory_data, {33657663488,8392777728,{<0.703.0>,3143528}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 31379876 kB\nBuffers: 420108 kB\nCached: 636340 kB\nSwapCached: 0 kB\nActive: 721516 kB\nInactive: 523252 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 31379876 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 1048 kB\nWriteback: 0 kB\nAnonPages: 188132 kB\nMapped: 15884 kB\nSlab: 206256 kB\nPageTables: 4940 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 1660424 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-02T18:54:44.484,ns_1@10.2.1.67:ns_doctor<0.18838.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.67', [{last_heard,{1351,907679,484931}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,58162416}, {processes,22196736}, {processes_used,21544736}, {system,35965680}, {atom,1414713}, {atom_used,1393952}, {binary,962568}, {code,14594040}, {ets,1106152}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,652034048}, {buffered_memory,430292992}, {free_memory,32074063872}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1888223,5000}}, {context_switches,{16960747,0}}, {garbage_collection,{3184620,15225994957,0}}, {io,{{input,25799846786},{output,2418992999}}}, {reductions,{6297035817,78987}}, {run_queue,0}, {runtime,{580660,230}}]}, {system_stats, [{cpu_utilization_rate,1.4962593516209477}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1888}, {memory_data,{33657663488,1584869376,{<0.703.0>,1943128}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 31322328 kB\nBuffers: 420208 kB\nCached: 636752 kB\nSwapCached: 0 kB\nActive: 780272 kB\nInactive: 523432 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 31322328 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 60 kB\nWriteback: 0 kB\nAnonPages: 246760 kB\nMapped: 16860 kB\nSlab: 204760 kB\nPageTables: 5396 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 1889992 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-02T18:55:44.484,ns_1@10.2.1.67:ns_doctor<0.18838.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.67', [{last_heard,{1351,907739,484964}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,58424216}, {processes,22533688}, {processes_used,21881688}, {system,35890528}, {atom,1414713}, {atom_used,1393952}, {binary,855392}, {code,14594040}, {ets,1138144}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,652115968}, {buffered_memory,430346240}, {free_memory,32073936896}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{1948223,5000}}, {context_switches,{16968368,0}}, {garbage_collection,{3185788,15228468436,0}}, {io,{{input,25799884176},{output,2419084945}}}, {reductions,{6298000103,84168}}, {run_queue,0}, {runtime,{583250,230}}]}, {system_stats, [{cpu_utilization_rate,1.2468827930174564}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1948}, {memory_data,{33657663488,1583726592,{<0.703.0>,1943128}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 31322204 kB\nBuffers: 420260 kB\nCached: 636832 kB\nSwapCached: 0 kB\nActive: 781440 kB\nInactive: 523480 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 31322204 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 40 kB\nWriteback: 0 kB\nAnonPages: 247916 kB\nMapped: 16860 kB\nSlab: 204540 kB\nPageTables: 5396 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 1892196 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-02T18:56:44.484,ns_1@10.2.1.67:ns_doctor<0.18838.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.67', [{last_heard,{1351,907799,484872}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,58459784}, {processes,22516624}, {processes_used,21864624}, {system,35943160}, {atom,1414713}, {atom_used,1393952}, {binary,879008}, {code,14594040}, {ets,1167136}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,652201984}, {buffered_memory,430379008}, {free_memory,32073682944}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2008223,5000}}, {context_switches,{16976029,0}}, {garbage_collection,{3186950,15231013161,0}}, {io,{{input,25799922419},{output,2419176876}}}, {reductions,{6298982818,84708}}, {run_queue,0}, {runtime,{585850,220}}]}, {system_stats, [{cpu_utilization_rate,1.492537313432836}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2008}, {memory_data,{33657663488,1583726592,{<0.703.0>,1801440}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 31321956 kB\nBuffers: 420292 kB\nCached: 636916 kB\nSwapCached: 0 kB\nActive: 782168 kB\nInactive: 523572 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 31321956 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 8 kB\nWriteback: 0 kB\nAnonPages: 248604 kB\nMapped: 16860 kB\nSlab: 204516 kB\nPageTables: 5396 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 1892196 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-02T18:57:23.795,ns_1@10.2.1.67:ns_doctor<0.19607.0>:ns_doctor:handle_info:133]Got initial status [{'ns_1@10.2.1.65', [{last_heard,{1351,907843,795110}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,82903456}, {processes,37192640}, {processes_used,35065024}, {system,45710816}, {atom,1502673}, {atom_used,1494165}, {binary,4296704}, {code,15250635}, {ets,6645784}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,620675072}, {buffered_memory,231276544}, {free_memory,27462590464}, {total_memory,33657663488}]}, {node_storage_conf, [{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2051968,23}}, {context_switches,{22796104,0}}, {garbage_collection,{1861963,25859347710,0}}, {io,{{input,37364496642},{output,7627388209}}}, {reductions,{9782787547,26775}}, {run_queue,0}, {runtime,{892470,10}}]}, {system_stats, [{cpu_utilization_rate,2.493765586034913}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2051}, {memory_data, {33657663488,6196224000,{<18077.697.0>,3143528}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 26818936 kB\nBuffers: 225856 kB\nCached: 606128 kB\nSwapCached: 0 kB\nActive: 5429588 kB\nInactive: 425632 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 26818936 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 620 kB\nWriteback: 64 kB\nAnonPages: 5023528 kB\nMapped: 16288 kB\nSlab: 147628 kB\nPageTables: 14340 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6728720 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,907843,795098}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,56631296}, {processes,20398008}, {processes_used,19700752}, {system,36233288}, {atom,1414713}, {atom_used,1394008}, {binary,1018080}, {code,14594040}, {ets,1261112}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,652242944}, {buffered_memory,430477312}, {free_memory,32132255744}, {total_memory,33657663488}]}, {node_storage_conf, [{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2052533,2}}, {context_switches,{16984077,0}}, {garbage_collection,{3188342,15234394392,0}}, {io,{{input,25800257452},{output,2419659608}}}, {reductions,{6300126730,51707}}, {run_queue,2}, {runtime,{587860,10}}]}, {system_stats, [{cpu_utilization_rate,0}, {swap_total,0}, {swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2052}, {memory_data, {33657663488,1584234496,{<0.703.0>,1801440}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 31379156 kB\nBuffers: 420388 kB\nCached: 636956 kB\nSwapCached: 0 kB\nActive: 724764 kB\nInactive: 523744 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 31379156 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 92 kB\nWriteback: 0 kB\nAnonPages: 191300 kB\nMapped: 16136 kB\nSlab: 204544 kB\nPageTables: 5024 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 1662608 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-02T18:57:40.508,ns_1@10.2.1.67:<0.19807.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,31,47}}) Stats for bucket "default": accepting_conns 1 auth_cmds 0 auth_errors 0 bucket_active_conns 1 bucket_conns 5 bytes 245040 bytes_read 226 bytes_written 800 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 0 conn_yields 0 connection_structures 5000 curr_connections 9 curr_conns_on_port_11209 6 curr_conns_on_port_11210 1 curr_items 0 curr_items_tot 0 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-03 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 0 ep_commit_time 0 ep_commit_time_total 0 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 0 ep_data_age_highwat 0 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data/default ep_degraded_mode 1 ep_diskqueue_drain 0 ep_diskqueue_fill 0 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 0 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 0 ep_io_read_bytes 0 ep_io_write_bytes 0 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 0 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 0 ep_max_checkpoints 2 ep_max_data_size 5242880000 ep_max_item_size 20971520 ep_max_size 5242880000 ep_max_txn_size 10000 ep_max_vbuckets 512 ep_mem_high_wat 3932160000 ep_mem_high_wat 3932160000 ep_mem_low_wat 3145728000 ep_mem_low_wat 3145728000 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 1152 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1351907858 ep_storage_age 0 ep_storage_age_highwat 0 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 0 ep_total_del_items 0 ep_total_enqueued 0 ep_total_new_items 0 ep_total_persisted 0 ep_uncommitted_items 0 ep_value_size 0 ep_vb0 0 ep_vb_snapshot_total 1 ep_vb_total 0 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_130_g8b09894 ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 729 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 245040 pid 17252 pointer_size 64 rejected_conns 0 rusage_system 0.016997 rusage_user 0.022996 threads 4 time 1351907859 total_connections 10 uptime 18 vb_active_curr_items 0 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 0 vb_active_itm_memory 0 vb_active_meta_data_memory 0 vb_active_num 0 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 0 vb_active_ops_create 0 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 0 vb_active_queue_age 0 vb_active_queue_drain 0 vb_active_queue_fill 0 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_meta_data_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-02T18:58:23.787,ns_1@10.2.1.67:ns_doctor<0.19607.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,907899,17534}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,105902152}, {processes,60855032}, {processes_used,59279512}, {system,45047120}, {atom,1502673}, {atom_used,1494429}, {binary,3469648}, {code,15256817}, {ets,6710328}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,711827456}, {buffered_memory,232013824}, {free_memory,27359436800}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2107194,5001}}, {context_switches,{23113074,0}}, {garbage_collection,{1896636,25928632230,0}}, {io,{{input,37366843009},{output,7630285257}}}, {reductions,{9804039675,8339954}}, {run_queue,0}, {runtime,{901530,3240}}]}, {system_stats, [{cpu_utilization_rate,39.5}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,36951323}, {couch_docs_data_size,20512351}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,19283}, {curr_items_tot,19283}, {mem_used,55254088}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2107}, {memory_data,{33657663488,6208962560,{<18077.697.0>,3143528}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 26718200 kB\nBuffers: 226576 kB\nCached: 695144 kB\nSwapCached: 0 kB\nActive: 5524316 kB\nInactive: 432684 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 26718200 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 1048 kB\nWriteback: 48 kB\nAnonPages: 5034756 kB\nMapped: 16324 kB\nSlab: 147856 kB\nPageTables: 14340 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6757816 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,907901,692362}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,91903320}, {processes,49646984}, {processes_used,49262664}, {system,42256336}, {atom,1413905}, {atom_used,1392509}, {binary,1715200}, {code,14575828}, {ets,6481616}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,743231488}, {buffered_memory,268754944}, {free_memory,27418796032}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2109440,5001}}, {context_switches,{24824121,0}}, {garbage_collection,{1979254,26041053925,0}}, {io,{{input,37020859217},{output,8048669704}}}, {reductions,{9746927654,6946433}}, {run_queue,0}, {runtime,{846470,3080}}]}, {system_stats, [{cpu_utilization_rate,40.75}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,102475940}, {couch_docs_data_size,55215381}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,28123}, {curr_items_tot,28123}, {mem_used,78622152}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2109}, {memory_data,{33657663488,6121803776,{<18117.695.0>,3143528}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 26776416 kB\nBuffers: 262456 kB\nCached: 725812 kB\nSwapCached: 140 kB\nActive: 5449580 kB\nInactive: 443660 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 26776416 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 1256 kB\nWriteback: 0 kB\nAnonPages: 4904356 kB\nMapped: 18020 kB\nSlab: 151416 kB\nPageTables: 14504 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6620980 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,907898,786334}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,88120672}, {processes,50719736}, {processes_used,50567360}, {system,37400936}, {atom,1414713}, {atom_used,1394174}, {binary,1532328}, {code,14610103}, {ets,1772928}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,744017920}, {buffered_memory,431136768}, {free_memory,31901065216}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2107524,4999}}, {context_switches,{17298049,0}}, {garbage_collection,{3232093,15278109066,0}}, {io,{{input,25802340876},{output,2422087529}}}, {reductions,{6317801413,8918722}}, {run_queue,0}, {runtime,{596340,3130}}]}, {system_stats, [{cpu_utilization_rate,39.25}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,35241216}, {couch_docs_data_size,20484328}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,19488}, {curr_items_tot,19488}, {mem_used,55781280}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2107}, {memory_data,{33657663488,1606639616,{<0.703.0>,3143528}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 31153508 kB\nBuffers: 421032 kB\nCached: 726580 kB\nSwapCached: 0 kB\nActive: 944776 kB\nInactive: 528564 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 31153508 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 1104 kB\nWriteback: 0 kB\nAnonPages: 325648 kB\nMapped: 17932 kB\nSlab: 205276 kB\nPageTables: 5424 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 2032988 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,907898,997594}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,96651464}, {processes,54133816}, {processes_used,53887976}, {system,42517648}, {atom,1413905}, {atom_used,1392509}, {binary,1914544}, {code,14613068}, {ets,6484456}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,706191360}, {buffered_memory,329953280}, {free_memory,27219312640}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2106430,5001}}, {context_switches,{23192581,0}}, {garbage_collection,{1994351,24860780082,0}}, {io,{{input,35802481592},{output,7535890710}}}, {reductions,{9232170742,8834823}}, {run_queue,0}, {runtime,{833990,3520}}]}, {system_stats, [{cpu_utilization_rate,38.44221105527638}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,35915940}, {couch_docs_data_size,20855542}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,19548}, {curr_items_tot,19548}, {mem_used,56037424}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2106}, {memory_data,{33657663488,6359126016,{<18118.694.0>,2173096}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 26581556 kB\nBuffers: 322220 kB\nCached: 689640 kB\nSwapCached: 12 kB\nActive: 5558728 kB\nInactive: 464676 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 26581556 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 976 kB\nWriteback: 0 kB\nAnonPages: 5011324 kB\nMapped: 18216 kB\nSlab: 215868 kB\nPageTables: 14696 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6722288 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-02T18:59:20.506,ns_1@10.2.1.67:<0.19807.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,31,47}}) Stats for bucket "default": accepting_conns 1 auth_cmds 1 auth_errors 0 bucket_active_conns 1 bucket_conns 6 bytes 510781048 bytes_read 460487579 bytes_written 9439772 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 1 cmd_set 191894 conn_yields 38 connection_structures 5000 curr_connections 10 curr_conns_on_port_11209 6 curr_conns_on_port_11210 2 curr_items 191894 curr_items_tot 191894 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-03 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 550 ep_commit_time 2 ep_commit_time_total 277 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 2 ep_data_age_highwat 4 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data/default ep_degraded_mode 0 ep_diskqueue_drain 190554 ep_diskqueue_fill 191894 ep_diskqueue_items 1341 ep_diskqueue_memory 42880 ep_diskqueue_pending 64320 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 69 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 215 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 190551 ep_io_read_bytes 0 ep_io_write_bytes 451143322 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 0 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 469370458 ep_max_checkpoints 2 ep_max_data_size 5242880000 ep_max_item_size 20971520 ep_max_size 5242880000 ep_max_txn_size 10000 ep_max_vbuckets 512 ep_mem_high_wat 3932160000 ep_mem_high_wat 3932160000 ep_mem_low_wat 3145728000 ep_mem_low_wat 3145728000 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 20292624 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 1126 ep_startup_time 1351907858 ep_storage_age 0 ep_storage_age_highwat 1 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 467067730 ep_total_del_items 0 ep_total_enqueued 191894 ep_total_new_items 190551 ep_total_persisted 190551 ep_uncommitted_items 351 ep_value_size 453564926 ep_vb0 0 ep_vb_snapshot_total 6 ep_vb_total 128 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_130_g8b09894 ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 729 get_hits 0 get_misses 1 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 510781048 pid 17252 pointer_size 64 rejected_conns 0 rusage_system 10.505402 rusage_user 24.711243 threads 4 time 1351907959 total_connections 11 uptime 118 vb_active_curr_items 191894 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 3214336 vb_active_itm_memory 467067730 vb_active_meta_data_memory 15159626 vb_active_num 128 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 191894 vb_active_ops_create 190553 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 100 vb_active_queue_age 43000 vb_active_queue_drain 190554 vb_active_queue_fill 191894 vb_active_queue_memory 42880 vb_active_queue_pending 64320 vb_active_queue_size 1340 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_meta_data_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-02T18:59:23.787,ns_1@10.2.1.67:ns_doctor<0.19607.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,907959,16298}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,117776800}, {processes,71254168}, {processes_used,69677408}, {system,46522632}, {atom,1502673}, {atom_used,1494429}, {binary,4780048}, {code,15256817}, {ets,6878496}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,1178976256}, {buffered_memory,234471424}, {free_memory,26879442944}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2167193,4998}}, {context_switches,{24341298,0}}, {garbage_collection,{2022391,26115489391,0}}, {io,{{input,37396751345},{output,7640012324}}}, {reductions,{9877500261,5186188}}, {run_queue,0}, {runtime,{930540,2240}}]}, {system_stats, [{cpu_utilization_rate,36.56716417910448}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,502563897}, {couch_docs_data_size,422296424}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,185327}, {curr_items_tot,185327}, {mem_used,493743816}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2167}, {memory_data,{33657663488,6695432192,{<18077.178.1>,7628384}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 26249456 kB\nBuffers: 228976 kB\nCached: 1151344 kB\nSwapCached: 0 kB\nActive: 5643276 kB\nInactive: 782072 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 26249456 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 2384 kB\nWriteback: 0 kB\nAnonPages: 5044548 kB\nMapped: 16312 kB\nSlab: 148092 kB\nPageTables: 14328 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6772760 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,907961,691926}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,100581744}, {processes,57068992}, {processes_used,56683032}, {system,43512752}, {atom,1413905}, {atom_used,1392509}, {binary,2806280}, {code,14575828}, {ets,6651552}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,1321709568}, {buffered_memory,270700544}, {free_memory,26833940480}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2169440,5000}}, {context_switches,{25916997,0}}, {garbage_collection,{2107767,26178666938,0}}, {io,{{input,37032510335},{output,8056718875}}}, {reductions,{9808314109,4002209}}, {run_queue,0}, {runtime,{870480,1740}}]}, {system_stats, [{cpu_utilization_rate,13.784461152882205}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,686749000}, {couch_docs_data_size,463949930}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,195535}, {curr_items_tot,195535}, {mem_used,520141368}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2169}, {memory_data,{33657663488,6718976000,{<18117.695.0>,3143528}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 26205144 kB\nBuffers: 264356 kB\nCached: 1290732 kB\nSwapCached: 140 kB\nActive: 5737924 kB\nInactive: 726276 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 26205144 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 2164 kB\nWriteback: 172 kB\nAnonPages: 4908348 kB\nMapped: 18020 kB\nSlab: 151732 kB\nPageTables: 14500 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6621672 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,907963,787454}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,93605240}, {processes,54918240}, {processes_used,54763984}, {system,38687000}, {atom,1414713}, {atom_used,1394174}, {binary,2473336}, {code,14610103}, {ets,2122672}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,1318096896}, {buffered_memory,433524736}, {free_memory,30812647424}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2172525,5000}}, {context_switches,{18816714,0}}, {garbage_collection,{3400447,15457701529,0}}, {io,{{input,25815645590},{output,2429439216}}}, {reductions,{6397146853,4039246}}, {run_queue,0}, {runtime,{628390,1750}}]}, {system_stats, [{cpu_utilization_rate,35.08771929824562}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,614110269}, {couch_docs_data_size,458907445}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,200207}, {curr_items_tot,200207}, {mem_used,532644440}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2172}, {memory_data,{33657663488,2586361856,{<0.703.0>,3143528}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 30090600 kB\nBuffers: 423364 kB\nCached: 1287204 kB\nSwapCached: 0 kB\nActive: 1668568 kB\nInactive: 866364 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 30090600 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 2160 kB\nWriteback: 180 kB\nAnonPages: 824320 kB\nMapped: 17960 kB\nSlab: 205796 kB\nPageTables: 6564 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 2532396 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,907958,998008}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,101832696}, {processes,58189944}, {processes_used,57942384}, {system,43642752}, {atom,1413905}, {atom_used,1392509}, {binary,2870064}, {code,14613068}, {ets,6652408}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,1165746176}, {buffered_memory,332214272}, {free_memory,26750902272}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2166431,5001}}, {context_switches,{24509041,0}}, {garbage_collection,{2144632,25019669553,0}}, {io,{{input,35818896900},{output,7545353341}}}, {reductions,{9302596624,5077374}}, {run_queue,0}, {runtime,{862780,2410}}]}, {system_stats, [{cpu_utilization_rate,38.90274314214464}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,495099239}, {couch_docs_data_size,424514256}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,186616}, {curr_items_tot,186616}, {mem_used,496815592}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2166}, {memory_data,{33657663488,6824996864,{<18118.19057.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 26124176 kB\nBuffers: 324428 kB\nCached: 1138424 kB\nSwapCached: 12 kB\nActive: 5647952 kB\nInactive: 832000 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 26124176 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 2196 kB\nWriteback: 48 kB\nAnonPages: 5016672 kB\nMapped: 18216 kB\nSlab: 216592 kB\nPageTables: 14708 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6722624 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:warn,2012-11-02T18:59:46.528,ns_1@10.2.1.67:system_stats_collector<0.19681.0>:system_stats_collector:handle_info:133]lost 1 ticks [stats:warn,2012-11-02T18:59:46.534,ns_1@10.2.1.67:<0.19807.0>:stats_collector:latest_tick:223]Dropped 1 ticks [ns_doctor:debug,2012-11-02T19:00:23.787,ns_1@10.2.1.67:ns_doctor<0.19607.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,908019,14460}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,118689680}, {processes,71510736}, {processes_used,69933256}, {system,47178944}, {atom,1502673}, {atom_used,1494429}, {binary,5369000}, {code,15256817}, {ets,6949888}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,1779167232}, {buffered_memory,235384832}, {free_memory,26275569664}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2227193,4999}}, {context_switches,{24754740,0}}, {garbage_collection,{2062305,26224688175,0}}, {io,{{input,37413584573},{output,7643240192}}}, {reductions,{9919304676,2933350}}, {run_queue,0}, {runtime,{944890,1000}}]}, {system_stats, [{cpu_utilization_rate,8.229426433915211}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,1143157995}, {couch_docs_data_size,831193738}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,351653}, {curr_items_tot,351653}, {mem_used,933075176}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2227}, {memory_data,{33657663488,7301210112,{<18077.178.1>,7628384}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 25659736 kB\nBuffers: 229868 kB\nCached: 1737468 kB\nSwapCached: 0 kB\nActive: 5970104 kB\nInactive: 1044828 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 25659736 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 1752 kB\nWriteback: 24 kB\nAnonPages: 5047636 kB\nMapped: 16312 kB\nSlab: 147956 kB\nPageTables: 14324 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6779444 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,908021,691990}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,101568768}, {processes,57964864}, {processes_used,57578048}, {system,43603904}, {atom,1413905}, {atom_used,1392509}, {binary,2827960}, {code,14575828}, {ets,6722264}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,1620426752}, {buffered_memory,272654336}, {free_memory,26527805440}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2229439,4998}}, {context_switches,{26314313,0}}, {garbage_collection,{2150340,26254566462,0}}, {io,{{input,37039558979},{output,8060112012}}}, {reductions,{9844204868,4483305}}, {run_queue,0}, {runtime,{882700,1940}}]}, {system_stats, [{cpu_utilization_rate,29.42643391521197}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,978044151}, {couch_docs_data_size,865761905}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,362421}, {curr_items_tot,362421}, {mem_used,961121720}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2229}, {memory_data,{33657663488,7019905024,{<18117.695.0>,3143528}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 25906060 kB\nBuffers: 266264 kB\nCached: 1582448 kB\nSwapCached: 140 kB\nActive: 5552112 kB\nInactive: 1211004 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 25906060 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 2164 kB\nWriteback: 0 kB\nAnonPages: 4913952 kB\nMapped: 18020 kB\nSlab: 151920 kB\nPageTables: 14496 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6627024 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,908018,787194}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,95728488}, {processes,56967832}, {processes_used,56812240}, {system,38760656}, {atom,1414713}, {atom_used,1394174}, {binary,2370608}, {code,14610103}, {ets,2300832}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,1899991040}, {buffered_memory,434196480}, {free_memory,29807898624}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2227525,5000}}, {context_switches,{19177665,0}}, {garbage_collection,{3437836,15525516874,0}}, {io,{{input,25820231113},{output,2433842992}}}, {reductions,{6428955288,3039034}}, {run_queue,0}, {runtime,{638890,860}}]}, {system_stats, [{cpu_utilization_rate,22.25}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,1178756026}, {couch_docs_data_size,817284217}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,353167}, {curr_items_tot,353167}, {mem_used,936715832}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2227}, {memory_data,{33657663488,3722280960,{<0.703.0>,3143528}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29109276 kB\nBuffers: 424020 kB\nCached: 1855460 kB\nSwapCached: 0 kB\nActive: 2388992 kB\nInactive: 1125924 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29109276 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 680 kB\nWriteback: 0 kB\nAnonPages: 1235384 kB\nMapped: 17960 kB\nSlab: 205744 kB\nPageTables: 7288 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 2941816 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,908018,998371}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,98944192}, {processes,55067768}, {processes_used,54818872}, {system,43876424}, {atom,1413905}, {atom_used,1392509}, {binary,3036296}, {code,14613068}, {ets,6721984}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,1806766080}, {buffered_memory,333168640}, {free_memory,26110177280}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2226432,5003}}, {context_switches,{24995110,0}}, {garbage_collection,{2195179,25101573254,0}}, {io,{{input,35821625964},{output,7548160470}}}, {reductions,{9340622670,2567012}}, {run_queue,0}, {runtime,{877080,670}}]}, {system_stats, [{cpu_utilization_rate,7.462686567164179}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,1145417192}, {couch_docs_data_size,816294883}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,353079}, {curr_items_tot,353079}, {mem_used,936132056}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2226}, {memory_data,{33657663488,7459246080,{<18118.19057.0>,2914384}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 25498344 kB\nBuffers: 325360 kB\nCached: 1764420 kB\nSwapCached: 12 kB\nActive: 6025624 kB\nInactive: 1079884 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 25498344 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 1968 kB\nWriteback: 0 kB\nAnonPages: 5015668 kB\nMapped: 18216 kB\nSlab: 216432 kB\nPageTables: 14700 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6718020 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-02T19:01:01.507,ns_1@10.2.1.67:<0.19807.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,31,48}}) Stats for bucket "default": accepting_conns 1 auth_cmds 1 auth_errors 0 bucket_active_conns 1 bucket_conns 6 bytes 1253003016 bytes_read 1135620246 bytes_written 22540404 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 1 cmd_set 473275 conn_yields 108 connection_structures 5000 curr_connections 10 curr_conns_on_port_11209 6 curr_conns_on_port_11210 2 curr_items 473275 curr_items_tot 473275 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-03 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 690 ep_commit_time 1 ep_commit_time_total 3954 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 0 ep_data_age_highwat 22 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data/default ep_degraded_mode 0 ep_diskqueue_drain 472618 ep_diskqueue_fill 473275 ep_diskqueue_items 658 ep_diskqueue_memory 21024 ep_diskqueue_pending 31536 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 171 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 221 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 472611 ep_io_read_bytes 0 ep_io_write_bytes 1118885148 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 0 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 1157566137 ep_max_checkpoints 2 ep_max_data_size 5242880000 ep_max_item_size 20971520 ep_max_size 5242880000 ep_max_txn_size 10000 ep_max_vbuckets 512 ep_mem_high_wat 3932160000 ep_mem_high_wat 3932160000 ep_mem_low_wat 3145728000 ep_mem_low_wat 3145728000 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 45041912 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 437 ep_startup_time 1351907858 ep_storage_age 0 ep_storage_age_highwat 20 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 1151886837 ep_total_del_items 0 ep_total_enqueued 473275 ep_total_new_items 472611 ep_total_persisted 472611 ep_uncommitted_items 456 ep_value_size 1118557325 ep_vb0 0 ep_vb_snapshot_total 6 ep_vb_total 128 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_130_g8b09894 ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 729 get_hits 0 get_misses 1 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 1253003016 pid 17252 pointer_size 64 rejected_conns 0 rusage_system 21.799685 rusage_user 53.909804 threads 4 time 1351908061 total_connections 11 uptime 220 vb_active_curr_items 473275 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 3202048 vb_active_itm_memory 1151886837 vb_active_meta_data_memory 37388725 vb_active_num 128 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 473275 vb_active_ops_create 472617 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 100 vb_active_queue_age 0 vb_active_queue_drain 472618 vb_active_queue_fill 473275 vb_active_queue_memory 21024 vb_active_queue_pending 31536 vb_active_queue_size 657 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_meta_data_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-02T19:01:23.787,ns_1@10.2.1.67:ns_doctor<0.19607.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,908079,13672}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,118735248}, {processes,71332128}, {processes_used,69755128}, {system,47403120}, {atom,1502673}, {atom_used,1494429}, {binary,5417568}, {code,15256817}, {ets,7123912}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,2436644864}, {buffered_memory,236072960}, {free_memory,25601835008}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2287194,5001}}, {context_switches,{25055264,0}}, {garbage_collection,{2091427,26324115968,0}}, {io,{{input,37430223321},{output,7646093775}}}, {reductions,{9957136501,2837960}}, {run_queue,0}, {runtime,{956360,830}}]}, {system_stats, [{cpu_utilization_rate,28.179551122194514}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,1753108647}, {couch_docs_data_size,1232966884}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,519643}, {curr_items_tot,519643}, {mem_used,1378372088}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2287}, {memory_data,{33657663488,7976976384,{<18077.178.1>,7628384}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 25001792 kB\nBuffers: 230540 kB\nCached: 2379536 kB\nSwapCached: 0 kB\nActive: 6279596 kB\nInactive: 1382372 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 25001792 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 2312 kB\nWriteback: 0 kB\nAnonPages: 5051500 kB\nMapped: 16312 kB\nSlab: 159580 kB\nPageTables: 14316 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6780380 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,908081,692439}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,103653152}, {processes,59942584}, {processes_used,59556088}, {system,43710568}, {atom,1413905}, {atom_used,1392509}, {binary,2756840}, {code,14575828}, {ets,6896856}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,2286092288}, {buffered_memory,273797120}, {free_memory,25854705664}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2289440,5000}}, {context_switches,{26674068,0}}, {garbage_collection,{2187430,26325694193,0}}, {io,{{input,37041735491},{output,8062553095}}}, {reductions,{9877731513,2319626}}, {run_queue,0}, {runtime,{893490,580}}]}, {system_stats, [{cpu_utilization_rate,26.75}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,1599042827}, {couch_docs_data_size,1249245251}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,530304}, {curr_items_tot,530304}, {mem_used,1406268712}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2289}, {memory_data,{33657663488,7673315328,{<18117.20373.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 25248860 kB\nBuffers: 267380 kB\nCached: 2232512 kB\nSwapCached: 140 kB\nActive: 5905976 kB\nInactive: 1511720 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 25248860 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 1616 kB\nWriteback: 0 kB\nAnonPages: 4917320 kB\nMapped: 18020 kB\nSlab: 154184 kB\nPageTables: 14520 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6632292 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,908078,787603}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,97404704}, {processes,58359232}, {processes_used,58204120}, {system,39045472}, {atom,1414713}, {atom_used,1394174}, {binary,2478120}, {code,14610103}, {ets,2475416}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,2081361920}, {buffered_memory,436363264}, {free_memory,29157273600}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2287525,4999}}, {context_switches,{19641875,0}}, {garbage_collection,{3488205,15607721759,0}}, {io,{{input,25830392530},{output,2437733543}}}, {reductions,{6467467866,5642816}}, {run_queue,0}, {runtime,{651040,2120}}]}, {system_stats, [{cpu_utilization_rate,32.83208020050125}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,1367532998}, {couch_docs_data_size,1235268167}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,520845}, {curr_items_tot,520845}, {mem_used,1380799080}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2287}, {memory_data,{33657663488,4346748928,{<0.19799.0>,4714984}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 28473900 kB\nBuffers: 426136 kB\nCached: 2032580 kB\nSwapCached: 0 kB\nActive: 2496732 kB\nInactive: 1652680 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 28473900 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 1100 kB\nWriteback: 0 kB\nAnonPages: 1690168 kB\nMapped: 17960 kB\nSlab: 206280 kB\nPageTables: 8172 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 3402160 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,908078,997152}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,99356184}, {processes,55773944}, {processes_used,55525528}, {system,43582240}, {atom,1413905}, {atom_used,1392509}, {binary,2564328}, {code,14613068}, {ets,6897600}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,2006278144}, {buffered_memory,335417344}, {free_memory,25906380800}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2286431,5000}}, {context_switches,{25317280,0}}, {garbage_collection,{2230197,25171392244,0}}, {io,{{input,35830676170},{output,7551710520}}}, {reductions,{9373927524,4614427}}, {run_queue,0}, {runtime,{887280,2150}}]}, {system_stats, [{cpu_utilization_rate,35.82089552238806}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,1333593367}, {couch_docs_data_size,1235391397}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,521084}, {curr_items_tot,521084}, {mem_used,1382136488}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2286}, {memory_data,{33657663488,7671042048,{<18118.19057.0>,2914384}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 25299324 kB\nBuffers: 327556 kB\nCached: 1959256 kB\nSwapCached: 12 kB\nActive: 5687220 kB\nInactive: 1617252 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 25299324 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 3220 kB\nWriteback: 0 kB\nAnonPages: 5016996 kB\nMapped: 18396 kB\nSlab: 216716 kB\nPageTables: 14820 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6720512 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-02T19:02:23.787,ns_1@10.2.1.67:ns_doctor<0.19607.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,908139,13187}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,115512256}, {processes,68644928}, {processes_used,67081328}, {system,46867328}, {atom,1502673}, {atom_used,1494429}, {binary,4682696}, {code,15256817}, {ets,7317256}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,2184847360}, {buffered_memory,238424064}, {free_memory,25865056256}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2347193,5000}}, {context_switches,{25191719,0}}, {garbage_collection,{2106933,26414003133,0}}, {io,{{input,37456812038},{output,7654941975}}}, {reductions,{9990429312,2499074}}, {run_queue,0}, {runtime,{964340,590}}]}, {system_stats, [{cpu_utilization_rate,17.206982543640898}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,1555688217}, {couch_docs_data_size,1477081825}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,688052}, {curr_items_tot,688052}, {mem_used,1770370568}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2347}, {memory_data,{33657663488,7789060096,{<18077.178.1>,7628384}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 25258968 kB\nBuffers: 232836 kB\nCached: 2133312 kB\nSwapCached: 0 kB\nActive: 5489936 kB\nInactive: 1919464 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 25258968 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 3784 kB\nWriteback: 44 kB\nAnonPages: 5042700 kB\nMapped: 16316 kB\nSlab: 154260 kB\nPageTables: 14384 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6769572 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,908141,693159}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,103018752}, {processes,59028256}, {processes_used,58641840}, {system,43990496}, {atom,1413905}, {atom_used,1392509}, {binary,2858504}, {code,14575828}, {ets,7075560}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,2960904192}, {buffered_memory,274591744}, {free_memory,25164083200}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2349440,5000}}, {context_switches,{26997686,0}}, {garbage_collection,{2221241,26394718901,0}}, {io,{{input,37047229729},{output,8070114815}}}, {reductions,{9910155813,2949788}}, {run_queue,0}, {runtime,{902740,860}}]}, {system_stats, [{cpu_utilization_rate,32.41206030150754}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,2303451781}, {couch_docs_data_size,1686985549}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,698542}, {curr_items_tot,698542}, {mem_used,1749121304}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2349}, {memory_data,{33657663488,8351494144,{<18117.20374.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 24574424 kB\nBuffers: 268156 kB\nCached: 2891508 kB\nSwapCached: 140 kB\nActive: 6258488 kB\nInactive: 1817196 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 24574424 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 844 kB\nWriteback: 0 kB\nAnonPages: 4915356 kB\nMapped: 18020 kB\nSlab: 170948 kB\nPageTables: 14512 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6627860 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,908138,787152}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,98679752}, {processes,59486128}, {processes_used,59330536}, {system,39193624}, {atom,1414713}, {atom_used,1394174}, {binary,2452632}, {code,14610103}, {ets,2651488}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,2877030400}, {buffered_memory,437473280}, {free_memory,27916460032}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2347525,5000}}, {context_switches,{20381211,0}}, {garbage_collection,{3564735,15712007111,0}}, {io,{{input,25834376854},{output,2441412801}}}, {reductions,{6514676358,2662966}}, {run_queue,0}, {runtime,{669140,730}}]}, {system_stats, [{cpu_utilization_rate,28.0}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,2135494618}, {couch_docs_data_size,1622644075}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,689661}, {curr_items_tot,689661}, {mem_used,1725024856}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2347}, {memory_data,{33657663488,5604831232,{<0.19799.0>,4714984}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 27262168 kB\nBuffers: 427220 kB\nCached: 2809600 kB\nSwapCached: 0 kB\nActive: 3507760 kB\nInactive: 1852816 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 27262168 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 1888 kB\nWriteback: 24 kB\nAnonPages: 2123596 kB\nMapped: 17960 kB\nSlab: 205988 kB\nPageTables: 9032 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 3837368 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,908138,997798}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,102667840}, {processes,58839368}, {processes_used,58590472}, {system,43828472}, {atom,1413905}, {atom_used,1392509}, {binary,2637056}, {code,14613068}, {ets,7075112}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,2653401088}, {buffered_memory,336179200}, {free_memory,25257267200}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2346432,5001}}, {context_switches,{25809261,0}}, {garbage_collection,{2279965,25255160860,0}}, {io,{{input,35836920623},{output,7560014153}}}, {reductions,{9412268603,2258347}}, {run_queue,0}, {runtime,{901350,850}}]}, {system_stats, [{cpu_utilization_rate,10.723192019950124}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,2040251690}, {couch_docs_data_size,1647918783}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,689529}, {curr_items_tot,689529}, {mem_used,1724178008}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2346}, {memory_data,{33657663488,8365727744,{<18118.19057.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 24665424 kB\nBuffers: 328300 kB\nCached: 2591212 kB\nSwapCached: 12 kB\nActive: 6084164 kB\nInactive: 1853952 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 24665424 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 704 kB\nWriteback: 0 kB\nAnonPages: 5018348 kB\nMapped: 18340 kB\nSlab: 216432 kB\nPageTables: 14612 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6724304 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-02T19:02:41.503,ns_1@10.2.1.67:<0.19807.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,31,48}}) Stats for bucket "default": accepting_conns 1 auth_cmds 1 auth_errors 0 bucket_active_conns 1 bucket_conns 6 bytes 1896156568 bytes_read 1809972816 bytes_written 35675444 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 1 cmd_set 754387 conn_yields 182 connection_structures 5000 curr_connections 10 curr_conns_on_port_11209 6 curr_conns_on_port_11210 2 curr_items 754387 curr_items_tot 754387 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-03 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 1008 ep_commit_time 2 ep_commit_time_total 4508 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 4 ep_data_age_highwat 22 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data/default ep_degraded_mode 0 ep_diskqueue_drain 749440 ep_diskqueue_fill 754387 ep_diskqueue_items 4948 ep_diskqueue_memory 158304 ep_diskqueue_pending 237456 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 270 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 3640 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 749368 ep_io_read_bytes 0 ep_io_write_bytes 1773898265 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 640256 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 1844976425 ep_max_checkpoints 2 ep_max_data_size 5242880000 ep_max_item_size 20971520 ep_max_size 5242880000 ep_max_txn_size 10000 ep_max_vbuckets 512 ep_mem_high_wat 3932160000 ep_mem_high_wat 3932160000 ep_mem_low_wat 3145728000 ep_mem_low_wat 3145728000 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 13487120 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 1308 ep_startup_time 1351907858 ep_storage_age 3 ep_storage_age_highwat 20 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 1835923781 ep_total_del_items 0 ep_total_enqueued 754387 ep_total_new_items 749368 ep_total_persisted 749368 ep_uncommitted_items 5189 ep_value_size 1782779373 ep_vb0 0 ep_vb_snapshot_total 44 ep_vb_total 128 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_130_g8b09894 ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 729 get_hits 0 get_misses 1 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 1896156568 pid 17252 pointer_size 64 rejected_conns 0 rusage_system 37.403313 rusage_user 96.334354 threads 4 time 1351908161 total_connections 11 uptime 320 vb_active_curr_items 754387 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 3202048 vb_active_itm_memory 1835923781 vb_active_meta_data_memory 59596573 vb_active_num 128 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 754387 vb_active_ops_create 749439 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 100 vb_active_queue_age 8273000 vb_active_queue_drain 749440 vb_active_queue_fill 754387 vb_active_queue_memory 158304 vb_active_queue_pending 237456 vb_active_queue_size 4947 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_meta_data_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-02T19:03:23.787,ns_1@10.2.1.67:ns_doctor<0.19607.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,908199,11933}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,118526520}, {processes,71497368}, {processes_used,69919888}, {system,47029152}, {atom,1502673}, {atom_used,1494429}, {binary,4690136}, {code,15256817}, {ets,7476640}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,3043262464}, {buffered_memory,239583232}, {free_memory,24974573568}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2407193,4997}}, {context_switches,{25716935,0}}, {garbage_collection,{2159013,26532775147,0}}, {io,{{input,37474685285},{output,7658622592}}}, {reductions,{10036447119,4730409}}, {run_queue,0}, {runtime,{978890,1740}}]}, {system_stats, [{cpu_utilization_rate,35.58897243107769}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,2359506213}, {couch_docs_data_size,2049239075}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,855792}, {curr_items_tot,855792}, {mem_used,2164595000}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2407}, {memory_data,{33657663488,8569827328,{<18077.178.1>,7628384}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 24389232 kB\nBuffers: 233968 kB\nCached: 2971936 kB\nSwapCached: 0 kB\nActive: 5973544 kB\nInactive: 2285064 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 24389232 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 848 kB\nWriteback: 0 kB\nAnonPages: 5052316 kB\nMapped: 16312 kB\nSlab: 174712 kB\nPageTables: 14320 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6779292 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,908201,692608}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.25886.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,17}, {started_on,1351908152}, {total_vbuckets,129}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351908165}, {vbuckets_done,22}]]}, {memory, [{total,105468520}, {processes,61651992}, {processes_used,61287968}, {system,43816528}, {atom,1413905}, {atom_used,1392509}, {binary,2489672}, {code,14575828}, {ets,7269616}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,3398651904}, {buffered_memory,275681280}, {free_memory,24734642176}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2409440,5000}}, {context_switches,{27264543,0}}, {garbage_collection,{2248444,26457930411,0}}, {io,{{input,37051388696},{output,8072622292}}}, {reductions,{9940543495,1977882}}, {run_queue,0}, {runtime,{910820,430}}]}, {system_stats, [{cpu_utilization_rate,6.030150753768845}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,2793638985}, {couch_docs_data_size,2037631117}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,867062}, {curr_items_tot,867062}, {mem_used,2195574792}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2409}, {memory_data,{33657663488,8987144192,{<18117.20373.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 24154924 kB\nBuffers: 269220 kB\nCached: 3318996 kB\nSwapCached: 140 kB\nActive: 6339132 kB\nInactive: 2143228 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 24154924 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 11928 kB\nWriteback: 56 kB\nAnonPages: 4894036 kB\nMapped: 18044 kB\nSlab: 182780 kB\nPageTables: 14648 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6624228 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,908203,787132}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,99906232}, {processes,60404536}, {processes_used,60248944}, {system,39501696}, {atom,1414713}, {atom_used,1394174}, {binary,2575984}, {code,14610103}, {ets,2835872}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,3663003648}, {buffered_memory,438177792}, {free_memory,26670956544}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2412525,5000}}, {context_switches,{20772476,0}}, {garbage_collection,{3604506,15789642382,0}}, {io,{{input,25840232652},{output,2445326326}}}, {reductions,{6550985290,2940730}}, {run_queue,0}, {runtime,{679880,860}}]}, {system_stats, [{cpu_utilization_rate,25.75}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,2934869452}, {couch_docs_data_size,2086107869}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,872415}, {curr_items_tot,872415}, {mem_used,2208669640}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2412}, {memory_data,{33657663488,6750023680,{<0.19799.0>,4714984}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 26045856 kB\nBuffers: 427908 kB\nCached: 3577152 kB\nSwapCached: 0 kB\nActive: 4403748 kB\nInactive: 2172220 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 26045856 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 836 kB\nWriteback: 0 kB\nAnonPages: 2570808 kB\nMapped: 17960 kB\nSlab: 205956 kB\nPageTables: 9892 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 4286832 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,908198,999271}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,102629240}, {processes,58672632}, {processes_used,58423736}, {system,43956608}, {atom,1413905}, {atom_used,1392509}, {binary,2590312}, {code,14613068}, {ets,7250448}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,3367927808}, {buffered_memory,337317888}, {free_memory,24543932416}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2406434,5004}}, {context_switches,{26163469,0}}, {garbage_collection,{2316243,25325675493,0}}, {io,{{input,35839034000},{output,7562416754}}}, {reductions,{9445654245,2481552}}, {run_queue,0}, {runtime,{912210,720}}]}, {system_stats, [{cpu_utilization_rate,6.7164179104477615}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,2741248568}, {couch_docs_data_size,2064750943}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,858288}, {curr_items_tot,858288}, {mem_used,2170061528}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2406}, {memory_data,{33657663488,9050124288,{<18118.19055.0>,2914384}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 23968808 kB\nBuffers: 329412 kB\nCached: 3288992 kB\nSwapCached: 12 kB\nActive: 6475084 kB\nInactive: 2160300 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 23968808 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 1940 kB\nWriteback: 0 kB\nAnonPages: 5016448 kB\nMapped: 18216 kB\nSlab: 216360 kB\nPageTables: 14700 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6722412 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-02T19:04:21.511,ns_1@10.2.1.67:<0.19807.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,31,48}}) Stats for bucket "default": accepting_conns 1 auth_cmds 1 auth_errors 0 bucket_active_conns 1 bucket_conns 6 bytes 2642157000 bytes_read 2485741731 bytes_written 48829778 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 1 cmd_set 1035533 conn_yields 235 connection_structures 5000 curr_connections 10 curr_conns_on_port_11209 6 curr_conns_on_port_11210 2 curr_items 1035533 curr_items_tot 1035533 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-03 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 1092 ep_commit_time 1095 ep_commit_time_total 6086 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 7 ep_data_age_highwat 37 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data/default ep_degraded_mode 0 ep_diskqueue_drain 960519 ep_diskqueue_fill 1035533 ep_diskqueue_items 75015 ep_diskqueue_memory 2400448 ep_diskqueue_pending 3600672 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 365 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 59293 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 960390 ep_io_read_bytes 0 ep_io_write_bytes 2274756457 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 640256 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 2533806423 ep_max_checkpoints 2 ep_max_data_size 5242880000 ep_max_item_size 20971520 ep_max_size 5242880000 ep_max_txn_size 10000 ep_max_vbuckets 512 ep_mem_high_wat 3932160000 ep_mem_high_wat 3932160000 ep_mem_low_wat 3145728000 ep_mem_low_wat 3145728000 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 41823016 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 15722 ep_startup_time 1351907858 ep_storage_age 7 ep_storage_age_highwat 36 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 2521380027 ep_total_del_items 0 ep_total_enqueued 1035533 ep_total_new_items 960204 ep_total_persisted 960208 ep_uncommitted_items 9064 ep_value_size 2448416542 ep_vb0 0 ep_vb_snapshot_total 44 ep_vb_total 128 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_130_g8b09894 ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 729 get_hits 0 get_misses 1 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 2642157000 pid 17252 pointer_size 64 rejected_conns 0 rusage_system 47.470783 rusage_user 122.116435 threads 4 time 1351908261 total_connections 11 uptime 420 vb_active_curr_items 1035533 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 6351872 vb_active_itm_memory 2521380027 vb_active_meta_data_memory 81807107 vb_active_num 128 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1035533 vb_active_ops_create 960518 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 100 vb_active_queue_age 1106049000 vb_active_queue_drain 960519 vb_active_queue_fill 1035533 vb_active_queue_memory 2400448 vb_active_queue_pending 3600672 vb_active_queue_size 75014 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_meta_data_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-02T19:04:23.788,ns_1@10.2.1.67:ns_doctor<0.19607.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,908259,10413}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,116892056}, {processes,69417232}, {processes_used,67839752}, {system,47474824}, {atom,1502673}, {atom_used,1494429}, {binary,5064496}, {code,15256817}, {ets,7546560}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,3711549440}, {buffered_memory,240234496}, {free_memory,24288141312}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2467193,5001}}, {context_switches,{26035438,0}}, {garbage_collection,{2190694,26634500002,0}}, {io,{{input,37491621698},{output,7661516422}}}, {reductions,{10075090319,2798431}}, {run_queue,0}, {runtime,{990890,720}}]}, {system_stats, [{cpu_utilization_rate,8.521303258145364}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,3029734669}, {couch_docs_data_size,2440910609}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1023987}, {curr_items_tot,1023987}, {mem_used,2610812024}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2467}, {memory_data,{33657663488,9262862336,{<18077.178.1>,7628384}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 23718888 kB\nBuffers: 234604 kB\nCached: 3624560 kB\nSwapCached: 0 kB\nActive: 6332748 kB\nInactive: 2579984 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 23718888 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 2060 kB\nWriteback: 8 kB\nAnonPages: 5053560 kB\nMapped: 16312 kB\nSlab: 191140 kB\nPageTables: 14324 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6784012 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,908261,693114}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.25886.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,86}, {started_on,1351908152}, {total_vbuckets,129}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351908226}, {vbuckets_done,112}]]}, {memory, [{total,104569288}, {processes,60930448}, {processes_used,60587584}, {system,43638840}, {atom,1413905}, {atom_used,1392509}, {binary,2215176}, {code,14575828}, {ets,7362480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,2977456128}, {buffered_memory,277688320}, {free_memory,25163313152}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2469440,5001}}, {context_switches,{27335674,0}}, {garbage_collection,{2256494,26505047675,0}}, {io,{{input,37057547843},{output,8074890473}}}, {reductions,{9964542610,1936935}}, {run_queue,0}, {runtime,{916020,420}}]}, {system_stats, [{cpu_utilization_rate,24.5}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,2388202993}, {couch_docs_data_size,2201595607}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1035454}, {curr_items_tot,1035454}, {mem_used,2641884744}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2469}, {memory_data,{33657663488,8559742976,{<18117.20373.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 24573672 kB\nBuffers: 271180 kB\nCached: 2907672 kB\nSwapCached: 140 kB\nActive: 5531964 kB\nInactive: 2542168 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 24573672 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 10056 kB\nWriteback: 248 kB\nAnonPages: 4895108 kB\nMapped: 18036 kB\nSlab: 172872 kB\nPageTables: 14488 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6624384 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,908263,786125}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,96220056}, {processes,56763584}, {processes_used,56635672}, {system,39456472}, {atom,1414713}, {atom_used,1394174}, {binary,2368560}, {code,14610103}, {ets,2992664}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,3074617344}, {buffered_memory,440930304}, {free_memory,26824089600}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2472524,5000}}, {context_switches,{20877277,0}}, {garbage_collection,{3616518,15841079385,0}}, {io,{{input,25853304210},{output,2448760987}}}, {reductions,{6576744287,2047818}}, {run_queue,0}, {runtime,{685170,340}}]}, {system_stats, [{cpu_utilization_rate,25.56390977443609}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,2409219655}, {couch_docs_data_size,2327060603}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1041100}, {curr_items_tot,1041100}, {mem_used,2656081320}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2472}, {memory_data,{33657663488,6816694272,{<0.19800.0>,4714984}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 26195400 kB\nBuffers: 430596 kB\nCached: 3002556 kB\nSwapCached: 0 kB\nActive: 3589336 kB\nInactive: 2834372 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 26195400 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 1412 kB\nWriteback: 0 kB\nAnonPages: 2990332 kB\nMapped: 17960 kB\nSlab: 206836 kB\nPageTables: 10740 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 4699208 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,908258,996157}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,104893384}, {processes,60521128}, {processes_used,60272232}, {system,44372256}, {atom,1413905}, {atom_used,1392509}, {binary,2935424}, {code,14613068}, {ets,7320712}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,4024991744}, {buffered_memory,338010112}, {free_memory,23884165120}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2466431,5001}}, {context_switches,{26413989,0}}, {garbage_collection,{2341379,25386927975,0}}, {io,{{input,35840753379},{output,7564485413}}}, {reductions,{9475297445,1847054}}, {run_queue,0}, {runtime,{920510,500}}]}, {system_stats, [{cpu_utilization_rate,7.231920199501247}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,3410343198}, {couch_docs_data_size,2471958629}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1026551}, {curr_items_tot,1026551}, {mem_used,2615874664}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2466}, {memory_data,{33657663488,9762205696,{<18118.19057.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 23324380 kB\nBuffers: 330088 kB\nCached: 3930656 kB\nSwapCached: 12 kB\nActive: 6805180 kB\nInactive: 2474292 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 23324380 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 2000 kB\nWriteback: 0 kB\nAnonPages: 5018348 kB\nMapped: 18216 kB\nSlab: 216204 kB\nPageTables: 14708 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6725476 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-02T19:05:23.787,ns_1@10.2.1.67:ns_doctor<0.19607.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,908319,8632}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,118928808}, {processes,71622696}, {processes_used,70044600}, {system,47306112}, {atom,1502673}, {atom_used,1494429}, {binary,4718824}, {code,15256817}, {ets,7724192}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,4417589248}, {buffered_memory,241393664}, {free_memory,23558914048}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2527193,5000}}, {context_switches,{26302774,0}}, {garbage_collection,{2216795,26731480765,0}}, {io,{{input,37511782436},{output,7667693156}}}, {reductions,{10111854150,2519866}}, {run_queue,0}, {runtime,{1001310,640}}]}, {system_stats, [{cpu_utilization_rate,22.8643216080402}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,3739825706}, {couch_docs_data_size,2869248345}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1192257}, {curr_items_tot,1192257}, {mem_used,3058106120}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2527}, {memory_data,{33657663488,10026373120,{<18077.178.1>,7628384}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 23006752 kB\nBuffers: 235736 kB\nCached: 4314052 kB\nSwapCached: 0 kB\nActive: 6682180 kB\nInactive: 2926060 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 23006752 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 1940 kB\nWriteback: 88 kB\nAnonPages: 5058364 kB\nMapped: 16312 kB\nSlab: 208104 kB\nPageTables: 14364 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6790268 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,908321,697030}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,103024576}, {processes,59038064}, {processes_used,58651648}, {system,43986512}, {atom,1413905}, {atom_used,1392509}, {binary,2433768}, {code,14575828}, {ets,7498344}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,3766067200}, {buffered_memory,278941696}, {free_memory,24359817216}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2529442,5002}}, {context_switches,{27524876,0}}, {garbage_collection,{2275965,26561882255,0}}, {io,{{input,37062551624},{output,8079883627}}}, {reductions,{9992148390,2473453}}, {run_queue,0}, {runtime,{923210,680}}]}, {system_stats, [{cpu_utilization_rate,28.92768079800499}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,3084100056}, {couch_docs_data_size,2879375932}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1203582}, {curr_items_tot,1203582}, {mem_used,3086975912}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2529}, {memory_data,{33657663488,9173790720,{<18117.20373.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 23788884 kB\nBuffers: 272404 kB\nCached: 3677800 kB\nSwapCached: 140 kB\nActive: 5650216 kB\nInactive: 3190792 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 23788884 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 1136 kB\nWriteback: 0 kB\nAnonPages: 4890156 kB\nMapped: 18020 kB\nSlab: 190992 kB\nPageTables: 14516 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6621272 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,908318,787413}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,101798792}, {processes,62111400}, {processes_used,61956128}, {system,39687392}, {atom,1414713}, {atom_used,1394174}, {binary,2465936}, {code,14610103}, {ets,3130640}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,3875540992}, {buffered_memory,441782272}, {free_memory,25589121024}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2527525,5000}}, {context_switches,{21317276,0}}, {garbage_collection,{3660858,15915590366,0}}, {io,{{input,25855894200},{output,2451370755}}}, {reductions,{6611334179,2837679}}, {run_queue,1}, {runtime,{696570,810}}]}, {system_stats, [{cpu_utilization_rate,31.0}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,3158508857}, {couch_docs_data_size,2863878808}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1196159}, {curr_items_tot,1196159}, {mem_used,3066481416}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2527}, {memory_data,{33657663488,7927472128,{<0.19799.0>,4714984}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 24989376 kB\nBuffers: 431428 kB\nCached: 3784708 kB\nSwapCached: 0 kB\nActive: 4437348 kB\nInactive: 3193520 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 24989376 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 1264 kB\nWriteback: 0 kB\nAnonPages: 3414464 kB\nMapped: 17960 kB\nSlab: 206116 kB\nPageTables: 11544 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 5120244 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,908318,993715}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,103640752}, {processes,59583024}, {processes_used,59334208}, {system,44057728}, {atom,1413905}, {atom_used,1392509}, {binary,2442400}, {code,14613068}, {ets,7498040}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,4749983744}, {buffered_memory,338771968}, {free_memory,23153164288}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2526430,4999}}, {context_switches,{26658971,0}}, {garbage_collection,{2366261,25448457692,0}}, {io,{{input,35845864793},{output,7569719577}}}, {reductions,{9504734347,2141938}}, {run_queue,0}, {runtime,{928400,580}}]}, {system_stats, [{cpu_utilization_rate,22.55639097744361}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,4086109884}, {couch_docs_data_size,2883387284}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1194813}, {curr_items_tot,1194813}, {mem_used,3064070248}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2526}, {memory_data,{33657663488,10456891392,{<18118.19055.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 22610512 kB\nBuffers: 330832 kB\nCached: 4638656 kB\nSwapCached: 12 kB\nActive: 7136520 kB\nInactive: 2842956 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 22610512 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 1488 kB\nWriteback: 0 kB\nAnonPages: 5009744 kB\nMapped: 18216 kB\nSlab: 229356 kB\nPageTables: 14684 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6718080 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-02T19:06:01.507,ns_1@10.2.1.67:<0.19807.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,31,48}}) Stats for bucket "default": accepting_conns 1 auth_cmds 1 auth_errors 0 bucket_active_conns 1 bucket_conns 6 bytes 3285133592 bytes_read 3161318490 bytes_written 62033655 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 1 cmd_set 1317254 conn_yields 297 connection_structures 5000 curr_connections 10 curr_conns_on_port_11209 6 curr_conns_on_port_11210 2 curr_items 1317233 curr_items_tot 1317233 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-03 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 1348 ep_commit_time 2 ep_commit_time_total 7110 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 0 ep_data_age_highwat 39 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data/default ep_degraded_mode 0 ep_diskqueue_drain 1316478 ep_diskqueue_fill 1317233 ep_diskqueue_items 767 ep_diskqueue_memory 24160 ep_diskqueue_pending 36240 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 471 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 701 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 1316477 ep_io_read_bytes 0 ep_io_write_bytes 3117264770 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 1275510 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 3222443913 ep_max_checkpoints 2 ep_max_data_size 5242880000 ep_max_item_size 20971520 ep_max_size 5242880000 ep_max_txn_size 10000 ep_max_vbuckets 512 ep_mem_high_wat 3932160000 ep_mem_high_wat 3932160000 ep_mem_low_wat 3145728000 ep_mem_low_wat 3145728000 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 10265950 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 66 ep_startup_time 1351907858 ep_storage_age 0 ep_storage_age_highwat 37 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 3206597639 ep_total_del_items 0 ep_total_enqueued 1317245 ep_total_new_items 1316477 ep_total_persisted 1316477 ep_uncommitted_items 74 ep_value_size 3113813849 ep_vb0 0 ep_vb_snapshot_total 101 ep_vb_total 128 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_130_g8b09894 ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 729 get_hits 0 get_misses 1 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 3285133592 pid 17252 pointer_size 64 rejected_conns 0 rusage_system 61.913587 rusage_user 162.943228 threads 4 time 1351908361 total_connections 11 uptime 520 vb_active_curr_items 1317233 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 6351872 vb_active_itm_memory 3206597639 vb_active_meta_data_memory 104061407 vb_active_num 128 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1317233 vb_active_ops_create 1316477 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 100 vb_active_queue_age 288000 vb_active_queue_drain 1316478 vb_active_queue_fill 1317233 vb_active_queue_memory 24160 vb_active_queue_pending 36240 vb_active_queue_size 755 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_meta_data_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-02T19:06:23.788,ns_1@10.2.1.67:ns_doctor<0.19607.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,908379,8298}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,121125952}, {processes,73324640}, {processes_used,71747480}, {system,47801312}, {atom,1502673}, {atom_used,1494429}, {binary,5140120}, {code,15256817}, {ets,7798240}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,5179400192}, {buffered_memory,242216960}, {free_memory,22797570048}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2587193,4999}}, {context_switches,{26680691,0}}, {garbage_collection,{2254278,26838021286,0}}, {io,{{input,37529028884},{output,7670846007}}}, {reductions,{10152702956,3980174}}, {run_queue,0}, {runtime,{1012770,1210}}]}, {system_stats, [{cpu_utilization_rate,23.19201995012469}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,4498523797}, {couch_docs_data_size,3297141346}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1361064}, {curr_items_tot,1361064}, {mem_used,3400856696}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2587}, {memory_data,{33657663488,10757750784,{<18077.178.1>,7628384}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 22263252 kB\nBuffers: 236540 kB\nCached: 5058008 kB\nSwapCached: 0 kB\nActive: 7120620 kB\nInactive: 3212616 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 22263252 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 1792 kB\nWriteback: 0 kB\nAnonPages: 5038596 kB\nMapped: 16312 kB\nSlab: 226984 kB\nPageTables: 14332 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6784636 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,908381,694298}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,105095480}, {processes,61033904}, {processes_used,60647088}, {system,44061576}, {atom,1413905}, {atom_used,1392509}, {binary,2436864}, {code,14575828}, {ets,7570256}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,4408918016}, {buffered_memory,279896064}, {free_memory,23694716928}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2589441,5002}}, {context_switches,{27802185,0}}, {garbage_collection,{2304048,26625596412,0}}, {io,{{input,37064303127},{output,8082026443}}}, {reductions,{10022788550,2403259}}, {run_queue,0}, {runtime,{931980,630}}]}, {system_stats, [{cpu_utilization_rate,6.5}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,3785729340}, {couch_docs_data_size,3308125337}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1371711}, {curr_items_tot,1371711}, {mem_used,3429178584}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2589}, {memory_data,{33657663488,9870000128,{<18117.20373.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 23139372 kB\nBuffers: 273336 kB\nCached: 4305584 kB\nSwapCached: 140 kB\nActive: 5983804 kB\nInactive: 3490948 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 23139372 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 1656 kB\nWriteback: 0 kB\nAnonPages: 4895712 kB\nMapped: 18020 kB\nSlab: 206528 kB\nPageTables: 14520 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6623836 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,908383,786997}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,102576160}, {processes,62501536}, {processes_used,62346024}, {system,40074624}, {atom,1414713}, {atom_used,1394174}, {binary,2670504}, {code,14610103}, {ets,3316056}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,4710416384}, {buffered_memory,442789888}, {free_memory,24272117760}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2592524,4998}}, {context_switches,{22035257,0}}, {garbage_collection,{3735262,16022106359,0}}, {io,{{input,25863277064},{output,2458249117}}}, {reductions,{6659633426,2337050}}, {run_queue,0}, {runtime,{714880,640}}]}, {system_stats, [{cpu_utilization_rate,7.231920199501247}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,4040830749}, {couch_docs_data_size,3319381546}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1379092}, {curr_items_tot,1379092}, {mem_used,3447775448}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2592}, {memory_data,{33657663488,9169682432,{<0.19799.0>,4714984}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 23703240 kB\nBuffers: 432412 kB\nCached: 4600016 kB\nSwapCached: 0 kB\nActive: 5478444 kB\nInactive: 3420312 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 23703240 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 1540 kB\nWriteback: 0 kB\nAnonPages: 3866352 kB\nMapped: 17960 kB\nSlab: 222876 kB\nPageTables: 12432 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 5581928 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,908378,993983}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.26755.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,39}, {started_on,1351908324}, {total_vbuckets,129}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351908370}, {vbuckets_done,51}]]}, {memory, [{total,102109584}, {processes,57942984}, {processes_used,57724240}, {system,44166600}, {atom,1413905}, {atom_used,1392509}, {binary,2447248}, {code,14613068}, {ets,7597744}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,4566437888}, {buffered_memory,340742144}, {free_memory,23335112704}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2586430,5000}}, {context_switches,{26779885,0}}, {garbage_collection,{2378919,25499868286,0}}, {io,{{input,35855111287},{output,7572547878}}}, {reductions,{9530501907,1913540}}, {run_queue,0}, {runtime,{934550,450}}]}, {system_stats, [{cpu_utilization_rate,23.0}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,3959977277}, {couch_docs_data_size,3110253835}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1363170}, {curr_items_tot,1363170}, {mem_used,3460023560}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2586}, {memory_data,{33657663488,10378682368,{<18118.19055.0>,2914384}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 22788196 kB\nBuffers: 332756 kB\nCached: 4459412 kB\nSwapCached: 12 kB\nActive: 6591456 kB\nInactive: 3209632 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 22788196 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 4404 kB\nWriteback: 1124 kB\nAnonPages: 5008940 kB\nMapped: 18220 kB\nSlab: 229820 kB\nPageTables: 14744 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6712456 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-02T19:07:23.788,ns_1@10.2.1.67:ns_doctor<0.19607.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,908439,5983}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.9970.1>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,3}, {started_on,1351908470}, {total_vbuckets,129}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351908471}, {vbuckets_done,4}]]}, {memory, [{total,119023248}, {processes,71445824}, {processes_used,69876416}, {system,47577424}, {atom,1502673}, {atom_used,1494429}, {binary,4737992}, {code,15256817}, {ets,7975600}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,5920669696}, {buffered_memory,243093504}, {free_memory,22036844544}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2647192,4998}}, {context_switches,{27096480,0}}, {garbage_collection,{2295337,26948073681,0}}, {io,{{input,37546543870},{output,7674038337}}}, {reductions,{10194731516,2613761}}, {run_queue,0}, {runtime,{1025860,730}}]}, {system_stats, [{cpu_utilization_rate,23.0}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,5298529589}, {couch_docs_data_size,3710196872}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1529175}, {curr_items_tot,1529175}, {mem_used,3845931768}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2647}, {memory_data,{33657663488,11591479296,{<18077.178.1>,7628384}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 21520356 kB\nBuffers: 237396 kB\nCached: 5781904 kB\nSwapCached: 0 kB\nActive: 7558156 kB\nInactive: 3498176 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 21520356 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 20312 kB\nWriteback: 0 kB\nAnonPages: 5036864 kB\nMapped: 16344 kB\nSlab: 246264 kB\nPageTables: 14228 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6784768 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,908441,694074}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,106509240}, {processes,62019520}, {processes_used,61633456}, {system,44489720}, {atom,1413905}, {atom_used,1392509}, {binary,2689400}, {code,14575828}, {ets,7744848}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,5101084672}, {buffered_memory,280735744}, {free_memory,22984413184}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2649440,5000}}, {context_switches,{27965835,0}}, {garbage_collection,{2320019,26679214621,0}}, {io,{{input,37065516405},{output,8083672642}}}, {reductions,{10049302891,2361587}}, {run_queue,0}, {runtime,{938950,670}}]}, {system_stats, [{cpu_utilization_rate,28.428927680798004}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,4407461194}, {couch_docs_data_size,3703550161}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1539876}, {curr_items_tot,1539876}, {mem_used,3874451416}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2649}, {memory_data,{33657663488,10558717952,{<18117.20373.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 22445716 kB\nBuffers: 274156 kB\nCached: 4981528 kB\nSwapCached: 140 kB\nActive: 6286912 kB\nInactive: 3865112 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 22445716 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 1676 kB\nWriteback: 0 kB\nAnonPages: 4896252 kB\nMapped: 18020 kB\nSlab: 223204 kB\nPageTables: 14520 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6624356 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,908438,787061}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,99561488}, {processes,59254272}, {processes_used,59099000}, {system,40307216}, {atom,1414713}, {atom_used,1394174}, {binary,2737072}, {code,14610103}, {ets,3483304}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,5462351872}, {buffered_memory,443584512}, {free_memory,23103057920}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2647525,5000}}, {context_switches,{22424126,0}}, {garbage_collection,{3774620,16092106661,0}}, {io,{{input,25865511795},{output,2460736974}}}, {reductions,{6692330364,3292347}}, {run_queue,0}, {runtime,{725270,1030}}]}, {system_stats, [{cpu_utilization_rate,29.925187032418954}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,4742503030}, {couch_docs_data_size,3701378027}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1533817}, {curr_items_tot,1533817}, {mem_used,3857449064}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2647}, {memory_data,{33657663488,10380648448,{<0.703.0>,3143528}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 22561580 kB\nBuffers: 433188 kB\nCached: 5334328 kB\nSwapCached: 0 kB\nActive: 6328048 kB\nInactive: 3693196 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 22561580 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 1428 kB\nWriteback: 96 kB\nAnonPages: 4253748 kB\nMapped: 17960 kB\nSlab: 240924 kB\nPageTables: 13048 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 5964432 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,908438,993384}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.26755.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,79}, {started_on,1351908324}, {total_vbuckets,129}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351908431}, {vbuckets_done,103}]]}, {memory, [{total,103035384}, {processes,58769848}, {processes_used,58563208}, {system,44265536}, {atom,1413905}, {atom_used,1392509}, {binary,2356680}, {code,14613068}, {ets,7784528}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,4148310016}, {buffered_memory,342261760}, {free_memory,23775846400}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2646430,4999}}, {context_switches,{26838773,0}}, {garbage_collection,{2385262,25544557646,0}}, {io,{{input,35856301925},{output,7573943823}}}, {reductions,{9553355951,1936374}}, {run_queue,0}, {runtime,{939830,390}}]}, {system_stats, [{cpu_utilization_rate,5.764411027568922}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,3580061581}, {couch_docs_data_size,3214430996}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1531780}, {curr_items_tot,1531780}, {mem_used,3853371368}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2646}, {memory_data,{33657663488,9942011904,{<18118.19055.0>,2914384}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 23218600 kB\nBuffers: 334240 kB\nCached: 4051084 kB\nSwapCached: 12 kB\nActive: 5871888 kB\nInactive: 3506192 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 23218600 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 16884 kB\nWriteback: 0 kB\nAnonPages: 4992672 kB\nMapped: 18220 kB\nSlab: 223172 kB\nPageTables: 14744 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6715668 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-02T19:07:41.499,ns_1@10.2.1.67:<0.19807.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,31,48}}) Stats for bucket "default": accepting_conns 1 auth_cmds 1 auth_errors 0 bucket_active_conns 1 bucket_conns 6 bytes 3526828376 bytes_read 3837382247 bytes_written 75244298 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 1 cmd_set 1598951 conn_yields 368 connection_structures 5000 curr_connections 10 curr_conns_on_port_11209 6 curr_conns_on_port_11210 2 curr_items 1598944 curr_items_tot 1598944 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-03 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 1550 ep_commit_time 2 ep_commit_time_total 7989 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 4 ep_data_age_highwat 39 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data/default ep_degraded_mode 0 ep_diskqueue_drain 1596108 ep_diskqueue_fill 1598944 ep_diskqueue_items 2838 ep_diskqueue_memory 90752 ep_diskqueue_pending 136128 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 569 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 482 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 1596099 ep_io_read_bytes 0 ep_io_write_bytes 3779420146 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 1567120 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 3462855782 ep_max_checkpoints 2 ep_max_data_size 5242880000 ep_max_item_size 20971520 ep_max_size 5242880000 ep_max_txn_size 10000 ep_max_vbuckets 512 ep_mem_high_wat 3932160000 ep_mem_high_wat 3932160000 ep_mem_low_wat 3145728000 ep_mem_low_wat 3145728000 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 48 ep_num_expiry_pager_runs 0 ep_num_non_resident 191023 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 2 ep_num_value_ejects 191023 ep_oom_errors 0 ep_overhead 9413896 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 2356 ep_startup_time 1351907858 ep_storage_age 2 ep_storage_age_highwat 37 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 3443662712 ep_total_del_items 0 ep_total_enqueued 1598945 ep_total_new_items 1596099 ep_total_persisted 1596099 ep_uncommitted_items 388 ep_value_size 3330888199 ep_vb0 0 ep_vb_snapshot_total 102 ep_vb_total 128 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_130_g8b09894 ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 729 get_hits 0 get_misses 1 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 3526828376 pid 17252 pointer_size 64 rejected_conns 0 rusage_system 75.770481 rusage_user 204.203956 threads 4 time 1351908461 total_connections 11 uptime 620 vb_active_curr_items 1598944 vb_active_eject 191023 vb_active_expired 0 vb_active_ht_memory 6351872 vb_active_itm_memory 3443662712 vb_active_meta_data_memory 126316576 vb_active_num 128 vb_active_num_non_resident 191023 vb_active_num_ref_ejects 191023 vb_active_num_ref_items 1598944 vb_active_ops_create 1596107 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 88 vb_active_queue_age 2640000 vb_active_queue_drain 1596108 vb_active_queue_fill 1598944 vb_active_queue_memory 90752 vb_active_queue_pending 136128 vb_active_queue_size 2836 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_meta_data_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-02T19:08:23.788,ns_1@10.2.1.67:ns_doctor<0.19607.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,908499,5921}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.9970.1>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,44}, {started_on,1351908470}, {total_vbuckets,129}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351908530}, {vbuckets_done,58}]]}, {memory, [{total,118147296}, {processes,69836312}, {processes_used,68300024}, {system,48310984}, {atom,1502673}, {atom_used,1494429}, {binary,5253128}, {code,15256817}, {ets,8188560}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,5370613760}, {buffered_memory,245551104}, {free_memory,22593634304}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2707193,4999}}, {context_switches,{27168612,0}}, {garbage_collection,{2303531,27030482532,0}}, {io,{{input,37571775655},{output,7683335312}}}, {reductions,{10225126343,2372627}}, {run_queue,0}, {runtime,{1032730,540}}]}, {system_stats, [{cpu_utilization_rate,18.65671641791045}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,4783356443}, {couch_docs_data_size,3805701749}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1697280}, {curr_items_tot,1697280}, {mem_used,3810555704}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2707}, {memory_data,{33657663488,11140849664,{<18077.178.1>,7628384}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 22064096 kB\nBuffers: 239796 kB\nCached: 5244740 kB\nSwapCached: 0 kB\nActive: 6685084 kB\nInactive: 3840280 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 22064096 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 5988 kB\nWriteback: 32 kB\nAnonPages: 5040728 kB\nMapped: 16352 kB\nSlab: 233420 kB\nPageTables: 14164 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6785264 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,908501,693950}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,102542680}, {processes,57786440}, {processes_used,57400024}, {system,44756240}, {atom,1413905}, {atom_used,1392509}, {binary,2778816}, {code,14575828}, {ets,7922832}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,5825015808}, {buffered_memory,281513984}, {free_memory,22243127296}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2709440,4999}}, {context_switches,{28247619,0}}, {garbage_collection,{2348582,26744772549,0}}, {io,{{input,37070751984},{output,8091948704}}}, {reductions,{10080373805,2517580}}, {run_queue,0}, {runtime,{947850,650}}]}, {system_stats, [{cpu_utilization_rate,29.0}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,5150713194}, {couch_docs_data_size,4132145121}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1707674}, {curr_items_tot,1707674}, {mem_used,3821672168}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2709}, {memory_data,{33657663488,11262959616,{<18117.695.0>,3143528}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 21721804 kB\nBuffers: 274916 kB\nCached: 5688492 kB\nSwapCached: 140 kB\nActive: 6678816 kB\nInactive: 4179300 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 21721804 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 1044 kB\nWriteback: 0 kB\nAnonPages: 4894468 kB\nMapped: 18020 kB\nSlab: 240880 kB\nPageTables: 14504 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6620208 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,908503,786186}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,101527856}, {processes,61350192}, {processes_used,61194680}, {system,40177664}, {atom,1414713}, {atom_used,1394174}, {binary,2519112}, {code,14610103}, {ets,3570960}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,6207864832}, {buffered_memory,444555264}, {free_memory,22214733824}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2712524,4999}}, {context_switches,{22717637,0}}, {garbage_collection,{3804254,16160706960,0}}, {io,{{input,25867437307},{output,2462994136}}}, {reductions,{6725055466,1937099}}, {run_queue,0}, {runtime,{734390,480}}]}, {system_stats, [{cpu_utilization_rate,9.25}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,5529932959}, {couch_docs_data_size,4137334030}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1717156}, {curr_items_tot,1717156}, {mem_used,3847442232}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2712}, {memory_data,{33657663488,11321413632,{<0.19800.0>,4714984}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 21694076 kB\nBuffers: 434136 kB\nCached: 6062368 kB\nSwapCached: 0 kB\nActive: 6854580 kB\nInactive: 4015016 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 21694076 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 2016 kB\nWriteback: 0 kB\nAnonPages: 4373224 kB\nMapped: 17964 kB\nSlab: 259356 kB\nPageTables: 13440 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 6085508 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,908498,993915}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,103469336}, {processes,58975728}, {processes_used,58726832}, {system,44493608}, {atom,1413905}, {atom_used,1392509}, {binary,2452256}, {code,14613068}, {ets,7922088}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,4439568384}, {buffered_memory,343678976}, {free_memory,23481118720}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2706431,5001}}, {context_switches,{26896419,0}}, {garbage_collection,{2392129,25590485789,0}}, {io,{{input,35860834372},{output,7581435046}}}, {reductions,{9576199907,1992314}}, {run_queue,0}, {runtime,{945080,560}}]}, {system_stats, [{cpu_utilization_rate,12.718204488778055}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,3758921096}, {couch_docs_data_size,3679309763}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1700820}, {curr_items_tot,1700820}, {mem_used,3855777336}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2706}, {memory_data,{33657663488,10057551872,{<18118.19055.0>,2914384}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 22930904 kB\nBuffers: 335624 kB\nCached: 4335516 kB\nSwapCached: 12 kB\nActive: 5568632 kB\nInactive: 4096976 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 22930904 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 1808 kB\nWriteback: 3984 kB\nAnonPages: 4994508 kB\nMapped: 18216 kB\nSlab: 222772 kB\nPageTables: 14688 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6719644 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-02T19:09:21.497,ns_1@10.2.1.67:<0.19807.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,31,48}}) Stats for bucket "default": accepting_conns 1 auth_cmds 1 auth_errors 0 bucket_active_conns 1 bucket_conns 6 bytes 3760155512 bytes_read 4515244847 bytes_written 88462354 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 1 cmd_set 1880814 conn_yields 438 connection_structures 5000 curr_connections 10 curr_conns_on_port_11209 6 curr_conns_on_port_11210 2 curr_items 1880814 curr_items_tot 1880814 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-03 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 1668 ep_commit_time 2 ep_commit_time_total 8629 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 7 ep_data_age_highwat 39 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data/default ep_degraded_mode 0 ep_diskqueue_drain 1869114 ep_diskqueue_fill 1880814 ep_diskqueue_items 11700 ep_diskqueue_memory 374400 ep_diskqueue_pending 561600 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 669 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 1869114 ep_io_read_bytes 0 ep_io_write_bytes 4427297183 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 1776734 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 3675125098 ep_max_checkpoints 2 ep_max_data_size 5242880000 ep_max_item_size 20971520 ep_max_size 5242880000 ep_max_txn_size 10000 ep_max_vbuckets 512 ep_mem_high_wat 3932160000 ep_mem_high_wat 3932160000 ep_mem_low_wat 3145728000 ep_mem_low_wat 3145728000 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 25456 ep_num_expiry_pager_runs 0 ep_num_non_resident 394839 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 4 ep_num_value_ejects 394839 ep_oom_errors 0 ep_overhead 22183224 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 11700 ep_startup_time 1351907858 ep_storage_age 4 ep_storage_age_highwat 37 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 3652555330 ep_total_del_items 0 ep_total_enqueued 1880814 ep_total_new_items 1869114 ep_total_persisted 1869114 ep_uncommitted_items 0 ep_value_size 3519797864 ep_vb0 0 ep_vb_snapshot_total 107 ep_vb_total 128 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_130_g8b09894 ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 729 get_hits 0 get_misses 1 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 3760155512 pid 17252 pointer_size 64 rejected_conns 0 rusage_system 87.180746 rusage_user 238.163793 threads 4 time 1351908561 total_connections 11 uptime 720 vb_active_curr_items 1880814 vb_active_eject 394839 vb_active_expired 0 vb_active_ht_memory 12645376 vb_active_itm_memory 3652555330 vb_active_meta_data_memory 148584306 vb_active_num 128 vb_active_num_non_resident 394839 vb_active_num_ref_ejects 394839 vb_active_num_ref_items 1880814 vb_active_ops_create 1869114 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 79 vb_active_queue_age 30313000 vb_active_queue_drain 1869114 vb_active_queue_fill 1880814 vb_active_queue_memory 374400 vb_active_queue_pending 561600 vb_active_queue_size 11700 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_meta_data_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-02T19:09:23.787,ns_1@10.2.1.67:ns_doctor<0.19607.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,908559,5125}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,119320928}, {processes,70636584}, {processes_used,69081184}, {system,48684344}, {atom,1502673}, {atom_used,1494429}, {binary,5468280}, {code,15256817}, {ets,8349040}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,4763144192}, {buffered_memory,247988224}, {free_memory,23216967680}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2767193,4999}}, {context_switches,{27236266,0}}, {garbage_collection,{2310998,27110817075,0}}, {io,{{input,37588383015},{output,7685479084}}}, {reductions,{10254772067,2438416}}, {run_queue,0}, {runtime,{1039150,570}}]}, {system_stats, [{cpu_utilization_rate,10.5}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,4161050631}, {couch_docs_data_size,4036985906}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1865872}, {curr_items_tot,1865872}, {mem_used,3773355768}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2767}, {memory_data,{33657663488,10492383232,{<18077.178.1>,7628384}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 22672820 kB\nBuffers: 242176 kB\nCached: 4651508 kB\nSwapCached: 0 kB\nActive: 5528484 kB\nInactive: 4403488 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 22672820 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 6876 kB\nWriteback: 828 kB\nAnonPages: 5038012 kB\nMapped: 16236 kB\nSlab: 218476 kB\nPageTables: 14548 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6789268 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,908561,693913}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,103026288}, {processes,58384320}, {processes_used,57997504}, {system,44641968}, {atom,1413905}, {atom_used,1392509}, {binary,2491200}, {code,14575828}, {ets,8097120}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,6537859072}, {buffered_memory,282460160}, {free_memory,21513396224}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2769440,5000}}, {context_switches,{28483619,0}}, {garbage_collection,{2372109,26805182123,0}}, {io,{{input,37072418360},{output,8093901919}}}, {reductions,{10109517164,2766636}}, {run_queue,0}, {runtime,{955680,800}}]}, {system_stats, [{cpu_utilization_rate,31.25}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,5877708148}, {couch_docs_data_size,4557715130}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1875112}, {curr_items_tot,1875112}, {mem_used,3754097464}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2769}, {memory_data,{33657663488,12011839488,{<18117.20373.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 21009176 kB\nBuffers: 275840 kB\nCached: 6384628 kB\nSwapCached: 140 kB\nActive: 7044116 kB\nInactive: 4509508 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 21009176 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 1956 kB\nWriteback: 32 kB\nAnonPages: 4892764 kB\nMapped: 18020 kB\nSlab: 258296 kB\nPageTables: 14508 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6619588 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,908558,787292}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,102821856}, {processes,61802264}, {processes_used,61646672}, {system,41019592}, {atom,1414713}, {atom_used,1394174}, {binary,3191688}, {code,14610103}, {ets,3740744}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,6970507264}, {buffered_memory,445308928}, {free_memory,21386088448}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2767525,4999}}, {context_switches,{23032464,0}}, {garbage_collection,{3833124,16225028380,0}}, {io,{{input,25872806257},{output,2467150741}}}, {reductions,{6755186732,2930130}}, {run_queue,0}, {runtime,{742940,790}}]}, {system_stats, [{cpu_utilization_rate,6.965174129353234}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,6252344705}, {couch_docs_data_size,4531258116}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1872384}, {curr_items_tot,1872384}, {mem_used,3737878360}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2767}, {memory_data,{33657663488,12187262976,{<0.19800.0>,4714984}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 20884976 kB\nBuffers: 434872 kB\nCached: 6807136 kB\nSwapCached: 0 kB\nActive: 7346440 kB\nInactive: 4314640 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 20884976 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 776 kB\nWriteback: 0 kB\nAnonPages: 4418976 kB\nMapped: 17964 kB\nSlab: 278156 kB\nPageTables: 13536 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 6130064 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,908558,993387}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,106428352}, {processes,61642040}, {processes_used,61393144}, {system,44786312}, {atom,1413905}, {atom_used,1392509}, {binary,2569184}, {code,14613068}, {ets,8096824}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,5456592896}, {buffered_memory,344764416}, {free_memory,22437146624}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2766431,5001}}, {context_switches,{27024691,0}}, {garbage_collection,{2404946,25641176250,0}}, {io,{{input,35861872728},{output,7583081035}}}, {reductions,{9601537598,2882851}}, {run_queue,0}, {runtime,{951050,740}}]}, {system_stats, [{cpu_utilization_rate,28.25}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,4769617232}, {couch_docs_data_size,4493494777}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1869666}, {curr_items_tot,1869666}, {mem_used,3812841304}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2766}, {memory_data,{33657663488,11145101312,{<18118.19055.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,1}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 21911276 kB\nBuffers: 336684 kB\nCached: 5328704 kB\nSwapCached: 12 kB\nActive: 5884784 kB\nInactive: 4777588 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 21911276 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 2428 kB\nWriteback: 0 kB\nAnonPages: 4996512 kB\nMapped: 18216 kB\nSlab: 246916 kB\nPageTables: 14676 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6717900 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-02T19:10:23.788,ns_1@10.2.1.67:ns_doctor<0.19607.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,908619,4115}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,118016520}, {processes,70455704}, {processes_used,68877608}, {system,47560816}, {atom,1502673}, {atom_used,1494429}, {binary,4299960}, {code,15256817}, {ets,8396136}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,5872205824}, {buffered_memory,249200640}, {free_memory,22074183680}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2827193,5001}}, {context_switches,{27468312,0}}, {garbage_collection,{2333475,27203130337,0}}, {io,{{input,37604960551},{output,7688175982}}}, {reductions,{10290034234,4498193}}, {run_queue,0}, {runtime,{1048320,1630}}]}, {system_stats, [{cpu_utilization_rate,33.25}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,5186975140}, {couch_docs_data_size,4903159399}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2034101}, {curr_items_tot,2034101}, {mem_used,3683054504}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2827}, {memory_data,{33657663488,11451678720,{<18077.178.1>,7628384}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 21556820 kB\nBuffers: 243360 kB\nCached: 5734576 kB\nSwapCached: 0 kB\nActive: 5914364 kB\nInactive: 5107412 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 21556820 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 4004 kB\nWriteback: 0 kB\nAnonPages: 5043944 kB\nMapped: 16312 kB\nSlab: 244632 kB\nPageTables: 14336 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6788672 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,908621,695087}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,102609560}, {processes,57923200}, {processes_used,57536384}, {system,44686360}, {atom,1413905}, {atom_used,1392509}, {binary,2463536}, {code,14575828}, {ets,8168696}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,7269941248}, {buffered_memory,283336704}, {free_memory,20760043520}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2829441,5001}}, {context_switches,{28763704,0}}, {garbage_collection,{2400178,26868961042,0}}, {io,{{input,37074263538},{output,8096026179}}}, {reductions,{10140353443,2880718}}, {run_queue,0}, {runtime,{963920,890}}]}, {system_stats, [{cpu_utilization_rate,5.985037406483791}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,6646502756}, {couch_docs_data_size,4980474101}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2043506}, {curr_items_tot,2043506}, {mem_used,3697070632}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2829}, {memory_data,{33657663488,12787023872,{<18117.695.0>,3143528}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 20273480 kB\nBuffers: 276696 kB\nCached: 7099552 kB\nSwapCached: 140 kB\nActive: 7451128 kB\nInactive: 4819536 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 20273480 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 1672 kB\nWriteback: 0 kB\nAnonPages: 4894140 kB\nMapped: 18020 kB\nSlab: 276108 kB\nPageTables: 14508 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6618124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,908623,786282}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.30462.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,1}, {started_on,1351908622}, {total_vbuckets,129}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351908623}, {vbuckets_done,2}]]}, {memory, [{total,103314920}, {processes,62352352}, {processes_used,62203824}, {system,40962568}, {atom,1414713}, {atom_used,1394174}, {binary,3053152}, {code,14610103}, {ets,3820304}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,7838814208}, {buffered_memory,446455808}, {free_memory,20495343616}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2832524,5000}}, {context_switches,{23446411,0}}, {garbage_collection,{3866979,16304299882,0}}, {io,{{input,25875264108},{output,2469887616}}}, {reductions,{6792499064,2894770}}, {run_queue,0}, {runtime,{753370,820}}]}, {system_stats, [{cpu_utilization_rate,11.25}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7159522689}, {couch_docs_data_size,4994670107}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2054556}, {curr_items_tot,2054556}, {mem_used,3720191384}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2832}, {memory_data,{33657663488,13009813504,{<0.703.0>,3143528}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 20014984 kB\nBuffers: 435992 kB\nCached: 7655092 kB\nSwapCached: 0 kB\nActive: 7874548 kB\nInactive: 4633096 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 20014984 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 10928 kB\nWriteback: 0 kB\nAnonPages: 4416328 kB\nMapped: 17968 kB\nSlab: 301504 kB\nPageTables: 13616 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 6129804 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,908618,993260}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,105941576}, {processes,61134368}, {processes_used,60885792}, {system,44807208}, {atom,1413905}, {atom_used,1392509}, {binary,2519104}, {code,14613068}, {ets,8167520}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,6108745728}, {buffered_memory,345673728}, {free_memory,21768364032}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2826432,5002}}, {context_switches,{27280343,0}}, {garbage_collection,{2430745,25703236904,0}}, {io,{{input,35863701915},{output,7585217969}}}, {reductions,{9631509434,1881555}}, {run_queue,0}, {runtime,{960200,470}}]}, {system_stats, [{cpu_utilization_rate,6.5}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,5492229526}, {couch_docs_data_size,4911941980}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2038665}, {curr_items_tot,2038665}, {mem_used,3764904776}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2826}, {memory_data,{33657663488,11882070016,{<18118.19054.0>,2914384}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 21258416 kB\nBuffers: 337572 kB\nCached: 5965572 kB\nSwapCached: 12 kB\nActive: 6226316 kB\nInactive: 5072544 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 21258416 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 1592 kB\nWriteback: 0 kB\nAnonPages: 4995492 kB\nMapped: 18216 kB\nSlab: 262600 kB\nPageTables: 14680 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6719968 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-02T19:11:01.495,ns_1@10.2.1.67:<0.19807.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,31,48}}) Stats for bucket "default": accepting_conns 1 auth_cmds 1 auth_errors 0 bucket_active_conns 1 bucket_conns 6 bytes 3553279800 bytes_read 5189097444 bytes_written 101644983 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 1 cmd_set 2161175 conn_yields 491 connection_structures 5000 curr_connections 10 curr_conns_on_port_11209 6 curr_conns_on_port_11210 2 curr_items 2161175 curr_items_tot 2161175 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-03 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 1775 ep_commit_time 13 ep_commit_time_total 8996 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 23 ep_data_age_highwat 39 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data/default ep_degraded_mode 0 ep_diskqueue_drain 2079727 ep_diskqueue_fill 2161175 ep_diskqueue_items 81449 ep_diskqueue_memory 2606336 ep_diskqueue_pending 3909504 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 758 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 52444 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 2079286 ep_io_read_bytes 0 ep_io_write_bytes 4925840262 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 1961841 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 3456994989 ep_max_checkpoints 2 ep_max_data_size 5242880000 ep_max_item_size 20971520 ep_max_size 5242880000 ep_max_txn_size 10000 ep_max_vbuckets 512 ep_mem_high_wat 3932160000 ep_mem_high_wat 3932160000 ep_mem_low_wat 3145728000 ep_mem_low_wat 3145728000 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 166799 ep_num_expiry_pager_runs 0 ep_num_non_resident 779945 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 8 ep_num_value_ejects 779945 ep_oom_errors 0 ep_overhead 31012216 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 29005 ep_startup_time 1351907858 ep_storage_age 22 ep_storage_age_highwat 37 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 3431060889 ep_total_del_items 0 ep_total_enqueued 2161175 ep_total_new_items 2079286 ep_total_persisted 2079286 ep_uncommitted_items 6965 ep_value_size 3278335419 ep_vb0 0 ep_vb_snapshot_total 110 ep_vb_total 128 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_130_g8b09894 ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 729 get_hits 0 get_misses 1 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 3553279800 pid 17252 pointer_size 64 rejected_conns 0 rusage_system 97.802131 rusage_user 269.238069 threads 4 time 1351908660 total_connections 11 uptime 819 vb_active_curr_items 2161175 vb_active_eject 779945 vb_active_expired 0 vb_active_ht_memory 12645376 vb_active_itm_memory 3431060889 vb_active_meta_data_memory 170732825 vb_active_num 128 vb_active_num_non_resident 779945 vb_active_num_ref_ejects 779945 vb_active_num_ref_items 2161175 vb_active_ops_create 2079726 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 63 vb_active_queue_age 1206481000 vb_active_queue_drain 2079727 vb_active_queue_fill 2161175 vb_active_queue_memory 2606336 vb_active_queue_pending 3909504 vb_active_queue_size 81448 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_meta_data_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-02T19:11:23.787,ns_1@10.2.1.67:ns_doctor<0.19607.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,908679,3636}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,122013616}, {processes,74013536}, {processes_used,72436056}, {system,48000080}, {atom,1502673}, {atom_used,1494429}, {binary,4564536}, {code,15256817}, {ets,8573272}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,6558834688}, {buffered_memory,250327040}, {free_memory,21369978880}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2887193,5000}}, {context_switches,{27787911,0}}, {garbage_collection,{2365203,27305787321,0}}, {io,{{input,37625547003},{output,7694591875}}}, {reductions,{10328639828,3236905}}, {run_queue,0}, {runtime,{1060000,990}}]}, {system_stats, [{cpu_utilization_rate,30.75}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,5853105655}, {couch_docs_data_size,5291710292}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2202858}, {curr_items_tot,2202858}, {mem_used,3637370280}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2887}, {memory_data,{33657663488,12201975808,{<18077.178.1>,7628384}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 20869120 kB\nBuffers: 244460 kB\nCached: 6405112 kB\nSwapCached: 0 kB\nActive: 6275856 kB\nInactive: 5416568 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 20869120 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 4296 kB\nWriteback: 0 kB\nAnonPages: 5042656 kB\nMapped: 16312 kB\nSlab: 261164 kB\nPageTables: 14292 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6781232 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,908681,695274}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,104148856}, {processes,59123448}, {processes_used,58736632}, {system,45025408}, {atom,1413905}, {atom_used,1392509}, {binary,2625192}, {code,14575828}, {ets,8345864}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,8053858304}, {buffered_memory,284131328}, {free_memory,19955777536}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2889441,5000}}, {context_switches,{29085793,0}}, {garbage_collection,{2432814,26937471989,0}}, {io,{{input,37079756453},{output,8101613740}}}, {reductions,{10172655194,2909252}}, {run_queue,0}, {runtime,{973580,850}}]}, {system_stats, [{cpu_utilization_rate,6.25}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,7419108142}, {couch_docs_data_size,5394244255}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2211463}, {curr_items_tot,2211463}, {mem_used,3653099432}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2889}, {memory_data,{33657663488,13572243456,{<18117.695.0>,3143528}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 19488188 kB\nBuffers: 277472 kB\nCached: 7865096 kB\nSwapCached: 140 kB\nActive: 7904736 kB\nInactive: 5132120 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 19488188 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 1264 kB\nWriteback: 104 kB\nAnonPages: 4894384 kB\nMapped: 18020 kB\nSlab: 295284 kB\nPageTables: 14508 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6619148 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,908683,787533}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.30462.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,31}, {started_on,1351908622}, {total_vbuckets,129}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351908683}, {vbuckets_done,41}]]}, {memory, [{total,103156432}, {processes,62325536}, {processes_used,62176176}, {system,40830896}, {atom,1414713}, {atom_used,1394174}, {binary,2746304}, {code,14610103}, {ets,3994880}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,7328870400}, {buffered_memory,448188416}, {free_memory,20980002816}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2892525,4997}}, {context_switches,{23520224,0}}, {garbage_collection,{3874425,16351900067,0}}, {io,{{input,25890150926},{output,2473504416}}}, {reductions,{6816802241,2830765}}, {run_queue,0}, {runtime,{758610,520}}]}, {system_stats, [{cpu_utilization_rate,12.280701754385966}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,6682698978}, {couch_docs_data_size,5103361767}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2222830}, {curr_items_tot,2222830}, {mem_used,3715493048}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2892}, {memory_data,{33657663488,12760322048,{<0.19799.0>,4714984}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 20488284 kB\nBuffers: 437684 kB\nCached: 7157100 kB\nSwapCached: 0 kB\nActive: 7076604 kB\nInactive: 4968676 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 20488284 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 31216 kB\nWriteback: 0 kB\nAnonPages: 4450580 kB\nMapped: 17968 kB\nSlab: 289056 kB\nPageTables: 13672 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 6163044 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,908678,992754}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,108209712}, {processes,63054896}, {processes_used,62806000}, {system,45154816}, {atom,1413905}, {atom_used,1392509}, {binary,2690752}, {code,14613068}, {ets,8345400}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,6803615744}, {buffered_memory,346431488}, {free_memory,21047136256}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2886432,5001}}, {context_switches,{27446578,0}}, {garbage_collection,{2447440,25757877432,0}}, {io,{{input,35868462422},{output,7590202160}}}, {reductions,{9658165604,2112919}}, {run_queue,0}, {runtime,{967080,620}}]}, {system_stats, [{cpu_utilization_rate,9.5}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,6185989518}, {couch_docs_data_size,5337723585}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2207437}, {curr_items_tot,2207437}, {mem_used,3711575144}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2886}, {memory_data,{33657663488,12554403840,{<18118.19055.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 20553844 kB\nBuffers: 338312 kB\nCached: 6644156 kB\nSwapCached: 12 kB\nActive: 6550284 kB\nInactive: 5435700 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 20553844 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 1820 kB\nWriteback: 0 kB\nAnonPages: 5003332 kB\nMapped: 18216 kB\nSlab: 279672 kB\nPageTables: 14688 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6725712 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-02T19:12:23.788,ns_1@10.2.1.67:ns_doctor<0.19607.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,908739,2228}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,119506352}, {processes,71633456}, {processes_used,70055976}, {system,47872896}, {atom,1502673}, {atom_used,1494429}, {binary,4373696}, {code,15256817}, {ets,8636984}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,7320002560}, {buffered_memory,251170816}, {free_memory,20590342144}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2947193,5000}}, {context_switches,{28171046,0}}, {garbage_collection,{2403485,27413280373,0}}, {io,{{input,37642863639},{output,7697664084}}}, {reductions,{10369519932,2572642}}, {run_queue,0}, {runtime,{1072630,740}}]}, {system_stats, [{cpu_utilization_rate,25.5}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,6624507480}, {couch_docs_data_size,5709356973}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2371653}, {curr_items_tot,2371653}, {mem_used,3599316728}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2947}, {memory_data,{33657663488,12996341760,{<18077.178.1>,7628384}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 20107756 kB\nBuffers: 245284 kB\nCached: 7148440 kB\nSwapCached: 0 kB\nActive: 6718152 kB\nInactive: 5717900 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 20107756 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 3960 kB\nWriteback: 0 kB\nAnonPages: 5042280 kB\nMapped: 16312 kB\nSlab: 279576 kB\nPageTables: 14292 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6774256 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,908741,695004}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,106881224}, {processes,61908496}, {processes_used,61521680}, {system,44972728}, {atom,1413905}, {atom_used,1392509}, {binary,2512056}, {code,14575828}, {ets,8407648}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,8825663488}, {buffered_memory,285257728}, {free_memory,19160903680}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2949440,5000}}, {context_switches,{29345091,0}}, {garbage_collection,{2458897,26999717315,0}}, {io,{{input,37081508795},{output,8103758593}}}, {reductions,{10202480765,2753795}}, {run_queue,0}, {runtime,{981690,710}}]}, {system_stats, [{cpu_utilization_rate,21.55388471177945}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,8177210743}, {couch_docs_data_size,5808230149}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2379674}, {curr_items_tot,2379674}, {mem_used,3619959256}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2949}, {memory_data,{33657663488,14360768512,{<18117.20373.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 18711820 kB\nBuffers: 278572 kB\nCached: 8618812 kB\nSwapCached: 140 kB\nActive: 8332540 kB\nInactive: 5461060 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 18711820 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 1812 kB\nWriteback: 20 kB\nAnonPages: 4896224 kB\nMapped: 18020 kB\nSlab: 315040 kB\nPageTables: 14512 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6625912 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,908743,786431}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.30462.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,71}, {started_on,1351908622}, {total_vbuckets,129}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351908743}, {vbuckets_done,92}]]}, {memory, [{total,102671232}, {processes,61763760}, {processes_used,61661320}, {system,40907472}, {atom,1414713}, {atom_used,1394174}, {binary,2587232}, {code,14610103}, {ets,4223672}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,6576635904}, {buffered_memory,450560000}, {free_memory,21708464128}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2952525,5000}}, {context_switches,{23578547,0}}, {garbage_collection,{3880456,16397205490,0}}, {io,{{input,25894888714},{output,2478145000}}}, {reductions,{6839707670,1912263}}, {run_queue,0}, {runtime,{763600,420}}]}, {system_stats, [{cpu_utilization_rate,27.06766917293233}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,5936088467}, {couch_docs_data_size,5216570947}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2391416}, {curr_items_tot,2391416}, {mem_used,3702743416}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2952}, {memory_data,{33657663488,12111220736,{<0.19799.0>,4714984}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 21199796 kB\nBuffers: 440000 kB\nCached: 6422168 kB\nSwapCached: 0 kB\nActive: 5945624 kB\nInactive: 5406056 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 21199796 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 29132 kB\nWriteback: 0 kB\nAnonPages: 4489236 kB\nMapped: 17968 kB\nSlab: 271496 kB\nPageTables: 13672 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 6201592 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,908738,992715}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,106333608}, {processes,61091712}, {processes_used,60842960}, {system,45241896}, {atom,1413905}, {atom_used,1392509}, {binary,2716560}, {code,14613068}, {ets,8407648}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,7560282112}, {buffered_memory,347217920}, {free_memory,20273725440}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{2946432,5002}}, {context_switches,{27646428,0}}, {garbage_collection,{2467655,25814854318,0}}, {io,{{input,35869933448},{output,7592016978}}}, {reductions,{9685884838,2810879}}, {run_queue,0}, {runtime,{973890,750}}]}, {system_stats, [{cpu_utilization_rate,27.318295739348372}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,6864332674}, {couch_docs_data_size,5743118204}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2376176}, {curr_items_tot,2376176}, {mem_used,3675730680}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2946}, {memory_data,{33657663488,13303181312,{<18118.19054.0>,2914384}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 19798684 kB\nBuffers: 339080 kB\nCached: 7383088 kB\nSwapCached: 12 kB\nActive: 6921244 kB\nInactive: 5802372 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 19798684 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 2476 kB\nWriteback: 136 kB\nAnonPages: 5001312 kB\nMapped: 18216 kB\nSlab: 297964 kB\nPageTables: 14684 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6726184 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:warn,2012-11-02T19:12:31.539,ns_1@10.2.1.67:system_stats_collector<0.19681.0>:system_stats_collector:handle_info:133]lost 1 ticks [stats:warn,2012-11-02T19:12:31.544,ns_1@10.2.1.67:<0.19807.0>:stats_collector:latest_tick:223]Dropped 1 ticks [stats:debug,2012-11-02T19:12:42.492,ns_1@10.2.1.67:<0.19807.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,31,48}}) Stats for bucket "default": accepting_conns 1 auth_cmds 1 auth_errors 0 bucket_active_conns 1 bucket_conns 6 bytes 3842692952 bytes_read 5870921788 bytes_written 114934796 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 1 cmd_set 2444882 conn_yields 540 connection_structures 5000 curr_connections 10 curr_conns_on_port_11209 6 curr_conns_on_port_11210 2 curr_items 2444882 curr_items_tot 2444882 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-03 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 1787 ep_commit_time 977 ep_commit_time_total 21690 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 104 ep_data_age_highwat 112 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data/default ep_degraded_mode 0 ep_diskqueue_drain 2202144 ep_diskqueue_fill 2444882 ep_diskqueue_items 242917 ep_diskqueue_memory 7767616 ep_diskqueue_pending 11651424 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 840 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 160284 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 2200405 ep_io_read_bytes 0 ep_io_write_bytes 5212822831 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 2145047 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 3725046414 ep_max_checkpoints 2 ep_max_data_size 5242880000 ep_max_item_size 20971520 ep_max_size 5242880000 ep_max_txn_size 10000 ep_max_vbuckets 512 ep_mem_high_wat 3932160000 ep_mem_high_wat 3932160000 ep_mem_low_wat 3145728000 ep_mem_low_wat 3145728000 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 306006 ep_num_expiry_pager_runs 0 ep_num_non_resident 961820 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 10 ep_num_value_ejects 961820 ep_oom_errors 0 ep_overhead 41067886 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 82633 ep_startup_time 1351907858 ep_storage_age 102 ep_storage_age_highwat 111 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 3695707830 ep_total_del_items 0 ep_total_enqueued 2444882 ep_total_new_items 2200405 ep_total_persisted 2200405 ep_uncommitted_items 5365 ep_value_size 3522889660 ep_vb0 0 ep_vb_snapshot_total 110 ep_vb_total 128 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_130_g8b09894 ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 729 get_hits 0 get_misses 1 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 3842692952 pid 17252 pointer_size 64 rejected_conns 0 rusage_system 104.522110 rusage_user 282.737017 threads 4 time 1351908761 total_connections 11 uptime 920 vb_active_curr_items 2444882 vb_active_eject 961820 vb_active_expired 0 vb_active_ht_memory 12645376 vb_active_itm_memory 3695707830 vb_active_meta_data_memory 193145678 vb_active_num 128 vb_active_num_non_resident 961820 vb_active_num_ref_ejects 961820 vb_active_num_ref_items 2444882 vb_active_ops_create 2202143 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 60 vb_active_queue_age 12398008000 vb_active_queue_drain 2202144 vb_active_queue_fill 2444882 vb_active_queue_memory 7767616 vb_active_queue_pending 11651424 vb_active_queue_size 242738 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_meta_data_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-02T19:13:23.787,ns_1@10.2.1.67:ns_doctor<0.19607.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,908799,807}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,120098632}, {processes,71130640}, {processes_used,69553160}, {system,48967992}, {atom,1502673}, {atom_used,1494429}, {binary,5294704}, {code,15256817}, {ets,8812048}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,7909191680}, {buffered_memory,251854848}, {free_memory,19985428480}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{3007193,4998}}, {context_switches,{28440442,0}}, {garbage_collection,{2429943,27515115223,0}}, {io,{{input,37660634855},{output,7700745974}}}, {reductions,{10409366584,3041767}}, {run_queue,0}, {runtime,{1083100,670}}]}, {system_stats, [{cpu_utilization_rate,5.0}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,7281055106}, {couch_docs_data_size,6068913143}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2497553}, {curr_items_tot,2497553}, {mem_used,3930588616}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3007}, {memory_data,{33657663488,13674139648,{<18077.8103.1>,7628384}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 19517020 kB\nBuffers: 245952 kB\nCached: 7723820 kB\nSwapCached: 0 kB\nActive: 7066128 kB\nInactive: 5946132 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 19517020 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 3740 kB\nWriteback: 0 kB\nAnonPages: 5042400 kB\nMapped: 16312 kB\nSlab: 293596 kB\nPageTables: 14288 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6777320 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,908801,694534}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.30141.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,21}, {started_on,1351908719}, {total_vbuckets,129}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351908765}, {vbuckets_done,28}]]}, {memory, [{total,105372664}, {processes,60242648}, {processes_used,59873104}, {system,45130016}, {atom,1413905}, {atom_used,1392509}, {binary,2477152}, {code,14575828}, {ets,8595392}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,8615895040}, {buffered_memory,286883840}, {free_memory,19371810816}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{3009439,5000}}, {context_switches,{29457578,0}}, {garbage_collection,{2470452,27050788851,0}}, {io,{{input,37093808855},{output,8106930955}}}, {reductions,{10227947385,1895962}}, {run_queue,0}, {runtime,{987480,410}}]}, {system_stats, [{cpu_utilization_rate,3.75}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,8009508390}, {couch_docs_data_size,5979549171}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2497102}, {curr_items_tot,2497102}, {mem_used,3930689352}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3009}, {memory_data,{33657663488,14401527808,{<18117.20373.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 18917784 kB\nBuffers: 280160 kB\nCached: 8413960 kB\nSwapCached: 140 kB\nActive: 7834196 kB\nInactive: 5754948 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 18917784 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 49088 kB\nWriteback: 0 kB\nAnonPages: 4895040 kB\nMapped: 18024 kB\nSlab: 312504 kB\nPageTables: 14588 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6626916 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,908798,787188}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,101155632}, {processes,60284144}, {processes_used,60128552}, {system,40871488}, {atom,1414713}, {atom_used,1394174}, {binary,2443032}, {code,14610103}, {ets,4338712}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,6551425024}, {buffered_memory,452739072}, {free_memory,21729169408}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{3007525,5000}}, {context_switches,{23635022,0}}, {garbage_collection,{3886187,16438565414,0}}, {io,{{input,25895952646},{output,2479521912}}}, {reductions,{6860615449,1985739}}, {run_queue,0}, {runtime,{767930,490}}]}, {system_stats, [{cpu_utilization_rate,6.467661691542289}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,5808215475}, {couch_docs_data_size,5717570991}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2503544}, {curr_items_tot,2503544}, {mem_used,3600165864}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3007}, {memory_data,{33657663488,11790852096,{<0.19799.0>,4714984}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 21219892 kB\nBuffers: 442128 kB\nCached: 6397876 kB\nSwapCached: 0 kB\nActive: 5184196 kB\nInactive: 6149440 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 21219892 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 5808 kB\nWriteback: 0 kB\nAnonPages: 4493400 kB\nMapped: 17968 kB\nSlab: 269448 kB\nPageTables: 13680 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 6204208 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,908798,991281}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,107849248}, {processes,62479128}, {processes_used,62230232}, {system,45370120}, {atom,1413905}, {atom_used,1392509}, {binary,2669152}, {code,14613068}, {ets,8582592}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,8099512320}, {buffered_memory,347865088}, {free_memory,19719475200}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{3006431,5001}}, {context_switches,{27793122,0}}, {garbage_collection,{2482082,25867510702,0}}, {io,{{input,35871091999},{output,7593615352}}}, {reductions,{9711660274,1842973}}, {run_queue,0}, {runtime,{980400,460}}]}, {system_stats, [{cpu_utilization_rate,2.5}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,7480853905}, {couch_docs_data_size,6098523648}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2501801}, {curr_items_tot,2501801}, {mem_used,3522567176}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3006}, {memory_data,{33657663488,13937172480,{<18118.19054.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 19257300 kB\nBuffers: 339712 kB\nCached: 7909680 kB\nSwapCached: 12 kB\nActive: 7182364 kB\nInactive: 6068220 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 19257300 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 604 kB\nWriteback: 0 kB\nAnonPages: 5001472 kB\nMapped: 18216 kB\nSlab: 311204 kB\nPageTables: 14700 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6728504 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-02T19:14:22.492,ns_1@10.2.1.67:<0.19807.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,31,48}}) Stats for bucket "default": accepting_conns 1 auth_cmds 1 auth_errors 0 bucket_active_conns 1 bucket_conns 5 bytes 3545759256 bytes_read 6011794400 bytes_written 122867706 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 1 cmd_set 2503544 conn_yields 550 connection_structures 5000 curr_connections 9 curr_conns_on_port_11209 6 curr_conns_on_port_11210 1 curr_items 2503544 curr_items_tot 2503544 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-03 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 1817 ep_commit_time 43 ep_commit_time_total 28269 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 40 ep_data_age_highwat 145 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data/default ep_degraded_mode 0 ep_diskqueue_drain 2503544 ep_diskqueue_fill 2503544 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 922 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 2503544 ep_io_read_bytes 0 ep_io_write_bytes 5931572051 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 2482667 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 3479871032 ep_max_checkpoints 2 ep_max_data_size 5242880000 ep_max_item_size 20971520 ep_max_size 5242880000 ep_max_txn_size 10000 ep_max_vbuckets 512 ep_mem_high_wat 3932160000 ep_mem_high_wat 3932160000 ep_mem_low_wat 3145728000 ep_mem_low_wat 3145728000 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 465954 ep_num_expiry_pager_runs 0 ep_num_non_resident 1127724 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 12 ep_num_value_ejects 1127724 ep_oom_errors 0 ep_overhead 14881464 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1351907858 ep_storage_age 39 ep_storage_age_highwat 144 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 3449828504 ep_total_del_items 0 ep_total_enqueued 2503544 ep_total_new_items 2503544 ep_total_persisted 2503544 ep_uncommitted_items 0 ep_value_size 3272788504 ep_vb0 0 ep_vb_snapshot_total 112 ep_vb_total 128 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_130_g8b09894 ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 729 get_hits 0 get_misses 1 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 3545759256 pid 17252 pointer_size 64 rejected_conns 0 rusage_system 107.397673 rusage_user 298.077685 threads 4 time 1351908861 total_connections 11 uptime 1020 vb_active_curr_items 2503544 vb_active_eject 1127724 vb_active_expired 0 vb_active_ht_memory 12645376 vb_active_itm_memory 3449828504 vb_active_meta_data_memory 197779976 vb_active_num 128 vb_active_num_non_resident 1127724 vb_active_num_ref_ejects 1127724 vb_active_num_ref_items 2503544 vb_active_ops_create 2503544 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 54 vb_active_queue_age 0 vb_active_queue_drain 2503544 vb_active_queue_fill 2503544 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_meta_data_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-02T19:14:23.787,ns_1@10.2.1.67:ns_doctor<0.19607.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,908858,999805}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,121147648}, {processes,71761768}, {processes_used,70184288}, {system,49385880}, {atom,1502673}, {atom_used,1494429}, {binary,5535848}, {code,15256817}, {ets,8989152}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,7904722944}, {buffered_memory,252112896}, {free_memory,19991281664}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{3067193,5000}}, {context_switches,{28498551,0}}, {garbage_collection,{2435553,27607773403,0}}, {io,{{input,37682918387},{output,7710478355}}}, {reductions,{10447873314,3101280}}, {run_queue,0}, {runtime,{1091430,690}}]}, {system_stats, [{cpu_utilization_rate,4.987531172069826}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,7281055082}, {couch_docs_data_size,6068913143}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2497553}, {curr_items_tot,2497553}, {mem_used,3930588600}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3067}, {memory_data,{33657663488,13662699520,{<18077.8103.1>,7628384}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 19522736 kB\nBuffers: 246204 kB\nCached: 7719456 kB\nSwapCached: 0 kB\nActive: 7064060 kB\nInactive: 5942640 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 19522736 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 408 kB\nWriteback: 0 kB\nAnonPages: 5041276 kB\nMapped: 16312 kB\nSlab: 293236 kB\nPageTables: 14376 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6789800 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,908861,694682}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.30141.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,49}, {started_on,1351908719}, {total_vbuckets,129}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351908824}, {vbuckets_done,64}]]}, {memory, [{total,104850000}, {processes,59502272}, {processes_used,59165848}, {system,45347728}, {atom,1413905}, {atom_used,1392509}, {binary,2477128}, {code,14575828}, {ets,8808632}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,7992963072}, {buffered_memory,289034240}, {free_memory,20007706624}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{3069439,4999}}, {context_switches,{29512771,0}}, {garbage_collection,{2476242,27097529657,0}}, {io,{{input,37098364920},{output,8115239374}}}, {reductions,{10250881827,1913770}}, {run_queue,0}, {runtime,{992700,460}}]}, {system_stats, [{cpu_utilization_rate,2.756892230576441}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,7372519892}, {couch_docs_data_size,6042553679}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2497102}, {curr_items_tot,2497102}, {mem_used,3930392104}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3069}, {memory_data,{33657663488,13730230272,{<18117.20373.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 19538776 kB\nBuffers: 282260 kB\nCached: 7805628 kB\nSwapCached: 140 kB\nActive: 6977556 kB\nInactive: 6006100 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 19538776 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 21820 kB\nWriteback: 10684 kB\nAnonPages: 4895676 kB\nMapped: 18024 kB\nSlab: 298032 kB\nPageTables: 14588 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6624444 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,908858,787121}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,100274640}, {processes,59436296}, {processes_used,59280704}, {system,40838344}, {atom,1414713}, {atom_used,1394174}, {binary,2347728}, {code,14610103}, {ets,4400448}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,6897516544}, {buffered_memory,453263360}, {free_memory,21376303104}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{3067525,5000}}, {context_switches,{23688367,0}}, {garbage_collection,{3890867,16483619398,0}}, {io,{{input,25896712529},{output,2480808495}}}, {reductions,{6883007320,1839038}}, {run_queue,0}, {runtime,{773270,490}}]}, {system_stats, [{cpu_utilization_rate,3.0},{swap_total,10701078528},{swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,6236677511}, {couch_docs_data_size,6073140266}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2503544}, {curr_items_tot,2503544}, {mem_used,3545759256}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3067}, {memory_data,{33657663488,12281233408,{<0.703.0>,3143528}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 20875296 kB\nBuffers: 442640 kB\nCached: 6735856 kB\nSwapCached: 0 kB\nActive: 5258536 kB\nInactive: 6412880 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 20875296 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 292 kB\nWriteback: 0 kB\nAnonPages: 4492888 kB\nMapped: 17968 kB\nSlab: 276340 kB\nPageTables: 13684 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 6203916 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,908858,990683}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,106670600}, {processes,61205704}, {processes_used,60956808}, {system,45464896}, {atom,1413905}, {atom_used,1392509}, {binary,2587944}, {code,14613068}, {ets,8759424}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,8094978048}, {buffered_memory,348090368}, {free_memory,19715010560}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{3066431,5001}}, {context_switches,{27840492,0}}, {garbage_collection,{2486583,25913476031,0}}, {io,{{input,35875260994},{output,7601884270}}}, {reductions,{9734007270,1966771}}, {run_queue,0}, {runtime,{986200,600}}]}, {system_stats, [{cpu_utilization_rate,6.75}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,7480853871}, {couch_docs_data_size,6098523648}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2501801}, {curr_items_tot,2501801}, {mem_used,3522567160}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3066}, {memory_data,{33657663488,13939707904,{<18118.19054.0>,2914384}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 19252940 kB\nBuffers: 339932 kB\nCached: 7905252 kB\nSwapCached: 12 kB\nActive: 7190256 kB\nInactive: 6064628 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 19252940 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 144 kB\nWriteback: 0 kB\nAnonPages: 5009824 kB\nMapped: 18216 kB\nSlab: 311068 kB\nPageTables: 14720 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6738864 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-02T19:15:23.788,ns_1@10.2.1.67:ns_doctor<0.19607.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,908918,999723}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.15929.1>">>}, {changes_done,6760}, {design_documents,[<<"_design/A">>]}, {indexer_type,main}, {initial_build,true}, {progress,0}, {set,<<"default">>}, {signature,<<"afeb9408e40add49ae0c9bf58f0c5f26">>}, {started_on,1351908950}, {total_changes,2497553}, {type,indexer}, {updated_on,1351908950}]]}, {memory, [{total,155636488}, {processes,100875904}, {processes_used,99347216}, {system,54760584}, {atom,1502673}, {atom_used,1494429}, {binary,10713904}, {code,15256817}, {ets,9181256}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,7905832960}, {buffered_memory,252252160}, {free_memory,19961298944}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{3127194,5001}}, {context_switches,{28615575,0}}, {garbage_collection,{2447081,27726814054,0}}, {io,{{input,37878287627},{output,7713626841}}}, {reductions,{10498999059,19089837}}, {run_queue,1}, {runtime,{1100640,1920}}]}, {system_stats, [{cpu_utilization_rate,20.0}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,7281059183}, {couch_docs_data_size,6068913143}, {couch_views_actual_disk_size,8775}, {couch_views_data_size,8775}, {curr_items,2497553}, {curr_items_tot,2497553}, {mem_used,3930588600}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3127}, {memory_data,{33657663488,13668794368,{<18077.8103.1>,7628384}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 19493580 kB\nBuffers: 246340 kB\nCached: 7720540 kB\nSwapCached: 0 kB\nActive: 7152708 kB\nInactive: 5883360 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 19493580 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 376 kB\nWriteback: 0 kB\nAnonPages: 5068860 kB\nMapped: 16312 kB\nSlab: 293224 kB\nPageTables: 14404 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6820228 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,908921,695525}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.30141.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,72}, {started_on,1351908719}, {total_vbuckets,129}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351908885}, {vbuckets_done,93}], [{pid,<<"<0.31992.0>">>}, {changes_done,6765}, {design_documents,[<<"_design/A">>]}, {indexer_type,main}, {initial_build,true}, {progress,0}, {set,<<"default">>}, {signature,<<"afeb9408e40add49ae0c9bf58f0c5f26">>}, {started_on,1351908881}, {total_changes,2497102}, {type,indexer}, {updated_on,1351908882}]]}, {memory, [{total,145442152}, {processes,94025088}, {processes_used,93714168}, {system,51417064}, {atom,1413905}, {atom_used,1392509}, {binary,8379040}, {code,14575828}, {ets,8975528}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,7474479104}, {buffered_memory,290643968}, {free_memory,20496670720}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{3129440,4999}}, {context_switches,{29875864,0}}, {garbage_collection,{2508412,27333929874,0}}, {io,{{input,37878758339},{output,8127181841}}}, {reductions,{10371184805,90793243}}, {run_queue,0}, {runtime,{1006090,7920}}]}, {system_stats, [{cpu_utilization_rate,47.61904761904762}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,6866188359}, {couch_docs_data_size,6073295945}, {couch_views_actual_disk_size,2655920}, {couch_views_data_size,8776}, {curr_items,2497102}, {curr_items_tot,2497102}, {mem_used,3929970856}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3129}, {memory_data,{33657663488,13203128320,{<18117.20373.0>,4714984}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 20016280 kB\nBuffers: 283832 kB\nCached: 7299296 kB\nSwapCached: 140 kB\nActive: 6642428 kB\nInactive: 5876836 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 20016280 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 48360 kB\nWriteback: 5248 kB\nAnonPages: 4935908 kB\nMapped: 18028 kB\nSlab: 284728 kB\nPageTables: 14788 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6674480 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,908923,786190}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.647.1>">>}, {changes_done,130539}, {design_documents,[<<"_design/A">>]}, {indexer_type,main}, {initial_build,true}, {progress,5}, {set,<<"default">>}, {signature,<<"afeb9408e40add49ae0c9bf58f0c5f26">>}, {started_on,1351908916}, {total_changes,2503544}, {type,indexer}, {updated_on,1351908922}]]}, {memory, [{total,158348888}, {processes,110956368}, {processes_used,110850056}, {system,47392520}, {atom,1414713}, {atom_used,1394174}, {binary,8695376}, {code,14610103}, {ets,4603544}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,6905212928}, {buffered_memory,453500928}, {free_memory,21296668672}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{3132524,4999}}, {context_switches,{24145090,0}}, {garbage_collection,{3933380,16815182721,0}}, {io,{{input,27086423886},{output,2501711092}}}, {reductions,{7055040522,100982041}}, {run_queue,0}, {runtime,{791250,8470}}]}, {system_stats, [{cpu_utilization_rate,47.36842105263158}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,6236681601}, {couch_docs_data_size,6073140266}, {couch_views_actual_disk_size,3817506}, {couch_views_data_size,8782}, {curr_items,2503544}, {curr_items_tot,2503544}, {mem_used,3545759256}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3132}, {memory_data,{33657663488,12287201280,{<0.19800.0>,4714984}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 20797528 kB\nBuffers: 442872 kB\nCached: 6743372 kB\nSwapCached: 0 kB\nActive: 5744616 kB\nInactive: 6005284 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 20797528 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 11684 kB\nWriteback: 0 kB\nAnonPages: 4563556 kB\nMapped: 17968 kB\nSlab: 276336 kB\nPageTables: 13816 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 6293316 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,908918,989428}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.31180.0>">>}, {changes_done,6566}, {design_documents,[<<"_design/A">>]}, {indexer_type,main}, {initial_build,true}, {progress,0}, {set,<<"default">>}, {signature,<<"afeb9408e40add49ae0c9bf58f0c5f26">>}, {started_on,1351908907}, {total_changes,2501801}, {type,indexer}, {updated_on,1351908907}]]}, {memory, [{total,141274648}, {processes,91996712}, {processes_used,91797240}, {system,49277936}, {atom,1413905}, {atom_used,1392509}, {binary,6204800}, {code,14613068}, {ets,8951200}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,8101400576}, {buffered_memory,348221440}, {free_memory,19659276288}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{3126431,5001}}, {context_switches,{28110865,0}}, {garbage_collection,{2511803,26111019561,0}}, {io,{{input,36519149127},{output,7611663011}}}, {reductions,{9841078776,75345424}}, {run_queue,0}, {runtime,{998680,6540}}]}, {system_stats, [{cpu_utilization_rate,45.5}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,7480857977}, {couch_docs_data_size,6098523648}, {couch_views_actual_disk_size,8774}, {couch_views_data_size,8774}, {curr_items,2501801}, {curr_items_tot,2501801}, {mem_used,3522567160}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3126}, {memory_data,{33657663488,13945294848,{<18118.19054.0>,2914384}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 19197892 kB\nBuffers: 340060 kB\nCached: 7911524 kB\nSwapCached: 12 kB\nActive: 7457836 kB\nInactive: 5852916 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 19197892 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 5860 kB\nWriteback: 0 kB\nAnonPages: 5059280 kB\nMapped: 18216 kB\nSlab: 310972 kB\nPageTables: 14860 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6801212 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-02T19:16:02.487,ns_1@10.2.1.67:<0.19807.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,31,48}}) Stats for bucket "default": accepting_conns 1 auth_cmds 1 auth_errors 0 bucket_active_conns 1 bucket_conns 5 bytes 3545759256 bytes_read 6011804890 bytes_written 129428471 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 1 cmd_set 2503544 conn_yields 550 connection_structures 5000 curr_connections 9 curr_conns_on_port_11209 6 curr_conns_on_port_11210 1 curr_items 2503544 curr_items_tot 2503544 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-03 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 1817 ep_commit_time 43 ep_commit_time_total 28269 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 40 ep_data_age_highwat 145 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data/default ep_degraded_mode 0 ep_diskqueue_drain 2503544 ep_diskqueue_fill 2503544 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 922 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 2503544 ep_io_read_bytes 0 ep_io_write_bytes 5931572051 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 2482667 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 3479871032 ep_max_checkpoints 2 ep_max_data_size 5242880000 ep_max_item_size 20971520 ep_max_size 5242880000 ep_max_txn_size 10000 ep_max_vbuckets 512 ep_mem_high_wat 3932160000 ep_mem_high_wat 3932160000 ep_mem_low_wat 3145728000 ep_mem_low_wat 3145728000 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 465954 ep_num_expiry_pager_runs 0 ep_num_non_resident 1127724 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 12 ep_num_value_ejects 1127724 ep_oom_errors 0 ep_overhead 14881464 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1351907858 ep_storage_age 39 ep_storage_age_highwat 144 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 3449828504 ep_total_del_items 0 ep_total_enqueued 2503544 ep_total_new_items 2503544 ep_total_persisted 2503544 ep_uncommitted_items 0 ep_value_size 3272788504 ep_vb0 0 ep_vb_snapshot_total 112 ep_vb_total 128 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_130_g8b09894 ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 729 get_hits 0 get_misses 1 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 3545759256 pid 17252 pointer_size 64 rejected_conns 0 rusage_system 107.414670 rusage_user 298.245659 threads 4 time 1351908961 total_connections 11 uptime 1120 vb_active_curr_items 2503544 vb_active_eject 1127724 vb_active_expired 0 vb_active_ht_memory 12645376 vb_active_itm_memory 3449828504 vb_active_meta_data_memory 197779976 vb_active_num 128 vb_active_num_non_resident 1127724 vb_active_num_ref_ejects 1127724 vb_active_num_ref_items 2503544 vb_active_ops_create 2503544 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 54 vb_active_queue_age 0 vb_active_queue_drain 2503544 vb_active_queue_fill 2503544 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_meta_data_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-02T19:16:23.788,ns_1@10.2.1.67:ns_doctor<0.19607.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,908978,999872}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.15929.1>">>}, {changes_done,1340392}, {design_documents,[<<"_design/A">>]}, {indexer_type,main}, {initial_build,true}, {progress,53}, {set,<<"default">>}, {signature,<<"afeb9408e40add49ae0c9bf58f0c5f26">>}, {started_on,1351908950}, {total_changes,2497553}, {type,indexer}, {updated_on,1351909008}]]}, {memory, [{total,168611288}, {processes,111003152}, {processes_used,109472624}, {system,57608136}, {atom,1502673}, {atom_used,1494429}, {binary,13473200}, {code,15256817}, {ets,9269760}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,8006930432}, {buffered_memory,252796928}, {free_memory,19821621248}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{3187193,5000}}, {context_switches,{31826709,0}}, {garbage_collection,{2743915,30257112509,0}}, {io,{{input,47173158266},{output,7935296254}}}, {reductions,{11743494906,98755150}}, {run_queue,1}, {runtime,{1206830,8480}}]}, {system_stats, [{cpu_utilization_rate,45.13715710723192}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,7281059183}, {couch_docs_data_size,6068913143}, {couch_views_actual_disk_size,93076087}, {couch_views_data_size,8775}, {curr_items,2497553}, {curr_items_tot,2497553}, {mem_used,3930588600}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3187}, {memory_data,{33657663488,13841629184,{<18077.15929.1>,58013176}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 19357548 kB\nBuffers: 246872 kB\nCached: 7819268 kB\nSwapCached: 0 kB\nActive: 10267120 kB\nInactive: 2901888 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 19357548 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 94456 kB\nWriteback: 0 kB\nAnonPages: 5102744 kB\nMapped: 16312 kB\nSlab: 295968 kB\nPageTables: 14480 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6866316 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,908981,696414}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.30141.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,93}, {started_on,1351908719}, {total_vbuckets,129}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1351908944}, {vbuckets_done,121}], [{pid,<<"<0.31992.0>">>}, {changes_done,1318990}, {design_documents,[<<"_design/A">>]}, {indexer_type,main}, {initial_build,true}, {progress,52}, {set,<<"default">>}, {signature,<<"afeb9408e40add49ae0c9bf58f0c5f26">>}, {started_on,1351908881}, {total_changes,2497102}, {type,indexer}, {updated_on,1351908945}]]}, {memory, [{total,169733416}, {processes,116017408}, {processes_used,115732248}, {system,53716008}, {atom,1413905}, {atom_used,1392509}, {binary,10559624}, {code,14575828}, {ets,9090080}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,6997495808}, {buffered_memory,292503552}, {free_memory,20969660416}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{3189440,4943}}, {context_switches,{33346442,0}}, {garbage_collection,{2806827,29844953552,0}}, {io,{{input,46115795744},{output,8332292874}}}, {reductions,{11485305937,91957917}}, {run_queue,0}, {runtime,{1103760,8090}}]}, {system_stats, [{cpu_utilization_rate,44.75}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,6279048657}, {couch_docs_data_size,6060675254}, {couch_views_actual_disk_size,91576812}, {couch_views_data_size,8776}, {curr_items,2497102}, {curr_items_tot,2497102}, {mem_used,3929970856}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3189}, {memory_data,{33657663488,12798726144,{<18117.31992.0>,62127056}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 20478184 kB\nBuffers: 285648 kB\nCached: 6833492 kB\nSwapCached: 140 kB\nActive: 8830004 kB\nInactive: 3238476 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 20478184 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 131120 kB\nWriteback: 5340 kB\nAnonPages: 4949300 kB\nMapped: 18024 kB\nSlab: 274016 kB\nPageTables: 14844 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6712236 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,908983,786162}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.647.1>">>}, {changes_done,1525230}, {design_documents,[<<"_design/A">>]}, {indexer_type,main}, {initial_build,true}, {progress,60}, {set,<<"default">>}, {signature,<<"afeb9408e40add49ae0c9bf58f0c5f26">>}, {started_on,1351908916}, {total_changes,2503544}, {type,indexer}, {updated_on,1351908981}]]}, {memory, [{total,139727784}, {processes,92988184}, {processes_used,92882016}, {system,46739600}, {atom,1414713}, {atom_used,1394174}, {binary,7954440}, {code,14610103}, {ets,4691440}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,7007408128}, {buffered_memory,454045696}, {free_memory,21189885952}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{3192524,5000}}, {context_switches,{27394520,0}}, {garbage_collection,{4235154,19437670288,0}}, {io,{{input,36511947258},{output,2739349804}}}, {reductions,{8334190774,119935107}}, {run_queue,0}, {runtime,{898150,9910}}]}, {system_stats, [{cpu_utilization_rate,41.64588528678304}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,6236681601}, {couch_docs_data_size,6073140266}, {couch_views_actual_disk_size,112744208}, {couch_views_data_size,8782}, {curr_items,2503544}, {curr_items_tot,2503544}, {mem_used,3545759256}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3192}, {memory_data,{33657663488,12472475648,{<0.647.1>,47742000}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 20693248 kB\nBuffers: 443404 kB\nCached: 6843172 kB\nSwapCached: 0 kB\nActive: 9026400 kB\nInactive: 2824800 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 20693248 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 102476 kB\nWriteback: 0 kB\nAnonPages: 4564424 kB\nMapped: 17968 kB\nSlab: 279352 kB\nPageTables: 13928 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 6294736 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,908978,990082}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.31180.0>">>}, {changes_done,1369606}, {design_documents,[<<"_design/A">>]}, {indexer_type,main}, {initial_build,true}, {progress,54}, {set,<<"default">>}, {signature,<<"afeb9408e40add49ae0c9bf58f0c5f26">>}, {started_on,1351908907}, {total_changes,2501801}, {type,indexer}, {updated_on,1351908967}]]}, {memory, [{total,172610880}, {processes,119187744}, {processes_used,118990104}, {system,53423136}, {atom,1413905}, {atom_used,1392509}, {binary,10058472}, {code,14613068}, {ets,9041872}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,8202797056}, {buffered_memory,348712960}, {free_memory,19541569536}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{3186431,5000}}, {context_switches,{31284428,0}}, {garbage_collection,{2802022,28613943476,0}}, {io,{{input,45669231773},{output,7830083590}}}, {reductions,{11072658608,97712654}}, {run_queue,0}, {runtime,{1105270,8460}}]}, {system_stats, [{cpu_utilization_rate,48.25}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,7480857977}, {couch_docs_data_size,6098523648}, {couch_views_actual_disk_size,96045324}, {couch_views_data_size,8774}, {curr_items,2501801}, {curr_items_tot,2501801}, {mem_used,3522567160}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3186}, {memory_data,{33657663488,14118133760,{<18118.31180.0>,43638912}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 19083564 kB\nBuffers: 340540 kB\nCached: 8010544 kB\nSwapCached: 12 kB\nActive: 10561668 kB\nInactive: 2861048 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 19083564 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 95340 kB\nWriteback: 0 kB\nAnonPages: 5071664 kB\nMapped: 18216 kB\nSlab: 313564 kB\nPageTables: 14912 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6815108 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-02T19:17:23.787,ns_1@10.2.1.67:ns_doctor<0.19607.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.65', [{last_heard,{1351,909038,997302}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.15929.1>">>}, {changes_done,2436953}, {design_documents,[<<"_design/A">>]}, {indexer_type,main}, {initial_build,true}, {progress,97}, {set,<<"default">>}, {signature,<<"afeb9408e40add49ae0c9bf58f0c5f26">>}, {started_on,1351908950}, {total_changes,2497553}, {type,indexer}, {updated_on,1351909055}]]}, {memory, [{total,152997128}, {processes,91618088}, {processes_used,90084168}, {system,61379040}, {atom,1502673}, {atom_used,1494429}, {binary,17061616}, {code,15256817}, {ets,9452440}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,11350716416}, {total_swap,11350876160}, {cached_memory,8064098304}, {buffered_memory,253616128}, {free_memory,19801161728}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{3247192,5000}}, {context_switches,{34463424,0}}, {garbage_collection,{2986442,32905949778,0}}, {io,{{input,54569821165},{output,8338175539}}}, {reductions,{13007479266,158069298}}, {run_queue,0}, {runtime,{1303640,4250}}]}, {system_stats, [{cpu_utilization_rate,24.25}, {swap_total,11350876160}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,7281059183}, {couch_docs_data_size,6068913143}, {couch_views_actual_disk_size,255587111}, {couch_views_data_size,8775}, {curr_items,2497553}, {curr_items_tot,2497553}, {mem_used,3930588600}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3247}, {memory_data,{33657663488,13928333312,{<18077.15929.1>,25150296}}}, {disk_data, [{"/",51088800,10}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 19337072 kB\nBuffers: 247672 kB\nCached: 7875096 kB\nSwapCached: 0 kB\nActive: 12600196 kB\nInactive: 585280 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 19337072 kB\nSwapTotal: 11084840 kB\nSwapFree: 11084684 kB\nDirty: 104528 kB\nWriteback: 0 kB\nAnonPages: 5062528 kB\nMapped: 16312 kB\nSlab: 298640 kB\nPageTables: 14404 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 27519244 kB\nCommitted_AS: 6809240 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6728 kB\nVmallocChunk: 34359730275 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.66', [{last_heard,{1351,909041,694951}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.31992.0>">>}, {changes_done,2396218}, {design_documents,[<<"_design/A">>]}, {indexer_type,main}, {initial_build,true}, {progress,95}, {set,<<"default">>}, {signature,<<"afeb9408e40add49ae0c9bf58f0c5f26">>}, {started_on,1351908881}, {total_changes,2497102}, {type,indexer}, {updated_on,1351908999}]]}, {memory, [{total,142300648}, {processes,94434592}, {processes_used,94149744}, {system,47866056}, {atom,1413905}, {atom_used,1392509}, {binary,4362144}, {code,14575828}, {ets,9280792}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684469248}, {total_swap,10684628992}, {cached_memory,6886453248}, {buffered_memory,293605376}, {free_memory,21112152064}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{3249439,5000}}, {context_switches,{36639694,0}}, {garbage_collection,{3089479,32403209658,0}}, {io,{{input,53930002836},{output,8635313915}}}, {reductions,{12590266270,75884201}}, {run_queue,0}, {runtime,{1199390,5500}}]}, {system_stats, [{cpu_utilization_rate,26.184538653366584}, {swap_total,10684628992}, {swap_used,159744}]}, {interesting_stats, [{couch_docs_actual_disk_size,6071129467}, {couch_docs_data_size,6057227702}, {couch_views_actual_disk_size,227984036}, {couch_views_data_size,8776}, {curr_items,2497102}, {curr_items_tot,2497102}, {mem_used,3929970856}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3249}, {memory_data,{33657663488,12564193280,{<18117.31992.0>,24233856}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 20617336 kB\nBuffers: 286724 kB\nCached: 6725052 kB\nSwapCached: 140 kB\nActive: 11253364 kB\nInactive: 677004 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 20617336 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434052 kB\nDirty: 164652 kB\nWriteback: 0 kB\nAnonPages: 4918080 kB\nMapped: 18020 kB\nSlab: 271616 kB\nPageTables: 14592 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6658340 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.67', [{last_heard,{1351,909043,786647}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.647.1>">>}, {changes_done,2412223}, {design_documents,[<<"_design/A">>]}, {indexer_type,main}, {initial_build,true}, {progress,96}, {set,<<"default">>}, {signature,<<"afeb9408e40add49ae0c9bf58f0c5f26">>}, {started_on,1351908916}, {total_changes,2503544}, {type,indexer}, {updated_on,1351909019}]]}, {memory, [{total,202803832}, {processes,138326528}, {processes_used,138216824}, {system,64477304}, {atom,1414713}, {atom_used,1394174}, {binary,25510768}, {code,14610103}, {ets,4872592}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10701078528}, {total_swap,10701078528}, {cached_memory,7039221760}, {buffered_memory,454692864}, {free_memory,21137100800}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{3252525,5001}}, {context_switches,{29911878,0}}, {garbage_collection,{4442523,22121479441,0}}, {io,{{input,42936124809},{output,3183491798}}}, {reductions,{9927304769,297426670}}, {run_queue,0}, {runtime,{987050,4520}}]}, {system_stats, [{cpu_utilization_rate,23.19201995012469}, {swap_total,10701078528}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,6236681601}, {couch_docs_data_size,6073140266}, {couch_views_actual_disk_size,214117483}, {couch_views_data_size,0}, {curr_items,2503544}, {curr_items_tot,2503544}, {mem_used,3545759256}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3252}, {memory_data,{33657663488,12603371520,{<0.647.1>,74966056}}}, {disk_data, [{"/",49781912,9}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 20641760 kB\nBuffers: 444036 kB\nCached: 6874240 kB\nSwapCached: 0 kB\nActive: 11171612 kB\nInactive: 725480 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 20641760 kB\nSwapTotal: 10450272 kB\nSwapFree: 10450272 kB\nDirty: 55196 kB\nWriteback: 0 kB\nAnonPages: 4578720 kB\nMapped: 17968 kB\nSlab: 283224 kB\nPageTables: 13968 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26884676 kB\nCommitted_AS: 6333704 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6416 kB\nVmallocChunk: 34359730587 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.68', [{last_heard,{1351,909038,989186}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.31180.0>">>}, {changes_done,2452279}, {design_documents,[<<"_design/A">>]}, {indexer_type,main}, {initial_build,true}, {progress,98}, {set,<<"default">>}, {signature,<<"afeb9408e40add49ae0c9bf58f0c5f26">>}, {started_on,1351908907}, {total_changes,2501801}, {type,indexer}, {updated_on,1351909014}]]}, {memory, [{total,178342592}, {processes,119421040}, {processes_used,119218032}, {system,58921552}, {atom,1413905}, {atom_used,1392509}, {binary,15574584}, {code,14613068}, {ets,9224464}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684616704}, {total_swap,10684628992}, {cached_memory,8235151360}, {buffered_memory,349507584}, {free_memory,19534831616}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data"},{index_path,"/data2"}]}, {statistics, [{wall_clock,{3246431,5002}}, {context_switches,{33862000,0}}, {garbage_collection,{3028295,31258243778,0}}, {io,{{input,52732704935},{output,8240321795}}}, {reductions,{12392835937,245611998}}, {run_queue,0}, {runtime,{1200890,4600}}]}, {system_stats, [{cpu_utilization_rate,25.75}, {swap_total,10684628992}, {swap_used,12288}]}, {interesting_stats, [{couch_docs_actual_disk_size,7480857977}, {couch_docs_data_size,6098523648}, {couch_views_actual_disk_size,269949736}, {couch_views_data_size,8774}, {curr_items,2501801}, {curr_items_tot,2501801}, {mem_used,3522567160}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-47a991b-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-47a991b-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-47a991b-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1925-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3246}, {memory_data,{33657663488,14231638016,{<18118.31180.0>,58013264}}}, {disk_data, [{"/",49781912,5}, {"/dev/shm",16434404,0}, {"/data",492257288,2}, {"/data2",246132896,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 19076984 kB\nBuffers: 341316 kB\nCached: 8042140 kB\nSwapCached: 12 kB\nActive: 12826868 kB\nInactive: 598164 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 19076984 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434196 kB\nDirty: 113696 kB\nWriteback: 0 kB\nAnonPages: 5041680 kB\nMapped: 18216 kB\nSlab: 316416 kB\nPageTables: 14876 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6804640 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6408 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-02T19:17:42.488,ns_1@10.2.1.67:<0.19807.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,31,49}}) Stats for bucket "default": accepting_conns 1 auth_cmds 1 auth_errors 0 bucket_active_conns 1 bucket_conns 5 bytes 3545759272 bytes_read 6011815380 bytes_written 135995776 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 1 cmd_set 2503544 conn_yields 550 connection_structures 5000 curr_connections 9 curr_conns_on_port_11209 6 curr_conns_on_port_11210 1 curr_items 2503544 curr_items_tot 2503544 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-03 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 1817 ep_commit_time 43 ep_commit_time_total 28269 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 40 ep_data_age_highwat 145 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data/default ep_degraded_mode 0 ep_diskqueue_drain 2503544 ep_diskqueue_fill 2503544 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 922 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 2503544 ep_io_read_bytes 0 ep_io_write_bytes 5931572051 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 2482667 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 3479871032 ep_max_checkpoints 2 ep_max_data_size 5242880000 ep_max_item_size 20971520 ep_max_size 5242880000 ep_max_txn_size 10000 ep_max_vbuckets 512 ep_mem_high_wat 3932160000 ep_mem_high_wat 3932160000 ep_mem_low_wat 3145728000 ep_mem_low_wat 3145728000 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 465954 ep_num_expiry_pager_runs 0 ep_num_non_resident 1127724 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 12 ep_num_value_ejects 1127724 ep_oom_errors 0 ep_overhead 14881464 e