============================================================================== couchbase logs (stats) cbbrowse_logs stats ============================================================================== [ns_doctor:debug,2012-11-14T10:46:47.286,ns_1@127.0.0.1:ns_doctor<0.375.0>:ns_doctor:handle_info:133]Got initial status [{'ns_1@127.0.0.1', [{last_heard,{1352,918807,286201}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,40156352}, {processes,9378880}, {processes_used,9368112}, {system,30777472}, {atom,1171065}, {atom_used,1138716}, {binary,148360}, {code,11321108}, {ets,875056}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684477440}, {total_swap,10684628992}, {cached_memory,15226200064}, {buffered_memory,286932992}, {free_memory,17481834496}, {total_memory,33657663488}]}, {node_storage_conf, [{db_path, "/opt/couchbase/var/lib/couchbase/data/"}, {index_path, "/opt/couchbase/var/lib/couchbase/data/"}]}, {statistics, [{wall_clock,{1908,5}}, {context_switches,{26358,0}}, {garbage_collection,{7294,33536720,0}}, {io,{{input,6771338},{output,663234}}}, {reductions,{19604958,19573}}, {run_queue,0}, {runtime,{1140,0}}]}, {system_stats, [{cpu_utilization_rate,0}, {swap_total,0}, {swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,1}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1}, {memory_data, {33657663488,16157106176,{<0.6.0>,601272}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,2}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 17072228 kB\nBuffers: 280208 kB\nCached: 14869336 kB\nSwapCached: 0 kB\nActive: 2136132 kB\nInactive: 13139360 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 17072228 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434060 kB\nDirty: 419692 kB\nWriteback: 0 kB\nAnonPages: 125612 kB\nMapped: 13548 kB\nSlab: 482684 kB\nPageTables: 4232 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 1572744 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-14T10:47:04.344,ns_1@10.2.1.58:ns_doctor<0.374.0>:ns_doctor:handle_info:133]Got initial status [{'ns_1@10.2.1.58', [{last_heard,{1352,918824,344688}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,39584592}, {processes,8835880}, {processes_used,8825112}, {system,30748712}, {atom,1171873}, {atom_used,1138823}, {binary,138744}, {code,11302596}, {ets,875288}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684477440}, {total_swap,10684628992}, {cached_memory,15226830848}, {buffered_memory,287322112}, {free_memory,17481838592}, {total_memory,33657663488}]}, {node_storage_conf, [{db_path, "/opt/couchbase/var/lib/couchbase/data/"}, {index_path, "/opt/couchbase/var/lib/couchbase/data/"}]}, {statistics, [{wall_clock,{1857,2}}, {context_switches,{24990,0}}, {garbage_collection,{6813,29013159,0}}, {io,{{input,6772553},{output,659956}}}, {reductions,{16908705,17718}}, {run_queue,1}, {runtime,{1070,10}}]}, {system_stats, [{cpu_utilization_rate,0}, {swap_total,0}, {swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,1}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1}, {memory_data, {33657663488,16157888512,{<0.6.0>,601320}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,2}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 17072232 kB\nBuffers: 280580 kB\nCached: 14869960 kB\nSwapCached: 0 kB\nActive: 2139268 kB\nInactive: 13136180 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 17072232 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434060 kB\nDirty: 420284 kB\nWriteback: 0 kB\nAnonPages: 124592 kB\nMapped: 14092 kB\nSlab: 483388 kB\nPageTables: 4784 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 1575476 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-14T10:47:13.681,ns_1@10.2.1.58:ns_doctor<0.731.0>:ns_doctor:handle_info:133]Got initial status [{'ns_1@10.2.1.58', [{last_heard,{1352,918833,681331}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,41921928}, {processes,9540608}, {processes_used,9463648}, {system,32381320}, {atom,1242281}, {atom_used,1241108}, {binary,171584}, {code,12669653}, {ets,997280}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684477440}, {total_swap,10684628992}, {cached_memory,627023872}, {buffered_memory,287731712}, {free_memory,32254631936}, {total_memory,33657663488}]}, {node_storage_conf, [{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{11193,2}}, {context_switches,{32459,0}}, {garbage_collection,{8900,32527298,0}}, {io,{{input,7653613},{output,1060596}}}, {reductions,{18222735,47578}}, {run_queue,1}, {runtime,{1670,10}}]}, {system_stats, [{cpu_utilization_rate,0}, {swap_total,0}, {swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,11}, {memory_data, {33657663488,16157888512,{<0.6.0>,601320}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,2}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 31498664 kB\nBuffers: 280988 kB\nCached: 612328 kB\nSwapCached: 0 kB\nActive: 529464 kB\nInactive: 493052 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 31498664 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434060 kB\nDirty: 243168 kB\nWriteback: 0 kB\nAnonPages: 129132 kB\nMapped: 15852 kB\nSlab: 307900 kB\nPageTables: 4820 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 1600488 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-14T10:47:31.453,ns_1@10.2.1.58:ns_doctor<0.407.0>:ns_doctor:handle_info:133]Got initial status [{'ns_1@10.2.1.58', [{last_heard,{1352,918851,453039}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,39523160}, {processes,8894544}, {processes_used,8887024}, {system,30628616}, {atom,1122825}, {atom_used,1120514}, {binary,194744}, {code,11125274}, {ets,922680}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684477440}, {total_swap,10684628992}, {cached_memory,627372032}, {buffered_memory,288133120}, {free_memory,32262676480}, {total_memory,33657663488}]}, {node_storage_conf, [{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1873,6}}, {context_switches,{25924,0}}, {garbage_collection,{7193,32809358,0}}, {io,{{input,6679658},{output,619937}}}, {reductions,{19281153,26940}}, {run_queue,0}, {runtime,{1150,10}}]}, {system_stats, [{cpu_utilization_rate,0}, {swap_total,0}, {swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1}, {memory_data, {33657663488,1377542144,{<0.6.0>,601272}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 31506520 kB\nBuffers: 281380 kB\nCached: 612668 kB\nSwapCached: 0 kB\nActive: 524164 kB\nInactive: 493520 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 31506520 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434060 kB\nDirty: 239720 kB\nWriteback: 0 kB\nAnonPages: 123896 kB\nMapped: 13248 kB\nSlab: 306704 kB\nPageTables: 4552 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 1571604 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-14T10:48:31.431,ns_1@10.2.1.58:ns_doctor<0.407.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,918911,428462}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,45508088}, {processes,13252144}, {processes_used,13245688}, {system,32255944}, {atom,1234201}, {atom_used,1230199}, {binary,164880}, {code,12490396}, {ets,1065328}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,20967424}, {buffered_memory,2654208}, {free_memory,33365651456}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{61848,5001}}, {context_switches,{34864,0}}, {garbage_collection,{8680,36046784,0}}, {io,{{input,7424183},{output,794512}}}, {reductions,{20315046,63148}}, {run_queue,0}, {runtime,{3950,210}}]}, {system_stats, [{cpu_utilization_rate,1.4962593516209477}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,61}, {memory_data,{33657663488,291950592,{<0.6.0>,601480}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 32583644 kB\nBuffers: 2592 kB\nCached: 20476 kB\nSwapCached: 0 kB\nActive: 208900 kB\nInactive: 3988 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 32583644 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 440 kB\nWriteback: 0 kB\nAnonPages: 189864 kB\nMapped: 15300 kB\nSlab: 32144 kB\nPageTables: 5184 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 1806324 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-14T10:49:31.431,ns_1@10.2.1.58:ns_doctor<0.407.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,918971,427499}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,46083352}, {processes,13741728}, {processes_used,13734472}, {system,32341624}, {atom,1235009}, {atom_used,1231796}, {binary,192080}, {code,12514649}, {ets,1095568}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,21282816}, {buffered_memory,2785280}, {free_memory,33366163456}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{121847,5000}}, {context_switches,{42526,0}}, {garbage_collection,{9641,38357456,0}}, {io,{{input,7477816},{output,887788}}}, {reductions,{21159614,65154}}, {run_queue,0}, {runtime,{6590,210}}]}, {system_stats, [{cpu_utilization_rate,1.5037593984962405}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,121}, {memory_data,{33657663488,291500032,{<0.6.0>,601480}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 32584144 kB\nBuffers: 2720 kB\nCached: 20784 kB\nSwapCached: 0 kB\nActive: 209348 kB\nInactive: 4248 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 32584144 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 64 kB\nWriteback: 0 kB\nAnonPages: 190092 kB\nMapped: 15316 kB\nSlab: 31740 kB\nPageTables: 5136 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 1803168 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-14T10:50:31.431,ns_1@10.2.1.58:ns_doctor<0.407.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,919031,427488}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,46098712}, {processes,13724880}, {processes_used,13702904}, {system,32373832}, {atom,1235009}, {atom_used,1231796}, {binary,190504}, {code,12514649}, {ets,1127416}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,21565440}, {buffered_memory,3010560}, {free_memory,33364647936}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{181847,5000}}, {context_switches,{50501,0}}, {garbage_collection,{10576,40772268,0}}, {io,{{input,7750918},{output,1193841}}}, {reductions,{22022139,95730}}, {run_queue,0}, {runtime,{9270,240}}]}, {system_stats, [{cpu_utilization_rate,2.2443890274314215}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,181}, {memory_data,{33657663488,292954112,{<0.6.0>,601480}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 32582664 kB\nBuffers: 2940 kB\nCached: 21060 kB\nSwapCached: 0 kB\nActive: 210040 kB\nInactive: 4800 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 32582664 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 56 kB\nWriteback: 0 kB\nAnonPages: 190812 kB\nMapped: 15316 kB\nSlab: 31764 kB\nPageTables: 5136 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 1804468 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-14T10:51:31.431,ns_1@10.2.1.58:ns_doctor<0.407.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,919091,427555}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,46336968}, {processes,13911160}, {processes_used,13889184}, {system,32425808}, {atom,1235009}, {atom_used,1231796}, {binary,211472}, {code,12514649}, {ets,1155920}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,21725184}, {buffered_memory,3444736}, {free_memory,33364099072}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{241847,5000}}, {context_switches,{58175,0}}, {garbage_collection,{11533,43165576,0}}, {io,{{input,7789099},{output,1283439}}}, {reductions,{22908877,73514}}, {run_queue,0}, {runtime,{12150,240}}]}, {system_stats, [{cpu_utilization_rate,7.4812967581047385}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,241}, {memory_data,{33657663488,293953536,{<0.6.0>,601480}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 32582128 kB\nBuffers: 3364 kB\nCached: 21216 kB\nSwapCached: 0 kB\nActive: 208924 kB\nInactive: 5148 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 32582128 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 392 kB\nWriteback: 20 kB\nAnonPages: 189308 kB\nMapped: 14644 kB\nSlab: 31792 kB\nPageTables: 5204 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 1804484 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-14T10:52:31.431,ns_1@10.2.1.58:ns_doctor<0.407.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,919151,427478}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,46308256}, {processes,13847960}, {processes_used,13825984}, {system,32460296}, {atom,1235009}, {atom_used,1231796}, {binary,214528}, {code,12514649}, {ets,1185088}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,21803008}, {buffered_memory,3588096}, {free_memory,33362862080}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{301847,5000}}, {context_switches,{65964,0}}, {garbage_collection,{12444,45653576,0}}, {io,{{input,7826427},{output,1380390}}}, {reductions,{23778237,53955}}, {run_queue,0}, {runtime,{14870,200}}]}, {system_stats, [{cpu_utilization_rate,2.2443890274314215}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,301}, {memory_data,{33657663488,294801408,{<0.6.0>,601480}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 32580920 kB\nBuffers: 3504 kB\nCached: 21292 kB\nSwapCached: 0 kB\nActive: 211880 kB\nInactive: 4920 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 32580920 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 72 kB\nWriteback: 0 kB\nAnonPages: 192004 kB\nMapped: 15316 kB\nSlab: 31700 kB\nPageTables: 5136 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 1805772 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-14T10:53:31.431,ns_1@10.2.1.58:ns_doctor<0.407.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,919211,427520}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,46437112}, {processes,13946680}, {processes_used,13924704}, {system,32490432}, {atom,1235009}, {atom_used,1231796}, {binary,211240}, {code,12514649}, {ets,1216608}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,21528576}, {buffered_memory,3727360}, {free_memory,33362608128}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{361847,5000}}, {context_switches,{73815,0}}, {garbage_collection,{13392,48092965,0}}, {io,{{input,8110368},{output,1549711}}}, {reductions,{24661036,97253}}, {run_queue,0}, {runtime,{17530,240}}]}, {system_stats, [{cpu_utilization_rate,2.5},{swap_total,10684628992},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,361}, {memory_data,{33657663488,294928384,{<0.6.0>,601480}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 32580672 kB\nBuffers: 3640 kB\nCached: 21024 kB\nSwapCached: 0 kB\nActive: 212216 kB\nInactive: 4800 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 32580672 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 64 kB\nWriteback: 0 kB\nAnonPages: 192172 kB\nMapped: 15316 kB\nSlab: 31632 kB\nPageTables: 5136 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 1806280 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-14T10:54:31.431,ns_1@10.2.1.58:ns_doctor<0.407.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,919271,427524}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,46513408}, {processes,13978464}, {processes_used,13956488}, {system,32534944}, {atom,1235009}, {atom_used,1231796}, {binary,223768}, {code,12514649}, {ets,1246008}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,21647360}, {buffered_memory,3809280}, {free_memory,33362227200}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{421847,5000}}, {context_switches,{81503,0}}, {garbage_collection,{14219,50424426,0}}, {io,{{input,8147687},{output,1640893}}}, {reductions,{25492130,58546}}, {run_queue,0}, {runtime,{20180,210}}]}, {system_stats, [{cpu_utilization_rate,2.0},{swap_total,10684628992},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,421}, {memory_data,{33657663488,295436288,{<0.6.0>,601480}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 32580300 kB\nBuffers: 3720 kB\nCached: 21140 kB\nSwapCached: 0 kB\nActive: 211804 kB\nInactive: 4980 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 32580300 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 116 kB\nWriteback: 0 kB\nAnonPages: 191920 kB\nMapped: 15316 kB\nSlab: 31624 kB\nPageTables: 5136 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 1806280 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-14T10:55:31.431,ns_1@10.2.1.58:ns_doctor<0.407.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,919331,427491}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,46380184}, {processes,13811008}, {processes_used,13789032}, {system,32569176}, {atom,1235009}, {atom_used,1231796}, {binary,227496}, {code,12514649}, {ets,1274616}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,21848064}, {buffered_memory,4886528}, {free_memory,33359867904}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{481847,5000}}, {context_switches,{89147,0}}, {garbage_collection,{15012,52697903,0}}, {io,{{input,8185877},{output,1733150}}}, {reductions,{26291165,54094}}, {run_queue,0}, {runtime,{23020,240}}]}, {system_stats, [{cpu_utilization_rate,2.0},{swap_total,10684628992},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,481}, {memory_data,{33657663488,297922560,{<0.6.0>,601480}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 32577996 kB\nBuffers: 4772 kB\nCached: 21336 kB\nSwapCached: 0 kB\nActive: 210900 kB\nInactive: 6044 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 32577996 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 372 kB\nWriteback: 0 kB\nAnonPages: 190840 kB\nMapped: 14396 kB\nSlab: 33132 kB\nPageTables: 5064 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 1803556 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-14T10:56:22.089,ns_1@10.2.1.58:ns_doctor<0.1953.0>:ns_doctor:handle_info:133]Got initial status [{'ns_1@10.2.1.58', [{last_heard,{1352,919382,82195}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,47212848}, {processes,14211432}, {processes_used,14135128}, {system,33001416}, {atom,1237433}, {atom_used,1234483}, {binary,439048}, {code,12563554}, {ets,1375872}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,22536192}, {buffered_memory,5992448}, {free_memory,33408319488}, {total_memory,33657663488}]}, {node_storage_conf, [{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{532502,3}}, {context_switches,{97958,0}}, {garbage_collection,{16522,56469838,0}}, {io,{{input,8332977},{output,2129549}}}, {reductions,{27464272,37120}}, {run_queue,0}, {runtime,{25280,0}}]}, {system_stats, [{cpu_utilization_rate,0}, {swap_total,0}, {swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,532}, {memory_data, {33657663488,297922560,{<0.6.0>,601480}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 32625684 kB\nBuffers: 5852 kB\nCached: 22008 kB\nSwapCached: 0 kB\nActive: 161136 kB\nInactive: 7480 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 32625684 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 460 kB\nWriteback: 0 kB\nAnonPages: 140904 kB\nMapped: 14544 kB\nSlab: 33660 kB\nPageTables: 6464 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 1614284 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,919382,82207}}, {outgoing_replications_safeness_level, [{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,203101200}, {processes,154440376}, {processes_used,154408152}, {system,48660824}, {atom,1393705}, {atom_used,1362981}, {binary,10163392}, {code,13894422}, {ets,4493624}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7328968704}, {buffered_memory,156725248}, {free_memory,17127505920}, {total_memory,33657663488}]}, {node_storage_conf, [{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{534129,40}}, {context_switches,{5155949,0}}, {garbage_collection,{360695,1552300073,0}}, {io,{{input,92058053},{output,5870112283}}}, {reductions,{908382198,54550}}, {run_queue,0}, {runtime,{103310,10}}]}, {system_stats, [{cpu_utilization_rate,3.75}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7288663077}, {couch_docs_data_size,7286122026}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499537}, {curr_items_tot,3000003}, {mem_used,7948944296}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,534}, {memory_data, {33657663488,16521998336, {<17879.619.0>,2914384}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16726080 kB\nBuffers: 153052 kB\nCached: 7157196 kB\nSwapCached: 0 kB\nActive: 8674204 kB\nInactive: 7170780 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16726080 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 18948 kB\nWriteback: 0 kB\nAnonPages: 8534560 kB\nMapped: 16464 kB\nSlab: 241828 kB\nPageTables: 22072 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10228636 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-14T10:56:22.849,ns_1@10.2.1.58:<0.2072.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,48,39}}) Stats for bucket "default": accepting_conns 1 auth_cmds 0 auth_errors 0 bucket_active_conns 1 bucket_conns 5 bytes 255856 bytes_read 196 bytes_written 120 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 0 conn_yields 0 connection_structures 5000 curr_connections 9 curr_conns_on_port_11209 6 curr_conns_on_port_11210 1 curr_items 0 curr_items_tot 0 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-15 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data2/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 0 ep_commit_time 0 ep_commit_time_total 0 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 0 ep_data_age_highwat 0 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data2/default ep_degraded_mode 1 ep_diskqueue_drain 0 ep_diskqueue_fill 0 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 0 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 0 ep_io_read_bytes 0 ep_io_write_bytes 0 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 0 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 0 ep_max_checkpoints 2 ep_max_data_size 12582912000 ep_max_item_size 20971520 ep_max_size 12582912000 ep_max_txn_size 10000 ep_max_vbuckets 1024 ep_mem_high_wat 9437184000 ep_mem_high_wat 9437184000 ep_mem_low_wat 7549747200 ep_mem_low_wat 7549747200 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 1152 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1352919382 ep_storage_age 0 ep_storage_age_highwat 0 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 0 ep_total_del_items 0 ep_total_enqueued 0 ep_total_new_items 0 ep_total_persisted 0 ep_uncommitted_items 0 ep_value_size 0 ep_vb0 0 ep_vb_snapshot_total 1 ep_vb_total 0 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_135_g40a391b ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 1431 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 255856 pid 29361 pointer_size 64 rejected_conns 0 rusage_system 0.025996 rusage_user 0.035994 threads 4 time 1352919382 total_connections 9 uptime 2 vb_active_curr_items 0 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 0 vb_active_itm_memory 0 vb_active_meta_data_memory 0 vb_active_num 0 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 0 vb_active_ops_create 0 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 0 vb_active_queue_age 0 vb_active_queue_drain 0 vb_active_queue_fill 0 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_meta_data_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-14T10:57:22.074,ns_1@10.2.1.58:ns_doctor<0.1953.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,919437,75155}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',96796817}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,96590656}, {processes,59321840}, {processes_used,59314584}, {system,37268816}, {atom,1332537}, {atom_used,1324944}, {binary,2533216}, {code,13364289}, {ets,2240032}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,1452191744}, {buffered_memory,44220416}, {free_memory,30587596800}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{587495,4999}}, {context_switches,{915593,0}}, {garbage_collection,{369514,581077593,0}}, {io,{{input,27423139},{output,2556962345}}}, {reductions,{132800162,9095518}}, {run_queue,0}, {runtime,{47720,1920}}]}, {system_stats, [{cpu_utilization_rate,5.25}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,1259228125}, {couch_docs_data_size,957783107}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,222106}, {curr_items_tot,429397}, {mem_used,1135991872}, {vb_replica_curr_items,207291}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,587}, {memory_data,{33657663488,825831424,{<0.367.0>,4114672}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29870700 kB\nBuffers: 43184 kB\nCached: 1418156 kB\nSwapCached: 0 kB\nActive: 1500524 kB\nInactive: 1384048 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29870700 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 4688 kB\nWriteback: 0 kB\nAnonPages: 1422568 kB\nMapped: 15588 kB\nSlab: 72248 kB\nPageTables: 7600 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 3110304 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,919439,824323}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',106169796}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,216386136}, {processes,170278904}, {processes_used,169481400}, {system,46107232}, {atom,1401785}, {atom_used,1374887}, {binary,7415728}, {code,14061510}, {ets,4616648}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,6243500032}, {buffered_memory,159866880}, {free_memory,18177478656}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{591876,4986}}, {context_switches,{5509922,0}}, {garbage_collection,{570085,3374877032,0}}, {io,{{input,102379398},{output,5920024319}}}, {reductions,{1037361240,11258237}}, {run_queue,0}, {runtime,{135060,3410}}]}, {system_stats, [{cpu_utilization_rate,37.37623762376238}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,6294962162}, {couch_docs_data_size,6292595455}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1289398}, {curr_items_tot,2558884}, {mem_used,6804000888}, {vb_replica_curr_items,1269486}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,591}, {memory_data,{33657663488,16467406848,{<17879.67.0>,12342376}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 17751444 kB\nBuffers: 156120 kB\nCached: 6097168 kB\nSwapCached: 0 kB\nActive: 8724432 kB\nInactive: 6110076 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 17751444 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 20112 kB\nWriteback: 192 kB\nAnonPages: 8581312 kB\nMapped: 16820 kB\nSlab: 225572 kB\nPageTables: 22136 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10296316 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,919441,478399}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default", [{'ns_1@10.2.1.58',5177542},{'ns_1@10.2.1.61',37315782}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,208255144}, {processes,160801776}, {processes_used,160375352}, {system,47453368}, {atom,1338193}, {atom_used,1334188}, {binary,9993272}, {code,13493302}, {ets,3989512}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,8600858624}, {buffered_memory,166080512}, {free_memory,15754137600}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{589941,4997}}, {context_switches,{5463975,0}}, {garbage_collection,{680608,1950021577,0}}, {io,{{input,105532244},{output,6957695789}}}, {reductions,{781367975,11109597}}, {run_queue,0}, {runtime,{117070,1840}}]}, {system_stats, [{cpu_utilization_rate,39.75308641975309}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,8514518192}, {couch_docs_data_size,7285311926}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1476590}, {curr_items_tot,3000003}, {mem_used,7974187992}, {vb_replica_curr_items,1523413}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,589}, {memory_data,{33657663488,16794689536,{<17926.368.0>,5085816}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 15384900 kB\nBuffers: 162188 kB\nCached: 8399276 kB\nSwapCached: 0 kB\nActive: 9341156 kB\nInactive: 7813804 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 15384900 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 20008 kB\nWriteback: 0 kB\nAnonPages: 8593104 kB\nMapped: 16404 kB\nSlab: 272372 kB\nPageTables: 22444 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10306880 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-14T10:58:02.854,ns_1@10.2.1.58:<0.2072.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,48,39}}) Stats for bucket "default": accepting_conns 1 auth_cmds 0 auth_errors 0 bucket_active_conns 1 bucket_conns 12 bytes 1932450008 bytes_read 4302605083 bytes_written 898027477 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 0 conn_yields 6435 connection_structures 5000 curr_connections 15 curr_conns_on_port_11209 12 curr_conns_on_port_11210 1 curr_items 385955 curr_items_tot 730866 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-15 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data2/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 2333 ep_commit_time 122 ep_commit_time_total 55967 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 2 ep_data_age_highwat 5 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data2/default ep_degraded_mode 0 ep_diskqueue_drain 2385027 ep_diskqueue_fill 1703746 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 75 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 1428165 ep_io_read_bytes 0 ep_io_write_bytes 3381161603 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 43769 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 1787452502 ep_max_checkpoints 2 ep_max_data_size 12582912000 ep_max_item_size 20971520 ep_max_size 12582912000 ep_max_txn_size 10000 ep_max_vbuckets 1024 ep_mem_high_wat 9437184000 ep_mem_high_wat 9437184000 ep_mem_low_wat 7549747200 ep_mem_low_wat 7549747200 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 63261234 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1352919382 ep_storage_age 1 ep_storage_age_highwat 2 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 1778682110 ep_total_del_items 0 ep_total_enqueued 1703746 ep_total_new_items 730866 ep_total_persisted 1428165 ep_uncommitted_items 0 ep_value_size 1727192781 ep_vb0 0 ep_vb_snapshot_total 504 ep_vb_total 251 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_135_g40a391b ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 1431 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 1932450008 pid 29361 pointer_size 64 rejected_conns 0 rusage_system 21.009806 rusage_user 76.878312 tap_checkpoint_end_received 359 tap_checkpoint_end_sent 123 tap_checkpoint_start_received 747 tap_checkpoint_start_sent 132 tap_connect_received 1 tap_mutation_received 1780439 tap_mutation_sent 362494 tap_opaque_received 1751 tap_opaque_sent 133 tap_vbucket_set_received 264 threads 4 time 1352919482 total_connections 1279 uptime 102 vb_active_curr_items 385955 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 3314784 vb_active_itm_memory 939279661 vb_active_meta_data_memory 30490445 vb_active_num 132 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 385955 vb_active_ops_create 385955 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 202249 vb_active_perc_mem_resident 100 vb_active_queue_age 0 vb_active_queue_drain 1128541 vb_active_queue_fill 766024 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 344911 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 2988328 vb_replica_itm_memory 839402449 vb_replica_meta_data_memory 27247969 vb_replica_num 119 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 344911 vb_replica_ops_create 344911 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 495050 vb_replica_perc_mem_resident 100 vb_replica_queue_age 0 vb_replica_queue_drain 1256486 vb_replica_queue_fill 937722 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-14T10:58:22.075,ns_1@10.2.1.58:ns_doctor<0.1953.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,919497,77729}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',122861031}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.7345.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,75}, {started_on,1352919472}, {total_vbuckets,229}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1352919497}, {vbuckets_done,172}]]}, {memory, [{total,113021256}, {processes,74057096}, {processes_used,74050976}, {system,38964160}, {atom,1332537}, {atom_used,1324944}, {binary,3612088}, {code,13364289}, {ets,2515416}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,2544250880}, {buffered_memory,72273920}, {free_memory,28352557056}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{647497,5002}}, {context_switches,{1685919,0}}, {garbage_collection,{661540,1054764981,0}}, {io,{{input,44828088},{output,4816880283}}}, {reductions,{232279077,7025265}}, {run_queue,0}, {runtime,{68110,1540}}]}, {system_stats, [{cpu_utilization_rate,42.01030927835052}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,2447154156}, {couch_docs_data_size,1902337315}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,427344}, {curr_items_tot,809761}, {mem_used,2145192448}, {vb_replica_curr_items,382417}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,647}, {memory_data,{33657663488,3829891072,{<0.367.0>,5085816}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 27688168 kB\nBuffers: 70580 kB\nCached: 2484620 kB\nSwapCached: 0 kB\nActive: 2603596 kB\nInactive: 2428916 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 27688168 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 7832 kB\nWriteback: 0 kB\nAnonPages: 2477444 kB\nMapped: 15596 kB\nSlab: 103736 kB\nPageTables: 9812 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 4196604 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,919499,825529}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',114769557}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,208459184}, {processes,163555216}, {processes_used,162131904}, {system,44903968}, {atom,1401785}, {atom_used,1374887}, {binary,6434376}, {code,14061510}, {ets,4481624}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,5353164800}, {buffered_memory,161992704}, {free_memory,19062390784}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{651877,4996}}, {context_switches,{5803659,0}}, {garbage_collection,{719237,5807824524,0}}, {io,{{input,107781229},{output,5957129110}}}, {reductions,{1141233455,8291331}}, {run_queue,0}, {runtime,{169580,2720}}]}, {system_stats, [{cpu_utilization_rate,20.895522388059703}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,5363378136}, {couch_docs_data_size,5361346057}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1116559}, {curr_items_tot,2183722}, {mem_used,5837782392}, {vb_replica_curr_items,1067163}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,651}, {memory_data,{33657663488,15336648704,{<17879.427.0>,9356040}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 18615616 kB\nBuffers: 158196 kB\nCached: 5227700 kB\nSwapCached: 0 kB\nActive: 8746540 kB\nInactive: 5241008 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 18615616 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 17972 kB\nWriteback: 4 kB\nAnonPages: 8601876 kB\nMapped: 16380 kB\nSlab: 208336 kB\nPageTables: 22268 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10323100 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,919501,479020}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default", [{'ns_1@10.2.1.58',117893590},{'ns_1@10.2.1.61',60644940}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,213863624}, {processes,164211192}, {processes_used,163790416}, {system,49652432}, {atom,1338193}, {atom_used,1334188}, {binary,12159272}, {code,13493302}, {ets,4009984}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,9672146944}, {buffered_memory,171929600}, {free_memory,14615805952}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{649942,5000}}, {context_switches,{6065919,0}}, {garbage_collection,{957016,2450266886,0}}, {io,{{input,122013491},{output,7855700439}}}, {reductions,{892747439,7744252}}, {run_queue,0}, {runtime,{135690,1260}}]}, {system_stats, [{cpu_utilization_rate,26.25}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,9587859630}, {couch_docs_data_size,7284716577}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1447306}, {curr_items_tot,3000003}, {mem_used,7995330888}, {vb_replica_curr_items,1552697}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,649}, {memory_data,{33657663488,18123296768,{<17926.368.0>,5085816}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 14273248 kB\nBuffers: 167900 kB\nCached: 9445456 kB\nSwapCached: 0 kB\nActive: 9844712 kB\nInactive: 8395952 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 14273248 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 13152 kB\nWriteback: 0 kB\nAnonPages: 8627664 kB\nMapped: 16404 kB\nSlab: 298628 kB\nPageTables: 22168 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10340772 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-14T10:59:22.074,ns_1@10.2.1.58:ns_doctor<0.1953.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,919557,75821}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',50274706}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,129585120}, {processes,90033760}, {processes_used,90023936}, {system,39551360}, {atom,1332537}, {atom_used,1324944}, {binary,3700488}, {code,13364289}, {ets,2834912}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,3232829440}, {buffered_memory,94752768}, {free_memory,26807132160}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{707496,4997}}, {context_switches,{2330005,0}}, {garbage_collection,{881587,1433131479,0}}, {io,{{input,59024607},{output,6530610339}}}, {reductions,{314796729,5893244}}, {run_queue,0}, {runtime,{85710,1320}}]}, {system_stats, [{cpu_utilization_rate,28.75}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,3085761041}, {couch_docs_data_size,2614251695}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,576729}, {curr_items_tot,1097809}, {mem_used,2901524472}, {vb_replica_curr_items,521080}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,707}, {memory_data,{33657663488,5408337920,{<0.367.0>,5085816}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 26178840 kB\nBuffers: 92532 kB\nCached: 3157060 kB\nSwapCached: 0 kB\nActive: 3407344 kB\nInactive: 3114440 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 26178840 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 10824 kB\nWriteback: 0 kB\nAnonPages: 3271844 kB\nMapped: 15592 kB\nSlab: 122616 kB\nPageTables: 11392 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 4977548 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,919559,825489}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',38866278}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,195242848}, {processes,150825408}, {processes_used,148902232}, {system,44417440}, {atom,1401785}, {atom_used,1374887}, {binary,5961848}, {code,14061510}, {ets,4535616}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,4651712512}, {buffered_memory,163487744}, {free_memory,19790233600}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{711877,5000}}, {context_switches,{6048705,0}}, {garbage_collection,{846759,7867066665,0}}, {io,{{input,112417422},{output,5986320279}}}, {reductions,{1224761090,7476213}}, {run_queue,0}, {runtime,{199600,2450}}]}, {system_stats, [{cpu_utilization_rate,29.601990049751244}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,4651414658}, {couch_docs_data_size,4649582877}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,972499}, {curr_items_tot,1893365}, {mem_used,5076992168}, {vb_replica_curr_items,920866}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,711}, {memory_data,{33657663488,14503849984,{<17879.427.0>,25150688}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 19326400 kB\nBuffers: 159656 kB\nCached: 4542688 kB\nSwapCached: 0 kB\nActive: 8734396 kB\nInactive: 4556520 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 19326400 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 10652 kB\nWriteback: 0 kB\nAnonPages: 8588980 kB\nMapped: 16492 kB\nSlab: 194560 kB\nPageTables: 22188 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10300496 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,919561,480568}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default", [{'ns_1@10.2.1.58',57806783},{'ns_1@10.2.1.61',83140945}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.24870.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,37}, {started_on,1352919499}, {total_vbuckets,1025}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1352919549}, {vbuckets_done,383}]]}, {memory, [{total,215809944}, {processes,165598544}, {processes_used,165511464}, {system,50211400}, {atom,1338193}, {atom_used,1334188}, {binary,12135096}, {code,13493302}, {ets,4538832}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,9117368320}, {buffered_memory,177831936}, {free_memory,15159365632}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{709943,5002}}, {context_switches,{6592458,0}}, {garbage_collection,{1187308,2868766697,0}}, {io,{{input,138911542},{output,8523523024}}}, {reductions,{990091892,7410390}}, {run_queue,0}, {runtime,{150780,1150}}]}, {system_stats, [{cpu_utilization_rate,48.029556650246306}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,9033454112}, {couch_docs_data_size,7285035162}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1444812}, {curr_items_tot,3000003}, {mem_used,8012640184}, {vb_replica_curr_items,1555191}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,709}, {memory_data,{33657663488,19198943232,{<17926.368.0>,5085816}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 14804192 kB\nBuffers: 173664 kB\nCached: 8903680 kB\nSwapCached: 0 kB\nActive: 9633872 kB\nInactive: 8088504 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 14804192 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 19268 kB\nWriteback: 20 kB\nAnonPages: 8644408 kB\nMapped: 16400 kB\nSlab: 285816 kB\nPageTables: 22328 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10360304 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-14T10:59:43.233,ns_1@10.2.1.58:<0.2072.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,48,39}}) Stats for bucket "default": accepting_conns 1 auth_cmds 0 auth_errors 0 bucket_active_conns 1 bucket_conns 12 bytes 3190381368 bytes_read 7102894042 bytes_written 1469405587 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 0 conn_yields 11398 connection_structures 5000 curr_connections 15 curr_conns_on_port_11209 12 curr_conns_on_port_11210 1 curr_items 632546 curr_items_tot 1206393 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-15 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data2/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 4171 ep_commit_time 94 ep_commit_time_total 112245 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 2 ep_data_age_highwat 5 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data2/default ep_degraded_mode 0 ep_diskqueue_drain 3936936 ep_diskqueue_fill 2832994 ep_diskqueue_items 221 ep_diskqueue_memory 6752 ep_diskqueue_pending 10128 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 140 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 2258880 ep_io_read_bytes 0 ep_io_write_bytes 5349381670 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 84888 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 2951286859 ep_max_checkpoints 2 ep_max_data_size 12582912000 ep_max_item_size 20971520 ep_max_size 12582912000 ep_max_txn_size 10000 ep_max_vbuckets 1024 ep_mem_high_wat 9437184000 ep_mem_high_wat 9437184000 ep_mem_low_wat 7549747200 ep_mem_low_wat 7549747200 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 104248950 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 221 ep_startup_time 1352919382 ep_storage_age 0 ep_storage_age_highwat 2 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 2936810143 ep_total_del_items 0 ep_total_enqueued 2833004 ep_total_new_items 1206393 ep_total_persisted 2258880 ep_uncommitted_items 410 ep_value_size 2851818247 ep_vb0 0 ep_vb_snapshot_total 855 ep_vb_total 413 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_135_g40a391b ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 1431 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 3190381368 pid 29361 pointer_size 64 rejected_conns 0 rusage_system 34.922690 rusage_user 151.439977 tap_checkpoint_end_received 591 tap_checkpoint_end_sent 199 tap_checkpoint_start_received 1230 tap_checkpoint_start_sent 216 tap_connect_received 1 tap_mutation_received 2938325 tap_mutation_sent 584124 tap_opaque_received 2884 tap_opaque_sent 217 tap_vbucket_set_received 432 threads 4 time 1352919582 total_connections 2095 uptime 202 vb_active_curr_items 632546 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 5424192 vb_active_itm_memory 1540092590 vb_active_meta_data_memory 49971134 vb_active_num 216 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 632546 vb_active_ops_create 632546 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 287315 vb_active_perc_mem_resident 100 vb_active_queue_age 0 vb_active_queue_drain 1833443 vb_active_queue_fill 1247608 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 573847 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 4947064 vb_replica_itm_memory 1396717553 vb_replica_meta_data_memory 45333913 vb_replica_num 197 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 573847 vb_replica_ops_create 573847 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 765582 vb_replica_perc_mem_resident 100 vb_replica_queue_age 0 vb_replica_queue_drain 2103493 vb_replica_queue_fill 1585386 vb_replica_queue_memory 6752 vb_replica_queue_pending 10128 vb_replica_queue_size 211 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-14T11:00:22.074,ns_1@10.2.1.58:ns_doctor<0.1953.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,919617,75267}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',13544851}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.16422.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,74}, {started_on,1352919575}, {total_vbuckets,402}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1352919616}, {vbuckets_done,300}]]}, {memory, [{total,139888336}, {processes,98859816}, {processes_used,98845616}, {system,41028520}, {atom,1332537}, {atom_used,1324944}, {binary,4870592}, {code,13364289}, {ets,3072072}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,3742031872}, {buffered_memory,114966528}, {free_memory,25569710080}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{767495,4998}}, {context_switches,{2897285,0}}, {garbage_collection,{1056865,1747330664,0}}, {io,{{input,74478940},{output,7945075756}}}, {reductions,{386053881,5317861}}, {run_queue,0}, {runtime,{100130,1050}}]}, {system_stats, [{cpu_utilization_rate,46.8671679197995}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,3636614717}, {couch_docs_data_size,3206096474}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,705657}, {curr_items_tot,1338035}, {mem_used,3539780864}, {vb_replica_curr_items,632378}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,767}, {memory_data,{33657663488,7342944256,{<0.367.0>,5085816}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 24970420 kB\nBuffers: 112272 kB\nCached: 3654328 kB\nSwapCached: 0 kB\nActive: 4096480 kB\nInactive: 3616428 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 24970420 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 17388 kB\nWriteback: 4 kB\nAnonPages: 3945592 kB\nMapped: 15596 kB\nSlab: 137380 kB\nPageTables: 12816 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 5655580 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,919619,834948}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',87368141}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,216994216}, {processes,172870952}, {processes_used,170540552}, {system,44123264}, {atom,1401785}, {atom_used,1374887}, {binary,5647152}, {code,14061510}, {ets,4612944}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,4065452032}, {buffered_memory,164896768}, {free_memory,20366450688}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{771880,5002}}, {context_switches,{6261021,0}}, {garbage_collection,{944719,9625378361,0}}, {io,{{input,119797715},{output,6012701456}}}, {reductions,{1296815695,7268178}}, {run_queue,0}, {runtime,{226220,2360}}]}, {system_stats, [{cpu_utilization_rate,14.463840399002494}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,4066293445}, {couch_docs_data_size,4064725113}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,867226}, {curr_items_tot,1656023}, {mem_used,4459218392}, {vb_replica_curr_items,788797}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,771}, {memory_data,{33657663488,13801148416,{<17879.427.0>,15599024}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 19889236 kB\nBuffers: 161032 kB\nCached: 3970168 kB\nSwapCached: 0 kB\nActive: 8752968 kB\nInactive: 3986848 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 19889236 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 9456 kB\nWriteback: 0 kB\nAnonPages: 8608864 kB\nMapped: 16796 kB\nSlab: 182880 kB\nPageTables: 22064 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10321184 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,919621,480295}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default", [{'ns_1@10.2.1.58',73125330},{'ns_1@10.2.1.61',110592434}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.24870.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,77}, {started_on,1352919499}, {total_vbuckets,1025}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1352919609}, {vbuckets_done,796}]]}, {memory, [{total,217501656}, {processes,169436464}, {processes_used,169421192}, {system,48065192}, {atom,1338193}, {atom_used,1334188}, {binary,9366000}, {code,13493302}, {ets,5114312}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7844134912}, {buffered_memory,183779328}, {free_memory,16435507200}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{769942,5000}}, {context_switches,{7085454,0}}, {garbage_collection,{1383780,3233325627,0}}, {io,{{input,159788065},{output,9119956971}}}, {reductions,{1077564264,8027214}}, {run_queue,0}, {runtime,{164660,1220}}]}, {system_stats, [{cpu_utilization_rate,47.75}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7760847332}, {couch_docs_data_size,7285744256}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1418140}, {curr_items_tot,3000003}, {mem_used,8024256312}, {vb_replica_curr_items,1581863}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,769}, {memory_data,{33657663488,18613084160,{<17926.368.0>,5085816}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16050300 kB\nBuffers: 179472 kB\nCached: 7660288 kB\nSwapCached: 0 kB\nActive: 9015236 kB\nInactive: 7489524 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16050300 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 21336 kB\nWriteback: 0 kB\nAnonPages: 8665152 kB\nMapped: 16400 kB\nSlab: 257284 kB\nPageTables: 22500 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10383712 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-14T11:01:22.074,ns_1@10.2.1.58:ns_doctor<0.1953.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,919677,75328}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',80446473}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,153189960}, {processes,110478184}, {processes_used,110475040}, {system,42711776}, {atom,1332537}, {atom_used,1324944}, {binary,6226448}, {code,13364289}, {ets,3338352}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,4688556032}, {buffered_memory,135544832}, {free_memory,23930880000}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{827495,5000}}, {context_switches,{3447702,0}}, {garbage_collection,{1215167,2042131067,0}}, {io,{{input,84590789},{output,9279552481}}}, {reductions,{453660713,5214343}}, {run_queue,0}, {runtime,{114790,1270}}]}, {system_stats, [{cpu_utilization_rate,24.25}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,4574606014}, {couch_docs_data_size,3747798941}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,819944}, {curr_items_tot,1563561}, {mem_used,4135185536}, {vb_replica_curr_items,743617}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,827}, {memory_data,{33657663488,8310542336,{<0.367.0>,5085816}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 23370124 kB\nBuffers: 132368 kB\nCached: 4578668 kB\nSwapCached: 0 kB\nActive: 4862840 kB\nInactive: 4424048 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 23370124 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 11864 kB\nWriteback: 0 kB\nAnonPages: 4575764 kB\nMapped: 15592 kB\nSlab: 163140 kB\nPageTables: 13948 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6281760 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,919679,824830}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',102331594}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,177079032}, {processes,133748336}, {processes_used,131042352}, {system,43330696}, {atom,1401785}, {atom_used,1374887}, {binary,4826256}, {code,14061510}, {ets,4695416}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,3533873152}, {buffered_memory,166133760}, {free_memory,20911796224}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{831876,4998}}, {context_switches,{6451041,0}}, {garbage_collection,{1036228,11242034615,0}}, {io,{{input,123232759},{output,6034621449}}}, {reductions,{1362145146,5860752}}, {run_queue,0}, {runtime,{250780,1750}}]}, {system_stats, [{cpu_utilization_rate,16.161616161616163}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,3517315945}, {couch_docs_data_size,3515914890}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,755869}, {curr_items_tot,1433450}, {mem_used,3888830648}, {vb_replica_curr_items,677581}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,831}, {memory_data,{33657663488,13209440256,{<17879.427.0>,15648032}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 20421676 kB\nBuffers: 162240 kB\nCached: 3451048 kB\nSwapCached: 0 kB\nActive: 8751324 kB\nInactive: 3466408 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 20421676 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 3928 kB\nWriteback: 0 kB\nAnonPages: 8604104 kB\nMapped: 16492 kB\nSlab: 172996 kB\nPageTables: 22232 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10330076 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,919681,477854}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default", [{'ns_1@10.2.1.58',82575332},{'ns_1@10.2.1.61',97705696}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,222189552}, {processes,172958264}, {processes_used,172876256}, {system,49231288}, {atom,1338193}, {atom_used,1334188}, {binary,10459600}, {code,13493302}, {ets,5186480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,8395063296}, {buffered_memory,187867136}, {free_memory,15848787968}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{829941,4994}}, {context_switches,{7547701,0}}, {garbage_collection,{1552535,3556018175,0}}, {io,{{input,172157023},{output,9607559402}}}, {reductions,{1157156860,6753620}}, {run_queue,0}, {runtime,{177450,1050}}]}, {system_stats, [{cpu_utilization_rate,27.5}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,8302900658}, {couch_docs_data_size,7285381515}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1418281}, {curr_items_tot,2997004}, {mem_used,8034980504}, {vb_replica_curr_items,1578723}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,829}, {memory_data,{33657663488,17321205760,{<17926.368.0>,5085816}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 15477332 kB\nBuffers: 183464 kB\nCached: 8198304 kB\nSwapCached: 0 kB\nActive: 9280004 kB\nInactive: 7787564 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 15477332 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 23048 kB\nWriteback: 0 kB\nAnonPages: 8685520 kB\nMapped: 16404 kB\nSlab: 268012 kB\nPageTables: 22320 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10407612 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-14T11:01:22.858,ns_1@10.2.1.58:<0.2072.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,48,39}}) Stats for bucket "default": accepting_conns 1 auth_cmds 0 auth_errors 0 bucket_active_conns 2 bucket_conns 15 bytes 4181241848 bytes_read 9287220313 bytes_written 1931588624 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 0 conn_yields 15881 connection_structures 5000 curr_connections 18 curr_conns_on_port_11209 15 curr_conns_on_port_11210 1 curr_items 828780 curr_items_tot 1581398 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-15 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data2/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 1 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 6896 ep_commit_time 5 ep_commit_time_total 160943 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 2 ep_data_age_highwat 5 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data2/default ep_degraded_mode 0 ep_diskqueue_drain 5150571 ep_diskqueue_fill 3724594 ep_diskqueue_items 171 ep_diskqueue_memory 5632 ep_diskqueue_pending 8448 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 201 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 56 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 2925803 ep_io_read_bytes 0 ep_io_write_bytes 6929674680 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 126143 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 3869538490 ep_max_checkpoints 2 ep_max_data_size 12582912000 ep_max_item_size 20971520 ep_max_size 12582912000 ep_max_txn_size 10000 ep_max_vbuckets 1024 ep_mem_high_wat 9437184000 ep_mem_high_wat 9437184000 ep_mem_low_wat 7549747200 ep_mem_low_wat 7549747200 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 135995294 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 115 ep_startup_time 1352919382 ep_storage_age 0 ep_storage_age_highwat 2 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 3850561714 ep_total_del_items 0 ep_total_enqueued 3724594 ep_total_new_items 1581230 ep_total_persisted 2925803 ep_uncommitted_items 31 ep_value_size 3739151082 ep_vb0 0 ep_vb_snapshot_total 1205 ep_vb_total 541 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_135_g40a391b ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 1431 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 4181241848 pid 29361 pointer_size 64 rejected_conns 0 rusage_system 46.101991 rusage_user 229.489112 tap_checkpoint_end_received 770 tap_checkpoint_end_sent 259 tap_checkpoint_start_received 1609 tap_checkpoint_start_sent 283 tap_connect_received 1 tap_mutation_received 3841084 tap_mutation_sent 758608 tap_opaque_received 3780 tap_opaque_sent 284 tap_vbucket_set_received 566 threads 4 time 1352919682 total_connections 2740 uptime 302 vb_active_curr_items 828780 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 7106696 vb_active_itm_memory 2018287060 vb_active_meta_data_memory 65473620 vb_active_num 283 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 828780 vb_active_ops_create 828780 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 363428 vb_active_perc_mem_resident 100 vb_active_queue_age 0 vb_active_queue_drain 2392969 vb_active_queue_fill 1634303 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 752618 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 6478896 vb_replica_itm_memory 1832274654 vb_replica_meta_data_memory 59456822 vb_replica_num 258 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 752618 vb_replica_ops_create 752450 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 981176 vb_replica_perc_mem_resident 100 vb_replica_queue_age 0 vb_replica_queue_drain 2757602 vb_replica_queue_fill 2090291 vb_replica_queue_memory 5632 vb_replica_queue_pending 8448 vb_replica_queue_size 176 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-14T11:02:22.074,ns_1@10.2.1.58:ns_doctor<0.1953.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,919737,76497}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',119263968}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,164504128}, {processes,120312792}, {processes_used,120299408}, {system,44191336}, {atom,1332537}, {atom_used,1324944}, {binary,7533080}, {code,13364289}, {ets,3465320}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,5908582400}, {buffered_memory,151957504}, {free_memory,22053556224}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{887496,5001}}, {context_switches,{4010859,0}}, {garbage_collection,{1358986,2324919913,0}}, {io,{{input,94144754},{output,10538903111}}}, {reductions,{520157524,6713064}}, {run_queue,0}, {runtime,{129530,1510}}]}, {system_stats, [{cpu_utilization_rate,43.5}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,5724421205}, {couch_docs_data_size,4252429843}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,934206}, {curr_items_tot,1777555}, {mem_used,4698049048}, {vb_replica_curr_items,843349}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,887}, {memory_data,{33657663488,10106941440,{<0.367.0>,5085816}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 21536800 kB\nBuffers: 148396 kB\nCached: 5770100 kB\nSwapCached: 0 kB\nActive: 5681916 kB\nInactive: 5403920 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 21536800 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 10016 kB\nWriteback: 72 kB\nAnonPages: 5167424 kB\nMapped: 15592 kB\nSlab: 196460 kB\nPageTables: 15128 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 6879628 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,919739,861339}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',1691425}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,201567856}, {processes,158868616}, {processes_used,155805488}, {system,42699240}, {atom,1401785}, {atom_used,1374887}, {binary,4251680}, {code,14061510}, {ets,4681160}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,3031072768}, {buffered_memory,167452672}, {free_memory,21436198912}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{891912,5033}}, {context_switches,{6623901,0}}, {garbage_collection,{1109769,12797039796,0}}, {io,{{input,126613703},{output,6055893193}}}, {reductions,{1423578393,5204626}}, {run_queue,1}, {runtime,{274790,1840}}]}, {system_stats, [{cpu_utilization_rate,4.271356783919598}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,3011871275}, {couch_docs_data_size,3010677501}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,659196}, {curr_items_tot,1222448}, {mem_used,3339831544}, {vb_replica_curr_items,563252}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,891}, {memory_data,{33657663488,12657483776,{<17879.427.0>,15610592}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 20933788 kB\nBuffers: 163528 kB\nCached: 2960032 kB\nSwapCached: 0 kB\nActive: 8738408 kB\nInactive: 2975960 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 20933788 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 1704 kB\nWriteback: 0 kB\nAnonPages: 8590620 kB\nMapped: 16772 kB\nSlab: 163132 kB\nPageTables: 22848 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10303980 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,919741,477907}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default", [{'ns_1@10.2.1.58',74087465},{'ns_1@10.2.1.61',82243698}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,226978656}, {processes,176541496}, {processes_used,176392088}, {system,50437160}, {atom,1338193}, {atom_used,1334188}, {binary,11675416}, {code,13493302}, {ets,5184456}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,8954552320}, {buffered_memory,190189568}, {free_memory,15258079232}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{889941,4999}}, {context_switches,{8000936,0}}, {garbage_collection,{1715087,3863517245,0}}, {io,{{input,182817338},{output,10076129758}}}, {reductions,{1233415950,5545560}}, {run_queue,0}, {runtime,{190480,1020}}]}, {system_stats, [{cpu_utilization_rate,36.38676844783715}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,8858490306}, {couch_docs_data_size,7285066029}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1403468}, {curr_items_tot,3000003}, {mem_used,8046724824}, {vb_replica_curr_items,1596535}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,889}, {memory_data,{33657663488,17915916288,{<17926.368.0>,5085816}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 14900468 kB\nBuffers: 185732 kB\nCached: 8744680 kB\nSwapCached: 0 kB\nActive: 9572280 kB\nInactive: 8057584 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 14900468 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 17832 kB\nWriteback: 0 kB\nAnonPages: 8699444 kB\nMapped: 16288 kB\nSlab: 281308 kB\nPageTables: 22340 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10419280 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-14T11:03:03.128,ns_1@10.2.1.58:<0.2072.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,48,39}}) Stats for bucket "default": accepting_conns 1 auth_cmds 0 auth_errors 0 bucket_active_conns 1 bucket_conns 16 bytes 5093627240 bytes_read 11303466747 bytes_written 2331838678 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 0 conn_yields 19504 connection_structures 5000 curr_connections 19 curr_conns_on_port_11209 16 curr_conns_on_port_11210 1 curr_items 1007860 curr_items_tot 1927559 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-15 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data2/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 1 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 12977 ep_commit_time 171 ep_commit_time_total 200587 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 2 ep_data_age_highwat 5 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data2/default ep_degraded_mode 0 ep_diskqueue_drain 6260091 ep_diskqueue_fill 4555122 ep_diskqueue_items 2974 ep_diskqueue_memory 95072 ep_diskqueue_pending 142608 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 255 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 2964 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 3625981 ep_io_read_bytes 0 ep_io_write_bytes 8587671973 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 184732 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 4716324397 ep_max_checkpoints 2 ep_max_data_size 12582912000 ep_max_item_size 20971520 ep_max_size 12582912000 ep_max_txn_size 10000 ep_max_vbuckets 1024 ep_mem_high_wat 9437184000 ep_mem_high_wat 9437184000 ep_mem_low_wat 7549747200 ep_mem_low_wat 7549747200 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 164572384 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 10 ep_startup_time 1352919382 ep_storage_age 0 ep_storage_age_highwat 2 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 4693193689 ep_total_del_items 0 ep_total_enqueued 4555122 ep_total_new_items 1926736 ep_total_persisted 3625981 ep_uncommitted_items 1 ep_value_size 4557397762 ep_vb0 0 ep_vb_snapshot_total 1547 ep_vb_total 659 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_135_g40a391b ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 1431 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 5093627240 pid 29361 pointer_size 64 rejected_conns 0 rusage_system 58.147160 rusage_user 317.636711 tap_checkpoint_end_received 938 tap_checkpoint_end_sent 308 tap_checkpoint_start_received 1959 tap_checkpoint_start_sent 344 tap_connect_received 1 tap_mutation_received 4675083 tap_mutation_sent 905415 tap_opaque_received 4606 tap_opaque_sent 345 tap_vbucket_set_received 688 threads 4 time 1352919782 total_connections 3335 uptime 402 vb_active_curr_items 1007860 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 8638336 vb_active_itm_memory 2454151964 vb_active_meta_data_memory 79620940 vb_active_num 344 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1007860 vb_active_ops_create 1007860 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 479862 vb_active_perc_mem_resident 100 vb_active_queue_age 0 vb_active_queue_drain 2892020 vb_active_queue_fill 1986541 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 919699 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 7910280 vb_replica_itm_memory 2239041725 vb_replica_meta_data_memory 72656221 vb_replica_num 315 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 919699 vb_replica_ops_create 918877 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 1219383 vb_replica_perc_mem_resident 100 vb_replica_queue_age 0 vb_replica_queue_drain 3368071 vb_replica_queue_fill 2568581 vb_replica_queue_memory 95072 vb_replica_queue_pending 142608 vb_replica_queue_size 2971 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-14T11:03:22.074,ns_1@10.2.1.58:ns_doctor<0.1953.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,919797,77200}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',55540468}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.26002.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,19}, {started_on,1352919780}, {total_vbuckets,654}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1352919797}, {vbuckets_done,130}]]}, {memory, [{total,167728000}, {processes,123774224}, {processes_used,123769192}, {system,43953776}, {atom,1332537}, {atom_used,1324944}, {binary,7037944}, {code,13364289}, {ets,3687768}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,6915538944}, {buffered_memory,163618816}, {free_memory,20476379136}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{947497,5002}}, {context_switches,{4544058,0}}, {garbage_collection,{1486857,2588460698,0}}, {io,{{input,107417749},{output,11657780680}}}, {reductions,{582885362,5058256}}, {run_queue,0}, {runtime,{142730,910}}]}, {system_stats, [{cpu_utilization_rate,41.60401002506266}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,6798407563}, {couch_docs_data_size,4729638200}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1031329}, {curr_items_tot,1964027}, {mem_used,5189960888}, {vb_replica_curr_items,932698}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,947}, {memory_data,{33657663488,11905548288,{<0.367.0>,5085816}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 19996464 kB\nBuffers: 159784 kB\nCached: 6753456 kB\nSwapCached: 0 kB\nActive: 6388124 kB\nInactive: 6209268 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 19996464 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 13248 kB\nWriteback: 0 kB\nAnonPages: 5683648 kB\nMapped: 15596 kB\nSlab: 223800 kB\nPageTables: 16264 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 7396684 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,919799,825854}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',37227077}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,155503568}, {processes,113149608}, {processes_used,109762680}, {system,42353960}, {atom,1401785}, {atom_used,1374887}, {binary,3839600}, {code,14061510}, {ets,4798560}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,2563231744}, {buffered_memory,168734720}, {free_memory,21910867968}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{951877,5000}}, {context_switches,{6780552,0}}, {garbage_collection,{1177318,14133684501,0}}, {io,{{input,133141760},{output,6078779428}}}, {reductions,{1480543589,4888636}}, {run_queue,0}, {runtime,{296490,1800}}]}, {system_stats, [{cpu_utilization_rate,11.5}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,2534614234}, {couch_docs_data_size,2533647679}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,562533}, {curr_items_tot,1031670}, {mem_used,2838654968}, {vb_replica_curr_items,469137}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,951}, {memory_data,{33657663488,12171030528,{<17879.427.0>,15604608}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 21397332 kB\nBuffers: 164780 kB\nCached: 2503156 kB\nSwapCached: 0 kB\nActive: 8740480 kB\nInactive: 2520080 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 21397332 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 1584 kB\nWriteback: 0 kB\nAnonPages: 8592632 kB\nMapped: 16492 kB\nSlab: 154736 kB\nPageTables: 22124 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10305004 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,919801,479281}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default", [{'ns_1@10.2.1.58',50226275},{'ns_1@10.2.1.61',126172544}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.8415.1>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,24}, {started_on,1352919760}, {total_vbuckets,1025}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1352919789}, {vbuckets_done,247}]]}, {memory, [{total,219925072}, {processes,169244520}, {processes_used,169123664}, {system,50680552}, {atom,1338193}, {atom_used,1334188}, {binary,11710504}, {code,13493302}, {ets,5385256}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,8842379264}, {buffered_memory,194093056}, {free_memory,15352176640}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{949942,5001}}, {context_switches,{8468133,0}}, {garbage_collection,{1868426,4159495105,0}}, {io,{{input,198842316},{output,10542188290}}}, {reductions,{1308901237,5574648}}, {run_queue,0}, {runtime,{202670,830}}]}, {system_stats, [{cpu_utilization_rate,43.0}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,8851161145}, {couch_docs_data_size,7285142515}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1406119}, {curr_items_tot,3000003}, {mem_used,8054830152}, {vb_replica_curr_items,1593884}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,949}, {memory_data,{33657663488,18504839168,{<17926.368.0>,5085816}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 14992608 kB\nBuffers: 189544 kB\nCached: 8635136 kB\nSwapCached: 0 kB\nActive: 9621556 kB\nInactive: 7912720 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 14992608 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 22164 kB\nWriteback: 0 kB\nAnonPages: 8709624 kB\nMapped: 16396 kB\nSlab: 285796 kB\nPageTables: 22220 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10425976 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-14T11:04:22.074,ns_1@10.2.1.58:ns_doctor<0.1953.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,919857,75275}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',1487474}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.26002.0>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,87}, {started_on,1352919780}, {total_vbuckets,654}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1352919856}, {vbuckets_done,574}]]}, {memory, [{total,176110696}, {processes,132233952}, {processes_used,132225368}, {system,43876744}, {atom,1332537}, {atom_used,1325003}, {binary,6596168}, {code,13364289}, {ets,3992864}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,6102429696}, {buffered_memory,172388352}, {free_memory,20805492736}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1007495,4999}}, {context_switches,{5066057,0}}, {garbage_collection,{1612095,2845464215,0}}, {io,{{input,119520084},{output,12721935536}}}, {reductions,{645839183,4897295}}, {run_queue,0}, {runtime,{155310,890}}]}, {system_stats, [{cpu_utilization_rate,44.75}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,6089574011}, {couch_docs_data_size,5169941453}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1119960}, {curr_items_tot,2140993}, {mem_used,5655172168}, {vb_replica_curr_items,1021033}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1007}, {memory_data,{33657663488,13134573568,{<0.367.0>,5085816}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 20317864 kB\nBuffers: 168348 kB\nCached: 5959404 kB\nSwapCached: 0 kB\nActive: 6493664 kB\nInactive: 5801580 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 20317864 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 11532 kB\nWriteback: 68 kB\nAnonPages: 6167684 kB\nMapped: 15596 kB\nSlab: 204476 kB\nPageTables: 17184 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 7881796 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,919859,826497}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',91203095}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,150044120}, {processes,107990672}, {processes_used,104297688}, {system,42053448}, {atom,1401785}, {atom_used,1374887}, {binary,3608464}, {code,14061510}, {ets,4768840}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,2133282816}, {buffered_memory,169844736}, {free_memory,22359490560}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1011877,5000}}, {context_switches,{6921068,0}}, {garbage_collection,{1239919,15400143063,0}}, {io,{{input,135631297},{output,6096641452}}}, {reductions,{1534169583,5660376}}, {run_queue,0}, {runtime,{317750,1880}}]}, {system_stats, [{cpu_utilization_rate,10.75}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,2108605618}, {couch_docs_data_size,2107825269}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,478408}, {curr_items_tot,856009}, {mem_used,2383339384}, {vb_replica_curr_items,377601}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1011}, {memory_data,{33657663488,11684450304,{<17879.427.0>,15604480}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 21835440 kB\nBuffers: 165864 kB\nCached: 2083284 kB\nSwapCached: 0 kB\nActive: 8730472 kB\nInactive: 2100256 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 21835440 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 1280 kB\nWriteback: 0 kB\nAnonPages: 8581708 kB\nMapped: 16492 kB\nSlab: 146028 kB\nPageTables: 22312 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10304680 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,919861,477679}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default", [{'ns_1@10.2.1.58',61695915},{'ns_1@10.2.1.61',103604338}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.8415.1>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,69}, {started_on,1352919760}, {total_vbuckets,1025}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1352919849}, {vbuckets_done,713}]]}, {memory, [{total,219036360}, {processes,168350712}, {processes_used,168245256}, {system,50685648}, {atom,1338193}, {atom_used,1334188}, {binary,11630192}, {code,13493302}, {ets,5468712}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,8677728256}, {buffered_memory,199327744}, {free_memory,15506055168}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1009941,4999}}, {context_switches,{8904390,0}}, {garbage_collection,{2014080,4442363714,0}}, {io,{{input,211004855},{output,10934343966}}}, {reductions,{1382587419,5888735}}, {run_queue,0}, {runtime,{213970,940}}]}, {system_stats, [{cpu_utilization_rate,31.5136476426799}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,8578117660}, {couch_docs_data_size,7285373160}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1395904}, {curr_items_tot,3000002}, {mem_used,8064008600}, {vb_replica_curr_items,1604098}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1009}, {memory_data,{33657663488,17768759296,{<17926.368.0>,5085816}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 15142632 kB\nBuffers: 194656 kB\nCached: 8474344 kB\nSwapCached: 0 kB\nActive: 9622600 kB\nInactive: 7771264 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 15142632 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 22772 kB\nWriteback: 1092 kB\nAnonPages: 8724248 kB\nMapped: 16424 kB\nSlab: 275848 kB\nPageTables: 22332 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10440112 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-14T11:04:43.742,ns_1@10.2.1.58:<0.2072.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,48,39}}) Stats for bucket "default": accepting_conns 1 auth_cmds 0 auth_errors 0 bucket_active_conns 1 bucket_conns 13 bytes 5863981208 bytes_read 13020130724 bytes_written 2687281807 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 0 conn_yields 23531 connection_structures 5000 curr_connections 16 curr_conns_on_port_11209 13 curr_conns_on_port_11210 1 curr_items 1163870 curr_items_tot 2220408 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-15 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data2/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 1 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 14844 ep_commit_time 2 ep_commit_time_total 251872 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 1 ep_data_age_highwat 5 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data2/default ep_degraded_mode 0 ep_diskqueue_drain 7208007 ep_diskqueue_fill 5254750 ep_diskqueue_items 707 ep_diskqueue_memory 22624 ep_diskqueue_pending 33936 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 314 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 4106191 ep_io_read_bytes 0 ep_io_write_bytes 9723266512 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 228783 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 5431964520 ep_max_checkpoints 2 ep_max_data_size 12582912000 ep_max_item_size 20971520 ep_max_size 12582912000 ep_max_txn_size 10000 ep_max_vbuckets 1024 ep_mem_high_wat 9437184000 ep_mem_high_wat 9437184000 ep_mem_low_wat 7549747200 ep_mem_low_wat 7549747200 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 188379872 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 707 ep_startup_time 1352919382 ep_storage_age 0 ep_storage_age_highwat 2 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 5405319624 ep_total_del_items 3 ep_total_enqueued 5254750 ep_total_new_items 2220408 ep_total_persisted 4106191 ep_uncommitted_items 0 ep_value_size 5248892905 ep_vb0 0 ep_vb_snapshot_total 1825 ep_vb_total 758 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_135_g40a391b ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 1431 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 5863981208 pid 29361 pointer_size 64 rejected_conns 0 rusage_system 66.642868 rusage_user 392.786287 tap_checkpoint_end_received 1079 tap_checkpoint_end_sent 352 tap_checkpoint_start_received 2257 tap_checkpoint_start_sent 397 tap_connect_received 1 tap_delete_received 2 tap_mutation_received 5385967 tap_mutation_sent 1033152 tap_opaque_received 5303 tap_opaque_sent 398 tap_vbucket_set_received 794 threads 4 time 1352919882 total_connections 3838 uptime 502 vb_active_curr_items 1163870 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 9969176 vb_active_itm_memory 2833779402 vb_active_meta_data_memory 91945730 vb_active_num 397 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1163870 vb_active_ops_create 1163870 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 511542 vb_active_perc_mem_resident 100 vb_active_queue_age 0 vb_active_queue_drain 3330615 vb_active_queue_fill 2295603 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 1056538 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 9065432 vb_replica_itm_memory 2571540222 vb_replica_meta_data_memory 83466502 vb_replica_num 361 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 1056538 vb_replica_ops_create 1056538 vb_replica_ops_delete 3 vb_replica_ops_reject 0 vb_replica_ops_update 1374238 vb_replica_perc_mem_resident 100 vb_replica_queue_age 0 vb_replica_queue_drain 3877392 vb_replica_queue_fill 2959147 vb_replica_queue_memory 22624 vb_replica_queue_pending 33936 vb_replica_queue_size 707 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-14T11:05:22.074,ns_1@10.2.1.58:ns_doctor<0.1953.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,919917,75861}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',84235418}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,191698776}, {processes,146235048}, {processes_used,146225232}, {system,45463728}, {atom,1332537}, {atom_used,1325003}, {binary,7867680}, {code,13364289}, {ets,4257992}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,6583398400}, {buffered_memory,179482624}, {free_memory,19836182528}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1067496,5000}}, {context_switches,{5578069,0}}, {garbage_collection,{1721367,3082868114,0}}, {io,{{input,128452475},{output,13726515984}}}, {reductions,{704123758,5106888}}, {run_queue,0}, {runtime,{167720,1060}}]}, {system_stats, [{cpu_utilization_rate,29.25}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,6448632936}, {couch_docs_data_size,5562014993}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1201837}, {curr_items_tot,2305247}, {mem_used,6089125288}, {vb_replica_curr_items,1103410}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1067}, {memory_data,{33657663488,12650778624,{<0.367.0>,5085816}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 19371396 kB\nBuffers: 175276 kB\nCached: 6429100 kB\nSwapCached: 0 kB\nActive: 7041172 kB\nInactive: 6187756 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 19371396 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 6936 kB\nWriteback: 0 kB\nAnonPages: 6624324 kB\nMapped: 15592 kB\nSlab: 216100 kB\nPageTables: 17976 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 8346716 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,919919,864915}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',36911220}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,154114248}, {processes,112462472}, {processes_used,108494432}, {system,41651776}, {atom,1401785}, {atom_used,1374887}, {binary,3065016}, {code,14061510}, {ets,4878048}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,1743282176}, {buffered_memory,170868736}, {free_memory,22759313408}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1071912,5034}}, {context_switches,{7044864,0}}, {garbage_collection,{1293140,16553933039,0}}, {io,{{input,138241584},{output,6113300811}}}, {reductions,{1582997635,4686711}}, {run_queue,0}, {runtime,{337290,1770}}]}, {system_stats, [{cpu_utilization_rate,8.521303258145364}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,1701934582}, {couch_docs_data_size,1701296458}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,393242}, {curr_items_tot,691871}, {mem_used,1957089224}, {vb_replica_curr_items,298629}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1071}, {memory_data,{33657663488,11256016896,{<17879.427.0>,15603072}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 22230356 kB\nBuffers: 166864 kB\nCached: 1702424 kB\nSwapCached: 0 kB\nActive: 8723840 kB\nInactive: 1719944 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 22230356 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 1484 kB\nWriteback: 0 kB\nAnonPages: 8574036 kB\nMapped: 16492 kB\nSlab: 137872 kB\nPageTables: 22284 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10302080 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,919921,479890}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default", [{'ns_1@10.2.1.58',69475481},{'ns_1@10.2.1.61',99595827}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,223555344}, {processes,174226920}, {processes_used,174174096}, {system,49328424}, {atom,1338193}, {atom_used,1334188}, {binary,9982256}, {code,13493302}, {ets,5747048}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7909146624}, {buffered_memory,203550720}, {free_memory,16266911744}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1069943,5000}}, {context_switches,{9346961,0}}, {garbage_collection,{2140721,4700267157,0}}, {io,{{input,226420858},{output,11323244070}}}, {reductions,{1452024380,5664208}}, {run_queue,0}, {runtime,{224630,1050}}]}, {system_stats, [{cpu_utilization_rate,33.75}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7804857153}, {couch_docs_data_size,7285797267}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1404967}, {curr_items_tot,3000002}, {mem_used,8071214904}, {vb_replica_curr_items,1595035}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1069}, {memory_data,{33657663488,17936011264,{<17926.368.0>,5085816}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 15885780 kB\nBuffers: 198780 kB\nCached: 7723776 kB\nSwapCached: 0 kB\nActive: 9170788 kB\nInactive: 7496944 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 15885780 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 27732 kB\nWriteback: 0 kB\nAnonPages: 8744456 kB\nMapped: 16404 kB\nSlab: 259740 kB\nPageTables: 22428 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10461332 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-14T11:06:22.074,ns_1@10.2.1.58:ns_doctor<0.1953.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,919977,76759}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',93255316}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,196848616}, {processes,150237736}, {processes_used,150228872}, {system,46610880}, {atom,1332537}, {atom_used,1325003}, {binary,8799200}, {code,13364289}, {ets,4440264}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7462068224}, {buffered_memory,183984128}, {free_memory,18496585728}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1127496,5001}}, {context_switches,{6079167,0}}, {garbage_collection,{1821201,3311985084,0}}, {io,{{input,140349681},{output,14661752193}}}, {reductions,{760311545,5588268}}, {run_queue,0}, {runtime,{180410,1160}}]}, {system_stats, [{cpu_utilization_rate,37.65586034912718}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7337048059}, {couch_docs_data_size,5944001882}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1271665}, {curr_items_tot,2465638}, {mem_used,6513371616}, {vb_replica_curr_items,1193973}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1127}, {memory_data,{33657663488,14136889344,{<0.367.0>,5085816}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 18063072 kB\nBuffers: 179672 kB\nCached: 7287176 kB\nSwapCached: 0 kB\nActive: 7631064 kB\nInactive: 6881164 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 18063072 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 4640 kB\nWriteback: 0 kB\nAnonPages: 7045048 kB\nMapped: 15592 kB\nSlab: 239712 kB\nPageTables: 18796 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 8753596 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,919979,826040}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',14313789}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,140202856}, {processes,99072688}, {processes_used,94839688}, {system,41130168}, {atom,1401785}, {atom_used,1374887}, {binary,2530920}, {code,14061510}, {ets,4998200}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,1364811776}, {buffered_memory,172023808}, {free_memory,23141519360}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1131877,4978}}, {context_switches,{7156527,0}}, {garbage_collection,{1343801,17620057272,0}}, {io,{{input,143656467},{output,6131392772}}}, {reductions,{1629471618,4475593}}, {run_queue,0}, {runtime,{355480,1390}}]}, {system_stats, [{cpu_utilization_rate,18.434343434343436}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,1326642115}, {couch_docs_data_size,1326130855}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,308536}, {curr_items_tot,537337}, {mem_used,1557665288}, {vb_replica_curr_items,228801}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1131}, {memory_data,{33657663488,10837528576,{<17879.427.0>,15584616}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 22599264 kB\nBuffers: 167992 kB\nCached: 1332824 kB\nSwapCached: 0 kB\nActive: 8731936 kB\nInactive: 1351504 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 22599264 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 964 kB\nWriteback: 12 kB\nAnonPages: 8582736 kB\nMapped: 16492 kB\nSlab: 129852 kB\nPageTables: 22140 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10298052 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,919981,477646}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default", [{'ns_1@10.2.1.58',52505608},{'ns_1@10.2.1.61',106525520}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,224026072}, {processes,173554280}, {processes_used,173454432}, {system,50471792}, {atom,1338193}, {atom_used,1334188}, {binary,10996792}, {code,13493302}, {ets,5879920}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,8361656320}, {buffered_memory,204976128}, {free_memory,15791886336}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1129941,5000}}, {context_switches,{9794723,0}}, {garbage_collection,{2249426,4936057839,0}}, {io,{{input,238854107},{output,11683108162}}}, {reductions,{1516662864,4549156}}, {run_queue,0}, {runtime,{236060,730}}]}, {system_stats, [{cpu_utilization_rate,29.75}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,8264754091}, {couch_docs_data_size,7285585039}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1419790}, {curr_items_tot,3000002}, {mem_used,8077762456}, {vb_replica_curr_items,1580212}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1129}, {memory_data,{33657663488,17469222912,{<17926.368.0>,5085816}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 15421764 kB\nBuffers: 200172 kB\nCached: 8165680 kB\nSwapCached: 0 kB\nActive: 9433436 kB\nInactive: 7688176 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 15421764 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 19316 kB\nWriteback: 0 kB\nAnonPages: 8755852 kB\nMapped: 16356 kB\nSlab: 267836 kB\nPageTables: 22412 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10463536 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-14T11:06:23.410,ns_1@10.2.1.58:<0.2072.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,48,39}}) Stats for bucket "default": accepting_conns 1 auth_cmds 0 auth_errors 0 bucket_active_conns 1 bucket_conns 12 bytes 6534842968 bytes_read 14545779625 bytes_written 2976188951 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 0 conn_yields 26403 connection_structures 5000 curr_connections 15 curr_conns_on_port_11209 12 curr_conns_on_port_11210 1 curr_items 1277622 curr_items_tot 2474423 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-15 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data2/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 20425 ep_commit_time 0 ep_commit_time_total 284819 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 0 ep_data_age_highwat 5 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data2/default ep_degraded_mode 0 ep_diskqueue_drain 8045065 ep_diskqueue_fill 5883575 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 363 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 4634338 ep_io_read_bytes 0 ep_io_write_bytes 10974140237 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 278175 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 6053390541 ep_max_checkpoints 2 ep_max_data_size 12582912000 ep_max_item_size 20971520 ep_max_size 12582912000 ep_max_txn_size 10000 ep_max_vbuckets 1024 ep_mem_high_wat 9437184000 ep_mem_high_wat 9437184000 ep_mem_low_wat 7549747200 ep_mem_low_wat 7549747200 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 210137492 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1352919382 ep_storage_age 0 ep_storage_age_highwat 2 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 6023697465 ep_total_del_items 3 ep_total_enqueued 5883575 ep_total_new_items 2474423 ep_total_persisted 4634338 ep_uncommitted_items 0 ep_value_size 5849376513 ep_vb0 0 ep_vb_snapshot_total 2061 ep_vb_total 846 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_135_g40a391b ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 1431 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 6534842968 pid 29361 pointer_size 64 rejected_conns 0 rusage_system 75.540516 rusage_user 473.798971 tap_checkpoint_end_received 1210 tap_checkpoint_end_sent 387 tap_checkpoint_start_received 2519 tap_checkpoint_start_sent 436 tap_connect_received 1 tap_delete_received 2 tap_mutation_received 6016887 tap_mutation_sent 1136152 tap_opaque_received 5916 tap_opaque_sent 437 tap_vbucket_set_received 872 threads 4 time 1352919982 total_connections 4267 uptime 602 vb_active_curr_items 1277622 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 10948544 vb_active_itm_memory 3110752242 vb_active_meta_data_memory 100932138 vb_active_num 436 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1277622 vb_active_ops_create 1277622 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 574476 vb_active_perc_mem_resident 100 vb_active_queue_age 0 vb_active_queue_drain 3660125 vb_active_queue_fill 2523107 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 1196801 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 10295920 vb_replica_itm_memory 2912945223 vb_replica_meta_data_memory 94547279 vb_replica_num 410 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 1196801 vb_replica_ops_create 1196801 vb_replica_ops_delete 3 vb_replica_ops_reject 0 vb_replica_ops_update 1585436 vb_replica_perc_mem_resident 100 vb_replica_queue_age 0 vb_replica_queue_drain 4384940 vb_replica_queue_fill 3360468 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-14T11:07:22.074,ns_1@10.2.1.58:ns_doctor<0.1953.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,920037,76497}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',15111250}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.4647.1>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,45}, {started_on,1352919988}, {total_vbuckets,850}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1352920036}, {vbuckets_done,390}]]}, {memory, [{total,200952024}, {processes,153908512}, {processes_used,153902424}, {system,47043512}, {atom,1332537}, {atom_used,1325003}, {binary,8915968}, {code,13364289}, {ets,4712896}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7190175744}, {buffered_memory,190341120}, {free_memory,18366173184}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1187496,4999}}, {context_switches,{6579226,0}}, {garbage_collection,{1921655,3534412220,0}}, {io,{{input,150817241},{output,15522039262}}}, {reductions,{818121580,4739447}}, {run_queue,0}, {runtime,{191510,1020}}]}, {system_stats, [{cpu_utilization_rate,33.0}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7578309619}, {couch_docs_data_size,6293001753}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1342091}, {curr_items_tot,2606475}, {mem_used,6883664664}, {vb_replica_curr_items,1264384}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1187}, {memory_data,{33657663488,15422648320,{<0.67.0>,11371232}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 17935840 kB\nBuffers: 185880 kB\nCached: 7021656 kB\nSwapCached: 0 kB\nActive: 7838080 kB\nInactive: 6804356 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 17935840 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 11172 kB\nWriteback: 180 kB\nAnonPages: 7434744 kB\nMapped: 15600 kB\nSlab: 236492 kB\nPageTables: 19404 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 9149452 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920039,826393}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',65119437}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,143745832}, {processes,102274120}, {processes_used,97803928}, {system,41471712}, {atom,1407441}, {atom_used,1381817}, {binary,2729544}, {code,14112737}, {ets,5116040}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,1025384448}, {buffered_memory,173219840}, {free_memory,23509241856}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1191877,5000}}, {context_switches,{7257239,0}}, {garbage_collection,{1391139,18645415241,0}}, {io,{{input,146717265},{output,6146340957}}}, {reductions,{1675784477,5444423}}, {run_queue,0}, {runtime,{373160,1380}}]}, {system_stats, [{cpu_utilization_rate,14.0}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,977891740}, {couch_docs_data_size,977500262}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,238085}, {curr_items_tot,393527}, {mem_used,1185327736}, {vb_replica_curr_items,155442}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1191}, {memory_data,{33657663488,10465226752,{<17879.427.0>,15576440}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 22958368 kB\nBuffers: 169160 kB\nCached: 1001352 kB\nSwapCached: 0 kB\nActive: 8710320 kB\nInactive: 1020420 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 22958368 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 996 kB\nWriteback: 0 kB\nAnonPages: 8560508 kB\nMapped: 16440 kB\nSlab: 123724 kB\nPageTables: 22244 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10267980 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,920041,479787}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default", [{'ns_1@10.2.1.58',124141396},{'ns_1@10.2.1.61',94169628}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,227061584}, {processes,176058072}, {processes_used,175913248}, {system,51003512}, {atom,1338193}, {atom_used,1334188}, {binary,11408720}, {code,13493302}, {ets,6005840}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,8793223168}, {buffered_memory,206217216}, {free_memory,15338082304}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1189943,5001}}, {context_switches,{10228955,0}}, {garbage_collection,{2355400,5173200329,0}}, {io,{{input,247739005},{output,12027734919}}}, {reductions,{1582068871,4807988}}, {run_queue,0}, {runtime,{247040,880}}]}, {system_stats, [{cpu_utilization_rate,31.0}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,8680145094}, {couch_docs_data_size,7285371261}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1416948}, {curr_items_tot,3000002}, {mem_used,8085415096}, {vb_replica_curr_items,1583054}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1189}, {memory_data,{33657663488,17953644544,{<17926.368.0>,5085816}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 14978596 kB\nBuffers: 201384 kB\nCached: 8587132 kB\nSwapCached: 0 kB\nActive: 9674424 kB\nInactive: 7882032 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 14978596 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 10868 kB\nWriteback: 204 kB\nAnonPages: 8767644 kB\nMapped: 16500 kB\nSlab: 277940 kB\nPageTables: 22584 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10486144 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:warn,2012-11-14T11:07:53.132,ns_1@10.2.1.58:<0.2072.0>:stats_collector:latest_tick:223]Dropped 1 ticks [stats:warn,2012-11-14T11:08:02.873,ns_1@10.2.1.58:<0.2072.0>:stats_collector:latest_tick:223]Dropped 1 ticks [stats:debug,2012-11-14T11:08:04.877,ns_1@10.2.1.58:<0.2072.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,48,40}}) Stats for bucket "default": accepting_conns 1 auth_cmds 0 auth_errors 0 bucket_active_conns 1 bucket_conns 14 bytes 7163478728 bytes_read 15961066072 bytes_written 3250319376 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 0 conn_yields 29664 connection_structures 5000 curr_connections 17 curr_conns_on_port_11209 14 curr_conns_on_port_11210 1 curr_items 1395037 curr_items_tot 2712876 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-15 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data2/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 22120 ep_commit_time 9 ep_commit_time_total 334232 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 3 ep_data_age_highwat 5 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data2/default ep_degraded_mode 0 ep_diskqueue_drain 8808123 ep_diskqueue_fill 6464840 ep_diskqueue_items 507 ep_diskqueue_memory 17248 ep_diskqueue_pending 25872 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 428 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 477 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 5021932 ep_io_read_bytes 0 ep_io_write_bytes 11892751864 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 327758 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 6637158804 ep_max_checkpoints 2 ep_max_data_size 12582912000 ep_max_item_size 20971520 ep_max_size 12582912000 ep_max_txn_size 10000 ep_max_vbuckets 1024 ep_mem_high_wat 9437184000 ep_mem_high_wat 9437184000 ep_mem_low_wat 7549747200 ep_mem_low_wat 7549747200 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 229705668 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 30 ep_startup_time 1352919382 ep_storage_age 0 ep_storage_age_highwat 2 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 6604604292 ep_total_del_items 3 ep_total_enqueued 6464840 ep_total_new_items 2712337 ep_total_persisted 5021932 ep_uncommitted_items 32 ep_value_size 6413482711 ep_vb0 0 ep_vb_snapshot_total 2279 ep_vb_total 927 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_135_g40a391b ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 1431 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 7163478728 pid 29361 pointer_size 64 rejected_conns 0 rusage_system 82.615440 rusage_user 542.387544 tap_checkpoint_end_received 1328 tap_checkpoint_end_sent 420 tap_checkpoint_start_received 2760 tap_checkpoint_start_sent 476 tap_connect_received 1 tap_delete_received 2 tap_mutation_received 6601652 tap_mutation_sent 1230891 tap_opaque_received 6482 tap_opaque_sent 477 tap_vbucket_set_received 952 threads 4 time 1352920083 total_connections 4671 uptime 703 vb_active_curr_items 1395037 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 11953024 vb_active_itm_memory 3396621715 vb_active_meta_data_memory 110207923 vb_active_num 476 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1395037 vb_active_ops_create 1395037 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 589981 vb_active_perc_mem_resident 100 vb_active_queue_age 0 vb_active_queue_drain 3985968 vb_active_queue_fill 2754973 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 1317839 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 11325512 vb_replica_itm_memory 3207982577 vb_replica_meta_data_memory 104109281 vb_replica_num 451 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 1317839 vb_replica_ops_create 1317300 vb_replica_ops_delete 3 vb_replica_ops_reject 0 vb_replica_ops_update 1719611 vb_replica_perc_mem_resident 100 vb_replica_queue_age 0 vb_replica_queue_drain 4822155 vb_replica_queue_fill 3709867 vb_replica_queue_memory 17248 vb_replica_queue_pending 25872 vb_replica_queue_size 539 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-14T11:08:22.074,ns_1@10.2.1.58:ns_doctor<0.1953.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,920097,78849}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',104945599}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.4647.1>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,96}, {started_on,1352919988}, {total_vbuckets,850}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1352920097}, {vbuckets_done,821}]]}, {memory, [{total,211471016}, {processes,164138680}, {processes_used,164120312}, {system,47332336}, {atom,1332537}, {atom_used,1325003}, {binary,8979328}, {code,13364289}, {ets,4895184}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7341547520}, {buffered_memory,196702208}, {free_memory,17817874432}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1247499,5003}}, {context_switches,{7082827,0}}, {garbage_collection,{2019562,3754650684,0}}, {io,{{input,161619761},{output,16354476479}}}, {reductions,{875104224,5951297}}, {run_queue,0}, {runtime,{202960,1090}}]}, {system_stats, [{cpu_utilization_rate,40.852130325814535}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7312878881}, {couch_docs_data_size,6621899292}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1409634}, {curr_items_tot,2744621}, {mem_used,7248513192}, {vb_replica_curr_items,1334987}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1247}, {memory_data,{33657663488,15230287872,{<0.367.0>,5085816}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 17400268 kB\nBuffers: 192092 kB\nCached: 7169480 kB\nSwapCached: 0 kB\nActive: 8166696 kB\nInactive: 7007896 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 17400268 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 14372 kB\nWriteback: 3480 kB\nAnonPages: 7812900 kB\nMapped: 15640 kB\nSlab: 239120 kB\nPageTables: 20168 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 9528884 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920099,826169}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',116225073}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,140588024}, {processes,99556552}, {processes_used,94846792}, {system,41031472}, {atom,1407441}, {atom_used,1381817}, {binary,2318880}, {code,14112737}, {ets,5119480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,683864064}, {buffered_memory,174120960}, {free_memory,23840006144}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1251877,5001}}, {context_switches,{7354509,0}}, {garbage_collection,{1435542,19632372694,0}}, {io,{{input,148289605},{output,6160774973}}}, {reductions,{1723730636,5160819}}, {run_queue,0}, {runtime,{390630,1680}}]}, {system_stats, [{cpu_utilization_rate,13.267813267813267}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,648603269}, {couch_docs_data_size,648346388}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,161608}, {curr_items_tot,252440}, {mem_used,818673128}, {vb_replica_curr_items,90832}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1251}, {memory_data,{33657663488,10108669952,{<17879.427.0>,15568680}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 23281256 kB\nBuffers: 170040 kB\nCached: 667836 kB\nSwapCached: 0 kB\nActive: 8728644 kB\nInactive: 687256 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 23281256 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 1540 kB\nWriteback: 0 kB\nAnonPages: 8577780 kB\nMapped: 16492 kB\nSlab: 115504 kB\nPageTables: 22200 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10294292 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,920101,478224}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default", [{'ns_1@10.2.1.58',78163240},{'ns_1@10.2.1.61',74304752}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,232074296}, {processes,179961184}, {processes_used,179774112}, {system,52113112}, {atom,1338193}, {atom_used,1334188}, {binary,12506904}, {code,13493302}, {ets,6018872}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,9189666816}, {buffered_memory,207257600}, {free_memory,14926307328}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1249942,5000}}, {context_switches,{10640059,0}}, {garbage_collection,{2449975,5392858862,0}}, {io,{{input,256416172},{output,12346352227}}}, {reductions,{1643310031,6005003}}, {run_queue,0}, {runtime,{257060,860}}]}, {system_stats, [{cpu_utilization_rate,42.394014962593516}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,9072764436}, {couch_docs_data_size,7285172080}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1428850}, {curr_items_tot,3000002}, {mem_used,8091739816}, {vb_replica_curr_items,1571152}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1249}, {memory_data,{33657663488,18380394496,{<17926.368.0>,5085816}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 14576472 kB\nBuffers: 202400 kB\nCached: 8974284 kB\nSwapCached: 0 kB\nActive: 9894136 kB\nInactive: 8054152 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 14576472 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 8296 kB\nWriteback: 0 kB\nAnonPages: 8771020 kB\nMapped: 16404 kB\nSlab: 288344 kB\nPageTables: 22496 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10496692 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-14T11:09:22.074,ns_1@10.2.1.58:ns_doctor<0.1953.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,920157,76514}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',90476866}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,219615520}, {processes,171551088}, {processes_used,171546472}, {system,48064432}, {atom,1332537}, {atom_used,1325003}, {binary,9452696}, {code,13364289}, {ets,5115760}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,8011603968}, {buffered_memory,200118272}, {free_memory,16703451136}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1307496,5001}}, {context_switches,{7580051,0}}, {garbage_collection,{2113231,3974761877,0}}, {io,{{input,173807027},{output,17239735742}}}, {reductions,{931450626,4057338}}, {run_queue,0}, {runtime,{214620,1000}}]}, {system_stats, [{cpu_utilization_rate,27.930174563591024}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7874236503}, {couch_docs_data_size,6992188796}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1482753}, {curr_items_tot,2891351}, {mem_used,7633817768}, {vb_replica_curr_items,1408598}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1307}, {memory_data,{33657663488,16027189248,{<0.367.0>,5085816}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16311964 kB\nBuffers: 195428 kB\nCached: 7823832 kB\nSwapCached: 0 kB\nActive: 8685504 kB\nInactive: 7559636 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16311964 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 10320 kB\nWriteback: 0 kB\nAnonPages: 8225836 kB\nMapped: 15920 kB\nSlab: 255920 kB\nPageTables: 21124 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 9942480 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920159,825972}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',1932124}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,129522312}, {processes,88976656}, {processes_used,84018248}, {system,40545656}, {atom,1407441}, {atom_used,1381817}, {binary,1752224}, {code,14112737}, {ets,5235272}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,400244736}, {buffered_memory,175595520}, {free_memory,24160313344}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1311877,4999}}, {context_switches,{7446381,0}}, {garbage_collection,{1490840,20672905344,0}}, {io,{{input,153170618},{output,6178676054}}}, {reductions,{1771894312,4491990}}, {run_queue,0}, {runtime,{408680,1340}}]}, {system_stats, [{cpu_utilization_rate,15.71072319201995}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,278008814}, {couch_docs_data_size,277883705}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,90938}, {curr_items_tot,108651}, {mem_used,446240024}, {vb_replica_curr_items,17713}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1311}, {memory_data,{33657663488,9761525760,{<17879.427.0>,25159424}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 23594056 kB\nBuffers: 171480 kB\nCached: 390864 kB\nSwapCached: 0 kB\nActive: 8740164 kB\nInactive: 369272 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 23594056 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 2852 kB\nWriteback: 0 kB\nAnonPages: 8547332 kB\nMapped: 16492 kB\nSlab: 108368 kB\nPageTables: 22160 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10266120 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,920161,477510}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default", [{'ns_1@10.2.1.58',25024159},{'ns_1@10.2.1.61',64924766}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,225644208}, {processes,172694056}, {processes_used,172465640}, {system,52950152}, {atom,1338193}, {atom_used,1334188}, {binary,13217672}, {code,13493302}, {ets,6148016}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,9656131584}, {buffered_memory,209317888}, {free_memory,14419202048}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1309941,4998}}, {context_switches,{11088811,0}}, {garbage_collection,{2551319,5626006373,0}}, {io,{{input,269081261},{output,12675504708}}}, {reductions,{1706851846,4399088}}, {run_queue,0}, {runtime,{267970,850}}]}, {system_stats, [{cpu_utilization_rate,25.75}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,9491749736}, {couch_docs_data_size,7284986471}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1426233}, {curr_items_tot,3000002}, {mem_used,8099011016}, {vb_replica_curr_items,1573769}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1309}, {memory_data,{33657663488,18821533696,{<17926.368.0>,5085816}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 14081376 kB\nBuffers: 204412 kB\nCached: 9429816 kB\nSwapCached: 0 kB\nActive: 10165488 kB\nInactive: 8264208 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 14081376 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 5852 kB\nWriteback: 0 kB\nAnonPages: 8795544 kB\nMapped: 19648 kB\nSlab: 300368 kB\nPageTables: 23040 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10532032 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-14T11:09:44.854,ns_1@10.2.1.58:<0.2072.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,48,40}}) Stats for bucket "default": accepting_conns 1 auth_cmds 0 auth_errors 0 bucket_active_conns 1 bucket_conns 13 bytes 7773803448 bytes_read 17357271743 bytes_written 3513479123 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 0 conn_yields 32738 connection_structures 5000 curr_connections 16 curr_conns_on_port_11209 13 curr_conns_on_port_11210 1 curr_items 1500466 curr_items_tot 2944447 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-15 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data2/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 1 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 26285 ep_commit_time 3 ep_commit_time_total 367525 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 0 ep_data_age_highwat 5 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data2/default ep_degraded_mode 0 ep_diskqueue_drain 9559788 ep_diskqueue_fill 7039597 ep_diskqueue_items 0 ep_diskqueue_memory 320 ep_diskqueue_pending 480 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 474 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 5474900 ep_io_read_bytes 0 ep_io_write_bytes 12963818791 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 380712 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 7203115397 ep_max_checkpoints 2 ep_max_data_size 12582912000 ep_max_item_size 20971520 ep_max_size 12582912000 ep_max_txn_size 10000 ep_max_vbuckets 1024 ep_mem_high_wat 9437184000 ep_mem_high_wat 9437184000 ep_mem_low_wat 7549747200 ep_mem_low_wat 7549747200 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 249148554 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1352919382 ep_storage_age 0 ep_storage_age_highwat 3 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 7167782033 ep_total_del_items 3 ep_total_enqueued 7039597 ep_total_new_items 2944447 ep_total_persisted 5474900 ep_uncommitted_items 10 ep_value_size 6960345856 ep_vb0 0 ep_vb_snapshot_total 2494 ep_vb_total 1005 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_135_g40a391b ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 1431 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 7773803448 pid 29361 pointer_size 64 rejected_conns 0 rusage_system 90.289273 rusage_user 620.161721 tap_checkpoint_end_received 1444 tap_checkpoint_end_sent 451 tap_checkpoint_start_received 2995 tap_checkpoint_start_sent 512 tap_connect_received 1 tap_delete_received 2 tap_mutation_received 7179792 tap_mutation_sent 1321512 tap_opaque_received 7032 tap_opaque_sent 513 tap_vbucket_set_received 1024 threads 4 time 1352920184 total_connections 5056 uptime 804 vb_active_curr_items 1500466 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 12857056 vb_active_itm_memory 3653305966 vb_active_meta_data_memory 118536814 vb_active_num 512 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1500466 vb_active_ops_create 1500466 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 639616 vb_active_perc_mem_resident 100 vb_active_queue_age 0 vb_active_queue_drain 4284459 vb_active_queue_fill 2962833 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 1443981 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 12380024 vb_replica_itm_memory 3514476067 vb_replica_meta_data_memory 114074499 vb_replica_num 493 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 1443981 vb_replica_ops_create 1443981 vb_replica_ops_delete 3 vb_replica_ops_reject 0 vb_replica_ops_update 1890834 vb_replica_perc_mem_resident 100 vb_replica_queue_age 0 vb_replica_queue_drain 5275329 vb_replica_queue_fill 4076764 vb_replica_queue_memory 320 vb_replica_queue_pending 480 vb_replica_queue_size 10 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-14T11:10:22.074,ns_1@10.2.1.58:ns_doctor<0.1953.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,920217,76393}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',97310940}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,225464360}, {processes,176113416}, {processes_used,176105480}, {system,49350944}, {atom,1332537}, {atom_used,1325003}, {binary,10659720}, {code,13364289}, {ets,5170728}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,8781049856}, {buffered_memory,202838016}, {free_memory,15618555904}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1367496,5000}}, {context_switches,{8047704,0}}, {garbage_collection,{2178691,4163253001,0}}, {io,{{input,181348887},{output,17944643368}}}, {reductions,{978386480,3895650}}, {run_queue,0}, {runtime,{227250,1120}}]}, {system_stats, [{cpu_utilization_rate,15.037593984962406}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,8580647255}, {couch_docs_data_size,7256191207}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,2997052}, {mem_used,7913286968}, {vb_replica_curr_items,1496586}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1367}, {memory_data,{33657663488,17268252672,{<0.367.0>,5085816}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 15252620 kB\nBuffers: 198084 kB\nCached: 8575244 kB\nSwapCached: 0 kB\nActive: 9167000 kB\nInactive: 8117224 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 15252620 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 17492 kB\nWriteback: 28 kB\nAnonPages: 8510332 kB\nMapped: 15592 kB\nSlab: 275516 kB\nPageTables: 21716 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10213548 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920219,826542}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes,[{"default",[]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,123356976}, {processes,83277104}, {processes_used,78127432}, {system,40079872}, {atom,1407441}, {atom_used,1381817}, {binary,1305304}, {code,14112737}, {ets,5254952}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145342464}, {buffered_memory,176275456}, {free_memory,24475975680}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1371877,5000}}, {context_switches,{7516166,0}}, {garbage_collection,{1508000,21444157715,0}}, {io,{{input,154544098},{output,6189947304}}}, {reductions,{1809935485,3878474}}, {run_queue,0}, {runtime,{423290,1170}}]}, {system_stats, [{cpu_utilization_rate,3.9800995024875623}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,21403929}, {couch_docs_data_size,21358256}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2950}, {curr_items_tot,2950}, {mem_used,176158616}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1371}, {memory_data,{33657663488,9472090112,{<17879.427.0>,15551296}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 23902320 kB\nBuffers: 172144 kB\nCached: 141936 kB\nSwapCached: 0 kB\nActive: 8685172 kB\nInactive: 122504 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 23902320 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 608 kB\nWriteback: 0 kB\nAnonPages: 8494056 kB\nMapped: 16492 kB\nSlab: 102544 kB\nPageTables: 22080 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10204144 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,920221,477487}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.25852.1>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,19}, {started_on,1352920189}, {total_vbuckets,1025}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1352920209}, {vbuckets_done,197}]]}, {memory, [{total,224215792}, {processes,171038232}, {processes_used,170775112}, {system,53177560}, {atom,1338193}, {atom_used,1334188}, {binary,13446576}, {code,13493302}, {ets,6170520}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,9979568128}, {buffered_memory,211849216}, {free_memory,14096252928}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1369941,4999}}, {context_switches,{11442866,0}}, {garbage_collection,{2600723,5768030735,0}}, {io,{{input,277327368},{output,12925602733}}}, {reductions,{1756331731,3865369}}, {run_queue,0}, {runtime,{275720,500}}]}, {system_stats, [{cpu_utilization_rate,42.5}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,9797251054}, {couch_docs_data_size,7284789472}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,8098714984}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1369}, {memory_data,{33657663488,19331203072,{<17926.368.0>,5085816}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 13765996 kB\nBuffers: 206884 kB\nCached: 9745672 kB\nSwapCached: 0 kB\nActive: 10315476 kB\nInactive: 8420948 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 13765996 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 21024 kB\nWriteback: 28 kB\nAnonPages: 8783964 kB\nMapped: 16404 kB\nSlab: 309124 kB\nPageTables: 22512 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10485996 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-14T11:11:22.074,ns_1@10.2.1.58:ns_doctor<0.1953.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,920277,75933}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,245555800}, {processes,195774016}, {processes_used,195754000}, {system,49781784}, {atom,1334153}, {atom_used,1326649}, {binary,10978480}, {code,13390578}, {ets,5292472}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,8822525952}, {buffered_memory,203276288}, {free_memory,15549087744}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1427496,5000}}, {context_switches,{8257821,0}}, {garbage_collection,{2189832,4219212824,0}}, {io,{{input,185469337},{output,17977591688}}}, {reductions,{1003486162,988729}}, {run_queue,0}, {runtime,{233800,510}}]}, {system_stats, [{cpu_utilization_rate,2.736318407960199}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,8662774620}, {couch_docs_data_size,7284405678}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7921143576}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1427}, {memory_data,{33657663488,18119847936,{<0.84.0>,11371232}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 15184656 kB\nBuffers: 198512 kB\nCached: 8615748 kB\nSwapCached: 0 kB\nActive: 9208864 kB\nInactive: 8144504 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 15184656 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 6452 kB\nWriteback: 0 kB\nAnonPages: 8538780 kB\nMapped: 15592 kB\nSlab: 274824 kB\nPageTables: 21780 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10247900 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920229,826001}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17879.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,920281,478228}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.25852.1>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,78}, {started_on,1352920189}, {total_vbuckets,1025}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1352920269}, {vbuckets_done,808}]]}, {memory, [{total,214435432}, {processes,163239888}, {processes_used,163146744}, {system,51195544}, {atom,1338193}, {atom_used,1334188}, {binary,11168360}, {code,13493302}, {ets,6480384}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,8547299328}, {buffered_memory,217767936}, {free_memory,15549161472}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1429942,5001}}, {context_switches,{11745950,0}}, {garbage_collection,{2631486,5860930252,0}}, {io,{{input,284709137},{output,12930320071}}}, {reductions,{1797166777,3243591}}, {run_queue,0}, {runtime,{281770,510}}]}, {system_stats, [{cpu_utilization_rate,21.5}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,8346166381}, {couch_docs_data_size,7285837870}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,8098639256}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1429}, {memory_data,{33657663488,19581845504,{<17926.84.0>,11742176}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 15184728 kB\nBuffers: 212664 kB\nCached: 8346972 kB\nSwapCached: 0 kB\nActive: 9822480 kB\nInactive: 7518728 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 15184728 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 21968 kB\nWriteback: 0 kB\nAnonPages: 8781432 kB\nMapped: 15584 kB\nSlab: 285992 kB\nPageTables: 21896 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10471036 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-14T11:11:33.093,ns_1@10.2.1.58:<0.2072.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,48,40}}) Stats for bucket "default": accepting_conns 1 auth_cmds 0 auth_errors 0 bucket_active_conns 1 bucket_conns 10 bytes 7921146680 bytes_read 17748147764 bytes_written 3517910494 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 0 conn_yields 33389 connection_structures 5000 curr_connections 13 curr_conns_on_port_11209 10 curr_conns_on_port_11210 1 curr_items 1500466 curr_items_tot 3000002 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-15 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data2/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 28828 ep_commit_time 3 ep_commit_time_total 379413 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 1 ep_data_age_highwat 5 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data2/default ep_degraded_mode 0 ep_diskqueue_drain 9763736 ep_diskqueue_fill 7199710 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 489 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 5613232 ep_io_read_bytes 0 ep_io_write_bytes 13292106458 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 395370 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 7339250270 ep_max_checkpoints 2 ep_max_data_size 12582912000 ep_max_item_size 20971520 ep_max_size 12582912000 ep_max_txn_size 10000 ep_max_vbuckets 1024 ep_mem_high_wat 9437184000 ep_mem_high_wat 9437184000 ep_mem_low_wat 7549747200 ep_mem_low_wat 7549747200 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 254245620 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1352919382 ep_storage_age 0 ep_storage_age_highwat 3 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 7303250246 ep_total_del_items 3 ep_total_enqueued 7199710 ep_total_new_items 3000002 ep_total_persisted 5613232 ep_uncommitted_items 0 ep_value_size 7091900551 ep_vb0 0 ep_vb_snapshot_total 2539 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_135_g40a391b ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 1431 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 7921146680 pid 29361 pointer_size 64 rejected_conns 0 rusage_system 92.671911 rusage_user 633.181741 tap_checkpoint_end_received 1482 tap_checkpoint_end_sent 451 tap_checkpoint_start_received 3054 tap_checkpoint_start_sent 512 tap_connect_received 1 tap_delete_received 2 tap_mutation_received 7341122 tap_mutation_sent 1321512 tap_opaque_received 7168 tap_opaque_sent 513 tap_vbucket_set_received 1024 threads 4 time 1352920292 total_connections 5133 uptime 912 vb_active_curr_items 1500466 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 12857056 vb_active_itm_memory 3653305966 vb_active_meta_data_memory 118536814 vb_active_num 512 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1500466 vb_active_ops_create 1500466 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 639616 vb_active_perc_mem_resident 100 vb_active_queue_age 0 vb_active_queue_drain 4284459 vb_active_queue_fill 2962833 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 1499536 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 12857152 vb_replica_itm_memory 3649944280 vb_replica_meta_data_memory 118463344 vb_replica_num 512 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 1499536 vb_replica_ops_create 1499536 vb_replica_ops_delete 3 vb_replica_ops_reject 0 vb_replica_ops_update 1973611 vb_replica_perc_mem_resident 100 vb_replica_queue_age 0 vb_replica_queue_drain 5479277 vb_replica_queue_fill 4236877 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-14T11:12:22.074,ns_1@10.2.1.58:ns_doctor<0.1953.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,920337,75744}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,243476616}, {processes,193890440}, {processes_used,193870424}, {system,49586176}, {atom,1334153}, {atom_used,1326649}, {binary,10606536}, {code,13390578}, {ets,5468968}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,8818102272}, {buffered_memory,203616256}, {free_memory,15548452864}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1487495,4999}}, {context_switches,{8453078,0}}, {garbage_collection,{2200073,4263587303,0}}, {io,{{input,192249423},{output,17982720342}}}, {reductions,{1029053397,1013656}}, {run_queue,0}, {runtime,{240210,500}}]}, {system_stats, [{cpu_utilization_rate,2.5},{swap_total,10684628992},{swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,8662772768}, {couch_docs_data_size,7284405678}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7921143576}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1487}, {memory_data,{33657663488,18108702720,{<0.84.0>,11371232}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 15184036 kB\nBuffers: 198844 kB\nCached: 8611428 kB\nSwapCached: 0 kB\nActive: 9213272 kB\nInactive: 8141220 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 15184036 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 296 kB\nWriteback: 0 kB\nAnonPages: 8544448 kB\nMapped: 15592 kB\nSlab: 274248 kB\nPageTables: 21776 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10250876 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920229,826001}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17879.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,920341,479002}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,214801248}, {processes,165442864}, {processes_used,165430888}, {system,49358384}, {atom,1338193}, {atom_used,1334188}, {binary,9018440}, {code,13493302}, {ets,6782632}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7471898624}, {buffered_memory,219922432}, {free_memory,16647299072}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1489942,5000}}, {context_switches,{11981408,0}}, {garbage_collection,{2649039,5936904098,0}}, {io,{{input,293226660},{output,12936444562}}}, {reductions,{1831108211,2621872}}, {run_queue,0}, {runtime,{288020,550}}]}, {system_stats, [{cpu_utilization_rate,3.0},{swap_total,10585923584},{swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7288618492}, {couch_docs_data_size,7286342852}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,8098639256}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1489}, {memory_data,{33657663488,18110406656,{<17926.676.0>,4714984}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16257128 kB\nBuffers: 214768 kB\nCached: 7296776 kB\nSwapCached: 0 kB\nActive: 9052480 kB\nInactive: 7237368 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16257128 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 11104 kB\nWriteback: 0 kB\nAnonPages: 8778376 kB\nMapped: 15596 kB\nSlab: 264844 kB\nPageTables: 22136 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10468964 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-14T11:13:13.092,ns_1@10.2.1.58:<0.2072.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,48,40}}) Stats for bucket "default": accepting_conns 1 auth_cmds 0 auth_errors 0 bucket_active_conns 1 bucket_conns 10 bytes 7921143544 bytes_read 17748158734 bytes_written 3520889718 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 0 conn_yields 33389 connection_structures 5000 curr_connections 13 curr_conns_on_port_11209 10 curr_conns_on_port_11210 1 curr_items 1500466 curr_items_tot 3000002 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-15 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data2/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 28828 ep_commit_time 3 ep_commit_time_total 379413 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 1 ep_data_age_highwat 5 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data2/default ep_degraded_mode 0 ep_diskqueue_drain 9763736 ep_diskqueue_fill 7199710 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 489 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 5613232 ep_io_read_bytes 0 ep_io_write_bytes 13292106458 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 395370 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 7339250270 ep_max_checkpoints 2 ep_max_data_size 12582912000 ep_max_item_size 20971520 ep_max_size 12582912000 ep_max_txn_size 10000 ep_max_vbuckets 1024 ep_mem_high_wat 9437184000 ep_mem_high_wat 9437184000 ep_mem_low_wat 7549747200 ep_mem_low_wat 7549747200 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 254245620 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1352919382 ep_storage_age 0 ep_storage_age_highwat 3 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 7303250246 ep_total_del_items 3 ep_total_enqueued 7199710 ep_total_new_items 3000002 ep_total_persisted 5613232 ep_uncommitted_items 0 ep_value_size 7091900551 ep_vb0 0 ep_vb_snapshot_total 2539 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_135_g40a391b ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 1431 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 7921143544 pid 29361 pointer_size 64 rejected_conns 0 rusage_system 92.696907 rusage_user 633.734657 tap_checkpoint_end_received 1482 tap_checkpoint_end_sent 451 tap_checkpoint_start_received 3054 tap_checkpoint_start_sent 512 tap_connect_received 1 tap_delete_received 2 tap_mutation_received 7341122 tap_mutation_sent 1321512 tap_opaque_received 7168 tap_opaque_sent 513 tap_vbucket_set_received 1024 threads 4 time 1352920392 total_connections 5133 uptime 1012 vb_active_curr_items 1500466 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 12857056 vb_active_itm_memory 3653305966 vb_active_meta_data_memory 118536814 vb_active_num 512 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1500466 vb_active_ops_create 1500466 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 639616 vb_active_perc_mem_resident 100 vb_active_queue_age 0 vb_active_queue_drain 4284459 vb_active_queue_fill 2962833 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 1499536 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 12857152 vb_replica_itm_memory 3649944280 vb_replica_meta_data_memory 118463344 vb_replica_num 512 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 1499536 vb_replica_ops_create 1499536 vb_replica_ops_delete 3 vb_replica_ops_reject 0 vb_replica_ops_update 1973611 vb_replica_perc_mem_resident 100 vb_replica_queue_age 0 vb_replica_queue_drain 5479277 vb_replica_queue_fill 4236877 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-14T11:13:22.074,ns_1@10.2.1.58:ns_doctor<0.1953.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,920397,75410}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.18653.1>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,23}, {started_on,1352920370}, {total_vbuckets,1025}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1352920397}, {vbuckets_done,237}]]}, {memory, [{total,238650432}, {processes,188534248}, {processes_used,188519544}, {system,50116184}, {atom,1334153}, {atom_used,1326649}, {binary,10953760}, {code,13390578}, {ets,5652120}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,8825135104}, {buffered_memory,206884864}, {free_memory,15526096896}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1547495,4999}}, {context_switches,{8688996,0}}, {garbage_collection,{2219897,4313950959,0}}, {io,{{input,196799623},{output,17985853654}}}, {reductions,{1057678103,1601317}}, {run_queue,0}, {runtime,{246760,560}}]}, {system_stats, [{cpu_utilization_rate,16.62531017369727}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,8655571909}, {couch_docs_data_size,7284405678}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7921143576}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1547}, {memory_data,{33657663488,18111750144,{<0.84.0>,11371232}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 15162204 kB\nBuffers: 202036 kB\nCached: 8618296 kB\nSwapCached: 0 kB\nActive: 9231216 kB\nInactive: 8142284 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 15162204 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 8864 kB\nWriteback: 180 kB\nAnonPages: 8553280 kB\nMapped: 15596 kB\nSlab: 275612 kB\nPageTables: 21860 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10260852 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920229,826001}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17879.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,920401,478095}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,216984376}, {processes,167637656}, {processes_used,167625680}, {system,49346720}, {atom,1338193}, {atom_used,1334188}, {binary,8832760}, {code,13493302}, {ets,6956656}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7473016832}, {buffered_memory,220016640}, {free_memory,16646537216}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1549942,5000}}, {context_switches,{12179519,0}}, {garbage_collection,{2659938,6006127230,0}}, {io,{{input,296686871},{output,12938394282}}}, {reductions,{1862166747,2601368}}, {run_queue,0}, {runtime,{294490,550}}]}, {system_stats, [{cpu_utilization_rate,2.756892230576441}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7288618492}, {couch_docs_data_size,7286342852}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,8098639256}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1549}, {memory_data,{33657663488,17010245632,{<17926.368.0>,4114672}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16256384 kB\nBuffers: 214860 kB\nCached: 7297868 kB\nSwapCached: 0 kB\nActive: 9053216 kB\nInactive: 7238388 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16256384 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 9516 kB\nWriteback: 0 kB\nAnonPages: 8778900 kB\nMapped: 15596 kB\nSlab: 264644 kB\nPageTables: 22136 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10469972 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-14T11:14:22.074,ns_1@10.2.1.58:ns_doctor<0.1953.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,920457,76488}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks, [[{pid,<<"<0.18653.1>">>}, {bucket,<<"default">>}, {original_target,{[{type,bucket}]}}, {progress,76}, {started_on,1352920370}, {total_vbuckets,1025}, {trigger_type,scheduled}, {type,bucket_compaction}, {updated_on,1352920457}, {vbuckets_done,780}]]}, {memory, [{total,208679480}, {processes,160267976}, {processes_used,160251952}, {system,48411504}, {atom,1334153}, {atom_used,1326649}, {binary,9074824}, {code,13390578}, {ets,5817832}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7620743168}, {buffered_memory,213049344}, {free_memory,16778334208}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1607496,5000}}, {context_switches,{8976569,0}}, {garbage_collection,{2250390,4375452391,0}}, {io,{{input,203490533},{output,17989427413}}}, {reductions,{1090051886,1609941}}, {run_queue,0}, {runtime,{253210,500}}]}, {system_stats, [{cpu_utilization_rate,20.100502512562816}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7468121121}, {couch_docs_data_size,7285046167}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7921143576}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1607}, {memory_data,{33657663488,18134740992,{<0.67.0>,11000288}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16385216 kB\nBuffers: 208056 kB\nCached: 7441804 kB\nSwapCached: 0 kB\nActive: 8823556 kB\nInactive: 7356208 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16385216 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 9936 kB\nWriteback: 0 kB\nAnonPages: 8529844 kB\nMapped: 15592 kB\nSlab: 246712 kB\nPageTables: 22060 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10234812 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920229,826001}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17879.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,920461,478439}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,212551920}, {processes,162926800}, {processes_used,162914824}, {system,49625120}, {atom,1338193}, {atom_used,1334188}, {binary,9048352}, {code,13493302}, {ets,7019256}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7474130944}, {buffered_memory,220114944}, {free_memory,16645648384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1609942,5000}}, {context_switches,{12377715,0}}, {garbage_collection,{2669670,6075464322,0}}, {io,{{input,300135454},{output,12940331742}}}, {reductions,{1893181319,2599196}}, {run_queue,0}, {runtime,{300820,550}}]}, {system_stats, [{cpu_utilization_rate,2.756892230576441}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7288618493}, {couch_docs_data_size,7286342852}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,8098639256}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1609}, {memory_data,{33657663488,17014943744,{<17926.676.0>,4714984}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16255516 kB\nBuffers: 214956 kB\nCached: 7298956 kB\nSwapCached: 0 kB\nActive: 9052348 kB\nInactive: 7239392 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16255516 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 8972 kB\nWriteback: 0 kB\nAnonPages: 8777844 kB\nMapped: 15596 kB\nSlab: 264628 kB\nPageTables: 22132 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10467488 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-14T11:14:53.092,ns_1@10.2.1.58:<0.2072.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,48,40}}) Stats for bucket "default": accepting_conns 1 auth_cmds 0 auth_errors 0 bucket_active_conns 1 bucket_conns 10 bytes 7921143544 bytes_read 17748169704 bytes_written 3523869014 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 0 conn_yields 33389 connection_structures 5000 curr_connections 13 curr_conns_on_port_11209 10 curr_conns_on_port_11210 1 curr_items 1500466 curr_items_tot 3000002 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-15 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data2/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 28828 ep_commit_time 3 ep_commit_time_total 379413 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 1 ep_data_age_highwat 5 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data2/default ep_degraded_mode 0 ep_diskqueue_drain 9763736 ep_diskqueue_fill 7199710 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 489 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 5613232 ep_io_read_bytes 0 ep_io_write_bytes 13292106458 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 395370 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 7339250270 ep_max_checkpoints 2 ep_max_data_size 12582912000 ep_max_item_size 20971520 ep_max_size 12582912000 ep_max_txn_size 10000 ep_max_vbuckets 1024 ep_mem_high_wat 9437184000 ep_mem_high_wat 9437184000 ep_mem_low_wat 7549747200 ep_mem_low_wat 7549747200 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 254245620 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1352919382 ep_storage_age 0 ep_storage_age_highwat 3 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 7303250246 ep_total_del_items 3 ep_total_enqueued 7199710 ep_total_new_items 3000002 ep_total_persisted 5613232 ep_uncommitted_items 0 ep_value_size 7091900551 ep_vb0 0 ep_vb_snapshot_total 2539 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_135_g40a391b ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 1431 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 7921143544 pid 29361 pointer_size 64 rejected_conns 0 rusage_system 92.723903 rusage_user 634.224583 tap_checkpoint_end_received 1482 tap_checkpoint_end_sent 451 tap_checkpoint_start_received 3054 tap_checkpoint_start_sent 512 tap_connect_received 1 tap_delete_received 2 tap_mutation_received 7341122 tap_mutation_sent 1321512 tap_opaque_received 7168 tap_opaque_sent 513 tap_vbucket_set_received 1024 threads 4 time 1352920492 total_connections 5133 uptime 1112 vb_active_curr_items 1500466 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 12857056 vb_active_itm_memory 3653305966 vb_active_meta_data_memory 118536814 vb_active_num 512 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1500466 vb_active_ops_create 1500466 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 639616 vb_active_perc_mem_resident 100 vb_active_queue_age 0 vb_active_queue_drain 4284459 vb_active_queue_fill 2962833 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 1499536 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 12857152 vb_replica_itm_memory 3649944280 vb_replica_meta_data_memory 118463344 vb_replica_num 512 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 1499536 vb_replica_ops_create 1499536 vb_replica_ops_delete 3 vb_replica_ops_reject 0 vb_replica_ops_update 1973611 vb_replica_perc_mem_resident 100 vb_replica_queue_age 0 vb_replica_queue_drain 5479277 vb_replica_queue_fill 4236877 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-14T11:15:22.074,ns_1@10.2.1.58:ns_doctor<0.1953.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,920517,75751}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,217313896}, {processes,168556576}, {processes_used,168532760}, {system,48757320}, {atom,1334153}, {atom_used,1326649}, {binary,9161944}, {code,13390578}, {ets,6071072}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7447289856}, {buffered_memory,215769088}, {free_memory,16950394880}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1667496,5000}}, {context_switches,{9214462,0}}, {garbage_collection,{2270901,4426542727,0}}, {io,{{input,211774505},{output,17995813587}}}, {reductions,{1118890207,1001722}}, {run_queue,0}, {runtime,{259650,540}}]}, {system_stats, [{cpu_utilization_rate,2.743142144638404}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7287798818}, {couch_docs_data_size,7285575572}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7921143576}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1667}, {memory_data,{33657663488,16883646464,{<0.367.0>,3143528}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16553120 kB\nBuffers: 210712 kB\nCached: 7272744 kB\nSwapCached: 0 kB\nActive: 8796996 kB\nInactive: 7221712 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16553120 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 10000 kB\nWriteback: 0 kB\nAnonPages: 8535016 kB\nMapped: 15592 kB\nSlab: 241240 kB\nPageTables: 21736 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10233296 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920229,826001}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17879.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,920521,478398}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,219495536}, {processes,170006232}, {processes_used,169991856}, {system,49489304}, {atom,1338193}, {atom_used,1334188}, {binary,8736184}, {code,13493302}, {ets,7196112}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7475212288}, {buffered_memory,220323840}, {free_memory,16641077248}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1669942,5000}}, {context_switches,{12577029,0}}, {garbage_collection,{2680540,6144922229,0}}, {io,{{input,306952991},{output,12945625336}}}, {reductions,{1924343411,2587428}}, {run_queue,0}, {runtime,{307130,520}}]}, {system_stats, [{cpu_utilization_rate,2.756892230576441}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7288616215}, {couch_docs_data_size,7286342852}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,8098628776}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1669}, {memory_data,{33657663488,17014943744,{<17926.368.0>,4114672}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16251052 kB\nBuffers: 215160 kB\nCached: 7300012 kB\nSwapCached: 0 kB\nActive: 9057016 kB\nInactive: 7240532 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16251052 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 7096 kB\nWriteback: 0 kB\nAnonPages: 8782488 kB\nMapped: 15596 kB\nSlab: 264508 kB\nPageTables: 22132 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10473108 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-14T11:16:22.074,ns_1@10.2.1.58:ns_doctor<0.1953.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,920577,75699}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,218287920}, {processes,169639176}, {processes_used,169615360}, {system,48648744}, {atom,1334153}, {atom_used,1326649}, {binary,8989680}, {code,13390578}, {ets,6134288}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7448408064}, {buffered_memory,215932928}, {free_memory,16944046080}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1727495,4999}}, {context_switches,{9409320,0}}, {garbage_collection,{2281386,4470145978,0}}, {io,{{input,215247154},{output,17998580461}}}, {reductions,{1144297324,1012973}}, {run_queue,0}, {runtime,{266170,530}}]}, {system_stats, [{cpu_utilization_rate,2.255639097744361}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7287798818}, {couch_docs_data_size,7285575572}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7921143576}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1727}, {memory_data,{33657663488,16709554176,{<0.367.0>,3143528}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16546920 kB\nBuffers: 210872 kB\nCached: 7273836 kB\nSwapCached: 0 kB\nActive: 8801516 kB\nInactive: 7222808 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16546920 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 8984 kB\nWriteback: 0 kB\nAnonPages: 8539572 kB\nMapped: 15592 kB\nSlab: 241020 kB\nPageTables: 21744 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10238380 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920229,826001}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17879.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,920581,478035}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,214139472}, {processes,164459728}, {processes_used,164445352}, {system,49679744}, {atom,1338193}, {atom_used,1334188}, {binary,8863960}, {code,13493302}, {ets,7258448}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7476326400}, {buffered_memory,220405760}, {free_memory,16637902848}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1729942,5000}}, {context_switches,{12775150,0}}, {garbage_collection,{2691218,6214661072,0}}, {io,{{input,310415934},{output,12947752529}}}, {reductions,{1955362781,2629716}}, {run_queue,0}, {runtime,{313370,540}}]}, {system_stats, [{cpu_utilization_rate,3.0},{swap_total,10585923584},{swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7288613936}, {couch_docs_data_size,7286342852}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,8098628776}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1729}, {memory_data,{33657663488,17020395520,{<17926.368.0>,4114672}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16247952 kB\nBuffers: 215240 kB\nCached: 7301100 kB\nSwapCached: 0 kB\nActive: 9058760 kB\nInactive: 7241532 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16247952 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 4924 kB\nWriteback: 0 kB\nAnonPages: 8784016 kB\nMapped: 15596 kB\nSlab: 264532 kB\nPageTables: 22132 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10473392 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-14T11:16:33.092,ns_1@10.2.1.58:<0.2072.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,48,40}}) Stats for bucket "default": accepting_conns 1 auth_cmds 0 auth_errors 0 bucket_active_conns 1 bucket_conns 10 bytes 7921143560 bytes_read 17748180674 bytes_written 3526848334 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 0 conn_yields 33389 connection_structures 5000 curr_connections 13 curr_conns_on_port_11209 10 curr_conns_on_port_11210 1 curr_items 1500466 curr_items_tot 3000002 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-15 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data2/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 28828 ep_commit_time 3 ep_commit_time_total 379413 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 1 ep_data_age_highwat 5 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data2/default ep_degraded_mode 0 ep_diskqueue_drain 9763736 ep_diskqueue_fill 7199710 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 489 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 5613232 ep_io_read_bytes 0 ep_io_write_bytes 13292106458 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 395370 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 7339250270 ep_max_checkpoints 2 ep_max_data_size 12582912000 ep_max_item_size 20971520 ep_max_size 12582912000 ep_max_txn_size 10000 ep_max_vbuckets 1024 ep_mem_high_wat 9437184000 ep_mem_high_wat 9437184000 ep_mem_low_wat 7549747200 ep_mem_low_wat 7549747200 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 254245620 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1352919382 ep_storage_age 0 ep_storage_age_highwat 3 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 7303250246 ep_total_del_items 3 ep_total_enqueued 7199710 ep_total_new_items 3000002 ep_total_persisted 5613232 ep_uncommitted_items 0 ep_value_size 7091900551 ep_vb0 0 ep_vb_snapshot_total 2539 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_135_g40a391b ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 1431 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 7921143560 pid 29361 pointer_size 64 rejected_conns 0 rusage_system 92.752899 rusage_user 634.890481 tap_checkpoint_end_received 1482 tap_checkpoint_end_sent 451 tap_checkpoint_start_received 3054 tap_checkpoint_start_sent 512 tap_connect_received 1 tap_delete_received 2 tap_mutation_received 7341122 tap_mutation_sent 1321512 tap_opaque_received 7168 tap_opaque_sent 513 tap_vbucket_set_received 1024 threads 4 time 1352920592 total_connections 5133 uptime 1212 vb_active_curr_items 1500466 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 12857056 vb_active_itm_memory 3653305966 vb_active_meta_data_memory 118536814 vb_active_num 512 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1500466 vb_active_ops_create 1500466 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 639616 vb_active_perc_mem_resident 100 vb_active_queue_age 0 vb_active_queue_drain 4284459 vb_active_queue_fill 2962833 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 1499536 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 12857152 vb_replica_itm_memory 3649944280 vb_replica_meta_data_memory 118463344 vb_replica_num 512 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 1499536 vb_replica_ops_create 1499536 vb_replica_ops_delete 3 vb_replica_ops_reject 0 vb_replica_ops_update 1973611 vb_replica_perc_mem_resident 100 vb_replica_queue_age 0 vb_replica_queue_drain 5479277 vb_replica_queue_fill 4236877 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-14T11:17:22.074,ns_1@10.2.1.58:ns_doctor<0.1953.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,920637,75739}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,217412432}, {processes,168559760}, {processes_used,168535944}, {system,48852672}, {atom,1334153}, {atom_used,1326649}, {binary,9019072}, {code,13390578}, {ets,6308880}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7449526272}, {buffered_memory,216076288}, {free_memory,16940744704}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1787496,5000}}, {context_switches,{9604166,0}}, {garbage_collection,{2293455,4513641945,0}}, {io,{{input,218535755},{output,18001110588}}}, {reductions,{1169822719,1013066}}, {run_queue,0}, {runtime,{272800,550}}]}, {system_stats, [{cpu_utilization_rate,2.743142144638404}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7287798818}, {couch_docs_data_size,7285575572}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7921143576}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1787}, {memory_data,{33657663488,16714379264,{<0.367.0>,3514472}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16543696 kB\nBuffers: 211012 kB\nCached: 7274928 kB\nSwapCached: 0 kB\nActive: 8803716 kB\nInactive: 7223888 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16543696 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 7644 kB\nWriteback: 0 kB\nAnonPages: 8541752 kB\nMapped: 15592 kB\nSlab: 241016 kB\nPageTables: 21744 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10238448 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920229,826001}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17879.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,920641,477941}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,213908320}, {processes,164020680}, {processes_used,164006304}, {system,49887640}, {atom,1338193}, {atom_used,1334188}, {binary,8897464}, {code,13493302}, {ets,7432944}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7482675200}, {buffered_memory,248147968}, {free_memory,16590467072}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1789942,5000}}, {context_switches,{12973194,0}}, {garbage_collection,{2702114,6283804604,0}}, {io,{{input,313787400},{output,12949680552}}}, {reductions,{1986376109,2621596}}, {run_queue,0}, {runtime,{319760,560}}]}, {system_stats, [{cpu_utilization_rate,2.756892230576441}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7288613936}, {couch_docs_data_size,7286342852}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,8098628776}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1789}, {memory_data,{33657663488,17019760640,{<17926.676.0>,4714984}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16201628 kB\nBuffers: 242332 kB\nCached: 7307300 kB\nSwapCached: 0 kB\nActive: 9088140 kB\nInactive: 7248516 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16201628 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 3488 kB\nWriteback: 0 kB\nAnonPages: 8787084 kB\nMapped: 16696 kB\nSlab: 273376 kB\nPageTables: 22492 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10483512 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-14T11:18:13.092,ns_1@10.2.1.58:<0.2072.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,48,40}}) Stats for bucket "default": accepting_conns 1 auth_cmds 0 auth_errors 0 bucket_active_conns 1 bucket_conns 10 bytes 7921143544 bytes_read 17748191705 bytes_written 3530415176 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 0 conn_yields 33389 connection_structures 5000 curr_connections 13 curr_conns_on_port_11209 10 curr_conns_on_port_11210 1 curr_items 1500466 curr_items_tot 3000002 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-15 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data2/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 28828 ep_commit_time 3 ep_commit_time_total 379413 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 1 ep_data_age_highwat 5 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data2/default ep_degraded_mode 0 ep_diskqueue_drain 9763736 ep_diskqueue_fill 7199710 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 489 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 5613232 ep_io_read_bytes 0 ep_io_write_bytes 13292106458 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 395370 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 7339250270 ep_max_checkpoints 2 ep_max_data_size 12582912000 ep_max_item_size 20971520 ep_max_size 12582912000 ep_max_txn_size 10000 ep_max_vbuckets 1024 ep_mem_high_wat 9437184000 ep_mem_high_wat 9437184000 ep_mem_low_wat 7549747200 ep_mem_low_wat 7549747200 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 254245620 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1352919382 ep_storage_age 0 ep_storage_age_highwat 3 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 7303250246 ep_total_del_items 3 ep_total_enqueued 7199710 ep_total_new_items 3000002 ep_total_persisted 5613232 ep_uncommitted_items 0 ep_value_size 7091900551 ep_vb0 0 ep_vb_snapshot_total 2539 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_135_g40a391b ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 1431 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 7921143544 pid 29361 pointer_size 64 rejected_conns 0 rusage_system 92.777895 rusage_user 635.561379 tap_checkpoint_end_received 1482 tap_checkpoint_end_sent 451 tap_checkpoint_start_received 3054 tap_checkpoint_start_sent 512 tap_connect_received 1 tap_delete_received 2 tap_mutation_received 7341122 tap_mutation_sent 1321512 tap_opaque_received 7168 tap_opaque_sent 513 tap_vbucket_set_received 1024 threads 4 time 1352920692 total_connections 5133 uptime 1312 vb_active_curr_items 1500466 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 12857056 vb_active_itm_memory 3653305966 vb_active_meta_data_memory 118536814 vb_active_num 512 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1500466 vb_active_ops_create 1500466 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 639616 vb_active_perc_mem_resident 100 vb_active_queue_age 0 vb_active_queue_drain 4284459 vb_active_queue_fill 2962833 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 1499536 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 12857152 vb_replica_itm_memory 3649944280 vb_replica_meta_data_memory 118463344 vb_replica_num 512 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 1499536 vb_replica_ops_create 1499536 vb_replica_ops_delete 3 vb_replica_ops_reject 0 vb_replica_ops_update 1973611 vb_replica_perc_mem_resident 100 vb_replica_queue_age 0 vb_replica_queue_drain 5479277 vb_replica_queue_fill 4236877 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-14T11:18:22.074,ns_1@10.2.1.58:ns_doctor<0.1953.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,920697,74717}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,241079448}, {processes,187879232}, {processes_used,187854616}, {system,53200216}, {atom,1334153}, {atom_used,1326813}, {binary,13178728}, {code,13399752}, {ets,6487048}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7448031232}, {buffered_memory,217260032}, {free_memory,16915619840}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1847495,4999}}, {context_switches,{9800518,0}}, {garbage_collection,{2304677,4561856756,0}}, {io,{{input,225279044},{output,18006998606}}}, {reductions,{1196938053,1009782}}, {run_queue,0}, {runtime,{279540,500}}]}, {system_stats, [{cpu_utilization_rate,8.521303258145364}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7287798818}, {couch_docs_data_size,7285575572}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7921143576}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1847}, {memory_data,{33657663488,16716537856,{<0.367.0>,3514472}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16520044 kB\nBuffers: 212168 kB\nCached: 7273468 kB\nSwapCached: 0 kB\nActive: 8827112 kB\nInactive: 7222848 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16520044 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 6260 kB\nWriteback: 0 kB\nAnonPages: 8564512 kB\nMapped: 17044 kB\nSlab: 241328 kB\nPageTables: 22272 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10279992 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920229,826001}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17879.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,920701,479069}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,234431640}, {processes,180162880}, {processes_used,180148504}, {system,54268760}, {atom,1349617}, {atom_used,1347167}, {binary,13025568}, {code,13553851}, {ets,7615832}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7592062976}, {buffered_memory,248602624}, {free_memory,16469618688}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1849943,5000}}, {context_switches,{13194536,0}}, {garbage_collection,{2713049,6429702678,0}}, {io,{{input,326044203},{output,12976039030}}}, {reductions,{2067777203,2592406}}, {run_queue,0}, {runtime,{328720,550}}]}, {system_stats, [{cpu_utilization_rate,3.0},{swap_total,10585923584},{swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7288613936}, {couch_docs_data_size,7286342852}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,8098628776}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1849}, {memory_data,{33657663488,17068589056,{<17926.368.0>,4114672}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16083612 kB\nBuffers: 242776 kB\nCached: 7414124 kB\nSwapCached: 0 kB\nActive: 9264800 kB\nInactive: 7190104 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16083612 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 89420 kB\nWriteback: 0 kB\nAnonPages: 8798044 kB\nMapped: 15608 kB\nSlab: 273544 kB\nPageTables: 22168 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10489696 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-14T11:19:22.074,ns_1@10.2.1.58:ns_doctor<0.1953.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,920757,75746}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,239167176}, {processes,185866496}, {processes_used,185841880}, {system,53300680}, {atom,1334153}, {atom_used,1326813}, {binary,13104488}, {code,13399752}, {ets,6661672}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7500627968}, {buffered_memory,243560448}, {free_memory,16816615424}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1907496,5000}}, {context_switches,{9995418,0}}, {garbage_collection,{2315104,4605704482,0}}, {io,{{input,228668137},{output,18009772918}}}, {reductions,{1222405200,1009435}}, {run_queue,0}, {runtime,{285630,500}}]}, {system_stats, [{cpu_utilization_rate,5.012531328320802}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7287798818}, {couch_docs_data_size,7285575572}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7921143576}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1907}, {memory_data,{33657663488,16779382784,{<0.11.0>,13466536}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16422476 kB\nBuffers: 237852 kB\nCached: 7324832 kB\nSwapCached: 0 kB\nActive: 8900336 kB\nInactive: 7236660 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16422476 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 50920 kB\nWriteback: 0 kB\nAnonPages: 8574576 kB\nMapped: 16668 kB\nSlab: 252212 kB\nPageTables: 22216 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10349944 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920229,826001}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17879.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,920761,478583}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,227716096}, {processes,173470496}, {processes_used,173456120}, {system,54245600}, {atom,1349617}, {atom_used,1347167}, {binary,12827984}, {code,13553851}, {ets,7790312}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7593234432}, {buffered_memory,248700928}, {free_memory,16470429696}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1909942,5000}}, {context_switches,{13392681,0}}, {garbage_collection,{2724100,6499281011,0}}, {io,{{input,329602353},{output,12978066023}}}, {reductions,{2098829793,2614492}}, {run_queue,0}, {runtime,{335020,560}}]}, {system_stats, [{cpu_utilization_rate,3.2418952618453867}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7288613936}, {couch_docs_data_size,7286342852}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,8098628776}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1909}, {memory_data,{33657663488,17187868672,{<17926.11.0>,13466536}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16084404 kB\nBuffers: 242872 kB\nCached: 7415268 kB\nSwapCached: 0 kB\nActive: 9263972 kB\nInactive: 7191136 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16084404 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 44004 kB\nWriteback: 0 kB\nAnonPages: 8796880 kB\nMapped: 15608 kB\nSlab: 272688 kB\nPageTables: 22160 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10486012 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-14T11:19:53.092,ns_1@10.2.1.58:<0.2072.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,48,40}}) Stats for bucket "default": accepting_conns 1 auth_cmds 0 auth_errors 0 bucket_active_conns 1 bucket_conns 10 bytes 7921143544 bytes_read 17748203396 bytes_written 3535300189 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 0 conn_yields 33389 connection_structures 5000 curr_connections 13 curr_conns_on_port_11209 10 curr_conns_on_port_11210 1 curr_items 1500466 curr_items_tot 3000002 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-15 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data2/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 28828 ep_commit_time 3 ep_commit_time_total 379413 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 1 ep_data_age_highwat 5 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data2/default ep_degraded_mode 0 ep_diskqueue_drain 9763736 ep_diskqueue_fill 7199710 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 489 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 5613232 ep_io_read_bytes 0 ep_io_write_bytes 13292106458 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 395370 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 7339250270 ep_max_checkpoints 2 ep_max_data_size 12582912000 ep_max_item_size 20971520 ep_max_size 12582912000 ep_max_txn_size 10000 ep_max_vbuckets 1024 ep_mem_high_wat 9437184000 ep_mem_high_wat 9437184000 ep_mem_low_wat 7549747200 ep_mem_low_wat 7549747200 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 254245620 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1352919382 ep_storage_age 0 ep_storage_age_highwat 3 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 7303250246 ep_total_del_items 3 ep_total_enqueued 7199710 ep_total_new_items 3000002 ep_total_persisted 5613232 ep_uncommitted_items 0 ep_value_size 7091900551 ep_vb0 0 ep_vb_snapshot_total 2539 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_135_g40a391b ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 1431 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 7921143544 pid 29361 pointer_size 64 rejected_conns 0 rusage_system 92.812890 rusage_user 636.595222 tap_checkpoint_end_received 1482 tap_checkpoint_end_sent 451 tap_checkpoint_start_received 3054 tap_checkpoint_start_sent 512 tap_connect_received 1 tap_delete_received 2 tap_mutation_received 7341122 tap_mutation_sent 1321512 tap_opaque_received 7168 tap_opaque_sent 513 tap_vbucket_set_received 1024 threads 4 time 1352920792 total_connections 5142 uptime 1412 vb_active_curr_items 1500466 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 12857056 vb_active_itm_memory 3653305966 vb_active_meta_data_memory 118536814 vb_active_num 512 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1500466 vb_active_ops_create 1500466 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 639616 vb_active_perc_mem_resident 100 vb_active_queue_age 0 vb_active_queue_drain 4284459 vb_active_queue_fill 2962833 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 1499536 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 12857152 vb_replica_itm_memory 3649944280 vb_replica_meta_data_memory 118463344 vb_replica_num 512 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 1499536 vb_replica_ops_create 1499536 vb_replica_ops_delete 3 vb_replica_ops_reject 0 vb_replica_ops_update 1973611 vb_replica_perc_mem_resident 100 vb_replica_queue_age 0 vb_replica_queue_drain 5479277 vb_replica_queue_fill 4236877 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-14T11:20:22.074,ns_1@10.2.1.58:ns_doctor<0.1953.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,920817,75734}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,265836520}, {processes,212378856}, {processes_used,212354240}, {system,53457664}, {atom,1337385}, {atom_used,1331436}, {binary,13135672}, {code,13451143}, {ets,6731520}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7528583168}, {buffered_memory,243830784}, {free_memory,16798756864}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1967495,4999}}, {context_switches,{10212760,0}}, {garbage_collection,{2326273,4726548920,0}}, {io,{{input,237485625},{output,18032381057}}}, {reductions,{1299645919,997022}}, {run_queue,0}, {runtime,{294480,510}}]}, {system_stats, [{cpu_utilization_rate,2.743142144638404}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7287798818}, {couch_docs_data_size,7285575572}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7921143576}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1967}, {memory_data,{33657663488,16862621696,{<0.11.0>,43123400}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16405036 kB\nBuffers: 238116 kB\nCached: 7352132 kB\nSwapCached: 0 kB\nActive: 8963272 kB\nInactive: 7192520 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16405036 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 35452 kB\nWriteback: 0 kB\nAnonPages: 8565224 kB\nMapped: 15604 kB\nSlab: 252004 kB\nPageTables: 21824 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10299592 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920229,826001}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17879.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,920821,477905}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,262638680}, {processes,208377880}, {processes_used,208363504}, {system,54260800}, {atom,1349617}, {atom_used,1347167}, {binary,12777360}, {code,13553851}, {ets,7855720}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7594364928}, {buffered_memory,248799232}, {free_memory,16470683648}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1969942,5000}}, {context_switches,{13591828,0}}, {garbage_collection,{2735133,6574011487,0}}, {io,{{input,333055677},{output,12980113570}}}, {reductions,{2131335787,2605178}}, {run_queue,0}, {runtime,{341610,600}}]}, {system_stats, [{cpu_utilization_rate,2.5},{swap_total,10585923584},{swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7288613936}, {couch_docs_data_size,7286342852}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,8098628776}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1969}, {memory_data,{33657663488,17189773312,{<17926.11.0>,43123400}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16084652 kB\nBuffers: 242968 kB\nCached: 7416372 kB\nSwapCached: 0 kB\nActive: 9263624 kB\nInactive: 7192144 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16084652 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 16136 kB\nWriteback: 0 kB\nAnonPages: 8796252 kB\nMapped: 15608 kB\nSlab: 272800 kB\nPageTables: 22136 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10519796 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-14T11:21:22.074,ns_1@10.2.1.58:ns_doctor<0.1953.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,920877,75720}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,265784440}, {processes,212161400}, {processes_used,212135184}, {system,53623040}, {atom,1337385}, {atom_used,1331436}, {binary,13115632}, {code,13451143}, {ets,6917128}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7529447424}, {buffered_memory,244117504}, {free_memory,16799899648}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2027496,5000}}, {context_switches,{10408681,0}}, {garbage_collection,{2337430,4770834013,0}}, {io,{{input,244328665},{output,18038326926}}}, {reductions,{1325492535,1009348}}, {run_queue,0}, {runtime,{300990,490}}]}, {system_stats, [{cpu_utilization_rate,2.255639097744361}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7287798818}, {couch_docs_data_size,7285575572}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7921143576}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2027}, {memory_data,{33657663488,16857890816,{<0.11.0>,43123400}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16406152 kB\nBuffers: 238396 kB\nCached: 7352976 kB\nSwapCached: 0 kB\nActive: 8961568 kB\nInactive: 7193548 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16406152 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 8224 kB\nWriteback: 0 kB\nAnonPages: 8563508 kB\nMapped: 15604 kB\nSlab: 251620 kB\nPageTables: 21824 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10299772 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920229,826001}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17879.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,920881,478571}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,262361856}, {processes,207630144}, {processes_used,207615768}, {system,54731712}, {atom,1349617}, {atom_used,1347167}, {binary,13071480}, {code,13553851}, {ets,8032952}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7595425792}, {buffered_memory,248999936}, {free_memory,16466137088}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2029942,5000}}, {context_switches,{13790955,0}}, {garbage_collection,{2745476,6643729521,0}}, {io,{{input,339966990},{output,12985382446}}}, {reductions,{2162482022,2614001}}, {run_queue,0}, {runtime,{348050,540}}]}, {system_stats, [{cpu_utilization_rate,2.5},{swap_total,10585923584},{swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7288613936}, {couch_docs_data_size,7286342852}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,8098628776}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2029}, {memory_data,{33657663488,17188122624,{<17926.11.0>,43123400}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16080212 kB\nBuffers: 243164 kB\nCached: 7417408 kB\nSwapCached: 0 kB\nActive: 9266164 kB\nInactive: 7193272 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16080212 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 5964 kB\nWriteback: 0 kB\nAnonPages: 8799020 kB\nMapped: 15608 kB\nSlab: 272588 kB\nPageTables: 22172 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10522844 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-14T11:21:33.092,ns_1@10.2.1.58:<0.2072.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,48,40}}) Stats for bucket "default": accepting_conns 1 auth_cmds 0 auth_errors 0 bucket_active_conns 1 bucket_conns 10 bytes 7921143560 bytes_read 17748214366 bytes_written 3538284685 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 0 conn_yields 33389 connection_structures 5000 curr_connections 13 curr_conns_on_port_11209 10 curr_conns_on_port_11210 1 curr_items 1500466 curr_items_tot 3000002 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-15 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data2/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 28828 ep_commit_time 3 ep_commit_time_total 379413 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 1 ep_data_age_highwat 5 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data2/default ep_degraded_mode 0 ep_diskqueue_drain 9763736 ep_diskqueue_fill 7199710 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 489 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 5613232 ep_io_read_bytes 0 ep_io_write_bytes 13292106458 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 395370 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 7339250270 ep_max_checkpoints 2 ep_max_data_size 12582912000 ep_max_item_size 20971520 ep_max_size 12582912000 ep_max_txn_size 10000 ep_max_vbuckets 1024 ep_mem_high_wat 9437184000 ep_mem_high_wat 9437184000 ep_mem_low_wat 7549747200 ep_mem_low_wat 7549747200 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 254245620 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1352919382 ep_storage_age 0 ep_storage_age_highwat 3 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 7303250246 ep_total_del_items 3 ep_total_enqueued 7199710 ep_total_new_items 3000002 ep_total_persisted 5613232 ep_uncommitted_items 0 ep_value_size 7091900551 ep_vb0 0 ep_vb_snapshot_total 2539 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_135_g40a391b ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 1431 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 7921143560 pid 29361 pointer_size 64 rejected_conns 0 rusage_system 92.836886 rusage_user 637.239124 tap_checkpoint_end_received 1482 tap_checkpoint_end_sent 451 tap_checkpoint_start_received 3054 tap_checkpoint_start_sent 512 tap_connect_received 1 tap_delete_received 2 tap_mutation_received 7341122 tap_mutation_sent 1321512 tap_opaque_received 7168 tap_opaque_sent 513 tap_vbucket_set_received 1024 threads 4 time 1352920892 total_connections 5142 uptime 1512 vb_active_curr_items 1500466 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 12857056 vb_active_itm_memory 3653305966 vb_active_meta_data_memory 118536814 vb_active_num 512 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1500466 vb_active_ops_create 1500466 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 639616 vb_active_perc_mem_resident 100 vb_active_queue_age 0 vb_active_queue_drain 4284459 vb_active_queue_fill 2962833 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 1499536 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 12857152 vb_replica_itm_memory 3649944280 vb_replica_meta_data_memory 118463344 vb_replica_num 512 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 1499536 vb_replica_ops_create 1499536 vb_replica_ops_delete 3 vb_replica_ops_reject 0 vb_replica_ops_update 1973611 vb_replica_perc_mem_resident 100 vb_replica_queue_age 0 vb_replica_queue_drain 5479277 vb_replica_queue_fill 4236877 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-14T11:22:22.074,ns_1@10.2.1.58:ns_doctor<0.1953.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,920937,75690}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,261228304}, {processes,207559816}, {processes_used,207533600}, {system,53668488}, {atom,1337385}, {atom_used,1331436}, {binary,13097640}, {code,13451143}, {ets,6980272}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7530569728}, {buffered_memory,244281344}, {free_memory,16797868032}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2087495,4999}}, {context_switches,{10603516,0}}, {garbage_collection,{2348454,4814826987,0}}, {io,{{input,247703691},{output,18041110259}}}, {reductions,{1351020345,1032589}}, {run_queue,0}, {runtime,{307220,510}}]}, {system_stats, [{cpu_utilization_rate,2.506265664160401}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7287798818}, {couch_docs_data_size,7285575572}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7921143576}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2087}, {memory_data,{33657663488,16858152960,{<0.11.0>,43123400}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16404168 kB\nBuffers: 238556 kB\nCached: 7354072 kB\nSwapCached: 0 kB\nActive: 8962276 kB\nInactive: 7194704 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16404168 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 132 kB\nWriteback: 0 kB\nAnonPages: 8564204 kB\nMapped: 15604 kB\nSlab: 251600 kB\nPageTables: 21828 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10300032 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920229,826001}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17879.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,920941,478455}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,260703672}, {processes,206088792}, {processes_used,206074416}, {system,54614880}, {atom,1349617}, {atom_used,1347167}, {binary,12882640}, {code,13553851}, {ets,8104632}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7596617728}, {buffered_memory,249098240}, {free_memory,16466137088}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2089942,5000}}, {context_switches,{13989084,0}}, {garbage_collection,{2755690,6712946873,0}}, {io,{{input,343330917},{output,12987412413}}}, {reductions,{2193434095,2605414}}, {run_queue,0}, {runtime,{354270,550}}]}, {system_stats, [{cpu_utilization_rate,2.75}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7288613936}, {couch_docs_data_size,7286342852}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,8098628776}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2089}, {memory_data,{33657663488,17192923136,{<17926.11.0>,43123400}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16080212 kB\nBuffers: 243260 kB\nCached: 7418572 kB\nSwapCached: 0 kB\nActive: 9265444 kB\nInactive: 7194304 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16080212 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 204 kB\nWriteback: 0 kB\nAnonPages: 8797836 kB\nMapped: 15608 kB\nSlab: 272424 kB\nPageTables: 22172 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10520720 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-14T11:23:13.092,ns_1@10.2.1.58:<0.2072.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,48,40}}) Stats for bucket "default": accepting_conns 1 auth_cmds 0 auth_errors 0 bucket_active_conns 1 bucket_conns 10 bytes 7921143544 bytes_read 17748225336 bytes_written 3541269262 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 0 conn_yields 33389 connection_structures 5000 curr_connections 13 curr_conns_on_port_11209 10 curr_conns_on_port_11210 1 curr_items 1500466 curr_items_tot 3000002 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-15 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data2/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 28828 ep_commit_time 3 ep_commit_time_total 379413 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 1 ep_data_age_highwat 5 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data2/default ep_degraded_mode 0 ep_diskqueue_drain 9763736 ep_diskqueue_fill 7199710 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 489 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 5613232 ep_io_read_bytes 0 ep_io_write_bytes 13292106458 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 395370 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 7339250270 ep_max_checkpoints 2 ep_max_data_size 12582912000 ep_max_item_size 20971520 ep_max_size 12582912000 ep_max_txn_size 10000 ep_max_vbuckets 1024 ep_mem_high_wat 9437184000 ep_mem_high_wat 9437184000 ep_mem_low_wat 7549747200 ep_mem_low_wat 7549747200 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 254245620 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1352919382 ep_storage_age 0 ep_storage_age_highwat 3 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 7303250246 ep_total_del_items 3 ep_total_enqueued 7199710 ep_total_new_items 3000002 ep_total_persisted 5613232 ep_uncommitted_items 0 ep_value_size 7091900551 ep_vb0 0 ep_vb_snapshot_total 2539 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_135_g40a391b ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 1431 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 7921143544 pid 29361 pointer_size 64 rejected_conns 0 rusage_system 92.869881 rusage_user 637.897024 tap_checkpoint_end_received 1482 tap_checkpoint_end_sent 451 tap_checkpoint_start_received 3054 tap_checkpoint_start_sent 512 tap_connect_received 1 tap_delete_received 2 tap_mutation_received 7341122 tap_mutation_sent 1321512 tap_opaque_received 7168 tap_opaque_sent 513 tap_vbucket_set_received 1024 threads 4 time 1352920992 total_connections 5142 uptime 1612 vb_active_curr_items 1500466 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 12857056 vb_active_itm_memory 3653305966 vb_active_meta_data_memory 118536814 vb_active_num 512 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1500466 vb_active_ops_create 1500466 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 639616 vb_active_perc_mem_resident 100 vb_active_queue_age 0 vb_active_queue_drain 4284459 vb_active_queue_fill 2962833 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 1499536 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 12857152 vb_replica_itm_memory 3649944280 vb_replica_meta_data_memory 118463344 vb_replica_num 512 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 1499536 vb_replica_ops_create 1499536 vb_replica_ops_delete 3 vb_replica_ops_reject 0 vb_replica_ops_update 1973611 vb_replica_perc_mem_resident 100 vb_replica_queue_age 0 vb_replica_queue_drain 5479277 vb_replica_queue_fill 4236877 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-14T11:23:22.074,ns_1@10.2.1.58:ns_doctor<0.1953.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,920997,75678}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,266475984}, {processes,212696816}, {processes_used,212670600}, {system,53779168}, {atom,1337385}, {atom_used,1331436}, {binary,13033528}, {code,13451143}, {ets,7155104}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7531683840}, {buffered_memory,244449280}, {free_memory,16797741056}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2147495,4999}}, {context_switches,{10798405,0}}, {garbage_collection,{2359181,4858661290,0}}, {io,{{input,250997072},{output,18043923539}}}, {reductions,{1376447601,1014813}}, {run_queue,0}, {runtime,{313600,530}}]}, {system_stats, [{cpu_utilization_rate,2.506265664160401}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7287798818}, {couch_docs_data_size,7285575572}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7921143576}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2147}, {memory_data,{33657663488,16861073408,{<0.11.0>,43123400}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16404044 kB\nBuffers: 238720 kB\nCached: 7355160 kB\nSwapCached: 0 kB\nActive: 8961420 kB\nInactive: 7195804 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16404044 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 600 kB\nWriteback: 0 kB\nAnonPages: 8563204 kB\nMapped: 15604 kB\nSlab: 251504 kB\nPageTables: 21812 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10295704 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920229,826001}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17879.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,921001,477891}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,267099880}, {processes,212040032}, {processes_used,212025656}, {system,55059848}, {atom,1349617}, {atom_used,1347167}, {binary,13152744}, {code,13553851}, {ets,8279488}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7597727744}, {buffered_memory,249159680}, {free_memory,16463851520}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2149942,4999}}, {context_switches,{14187171,0}}, {garbage_collection,{2765499,6781469434,0}}, {io,{{input,346895733},{output,12989347289}}}, {reductions,{2224440429,2605905}}, {run_queue,0}, {runtime,{360550,550}}]}, {system_stats, [{cpu_utilization_rate,2.5},{swap_total,10585923584},{swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7288613936}, {couch_docs_data_size,7286342852}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,8098628776}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2149}, {memory_data,{33657663488,17192923136,{<17926.11.0>,43123400}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16077980 kB\nBuffers: 243320 kB\nCached: 7419656 kB\nSwapCached: 0 kB\nActive: 9267216 kB\nInactive: 7195216 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16077980 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 96 kB\nWriteback: 0 kB\nAnonPages: 8799596 kB\nMapped: 15608 kB\nSlab: 272380 kB\nPageTables: 22172 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10521772 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-14T11:24:22.074,ns_1@10.2.1.58:ns_doctor<0.1953.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,921057,75746}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,265956368}, {processes,211816688}, {processes_used,211790472}, {system,54139680}, {atom,1337385}, {atom_used,1331436}, {binary,13217128}, {code,13451143}, {ets,7332440}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7527321600}, {buffered_memory,244707328}, {free_memory,16797360128}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2207495,4999}}, {context_switches,{10993708,0}}, {garbage_collection,{2369315,4903389302,0}}, {io,{{input,257727994},{output,18050462053}}}, {reductions,{1402001079,1009837}}, {run_queue,0}, {runtime,{320120,550}}]}, {system_stats, [{cpu_utilization_rate,2.743142144638404}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7287798818}, {couch_docs_data_size,7285575572}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7921143576}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2207}, {memory_data,{33657663488,16862334976,{<0.11.0>,43123400}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16403672 kB\nBuffers: 238972 kB\nCached: 7350900 kB\nSwapCached: 0 kB\nActive: 8965844 kB\nInactive: 7192240 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16403672 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 344 kB\nWriteback: 0 kB\nAnonPages: 8567980 kB\nMapped: 15604 kB\nSlab: 251520 kB\nPageTables: 21820 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10297824 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920229,826001}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17879.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,921061,477809}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,261688744}, {processes,206738376}, {processes_used,206724000}, {system,54950368}, {atom,1349617}, {atom_used,1347167}, {binary,12866536}, {code,13553851}, {ets,8456480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7593119744}, {buffered_memory,249344000}, {free_memory,16465883136}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2209942,4999}}, {context_switches,{14385875,0}}, {garbage_collection,{2776429,6852702962,0}}, {io,{{input,353790642},{output,12996626459}}}, {reductions,{2255842867,2610298}}, {run_queue,0}, {runtime,{367130,560}}]}, {system_stats, [{cpu_utilization_rate,3.0},{swap_total,10585923584},{swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7288613936}, {couch_docs_data_size,7286342852}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,8098628776}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2209}, {memory_data,{33657663488,17195843584,{<17926.11.0>,43123400}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16079964 kB\nBuffers: 243500 kB\nCached: 7415156 kB\nSwapCached: 0 kB\nActive: 9268864 kB\nInactive: 7191488 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16079964 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 292 kB\nWriteback: 0 kB\nAnonPages: 8801828 kB\nMapped: 15608 kB\nSlab: 272336 kB\nPageTables: 22176 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10524336 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-14T11:24:53.093,ns_1@10.2.1.58:<0.2072.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,48,41}}) Stats for bucket "default": accepting_conns 1 auth_cmds 0 auth_errors 0 bucket_active_conns 1 bucket_conns 10 bytes 7921143560 bytes_read 17748236306 bytes_written 3544253882 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 0 conn_yields 33389 connection_structures 5000 curr_connections 13 curr_conns_on_port_11209 10 curr_conns_on_port_11210 1 curr_items 1500466 curr_items_tot 3000002 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-15 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data2/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 28828 ep_commit_time 3 ep_commit_time_total 379413 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 1 ep_data_age_highwat 5 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data2/default ep_degraded_mode 0 ep_diskqueue_drain 9763736 ep_diskqueue_fill 7199710 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 489 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 5613232 ep_io_read_bytes 0 ep_io_write_bytes 13292106458 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 395370 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 7339250270 ep_max_checkpoints 2 ep_max_data_size 12582912000 ep_max_item_size 20971520 ep_max_size 12582912000 ep_max_txn_size 10000 ep_max_vbuckets 1024 ep_mem_high_wat 9437184000 ep_mem_high_wat 9437184000 ep_mem_low_wat 7549747200 ep_mem_low_wat 7549747200 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 254245620 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1352919382 ep_storage_age 0 ep_storage_age_highwat 3 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 7303250246 ep_total_del_items 3 ep_total_enqueued 7199710 ep_total_new_items 3000002 ep_total_persisted 5613232 ep_uncommitted_items 0 ep_value_size 7091900551 ep_vb0 0 ep_vb_snapshot_total 2539 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_135_g40a391b ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 1431 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 7921143560 pid 29361 pointer_size 64 rejected_conns 0 rusage_system 92.895877 rusage_user 638.531928 tap_checkpoint_end_received 1482 tap_checkpoint_end_sent 451 tap_checkpoint_start_received 3054 tap_checkpoint_start_sent 512 tap_connect_received 1 tap_delete_received 2 tap_mutation_received 7341122 tap_mutation_sent 1321512 tap_opaque_received 7168 tap_opaque_sent 513 tap_vbucket_set_received 1024 threads 4 time 1352921092 total_connections 5142 uptime 1712 vb_active_curr_items 1500466 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 12857056 vb_active_itm_memory 3653305966 vb_active_meta_data_memory 118536814 vb_active_num 512 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1500466 vb_active_ops_create 1500466 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 639616 vb_active_perc_mem_resident 100 vb_active_queue_age 0 vb_active_queue_drain 4284459 vb_active_queue_fill 2962833 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 1499536 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 12857152 vb_replica_itm_memory 3649944280 vb_replica_meta_data_memory 118463344 vb_replica_num 512 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 1499536 vb_replica_ops_create 1499536 vb_replica_ops_delete 3 vb_replica_ops_reject 0 vb_replica_ops_update 1973611 vb_replica_perc_mem_resident 100 vb_replica_queue_age 0 vb_replica_queue_drain 5479277 vb_replica_queue_fill 4236877 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-14T11:25:22.074,ns_1@10.2.1.58:ns_doctor<0.1953.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,921117,75960}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,264844312}, {processes,210515952}, {processes_used,210489736}, {system,54328360}, {atom,1337385}, {atom_used,1331436}, {binary,13232368}, {code,13451143}, {ets,7505952}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7528435712}, {buffered_memory,244858880}, {free_memory,16799010816}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2267496,5000}}, {context_switches,{11188607,0}}, {garbage_collection,{2380488,4946908603,0}}, {io,{{input,261031337},{output,18053266223}}}, {reductions,{1427506829,995540}}, {run_queue,0}, {runtime,{326450,500}}]}, {system_stats, [{cpu_utilization_rate,2.506265664160401}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7287798818}, {couch_docs_data_size,7285575572}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7921143576}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2267}, {memory_data,{33657663488,16861192192,{<0.11.0>,43123400}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16405284 kB\nBuffers: 239120 kB\nCached: 7351988 kB\nSwapCached: 0 kB\nActive: 8962316 kB\nInactive: 7193360 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16405284 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 136 kB\nWriteback: 0 kB\nAnonPages: 8564672 kB\nMapped: 15604 kB\nSlab: 251508 kB\nPageTables: 21816 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10291928 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920229,826001}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17879.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,921121,478441}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,263185416}, {processes,207998824}, {processes_used,207984448}, {system,55186592}, {atom,1349617}, {atom_used,1347167}, {binary,12928024}, {code,13553851}, {ets,8631248}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7594233856}, {buffered_memory,249434112}, {free_memory,16465502208}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2269942,4999}}, {context_switches,{14584042,0}}, {garbage_collection,{2787507,6922417451,0}}, {io,{{input,357336657},{output,12998578001}}}, {reductions,{2286918163,2618982}}, {run_queue,0}, {runtime,{373650,580}}]}, {system_stats, [{cpu_utilization_rate,2.756892230576441}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7288613936}, {couch_docs_data_size,7286342852}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,8098628776}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2269}, {memory_data,{33657663488,17192542208,{<17926.11.0>,43123400}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16079592 kB\nBuffers: 243588 kB\nCached: 7416244 kB\nSwapCached: 0 kB\nActive: 9268400 kB\nInactive: 7192516 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16079592 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 164 kB\nWriteback: 0 kB\nAnonPages: 8801144 kB\nMapped: 15608 kB\nSlab: 272300 kB\nPageTables: 22176 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10520388 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-14T11:26:22.074,ns_1@10.2.1.58:ns_doctor<0.1953.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,921177,75971}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,269094920}, {processes,214911736}, {processes_used,214885520}, {system,54183184}, {atom,1337385}, {atom_used,1331436}, {binary,13023080}, {code,13451143}, {ets,7569752}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7529504768}, {buffered_memory,245022720}, {free_memory,16793423872}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2327496,5000}}, {context_switches,{11383421,0}}, {garbage_collection,{2391058,4990051778,0}}, {io,{{input,264425744},{output,18056227741}}}, {reductions,{1452948171,1004365}}, {run_queue,0}, {runtime,{332850,520}}]}, {system_stats, [{cpu_utilization_rate,2.5},{swap_total,10684628992},{swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7287798818}, {couch_docs_data_size,7285575572}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7921143576}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2327}, {memory_data,{33657663488,16860684288,{<0.11.0>,43123400}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16399828 kB\nBuffers: 239280 kB\nCached: 7353032 kB\nSwapCached: 0 kB\nActive: 8967320 kB\nInactive: 7194464 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16399828 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 536 kB\nWriteback: 0 kB\nAnonPages: 8569240 kB\nMapped: 15604 kB\nSlab: 251520 kB\nPageTables: 21820 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10297364 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920229,826001}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17879.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,921181,477614}}, {outgoing_replications_safeness_level,[{"default",green}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,263944296}, {processes,208690512}, {processes_used,208676136}, {system,55253784}, {atom,1349617}, {atom_used,1347167}, {binary,12931880}, {code,13553851}, {ets,8694272}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7595376640}, {buffered_memory,249532416}, {free_memory,16464613376}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2329942,4999}}, {context_switches,{14782228,0}}, {garbage_collection,{2797649,6991096883,0}}, {io,{{input,360894825},{output,13000624098}}}, {reductions,{2317974116,2643958}}, {run_queue,0}, {runtime,{380130,570}}]}, {system_stats, [{cpu_utilization_rate,2.506265664160401}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7288613936}, {couch_docs_data_size,7286342852}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,8098628776}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2329}, {memory_data,{33657663488,17193938944,{<17926.11.0>,43123400}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16078724 kB\nBuffers: 243684 kB\nCached: 7417360 kB\nSwapCached: 0 kB\nActive: 9267916 kB\nInactive: 7193544 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16078724 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 644 kB\nWriteback: 0 kB\nAnonPages: 8800460 kB\nMapped: 15608 kB\nSlab: 272316 kB\nPageTables: 22176 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10523388 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-14T11:26:33.092,ns_1@10.2.1.58:<0.2072.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,48,41}}) Stats for bucket "default": accepting_conns 1 auth_cmds 0 auth_errors 0 bucket_active_conns 1 bucket_conns 10 bytes 7918884344 bytes_read 17748249220 bytes_written 3547239992 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 0 conn_yields 33389 connection_structures 5000 curr_connections 13 curr_conns_on_port_11209 10 curr_conns_on_port_11210 1 curr_items 1500466 curr_items_tot 3000002 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-15 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data2/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 28828 ep_commit_time 3 ep_commit_time_total 379413 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 1 ep_data_age_highwat 5 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data2/default ep_degraded_mode 0 ep_diskqueue_drain 9763736 ep_diskqueue_fill 7199710 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 489 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 5613232 ep_io_read_bytes 0 ep_io_write_bytes 13292106458 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 409989 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 7339250270 ep_max_checkpoints 2 ep_max_data_size 12582912000 ep_max_item_size 20971520 ep_max_size 12582912000 ep_max_txn_size 10000 ep_max_vbuckets 1024 ep_mem_high_wat 9437184000 ep_mem_high_wat 9437184000 ep_mem_low_wat 7549747200 ep_mem_low_wat 7549747200 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 253081428 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1352919382 ep_storage_age 0 ep_storage_age_highwat 3 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 7303250246 ep_total_del_items 3 ep_total_enqueued 7199710 ep_total_new_items 3000002 ep_total_persisted 5613232 ep_uncommitted_items 0 ep_value_size 7091900551 ep_vb0 0 ep_vb_snapshot_total 2542 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_135_g40a391b ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 1431 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 7918884344 pid 29361 pointer_size 64 rejected_conns 0 rusage_system 92.950869 rusage_user 639.233821 tap_checkpoint_end_received 1494 tap_checkpoint_end_sent 458 tap_checkpoint_start_received 3066 tap_checkpoint_start_sent 519 tap_connect_received 1 tap_delete_received 2 tap_mutation_received 7341122 tap_mutation_sent 1321512 tap_opaque_received 7168 tap_opaque_sent 513 tap_vbucket_set_received 1024 threads 4 time 1352921192 total_connections 5142 uptime 1812 vb_active_curr_items 1500466 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 12857056 vb_active_itm_memory 3653305966 vb_active_meta_data_memory 118536814 vb_active_num 512 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1500466 vb_active_ops_create 1500466 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 639616 vb_active_perc_mem_resident 100 vb_active_queue_age 0 vb_active_queue_drain 4284459 vb_active_queue_fill 2962833 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 1499536 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 12857152 vb_replica_itm_memory 3649944280 vb_replica_meta_data_memory 118463344 vb_replica_num 512 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 1499536 vb_replica_ops_create 1499536 vb_replica_ops_delete 3 vb_replica_ops_reject 0 vb_replica_ops_update 1973611 vb_replica_perc_mem_resident 100 vb_replica_queue_age 0 vb_replica_queue_drain 5479277 vb_replica_queue_fill 4236877 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-14T11:27:22.074,ns_1@10.2.1.58:ns_doctor<0.1953.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,921237,75812}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,265303360}, {processes,210778144}, {processes_used,210630488}, {system,54525216}, {atom,1337385}, {atom_used,1331436}, {binary,13336136}, {code,13451143}, {ets,7615240}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7530930176}, {buffered_memory,245305344}, {free_memory,16793550848}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2387496,5000}}, {context_switches,{11583795,0}}, {garbage_collection,{2402756,5035431025,0}}, {io,{{input,271383978},{output,18062194200}}}, {reductions,{1478945929,1026747}}, {run_queue,0}, {runtime,{339540,520}}]}, {system_stats, [{cpu_utilization_rate,2.5},{swap_total,10684628992},{swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7288290338}, {couch_docs_data_size,7285575571}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7866683976}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2387}, {memory_data,{33657663488,16864493568,{<0.11.0>,43123400}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16399952 kB\nBuffers: 239556 kB\nCached: 7354424 kB\nSwapCached: 0 kB\nActive: 8965684 kB\nInactive: 7195192 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16399952 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 540 kB\nWriteback: 0 kB\nAnonPages: 8566552 kB\nMapped: 15604 kB\nSlab: 251764 kB\nPageTables: 21820 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10305468 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920229,826001}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17879.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,921241,477569}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,259199752}, {processes,203914088}, {processes_used,203771832}, {system,55285664}, {atom,1349617}, {atom_used,1347167}, {binary,12943728}, {code,13553851}, {ets,8731608}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7597031424}, {buffered_memory,249679872}, {free_memory,16461438976}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2389942,4999}}, {context_switches,{14986071,0}}, {garbage_collection,{2809421,7061298671,0}}, {io,{{input,367742915},{output,13006018337}}}, {reductions,{2349279659,2621192}}, {run_queue,0}, {runtime,{386860,580}}]}, {system_stats, [{cpu_utilization_rate,2.743142144638404}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7289183282}, {couch_docs_data_size,7286342846}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,8041476184}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2389}, {memory_data,{33657663488,17195081728,{<17926.11.0>,43123400}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16075624 kB\nBuffers: 243828 kB\nCached: 7418976 kB\nSwapCached: 0 kB\nActive: 9270352 kB\nInactive: 7194312 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16075624 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 504 kB\nWriteback: 0 kB\nAnonPages: 8801900 kB\nMapped: 15608 kB\nSlab: 272544 kB\nPageTables: 22184 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10521344 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-14T11:28:13.092,ns_1@10.2.1.58:<0.2072.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,48,41}}) Stats for bucket "default": accepting_conns 1 auth_cmds 0 auth_errors 0 bucket_active_conns 1 bucket_conns 10 bytes 7812671528 bytes_read 17748280480 bytes_written 3550246521 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 0 conn_yields 33389 connection_structures 5000 curr_connections 13 curr_conns_on_port_11209 10 curr_conns_on_port_11210 1 curr_items 1500466 curr_items_tot 3000002 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-15 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data2/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 28828 ep_commit_time 3 ep_commit_time_total 379413 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 1 ep_data_age_highwat 5 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data2/default ep_degraded_mode 0 ep_diskqueue_drain 9763736 ep_diskqueue_fill 7199710 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 489 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 5613232 ep_io_read_bytes 0 ep_io_write_bytes 13292106458 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 1094864 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 7339250270 ep_max_checkpoints 2 ep_max_data_size 12582912000 ep_max_item_size 20971520 ep_max_size 12582912000 ep_max_txn_size 10000 ep_max_vbuckets 1024 ep_mem_high_wat 9437184000 ep_mem_high_wat 9437184000 ep_mem_low_wat 7549747200 ep_mem_low_wat 7549747200 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 198119490 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1352919382 ep_storage_age 0 ep_storage_age_highwat 3 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 7303250246 ep_total_del_items 3 ep_total_enqueued 7199710 ep_total_new_items 3000002 ep_total_persisted 5613232 ep_uncommitted_items 0 ep_value_size 7091900551 ep_vb0 0 ep_vb_snapshot_total 2583 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_135_g40a391b ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 1431 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 7812671528 pid 29361 pointer_size 64 rejected_conns 0 rusage_system 93.184833 rusage_user 640.934563 tap_checkpoint_end_received 1601 tap_checkpoint_end_sent 582 tap_checkpoint_start_received 3173 tap_checkpoint_start_sent 643 tap_connect_received 1 tap_delete_received 2 tap_mutation_received 7341122 tap_mutation_sent 1321512 tap_opaque_received 7168 tap_opaque_sent 513 tap_vbucket_set_received 1024 threads 4 time 1352921292 total_connections 5142 uptime 1912 vb_active_curr_items 1500466 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 12857056 vb_active_itm_memory 3653305966 vb_active_meta_data_memory 118536814 vb_active_num 512 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1500466 vb_active_ops_create 1500466 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 639616 vb_active_perc_mem_resident 100 vb_active_queue_age 0 vb_active_queue_drain 4284459 vb_active_queue_fill 2962833 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 1499536 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 12857152 vb_replica_itm_memory 3649944280 vb_replica_meta_data_memory 118463344 vb_replica_num 512 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 1499536 vb_replica_ops_create 1499536 vb_replica_ops_delete 3 vb_replica_ops_reject 0 vb_replica_ops_update 1973611 vb_replica_perc_mem_resident 100 vb_replica_queue_age 0 vb_replica_queue_drain 5479277 vb_replica_queue_fill 4236877 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-14T11:28:22.074,ns_1@10.2.1.58:ns_doctor<0.1953.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,921297,76098}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,262187128}, {processes,208209184}, {processes_used,207948368}, {system,53977944}, {atom,1337385}, {atom_used,1331436}, {binary,12864936}, {code,13451143}, {ets,7553928}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7532572672}, {buffered_memory,245456896}, {free_memory,16791900160}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2447496,5000}}, {context_switches,{11782661,0}}, {garbage_collection,{2414728,5079844945,0}}, {io,{{input,274782351},{output,18064903513}}}, {reductions,{1504586808,1040874}}, {run_queue,0}, {runtime,{346230,550}}]}, {system_stats, [{cpu_utilization_rate,2.506265664160401}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7288822820}, {couch_docs_data_size,7285575570}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7807653496}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2447}, {memory_data,{33657663488,16864239616,{<0.11.0>,43123400}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16398340 kB\nBuffers: 239704 kB\nCached: 7356028 kB\nSwapCached: 0 kB\nActive: 8965796 kB\nInactive: 7195968 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16398340 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 288 kB\nWriteback: 0 kB\nAnonPages: 8566092 kB\nMapped: 15604 kB\nSlab: 251696 kB\nPageTables: 21828 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10295704 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920229,826001}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17879.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,921301,478324}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,265449296}, {processes,209789576}, {processes_used,209534160}, {system,55659720}, {atom,1349617}, {atom_used,1347167}, {binary,13394872}, {code,13553851}, {ets,8669400}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7598657536}, {buffered_memory,249753600}, {free_memory,16459280384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2449942,5000}}, {context_switches,{15188148,0}}, {garbage_collection,{2820556,7131269758,0}}, {io,{{input,371121433},{output,13008081495}}}, {reductions,{2380460585,2623394}}, {run_queue,0}, {runtime,{393650,540}}]}, {system_stats, [{cpu_utilization_rate,3.0},{swap_total,10585923584},{swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7289687095}, {couch_docs_data_size,7286342845}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,7982832088}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2449}, {memory_data,{33657663488,17197875200,{<17926.11.0>,43123400}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16073516 kB\nBuffers: 243900 kB\nCached: 7420564 kB\nSwapCached: 0 kB\nActive: 9271048 kB\nInactive: 7195044 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16073516 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 192 kB\nWriteback: 0 kB\nAnonPages: 8801640 kB\nMapped: 15608 kB\nSlab: 272472 kB\nPageTables: 22184 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10521420 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-14T11:29:22.074,ns_1@10.2.1.58:ns_doctor<0.1953.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,921357,75950}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,262296608}, {processes,208158960}, {processes_used,207811664}, {system,54137648}, {atom,1337385}, {atom_used,1331436}, {binary,12957088}, {code,13451143}, {ets,7633200}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7534039040}, {buffered_memory,245628928}, {free_memory,16794820608}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2507496,5000}}, {context_switches,{11980544,0}}, {garbage_collection,{2426106,5123237076,0}}, {io,{{input,278086624},{output,18067708384}}}, {reductions,{1530207190,1018432}}, {run_queue,0}, {runtime,{352760,510}}]}, {system_stats, [{cpu_utilization_rate,2.506265664160401}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7289199657}, {couch_docs_data_size,7285575568}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7765740072}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2507}, {memory_data,{33657663488,16861954048,{<0.11.0>,43123400}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16401192 kB\nBuffers: 239872 kB\nCached: 7357460 kB\nSwapCached: 0 kB\nActive: 8962520 kB\nInactive: 7196736 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16401192 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 540 kB\nWriteback: 0 kB\nAnonPages: 8561852 kB\nMapped: 15604 kB\nSlab: 251680 kB\nPageTables: 21820 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10299384 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920229,826001}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17879.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,921361,478038}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,262443200}, {processes,206722000}, {processes_used,206383784}, {system,55721200}, {atom,1349617}, {atom_used,1347167}, {binary,13383768}, {code,13553851}, {ets,8753000}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7600103424}, {buffered_memory,249831424}, {free_memory,16458391552}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2509942,4999}}, {context_switches,{15389153,0}}, {garbage_collection,{2831095,7200597261,0}}, {io,{{input,374690659},{output,13010040991}}}, {reductions,{2411547611,2606768}}, {run_queue,0}, {runtime,{400230,570}}]}, {system_stats, [{cpu_utilization_rate,2.743142144638404}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7290055738}, {couch_docs_data_size,7286342842}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,7941853144}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2509}, {memory_data,{33657663488,17200541696,{<17926.11.0>,43123400}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16072648 kB\nBuffers: 243976 kB\nCached: 7421976 kB\nSwapCached: 0 kB\nActive: 9271584 kB\nInactive: 7195748 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16072648 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 836 kB\nWriteback: 0 kB\nAnonPages: 8801476 kB\nMapped: 15608 kB\nSlab: 272436 kB\nPageTables: 22180 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10519832 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-14T11:29:53.092,ns_1@10.2.1.58:<0.2072.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,48,41}}) Stats for bucket "default": accepting_conns 1 auth_cmds 0 auth_errors 0 bucket_active_conns 1 bucket_conns 10 bytes 7744807192 bytes_read 17748304516 bytes_written 3553244855 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 0 conn_yields 33389 connection_structures 5000 curr_connections 13 curr_conns_on_port_11209 10 curr_conns_on_port_11210 1 curr_items 1500466 curr_items_tot 3000002 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-15 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data2/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 28828 ep_commit_time 3 ep_commit_time_total 379413 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 1 ep_data_age_highwat 5 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data2/default ep_degraded_mode 0 ep_diskqueue_drain 9763736 ep_diskqueue_fill 7199710 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 489 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 5613232 ep_io_read_bytes 0 ep_io_write_bytes 13292106458 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 1532571 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 7339250270 ep_max_checkpoints 2 ep_max_data_size 12582912000 ep_max_item_size 20971520 ep_max_size 12582912000 ep_max_txn_size 10000 ep_max_vbuckets 1024 ep_mem_high_wat 9437184000 ep_mem_high_wat 9437184000 ep_mem_low_wat 7549747200 ep_mem_low_wat 7549747200 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 162890938 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1352919382 ep_storage_age 0 ep_storage_age_highwat 3 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 7303250246 ep_total_del_items 3 ep_total_enqueued 7199710 ep_total_new_items 3000002 ep_total_persisted 5613232 ep_uncommitted_items 0 ep_value_size 7091900551 ep_vb0 0 ep_vb_snapshot_total 2624 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_135_g40a391b ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 1431 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 7744807192 pid 29361 pointer_size 64 rejected_conns 0 rusage_system 93.361806 rusage_user 642.357346 tap_checkpoint_end_received 1672 tap_checkpoint_end_sent 656 tap_checkpoint_start_received 3244 tap_checkpoint_start_sent 717 tap_connect_received 1 tap_delete_received 2 tap_mutation_received 7341122 tap_mutation_sent 1321512 tap_opaque_received 7168 tap_opaque_sent 513 tap_vbucket_set_received 1024 threads 4 time 1352921393 total_connections 5142 uptime 2013 vb_active_curr_items 1500466 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 12857056 vb_active_itm_memory 3653305966 vb_active_meta_data_memory 118536814 vb_active_num 512 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1500466 vb_active_ops_create 1500466 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 639616 vb_active_perc_mem_resident 100 vb_active_queue_age 0 vb_active_queue_drain 4284459 vb_active_queue_fill 2962833 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 1499536 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 12857152 vb_replica_itm_memory 3649944280 vb_replica_meta_data_memory 118463344 vb_replica_num 512 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 1499536 vb_replica_ops_create 1499536 vb_replica_ops_delete 3 vb_replica_ops_reject 0 vb_replica_ops_update 1973611 vb_replica_perc_mem_resident 100 vb_replica_queue_age 0 vb_replica_queue_drain 5479277 vb_replica_queue_fill 4236877 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-14T11:30:22.074,ns_1@10.2.1.58:ns_doctor<0.1953.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,921417,75718}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,258794608}, {processes,204890176}, {processes_used,204472040}, {system,53904432}, {atom,1337385}, {atom_used,1331436}, {binary,12634952}, {code,13451143}, {ets,7732384}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7529996288}, {buffered_memory,245858304}, {free_memory,16798248960}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2567495,4999}}, {context_switches,{12178380,0}}, {garbage_collection,{2438482,5168392852,0}}, {io,{{input,284937956},{output,18075141431}}}, {reductions,{1555943184,1021834}}, {run_queue,0}, {runtime,{359320,540}}]}, {system_stats, [{cpu_utilization_rate,2.743142144638404}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7289519148}, {couch_docs_data_size,7285575566}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7730275704}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2567}, {memory_data,{33657663488,16862715904,{<0.11.0>,43123400}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16404540 kB\nBuffers: 240096 kB\nCached: 7353512 kB\nSwapCached: 0 kB\nActive: 8963944 kB\nInactive: 7192956 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16404540 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 632 kB\nWriteback: 0 kB\nAnonPages: 8563008 kB\nMapped: 15604 kB\nSlab: 251780 kB\nPageTables: 21812 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10294824 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920229,826001}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17879.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,921421,478218}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,261944472}, {processes,205999312}, {processes_used,205589336}, {system,55945160}, {atom,1349617}, {atom_used,1347167}, {binary,13518504}, {code,13553851}, {ets,8851856}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7595847680}, {buffered_memory,250028032}, {free_memory,16459915264}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2569943,5001}}, {context_switches,{15590409,0}}, {garbage_collection,{2842022,7271428041,0}}, {io,{{input,381500096},{output,13018256632}}}, {reductions,{2443033984,2613938}}, {run_queue,0}, {runtime,{406950,590}}]}, {system_stats, [{cpu_utilization_rate,2.75}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7290375226}, {couch_docs_data_size,7286342841}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,7905948968}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2569}, {memory_data,{33657663488,17201938432,{<17926.11.0>,43123400}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16074136 kB\nBuffers: 244168 kB\nCached: 7417820 kB\nSwapCached: 0 kB\nActive: 9273844 kB\nInactive: 7191872 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16074136 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 588 kB\nWriteback: 0 kB\nAnonPages: 8803868 kB\nMapped: 15608 kB\nSlab: 272508 kB\nPageTables: 22188 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10522596 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-14T11:31:22.074,ns_1@10.2.1.58:ns_doctor<0.1953.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,921477,75765}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,263927776}, {processes,209787296}, {processes_used,209307520}, {system,54140480}, {atom,1337385}, {atom_used,1331436}, {binary,12763320}, {code,13451143}, {ets,7848320}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7531384832}, {buffered_memory,246001664}, {free_memory,16797360128}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2627495,4999}}, {context_switches,{12375485,0}}, {garbage_collection,{2451050,5212657386,0}}, {io,{{input,288346538},{output,18078031036}}}, {reductions,{1581511847,1013548}}, {run_queue,0}, {runtime,{365820,530}}]}, {system_stats, [{cpu_utilization_rate,2.5},{swap_total,10684628992},{swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7289801772}, {couch_docs_data_size,7285575563}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7698901368}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2627}, {memory_data,{33657663488,16860176384,{<0.11.0>,43123400}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16403672 kB\nBuffers: 240236 kB\nCached: 7354868 kB\nSwapCached: 0 kB\nActive: 8963812 kB\nInactive: 7193852 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16403672 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 220 kB\nWriteback: 0 kB\nAnonPages: 8562440 kB\nMapped: 15604 kB\nSlab: 251696 kB\nPageTables: 21808 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10298236 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920229,826001}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17879.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,921481,478247}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,266403536}, {processes,210478936}, {processes_used,210009160}, {system,55924600}, {atom,1349617}, {atom_used,1347167}, {binary,13397552}, {code,13553851}, {ets,8960536}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7597236224}, {buffered_memory,250122240}, {free_memory,16459915264}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2629943,5001}}, {context_switches,{15790612,0}}, {garbage_collection,{2852586,7340680937,0}}, {io,{{input,385062758},{output,13020418320}}}, {reductions,{2474123576,2611854}}, {run_queue,0}, {runtime,{413320,600}}]}, {system_stats, [{cpu_utilization_rate,2.75}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7290641466}, {couch_docs_data_size,7286342841}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,7875453384}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2629}, {memory_data,{33657663488,17198637056,{<17926.11.0>,43123400}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16074136 kB\nBuffers: 244260 kB\nCached: 7419176 kB\nSwapCached: 0 kB\nActive: 9273280 kB\nInactive: 7192668 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16074136 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 204 kB\nWriteback: 0 kB\nAnonPages: 8802632 kB\nMapped: 15608 kB\nSlab: 272516 kB\nPageTables: 22180 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10520584 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-14T11:31:33.093,ns_1@10.2.1.58:<0.2072.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,48,41}}) Stats for bucket "default": accepting_conns 1 auth_cmds 0 auth_errors 0 bucket_active_conns 1 bucket_conns 10 bytes 7692087240 bytes_read 17748325774 bytes_written 3556240325 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 0 conn_yields 33389 connection_structures 5000 curr_connections 13 curr_conns_on_port_11209 10 curr_conns_on_port_11210 1 curr_items 1500466 curr_items_tot 3000002 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-15 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data2/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 28828 ep_commit_time 3 ep_commit_time_total 379413 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 1 ep_data_age_highwat 5 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data2/default ep_degraded_mode 0 ep_diskqueue_drain 9763736 ep_diskqueue_fill 7199710 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 489 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 5613232 ep_io_read_bytes 0 ep_io_write_bytes 13292106458 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 1872548 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 7339250270 ep_max_checkpoints 2 ep_max_data_size 12582912000 ep_max_item_size 20971520 ep_max_size 12582912000 ep_max_txn_size 10000 ep_max_vbuckets 1024 ep_mem_high_wat 9437184000 ep_mem_high_wat 9437184000 ep_mem_low_wat 7549747200 ep_mem_low_wat 7549747200 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 135498818 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1352919382 ep_storage_age 0 ep_storage_age_highwat 3 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 7303250246 ep_total_del_items 3 ep_total_enqueued 7199710 ep_total_new_items 3000002 ep_total_persisted 5613232 ep_uncommitted_items 0 ep_value_size 7091900551 ep_vb0 0 ep_vb_snapshot_total 2665 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_135_g40a391b ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 1431 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 7692087240 pid 29361 pointer_size 64 rejected_conns 0 rusage_system 93.493786 rusage_user 643.679145 tap_checkpoint_end_received 1728 tap_checkpoint_end_sent 714 tap_checkpoint_start_received 3300 tap_checkpoint_start_sent 775 tap_connect_received 1 tap_delete_received 2 tap_mutation_received 7341122 tap_mutation_sent 1321512 tap_opaque_received 7168 tap_opaque_sent 513 tap_vbucket_set_received 1024 threads 4 time 1352921492 total_connections 5142 uptime 2112 vb_active_curr_items 1500466 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 12857056 vb_active_itm_memory 3653305966 vb_active_meta_data_memory 118536814 vb_active_num 512 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1500466 vb_active_ops_create 1500466 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 639616 vb_active_perc_mem_resident 100 vb_active_queue_age 0 vb_active_queue_drain 4284459 vb_active_queue_fill 2962833 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 1499536 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 12857152 vb_replica_itm_memory 3649944280 vb_replica_meta_data_memory 118463344 vb_replica_num 512 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 1499536 vb_replica_ops_create 1499536 vb_replica_ops_delete 3 vb_replica_ops_reject 0 vb_replica_ops_update 1973611 vb_replica_perc_mem_resident 100 vb_replica_queue_age 0 vb_replica_queue_drain 5479277 vb_replica_queue_fill 4236877 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-14T11:32:22.074,ns_1@10.2.1.58:ns_doctor<0.1953.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,921537,75844}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,264243504}, {processes,210097176}, {processes_used,209560360}, {system,54146328}, {atom,1337385}, {atom_used,1331436}, {binary,12776544}, {code,13451143}, {ets,7848168}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7532744704}, {buffered_memory,246145024}, {free_memory,16795455488}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2687496,5000}}, {context_switches,{12572385,0}}, {garbage_collection,{2461925,5256414034,0}}, {io,{{input,291760676},{output,18080639636}}}, {reductions,{1607127736,1023984}}, {run_queue,0}, {runtime,{372540,540}}]}, {system_stats, [{cpu_utilization_rate,2.5},{swap_total,10684628992},{swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7290051628}, {couch_docs_data_size,7285575562}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7671163720}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2687}, {memory_data,{33657663488,16862081024,{<0.11.0>,43123400}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16401812 kB\nBuffers: 240376 kB\nCached: 7356196 kB\nSwapCached: 0 kB\nActive: 8964392 kB\nInactive: 7194684 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16401812 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 608 kB\nWriteback: 0 kB\nAnonPages: 8562492 kB\nMapped: 15604 kB\nSlab: 251664 kB\nPageTables: 21824 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10297004 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920229,826001}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17879.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,921541,477520}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,262870176}, {processes,207102792}, {processes_used,206575056}, {system,55767384}, {atom,1349617}, {atom_used,1347167}, {binary,13240136}, {code,13553851}, {ets,8968080}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7598608384}, {buffered_memory,250212352}, {free_memory,16460169216}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2689942,5000}}, {context_switches,{15990749,0}}, {garbage_collection,{2863353,7410520706,0}}, {io,{{input,388518056},{output,13022376109}}}, {reductions,{2505213631,2613323}}, {run_queue,0}, {runtime,{419830,550}}]}, {system_stats, [{cpu_utilization_rate,2.75}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7290899514}, {couch_docs_data_size,7286342841}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,7848187368}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2689}, {memory_data,{33657663488,17200287744,{<17926.11.0>,43123400}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16074384 kB\nBuffers: 244348 kB\nCached: 7420516 kB\nSwapCached: 0 kB\nActive: 9272688 kB\nInactive: 7193472 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16074384 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 800 kB\nWriteback: 0 kB\nAnonPages: 8801228 kB\nMapped: 15608 kB\nSlab: 272428 kB\nPageTables: 22184 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10521840 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-14T11:33:13.093,ns_1@10.2.1.58:<0.2072.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,48,41}}) Stats for bucket "default": accepting_conns 1 auth_cmds 0 auth_errors 0 bucket_active_conns 1 bucket_conns 10 bytes 7646990328 bytes_read 17748345576 bytes_written 3559234339 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 0 conn_yields 33389 connection_structures 5000 curr_connections 13 curr_conns_on_port_11209 10 curr_conns_on_port_11210 1 curr_items 1500466 curr_items_tot 3000002 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-15 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data2/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 28828 ep_commit_time 3 ep_commit_time_total 379413 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 1 ep_data_age_highwat 5 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data2/default ep_degraded_mode 0 ep_diskqueue_drain 9763736 ep_diskqueue_fill 7199710 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 489 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 5613232 ep_io_read_bytes 0 ep_io_write_bytes 13292106458 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 2163419 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 7339250270 ep_max_checkpoints 2 ep_max_data_size 12582912000 ep_max_item_size 20971520 ep_max_size 12582912000 ep_max_txn_size 10000 ep_max_vbuckets 1024 ep_mem_high_wat 9437184000 ep_mem_high_wat 9437184000 ep_mem_low_wat 7549747200 ep_mem_low_wat 7549747200 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 111987340 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1352919382 ep_storage_age 0 ep_storage_age_highwat 3 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 7303250246 ep_total_del_items 3 ep_total_enqueued 7199710 ep_total_new_items 3000002 ep_total_persisted 5613232 ep_uncommitted_items 0 ep_value_size 7091900551 ep_vb0 0 ep_vb_snapshot_total 2705 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_135_g40a391b ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 1431 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 7646990328 pid 29361 pointer_size 64 rejected_conns 0 rusage_system 93.607769 rusage_user 644.951952 tap_checkpoint_end_received 1776 tap_checkpoint_end_sent 764 tap_checkpoint_start_received 3348 tap_checkpoint_start_sent 825 tap_connect_received 1 tap_delete_received 2 tap_mutation_received 7341122 tap_mutation_sent 1321512 tap_opaque_received 7168 tap_opaque_sent 513 tap_vbucket_set_received 1024 threads 4 time 1352921592 total_connections 5142 uptime 2212 vb_active_curr_items 1500466 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 12857056 vb_active_itm_memory 3653305966 vb_active_meta_data_memory 118536814 vb_active_num 512 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1500466 vb_active_ops_create 1500466 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 639616 vb_active_perc_mem_resident 100 vb_active_queue_age 0 vb_active_queue_drain 4284459 vb_active_queue_fill 2962833 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 1499536 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 12857152 vb_replica_itm_memory 3649944280 vb_replica_meta_data_memory 118463344 vb_replica_num 512 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 1499536 vb_replica_ops_create 1499536 vb_replica_ops_delete 3 vb_replica_ops_reject 0 vb_replica_ops_update 1973611 vb_replica_perc_mem_resident 100 vb_replica_queue_age 0 vb_replica_queue_drain 5479277 vb_replica_queue_fill 4236877 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-14T11:33:22.074,ns_1@10.2.1.58:ns_doctor<0.1953.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,921597,75978}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,261081264}, {processes,207010512}, {processes_used,206419416}, {system,54070752}, {atom,1337385}, {atom_used,1331436}, {binary,12581512}, {code,13451143}, {ets,7975200}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7533875200}, {buffered_memory,246435840}, {free_memory,16793550848}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2747496,5000}}, {context_switches,{12770637,0}}, {garbage_collection,{2474238,5301479512,0}}, {io,{{input,298809237},{output,18087019394}}}, {reductions,{1633165911,1013519}}, {run_queue,0}, {runtime,{379210,520}}]}, {system_stats, [{cpu_utilization_rate,2.5},{swap_total,10684628992},{swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7290285100}, {couch_docs_data_size,7285575559}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7645163992}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2747}, {memory_data,{33657663488,16861954048,{<0.11.0>,43123400}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16399952 kB\nBuffers: 240660 kB\nCached: 7357300 kB\nSwapCached: 0 kB\nActive: 8965384 kB\nInactive: 7195564 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16399952 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 468 kB\nWriteback: 0 kB\nAnonPages: 8562952 kB\nMapped: 15604 kB\nSlab: 251756 kB\nPageTables: 21828 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10300384 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920229,826001}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17879.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,921601,478509}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,264271256}, {processes,208282208}, {processes_used,207702032}, {system,55989048}, {atom,1349617}, {atom_used,1347167}, {binary,13349648}, {code,13553851}, {ets,9087752}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7599919104}, {buffered_memory,250408960}, {free_memory,16456359936}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2749943,5002}}, {context_switches,{16191773,0}}, {garbage_collection,{2873822,7480433903,0}}, {io,{{input,395456073},{output,13027854323}}}, {reductions,{2536408910,2614032}}, {run_queue,0}, {runtime,{426200,530}}]}, {system_stats, [{cpu_utilization_rate,2.743142144638404}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7291132986}, {couch_docs_data_size,7286342839}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,7820845464}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2749}, {memory_data,{33657663488,17200414720,{<17926.11.0>,43123400}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16070664 kB\nBuffers: 244540 kB\nCached: 7421796 kB\nSwapCached: 0 kB\nActive: 9275180 kB\nInactive: 7194508 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16070664 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 496 kB\nWriteback: 0 kB\nAnonPages: 8803480 kB\nMapped: 15608 kB\nSlab: 272512 kB\nPageTables: 22176 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10521928 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-14T11:34:22.074,ns_1@10.2.1.58:ns_doctor<0.1953.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,921657,75860}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,266493704}, {processes,212130728}, {processes_used,211491792}, {system,54362976}, {atom,1337385}, {atom_used,1331436}, {binary,12870144}, {code,13451143}, {ets,7985184}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7535198208}, {buffered_memory,246566912}, {free_memory,16786440192}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2807495,5000}}, {context_switches,{12967185,0}}, {garbage_collection,{2485219,5344883589,0}}, {io,{{input,302193641},{output,18089725060}}}, {reductions,{1658720968,1015904}}, {run_queue,0}, {runtime,{385650,500}}]}, {system_stats, [{cpu_utilization_rate,2.5},{swap_total,10684628992},{swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7290510380}, {couch_docs_data_size,7285575557}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7620234248}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2807}, {memory_data,{33657663488,16864620544,{<0.11.0>,43123400}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16393008 kB\nBuffers: 240788 kB\nCached: 7358592 kB\nSwapCached: 0 kB\nActive: 8971552 kB\nInactive: 7196416 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16393008 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 336 kB\nWriteback: 0 kB\nAnonPages: 8568396 kB\nMapped: 15604 kB\nSlab: 251712 kB\nPageTables: 21840 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10305072 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920229,826001}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17879.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,921661,477404}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,264300400}, {processes,208427032}, {processes_used,207797176}, {system,55873368}, {atom,1349617}, {atom_used,1347167}, {binary,13232512}, {code,13553851}, {ets,9095648}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7601266688}, {buffered_memory,250494976}, {free_memory,16460169216}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2809942,5000}}, {context_switches,{16391638,0}}, {garbage_collection,{2883228,7549567471,0}}, {io,{{input,398835757},{output,13029885871}}}, {reductions,{2567430110,2604682}}, {run_queue,0}, {runtime,{432540,540}}]}, {system_stats, [{cpu_utilization_rate,2.743142144638404}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7291354170}, {couch_docs_data_size,7286342835}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,7796341992}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2809}, {memory_data,{33657663488,17201430528,{<17926.11.0>,43123400}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16074384 kB\nBuffers: 244624 kB\nCached: 7423112 kB\nSwapCached: 0 kB\nActive: 9270160 kB\nInactive: 7195308 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16074384 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 216 kB\nWriteback: 0 kB\nAnonPages: 8797808 kB\nMapped: 15608 kB\nSlab: 272460 kB\nPageTables: 22184 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10513480 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-14T11:34:53.092,ns_1@10.2.1.58:<0.2072.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,48,41}}) Stats for bucket "default": accepting_conns 1 auth_cmds 0 auth_errors 0 bucket_active_conns 1 bucket_conns 10 bytes 7607000152 bytes_read 17748364640 bytes_written 3562227222 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 0 conn_yields 33389 connection_structures 5000 curr_connections 13 curr_conns_on_port_11209 10 curr_conns_on_port_11210 1 curr_items 1500466 curr_items_tot 3000002 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-15 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data2/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 28828 ep_commit_time 3 ep_commit_time_total 379413 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 1 ep_data_age_highwat 5 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data2/default ep_degraded_mode 0 ep_diskqueue_drain 9763736 ep_diskqueue_fill 7199710 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 489 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 5613232 ep_io_read_bytes 0 ep_io_write_bytes 13292106458 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 2421310 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 7339250270 ep_max_checkpoints 2 ep_max_data_size 12582912000 ep_max_item_size 20971520 ep_max_size 12582912000 ep_max_txn_size 10000 ep_max_vbuckets 1024 ep_mem_high_wat 9437184000 ep_mem_high_wat 9437184000 ep_mem_low_wat 7549747200 ep_mem_low_wat 7549747200 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 91230248 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1352919382 ep_storage_age 0 ep_storage_age_highwat 3 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 7303250246 ep_total_del_items 3 ep_total_enqueued 7199710 ep_total_new_items 3000002 ep_total_persisted 5613232 ep_uncommitted_items 0 ep_value_size 7091900551 ep_vb0 0 ep_vb_snapshot_total 2745 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_135_g40a391b ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 1431 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 7607000152 pid 29361 pointer_size 64 rejected_conns 0 rusage_system 93.744748 rusage_user 646.204761 tap_checkpoint_end_received 1821 tap_checkpoint_end_sent 807 tap_checkpoint_start_received 3393 tap_checkpoint_start_sent 868 tap_connect_received 1 tap_delete_received 2 tap_mutation_received 7341122 tap_mutation_sent 1321512 tap_opaque_received 7168 tap_opaque_sent 513 tap_vbucket_set_received 1024 threads 4 time 1352921692 total_connections 5142 uptime 2312 vb_active_curr_items 1500466 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 12857056 vb_active_itm_memory 3653305966 vb_active_meta_data_memory 118536814 vb_active_num 512 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1500466 vb_active_ops_create 1500466 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 639616 vb_active_perc_mem_resident 100 vb_active_queue_age 0 vb_active_queue_drain 4284459 vb_active_queue_fill 2962833 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 1499536 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 12857152 vb_replica_itm_memory 3649944280 vb_replica_meta_data_memory 118463344 vb_replica_num 512 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 1499536 vb_replica_ops_create 1499536 vb_replica_ops_delete 3 vb_replica_ops_reject 0 vb_replica_ops_update 1973611 vb_replica_perc_mem_resident 100 vb_replica_queue_age 0 vb_replica_queue_drain 5479277 vb_replica_queue_fill 4236877 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-14T11:35:22.074,ns_1@10.2.1.58:ns_doctor<0.1953.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,921717,75725}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,262834072}, {processes,208473672}, {processes_used,207788736}, {system,54360400}, {atom,1337385}, {atom_used,1331436}, {binary,12749752}, {code,13451143}, {ets,8109384}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7536529408}, {buffered_memory,246718464}, {free_memory,16786694144}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2867495,4999}}, {context_switches,{13163702,0}}, {garbage_collection,{2497191,5389134925,0}}, {io,{{input,305500918},{output,18092454599}}}, {reductions,{1684325395,1015622}}, {run_queue,0}, {runtime,{392330,560}}]}, {system_stats, [{cpu_utilization_rate,2.75}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7290715179}, {couch_docs_data_size,7285575554}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7597441480}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2867}, {memory_data,{33657663488,16871604224,{<0.11.0>,43123400}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16393256 kB\nBuffers: 240936 kB\nCached: 7359892 kB\nSwapCached: 0 kB\nActive: 8970528 kB\nInactive: 7197348 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16393256 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 676 kB\nWriteback: 0 kB\nAnonPages: 8566768 kB\nMapped: 15604 kB\nSlab: 251760 kB\nPageTables: 21836 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10304280 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920229,826001}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17879.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,921721,477251}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,261214328}, {processes,205172600}, {processes_used,204498584}, {system,56041728}, {atom,1349617}, {atom_used,1347167}, {binary,13280512}, {code,13553851}, {ets,9222256}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7602556928}, {buffered_memory,250593280}, {free_memory,16459153408}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2869942,5000}}, {context_switches,{16591332,0}}, {garbage_collection,{2893987,7619079119,0}}, {io,{{input,402301989},{output,13031844488}}}, {reductions,{2598531897,2608139}}, {run_queue,0}, {runtime,{439030,600}}]}, {system_stats, [{cpu_utilization_rate,3.0},{swap_total,10585923584},{swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7291550777}, {couch_docs_data_size,7286342835}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,7774013368}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2869}, {memory_data,{33657663488,17198764032,{<17926.11.0>,43123400}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16073392 kB\nBuffers: 244720 kB\nCached: 7424372 kB\nSwapCached: 0 kB\nActive: 9271236 kB\nInactive: 7196196 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16073392 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 516 kB\nWriteback: 0 kB\nAnonPages: 8798388 kB\nMapped: 15608 kB\nSlab: 272440 kB\nPageTables: 22184 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10515716 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-14T11:36:22.074,ns_1@10.2.1.58:ns_doctor<0.1953.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,921777,76059}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,263004144}, {processes,208547032}, {processes_used,207820696}, {system,54457112}, {atom,1337385}, {atom_used,1331436}, {binary,12721680}, {code,13451143}, {ets,8240472}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7532335104}, {buffered_memory,246947840}, {free_memory,16788344832}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2927496,5000}}, {context_switches,{13360431,0}}, {garbage_collection,{2510301,5434210503,0}}, {io,{{input,312241951},{output,18100742451}}}, {reductions,{1710035727,1017855}}, {run_queue,0}, {runtime,{398900,490}}]}, {system_stats, [{cpu_utilization_rate,2.506265664160401}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7290895402}, {couch_docs_data_size,7285575553}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7577540856}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2927}, {memory_data,{33657663488,16873000960,{<0.11.0>,43123400}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16394868 kB\nBuffers: 241160 kB\nCached: 7355796 kB\nSwapCached: 0 kB\nActive: 8972216 kB\nInactive: 7193612 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16394868 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 448 kB\nWriteback: 0 kB\nAnonPages: 8568588 kB\nMapped: 15604 kB\nSlab: 251852 kB\nPageTables: 21836 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10303464 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920229,826001}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17879.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,921781,477734}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,267602704}, {processes,211486048}, {processes_used,210770632}, {system,56116656}, {atom,1349617}, {atom_used,1347167}, {binary,13220616}, {code,13553851}, {ets,9362976}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7598182400}, {buffered_memory,250769408}, {free_memory,16461692928}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2929942,5000}}, {context_switches,{16791556,0}}, {garbage_collection,{2904926,7690113962,0}}, {io,{{input,409204857},{output,13040884347}}}, {reductions,{2630072474,2607043}}, {run_queue,0}, {runtime,{445770,580}}]}, {system_stats, [{cpu_utilization_rate,2.9925187032418954}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7291735096}, {couch_docs_data_size,7286342835}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,7753673080}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2929}, {memory_data,{33657663488,17200033792,{<17926.11.0>,43123400}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16075872 kB\nBuffers: 244892 kB\nCached: 7420100 kB\nSwapCached: 0 kB\nActive: 9271164 kB\nInactive: 7192328 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16075872 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 512 kB\nWriteback: 0 kB\nAnonPages: 8798456 kB\nMapped: 15608 kB\nSlab: 272476 kB\nPageTables: 22192 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10517452 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-14T11:36:33.092,ns_1@10.2.1.58:<0.2072.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,48,41}}) Stats for bucket "default": accepting_conns 1 auth_cmds 0 auth_errors 0 bucket_active_conns 1 bucket_conns 10 bytes 7572095832 bytes_read 17748383100 bytes_written 3565219010 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 0 conn_yields 33389 connection_structures 5000 curr_connections 13 curr_conns_on_port_11209 10 curr_conns_on_port_11210 1 curr_items 1500466 curr_items_tot 3000002 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-15 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data2/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 28828 ep_commit_time 3 ep_commit_time_total 379413 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 1 ep_data_age_highwat 5 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data2/default ep_degraded_mode 0 ep_diskqueue_drain 9763736 ep_diskqueue_fill 7199710 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 489 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 5613232 ep_io_read_bytes 0 ep_io_write_bytes 13292106458 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 2646367 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 7339250270 ep_max_checkpoints 2 ep_max_data_size 12582912000 ep_max_item_size 20971520 ep_max_size 12582912000 ep_max_txn_size 10000 ep_max_vbuckets 1024 ep_mem_high_wat 9437184000 ep_mem_high_wat 9437184000 ep_mem_low_wat 7549747200 ep_mem_low_wat 7549747200 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 72938076 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1352919382 ep_storage_age 0 ep_storage_age_highwat 3 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 7303250246 ep_total_del_items 3 ep_total_enqueued 7199710 ep_total_new_items 3000002 ep_total_persisted 5613232 ep_uncommitted_items 0 ep_value_size 7091900551 ep_vb0 0 ep_vb_snapshot_total 2783 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_135_g40a391b ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 1431 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 7572095832 pid 29361 pointer_size 64 rejected_conns 0 rusage_system 93.831735 rusage_user 647.421576 tap_checkpoint_end_received 1864 tap_checkpoint_end_sent 843 tap_checkpoint_start_received 3436 tap_checkpoint_start_sent 904 tap_connect_received 1 tap_delete_received 2 tap_mutation_received 7341122 tap_mutation_sent 1321512 tap_opaque_received 7168 tap_opaque_sent 513 tap_vbucket_set_received 1024 threads 4 time 1352921792 total_connections 5142 uptime 2412 vb_active_curr_items 1500466 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 12857056 vb_active_itm_memory 3653305966 vb_active_meta_data_memory 118536814 vb_active_num 512 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1500466 vb_active_ops_create 1500466 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 639616 vb_active_perc_mem_resident 100 vb_active_queue_age 0 vb_active_queue_drain 4284459 vb_active_queue_fill 2962833 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 1499536 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 12857152 vb_replica_itm_memory 3649944280 vb_replica_meta_data_memory 118463344 vb_replica_num 512 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 1499536 vb_replica_ops_create 1499536 vb_replica_ops_delete 3 vb_replica_ops_reject 0 vb_replica_ops_update 1973611 vb_replica_perc_mem_resident 100 vb_replica_queue_age 0 vb_replica_queue_drain 5479277 vb_replica_queue_fill 4236877 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-14T11:37:22.074,ns_1@10.2.1.58:ns_doctor<0.1953.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,921837,75739}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,263226304}, {processes,208584440}, {processes_used,207815784}, {system,54641864}, {atom,1337385}, {atom_used,1331436}, {binary,12783736}, {code,13451143}, {ets,8368776}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7533613056}, {buffered_memory,247123968}, {free_memory,16789360640}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2987495,4999}}, {context_switches,{13556879,0}}, {garbage_collection,{2521465,5477817666,0}}, {io,{{input,315745375},{output,18103546327}}}, {reductions,{1735608520,1018089}}, {run_queue,0}, {runtime,{405140,520}}]}, {system_stats, [{cpu_utilization_rate,2.743142144638404}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7291096106}, {couch_docs_data_size,7285575553}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7555249384}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2987}, {memory_data,{33657663488,16867414016,{<0.11.0>,43123400}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16395860 kB\nBuffers: 241332 kB\nCached: 7357044 kB\nSwapCached: 0 kB\nActive: 8969836 kB\nInactive: 7194544 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16395860 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 340 kB\nWriteback: 0 kB\nAnonPages: 8566088 kB\nMapped: 15604 kB\nSlab: 251852 kB\nPageTables: 21832 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10292988 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920229,826001}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17879.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,921841,477848}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,263421144}, {processes,207425448}, {processes_used,206666792}, {system,55995696}, {atom,1349617}, {atom_used,1347167}, {binary,12978528}, {code,13553851}, {ets,9489888}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7599456256}, {buffered_memory,250843136}, {free_memory,16462200832}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{2989942,4999}}, {context_switches,{16991213,0}}, {garbage_collection,{2914792,7759482789,0}}, {io,{{input,412771895},{output,13042931003}}}, {reductions,{2661142268,2613130}}, {run_queue,0}, {runtime,{452230,550}}]}, {system_stats, [{cpu_utilization_rate,2.743142144638404}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7291927608}, {couch_docs_data_size,7286342834}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,7732736888}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,2989}, {memory_data,{33657663488,17196478464,{<17926.11.0>,43123400}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16076492 kB\nBuffers: 244964 kB\nCached: 7421344 kB\nSwapCached: 0 kB\nActive: 9271068 kB\nInactive: 7193140 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16076492 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 240 kB\nWriteback: 0 kB\nAnonPages: 8797992 kB\nMapped: 15608 kB\nSlab: 272456 kB\nPageTables: 22176 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10513700 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-14T11:38:13.092,ns_1@10.2.1.58:<0.2072.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,48,41}}) Stats for bucket "default": accepting_conns 1 auth_cmds 0 auth_errors 0 bucket_active_conns 1 bucket_conns 10 bytes 7540646184 bytes_read 17748400468 bytes_written 3568209706 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 0 conn_yields 33389 connection_structures 5000 curr_connections 13 curr_conns_on_port_11209 10 curr_conns_on_port_11210 1 curr_items 1500466 curr_items_tot 3000002 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-15 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data2/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 28828 ep_commit_time 3 ep_commit_time_total 379413 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 1 ep_data_age_highwat 5 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data2/default ep_degraded_mode 0 ep_diskqueue_drain 9763736 ep_diskqueue_fill 7199710 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 489 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 5613232 ep_io_read_bytes 0 ep_io_write_bytes 13292106458 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 2849218 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 7339250270 ep_max_checkpoints 2 ep_max_data_size 12582912000 ep_max_item_size 20971520 ep_max_size 12582912000 ep_max_txn_size 10000 ep_max_vbuckets 1024 ep_mem_high_wat 9437184000 ep_mem_high_wat 9437184000 ep_mem_low_wat 7549747200 ep_mem_low_wat 7549747200 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 56488592 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1352919382 ep_storage_age 0 ep_storage_age_highwat 3 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 7303250246 ep_total_del_items 3 ep_total_enqueued 7199710 ep_total_new_items 3000002 ep_total_persisted 5613232 ep_uncommitted_items 0 ep_value_size 7091900551 ep_vb0 0 ep_vb_snapshot_total 2819 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_135_g40a391b ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 1431 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 7540646184 pid 29361 pointer_size 64 rejected_conns 0 rusage_system 93.933719 rusage_user 648.588399 tap_checkpoint_end_received 1901 tap_checkpoint_end_sent 873 tap_checkpoint_start_received 3473 tap_checkpoint_start_sent 934 tap_connect_received 1 tap_delete_received 2 tap_mutation_received 7341122 tap_mutation_sent 1321512 tap_opaque_received 7168 tap_opaque_sent 513 tap_vbucket_set_received 1024 threads 4 time 1352921892 total_connections 5142 uptime 2512 vb_active_curr_items 1500466 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 12857056 vb_active_itm_memory 3653305966 vb_active_meta_data_memory 118536814 vb_active_num 512 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1500466 vb_active_ops_create 1500466 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 639616 vb_active_perc_mem_resident 100 vb_active_queue_age 0 vb_active_queue_drain 4284459 vb_active_queue_fill 2962833 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 1499536 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 12857152 vb_replica_itm_memory 3649944280 vb_replica_meta_data_memory 118463344 vb_replica_num 512 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 1499536 vb_replica_ops_create 1499536 vb_replica_ops_delete 3 vb_replica_ops_reject 0 vb_replica_ops_update 1973611 vb_replica_perc_mem_resident 100 vb_replica_queue_age 0 vb_replica_queue_drain 5479277 vb_replica_queue_fill 4236877 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-14T11:38:22.074,ns_1@10.2.1.58:ns_doctor<0.1953.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,921897,75820}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,261942256}, {processes,207278128}, {processes_used,206473592}, {system,54664128}, {atom,1337385}, {atom_used,1331436}, {binary,12786832}, {code,13451143}, {ets,8392440}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7534903296}, {buffered_memory,247238656}, {free_memory,16786186240}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{3047496,5001}}, {context_switches,{13752994,0}}, {garbage_collection,{2533704,5521541601,0}}, {io,{{input,319030361},{output,18106155364}}}, {reductions,{1761099199,1019580}}, {run_queue,0}, {runtime,{411510,530}}]}, {system_stats, [{cpu_utilization_rate,2.743142144638404}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7291243562}, {couch_docs_data_size,7285575552}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7538819128}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3047}, {memory_data,{33657663488,16868302848,{<0.11.0>,43123400}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16392760 kB\nBuffers: 241444 kB\nCached: 7358304 kB\nSwapCached: 0 kB\nActive: 8972664 kB\nInactive: 7195360 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16392760 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 696 kB\nWriteback: 0 kB\nAnonPages: 8567816 kB\nMapped: 15604 kB\nSlab: 251848 kB\nPageTables: 21844 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10303888 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920229,826001}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17879.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,921901,477881}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,261055960}, {processes,205048800}, {processes_used,204254264}, {system,56007160}, {atom,1349617}, {atom_used,1347167}, {binary,12970656}, {code,13553851}, {ets,9513416}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7600762880}, {buffered_memory,250945536}, {free_memory,16460804096}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{3049942,4999}}, {context_switches,{17190573,0}}, {garbage_collection,{2925848,7828906480,0}}, {io,{{input,416236492},{output,13044971129}}}, {reductions,{2692205435,2624432}}, {run_queue,0}, {runtime,{458800,580}}]}, {system_stats, [{cpu_utilization_rate,2.2613065326633164}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7292087352}, {couch_docs_data_size,7286342832}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,7715413912}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3049}, {memory_data,{33657663488,17197875200,{<17926.11.0>,43123400}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16075004 kB\nBuffers: 245064 kB\nCached: 7422620 kB\nSwapCached: 0 kB\nActive: 9271316 kB\nInactive: 7194040 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16075004 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 644 kB\nWriteback: 0 kB\nAnonPages: 8797740 kB\nMapped: 15608 kB\nSlab: 272496 kB\nPageTables: 22176 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10514976 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-14T11:39:22.074,ns_1@10.2.1.58:ns_doctor<0.1953.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,921957,75754}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,266513072}, {processes,211635416}, {processes_used,210790400}, {system,54877656}, {atom,1337385}, {atom_used,1331436}, {binary,12873184}, {code,13451143}, {ets,8524984}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7535980544}, {buffered_memory,247480320}, {free_memory,16784916480}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{3107495,4999}}, {context_switches,{13950679,0}}, {garbage_collection,{2545196,5566415311,0}}, {io,{{input,325993838},{output,18112461427}}}, {reductions,{1787106022,1013097}}, {run_queue,0}, {runtime,{418280,540}}]}, {system_stats, [{cpu_utilization_rate,2.5},{swap_total,10684628992},{swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7291431978}, {couch_docs_data_size,7285575551}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7517890136}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3107}, {memory_data,{33657663488,16870461440,{<0.11.0>,43123400}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16391520 kB\nBuffers: 241680 kB\nCached: 7359356 kB\nSwapCached: 0 kB\nActive: 8973092 kB\nInactive: 7196232 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16391520 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 336 kB\nWriteback: 0 kB\nAnonPages: 8568200 kB\nMapped: 15604 kB\nSlab: 251976 kB\nPageTables: 21844 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10301704 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920229,826001}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17879.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,921961,477116}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,266850264}, {processes,210742672}, {processes_used,209907656}, {system,56107592}, {atom,1349617}, {atom_used,1347167}, {binary,12945160}, {code,13553851}, {ets,9644760}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7601975296}, {buffered_memory,251150336}, {free_memory,16457375744}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{3109942,5000}}, {context_switches,{17391357,0}}, {garbage_collection,{2936901,7898497370,0}}, {io,{{input,423169614},{output,13050285362}}}, {reductions,{2723416203,2604569}}, {run_queue,0}, {runtime,{465510,570}}]}, {system_stats, [{cpu_utilization_rate,2.5},{swap_total,10585923584},{swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7292267576}, {couch_docs_data_size,7286342832}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,7694922856}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3109}, {memory_data,{33657663488,17199271936,{<17926.11.0>,43123400}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16071656 kB\nBuffers: 245264 kB\nCached: 7423804 kB\nSwapCached: 0 kB\nActive: 9273320 kB\nInactive: 7195092 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16071656 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 472 kB\nWriteback: 0 kB\nAnonPages: 8799364 kB\nMapped: 15608 kB\nSlab: 272468 kB\nPageTables: 22176 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10516332 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-14T11:39:53.094,ns_1@10.2.1.58:<0.2072.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,48,41}}) Stats for bucket "default": accepting_conns 1 auth_cmds 0 auth_errors 0 bucket_active_conns 1 bucket_conns 10 bytes 7508295032 bytes_read 17748418324 bytes_written 3571200460 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 0 conn_yields 33389 connection_structures 5000 curr_connections 13 curr_conns_on_port_11209 10 curr_conns_on_port_11210 1 curr_items 1500466 curr_items_tot 3000002 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-15 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data2/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 28828 ep_commit_time 3 ep_commit_time_total 379413 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 1 ep_data_age_highwat 5 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data2/default ep_degraded_mode 0 ep_diskqueue_drain 9763736 ep_diskqueue_fill 7199710 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 489 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 5613232 ep_io_read_bytes 0 ep_io_write_bytes 13292106458 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 3057885 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 7339250270 ep_max_checkpoints 2 ep_max_data_size 12582912000 ep_max_item_size 20971520 ep_max_size 12582912000 ep_max_txn_size 10000 ep_max_vbuckets 1024 ep_mem_high_wat 9437184000 ep_mem_high_wat 9437184000 ep_mem_low_wat 7549747200 ep_mem_low_wat 7549747200 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 39525182 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1352919382 ep_storage_age 0 ep_storage_age_highwat 3 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 7303250246 ep_total_del_items 3 ep_total_enqueued 7199710 ep_total_new_items 3000002 ep_total_persisted 5613232 ep_uncommitted_items 0 ep_value_size 7091900551 ep_vb0 0 ep_vb_snapshot_total 2855 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_135_g40a391b ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 1431 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 7508295032 pid 29361 pointer_size 64 rejected_conns 0 rusage_system 94.049702 rusage_user 649.752222 tap_checkpoint_end_received 1942 tap_checkpoint_end_sent 902 tap_checkpoint_start_received 3514 tap_checkpoint_start_sent 963 tap_connect_received 1 tap_delete_received 2 tap_mutation_received 7341122 tap_mutation_sent 1321512 tap_opaque_received 7168 tap_opaque_sent 513 tap_vbucket_set_received 1024 threads 4 time 1352921992 total_connections 5142 uptime 2612 vb_active_curr_items 1500466 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 12857056 vb_active_itm_memory 3653305966 vb_active_meta_data_memory 118536814 vb_active_num 512 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1500466 vb_active_ops_create 1500466 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 639616 vb_active_perc_mem_resident 100 vb_active_queue_age 0 vb_active_queue_drain 4284459 vb_active_queue_fill 2962833 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 1499536 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 12857152 vb_replica_itm_memory 3649944280 vb_replica_meta_data_memory 118463344 vb_replica_num 512 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 1499536 vb_replica_ops_create 1499536 vb_replica_ops_delete 3 vb_replica_ops_reject 0 vb_replica_ops_update 1973611 vb_replica_perc_mem_resident 100 vb_replica_queue_age 0 vb_replica_queue_drain 5479277 vb_replica_queue_fill 4236877 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-14T11:40:22.074,ns_1@10.2.1.58:ns_doctor<0.1953.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,922017,75981}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,264680704}, {processes,209575640}, {processes_used,208701184}, {system,55105064}, {atom,1337385}, {atom_used,1331436}, {binary,13075112}, {code,13451143}, {ets,8554008}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7537242112}, {buffered_memory,247631872}, {free_memory,16784027648}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{3167496,5000}}, {context_switches,{14146606,0}}, {garbage_collection,{2556445,5610644962,0}}, {io,{{input,329482931},{output,18115189097}}}, {reductions,{1812722803,1035975}}, {run_queue,0}, {runtime,{424780,510}}]}, {system_stats, [{cpu_utilization_rate,2.75}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7291567146}, {couch_docs_data_size,7285575551}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7502853032}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3167}, {memory_data,{33657663488,16873000960,{<0.11.0>,43123400}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16390652 kB\nBuffers: 241828 kB\nCached: 7360588 kB\nSwapCached: 0 kB\nActive: 8972636 kB\nInactive: 7197180 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16390652 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 220 kB\nWriteback: 0 kB\nAnonPages: 8567328 kB\nMapped: 15604 kB\nSlab: 251792 kB\nPageTables: 21844 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10305788 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920229,826001}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17879.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,922021,477109}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,262960824}, {processes,206856944}, {processes_used,205994328}, {system,56103880}, {atom,1349617}, {atom_used,1347167}, {binary,12912952}, {code,13553851}, {ets,9676600}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7603224576}, {buffered_memory,251236352}, {free_memory,16454836224}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{3169942,5000}}, {context_switches,{17590375,0}}, {garbage_collection,{2948267,7968345851,0}}, {io,{{input,426562366},{output,13052417311}}}, {reductions,{2754517020,2609271}}, {run_queue,0}, {runtime,{471930,540}}]}, {system_stats, [{cpu_utilization_rate,2.9925187032418954}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7292390456}, {couch_docs_data_size,7286342831}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,7680337656}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3169}, {memory_data,{33657663488,17201049600,{<17926.11.0>,43123400}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16069176 kB\nBuffers: 245348 kB\nCached: 7425024 kB\nSwapCached: 0 kB\nActive: 9274212 kB\nInactive: 7196020 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16069176 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 324 kB\nWriteback: 0 kB\nAnonPages: 8799952 kB\nMapped: 15608 kB\nSlab: 272488 kB\nPageTables: 22180 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10521312 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [ns_doctor:debug,2012-11-14T11:41:22.074,ns_1@10.2.1.58:ns_doctor<0.1953.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,922077,75972}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,262629544}, {processes,207601040}, {processes_used,206725664}, {system,55028504}, {atom,1337385}, {atom_used,1331436}, {binary,12816216}, {code,13451143}, {ets,8736480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7538348032}, {buffered_memory,247767040}, {free_memory,16784662528}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{3227496,5000}}, {context_switches,{14341514,0}}, {garbage_collection,{2567938,5654171109,0}}, {io,{{input,332773539},{output,18117876092}}}, {reductions,{1838224695,1005689}}, {run_queue,0}, {runtime,{431310,540}}]}, {system_stats, [{cpu_utilization_rate,2.5},{swap_total,10684628992},{swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7291579434}, {couch_docs_data_size,7285575551}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7501491416}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3227}, {memory_data,{33657663488,16872620032,{<0.11.0>,43123400}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16391272 kB\nBuffers: 241960 kB\nCached: 7361668 kB\nSwapCached: 0 kB\nActive: 8971676 kB\nInactive: 7198164 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16391272 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 600 kB\nWriteback: 0 kB\nAnonPages: 8566012 kB\nMapped: 15604 kB\nSlab: 251760 kB\nPageTables: 21828 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10293636 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920229,826001}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17879.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,922081,477092}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,265035376}, {processes,208567168}, {processes_used,207705920}, {system,56468208}, {atom,1353657}, {atom_used,1352629}, {binary,13026248}, {code,13603146}, {ets,9871360}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7605870592}, {buffered_memory,251371520}, {free_memory,16452476928}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{3229942,5000}}, {context_switches,{17790564,0}}, {garbage_collection,{2959520,8045341925,0}}, {io,{{input,431458310},{output,13055927632}}}, {reductions,{2787510313,2643002}}, {run_queue,0}, {runtime,{478500,560}}]}, {system_stats, [{cpu_utilization_rate,2.75}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7292394552}, {couch_docs_data_size,7286342831}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,7678974744}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3229}, {memory_data,{33657663488,17203462144,{<17926.11.0>,43123400}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16066872 kB\nBuffers: 245480 kB\nCached: 7427608 kB\nSwapCached: 0 kB\nActive: 9274240 kB\nInactive: 7198540 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16066872 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 608 kB\nWriteback: 0 kB\nAnonPages: 8799680 kB\nMapped: 15608 kB\nSlab: 272432 kB\nPageTables: 22180 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10521324 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-14T11:41:33.092,ns_1@10.2.1.58:<0.2072.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,48,42}}) Stats for bucket "default": accepting_conns 1 auth_cmds 0 auth_errors 0 bucket_active_conns 1 bucket_conns 10 bytes 7501491432 bytes_read 17748430902 bytes_written 3574186032 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 0 conn_yields 33389 connection_structures 5000 curr_connections 13 curr_conns_on_port_11209 10 curr_conns_on_port_11210 1 curr_items 1500466 curr_items_tot 3000002 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-15 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data2/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 28828 ep_commit_time 3 ep_commit_time_total 379413 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 1 ep_data_age_highwat 5 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data2/default ep_degraded_mode 0 ep_diskqueue_drain 9763736 ep_diskqueue_fill 7199710 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 489 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 5613232 ep_io_read_bytes 0 ep_io_write_bytes 13292106458 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 3101741 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 7339250270 ep_max_checkpoints 2 ep_max_data_size 12582912000 ep_max_item_size 20971520 ep_max_size 12582912000 ep_max_txn_size 10000 ep_max_vbuckets 1024 ep_mem_high_wat 9437184000 ep_mem_high_wat 9437184000 ep_mem_low_wat 7549747200 ep_mem_low_wat 7549747200 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 35900908 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1352919382 ep_storage_age 0 ep_storage_age_highwat 3 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 7303250246 ep_total_del_items 3 ep_total_enqueued 7199710 ep_total_new_items 3000002 ep_total_persisted 5613232 ep_uncommitted_items 0 ep_value_size 7091900551 ep_vb0 0 ep_vb_snapshot_total 2861 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_135_g40a391b ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 1431 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 7501491432 pid 29361 pointer_size 64 rejected_conns 0 rusage_system 94.099694 rusage_user 650.488110 tap_checkpoint_end_received 1954 tap_checkpoint_end_sent 902 tap_checkpoint_start_received 3526 tap_checkpoint_start_sent 963 tap_connect_received 1 tap_delete_received 2 tap_mutation_received 7341122 tap_mutation_sent 1321512 tap_opaque_received 7168 tap_opaque_sent 513 tap_vbucket_set_received 1024 threads 4 time 1352922092 total_connections 5142 uptime 2712 vb_active_curr_items 1500466 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 12857056 vb_active_itm_memory 3653305966 vb_active_meta_data_memory 118536814 vb_active_num 512 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1500466 vb_active_ops_create 1500466 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 639616 vb_active_perc_mem_resident 100 vb_active_queue_age 0 vb_active_queue_drain 4284459 vb_active_queue_fill 2962833 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 1499536 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 12857152 vb_replica_itm_memory 3649944280 vb_replica_meta_data_memory 118463344 vb_replica_num 512 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 1499536 vb_replica_ops_create 1499536 vb_replica_ops_delete 3 vb_replica_ops_reject 0 vb_replica_ops_update 1973611 vb_replica_perc_mem_resident 100 vb_replica_queue_age 0 vb_replica_queue_drain 5479277 vb_replica_queue_fill 4236877 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-14T11:42:22.074,ns_1@10.2.1.58:ns_doctor<0.1953.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,922137,75864}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,267676328}, {processes,212494384}, {processes_used,211619008}, {system,55181944}, {atom,1337385}, {atom_used,1331436}, {binary,12793496}, {code,13451143}, {ets,8913056}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7533936640}, {buffered_memory,247996416}, {free_memory,16786948096}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{3287496,5000}}, {context_switches,{14536852,0}}, {garbage_collection,{2579672,5698908656,0}}, {io,{{input,339518042},{output,18126994090}}}, {reductions,{1863876951,1006875}}, {run_queue,0}, {runtime,{437990,560}}]}, {system_stats, [{cpu_utilization_rate,2.75}, {swap_total,10684628992}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7291579434}, {couch_docs_data_size,7285575551}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7501491416}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3287}, {memory_data,{33657663488,16874905600,{<0.11.0>,43123400}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16393504 kB\nBuffers: 242184 kB\nCached: 7357360 kB\nSwapCached: 0 kB\nActive: 8973080 kB\nInactive: 7194588 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16393504 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 296 kB\nWriteback: 0 kB\nAnonPages: 8568228 kB\nMapped: 15604 kB\nSlab: 251800 kB\nPageTables: 21824 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10294800 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920229,826001}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17879.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,922141,477158}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,264735872}, {processes,207923048}, {processes_used,207061800}, {system,56812824}, {atom,1353657}, {atom_used,1352629}, {binary,13184936}, {code,13603146}, {ets,10057616}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7601266688}, {buffered_memory,251555840}, {free_memory,16457809920}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{3289942,5000}}, {context_switches,{17989998,0}}, {garbage_collection,{2968777,8119118377,0}}, {io,{{input,438355091},{output,13065713946}}}, {reductions,{2819510316,2623396}}, {run_queue,0}, {runtime,{485310,570}}]}, {system_stats, [{cpu_utilization_rate,2.75}, {swap_total,10585923584}, {swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7292394552}, {couch_docs_data_size,7286342831}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,7678974744}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3289}, {memory_data,{33657663488,17206202368,{<17926.11.0>,43123400}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16072080 kB\nBuffers: 245660 kB\nCached: 7423112 kB\nSwapCached: 0 kB\nActive: 9273468 kB\nInactive: 7194760 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16072080 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 360 kB\nWriteback: 0 kB\nAnonPages: 8799644 kB\nMapped: 15608 kB\nSlab: 272388 kB\nPageTables: 22184 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10516720 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}] [stats:debug,2012-11-14T11:43:13.092,ns_1@10.2.1.58:<0.2072.0>:stats_collector:maybe_log_stats:126](at {{1970,1,16},{7,48,42}}) Stats for bucket "default": accepting_conns 1 auth_cmds 0 auth_errors 0 bucket_active_conns 1 bucket_conns 10 bytes 7501491416 bytes_read 17748441872 bytes_written 3577171052 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 0 conn_yields 33389 connection_structures 5000 curr_connections 13 curr_conns_on_port_11209 10 curr_conns_on_port_11210 1 curr_items 1500466 curr_items_tot 3000002 curr_temp_items 0 daemon_connections 4 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_access_scanner_last_runtime 0 ep_access_scanner_num_items 0 ep_access_scanner_task_time 2012-11-15 10:00:00 ep_allow_data_loss_during_shutdown 1 ep_alog_block_size 4096 ep_alog_path /data2/default/access.log ep_alog_sleep_time 1440 ep_alog_task_time 10 ep_backend couchdb ep_bg_fetch_delay 0 ep_bg_fetched 0 ep_bg_meta_fetched 0 ep_bg_remaining_jobs 0 ep_chk_max_items 5000 ep_chk_period 1800 ep_chk_persistence_remains 0 ep_chk_persistence_timeout 10 ep_chk_remover_stime 5 ep_commit_num 28828 ep_commit_time 3 ep_commit_time_total 379413 ep_concurrentDB 1 ep_config_file ep_couch_bucket default ep_couch_host localhost ep_couch_port 11213 ep_couch_reconnect_sleeptime 250 ep_couch_response_timeout 180000 ep_data_age 1 ep_data_age_highwat 5 ep_data_traffic_enabled 0 ep_dbinit 0 ep_dbname /data2/default ep_degraded_mode 0 ep_diskqueue_drain 9763736 ep_diskqueue_fill 7199710 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_exp_pager_stime 3600 ep_expired_access 0 ep_expired_pager 0 ep_expiry_window 3 ep_failpartialwarmup 0 ep_flush_all false ep_flush_duration_total 489 ep_flushall_enabled 0 ep_flusher_state running ep_flusher_todo 0 ep_getl_default_timeout 15 ep_getl_max_timeout 30 ep_ht_locks 5 ep_ht_size 3079 ep_inconsistent_slave_chk 0 ep_initfile ep_io_num_read 0 ep_io_num_write 5613232 ep_io_read_bytes 0 ep_io_write_bytes 13292106458 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_item_num_based_new_chk 1 ep_items_rm_from_checkpoints 3101741 ep_keep_closed_chks 0 ep_klog_block_size 4096 ep_klog_compactor_queue_cap 500000 ep_klog_compactor_stime 3600 ep_klog_flush commit2 ep_klog_max_entry_ratio 10 ep_klog_max_log_size 2147483647 ep_klog_path ep_klog_sync commit2 ep_kv_size 7339250270 ep_max_checkpoints 2 ep_max_data_size 12582912000 ep_max_item_size 20971520 ep_max_size 12582912000 ep_max_txn_size 10000 ep_max_vbuckets 1024 ep_mem_high_wat 9437184000 ep_mem_high_wat 9437184000 ep_mem_low_wat 7549747200 ep_mem_low_wat 7549747200 ep_mem_tracker_enabled true ep_min_data_age 0 ep_mlog_compactor_runs 0 ep_mutation_mem_threshold 0 ep_num_access_scanner_runs 0 ep_num_eject_failures 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_ops_del_meta 0 ep_num_ops_get_meta 0 ep_num_ops_set_meta 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_oom_errors 0 ep_overhead 35900908 ep_pager_active_vb_pcnt 40 ep_pager_unbiased_period 60 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_postInitfile ep_queue_age_cap 900 ep_queue_size 0 ep_startup_time 1352919382 ep_storage_age 0 ep_storage_age_highwat 3 ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_stored_val_type ep_tap_ack_grace_period 300 ep_tap_ack_initial_sequence_number 1 ep_tap_ack_interval 1000 ep_tap_ack_window_size 10 ep_tap_backfill_resident 0.9 ep_tap_backlog_limit 5000 ep_tap_backoff_period 5 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_bg_max_pending 500 ep_tap_keepalive 300 ep_tap_noop_interval 20 ep_tap_requeue_sleep_time 0.1 ep_tap_throttle_cap_pcnt 10 ep_tap_throttle_queue_cap 1000000 ep_tap_throttle_threshold 90 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 7303250246 ep_total_del_items 3 ep_total_enqueued 7199710 ep_total_new_items 3000002 ep_total_persisted 5613232 ep_uncommitted_items 0 ep_value_size 7091900551 ep_vb0 0 ep_vb_snapshot_total 2861 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 2.0.0r_135_g40a391b ep_waitforwarmup 0 ep_warmup 1 ep_warmup_batch_size 1000 ep_warmup_dups 0 ep_warmup_min_items_threshold 100 ep_warmup_min_memory_threshold 100 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 1431 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 max_conns_on_port_11209 1000 max_conns_on_port_11210 9000 mem_used 7501491416 pid 29361 pointer_size 64 rejected_conns 0 rusage_system 94.132689 rusage_user 651.115015 tap_checkpoint_end_received 1954 tap_checkpoint_end_sent 902 tap_checkpoint_start_received 3526 tap_checkpoint_start_sent 963 tap_connect_received 1 tap_delete_received 2 tap_mutation_received 7341122 tap_mutation_sent 1321512 tap_opaque_received 7168 tap_opaque_sent 513 tap_vbucket_set_received 1024 threads 4 time 1352922192 total_connections 5142 uptime 2812 vb_active_curr_items 1500466 vb_active_eject 0 vb_active_expired 0 vb_active_ht_memory 12857056 vb_active_itm_memory 3653305966 vb_active_meta_data_memory 118536814 vb_active_num 512 vb_active_num_non_resident 0 vb_active_num_ref_ejects 0 vb_active_num_ref_items 1500466 vb_active_ops_create 1500466 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 639616 vb_active_perc_mem_resident 100 vb_active_queue_age 0 vb_active_queue_drain 4284459 vb_active_queue_fill 2962833 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_expired 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_meta_data_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_num_ref_ejects 0 vb_pending_num_ref_items 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 1499536 vb_replica_eject 0 vb_replica_expired 0 vb_replica_ht_memory 12857152 vb_replica_itm_memory 3649944280 vb_replica_meta_data_memory 118463344 vb_replica_num 512 vb_replica_num_non_resident 0 vb_replica_num_ref_ejects 0 vb_replica_num_ref_items 1499536 vb_replica_ops_create 1499536 vb_replica_ops_delete 3 vb_replica_ops_reject 0 vb_replica_ops_update 1973611 vb_replica_perc_mem_resident 100 vb_replica_queue_age 0 vb_replica_queue_drain 5479277 vb_replica_queue_fill 4236877 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_600_g7ea975a [ns_doctor:debug,2012-11-14T11:43:22.074,ns_1@10.2.1.58:ns_doctor<0.1953.0>:ns_doctor:handle_info:137]Current node statuses: [{'ns_1@10.2.1.58', [{last_heard,{1352,922197,75656}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.63',132836600}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,265078496}, {processes,209597496}, {processes_used,208722120}, {system,55481000}, {atom,1337385}, {atom_used,1331436}, {binary,12917528}, {code,13451143}, {ets,9088176}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,7535714304}, {buffered_memory,248238080}, {free_memory,16775241728}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{3347495,4999}}, {context_switches,{14731804,0}}, {garbage_collection,{2590962,5743072591,0}}, {io,{{input,342998414},{output,18129678158}}}, {reductions,{1889422365,1017848}}, {run_queue,0}, {runtime,{444550,490}}]}, {system_stats, [{cpu_utilization_rate,2.5},{swap_total,10684628992},{swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7291579434}, {couch_docs_data_size,7285575551}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1500466}, {curr_items_tot,3000002}, {mem_used,7501491416}, {vb_replica_curr_items,1499536}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3347}, {memory_data,{33657663488,16870715392,{<0.11.0>,43123400}}}, {disk_data, [{"/",49781912,6}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16382072 kB\nBuffers: 242420 kB\nCached: 7359096 kB\nSwapCached: 0 kB\nActive: 8983332 kB\nInactive: 7194556 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16382072 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 240 kB\nWriteback: 0 kB\nAnonPages: 8576244 kB\nMapped: 16716 kB\nSlab: 252012 kB\nPageTables: 22184 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10308588 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.61', [{last_heard,{1352,920229,826001}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,97173824}, {processes,57341120}, {processes_used,52147296}, {system,39832704}, {atom,1407441}, {atom_used,1381817}, {binary,1128752}, {code,14112737}, {ets,5194480}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10684628992}, {total_swap,10684628992}, {cached_memory,145350656}, {buffered_memory,176361472}, {free_memory,30178320384}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{1381877,8}}, {context_switches,{7522433,0}}, {garbage_collection,{1508791,21462389082,0}}, {io,{{input,154585148},{output,6191000892}}}, {reductions,{1814427345,14464}}, {run_queue,0}, {runtime,{424360,10}}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1381}, {memory_data,{33657663488,9177124864,{<17879.427.0>,15531672}}}, {disk_data, [{"/",49781912,20}, {"/data",886279600,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,0}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 29461468 kB\nBuffers: 172228 kB\nCached: 141944 kB\nSwapCached: 0 kB\nActive: 3125668 kB\nInactive: 122652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 29461468 kB\nSwapTotal: 10434208 kB\nSwapFree: 10434208 kB\nDirty: 796 kB\nWriteback: 0 kB\nAnonPages: 2932284 kB\nMapped: 15508 kB\nSlab: 101828 kB\nPageTables: 22016 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26868612 kB\nCommitted_AS: 10151124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6400 kB\nVmallocChunk: 34359730595 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}, {'ns_1@10.2.1.63', [{last_heard,{1352,922201,477070}}, {outgoing_replications_safeness_level,[{"default",yellow}]}, {incoming_replications_conf_hashes, [{"default",[{'ns_1@10.2.1.58',99799376}]}]}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {local_tasks,[]}, {memory, [{total,262188024}, {processes,205129872}, {processes_used,204268624}, {system,57058152}, {atom,1353657}, {atom_used,1352629}, {binary,13255712}, {code,13603146}, {ets,10232224}]}, {system_memory_data, [{system_total_memory,33657663488}, {free_swap,10585923584}, {total_swap,10585923584}, {cached_memory,7602417664}, {buffered_memory,251645952}, {free_memory,16455651328}, {total_memory,33657663488}]}, {node_storage_conf,[{db_path,"/data2"},{index_path,"/data"}]}, {statistics, [{wall_clock,{3349942,5000}}, {context_switches,{18188758,0}}, {garbage_collection,{2979880,8191820218,0}}, {io,{{input,441711317},{output,13067876302}}}, {reductions,{2851071583,2644052}}, {run_queue,0}, {runtime,{491970,580}}]}, {system_stats, [{cpu_utilization_rate,2.5},{swap_total,10585923584},{swap_used,0}]}, {interesting_stats, [{couch_docs_actual_disk_size,7292394552}, {couch_docs_data_size,7286342831}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,1499536}, {curr_items_tot,3000002}, {mem_used,7678974744}, {vb_replica_curr_items,1500466}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8cffe61"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-dd6de6a-git"}, {mnesia,"4.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-dd6de6a-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-dd6de6a-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.0.0-1945-rel-enterprise"}, {mochiweb,"1.4.1"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3349}, {memory_data,{33657663488,17199345664,{<17926.11.0>,43123400}}}, {disk_data, [{"/",49781912,19}, {"/data",886371884,1}, {"/dev/shm",16434404,0}, {"/data2",1922859824,1}]}, {meminfo, <<"MemTotal: 32868812 kB\nMemFree: 16069972 kB\nBuffers: 245748 kB\nCached: 7424236 kB\nSwapCached: 0 kB\nActive: 9273732 kB\nInactive: 7195816 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 32868812 kB\nLowFree: 16069972 kB\nSwapTotal: 10337816 kB\nSwapFree: 10337816 kB\nDirty: 256 kB\nWriteback: 0 kB\nAnonPages: 8799636 kB\nMapped: 15608 kB\nSlab: 272356 kB\nPageTables: 22180 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 26772220 kB\nCommitted_AS: 10521272 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 6352 kB\nVmallocChunk: 34359730643 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}]}]