============================================================================== membase logs mbbrowse_logs ============================================================================== rb: reading report...done. rb: reading report...done. rb: reading report...done. rb: reading report...done. rb: reading report...done. rb: reading report...done. rb: reading report...done. rb: reading report...done. rb: reading report...done. rb: reading report...done. rb: reading report...done. rb: reading report...done. rb: reading report...done. rb: reading report...done. rb: reading report...done. rb: reading report...done. rb: reading report...done. rb: reading report...done. rb: reading report...done. rb: reading report...done. INFO REPORT <5883.111.192> 2011-12-29 05:47:06 =============================================================================== ns_1@192.168.24.61:<5883.111.192>:stats_collector:83: Stats for bucket "JDPCachePrimary": accepting_conns 1 auth_cmds 10369 auth_errors 0 bucket_active_conns 1 bucket_conns 1 bytes_read 53441318550 bytes_written 45285002521 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 29975715 cmd_set 445274018 conn_yields 61205 connection_structures 1931 curr_connections 11 curr_items 297280049 curr_items_tot 297280049 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 466 delete_misses 174808 ep_bg_fetched 1661206 ep_bg_load 4365803470 ep_bg_load_avg 2628 ep_bg_max_load 2020165 ep_bg_max_wait 2929571 ep_bg_min_load 1 ep_bg_min_wait 10 ep_bg_num_samples 1661198 ep_bg_wait 48864908595 ep_bg_wait_avg 29415 ep_commit_num 818500 ep_commit_time 0 ep_commit_time_total 179775 ep_data_age 0 ep_data_age_highwat 2571 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 1 ep_dbname /opt/membase/var/lib/membase/data/JDPCachePrimary-data/JDPCachePrimary ep_dbshards 4 ep_diskqueue_drain 358215763 ep_diskqueue_fill 358215763 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 1256 ep_flush_duration_total 251908 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 1661206 ep_io_num_write 357979281 ep_io_read_bytes 145605718 ep_io_write_bytes 6012819006520 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 356086874 ep_kv_size 57356020515 ep_latency_arith_cmd 46093 ep_latency_get_cmd 101813590 ep_latency_store_cmd 515449025 ep_max_data_size 81788928000 ep_max_txn_size 1000 ep_mem_high_wat 61341696000 ep_mem_low_wat 49073356800 ep_min_data_age 0 ep_num_active_non_resident 94237869 ep_num_checkpoint_remover_runs354155 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 489 ep_num_non_resident 94237869 ep_num_not_my_vbuckets 0 ep_num_pager_runs 2 ep_num_value_ejects 95895079 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 1611440768 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 3597 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 62797498 ep_too_young 0 ep_total_cache_size 58930442359 ep_total_del_items 416 ep_total_enqueued 358215763 ep_total_new_items 297280465 ep_total_persisted 357979697 ep_uncommitted_items 0 ep_value_size 15808691965 ep_vb_total 512 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.1.1_39_g4bd26a3 ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 8853 get_hits 7311306 get_misses 22664409 incr_hits 44431 incr_misses 1662 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 58967461283 pid 17177 pointer_size 64 rejected_conns 0 rusage_system 91967.811767 rusage_user 249391.466734 threads 4 time 1325166426 total_connections 30202 uptime 3439357 vb_active_curr_items 297280049 vb_active_eject 95895079 vb_active_ht_memory 1610809344 vb_active_itm_memory 53788643543 vb_active_num 512 vb_active_num_non_resident 94237869 vb_active_ops_create 297280465 vb_active_ops_delete 416 vb_active_ops_reject 0 vb_active_ops_update 60698816 vb_active_perc_mem_resident 68 vb_active_queue_age 0 vb_active_queue_drain 358215763 vb_active_queue_fill 358215763 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version UNKNOWN INFO REPORT <5883.136.0> 2011-12-29 05:47:45 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 05:47:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,166466,408752}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,101865544}, {processes,31382032}, {processes_used,28707760}, {system,70483512}, {atom,903241}, {atom_used,880290}, {binary,1328752}, {code,8604866}, {ets,38547536}]}, {system_stats, [{cpu_utilization_rate,0.4993757802746567}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3439395}, {memory_data,{101368545280,100267139072,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1075956 kB\nBuffers: 283552 kB\nCached: 32671860 kB\nSwapCached: 5096 kB\nActive: 82503728 kB\nInactive: 13858456 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1075956 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 1040 kB\nWriteback: 0 kB\nAnonPages: 63399180 kB\nMapped: 16728 kB\nSlab: 1346340 kB\nPageTables: 171512 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86137112 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33455984640}, {buffered_memory,290357248}, {free_memory,1101778944}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3439380475,1}}, {context_switches,{1912208927,0}}, {garbage_collection,{247847346,3900366222527,0}}, {io,{{input,69507193998},{output,131864076949}}}, {reductions,{1585374288185,2420223}}, {run_queue,0}, {runtime,{130318410,200}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,166465,404750}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,83770424}, {processes,14615512}, {processes_used,14579816}, {system,69154912}, {atom,828337}, {atom_used,824728}, {binary,1164152}, {code,7914132}, {ets,38317128}]}, {system_stats, [{cpu_utilization_rate,0.25}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3439375}, {memory_data,{101368545280,100841299968,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 522060 kB\nBuffers: 283840 kB\nCached: 31388476 kB\nSwapCached: 3836 kB\nActive: 82923152 kB\nInactive: 13851768 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 522060 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 292 kB\nWriteback: 0 kB\nAnonPages: 65094328 kB\nMapped: 17876 kB\nSlab: 1381148 kB\nPageTables: 278864 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141208104 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32141799424}, {buffered_memory,290652160}, {free_memory,534589440}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3439365640,0}}, {context_switches,{750976312,0}}, {garbage_collection,{185603931,674531920537,0}}, {io,{{input,58219227978},{output,130442087828}}}, {reductions,{324444829292,1387758}}, {run_queue,0}, {runtime,{37289380,170}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 05:48:15 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 05:48:30 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.111.192> 2011-12-29 05:48:46 =============================================================================== ns_1@192.168.24.61:<5883.111.192>:stats_collector:83: Stats for bucket "JDPCachePrimary": accepting_conns 1 auth_cmds 10369 auth_errors 0 bucket_active_conns 1 bucket_conns 1 bytes_read 53441325710 bytes_written 45286670661 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 29975715 cmd_set 445274018 conn_yields 61205 connection_structures 1931 curr_connections 11 curr_items 297280049 curr_items_tot 297280049 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 466 delete_misses 174808 ep_bg_fetched 1661206 ep_bg_load 4365803470 ep_bg_load_avg 2628 ep_bg_max_load 2020165 ep_bg_max_wait 2929571 ep_bg_min_load 1 ep_bg_min_wait 10 ep_bg_num_samples 1661198 ep_bg_wait 48864908595 ep_bg_wait_avg 29415 ep_commit_num 818500 ep_commit_time 0 ep_commit_time_total 179775 ep_data_age 0 ep_data_age_highwat 2571 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 1 ep_dbname /opt/membase/var/lib/membase/data/JDPCachePrimary-data/JDPCachePrimary ep_dbshards 4 ep_diskqueue_drain 358215763 ep_diskqueue_fill 358215763 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 1256 ep_flush_duration_total 251908 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 1661206 ep_io_num_write 357979281 ep_io_read_bytes 145605718 ep_io_write_bytes 6012819006520 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 356086874 ep_kv_size 57356020515 ep_latency_arith_cmd 46093 ep_latency_get_cmd 101813590 ep_latency_store_cmd 515449025 ep_max_data_size 81788928000 ep_max_txn_size 1000 ep_mem_high_wat 61341696000 ep_mem_low_wat 49073356800 ep_min_data_age 0 ep_num_active_non_resident 94237869 ep_num_checkpoint_remover_runs354175 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 489 ep_num_non_resident 94237869 ep_num_not_my_vbuckets 0 ep_num_pager_runs 2 ep_num_value_ejects 95895079 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 1611440768 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 3597 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 62797498 ep_too_young 0 ep_total_cache_size 58930442359 ep_total_del_items 416 ep_total_enqueued 358215763 ep_total_new_items 297280465 ep_total_persisted 357979697 ep_uncommitted_items 0 ep_value_size 15808691965 ep_vb_total 512 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.1.1_39_g4bd26a3 ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 8853 get_hits 7311306 get_misses 22664409 incr_hits 44431 incr_misses 1662 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 58967461283 pid 17177 pointer_size 64 rejected_conns 0 rusage_system 91967.833764 rusage_user 249391.564719 threads 4 time 1325166526 total_connections 30202 uptime 3439457 vb_active_curr_items 297280049 vb_active_eject 95895079 vb_active_ht_memory 1610809344 vb_active_itm_memory 53788643543 vb_active_num 512 vb_active_num_non_resident 94237869 vb_active_ops_create 297280465 vb_active_ops_delete 416 vb_active_ops_reject 0 vb_active_ops_update 60698816 vb_active_perc_mem_resident 68 vb_active_queue_age 0 vb_active_queue_drain 358215763 vb_active_queue_fill 358215763 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version UNKNOWN INFO REPORT <5883.1095.0> 2011-12-29 05:48:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,166526,409577}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,101798392}, {processes,31673304}, {processes_used,28999032}, {system,70125088}, {atom,903241}, {atom_used,880290}, {binary,1440040}, {code,8604866}, {ets,38078024}]}, {system_stats, [{cpu_utilization_rate,0.625}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3439455}, {memory_data,{101368545280,100266594304,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1075520 kB\nBuffers: 283552 kB\nCached: 32672628 kB\nSwapCached: 5096 kB\nActive: 82503488 kB\nInactive: 13859116 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1075520 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 4 kB\nWriteback: 0 kB\nAnonPages: 63398440 kB\nMapped: 16728 kB\nSlab: 1346124 kB\nPageTables: 171512 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86136540 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33456771072}, {buffered_memory,290357248}, {free_memory,1101332480}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3439440476,2}}, {context_switches,{1912225320,0}}, {garbage_collection,{247850652,3900378235230,0}}, {io,{{input,69507451382},{output,131864928318}}}, {reductions,{1585386331673,2514491}}, {run_queue,0}, {runtime,{130319150,170}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,166525,404545}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,83597112}, {processes,14755368}, {processes_used,14719672}, {system,68841744}, {atom,828337}, {atom_used,824728}, {binary,1261576}, {code,7914132}, {ets,37906624}]}, {system_stats, [{cpu_utilization_rate,0.625}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3439435}, {memory_data,{101368545280,100834684928,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 520428 kB\nBuffers: 283840 kB\nCached: 31389228 kB\nSwapCached: 3836 kB\nActive: 82924064 kB\nInactive: 13852500 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 520428 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 176 kB\nWriteback: 0 kB\nAnonPages: 65095428 kB\nMapped: 17876 kB\nSlab: 1381040 kB\nPageTables: 278864 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141211052 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32142569472}, {buffered_memory,290652160}, {free_memory,532918272}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3439425641,1}}, {context_switches,{750988960,0}}, {garbage_collection,{185607046,674543058217,0}}, {io,{{input,58219387307},{output,130443008143}}}, {reductions,{324450282097,1395796}}, {run_queue,0}, {runtime,{37289980,150}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 05:49:02 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 05:49:47 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 05:49:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,166586,408634}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102080032}, {processes,31964656}, {processes_used,29290384}, {system,70115376}, {atom,903241}, {atom_used,880290}, {binary,1396992}, {code,8604866}, {ets,38111248}]}, {system_stats, [{cpu_utilization_rate,0.25}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3439515}, {memory_data,{101368545280,100265304064,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1077264 kB\nBuffers: 283552 kB\nCached: 32673416 kB\nSwapCached: 5096 kB\nActive: 82502148 kB\nInactive: 13859948 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1077264 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 512 kB\nWriteback: 0 kB\nAnonPages: 63397428 kB\nMapped: 16728 kB\nSlab: 1346012 kB\nPageTables: 171512 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86135896 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33457577984}, {buffered_memory,290357248}, {free_memory,1103118336}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3439500476,2}}, {context_switches,{1912242288,0}}, {garbage_collection,{247853962,3900390614785,0}}, {io,{{input,69507641599},{output,131865845097}}}, {reductions,{1585398872208,2573860}}, {run_queue,0}, {runtime,{130319960,180}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,166585,404508}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,82996680}, {processes,14201160}, {processes_used,14165464}, {system,68795520}, {atom,828337}, {atom_used,824728}, {binary,1180912}, {code,7914132}, {ets,37940752}]}, {system_stats, [{cpu_utilization_rate,0.375}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3439495}, {memory_data,{101368545280,100836032512,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 519848 kB\nBuffers: 283840 kB\nCached: 31390016 kB\nSwapCached: 3836 kB\nActive: 82923752 kB\nInactive: 13853196 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 519848 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 488 kB\nWriteback: 0 kB\nAnonPages: 65095376 kB\nMapped: 17876 kB\nSlab: 1380992 kB\nPageTables: 278864 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141208064 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32143376384}, {buffered_memory,290652160}, {free_memory,532324352}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3439485640,0}}, {context_switches,{751001828,0}}, {garbage_collection,{185610324,674554418514,0}}, {io,{{input,58219587818},{output,130443877304}}}, {reductions,{324456016012,1391906}}, {run_queue,0}, {runtime,{37290610,160}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 05:50:24 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.111.192> 2011-12-29 05:50:26 =============================================================================== ns_1@192.168.24.61:<5883.111.192>:stats_collector:83: Stats for bucket "JDPCachePrimary": accepting_conns 1 auth_cmds 10369 auth_errors 0 bucket_active_conns 1 bucket_conns 1 bytes_read 53441332870 bytes_written 45288338801 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 29975715 cmd_set 445274018 conn_yields 61205 connection_structures 1931 curr_connections 11 curr_items 297280049 curr_items_tot 297280049 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 466 delete_misses 174808 ep_bg_fetched 1661206 ep_bg_load 4365803470 ep_bg_load_avg 2628 ep_bg_max_load 2020165 ep_bg_max_wait 2929571 ep_bg_min_load 1 ep_bg_min_wait 10 ep_bg_num_samples 1661198 ep_bg_wait 48864908595 ep_bg_wait_avg 29415 ep_commit_num 818500 ep_commit_time 0 ep_commit_time_total 179775 ep_data_age 0 ep_data_age_highwat 2571 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 1 ep_dbname /opt/membase/var/lib/membase/data/JDPCachePrimary-data/JDPCachePrimary ep_dbshards 4 ep_diskqueue_drain 358215763 ep_diskqueue_fill 358215763 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 1256 ep_flush_duration_total 251908 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 1661206 ep_io_num_write 357979281 ep_io_read_bytes 145605718 ep_io_write_bytes 6012819006520 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 356086874 ep_kv_size 57356020515 ep_latency_arith_cmd 46093 ep_latency_get_cmd 101813590 ep_latency_store_cmd 515449025 ep_max_data_size 81788928000 ep_max_txn_size 1000 ep_mem_high_wat 61341696000 ep_mem_low_wat 49073356800 ep_min_data_age 0 ep_num_active_non_resident 94237869 ep_num_checkpoint_remover_runs354194 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 489 ep_num_non_resident 94237869 ep_num_not_my_vbuckets 0 ep_num_pager_runs 2 ep_num_value_ejects 95895079 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 1611440768 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 3597 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 62797498 ep_too_young 0 ep_total_cache_size 58930442359 ep_total_del_items 416 ep_total_enqueued 358215763 ep_total_new_items 297280465 ep_total_persisted 357979697 ep_uncommitted_items 0 ep_value_size 15808691965 ep_vb_total 512 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.1.1_39_g4bd26a3 ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 8853 get_hits 7311306 get_misses 22664409 incr_hits 44431 incr_misses 1662 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 58967461283 pid 17177 pointer_size 64 rejected_conns 0 rusage_system 91967.865759 rusage_user 249391.638708 threads 4 time 1325166626 total_connections 30202 uptime 3439557 vb_active_curr_items 297280049 vb_active_eject 95895079 vb_active_ht_memory 1610809344 vb_active_itm_memory 53788643543 vb_active_num 512 vb_active_num_non_resident 94237869 vb_active_ops_create 297280465 vb_active_ops_delete 416 vb_active_ops_reject 0 vb_active_ops_update 60698816 vb_active_perc_mem_resident 68 vb_active_queue_age 0 vb_active_queue_drain 358215763 vb_active_queue_fill 358215763 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version UNKNOWN INFO REPORT <5883.1095.0> 2011-12-29 05:50:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,166646,409570}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102199256}, {processes,31906336}, {processes_used,29232064}, {system,70292920}, {atom,903241}, {atom_used,880290}, {binary,1417856}, {code,8604866}, {ets,38268336}]}, {system_stats, [{cpu_utilization_rate,15.39424280350438}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3439575}, {memory_data,{101368545280,100265009152,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1077044 kB\nBuffers: 283560 kB\nCached: 32673068 kB\nSwapCached: 5096 kB\nActive: 82502884 kB\nInactive: 13859512 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1077044 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 364 kB\nWriteback: 0 kB\nAnonPages: 63397556 kB\nMapped: 16728 kB\nSlab: 1346040 kB\nPageTables: 171516 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86137216 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33457221632}, {buffered_memory,290365440}, {free_memory,1102893056}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3439560474,0}}, {context_switches,{1912259290,0}}, {garbage_collection,{247857339,3900403201692,0}}, {io,{{input,69510354794},{output,131868717460}}}, {reductions,{1585410859061,2505382}}, {run_queue,0}, {runtime,{130320820,180}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,166645,404343}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,83676144}, {processes,14672680}, {processes_used,14636984}, {system,69003464}, {atom,828337}, {atom_used,824728}, {binary,1234960}, {code,7914132}, {ets,38095432}]}, {system_stats, [{cpu_utilization_rate,0.5006257822277848}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3439555}, {memory_data,{101368545280,100836020224,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 520200 kB\nBuffers: 283844 kB\nCached: 31389656 kB\nSwapCached: 3836 kB\nActive: 82923944 kB\nInactive: 13852816 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 520200 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 292 kB\nWriteback: 0 kB\nAnonPages: 65095208 kB\nMapped: 17876 kB\nSlab: 1381048 kB\nPageTables: 278856 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141207152 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32143007744}, {buffered_memory,290656256}, {free_memory,532684800}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3439545641,1}}, {context_switches,{751015367,0}}, {garbage_collection,{185613780,674566149493,0}}, {io,{{input,58222338410},{output,130446713566}}}, {reductions,{324461611167,1422525}}, {run_queue,0}, {runtime,{37291290,140}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 05:51:10 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 05:51:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,166706,408421}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102579696}, {processes,32155552}, {processes_used,29481280}, {system,70424144}, {atom,903241}, {atom_used,880290}, {binary,1516752}, {code,8604866}, {ets,38300384}]}, {system_stats, [{cpu_utilization_rate,0.375}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3439635}, {memory_data,{101368545280,100318466048,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 965296 kB\nBuffers: 283560 kB\nCached: 32673932 kB\nSwapCached: 5096 kB\nActive: 82611800 kB\nInactive: 13860080 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 965296 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 3444 kB\nWriteback: 0 kB\nAnonPages: 63506060 kB\nMapped: 24076 kB\nSlab: 1347244 kB\nPageTables: 172444 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86243668 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33458106368}, {buffered_memory,290365440}, {free_memory,988463104}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3439620476,1}}, {context_switches,{1912275559,0}}, {garbage_collection,{247860544,3900415213416,0}}, {io,{{input,69510531716},{output,131869633551}}}, {reductions,{1585422778081,2434931}}, {run_queue,0}, {runtime,{130321590,190}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,166705,404627}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,83450912}, {processes,14415776}, {processes_used,14380080}, {system,69035136}, {atom,828337}, {atom_used,824728}, {binary,1234368}, {code,7914132}, {ets,38127440}]}, {system_stats, [{cpu_utilization_rate,13.14142678347935}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3439615}, {memory_data,{101368545280,100836270080,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 432308 kB\nBuffers: 283848 kB\nCached: 31390476 kB\nSwapCached: 3836 kB\nActive: 83009420 kB\nInactive: 13853520 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 432308 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 3280 kB\nWriteback: 0 kB\nAnonPages: 65180632 kB\nMapped: 25208 kB\nSlab: 1381780 kB\nPageTables: 279700 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141293068 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32143847424}, {buffered_memory,290660352}, {free_memory,442683392}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3439605640,0}}, {context_switches,{751027894,0}}, {garbage_collection,{185617138,674577322154,0}}, {io,{{input,58222536717},{output,130447551359}}}, {reductions,{324467065797,1364812}}, {run_queue,0}, {runtime,{37291890,140}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 05:52:05 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.111.192> 2011-12-29 05:52:06 =============================================================================== ns_1@192.168.24.61:<5883.111.192>:stats_collector:83: Stats for bucket "JDPCachePrimary": accepting_conns 1 auth_cmds 10369 auth_errors 0 bucket_active_conns 1 bucket_conns 1 bytes_read 53441340030 bytes_written 45290006941 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 29975715 cmd_set 445274018 conn_yields 61205 connection_structures 1931 curr_connections 11 curr_items 297280049 curr_items_tot 297280049 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 466 delete_misses 174808 ep_bg_fetched 1661206 ep_bg_load 4365803470 ep_bg_load_avg 2628 ep_bg_max_load 2020165 ep_bg_max_wait 2929571 ep_bg_min_load 1 ep_bg_min_wait 10 ep_bg_num_samples 1661198 ep_bg_wait 48864908595 ep_bg_wait_avg 29415 ep_commit_num 818500 ep_commit_time 0 ep_commit_time_total 179775 ep_data_age 0 ep_data_age_highwat 2571 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 1 ep_dbname /opt/membase/var/lib/membase/data/JDPCachePrimary-data/JDPCachePrimary ep_dbshards 4 ep_diskqueue_drain 358215763 ep_diskqueue_fill 358215763 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 1256 ep_flush_duration_total 251908 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 1661206 ep_io_num_write 357979281 ep_io_read_bytes 145605718 ep_io_write_bytes 6012819006520 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 356086874 ep_kv_size 57356020515 ep_latency_arith_cmd 46093 ep_latency_get_cmd 101813590 ep_latency_store_cmd 515449025 ep_max_data_size 81788928000 ep_max_txn_size 1000 ep_mem_high_wat 61341696000 ep_mem_low_wat 49073356800 ep_min_data_age 0 ep_num_active_non_resident 94237869 ep_num_checkpoint_remover_runs354214 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 489 ep_num_non_resident 94237869 ep_num_not_my_vbuckets 0 ep_num_pager_runs 2 ep_num_value_ejects 95895079 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 1611440768 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 3597 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 62797498 ep_too_young 0 ep_total_cache_size 58930442359 ep_total_del_items 416 ep_total_enqueued 358215763 ep_total_new_items 297280465 ep_total_persisted 357979697 ep_uncommitted_items 0 ep_value_size 15808691965 ep_vb_total 512 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.1.1_39_g4bd26a3 ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 8853 get_hits 7311306 get_misses 22664409 incr_hits 44431 incr_misses 1662 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 58967461283 pid 17177 pointer_size 64 rejected_conns 0 rusage_system 91967.894754 rusage_user 249391.729694 threads 4 time 1325166726 total_connections 30202 uptime 3439657 vb_active_curr_items 297280049 vb_active_eject 95895079 vb_active_ht_memory 1610809344 vb_active_itm_memory 53788643543 vb_active_num 512 vb_active_num_non_resident 94237869 vb_active_ops_create 297280465 vb_active_ops_delete 416 vb_active_ops_reject 0 vb_active_ops_update 60698816 vb_active_perc_mem_resident 68 vb_active_queue_age 0 vb_active_queue_drain 358215763 vb_active_queue_fill 358215763 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version UNKNOWN INFO REPORT <5883.136.0> 2011-12-29 05:52:48 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 05:52:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,166766,408322}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102282064}, {processes,31588800}, {processes_used,28914528}, {system,70693264}, {atom,903241}, {atom_used,880290}, {binary,1639624}, {code,8604866}, {ets,38446624}]}, {system_stats, [{cpu_utilization_rate,0.2503128911138924}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3439695}, {memory_data,{101368545280,100268077056,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1073704 kB\nBuffers: 283560 kB\nCached: 32674704 kB\nSwapCached: 5096 kB\nActive: 82504640 kB\nInactive: 13860864 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1073704 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 564 kB\nWriteback: 0 kB\nAnonPages: 63399620 kB\nMapped: 16728 kB\nSlab: 1346032 kB\nPageTables: 171516 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86135692 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33458896896}, {buffered_memory,290365440}, {free_memory,1099472896}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3439680476,2}}, {context_switches,{1912291881,0}}, {garbage_collection,{247863721,3900427192210,0}}, {io,{{input,69510713957},{output,131870486714}}}, {reductions,{1585434818661,2511456}}, {run_queue,2}, {runtime,{130322390,200}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,166765,404481}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,84017264}, {processes,14835008}, {processes_used,14799312}, {system,69182256}, {atom,828337}, {atom_used,824728}, {binary,1235448}, {code,7914132}, {ets,38273568}]}, {system_stats, [{cpu_utilization_rate,0.2503128911138924}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3439675}, {memory_data,{101368545280,100947390464,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 518512 kB\nBuffers: 283848 kB\nCached: 31391304 kB\nSwapCached: 3836 kB\nActive: 82924284 kB\nInactive: 13854228 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 518512 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 660 kB\nWriteback: 0 kB\nAnonPages: 65095664 kB\nMapped: 17876 kB\nSlab: 1381132 kB\nPageTables: 278856 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141208444 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32144695296}, {buffered_memory,290660352}, {free_memory,530956288}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3439665641,1}}, {context_switches,{751040318,0}}, {garbage_collection,{185620294,674588514795,0}}, {io,{{input,58222690294},{output,130448400752}}}, {reductions,{324472565081,1402573}}, {run_queue,0}, {runtime,{37292500,160}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 05:53:35 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 05:53:43 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.111.192> 2011-12-29 05:53:46 =============================================================================== ns_1@192.168.24.61:<5883.111.192>:stats_collector:83: Stats for bucket "JDPCachePrimary": accepting_conns 1 auth_cmds 10369 auth_errors 0 bucket_active_conns 1 bucket_conns 1 bytes_read 53441347190 bytes_written 45291675081 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 29975715 cmd_set 445274018 conn_yields 61205 connection_structures 1931 curr_connections 11 curr_items 297280049 curr_items_tot 297280049 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 466 delete_misses 174808 ep_bg_fetched 1661206 ep_bg_load 4365803470 ep_bg_load_avg 2628 ep_bg_max_load 2020165 ep_bg_max_wait 2929571 ep_bg_min_load 1 ep_bg_min_wait 10 ep_bg_num_samples 1661198 ep_bg_wait 48864908595 ep_bg_wait_avg 29415 ep_commit_num 818500 ep_commit_time 0 ep_commit_time_total 179775 ep_data_age 0 ep_data_age_highwat 2571 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 1 ep_dbname /opt/membase/var/lib/membase/data/JDPCachePrimary-data/JDPCachePrimary ep_dbshards 4 ep_diskqueue_drain 358215763 ep_diskqueue_fill 358215763 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 1256 ep_flush_duration_total 251908 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 1661206 ep_io_num_write 357979281 ep_io_read_bytes 145605718 ep_io_write_bytes 6012819006520 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 356086874 ep_kv_size 57356020515 ep_latency_arith_cmd 46093 ep_latency_get_cmd 101813590 ep_latency_store_cmd 515449025 ep_max_data_size 81788928000 ep_max_txn_size 1000 ep_mem_high_wat 61341696000 ep_mem_low_wat 49073356800 ep_min_data_age 0 ep_num_active_non_resident 94237869 ep_num_checkpoint_remover_runs354234 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 489 ep_num_non_resident 94237869 ep_num_not_my_vbuckets 0 ep_num_pager_runs 2 ep_num_value_ejects 95895079 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 1611440768 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 3597 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 62797498 ep_too_young 0 ep_total_cache_size 58930442359 ep_total_del_items 416 ep_total_enqueued 358215763 ep_total_new_items 297280465 ep_total_persisted 357979697 ep_uncommitted_items 0 ep_value_size 15808691965 ep_vb_total 512 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.1.1_39_g4bd26a3 ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 8853 get_hits 7311306 get_misses 22664409 incr_hits 44431 incr_misses 1662 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 58967461283 pid 17177 pointer_size 64 rejected_conns 0 rusage_system 91967.917751 rusage_user 249391.831678 threads 4 time 1325166826 total_connections 30202 uptime 3439757 vb_active_curr_items 297280049 vb_active_eject 95895079 vb_active_ht_memory 1610809344 vb_active_itm_memory 53788643543 vb_active_num 512 vb_active_num_non_resident 94237869 vb_active_ops_create 297280465 vb_active_ops_delete 416 vb_active_ops_reject 0 vb_active_ops_update 60698816 vb_active_perc_mem_resident 68 vb_active_queue_age 0 vb_active_queue_drain 358215763 vb_active_queue_fill 358215763 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version UNKNOWN INFO REPORT <5883.1095.0> 2011-12-29 05:53:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,166826,408907}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102090296}, {processes,31330912}, {processes_used,28656640}, {system,70759384}, {atom,903241}, {atom_used,880290}, {binary,1557808}, {code,8604866}, {ets,38594768}]}, {system_stats, [{cpu_utilization_rate,0.2503128911138924}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3439755}, {memory_data,{101368545280,100266827776,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1075784 kB\nBuffers: 283560 kB\nCached: 32674988 kB\nSwapCached: 5096 kB\nActive: 82502524 kB\nInactive: 13861316 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1075784 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 272 kB\nWriteback: 0 kB\nAnonPages: 63397508 kB\nMapped: 16728 kB\nSlab: 1345968 kB\nPageTables: 171520 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86135020 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33459187712}, {buffered_memory,290365440}, {free_memory,1101602816}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3439740475,0}}, {context_switches,{1912308798,0}}, {garbage_collection,{247867077,3900439840244,0}}, {io,{{input,69513371957},{output,131873970076}}}, {reductions,{1585446773143,2446483}}, {run_queue,0}, {runtime,{130323280,190}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,166825,404399}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,84238160}, {processes,14833144}, {processes_used,14797448}, {system,69405016}, {atom,828337}, {atom_used,824728}, {binary,1367712}, {code,7914132}, {ets,38363992}]}, {system_stats, [{cpu_utilization_rate,0.25}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3439735}, {memory_data,{101368545280,100837347328,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 518796 kB\nBuffers: 283848 kB\nCached: 31391612 kB\nSwapCached: 3836 kB\nActive: 82923880 kB\nInactive: 13854680 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 518796 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 312 kB\nWriteback: 0 kB\nAnonPages: 65094832 kB\nMapped: 17876 kB\nSlab: 1381024 kB\nPageTables: 278860 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141210884 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32145010688}, {buffered_memory,290660352}, {free_memory,531374080}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3439725641,0}}, {context_switches,{751053853,0}}, {garbage_collection,{185623690,674600285302,0}}, {io,{{input,58225342381},{output,130451872871}}}, {reductions,{324478370570,1386795}}, {run_queue,0}, {runtime,{37293180,160}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 05:53:54 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 05:54:48 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 05:54:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,166886,409148}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102377168}, {processes,31954816}, {processes_used,29280544}, {system,70422352}, {atom,903241}, {atom_used,880290}, {binary,1690008}, {code,8604866}, {ets,38125576}]}, {system_stats, [{cpu_utilization_rate,0.5}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3439815}, {memory_data,{101368545280,100267499520,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1074756 kB\nBuffers: 283560 kB\nCached: 32675764 kB\nSwapCached: 5096 kB\nActive: 82503244 kB\nInactive: 13861908 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1074756 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 88 kB\nWriteback: 0 kB\nAnonPages: 63397596 kB\nMapped: 16728 kB\nSlab: 1346008 kB\nPageTables: 171516 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86134708 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33459982336}, {buffered_memory,290365440}, {free_memory,1100550144}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3439800475,1}}, {context_switches,{1912325046,0}}, {garbage_collection,{247870424,3900451851410,0}}, {io,{{input,69513640919},{output,131874842273}}}, {reductions,{1585458709572,2404422}}, {run_queue,0}, {runtime,{130324080,190}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,166885,404420}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,83134416}, {processes,14154920}, {processes_used,14119224}, {system,68979496}, {atom,828337}, {atom_used,824728}, {binary,1353376}, {code,7914132}, {ets,37952896}]}, {system_stats, [{cpu_utilization_rate,0.6242197253433208}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3439795}, {memory_data,{101368545280,100837298176,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 517592 kB\nBuffers: 283848 kB\nCached: 31392372 kB\nSwapCached: 3836 kB\nActive: 82924692 kB\nInactive: 13855424 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 517592 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 44 kB\nWriteback: 0 kB\nAnonPages: 65095776 kB\nMapped: 17876 kB\nSlab: 1380996 kB\nPageTables: 278860 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141207740 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32145788928}, {buffered_memory,290660352}, {free_memory,530014208}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3439785641,1}}, {context_switches,{751066230,0}}, {garbage_collection,{185626875,674611428219,0}}, {io,{{input,58225537377},{output,130452798860}}}, {reductions,{324483854343,1407483}}, {run_queue,1}, {runtime,{37293780,160}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 05:55:22 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.111.192> 2011-12-29 05:55:26 =============================================================================== ns_1@192.168.24.61:<5883.111.192>:stats_collector:83: Stats for bucket "JDPCachePrimary": accepting_conns 1 auth_cmds 10369 auth_errors 0 bucket_active_conns 1 bucket_conns 1 bytes_read 53441354350 bytes_written 45293343221 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 29975715 cmd_set 445274018 conn_yields 61205 connection_structures 1931 curr_connections 11 curr_items 297280049 curr_items_tot 297280049 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 466 delete_misses 174808 ep_bg_fetched 1661206 ep_bg_load 4365803470 ep_bg_load_avg 2628 ep_bg_max_load 2020165 ep_bg_max_wait 2929571 ep_bg_min_load 1 ep_bg_min_wait 10 ep_bg_num_samples 1661198 ep_bg_wait 48864908595 ep_bg_wait_avg 29415 ep_commit_num 818500 ep_commit_time 0 ep_commit_time_total 179775 ep_data_age 0 ep_data_age_highwat 2571 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 1 ep_dbname /opt/membase/var/lib/membase/data/JDPCachePrimary-data/JDPCachePrimary ep_dbshards 4 ep_diskqueue_drain 358215763 ep_diskqueue_fill 358215763 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 1256 ep_flush_duration_total 251908 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 1661206 ep_io_num_write 357979281 ep_io_read_bytes 145605718 ep_io_write_bytes 6012819006520 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 356086874 ep_kv_size 57356020515 ep_latency_arith_cmd 46093 ep_latency_get_cmd 101813590 ep_latency_store_cmd 515449025 ep_max_data_size 81788928000 ep_max_txn_size 1000 ep_mem_high_wat 61341696000 ep_mem_low_wat 49073356800 ep_min_data_age 0 ep_num_active_non_resident 94237869 ep_num_checkpoint_remover_runs354254 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 489 ep_num_non_resident 94237869 ep_num_not_my_vbuckets 0 ep_num_pager_runs 2 ep_num_value_ejects 95895079 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 1611440768 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 3597 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 62797498 ep_too_young 0 ep_total_cache_size 58930442359 ep_total_del_items 416 ep_total_enqueued 358215763 ep_total_new_items 297280465 ep_total_persisted 357979697 ep_uncommitted_items 0 ep_value_size 15808691965 ep_vb_total 512 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.1.1_39_g4bd26a3 ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 8853 get_hits 7311306 get_misses 22664409 incr_hits 44431 incr_misses 1662 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 58967461283 pid 17177 pointer_size 64 rejected_conns 0 rusage_system 91967.943747 rusage_user 249391.913666 threads 4 time 1325166926 total_connections 30202 uptime 3439857 vb_active_curr_items 297280049 vb_active_eject 95895079 vb_active_ht_memory 1610809344 vb_active_itm_memory 53788643543 vb_active_num 512 vb_active_num_non_resident 94237869 vb_active_ops_create 297280465 vb_active_ops_delete 416 vb_active_ops_reject 0 vb_active_ops_update 60698816 vb_active_perc_mem_resident 68 vb_active_queue_age 0 vb_active_queue_drain 358215763 vb_active_queue_fill 358215763 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version UNKNOWN INFO REPORT <5883.136.0> 2011-12-29 05:55:33 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 05:55:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,166946,409068}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102509296}, {processes,32116040}, {processes_used,29441768}, {system,70393256}, {atom,903241}, {atom_used,880290}, {binary,1627792}, {code,8604866}, {ets,38158408}]}, {system_stats, [{cpu_utilization_rate,0.2503128911138924}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3439875}, {memory_data,{101368545280,100269920256,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1072152 kB\nBuffers: 283560 kB\nCached: 32676564 kB\nSwapCached: 5096 kB\nActive: 82505004 kB\nInactive: 13862512 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1072152 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 488 kB\nWriteback: 0 kB\nAnonPages: 63399684 kB\nMapped: 16728 kB\nSlab: 1345960 kB\nPageTables: 171520 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86137832 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33460801536}, {buffered_memory,290365440}, {free_memory,1097883648}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3439860476,2}}, {context_switches,{1912342003,0}}, {garbage_collection,{247873721,3900464141793,0}}, {io,{{input,69513853314},{output,131875729563}}}, {reductions,{1585471192570,2441015}}, {run_queue,0}, {runtime,{130324920,180}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,166945,404525}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,83788864}, {processes,14814320}, {processes_used,14778624}, {system,68974544}, {atom,828337}, {atom_used,824728}, {binary,1315400}, {code,7914132}, {ets,37985520}]}, {system_stats, [{cpu_utilization_rate,0.37546933667083854}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3439855}, {memory_data,{101368545280,100838772736,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 516444 kB\nBuffers: 283848 kB\nCached: 31393160 kB\nSwapCached: 3836 kB\nActive: 82925068 kB\nInactive: 13856120 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 516444 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 576 kB\nWriteback: 0 kB\nAnonPages: 65096320 kB\nMapped: 17876 kB\nSlab: 1380880 kB\nPageTables: 278860 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141213212 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32146595840}, {buffered_memory,290660352}, {free_memory,528838656}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3439845641,1}}, {context_switches,{751079003,0}}, {garbage_collection,{185630107,674622660784,0}}, {io,{{input,58225730440},{output,130453740453}}}, {reductions,{324489565471,1412090}}, {run_queue,0}, {runtime,{37294390,180}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 05:55:57 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 05:56:06 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 05:56:47 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 05:56:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,167006,408394}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102266320}, {processes,32096520}, {processes_used,29422248}, {system,70169800}, {atom,903241}, {atom_used,880290}, {binary,1255576}, {code,8604866}, {ets,38308056}]}, {system_stats, [{cpu_utilization_rate,15.0}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3439935}, {memory_data,{101368545280,100311326720,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1000236 kB\nBuffers: 283568 kB\nCached: 32670804 kB\nSwapCached: 5096 kB\nActive: 82579020 kB\nInactive: 13858156 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1000236 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 4332 kB\nWriteback: 0 kB\nAnonPages: 63475032 kB\nMapped: 23692 kB\nSlab: 1346616 kB\nPageTables: 172224 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86214320 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33454903296}, {buffered_memory,290373632}, {free_memory,1024368640}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3439920474,0}}, {context_switches,{1912359102,0}}, {garbage_collection,{247877162,3900477490151,0}}, {io,{{input,69516601774},{output,131882828545}}}, {reductions,{1585483326348,2501921}}, {run_queue,0}, {runtime,{130325900,180}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,167005,405649}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,83916720}, {processes,14830200}, {processes_used,14794504}, {system,69086520}, {atom,828337}, {atom_used,824728}, {binary,1279168}, {code,7914132}, {ets,38134256}]}, {system_stats, [{cpu_utilization_rate,13.0}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3439915}, {memory_data,{101368545280,100840357888,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 464248 kB\nBuffers: 283860 kB\nCached: 31388420 kB\nSwapCached: 3836 kB\nActive: 82979708 kB\nInactive: 13851420 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 464248 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 2672 kB\nWriteback: 0 kB\nAnonPages: 65151112 kB\nMapped: 24700 kB\nSlab: 1381460 kB\nPageTables: 279340 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141262060 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32141742080}, {buffered_memory,290672640}, {free_memory,475389952}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3439905644,1}}, {context_switches,{751092338,0}}, {garbage_collection,{185633478,674635186588,0}}, {io,{{input,58228410097},{output,130460775055}}}, {reductions,{324495169014,1379900}}, {run_queue,0}, {runtime,{37295110,140}}]}]}] INFO REPORT <5883.111.192> 2011-12-29 05:57:06 =============================================================================== ns_1@192.168.24.61:<5883.111.192>:stats_collector:83: Stats for bucket "JDPCachePrimary": accepting_conns 1 auth_cmds 10369 auth_errors 0 bucket_active_conns 1 bucket_conns 1 bytes_read 53441361510 bytes_written 45295011361 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 29975715 cmd_set 445274018 conn_yields 61205 connection_structures 1931 curr_connections 11 curr_items 297280049 curr_items_tot 297280049 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 466 delete_misses 174808 ep_bg_fetched 1661206 ep_bg_load 4365803470 ep_bg_load_avg 2628 ep_bg_max_load 2020165 ep_bg_max_wait 2929571 ep_bg_min_load 1 ep_bg_min_wait 10 ep_bg_num_samples 1661198 ep_bg_wait 48864908595 ep_bg_wait_avg 29415 ep_commit_num 818500 ep_commit_time 0 ep_commit_time_total 179775 ep_data_age 0 ep_data_age_highwat 2571 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 1 ep_dbname /opt/membase/var/lib/membase/data/JDPCachePrimary-data/JDPCachePrimary ep_dbshards 4 ep_diskqueue_drain 358215763 ep_diskqueue_fill 358215763 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 1256 ep_flush_duration_total 251908 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 1661206 ep_io_num_write 357979281 ep_io_read_bytes 145605718 ep_io_write_bytes 6012819006520 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 356086874 ep_kv_size 57356020515 ep_latency_arith_cmd 46093 ep_latency_get_cmd 101813590 ep_latency_store_cmd 515449025 ep_max_data_size 81788928000 ep_max_txn_size 1000 ep_mem_high_wat 61341696000 ep_mem_low_wat 49073356800 ep_min_data_age 0 ep_num_active_non_resident 94237869 ep_num_checkpoint_remover_runs354274 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 489 ep_num_non_resident 94237869 ep_num_not_my_vbuckets 0 ep_num_pager_runs 2 ep_num_value_ejects 95895079 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 1611440768 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 3597 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 62797498 ep_too_young 0 ep_total_cache_size 58930442359 ep_total_del_items 416 ep_total_enqueued 358215763 ep_total_new_items 297280465 ep_total_persisted 357979697 ep_uncommitted_items 0 ep_value_size 15808691965 ep_vb_total 512 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.1.1_39_g4bd26a3 ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 8853 get_hits 7311306 get_misses 22664409 incr_hits 44431 incr_misses 1662 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 58967461283 pid 17177 pointer_size 64 rejected_conns 0 rusage_system 91967.970743 rusage_user 249392.022649 threads 4 time 1325167026 total_connections 30202 uptime 3439957 vb_active_curr_items 297280049 vb_active_eject 95895079 vb_active_ht_memory 1610809344 vb_active_itm_memory 53788643543 vb_active_num 512 vb_active_num_non_resident 94237869 vb_active_ops_create 297280465 vb_active_ops_delete 416 vb_active_ops_reject 0 vb_active_ops_update 60698816 vb_active_perc_mem_resident 68 vb_active_queue_age 0 vb_active_queue_drain 358215763 vb_active_queue_fill 358215763 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version UNKNOWN INFO REPORT <5883.136.0> 2011-12-29 05:57:26 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 05:57:39 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 05:57:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,167066,408670}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,101483312}, {processes,31209912}, {processes_used,28535640}, {system,70273400}, {atom,903241}, {atom_used,880290}, {binary,1325696}, {code,8604866}, {ets,38340872}]}, {system_stats, [{cpu_utilization_rate,0.1251564455569462}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3439995}, {memory_data,{101368545280,100266893312,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1075936 kB\nBuffers: 283572 kB\nCached: 32671668 kB\nSwapCached: 5096 kB\nActive: 82504692 kB\nInactive: 13858788 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1075936 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 304 kB\nWriteback: 0 kB\nAnonPages: 63400656 kB\nMapped: 16728 kB\nSlab: 1346316 kB\nPageTables: 171520 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86136388 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33455788032}, {buffered_memory,290377728}, {free_memory,1101758464}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3439980476,0}}, {context_switches,{1912375433,0}}, {garbage_collection,{247880571,3900489524055,0}}, {io,{{input,69516822536},{output,131883711827}}}, {reductions,{1585495255968,2453980}}, {run_queue,0}, {runtime,{130326720,200}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,167065,404877}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,83541912}, {processes,14395360}, {processes_used,14359664}, {system,69146552}, {atom,828337}, {atom_used,824728}, {binary,1305128}, {code,7914132}, {ets,38168144}]}, {system_stats, [{cpu_utilization_rate,0.5}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3439975}, {memory_data,{101368545280,100927582208,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 519500 kB\nBuffers: 283860 kB\nCached: 31389292 kB\nSwapCached: 3836 kB\nActive: 82925256 kB\nInactive: 13852088 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 519500 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 220 kB\nWriteback: 0 kB\nAnonPages: 65096580 kB\nMapped: 17876 kB\nSlab: 1381068 kB\nPageTables: 278860 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141208548 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32142635008}, {buffered_memory,290672640}, {free_memory,531968000}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3439965641,0}}, {context_switches,{751104904,0}}, {garbage_collection,{185636786,674646331034,0}}, {io,{{input,58228611639},{output,130461676531}}}, {reductions,{324500670872,1401137}}, {run_queue,0}, {runtime,{37295700,150}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 05:58:21 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.111.192> 2011-12-29 05:58:46 =============================================================================== ns_1@192.168.24.61:<5883.111.192>:stats_collector:83: Stats for bucket "JDPCachePrimary": accepting_conns 1 auth_cmds 10369 auth_errors 0 bucket_active_conns 1 bucket_conns 1 bytes_read 53441368670 bytes_written 45296679501 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 29975715 cmd_set 445274018 conn_yields 61205 connection_structures 1931 curr_connections 11 curr_items 297280049 curr_items_tot 297280049 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 466 delete_misses 174808 ep_bg_fetched 1661206 ep_bg_load 4365803470 ep_bg_load_avg 2628 ep_bg_max_load 2020165 ep_bg_max_wait 2929571 ep_bg_min_load 1 ep_bg_min_wait 10 ep_bg_num_samples 1661198 ep_bg_wait 48864908595 ep_bg_wait_avg 29415 ep_commit_num 818500 ep_commit_time 0 ep_commit_time_total 179775 ep_data_age 0 ep_data_age_highwat 2571 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 1 ep_dbname /opt/membase/var/lib/membase/data/JDPCachePrimary-data/JDPCachePrimary ep_dbshards 4 ep_diskqueue_drain 358215763 ep_diskqueue_fill 358215763 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 1256 ep_flush_duration_total 251908 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 1661206 ep_io_num_write 357979281 ep_io_read_bytes 145605718 ep_io_write_bytes 6012819006520 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 356086874 ep_kv_size 57356020515 ep_latency_arith_cmd 46093 ep_latency_get_cmd 101813590 ep_latency_store_cmd 515449025 ep_max_data_size 81788928000 ep_max_txn_size 1000 ep_mem_high_wat 61341696000 ep_mem_low_wat 49073356800 ep_min_data_age 0 ep_num_active_non_resident 94237869 ep_num_checkpoint_remover_runs354294 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 489 ep_num_non_resident 94237869 ep_num_not_my_vbuckets 0 ep_num_pager_runs 2 ep_num_value_ejects 95895079 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 1611440768 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 3597 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 62797498 ep_too_young 0 ep_total_cache_size 58930442359 ep_total_del_items 416 ep_total_enqueued 358215763 ep_total_new_items 297280465 ep_total_persisted 357979697 ep_uncommitted_items 0 ep_value_size 15808691965 ep_vb_total 512 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.1.1_39_g4bd26a3 ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 8853 get_hits 7311306 get_misses 22664409 incr_hits 44431 incr_misses 1662 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 58967461283 pid 17177 pointer_size 64 rejected_conns 0 rusage_system 91967.994739 rusage_user 249392.111636 threads 4 time 1325167126 total_connections 30202 uptime 3440057 vb_active_curr_items 297280049 vb_active_eject 95895079 vb_active_ht_memory 1610809344 vb_active_itm_memory 53788643543 vb_active_num 512 vb_active_num_non_resident 94237869 vb_active_ops_create 297280465 vb_active_ops_delete 416 vb_active_ops_reject 0 vb_active_ops_update 60698816 vb_active_perc_mem_resident 68 vb_active_queue_age 0 vb_active_queue_drain 358215763 vb_active_queue_fill 358215763 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version UNKNOWN INFO REPORT <5883.1095.0> 2011-12-29 05:58:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,167126,409208}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,101834472}, {processes,31526736}, {processes_used,28852464}, {system,70307736}, {atom,903241}, {atom_used,880290}, {binary,1213376}, {code,8604866}, {ets,38487528}]}, {system_stats, [{cpu_utilization_rate,0.37546933667083854}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3440055}, {memory_data,{101368545280,100264714240,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1078072 kB\nBuffers: 283572 kB\nCached: 32672432 kB\nSwapCached: 5096 kB\nActive: 82501412 kB\nInactive: 13859560 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1078072 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 344 kB\nWriteback: 0 kB\nAnonPages: 63397252 kB\nMapped: 16728 kB\nSlab: 1346080 kB\nPageTables: 171516 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86134816 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33456570368}, {buffered_memory,290377728}, {free_memory,1103945728}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3440040476,1}}, {context_switches,{1912391682,0}}, {garbage_collection,{247883879,3900501724393,0}}, {io,{{input,69517037760},{output,131884586580}}}, {reductions,{1585507246960,2455464}}, {run_queue,0}, {runtime,{130327530,200}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,167125,405011}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,84083496}, {processes,14769808}, {processes_used,14734112}, {system,69313688}, {atom,828337}, {atom_used,824728}, {binary,1326152}, {code,7914132}, {ets,38314400}]}, {system_stats, [{cpu_utilization_rate,0.5006257822277848}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3440035}, {memory_data,{101368545280,100836405248,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 519272 kB\nBuffers: 283860 kB\nCached: 31390048 kB\nSwapCached: 3836 kB\nActive: 82924588 kB\nInactive: 13852800 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 519272 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 540 kB\nWriteback: 0 kB\nAnonPages: 65095340 kB\nMapped: 17876 kB\nSlab: 1380988 kB\nPageTables: 278860 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141211884 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32143409152}, {buffered_memory,290672640}, {free_memory,531734528}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3440025641,0}}, {context_switches,{751117380,0}}, {garbage_collection,{185639941,674657506600,0}}, {io,{{input,58228812441},{output,130462541013}}}, {reductions,{324506104159,1386265}}, {run_queue,1}, {runtime,{37296280,140}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 05:59:08 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 05:59:28 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 05:59:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,167186,408294}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102618976}, {processes,32117176}, {processes_used,29442904}, {system,70501800}, {atom,903241}, {atom_used,880290}, {binary,1259328}, {code,8604866}, {ets,38636336}]}, {system_stats, [{cpu_utilization_rate,0.37453183520599254}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3440115}, {memory_data,{101368545280,100264427520,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1077744 kB\nBuffers: 283580 kB\nCached: 32672724 kB\nSwapCached: 5096 kB\nActive: 82501740 kB\nInactive: 13859912 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1077744 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 336 kB\nWriteback: 0 kB\nAnonPages: 63397224 kB\nMapped: 16728 kB\nSlab: 1346040 kB\nPageTables: 171512 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86133736 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33456869376}, {buffered_memory,290385920}, {free_memory,1103609856}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3440100476,1}}, {context_switches,{1912408660,0}}, {garbage_collection,{247887430,3900514361133,0}}, {io,{{input,69519748063},{output,131888128773}}}, {reductions,{1585519341463,2443383}}, {run_queue,0}, {runtime,{130328420,190}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,167185,404440}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,84156816}, {processes,14664816}, {processes_used,14629120}, {system,69492000}, {atom,828337}, {atom_used,824728}, {binary,1415424}, {code,7914132}, {ets,38403792}]}, {system_stats, [{cpu_utilization_rate,0.37546933667083854}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3440095}, {memory_data,{101368545280,100836917248,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 518788 kB\nBuffers: 283860 kB\nCached: 31390356 kB\nSwapCached: 3836 kB\nActive: 82925096 kB\nInactive: 13853236 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 518788 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 296 kB\nWriteback: 0 kB\nAnonPages: 65096432 kB\nMapped: 17876 kB\nSlab: 1380992 kB\nPageTables: 278864 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141211440 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32143724544}, {buffered_memory,290672640}, {free_memory,531238912}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3440085640,0}}, {context_switches,{751131013,0}}, {garbage_collection,{185643439,674669308677,0}}, {io,{{input,58231501167},{output,130466074001}}}, {reductions,{324511937638,1396144}}, {run_queue,0}, {runtime,{37297000,130}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 06:00:12 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.111.192> 2011-12-29 06:00:26 =============================================================================== ns_1@192.168.24.61:<5883.111.192>:stats_collector:83: Stats for bucket "JDPCachePrimary": accepting_conns 1 auth_cmds 10369 auth_errors 0 bucket_active_conns 1 bucket_conns 1 bytes_read 53441375830 bytes_written 45298347641 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 29975715 cmd_set 445274018 conn_yields 61205 connection_structures 1931 curr_connections 11 curr_items 297280049 curr_items_tot 297280049 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 466 delete_misses 174808 ep_bg_fetched 1661206 ep_bg_load 4365803470 ep_bg_load_avg 2628 ep_bg_max_load 2020165 ep_bg_max_wait 2929571 ep_bg_min_load 1 ep_bg_min_wait 10 ep_bg_num_samples 1661198 ep_bg_wait 48864908595 ep_bg_wait_avg 29415 ep_commit_num 818500 ep_commit_time 0 ep_commit_time_total 179775 ep_data_age 0 ep_data_age_highwat 2571 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 1 ep_dbname /opt/membase/var/lib/membase/data/JDPCachePrimary-data/JDPCachePrimary ep_dbshards 4 ep_diskqueue_drain 358215763 ep_diskqueue_fill 358215763 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 1256 ep_flush_duration_total 251908 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 1661206 ep_io_num_write 357979281 ep_io_read_bytes 145605718 ep_io_write_bytes 6012819006520 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 356086874 ep_kv_size 57356020515 ep_latency_arith_cmd 46093 ep_latency_get_cmd 101813590 ep_latency_store_cmd 515449025 ep_max_data_size 81788928000 ep_max_txn_size 1000 ep_mem_high_wat 61341696000 ep_mem_low_wat 49073356800 ep_min_data_age 0 ep_num_active_non_resident 94237869 ep_num_checkpoint_remover_runs354314 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 489 ep_num_non_resident 94237869 ep_num_not_my_vbuckets 0 ep_num_pager_runs 2 ep_num_value_ejects 95895079 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 1611440768 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 3597 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 62797498 ep_too_young 0 ep_total_cache_size 58930442359 ep_total_del_items 416 ep_total_enqueued 358215763 ep_total_new_items 297280465 ep_total_persisted 357979697 ep_uncommitted_items 0 ep_value_size 15808691965 ep_vb_total 512 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.1.1_39_g4bd26a3 ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 8853 get_hits 7311306 get_misses 22664409 incr_hits 44431 incr_misses 1662 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 58967461283 pid 17177 pointer_size 64 rejected_conns 0 rusage_system 91968.019735 rusage_user 249392.191624 threads 4 time 1325167226 total_connections 30202 uptime 3440157 vb_active_curr_items 297280049 vb_active_eject 95895079 vb_active_ht_memory 1610809344 vb_active_itm_memory 53788643543 vb_active_num 512 vb_active_num_non_resident 94237869 vb_active_ops_create 297280465 vb_active_ops_delete 416 vb_active_ops_reject 0 vb_active_ops_update 60698816 vb_active_perc_mem_resident 68 vb_active_queue_age 0 vb_active_queue_drain 358215763 vb_active_queue_fill 358215763 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version UNKNOWN INFO REPORT <5883.136.0> 2011-12-29 06:00:43 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 06:00:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,167246,409136}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,101993704}, {processes,31964576}, {processes_used,29290304}, {system,70029128}, {atom,903241}, {atom_used,880290}, {binary,1241640}, {code,8604866}, {ets,38181184}]}, {system_stats, [{cpu_utilization_rate,13.0}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3440175}, {memory_data,{101368545280,100266094592,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1063796 kB\nBuffers: 283580 kB\nCached: 32673496 kB\nSwapCached: 5096 kB\nActive: 82514956 kB\nInactive: 13860664 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1063796 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 1148 kB\nWriteback: 0 kB\nAnonPages: 63410736 kB\nMapped: 18188 kB\nSlab: 1346148 kB\nPageTables: 171848 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86143516 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33457659904}, {buffered_memory,290385920}, {free_memory,1089327104}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3440160475,0}}, {context_switches,{1912424965,0}}, {garbage_collection,{247890819,3900526491004,0}}, {io,{{input,69519930515},{output,131888968166}}}, {reductions,{1585531394219,2473062}}, {run_queue,0}, {runtime,{130329210,180}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,167245,404700}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,83509184}, {processes,14541920}, {processes_used,14506224}, {system,68967264}, {atom,828337}, {atom_used,824728}, {binary,1288544}, {code,7914132}, {ets,38006040}]}, {system_stats, [{cpu_utilization_rate,13.125}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3440155}, {memory_data,{101368545280,100837670912,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 488824 kB\nBuffers: 283860 kB\nCached: 31391144 kB\nSwapCached: 3836 kB\nActive: 82951548 kB\nInactive: 13853924 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 488824 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 3124 kB\nWriteback: 0 kB\nAnonPages: 65123312 kB\nMapped: 19820 kB\nSlab: 1381260 kB\nPageTables: 279296 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141237380 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32144531456}, {buffered_memory,290672640}, {free_memory,500555776}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3440145642,1}}, {context_switches,{751143345,0}}, {garbage_collection,{185646606,674680437908,0}}, {io,{{input,58231661021},{output,130466913213}}}, {reductions,{324517403296,1416020}}, {run_queue,1}, {runtime,{37297590,150}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 06:01:43 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 06:01:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,167306,408424}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,101985056}, {processes,31864232}, {processes_used,29189960}, {system,70120824}, {atom,903241}, {atom_used,880290}, {binary,1300656}, {code,8604866}, {ets,38213104}]}, {system_stats, [{cpu_utilization_rate,0.2503128911138924}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3440235}, {memory_data,{101368545280,100363325440,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1071676 kB\nBuffers: 283580 kB\nCached: 32674400 kB\nSwapCached: 5096 kB\nActive: 82502276 kB\nInactive: 13861336 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1071676 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 4348 kB\nWriteback: 0 kB\nAnonPages: 63398052 kB\nMapped: 16728 kB\nSlab: 1347304 kB\nPageTables: 171516 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86133340 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33458585600}, {buffered_memory,290385920}, {free_memory,1097396224}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3440220475,1}}, {context_switches,{1912441851,0}}, {garbage_collection,{247894009,3900538667298,0}}, {io,{{input,69520105412},{output,131889869030}}}, {reductions,{1585543855802,2423285}}, {run_queue,0}, {runtime,{130329980,180}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,167305,404803}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,83649840}, {processes,14572352}, {processes_used,14536656}, {system,69077488}, {atom,828337}, {atom_used,824728}, {binary,1363408}, {code,7914132}, {ets,38041072}]}, {system_stats, [{cpu_utilization_rate,0.37546933667083854}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3440215}, {memory_data,{101368545280,100900978688,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 516132 kB\nBuffers: 283860 kB\nCached: 31392016 kB\nSwapCached: 3836 kB\nActive: 82925272 kB\nInactive: 13854572 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 516132 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 996 kB\nWriteback: 0 kB\nAnonPages: 65096352 kB\nMapped: 17876 kB\nSlab: 1381428 kB\nPageTables: 278860 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141209972 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32145424384}, {buffered_memory,290672640}, {free_memory,528519168}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3440205642,1}}, {context_switches,{751156165,0}}, {garbage_collection,{185650018,674691733683,0}}, {io,{{input,58231856405},{output,130467785158}}}, {reductions,{324523140266,1412520}}, {run_queue,0}, {runtime,{37298210,140}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 06:01:52 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.111.192> 2011-12-29 06:02:06 =============================================================================== ns_1@192.168.24.61:<5883.111.192>:stats_collector:83: Stats for bucket "JDPCachePrimary": accepting_conns 1 auth_cmds 10369 auth_errors 0 bucket_active_conns 1 bucket_conns 1 bytes_read 53441382990 bytes_written 45300015781 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 29975715 cmd_set 445274018 conn_yields 61205 connection_structures 1931 curr_connections 11 curr_items 297280049 curr_items_tot 297280049 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 466 delete_misses 174808 ep_bg_fetched 1661206 ep_bg_load 4365803470 ep_bg_load_avg 2628 ep_bg_max_load 2020165 ep_bg_max_wait 2929571 ep_bg_min_load 1 ep_bg_min_wait 10 ep_bg_num_samples 1661198 ep_bg_wait 48864908595 ep_bg_wait_avg 29415 ep_commit_num 818500 ep_commit_time 0 ep_commit_time_total 179775 ep_data_age 0 ep_data_age_highwat 2571 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 1 ep_dbname /opt/membase/var/lib/membase/data/JDPCachePrimary-data/JDPCachePrimary ep_dbshards 4 ep_diskqueue_drain 358215763 ep_diskqueue_fill 358215763 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 1256 ep_flush_duration_total 251908 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 1661206 ep_io_num_write 357979281 ep_io_read_bytes 145605718 ep_io_write_bytes 6012819006520 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 356086874 ep_kv_size 57356020515 ep_latency_arith_cmd 46093 ep_latency_get_cmd 101813590 ep_latency_store_cmd 515449025 ep_max_data_size 81788928000 ep_max_txn_size 1000 ep_mem_high_wat 61341696000 ep_mem_low_wat 49073356800 ep_min_data_age 0 ep_num_active_non_resident 94237869 ep_num_checkpoint_remover_runs354334 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 489 ep_num_non_resident 94237869 ep_num_not_my_vbuckets 0 ep_num_pager_runs 2 ep_num_value_ejects 95895079 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 1611440768 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 3597 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 62797498 ep_too_young 0 ep_total_cache_size 58930442359 ep_total_del_items 416 ep_total_enqueued 358215763 ep_total_new_items 297280465 ep_total_persisted 357979697 ep_uncommitted_items 0 ep_value_size 15808691965 ep_vb_total 512 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.1.1_39_g4bd26a3 ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 8853 get_hits 7311306 get_misses 22664409 incr_hits 44431 incr_misses 1662 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 58967461283 pid 17177 pointer_size 64 rejected_conns 0 rusage_system 91968.049731 rusage_user 249392.295608 threads 4 time 1325167326 total_connections 30202 uptime 3440257 vb_active_curr_items 297280049 vb_active_eject 95895079 vb_active_ht_memory 1610809344 vb_active_itm_memory 53788643543 vb_active_num 512 vb_active_num_non_resident 94237869 vb_active_ops_create 297280465 vb_active_ops_delete 416 vb_active_ops_reject 0 vb_active_ops_update 60698816 vb_active_perc_mem_resident 68 vb_active_queue_age 0 vb_active_queue_drain 358215763 vb_active_queue_fill 358215763 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version UNKNOWN INFO REPORT <5883.136.0> 2011-12-29 06:02:15 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 06:02:40 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 06:02:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,167366,408480}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,101605024}, {processes,31391824}, {processes_used,28717552}, {system,70213200}, {atom,903241}, {atom_used,880290}, {binary,1243240}, {code,8604866}, {ets,38362752}]}, {system_stats, [{cpu_utilization_rate,0.2503128911138924}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3440295}, {memory_data,{101368545280,100266573824,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1076040 kB\nBuffers: 283584 kB\nCached: 32674032 kB\nSwapCached: 5096 kB\nActive: 82502100 kB\nInactive: 13861076 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1076040 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 408 kB\nWriteback: 0 kB\nAnonPages: 63397552 kB\nMapped: 16728 kB\nSlab: 1346392 kB\nPageTables: 171512 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86138096 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33458208768}, {buffered_memory,290390016}, {free_memory,1101864960}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3440280476,1}}, {context_switches,{1912458934,0}}, {garbage_collection,{247897559,3900551456128,0}}, {io,{{input,69522858655},{output,131892658124}}}, {reductions,{1585555941084,2444247}}, {run_queue,0}, {runtime,{130330840,200}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,167365,404788}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,83599392}, {processes,14435344}, {processes_used,14399648}, {system,69164048}, {atom,828337}, {atom_used,824728}, {binary,1301752}, {code,7914132}, {ets,38189216}]}, {system_stats, [{cpu_utilization_rate,0.37546933667083854}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3440275}, {memory_data,{101368545280,100840083456,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 517692 kB\nBuffers: 283864 kB\nCached: 31391648 kB\nSwapCached: 3836 kB\nActive: 82924744 kB\nInactive: 13854344 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 517692 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 488 kB\nWriteback: 0 kB\nAnonPages: 65095416 kB\nMapped: 17876 kB\nSlab: 1380964 kB\nPageTables: 278860 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141211336 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32145047552}, {buffered_memory,290676736}, {free_memory,530116608}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3440265641,0}}, {context_switches,{751169590,0}}, {garbage_collection,{185653318,674703473033,0}}, {io,{{input,58234471845},{output,130470632569}}}, {reductions,{324528696918,1402153}}, {run_queue,1}, {runtime,{37298850,150}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 06:03:05 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 06:03:45 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.111.192> 2011-12-29 06:03:46 =============================================================================== ns_1@192.168.24.61:<5883.111.192>:stats_collector:83: Stats for bucket "JDPCachePrimary": accepting_conns 1 auth_cmds 10369 auth_errors 0 bucket_active_conns 1 bucket_conns 1 bytes_read 53441390150 bytes_written 45301683921 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 29975715 cmd_set 445274018 conn_yields 61205 connection_structures 1931 curr_connections 11 curr_items 297280049 curr_items_tot 297280049 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 466 delete_misses 174808 ep_bg_fetched 1661206 ep_bg_load 4365803470 ep_bg_load_avg 2628 ep_bg_max_load 2020165 ep_bg_max_wait 2929571 ep_bg_min_load 1 ep_bg_min_wait 10 ep_bg_num_samples 1661198 ep_bg_wait 48864908595 ep_bg_wait_avg 29415 ep_commit_num 818500 ep_commit_time 0 ep_commit_time_total 179775 ep_data_age 0 ep_data_age_highwat 2571 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 1 ep_dbname /opt/membase/var/lib/membase/data/JDPCachePrimary-data/JDPCachePrimary ep_dbshards 4 ep_diskqueue_drain 358215763 ep_diskqueue_fill 358215763 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 1256 ep_flush_duration_total 251908 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 1661206 ep_io_num_write 357979281 ep_io_read_bytes 145605718 ep_io_write_bytes 6012819006520 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 356086874 ep_kv_size 57356020515 ep_latency_arith_cmd 46093 ep_latency_get_cmd 101813590 ep_latency_store_cmd 515449025 ep_max_data_size 81788928000 ep_max_txn_size 1000 ep_mem_high_wat 61341696000 ep_mem_low_wat 49073356800 ep_min_data_age 0 ep_num_active_non_resident 94237869 ep_num_checkpoint_remover_runs354354 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 489 ep_num_non_resident 94237869 ep_num_not_my_vbuckets 0 ep_num_pager_runs 2 ep_num_value_ejects 95895079 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 1611440768 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 3597 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 62797498 ep_too_young 0 ep_total_cache_size 58930442359 ep_total_del_items 416 ep_total_enqueued 358215763 ep_total_new_items 297280465 ep_total_persisted 357979697 ep_uncommitted_items 0 ep_value_size 15808691965 ep_vb_total 512 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.1.1_39_g4bd26a3 ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 8853 get_hits 7311306 get_misses 22664409 incr_hits 44431 incr_misses 1662 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 58967461283 pid 17177 pointer_size 64 rejected_conns 0 rusage_system 91968.081726 rusage_user 249392.398592 threads 4 time 1325167426 total_connections 30202 uptime 3440357 vb_active_curr_items 297280049 vb_active_eject 95895079 vb_active_ht_memory 1610809344 vb_active_itm_memory 53788643543 vb_active_num 512 vb_active_num_non_resident 94237869 vb_active_ops_create 297280465 vb_active_ops_delete 416 vb_active_ops_reject 0 vb_active_ops_update 60698816 vb_active_perc_mem_resident 68 vb_active_queue_age 0 vb_active_queue_drain 358215763 vb_active_queue_fill 358215763 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version UNKNOWN INFO REPORT <5883.1095.0> 2011-12-29 06:03:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,167426,408988}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,101701776}, {processes,31383216}, {processes_used,28708944}, {system,70318560}, {atom,903241}, {atom_used,880290}, {binary,1313760}, {code,8604866}, {ets,38396368}]}, {system_stats, [{cpu_utilization_rate,0.25}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3440355}, {memory_data,{101368545280,100267417600,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1074832 kB\nBuffers: 283584 kB\nCached: 32674796 kB\nSwapCached: 5096 kB\nActive: 82502168 kB\nInactive: 13861736 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1074832 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 132 kB\nWriteback: 0 kB\nAnonPages: 63397328 kB\nMapped: 16728 kB\nSlab: 1346188 kB\nPageTables: 171512 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86136896 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33458991104}, {buffered_memory,290390016}, {free_memory,1100627968}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3440340474,0}}, {context_switches,{1912475190,0}}, {garbage_collection,{247900832,3900563406266,0}}, {io,{{input,69523068148},{output,131893535342}}}, {reductions,{1585567932006,2429353}}, {run_queue,0}, {runtime,{130331600,180}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,167425,404517}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,84181288}, {processes,14835024}, {processes_used,14799328}, {system,69346264}, {atom,828337}, {atom_used,824728}, {binary,1450880}, {code,7914132}, {ets,38221832}]}, {system_stats, [{cpu_utilization_rate,0.375}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3440335}, {memory_data,{101368545280,100837593088,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 517720 kB\nBuffers: 283864 kB\nCached: 31392408 kB\nSwapCached: 3836 kB\nActive: 82923884 kB\nInactive: 13855052 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 517720 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 160 kB\nWriteback: 0 kB\nAnonPages: 65094892 kB\nMapped: 17876 kB\nSlab: 1380964 kB\nPageTables: 278856 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141208784 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32145825792}, {buffered_memory,290676736}, {free_memory,530145280}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3440325641,1}}, {context_switches,{751182145,0}}, {garbage_collection,{185656491,674714562252,0}}, {io,{{input,58234662415},{output,130471517219}}}, {reductions,{324534141759,1371193}}, {run_queue,1}, {runtime,{37299450,150}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 06:04:18 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 06:04:39 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 06:04:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,167486,409218}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102486736}, {processes,31864296}, {processes_used,29190024}, {system,70622440}, {atom,903241}, {atom_used,880290}, {binary,1471544}, {code,8604866}, {ets,38542608}]}, {system_stats, [{cpu_utilization_rate,0.2503128911138924}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3440415}, {memory_data,{101368545280,100268171264,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1074336 kB\nBuffers: 283584 kB\nCached: 32675556 kB\nSwapCached: 5096 kB\nActive: 82502064 kB\nInactive: 13862456 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1074336 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 352 kB\nWriteback: 0 kB\nAnonPages: 63397820 kB\nMapped: 16728 kB\nSlab: 1346100 kB\nPageTables: 171512 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86137796 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33459769344}, {buffered_memory,290390016}, {free_memory,1100120064}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3440400475,1}}, {context_switches,{1912491539,0}}, {garbage_collection,{247904117,3900575470416,0}}, {io,{{input,69523289876},{output,131894390925}}}, {reductions,{1585579993334,2513196}}, {run_queue,0}, {runtime,{130332380,190}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,167485,404870}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,83980960}, {processes,14556296}, {processes_used,14520600}, {system,69424664}, {atom,828337}, {atom_used,824728}, {binary,1383224}, {code,7914132}, {ets,38367912}]}, {system_stats, [{cpu_utilization_rate,0.375}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3440395}, {memory_data,{101368545280,100839247872,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 515532 kB\nBuffers: 283864 kB\nCached: 31393184 kB\nSwapCached: 3836 kB\nActive: 82925220 kB\nInactive: 13855772 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 515532 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 472 kB\nWriteback: 0 kB\nAnonPages: 65096232 kB\nMapped: 17876 kB\nSlab: 1380936 kB\nPageTables: 278860 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141212724 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32146620416}, {buffered_memory,290676736}, {free_memory,527904768}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3440385642,1}}, {context_switches,{751194863,0}}, {garbage_collection,{185659667,674725742573,0}}, {io,{{input,58234819813},{output,130472389210}}}, {reductions,{324539648025,1410302}}, {run_queue,1}, {runtime,{37300050,170}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 06:05:18 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.111.192> 2011-12-29 06:05:26 =============================================================================== ns_1@192.168.24.61:<5883.111.192>:stats_collector:83: Stats for bucket "JDPCachePrimary": accepting_conns 1 auth_cmds 10369 auth_errors 0 bucket_active_conns 1 bucket_conns 1 bytes_read 53441397310 bytes_written 45303352061 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 29975715 cmd_set 445274018 conn_yields 61205 connection_structures 1931 curr_connections 11 curr_items 297280049 curr_items_tot 297280049 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 466 delete_misses 174808 ep_bg_fetched 1661206 ep_bg_load 4365803470 ep_bg_load_avg 2628 ep_bg_max_load 2020165 ep_bg_max_wait 2929571 ep_bg_min_load 1 ep_bg_min_wait 10 ep_bg_num_samples 1661198 ep_bg_wait 48864908595 ep_bg_wait_avg 29415 ep_commit_num 818500 ep_commit_time 0 ep_commit_time_total 179775 ep_data_age 0 ep_data_age_highwat 2571 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 1 ep_dbname /opt/membase/var/lib/membase/data/JDPCachePrimary-data/JDPCachePrimary ep_dbshards 4 ep_diskqueue_drain 358215763 ep_diskqueue_fill 358215763 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 1256 ep_flush_duration_total 251908 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 1661206 ep_io_num_write 357979281 ep_io_read_bytes 145605718 ep_io_write_bytes 6012819006520 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 356086874 ep_kv_size 57356020515 ep_latency_arith_cmd 46093 ep_latency_get_cmd 101813590 ep_latency_store_cmd 515449025 ep_max_data_size 81788928000 ep_max_txn_size 1000 ep_mem_high_wat 61341696000 ep_mem_low_wat 49073356800 ep_min_data_age 0 ep_num_active_non_resident 94237869 ep_num_checkpoint_remover_runs354374 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 489 ep_num_non_resident 94237869 ep_num_not_my_vbuckets 0 ep_num_pager_runs 2 ep_num_value_ejects 95895079 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 1611440768 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 3597 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 62797498 ep_too_young 0 ep_total_cache_size 58930442359 ep_total_del_items 416 ep_total_enqueued 358215763 ep_total_new_items 297280465 ep_total_persisted 357979697 ep_uncommitted_items 0 ep_value_size 15808691965 ep_vb_total 512 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.1.1_39_g4bd26a3 ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 8853 get_hits 7311306 get_misses 22664409 incr_hits 44431 incr_misses 1662 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 58967461283 pid 17177 pointer_size 64 rejected_conns 0 rusage_system 91968.116721 rusage_user 249392.489578 threads 4 time 1325167526 total_connections 30202 uptime 3440457 vb_active_curr_items 297280049 vb_active_eject 95895079 vb_active_ht_memory 1610809344 vb_active_itm_memory 53788643543 vb_active_num 512 vb_active_num_non_resident 94237869 vb_active_ops_create 297280465 vb_active_ops_delete 416 vb_active_ops_reject 0 vb_active_ops_update 60698816 vb_active_perc_mem_resident 68 vb_active_queue_age 0 vb_active_queue_drain 358215763 vb_active_queue_fill 358215763 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version UNKNOWN INFO REPORT <5883.1095.0> 2011-12-29 06:05:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,167546,409122}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102670048}, {processes,31979280}, {processes_used,29305008}, {system,70690768}, {atom,903241}, {atom_used,880290}, {binary,1391432}, {code,8604866}, {ets,38691688}]}, {system_stats, [{cpu_utilization_rate,0.1251564455569462}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3440475}, {memory_data,{101368545280,100264538112,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1077636 kB\nBuffers: 283596 kB\nCached: 32672008 kB\nSwapCached: 5096 kB\nActive: 82502260 kB\nInactive: 13859084 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1077636 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 400 kB\nWriteback: 0 kB\nAnonPages: 63397948 kB\nMapped: 16728 kB\nSlab: 1346212 kB\nPageTables: 171512 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86135648 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33456136192}, {buffered_memory,290402304}, {free_memory,1103499264}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3440460475,1}}, {context_switches,{1912508505,0}}, {garbage_collection,{247907564,3900588833065,0}}, {io,{{input,69525978877},{output,131902388697}}}, {reductions,{1585592080553,2522430}}, {run_queue,0}, {runtime,{130333320,170}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,167545,404891}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,84280392}, {processes,14752416}, {processes_used,14716720}, {system,69527976}, {atom,828337}, {atom_used,824728}, {binary,1394800}, {code,7914132}, {ets,38460128}]}, {system_stats, [{cpu_utilization_rate,13.642052565707134}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3440455}, {memory_data,{101368545280,100839899136,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 445788 kB\nBuffers: 283876 kB\nCached: 31389672 kB\nSwapCached: 3836 kB\nActive: 82998220 kB\nInactive: 13852392 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 445788 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 3392 kB\nWriteback: 0 kB\nAnonPages: 65168308 kB\nMapped: 25144 kB\nSlab: 1381112 kB\nPageTables: 279724 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141279804 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32143024128}, {buffered_memory,290689024}, {free_memory,456486912}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3440445641,1}}, {context_switches,{751208461,0}}, {garbage_collection,{185663163,674738313907,0}}, {io,{{input,58237474119},{output,130480424645}}}, {reductions,{324545557980,1411228}}, {run_queue,1}, {runtime,{37300870,160}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 06:05:50 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 06:06:49 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 06:06:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,167606,409714}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102484424}, {processes,32129984}, {processes_used,29455712}, {system,70354440}, {atom,903241}, {atom_used,880290}, {binary,1523952}, {code,8604866}, {ets,38222832}]}, {system_stats, [{cpu_utilization_rate,13.14142678347935}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3440535}, {memory_data,{101368545280,100321751040,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 983440 kB\nBuffers: 283596 kB\nCached: 32672820 kB\nSwapCached: 5096 kB\nActive: 82594048 kB\nInactive: 13859748 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 983440 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 1200 kB\nWriteback: 0 kB\nAnonPages: 63489448 kB\nMapped: 24076 kB\nSlab: 1346628 kB\nPageTables: 172404 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86227236 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33456967680}, {buffered_memory,290402304}, {free_memory,1007169536}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3440520475,0}}, {context_switches,{1912524767,0}}, {garbage_collection,{247910869,3900600780471,0}}, {io,{{input,69526166202},{output,131903229076}}}, {reductions,{1585603972575,2439021}}, {run_queue,1}, {runtime,{130334090,170}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,167605,405558}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,84137720}, {processes,14871488}, {processes_used,14835872}, {system,69266232}, {atom,828337}, {atom_used,824728}, {binary,1536224}, {code,7914132}, {ets,38057272}]}, {system_stats, [{cpu_utilization_rate,0.5006257822277848}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3440515}, {memory_data,{101368545280,100934701056,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 516280 kB\nBuffers: 283876 kB\nCached: 31390524 kB\nSwapCached: 3836 kB\nActive: 82925888 kB\nInactive: 13853076 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 516280 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 1356 kB\nWriteback: 0 kB\nAnonPages: 65096440 kB\nMapped: 17876 kB\nSlab: 1381520 kB\nPageTables: 278868 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141209868 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32143896576}, {buffered_memory,290689024}, {free_memory,528678912}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3440505641,1}}, {context_switches,{751221005,0}}, {garbage_collection,{185666323,674749486783,0}}, {io,{{input,58237634245},{output,130481268173}}}, {reductions,{324551012148,1380407}}, {run_queue,0}, {runtime,{37301480,170}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 06:07:04 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.111.192> 2011-12-29 06:07:06 =============================================================================== ns_1@192.168.24.61:<5883.111.192>:stats_collector:83: Stats for bucket "JDPCachePrimary": accepting_conns 1 auth_cmds 10369 auth_errors 0 bucket_active_conns 1 bucket_conns 1 bytes_read 53441404470 bytes_written 45305020201 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 29975715 cmd_set 445274018 conn_yields 61205 connection_structures 1931 curr_connections 11 curr_items 297280049 curr_items_tot 297280049 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 466 delete_misses 174808 ep_bg_fetched 1661206 ep_bg_load 4365803470 ep_bg_load_avg 2628 ep_bg_max_load 2020165 ep_bg_max_wait 2929571 ep_bg_min_load 1 ep_bg_min_wait 10 ep_bg_num_samples 1661198 ep_bg_wait 48864908595 ep_bg_wait_avg 29415 ep_commit_num 818500 ep_commit_time 0 ep_commit_time_total 179775 ep_data_age 0 ep_data_age_highwat 2571 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 1 ep_dbname /opt/membase/var/lib/membase/data/JDPCachePrimary-data/JDPCachePrimary ep_dbshards 4 ep_diskqueue_drain 358215763 ep_diskqueue_fill 358215763 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 1256 ep_flush_duration_total 251908 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 1661206 ep_io_num_write 357979281 ep_io_read_bytes 145605718 ep_io_write_bytes 6012819006520 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 356086874 ep_kv_size 57356020515 ep_latency_arith_cmd 46093 ep_latency_get_cmd 101813590 ep_latency_store_cmd 515449025 ep_max_data_size 81788928000 ep_max_txn_size 1000 ep_mem_high_wat 61341696000 ep_mem_low_wat 49073356800 ep_min_data_age 0 ep_num_active_non_resident 94237869 ep_num_checkpoint_remover_runs354394 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 489 ep_num_non_resident 94237869 ep_num_not_my_vbuckets 0 ep_num_pager_runs 2 ep_num_value_ejects 95895079 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 1611440768 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 3597 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 62797498 ep_too_young 0 ep_total_cache_size 58930442359 ep_total_del_items 416 ep_total_enqueued 358215763 ep_total_new_items 297280465 ep_total_persisted 357979697 ep_uncommitted_items 0 ep_value_size 15808691965 ep_vb_total 512 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.1.1_39_g4bd26a3 ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 8853 get_hits 7311306 get_misses 22664409 incr_hits 44431 incr_misses 1662 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 58967461283 pid 17177 pointer_size 64 rejected_conns 0 rusage_system 91968.154715 rusage_user 249392.575565 threads 4 time 1325167626 total_connections 30202 uptime 3440557 vb_active_curr_items 297280049 vb_active_eject 95895079 vb_active_ht_memory 1610809344 vb_active_itm_memory 53788643543 vb_active_num 512 vb_active_num_non_resident 94237869 vb_active_ops_create 297280465 vb_active_ops_delete 416 vb_active_ops_reject 0 vb_active_ops_update 60698816 vb_active_perc_mem_resident 68 vb_active_queue_age 0 vb_active_queue_drain 358215763 vb_active_queue_fill 358215763 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version UNKNOWN INFO REPORT <5883.136.0> 2011-12-29 06:07:20 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 06:07:26 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 06:07:49 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 06:07:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,167666,409168}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,101985288}, {processes,31617336}, {processes_used,28943064}, {system,70367952}, {atom,903241}, {atom_used,880290}, {binary,1504376}, {code,8604866}, {ets,38255568}]}, {system_stats, [{cpu_utilization_rate,0.37453183520599254}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3440595}, {memory_data,{101368545280,100266758144,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1075720 kB\nBuffers: 283596 kB\nCached: 32673700 kB\nSwapCached: 5096 kB\nActive: 82503212 kB\nInactive: 13860488 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1075720 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 456 kB\nWriteback: 0 kB\nAnonPages: 63398916 kB\nMapped: 16728 kB\nSlab: 1346476 kB\nPageTables: 171524 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86138672 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33457868800}, {buffered_memory,290402304}, {free_memory,1101537280}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3440580476,1}}, {context_switches,{1912541702,0}}, {garbage_collection,{247914214,3900613196044,0}}, {io,{{input,69526452123},{output,131904154204}}}, {reductions,{1585616487124,2477547}}, {run_queue,0}, {runtime,{130334900,170}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,167665,404498}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,83705424}, {processes,14506576}, {processes_used,14470880}, {system,69198848}, {atom,828337}, {atom_used,824728}, {binary,1442104}, {code,7914132}, {ets,38083416}]}, {system_stats, [{cpu_utilization_rate,0.5}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3440575}, {memory_data,{101368545280,100838932480,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 517396 kB\nBuffers: 283876 kB\nCached: 31391320 kB\nSwapCached: 3836 kB\nActive: 82925424 kB\nInactive: 13853792 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 517396 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 548 kB\nWriteback: 0 kB\nAnonPages: 65096348 kB\nMapped: 17876 kB\nSlab: 1381000 kB\nPageTables: 278856 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141209584 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32144711680}, {buffered_memory,290689024}, {free_memory,529813504}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3440565640,0}}, {context_switches,{751234038,0}}, {garbage_collection,{185669507,674760802387,0}}, {io,{{input,58237834206},{output,130482236604}}}, {reductions,{324556749008,1416662}}, {run_queue,0}, {runtime,{37302100,130}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 06:08:20 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.111.192> 2011-12-29 06:08:46 =============================================================================== ns_1@192.168.24.61:<5883.111.192>:stats_collector:83: Stats for bucket "JDPCachePrimary": accepting_conns 1 auth_cmds 10369 auth_errors 0 bucket_active_conns 1 bucket_conns 1 bytes_read 53441411630 bytes_written 45306688341 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 29975715 cmd_set 445274018 conn_yields 61205 connection_structures 1931 curr_connections 11 curr_items 297280049 curr_items_tot 297280049 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 466 delete_misses 174808 ep_bg_fetched 1661206 ep_bg_load 4365803470 ep_bg_load_avg 2628 ep_bg_max_load 2020165 ep_bg_max_wait 2929571 ep_bg_min_load 1 ep_bg_min_wait 10 ep_bg_num_samples 1661198 ep_bg_wait 48864908595 ep_bg_wait_avg 29415 ep_commit_num 818500 ep_commit_time 0 ep_commit_time_total 179775 ep_data_age 0 ep_data_age_highwat 2571 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 1 ep_dbname /opt/membase/var/lib/membase/data/JDPCachePrimary-data/JDPCachePrimary ep_dbshards 4 ep_diskqueue_drain 358215763 ep_diskqueue_fill 358215763 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 1256 ep_flush_duration_total 251908 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 1661206 ep_io_num_write 357979281 ep_io_read_bytes 145605718 ep_io_write_bytes 6012819006520 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 356086874 ep_kv_size 57356020515 ep_latency_arith_cmd 46093 ep_latency_get_cmd 101813590 ep_latency_store_cmd 515449025 ep_max_data_size 81788928000 ep_max_txn_size 1000 ep_mem_high_wat 61341696000 ep_mem_low_wat 49073356800 ep_min_data_age 0 ep_num_active_non_resident 94237869 ep_num_checkpoint_remover_runs354414 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 489 ep_num_non_resident 94237869 ep_num_not_my_vbuckets 0 ep_num_pager_runs 2 ep_num_value_ejects 95895079 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 1611440768 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 3597 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 62797498 ep_too_young 0 ep_total_cache_size 58930442359 ep_total_del_items 416 ep_total_enqueued 358215763 ep_total_new_items 297280465 ep_total_persisted 357979697 ep_uncommitted_items 0 ep_value_size 15808691965 ep_vb_total 512 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.1.1_39_g4bd26a3 ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 8853 get_hits 7311306 get_misses 22664409 incr_hits 44431 incr_misses 1662 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 58967461283 pid 17177 pointer_size 64 rejected_conns 0 rusage_system 91968.182711 rusage_user 249392.672550 threads 4 time 1325167726 total_connections 30202 uptime 3440657 vb_active_curr_items 297280049 vb_active_eject 95895079 vb_active_ht_memory 1610809344 vb_active_itm_memory 53788643543 vb_active_num 512 vb_active_num_non_resident 94237869 vb_active_ops_create 297280465 vb_active_ops_delete 416 vb_active_ops_reject 0 vb_active_ops_update 60698816 vb_active_perc_mem_resident 68 vb_active_queue_age 0 vb_active_queue_drain 358215763 vb_active_queue_fill 358215763 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version UNKNOWN INFO REPORT <5883.1095.0> 2011-12-29 06:08:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,167726,408997}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102535288}, {processes,32020576}, {processes_used,29346304}, {system,70514712}, {atom,903241}, {atom_used,880290}, {binary,1502848}, {code,8604866}, {ets,38404008}]}, {system_stats, [{cpu_utilization_rate,0.375}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3440655}, {memory_data,{101368545280,100266459136,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1076384 kB\nBuffers: 283600 kB\nCached: 32673320 kB\nSwapCached: 5096 kB\nActive: 82502360 kB\nInactive: 13860172 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1076384 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 292 kB\nWriteback: 0 kB\nAnonPages: 63397856 kB\nMapped: 16728 kB\nSlab: 1346192 kB\nPageTables: 171516 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86135092 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33457479680}, {buffered_memory,290406400}, {free_memory,1102217216}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3440640475,1}}, {context_switches,{1912558802,0}}, {garbage_collection,{247917728,3900625785748,0}}, {io,{{input,69529157217},{output,131906965849}}}, {reductions,{1585628595604,2485402}}, {run_queue,1}, {runtime,{130335730,170}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,167725,404481}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,83894808}, {processes,14589992}, {processes_used,14554296}, {system,69304816}, {atom,828337}, {atom_used,824728}, {binary,1398584}, {code,7914132}, {ets,38233224}]}, {system_stats, [{cpu_utilization_rate,0.375}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3440635}, {memory_data,{101368545280,100839239680,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 518768 kB\nBuffers: 283880 kB\nCached: 31390940 kB\nSwapCached: 3836 kB\nActive: 82924872 kB\nInactive: 13853512 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 518768 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 156 kB\nWriteback: 0 kB\nAnonPages: 65095908 kB\nMapped: 17876 kB\nSlab: 1381028 kB\nPageTables: 278860 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141210816 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32144322560}, {buffered_memory,290693120}, {free_memory,531218432}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3440625641,0}}, {context_switches,{751247504,0}}, {garbage_collection,{185672833,674772530427,0}}, {io,{{input,58240482288},{output,130485041077}}}, {reductions,{324562287839,1389177}}, {run_queue,0}, {runtime,{37302750,120}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 06:08:51 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 06:08:59 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 06:09:17 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 06:09:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,167786,409070}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102583200}, {processes,32064952}, {processes_used,29390680}, {system,70518248}, {atom,903241}, {atom_used,880290}, {binary,1474272}, {code,8604866}, {ets,38435800}]}, {system_stats, [{cpu_utilization_rate,0.2503128911138924}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3440715}, {memory_data,{101368545280,100267393024,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1075092 kB\nBuffers: 283600 kB\nCached: 32674096 kB\nSwapCached: 5096 kB\nActive: 82502664 kB\nInactive: 13860916 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1075092 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 128 kB\nWriteback: 0 kB\nAnonPages: 63397780 kB\nMapped: 16728 kB\nSlab: 1346200 kB\nPageTables: 171512 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86138656 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33458274304}, {buffered_memory,290406400}, {free_memory,1100894208}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3440700475,1}}, {context_switches,{1912575155,0}}, {garbage_collection,{247921115,3900637896956,0}}, {io,{{input,69529404819},{output,131907818821}}}, {reductions,{1585640597112,2458779}}, {run_queue,0}, {runtime,{130336500,180}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,167785,404902}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,83425128}, {processes,14113824}, {processes_used,14078128}, {system,69311304}, {atom,828337}, {atom_used,824728}, {binary,1372296}, {code,7914132}, {ets,38265280}]}, {system_stats, [{cpu_utilization_rate,0.2506265664160401}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3440695}, {memory_data,{101368545280,100837335040,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 517648 kB\nBuffers: 283880 kB\nCached: 31391712 kB\nSwapCached: 3836 kB\nActive: 82925324 kB\nInactive: 13854240 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 517648 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 208 kB\nWriteback: 0 kB\nAnonPages: 65095740 kB\nMapped: 17876 kB\nSlab: 1380904 kB\nPageTables: 278864 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141210600 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32145113088}, {buffered_memory,290693120}, {free_memory,530071552}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3440685642,1}}, {context_switches,{751260205,0}}, {garbage_collection,{185676019,674783670812,0}}, {io,{{input,58240640764},{output,130485968417}}}, {reductions,{324567761515,1411963}}, {run_queue,0}, {runtime,{37303330,140}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 06:09:58 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 06:10:04 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 06:10:22 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.111.192> 2011-12-29 06:10:26 =============================================================================== ns_1@192.168.24.61:<5883.111.192>:stats_collector:83: Stats for bucket "JDPCachePrimary": accepting_conns 1 auth_cmds 10369 auth_errors 0 bucket_active_conns 1 bucket_conns 1 bytes_read 53441418790 bytes_written 45308356481 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 29975715 cmd_set 445274018 conn_yields 61205 connection_structures 1931 curr_connections 11 curr_items 297280049 curr_items_tot 297280049 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 466 delete_misses 174808 ep_bg_fetched 1661206 ep_bg_load 4365803470 ep_bg_load_avg 2628 ep_bg_max_load 2020165 ep_bg_max_wait 2929571 ep_bg_min_load 1 ep_bg_min_wait 10 ep_bg_num_samples 1661198 ep_bg_wait 48864908595 ep_bg_wait_avg 29415 ep_commit_num 818500 ep_commit_time 0 ep_commit_time_total 179775 ep_data_age 0 ep_data_age_highwat 2571 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 1 ep_dbname /opt/membase/var/lib/membase/data/JDPCachePrimary-data/JDPCachePrimary ep_dbshards 4 ep_diskqueue_drain 358215763 ep_diskqueue_fill 358215763 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 1256 ep_flush_duration_total 251908 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 1661206 ep_io_num_write 357979281 ep_io_read_bytes 145605718 ep_io_write_bytes 6012819006520 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 356086874 ep_kv_size 57356020515 ep_latency_arith_cmd 46093 ep_latency_get_cmd 101813590 ep_latency_store_cmd 515449025 ep_max_data_size 81788928000 ep_max_txn_size 1000 ep_mem_high_wat 61341696000 ep_mem_low_wat 49073356800 ep_min_data_age 0 ep_num_active_non_resident 94237869 ep_num_checkpoint_remover_runs354434 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 489 ep_num_non_resident 94237869 ep_num_not_my_vbuckets 0 ep_num_pager_runs 2 ep_num_value_ejects 95895079 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 1611440768 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 3597 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 62797498 ep_too_young 0 ep_total_cache_size 58930442359 ep_total_del_items 416 ep_total_enqueued 358215763 ep_total_new_items 297280465 ep_total_persisted 357979697 ep_uncommitted_items 0 ep_value_size 15808691965 ep_vb_total 512 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.1.1_39_g4bd26a3 ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 8853 get_hits 7311306 get_misses 22664409 incr_hits 44431 incr_misses 1662 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 58967461283 pid 17177 pointer_size 64 rejected_conns 0 rusage_system 91968.206707 rusage_user 249392.761537 threads 4 time 1325167826 total_connections 30202 uptime 3440757 vb_active_curr_items 297280049 vb_active_eject 95895079 vb_active_ht_memory 1610809344 vb_active_itm_memory 53788643543 vb_active_num 512 vb_active_num_non_resident 94237869 vb_active_ops_create 297280465 vb_active_ops_delete 416 vb_active_ops_reject 0 vb_active_ops_update 60698816 vb_active_perc_mem_resident 68 vb_active_queue_age 0 vb_active_queue_drain 358215763 vb_active_queue_fill 358215763 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version UNKNOWN INFO REPORT <5883.1095.0> 2011-12-29 06:10:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,167846,409506}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102700792}, {processes,31917032}, {processes_used,29242760}, {system,70783760}, {atom,903241}, {atom_used,880290}, {binary,1585784}, {code,8604866}, {ets,38589816}]}, {system_stats, [{cpu_utilization_rate,0.25}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3440775}, {memory_data,{101368545280,100269240320,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1073164 kB\nBuffers: 283600 kB\nCached: 32674860 kB\nSwapCached: 5096 kB\nActive: 82503204 kB\nInactive: 13861668 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1073164 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 372 kB\nWriteback: 0 kB\nAnonPages: 63398280 kB\nMapped: 16728 kB\nSlab: 1346188 kB\nPageTables: 171516 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86137268 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33459056640}, {buffered_memory,290406400}, {free_memory,1098919936}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3440760476,0}}, {context_switches,{1912591506,0}}, {garbage_collection,{247924547,3900650063725,0}}, {io,{{input,69529659868},{output,131908677322}}}, {reductions,{1585652645775,2478516}}, {run_queue,0}, {runtime,{130337280,210}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,167845,405111}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,83827088}, {processes,14283800}, {processes_used,14248104}, {system,69543288}, {atom,828337}, {atom_used,824728}, {binary,1451088}, {code,7914132}, {ets,38418520}]}, {system_stats, [{cpu_utilization_rate,12.875}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3440755}, {memory_data,{101368545280,100838490112,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 472432 kB\nBuffers: 283880 kB\nCached: 31392496 kB\nSwapCached: 3836 kB\nActive: 82968916 kB\nInactive: 13854932 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 472432 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 2964 kB\nWriteback: 0 kB\nAnonPages: 65139004 kB\nMapped: 24660 kB\nSlab: 1381128 kB\nPageTables: 279236 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141249692 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32145915904}, {buffered_memory,290693120}, {free_memory,483770368}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3440745641,0}}, {context_switches,{751272889,0}}, {garbage_collection,{185679053,674794743507,0}}, {io,{{input,58240813037},{output,130486872336}}}, {reductions,{324573191750,1392276}}, {run_queue,0}, {runtime,{37303910,150}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 06:11:08 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 06:11:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,167906,409153}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102550424}, {processes,31771816}, {processes_used,29097544}, {system,70778608}, {atom,903241}, {atom_used,880290}, {binary,1431432}, {code,8604866}, {ets,38739568}]}, {system_stats, [{cpu_utilization_rate,12.85892634207241}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3440835}, {memory_data,{101368545280,100281065472,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1042496 kB\nBuffers: 283604 kB\nCached: 32675160 kB\nSwapCached: 5096 kB\nActive: 82533648 kB\nInactive: 13861984 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1042496 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 3268 kB\nWriteback: 0 kB\nAnonPages: 63428384 kB\nMapped: 22708 kB\nSlab: 1346340 kB\nPageTables: 171848 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86163204 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33459363840}, {buffered_memory,290410496}, {free_memory,1067642880}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3440820476,0}}, {context_switches,{1912608509,0}}, {garbage_collection,{247928013,3900662780241,0}}, {io,{{input,69532310970},{output,131912126721}}}, {reductions,{1585664720031,2445294}}, {run_queue,0}, {runtime,{130338120,170}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,167905,404862}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,84004664}, {processes,14449488}, {processes_used,14413792}, {system,69555176}, {atom,828337}, {atom_used,824728}, {binary,1372536}, {code,7914132}, {ets,38509352}]}, {system_stats, [{cpu_utilization_rate,0.37546933667083854}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3440815}, {memory_data,{101368545280,100911476736,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 513472 kB\nBuffers: 283880 kB\nCached: 31392900 kB\nSwapCached: 3836 kB\nActive: 82924316 kB\nInactive: 13855200 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 513472 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 3972 kB\nWriteback: 0 kB\nAnonPages: 65094568 kB\nMapped: 17876 kB\nSlab: 1381816 kB\nPageTables: 278856 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141210384 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32146329600}, {buffered_memory,290693120}, {free_memory,525832192}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3440805641,1}}, {context_switches,{751286375,0}}, {garbage_collection,{185682596,674806568849,0}}, {io,{{input,58243427327},{output,130490352819}}}, {reductions,{324579055600,1390406}}, {run_queue,2}, {runtime,{37304630,160}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 06:12:00 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.111.192> 2011-12-29 06:12:06 =============================================================================== ns_1@192.168.24.61:<5883.111.192>:stats_collector:83: Stats for bucket "JDPCachePrimary": accepting_conns 1 auth_cmds 10369 auth_errors 0 bucket_active_conns 1 bucket_conns 1 bytes_read 53441425950 bytes_written 45310024621 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 29975715 cmd_set 445274018 conn_yields 61205 connection_structures 1931 curr_connections 11 curr_items 297280049 curr_items_tot 297280049 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 466 delete_misses 174808 ep_bg_fetched 1661206 ep_bg_load 4365803470 ep_bg_load_avg 2628 ep_bg_max_load 2020165 ep_bg_max_wait 2929571 ep_bg_min_load 1 ep_bg_min_wait 10 ep_bg_num_samples 1661198 ep_bg_wait 48864908595 ep_bg_wait_avg 29415 ep_commit_num 818500 ep_commit_time 0 ep_commit_time_total 179775 ep_data_age 0 ep_data_age_highwat 2571 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 1 ep_dbname /opt/membase/var/lib/membase/data/JDPCachePrimary-data/JDPCachePrimary ep_dbshards 4 ep_diskqueue_drain 358215763 ep_diskqueue_fill 358215763 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 1256 ep_flush_duration_total 251908 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 1661206 ep_io_num_write 357979281 ep_io_read_bytes 145605718 ep_io_write_bytes 6012819006520 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 356086874 ep_kv_size 57356020515 ep_latency_arith_cmd 46093 ep_latency_get_cmd 101813590 ep_latency_store_cmd 515449025 ep_max_data_size 81788928000 ep_max_txn_size 1000 ep_mem_high_wat 61341696000 ep_mem_low_wat 49073356800 ep_min_data_age 0 ep_num_active_non_resident 94237869 ep_num_checkpoint_remover_runs354454 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 489 ep_num_non_resident 94237869 ep_num_not_my_vbuckets 0 ep_num_pager_runs 2 ep_num_value_ejects 95895079 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 1611440768 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 3597 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 62797498 ep_too_young 0 ep_total_cache_size 58930442359 ep_total_del_items 416 ep_total_enqueued 358215763 ep_total_new_items 297280465 ep_total_persisted 357979697 ep_uncommitted_items 0 ep_value_size 15808691965 ep_vb_total 512 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.1.1_39_g4bd26a3 ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 8853 get_hits 7311306 get_misses 22664409 incr_hits 44431 incr_misses 1662 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 58967461283 pid 17177 pointer_size 64 rejected_conns 0 rusage_system 91968.230703 rusage_user 249392.852523 threads 4 time 1325167926 total_connections 30202 uptime 3440857 vb_active_curr_items 297280049 vb_active_eject 95895079 vb_active_ht_memory 1610809344 vb_active_itm_memory 53788643543 vb_active_num 512 vb_active_num_non_resident 94237869 vb_active_ops_create 297280465 vb_active_ops_delete 416 vb_active_ops_reject 0 vb_active_ops_update 60698816 vb_active_perc_mem_resident 68 vb_active_queue_age 0 vb_active_queue_drain 358215763 vb_active_queue_fill 358215763 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version UNKNOWN INFO REPORT <5883.136.0> 2011-12-29 06:12:23 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 06:12:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,167966,408492}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,101970792}, {processes,31407616}, {processes_used,28733344}, {system,70563176}, {atom,903241}, {atom_used,880290}, {binary,1685248}, {code,8604866}, {ets,38270360}]}, {system_stats, [{cpu_utilization_rate,0.4993757802746567}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3440895}, {memory_data,{101368545280,100272226304,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1070912 kB\nBuffers: 283604 kB\nCached: 32676040 kB\nSwapCached: 5096 kB\nActive: 82502820 kB\nInactive: 13862620 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1070912 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 3764 kB\nWriteback: 0 kB\nAnonPages: 63398040 kB\nMapped: 16728 kB\nSlab: 1346932 kB\nPageTables: 171512 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86132824 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33460264960}, {buffered_memory,290410496}, {free_memory,1096613888}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3440880476,1}}, {context_switches,{1912624764,0}}, {garbage_collection,{247931311,3900674745085,0}}, {io,{{input,69532533537},{output,131913046139}}}, {reductions,{1585676671657,2444403}}, {run_queue,0}, {runtime,{130338940,180}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,167965,404793}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,83779520}, {processes,14575368}, {processes_used,14539672}, {system,69204152}, {atom,828337}, {atom_used,824728}, {binary,1432352}, {code,7914132}, {ets,38098440}]}, {system_stats, [{cpu_utilization_rate,0.5}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3440875}, {memory_data,{101368545280,100839854080,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 514816 kB\nBuffers: 283880 kB\nCached: 31393668 kB\nSwapCached: 3836 kB\nActive: 82925272 kB\nInactive: 13855876 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 514816 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 416 kB\nWriteback: 0 kB\nAnonPages: 65095404 kB\nMapped: 17876 kB\nSlab: 1381044 kB\nPageTables: 278860 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141213432 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32147116032}, {buffered_memory,290693120}, {free_memory,527171584}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3440865641,0}}, {context_switches,{751298921,0}}, {garbage_collection,{185685951,674817853244,0}}, {io,{{input,58243664726},{output,130491230177}}}, {reductions,{324584555991,1433060}}, {run_queue,0}, {runtime,{37305210,150}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 06:12:56 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 06:13:10 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 06:13:26 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 06:13:39 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.111.192> 2011-12-29 06:13:46 =============================================================================== ns_1@192.168.24.61:<5883.111.192>:stats_collector:83: Stats for bucket "JDPCachePrimary": accepting_conns 1 auth_cmds 10369 auth_errors 0 bucket_active_conns 1 bucket_conns 1 bytes_read 53441433110 bytes_written 45311692761 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 29975715 cmd_set 445274018 conn_yields 61205 connection_structures 1931 curr_connections 11 curr_items 297280049 curr_items_tot 297280049 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 466 delete_misses 174808 ep_bg_fetched 1661206 ep_bg_load 4365803470 ep_bg_load_avg 2628 ep_bg_max_load 2020165 ep_bg_max_wait 2929571 ep_bg_min_load 1 ep_bg_min_wait 10 ep_bg_num_samples 1661198 ep_bg_wait 48864908595 ep_bg_wait_avg 29415 ep_commit_num 818500 ep_commit_time 0 ep_commit_time_total 179775 ep_data_age 0 ep_data_age_highwat 2571 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 1 ep_dbname /opt/membase/var/lib/membase/data/JDPCachePrimary-data/JDPCachePrimary ep_dbshards 4 ep_diskqueue_drain 358215763 ep_diskqueue_fill 358215763 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 1256 ep_flush_duration_total 251908 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 1661206 ep_io_num_write 357979281 ep_io_read_bytes 145605718 ep_io_write_bytes 6012819006520 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 356086874 ep_kv_size 57356020515 ep_latency_arith_cmd 46093 ep_latency_get_cmd 101813590 ep_latency_store_cmd 515449025 ep_max_data_size 81788928000 ep_max_txn_size 1000 ep_mem_high_wat 61341696000 ep_mem_low_wat 49073356800 ep_min_data_age 0 ep_num_active_non_resident 94237869 ep_num_checkpoint_remover_runs354474 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 489 ep_num_non_resident 94237869 ep_num_not_my_vbuckets 0 ep_num_pager_runs 2 ep_num_value_ejects 95895079 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 1611440768 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 3597 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 62797498 ep_too_young 0 ep_total_cache_size 58930442359 ep_total_del_items 416 ep_total_enqueued 358215763 ep_total_new_items 297280465 ep_total_persisted 357979697 ep_uncommitted_items 0 ep_value_size 15808691965 ep_vb_total 512 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.1.1_39_g4bd26a3 ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 8853 get_hits 7311306 get_misses 22664409 incr_hits 44431 incr_misses 1662 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 58967461283 pid 17177 pointer_size 64 rejected_conns 0 rusage_system 91968.265698 rusage_user 249392.950508 threads 4 time 1325168026 total_connections 30202 uptime 3440957 vb_active_curr_items 297280049 vb_active_eject 95895079 vb_active_ht_memory 1610809344 vb_active_itm_memory 53788643543 vb_active_num 512 vb_active_num_non_resident 94237869 vb_active_ops_create 297280465 vb_active_ops_delete 416 vb_active_ops_reject 0 vb_active_ops_update 60698816 vb_active_perc_mem_resident 68 vb_active_queue_age 0 vb_active_queue_drain 358215763 vb_active_queue_fill 358215763 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version UNKNOWN INFO REPORT <5883.1095.0> 2011-12-29 06:13:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,168026,409050}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102325360}, {processes,31913288}, {processes_used,29239016}, {system,70412072}, {atom,903241}, {atom_used,880290}, {binary,1500920}, {code,8604866}, {ets,38303448}]}, {system_stats, [{cpu_utilization_rate,0.25}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3440955}, {memory_data,{101368545280,100269809664,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1072744 kB\nBuffers: 283604 kB\nCached: 32676824 kB\nSwapCached: 5096 kB\nActive: 82502888 kB\nInactive: 13863352 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1072744 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 492 kB\nWriteback: 0 kB\nAnonPages: 63397976 kB\nMapped: 16728 kB\nSlab: 1346240 kB\nPageTables: 171512 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86135336 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33461067776}, {buffered_memory,290410496}, {free_memory,1098489856}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3440940476,2}}, {context_switches,{1912641769,0}}, {garbage_collection,{247934805,3900687084916,0}}, {io,{{input,69532787728},{output,131913925589}}}, {reductions,{1585689237287,2485988}}, {run_queue,0}, {runtime,{130339760,180}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,168025,405869}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,83731104}, {processes,14425704}, {processes_used,14390296}, {system,69305400}, {atom,828337}, {atom_used,824728}, {binary,1495112}, {code,7914132}, {ets,38136544}]}, {system_stats, [{cpu_utilization_rate,0.375}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3440935}, {memory_data,{101368545280,100841385984,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 514100 kB\nBuffers: 283880 kB\nCached: 31394460 kB\nSwapCached: 3836 kB\nActive: 82925836 kB\nInactive: 13856632 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 514100 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 544 kB\nWriteback: 0 kB\nAnonPages: 65096472 kB\nMapped: 17876 kB\nSlab: 1380856 kB\nPageTables: 278860 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141211400 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32147927040}, {buffered_memory,290693120}, {free_memory,526438400}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3440925642,1}}, {context_switches,{751311757,0}}, {garbage_collection,{185689174,674829129122,0}}, {io,{{input,58243831892},{output,130492164184}}}, {reductions,{324590273087,1386288}}, {run_queue,0}, {runtime,{37305780,150}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 06:14:21 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 06:14:33 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 06:14:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,168086,409067}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102420376}, {processes,31797352}, {processes_used,29123080}, {system,70623024}, {atom,903241}, {atom_used,880290}, {binary,1563000}, {code,8604866}, {ets,38452624}]}, {system_stats, [{cpu_utilization_rate,0.37453183520599254}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3441015}, {memory_data,{101368545280,100265254912,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1077660 kB\nBuffers: 283608 kB\nCached: 32672076 kB\nSwapCached: 5096 kB\nActive: 82502668 kB\nInactive: 13858788 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1077660 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 164 kB\nWriteback: 176 kB\nAnonPages: 63397664 kB\nMapped: 16728 kB\nSlab: 1346192 kB\nPageTables: 171512 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86133556 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33456205824}, {buffered_memory,290414592}, {free_memory,1103523840}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3441000475,0}}, {context_switches,{1912658773,0}}, {garbage_collection,{247938484,3900700411416,0}}, {io,{{input,69535490033},{output,131920922570}}}, {reductions,{1585701339810,2435685}}, {run_queue,0}, {runtime,{130340720,180}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,168085,404710}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,83436384}, {processes,14042000}, {processes_used,14006304}, {system,69394384}, {atom,828337}, {atom_used,824728}, {binary,1440120}, {code,7914132}, {ets,38280912}]}, {system_stats, [{cpu_utilization_rate,0.375}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3440995}, {memory_data,{101368545280,100841586688,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 519312 kB\nBuffers: 283896 kB\nCached: 31389704 kB\nSwapCached: 3836 kB\nActive: 82925416 kB\nInactive: 13852040 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 519312 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 320 kB\nWriteback: 0 kB\nAnonPages: 65096004 kB\nMapped: 17876 kB\nSlab: 1381000 kB\nPageTables: 278868 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141211264 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32143056896}, {buffered_memory,290709504}, {free_memory,531775488}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3440985642,1}}, {context_switches,{751325371,0}}, {garbage_collection,{185692671,674841519444,0}}, {io,{{input,58246441807},{output,130499232082}}}, {reductions,{324595875194,1400402}}, {run_queue,0}, {runtime,{37306550,160}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 06:15:06 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.111.192> 2011-12-29 06:15:26 =============================================================================== ns_1@192.168.24.61:<5883.111.192>:stats_collector:83: Stats for bucket "JDPCachePrimary": accepting_conns 1 auth_cmds 10369 auth_errors 0 bucket_active_conns 1 bucket_conns 1 bytes_read 53441440270 bytes_written 45313360901 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 29975715 cmd_set 445274018 conn_yields 61205 connection_structures 1931 curr_connections 11 curr_items 297280049 curr_items_tot 297280049 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 466 delete_misses 174808 ep_bg_fetched 1661206 ep_bg_load 4365803470 ep_bg_load_avg 2628 ep_bg_max_load 2020165 ep_bg_max_wait 2929571 ep_bg_min_load 1 ep_bg_min_wait 10 ep_bg_num_samples 1661198 ep_bg_wait 48864908595 ep_bg_wait_avg 29415 ep_commit_num 818500 ep_commit_time 0 ep_commit_time_total 179775 ep_data_age 0 ep_data_age_highwat 2571 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 1 ep_dbname /opt/membase/var/lib/membase/data/JDPCachePrimary-data/JDPCachePrimary ep_dbshards 4 ep_diskqueue_drain 358215763 ep_diskqueue_fill 358215763 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 1256 ep_flush_duration_total 251908 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 1661206 ep_io_num_write 357979281 ep_io_read_bytes 145605718 ep_io_write_bytes 6012819006520 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 356086874 ep_kv_size 57356020515 ep_latency_arith_cmd 46093 ep_latency_get_cmd 101813590 ep_latency_store_cmd 515449025 ep_max_data_size 81788928000 ep_max_txn_size 1000 ep_mem_high_wat 61341696000 ep_mem_low_wat 49073356800 ep_min_data_age 0 ep_num_active_non_resident 94237869 ep_num_checkpoint_remover_runs354494 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 489 ep_num_non_resident 94237869 ep_num_not_my_vbuckets 0 ep_num_pager_runs 2 ep_num_value_ejects 95895079 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 1611440768 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 3597 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 62797498 ep_too_young 0 ep_total_cache_size 58930442359 ep_total_del_items 416 ep_total_enqueued 358215763 ep_total_new_items 297280465 ep_total_persisted 357979697 ep_uncommitted_items 0 ep_value_size 15808691965 ep_vb_total 512 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.1.1_39_g4bd26a3 ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 8853 get_hits 7311306 get_misses 22664409 incr_hits 44431 incr_misses 1662 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 58967461283 pid 17177 pointer_size 64 rejected_conns 0 rusage_system 91968.299693 rusage_user 249393.047493 threads 4 time 1325168126 total_connections 30202 uptime 3441057 vb_active_curr_items 297280049 vb_active_eject 95895079 vb_active_ht_memory 1610809344 vb_active_itm_memory 53788643543 vb_active_num 512 vb_active_num_non_resident 94237869 vb_active_ops_create 297280465 vb_active_ops_delete 416 vb_active_ops_reject 0 vb_active_ops_update 60698816 vb_active_perc_mem_resident 68 vb_active_queue_age 0 vb_active_queue_drain 358215763 vb_active_queue_fill 358215763 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version UNKNOWN INFO REPORT <5883.1095.0> 2011-12-29 06:15:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,168146,409274}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102302880}, {processes,31754144}, {processes_used,29079872}, {system,70548736}, {atom,903241}, {atom_used,880290}, {binary,1455688}, {code,8604866}, {ets,38485472}]}, {system_stats, [{cpu_utilization_rate,0.25}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3441075}, {memory_data,{101368545280,100266364928,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1076340 kB\nBuffers: 283608 kB\nCached: 32672852 kB\nSwapCached: 5096 kB\nActive: 82503412 kB\nInactive: 13859540 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1076340 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 212 kB\nWriteback: 64 kB\nAnonPages: 63398676 kB\nMapped: 16728 kB\nSlab: 1346100 kB\nPageTables: 171512 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86134904 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33457000448}, {buffered_memory,290414592}, {free_memory,1102172160}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3441060476,1}}, {context_switches,{1912675117,0}}, {garbage_collection,{247941792,3900712437087,0}}, {io,{{input,69535709937},{output,131921819112}}}, {reductions,{1585713347439,2479595}}, {run_queue,0}, {runtime,{130341480,180}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,168145,405024}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,83860264}, {processes,14637288}, {processes_used,14601592}, {system,69222976}, {atom,828337}, {atom_used,824728}, {binary,1235920}, {code,7914132}, {ets,38313376}]}, {system_stats, [{cpu_utilization_rate,12.875}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3441055}, {memory_data,{101368545280,100836900864,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 519676 kB\nBuffers: 283896 kB\nCached: 31390468 kB\nSwapCached: 3836 kB\nActive: 82925092 kB\nInactive: 13852720 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 519676 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 44 kB\nWriteback: 0 kB\nAnonPages: 65095868 kB\nMapped: 17876 kB\nSlab: 1380904 kB\nPageTables: 278860 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141209616 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32143839232}, {buffered_memory,290709504}, {free_memory,532148224}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3441045641,1}}, {context_switches,{751338022,0}}, {garbage_collection,{185695720,674852632695,0}}, {io,{{input,58246672383},{output,130500100950}}}, {reductions,{324601300700,1404787}}, {run_queue,0}, {runtime,{37307150,180}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 06:15:56 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 06:16:29 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 06:16:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,168206,408729}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102756280}, {processes,31953680}, {processes_used,29279488}, {system,70802600}, {atom,903241}, {atom_used,880290}, {binary,1565152}, {code,8604866}, {ets,38630208}]}, {system_stats, [{cpu_utilization_rate,12.875}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3441135}, {memory_data,{101368545280,100311924736,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1000460 kB\nBuffers: 283608 kB\nCached: 32673648 kB\nSwapCached: 5096 kB\nActive: 82576272 kB\nInactive: 13860216 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1000460 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 3484 kB\nWriteback: 12 kB\nAnonPages: 63471544 kB\nMapped: 23808 kB\nSlab: 1347056 kB\nPageTables: 172668 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86208876 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33457815552}, {buffered_memory,290414592}, {free_memory,1024471040}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3441120476,1}}, {context_switches,{1912691389,0}}, {garbage_collection,{247945108,3900724651016,0}}, {io,{{input,69535927824},{output,131922730539}}}, {reductions,{1585725341436,2497152}}, {run_queue,0}, {runtime,{130342290,190}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,168205,407295}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,84318528}, {processes,14795496}, {processes_used,14759800}, {system,69523032}, {atom,828337}, {atom_used,824728}, {binary,1384584}, {code,7914132}, {ets,38465056}]}, {system_stats, [{cpu_utilization_rate,0.5}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3441115}, {memory_data,{101368545280,100867567616,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 421200 kB\nBuffers: 283896 kB\nCached: 31391332 kB\nSwapCached: 3836 kB\nActive: 83017404 kB\nInactive: 13853400 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 421200 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 3760 kB\nWriteback: 0 kB\nAnonPages: 65187772 kB\nMapped: 25348 kB\nSlab: 1382236 kB\nPageTables: 279584 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141302776 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32144723968}, {buffered_memory,290709504}, {free_memory,431308800}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3441105642,1}}, {context_switches,{751350433,0}}, {garbage_collection,{185698929,674863702344,0}}, {io,{{input,58246874648},{output,130500966045}}}, {reductions,{324606754445,1397146}}, {run_queue,0}, {runtime,{37307750,180}}]}]}] INFO REPORT <5883.111.192> 2011-12-29 06:17:06 =============================================================================== ns_1@192.168.24.61:<5883.111.192>:stats_collector:83: Stats for bucket "JDPCachePrimary": accepting_conns 1 auth_cmds 10369 auth_errors 0 bucket_active_conns 1 bucket_conns 1 bytes_read 53441447430 bytes_written 45315029041 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 29975715 cmd_set 445274018 conn_yields 61205 connection_structures 1931 curr_connections 11 curr_items 297280049 curr_items_tot 297280049 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 466 delete_misses 174808 ep_bg_fetched 1661206 ep_bg_load 4365803470 ep_bg_load_avg 2628 ep_bg_max_load 2020165 ep_bg_max_wait 2929571 ep_bg_min_load 1 ep_bg_min_wait 10 ep_bg_num_samples 1661198 ep_bg_wait 48864908595 ep_bg_wait_avg 29415 ep_commit_num 818500 ep_commit_time 0 ep_commit_time_total 179775 ep_data_age 0 ep_data_age_highwat 2571 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 1 ep_dbname /opt/membase/var/lib/membase/data/JDPCachePrimary-data/JDPCachePrimary ep_dbshards 4 ep_diskqueue_drain 358215763 ep_diskqueue_fill 358215763 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 1256 ep_flush_duration_total 251908 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 1661206 ep_io_num_write 357979281 ep_io_read_bytes 145605718 ep_io_write_bytes 6012819006520 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 356086874 ep_kv_size 57356020515 ep_latency_arith_cmd 46093 ep_latency_get_cmd 101813590 ep_latency_store_cmd 515449025 ep_max_data_size 81788928000 ep_max_txn_size 1000 ep_mem_high_wat 61341696000 ep_mem_low_wat 49073356800 ep_min_data_age 0 ep_num_active_non_resident 94237869 ep_num_checkpoint_remover_runs354514 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 489 ep_num_non_resident 94237869 ep_num_not_my_vbuckets 0 ep_num_pager_runs 2 ep_num_value_ejects 95895079 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 1611440768 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 3597 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 62797498 ep_too_young 0 ep_total_cache_size 58930442359 ep_total_del_items 416 ep_total_enqueued 358215763 ep_total_new_items 297280465 ep_total_persisted 357979697 ep_uncommitted_items 0 ep_value_size 15808691965 ep_vb_total 512 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.1.1_39_g4bd26a3 ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 8853 get_hits 7311306 get_misses 22664409 incr_hits 44431 incr_misses 1662 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 58967461283 pid 17177 pointer_size 64 rejected_conns 0 rusage_system 91968.333688 rusage_user 249393.128481 threads 4 time 1325168226 total_connections 30202 uptime 3441157 vb_active_curr_items 297280049 vb_active_eject 95895079 vb_active_ht_memory 1610809344 vb_active_itm_memory 53788643543 vb_active_num 512 vb_active_num_non_resident 94237869 vb_active_ops_create 297280465 vb_active_ops_delete 416 vb_active_ops_reject 0 vb_active_ops_update 60698816 vb_active_perc_mem_resident 68 vb_active_queue_age 0 vb_active_queue_drain 358215763 vb_active_queue_fill 358215763 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version UNKNOWN INFO REPORT <5883.136.0> 2011-12-29 06:17:07 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 06:17:27 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 06:17:40 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 06:17:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,168266,408505}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102430984}, {processes,31555320}, {processes_used,28881048}, {system,70875664}, {atom,903241}, {atom_used,880290}, {binary,1487472}, {code,8604866}, {ets,38780664}]}, {system_stats, [{cpu_utilization_rate,0.1251564455569462}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3441195}, {memory_data,{101368545280,100267466752,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1075768 kB\nBuffers: 283608 kB\nCached: 32674016 kB\nSwapCached: 5096 kB\nActive: 82502912 kB\nInactive: 13860620 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1075768 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 264 kB\nWriteback: 0 kB\nAnonPages: 63398004 kB\nMapped: 16728 kB\nSlab: 1346292 kB\nPageTables: 171516 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86135008 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33458192384}, {buffered_memory,290414592}, {free_memory,1101586432}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3441180476,2}}, {context_switches,{1912708420,0}}, {garbage_collection,{247948648,3900737220694,0}}, {io,{{input,69538618811},{output,131926189632}}}, {reductions,{1585737529242,2585330}}, {run_queue,3}, {runtime,{130343170,190}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,168265,404716}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,84597456}, {processes,15019904}, {processes_used,14984208}, {system,69577552}, {atom,828337}, {atom_used,824728}, {binary,1353456}, {code,7914132}, {ets,38550664}]}, {system_stats, [{cpu_utilization_rate,0.5}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3441175}, {memory_data,{101368545280,100841934848,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 517628 kB\nBuffers: 283896 kB\nCached: 31391644 kB\nSwapCached: 3836 kB\nActive: 82925692 kB\nInactive: 13853868 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 517628 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 444 kB\nWriteback: 0 kB\nAnonPages: 65096300 kB\nMapped: 17876 kB\nSlab: 1381212 kB\nPageTables: 278860 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141211728 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32145043456}, {buffered_memory,290709504}, {free_memory,530051072}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3441165641,1}}, {context_switches,{751363855,0}}, {garbage_collection,{185702345,674875598034,0}}, {io,{{input,58249493676},{output,130504486578}}}, {reductions,{324612587543,1413023}}, {run_queue,2}, {runtime,{37308440,150}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 06:17:54 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 06:18:15 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.111.192> 2011-12-29 06:18:46 =============================================================================== ns_1@192.168.24.61:<5883.111.192>:stats_collector:83: Stats for bucket "JDPCachePrimary": accepting_conns 1 auth_cmds 10369 auth_errors 0 bucket_active_conns 1 bucket_conns 1 bytes_read 53441454590 bytes_written 45316697181 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 29975715 cmd_set 445274018 conn_yields 61205 connection_structures 1931 curr_connections 11 curr_items 297280049 curr_items_tot 297280049 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 466 delete_misses 174808 ep_bg_fetched 1661206 ep_bg_load 4365803470 ep_bg_load_avg 2628 ep_bg_max_load 2020165 ep_bg_max_wait 2929571 ep_bg_min_load 1 ep_bg_min_wait 10 ep_bg_num_samples 1661198 ep_bg_wait 48864908595 ep_bg_wait_avg 29415 ep_commit_num 818500 ep_commit_time 0 ep_commit_time_total 179775 ep_data_age 0 ep_data_age_highwat 2571 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 1 ep_dbname /opt/membase/var/lib/membase/data/JDPCachePrimary-data/JDPCachePrimary ep_dbshards 4 ep_diskqueue_drain 358215763 ep_diskqueue_fill 358215763 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 1256 ep_flush_duration_total 251908 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 1661206 ep_io_num_write 357979281 ep_io_read_bytes 145605718 ep_io_write_bytes 6012819006520 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 356086874 ep_kv_size 57356020515 ep_latency_arith_cmd 46093 ep_latency_get_cmd 101813590 ep_latency_store_cmd 515449025 ep_max_data_size 81788928000 ep_max_txn_size 1000 ep_mem_high_wat 61341696000 ep_mem_low_wat 49073356800 ep_min_data_age 0 ep_num_active_non_resident 94237869 ep_num_checkpoint_remover_runs354534 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 489 ep_num_non_resident 94237869 ep_num_not_my_vbuckets 0 ep_num_pager_runs 2 ep_num_value_ejects 95895079 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 1611440768 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 3597 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 62797498 ep_too_young 0 ep_total_cache_size 58930442359 ep_total_del_items 416 ep_total_enqueued 358215763 ep_total_new_items 297280465 ep_total_persisted 357979697 ep_uncommitted_items 0 ep_value_size 15808691965 ep_vb_total 512 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.1.1_39_g4bd26a3 ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 8853 get_hits 7311306 get_misses 22664409 incr_hits 44431 incr_misses 1662 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 58967461283 pid 17177 pointer_size 64 rejected_conns 0 rusage_system 91968.349685 rusage_user 249393.223467 threads 4 time 1325168326 total_connections 30202 uptime 3441257 vb_active_curr_items 297280049 vb_active_eject 95895079 vb_active_ht_memory 1610809344 vb_active_itm_memory 53788643543 vb_active_num 512 vb_active_num_non_resident 94237869 vb_active_ops_create 297280465 vb_active_ops_delete 416 vb_active_ops_reject 0 vb_active_ops_update 60698816 vb_active_perc_mem_resident 68 vb_active_queue_age 0 vb_active_queue_drain 358215763 vb_active_queue_fill 358215763 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version UNKNOWN INFO REPORT <5883.136.0> 2011-12-29 06:18:48 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 06:18:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,168326,409095}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102306840}, {processes,31778312}, {processes_used,29104040}, {system,70528528}, {atom,903241}, {atom_used,880290}, {binary,1609496}, {code,8604866}, {ets,38311536}]}, {system_stats, [{cpu_utilization_rate,0.5}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3441255}, {memory_data,{101368545280,100267347968,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1075544 kB\nBuffers: 283608 kB\nCached: 32674764 kB\nSwapCached: 5096 kB\nActive: 82503120 kB\nInactive: 13861352 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1075544 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 184 kB\nWriteback: 0 kB\nAnonPages: 63398008 kB\nMapped: 16728 kB\nSlab: 1345984 kB\nPageTables: 171516 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86134480 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33458958336}, {buffered_memory,290414592}, {free_memory,1101357056}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3441240476,1}}, {context_switches,{1912724681,0}}, {garbage_collection,{247951925,3900749280895,0}}, {io,{{input,69538869269},{output,131927065645}}}, {reductions,{1585749404894,2475328}}, {run_queue,0}, {runtime,{130343940,180}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,168325,404628}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,83463240}, {processes,14365320}, {processes_used,14329624}, {system,69097920}, {atom,828337}, {atom_used,824728}, {binary,1284848}, {code,7914132}, {ets,38139600}]}, {system_stats, [{cpu_utilization_rate,0.7490636704119851}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3441235}, {memory_data,{101368545280,100838375424,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 517756 kB\nBuffers: 283896 kB\nCached: 31392420 kB\nSwapCached: 3836 kB\nActive: 82925356 kB\nInactive: 13854608 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 517756 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 180 kB\nWriteback: 0 kB\nAnonPages: 65095932 kB\nMapped: 17876 kB\nSlab: 1381020 kB\nPageTables: 278860 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141211708 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32145838080}, {buffered_memory,290709504}, {free_memory,530182144}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3441225641,0}}, {context_switches,{751376296,0}}, {garbage_collection,{185705584,674886786997,0}}, {io,{{input,58249722204},{output,130505384999}}}, {reductions,{324618054272,1394602}}, {run_queue,0}, {runtime,{37309040,130}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 06:19:43 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 06:19:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,168386,408970}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102215720}, {processes,31712600}, {processes_used,29038328}, {system,70503120}, {atom,903241}, {atom_used,880290}, {binary,1550304}, {code,8604866}, {ets,38344864}]}, {system_stats, [{cpu_utilization_rate,12.640801001251564}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3441315}, {memory_data,{101368545280,100268322816,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1074556 kB\nBuffers: 283608 kB\nCached: 32675564 kB\nSwapCached: 5096 kB\nActive: 82503420 kB\nInactive: 13862132 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1074556 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 476 kB\nWriteback: 0 kB\nAnonPages: 63398532 kB\nMapped: 16728 kB\nSlab: 1346036 kB\nPageTables: 171516 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86135588 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33459777536}, {buffered_memory,290414592}, {free_memory,1100345344}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3441300476,1}}, {context_switches,{1912741605,0}}, {garbage_collection,{247955316,3900761457745,0}}, {io,{{input,69539058637},{output,131927978838}}}, {reductions,{1585761862915,2431981}}, {run_queue,0}, {runtime,{130344730,160}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,168385,404777}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,83374112}, {processes,14190416}, {processes_used,14154720}, {system,69183696}, {atom,828337}, {atom_used,824728}, {binary,1338048}, {code,7914132}, {ets,38171880}]}, {system_stats, [{cpu_utilization_rate,12.875}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3441295}, {memory_data,{101368545280,100838764544,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 515656 kB\nBuffers: 283896 kB\nCached: 31393196 kB\nSwapCached: 3836 kB\nActive: 82926404 kB\nInactive: 13855384 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 515656 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 516 kB\nWriteback: 0 kB\nAnonPages: 65096968 kB\nMapped: 17876 kB\nSlab: 1380928 kB\nPageTables: 278864 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141212028 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32146632704}, {buffered_memory,290709504}, {free_memory,528031744}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3441285641,0}}, {context_switches,{751389135,0}}, {garbage_collection,{185708901,674898073590,0}}, {io,{{input,58249882739},{output,130506269776}}}, {reductions,{324623803684,1413225}}, {run_queue,0}, {runtime,{37309620,140}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 06:20:05 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 06:20:22 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.111.192> 2011-12-29 06:20:26 =============================================================================== ns_1@192.168.24.61:<5883.111.192>:stats_collector:83: Stats for bucket "JDPCachePrimary": accepting_conns 1 auth_cmds 10369 auth_errors 0 bucket_active_conns 1 bucket_conns 1 bytes_read 53441461750 bytes_written 45318365321 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 29975715 cmd_set 445274018 conn_yields 61205 connection_structures 1931 curr_connections 11 curr_items 297280049 curr_items_tot 297280049 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 466 delete_misses 174808 ep_bg_fetched 1661206 ep_bg_load 4365803470 ep_bg_load_avg 2628 ep_bg_max_load 2020165 ep_bg_max_wait 2929571 ep_bg_min_load 1 ep_bg_min_wait 10 ep_bg_num_samples 1661198 ep_bg_wait 48864908595 ep_bg_wait_avg 29415 ep_commit_num 818500 ep_commit_time 0 ep_commit_time_total 179775 ep_data_age 0 ep_data_age_highwat 2571 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 1 ep_dbname /opt/membase/var/lib/membase/data/JDPCachePrimary-data/JDPCachePrimary ep_dbshards 4 ep_diskqueue_drain 358215763 ep_diskqueue_fill 358215763 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 1256 ep_flush_duration_total 251908 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 1661206 ep_io_num_write 357979281 ep_io_read_bytes 145605718 ep_io_write_bytes 6012819006520 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 356086874 ep_kv_size 57356020515 ep_latency_arith_cmd 46093 ep_latency_get_cmd 101813590 ep_latency_store_cmd 515449025 ep_max_data_size 81788928000 ep_max_txn_size 1000 ep_mem_high_wat 61341696000 ep_mem_low_wat 49073356800 ep_min_data_age 0 ep_num_active_non_resident 94237869 ep_num_checkpoint_remover_runs354554 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 490 ep_num_non_resident 94237869 ep_num_not_my_vbuckets 0 ep_num_pager_runs 2 ep_num_value_ejects 95895079 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 1611440768 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 3597 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 62797498 ep_too_young 0 ep_total_cache_size 58930442359 ep_total_del_items 416 ep_total_enqueued 358215763 ep_total_new_items 297280465 ep_total_persisted 357979697 ep_uncommitted_items 0 ep_value_size 15808691965 ep_vb_total 512 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.1.1_39_g4bd26a3 ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 8853 get_hits 7311306 get_misses 22664409 incr_hits 44431 incr_misses 1662 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 58967461283 pid 17177 pointer_size 64 rejected_conns 0 rusage_system 91968.410676 rusage_user 249492.528370 threads 4 time 1325168426 total_connections 30202 uptime 3441357 vb_active_curr_items 297280049 vb_active_eject 95895079 vb_active_ht_memory 1610809344 vb_active_itm_memory 53788643543 vb_active_num 512 vb_active_num_non_resident 94237869 vb_active_ops_create 297280465 vb_active_ops_delete 416 vb_active_ops_reject 0 vb_active_ops_update 60698816 vb_active_perc_mem_resident 68 vb_active_queue_age 0 vb_active_queue_drain 358215763 vb_active_queue_fill 358215763 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version UNKNOWN INFO REPORT <5883.136.0> 2011-12-29 06:20:41 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 06:20:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,168446,409126}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102248120}, {processes,31867248}, {processes_used,29192976}, {system,70380872}, {atom,903241}, {atom_used,880290}, {binary,1273328}, {code,8604866}, {ets,38499920}]}, {system_stats, [{cpu_utilization_rate,0.37546933667083854}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3441375}, {memory_data,{101368545280,100266848256,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1075860 kB\nBuffers: 283616 kB\nCached: 32675216 kB\nSwapCached: 5096 kB\nActive: 82502120 kB\nInactive: 13861720 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1075860 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 224 kB\nWriteback: 0 kB\nAnonPages: 63397256 kB\nMapped: 16728 kB\nSlab: 1345904 kB\nPageTables: 171512 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86137400 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33459421184}, {buffered_memory,290422784}, {free_memory,1101807616}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3441360476,2}}, {context_switches,{1912758760,0}}, {garbage_collection,{247958861,3900774357951,0}}, {io,{{input,69541804334},{output,131930801018}}}, {reductions,{1585773965171,2498117}}, {run_queue,1}, {runtime,{130345510,160}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,168445,404813}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,83814768}, {processes,14524664}, {processes_used,14488968}, {system,69290104}, {atom,828337}, {atom_used,824728}, {binary,1288944}, {code,7914132}, {ets,38327552}]}, {system_stats, [{cpu_utilization_rate,0.4993757802746567}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3441355}, {memory_data,{101368545280,100840742912,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 517640 kB\nBuffers: 283900 kB\nCached: 31392856 kB\nSwapCached: 3836 kB\nActive: 82925072 kB\nInactive: 13854984 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 517640 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 384 kB\nWriteback: 0 kB\nAnonPages: 65095196 kB\nMapped: 17876 kB\nSlab: 1380988 kB\nPageTables: 278860 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141213488 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32146284544}, {buffered_memory,290713600}, {free_memory,530063360}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3441345641,1}}, {context_switches,{751402619,0}}, {garbage_collection,{185712309,674909725427,0}}, {io,{{input,58252534354},{output,130509115563}}}, {reductions,{324629354626,1413286}}, {run_queue,0}, {runtime,{37310250,150}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 06:21:04 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 06:21:33 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 06:21:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,168506,408523}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102387592}, {processes,31886624}, {processes_used,29212496}, {system,70500968}, {atom,903241}, {atom_used,880290}, {binary,1360904}, {code,8604866}, {ets,38532464}]}, {system_stats, [{cpu_utilization_rate,12.875}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3441435}, {memory_data,{101368545280,100269436928,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1030288 kB\nBuffers: 283616 kB\nCached: 32675988 kB\nSwapCached: 5096 kB\nActive: 82545316 kB\nInactive: 13862512 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1030288 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 2704 kB\nWriteback: 0 kB\nAnonPages: 63440208 kB\nMapped: 23140 kB\nSlab: 1346208 kB\nPageTables: 171816 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86177680 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33460211712}, {buffered_memory,290422784}, {free_memory,1055014912}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3441420476,1}}, {context_switches,{1912775087,0}}, {garbage_collection,{247962093,3900786383859,0}}, {io,{{input,69542013518},{output,131931645395}}}, {reductions,{1585785941525,2446915}}, {run_queue,0}, {runtime,{130346290,170}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,168505,404825}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,84157104}, {processes,14712112}, {processes_used,14676416}, {system,69444992}, {atom,828337}, {atom_used,824728}, {binary,1410096}, {code,7914132}, {ets,38360880}]}, {system_stats, [{cpu_utilization_rate,12.983770287141073}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3441415}, {memory_data,{101368545280,100841734144,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 438436 kB\nBuffers: 283900 kB\nCached: 31393660 kB\nSwapCached: 3836 kB\nActive: 83000120 kB\nInactive: 13855628 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 438436 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 2952 kB\nWriteback: 0 kB\nAnonPages: 65169764 kB\nMapped: 24396 kB\nSlab: 1381772 kB\nPageTables: 279844 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141287240 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32147107840}, {buffered_memory,290713600}, {free_memory,448958464}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3441405641,1}}, {context_switches,{751415191,0}}, {garbage_collection,{185715570,674920928751,0}}, {io,{{input,58252737336},{output,130510035956}}}, {reductions,{324634870159,1413400}}, {run_queue,0}, {runtime,{37310830,150}}]}]}] INFO REPORT <5883.111.192> 2011-12-29 06:22:06 =============================================================================== ns_1@192.168.24.61:<5883.111.192>:stats_collector:83: Stats for bucket "JDPCachePrimary": accepting_conns 1 auth_cmds 10369 auth_errors 0 bucket_active_conns 1 bucket_conns 1 bytes_read 53441468910 bytes_written 45320033461 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 29975715 cmd_set 445274018 conn_yields 61205 connection_structures 1931 curr_connections 11 curr_items 297280049 curr_items_tot 297280049 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 466 delete_misses 174808 ep_bg_fetched 1661206 ep_bg_load 4365803470 ep_bg_load_avg 2628 ep_bg_max_load 2020165 ep_bg_max_wait 2929571 ep_bg_min_load 1 ep_bg_min_wait 10 ep_bg_num_samples 1661198 ep_bg_wait 48864908595 ep_bg_wait_avg 29415 ep_commit_num 818500 ep_commit_time 0 ep_commit_time_total 179775 ep_data_age 0 ep_data_age_highwat 2571 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 1 ep_dbname /opt/membase/var/lib/membase/data/JDPCachePrimary-data/JDPCachePrimary ep_dbshards 4 ep_diskqueue_drain 358215763 ep_diskqueue_fill 358215763 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 1256 ep_flush_duration_total 251908 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 1661206 ep_io_num_write 357979281 ep_io_read_bytes 145605718 ep_io_write_bytes 6012819006520 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 356086874 ep_kv_size 57356020515 ep_latency_arith_cmd 46093 ep_latency_get_cmd 101813590 ep_latency_store_cmd 515449025 ep_max_data_size 81788928000 ep_max_txn_size 1000 ep_mem_high_wat 61341696000 ep_mem_low_wat 49073356800 ep_min_data_age 0 ep_num_active_non_resident 94237869 ep_num_checkpoint_remover_runs354574 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 490 ep_num_non_resident 94237869 ep_num_not_my_vbuckets 0 ep_num_pager_runs 2 ep_num_value_ejects 95895079 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 1611440768 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 3597 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 62797498 ep_too_young 0 ep_total_cache_size 58930442359 ep_total_del_items 416 ep_total_enqueued 358215763 ep_total_new_items 297280465 ep_total_persisted 357979697 ep_uncommitted_items 0 ep_value_size 15808691965 ep_vb_total 512 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.1.1_39_g4bd26a3 ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 8853 get_hits 7311306 get_misses 22664409 incr_hits 44431 incr_misses 1662 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 58967461283 pid 17177 pointer_size 64 rejected_conns 0 rusage_system 91968.449670 rusage_user 249502.003930 threads 4 time 1325168526 total_connections 30202 uptime 3441457 vb_active_curr_items 297280049 vb_active_eject 95895079 vb_active_ht_memory 1610809344 vb_active_itm_memory 53788643543 vb_active_num 512 vb_active_num_non_resident 94237869 vb_active_ops_create 297280465 vb_active_ops_delete 416 vb_active_ops_reject 0 vb_active_ops_update 60698816 vb_active_perc_mem_resident 68 vb_active_queue_age 0 vb_active_queue_drain 358215763 vb_active_queue_fill 358215763 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version UNKNOWN INFO REPORT <5883.136.0> 2011-12-29 06:22:28 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 06:22:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,168566,408685}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102241168}, {processes,31667272}, {processes_used,28993000}, {system,70573896}, {atom,903241}, {atom_used,880290}, {binary,1287224}, {code,8604866}, {ets,38678800}]}, {system_stats, [{cpu_utilization_rate,0.375}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3441495}, {memory_data,{101368545280,100272209920,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1071324 kB\nBuffers: 283616 kB\nCached: 32676864 kB\nSwapCached: 5096 kB\nActive: 82503192 kB\nInactive: 13863124 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1071324 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 4336 kB\nWriteback: 0 kB\nAnonPages: 63397736 kB\nMapped: 16728 kB\nSlab: 1346972 kB\nPageTables: 171512 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86133308 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33461108736}, {buffered_memory,290422784}, {free_memory,1097035776}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3441480476,2}}, {context_switches,{1912791144,0}}, {garbage_collection,{247965317,3900798366517,0}}, {io,{{input,69542237262},{output,131932531044}}}, {reductions,{1585798012216,2527197}}, {run_queue,0}, {runtime,{130347060,180}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,168565,404694}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,84170624}, {processes,14725584}, {processes_used,14689888}, {system,69445040}, {atom,828337}, {atom_used,824728}, {binary,1264040}, {code,7914132}, {ets,38506888}]}, {system_stats, [{cpu_utilization_rate,0.5}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3441475}, {memory_data,{101368545280,100845928448,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 514664 kB\nBuffers: 283900 kB\nCached: 31394496 kB\nSwapCached: 3836 kB\nActive: 82926368 kB\nInactive: 13856332 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 514664 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 504 kB\nWriteback: 0 kB\nAnonPages: 65096704 kB\nMapped: 17876 kB\nSlab: 1381112 kB\nPageTables: 278864 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141212208 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32147963904}, {buffered_memory,290713600}, {free_memory,527015936}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3441465641,0}}, {context_switches,{751427631,0}}, {garbage_collection,{185718716,674932206713,0}}, {io,{{input,58252890759},{output,130510873729}}}, {reductions,{324640377464,1414035}}, {run_queue,0}, {runtime,{37311410,160}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 06:23:14 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 06:23:40 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.111.192> 2011-12-29 06:23:46 =============================================================================== ns_1@192.168.24.61:<5883.111.192>:stats_collector:83: Stats for bucket "JDPCachePrimary": accepting_conns 1 auth_cmds 10369 auth_errors 0 bucket_active_conns 1 bucket_conns 1 bytes_read 53441476070 bytes_written 45321701601 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 29975715 cmd_set 445274018 conn_yields 61205 connection_structures 1931 curr_connections 11 curr_items 297280049 curr_items_tot 297280049 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 466 delete_misses 174808 ep_bg_fetched 1661206 ep_bg_load 4365803470 ep_bg_load_avg 2628 ep_bg_max_load 2020165 ep_bg_max_wait 2929571 ep_bg_min_load 1 ep_bg_min_wait 10 ep_bg_num_samples 1661198 ep_bg_wait 48864908595 ep_bg_wait_avg 29415 ep_commit_num 818500 ep_commit_time 0 ep_commit_time_total 179775 ep_data_age 0 ep_data_age_highwat 2571 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 1 ep_dbname /opt/membase/var/lib/membase/data/JDPCachePrimary-data/JDPCachePrimary ep_dbshards 4 ep_diskqueue_drain 358215763 ep_diskqueue_fill 358215763 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 1256 ep_flush_duration_total 251908 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 1661206 ep_io_num_write 357979281 ep_io_read_bytes 145605718 ep_io_write_bytes 6012819006520 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 356086874 ep_kv_size 57356020515 ep_latency_arith_cmd 46093 ep_latency_get_cmd 101813590 ep_latency_store_cmd 515449025 ep_max_data_size 81788928000 ep_max_txn_size 1000 ep_mem_high_wat 61341696000 ep_mem_low_wat 49073356800 ep_min_data_age 0 ep_num_active_non_resident 94237869 ep_num_checkpoint_remover_runs354594 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 490 ep_num_non_resident 94237869 ep_num_not_my_vbuckets 0 ep_num_pager_runs 2 ep_num_value_ejects 95895079 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 1611440768 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 3597 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 62797498 ep_too_young 0 ep_total_cache_size 58930442359 ep_total_del_items 416 ep_total_enqueued 358215763 ep_total_new_items 297280465 ep_total_persisted 357979697 ep_uncommitted_items 0 ep_value_size 15808691965 ep_vb_total 512 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.1.1_39_g4bd26a3 ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 8853 get_hits 7311306 get_misses 22664409 incr_hits 44431 incr_misses 1662 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 58967461283 pid 17177 pointer_size 64 rejected_conns 0 rusage_system 91968.474666 rusage_user 249502.111913 threads 4 time 1325168626 total_connections 30202 uptime 3441557 vb_active_curr_items 297280049 vb_active_eject 95895079 vb_active_ht_memory 1610809344 vb_active_itm_memory 53788643543 vb_active_num 512 vb_active_num_non_resident 94237869 vb_active_ops_create 297280465 vb_active_ops_delete 416 vb_active_ops_reject 0 vb_active_ops_update 60698816 vb_active_perc_mem_resident 68 vb_active_queue_age 0 vb_active_queue_drain 358215763 vb_active_queue_fill 358215763 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version UNKNOWN INFO REPORT <5883.1095.0> 2011-12-29 06:23:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,168626,409181}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102673472}, {processes,31908376}, {processes_used,29234104}, {system,70765096}, {atom,903241}, {atom_used,880290}, {binary,1330144}, {code,8604866}, {ets,38827608}]}, {system_stats, [{cpu_utilization_rate,0.25}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3441555}, {memory_data,{101368545280,100266303488,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1076536 kB\nBuffers: 283620 kB\nCached: 32673316 kB\nSwapCached: 5096 kB\nActive: 82503348 kB\nInactive: 13859736 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1076536 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 304 kB\nWriteback: 0 kB\nAnonPages: 63398228 kB\nMapped: 16728 kB\nSlab: 1346204 kB\nPageTables: 171512 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86134220 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33457475584}, {buffered_memory,290426880}, {free_memory,1102372864}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3441540475,1}}, {context_switches,{1912808012,0}}, {garbage_collection,{247968687,3900811681576,0}}, {io,{{input,69544897707},{output,131940560809}}}, {reductions,{1585810086785,2510396}}, {run_queue,0}, {runtime,{130348050,190}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,168625,404755}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,84015952}, {processes,14371600}, {processes_used,14335904}, {system,69644352}, {atom,828337}, {atom_used,824728}, {binary,1373416}, {code,7914132}, {ets,38597312}]}, {system_stats, [{cpu_utilization_rate,0.2503128911138924}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3441535}, {memory_data,{101368545280,100842016768,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 518608 kB\nBuffers: 283908 kB\nCached: 31390960 kB\nSwapCached: 3836 kB\nActive: 82924976 kB\nInactive: 13852976 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 518608 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 300 kB\nWriteback: 0 kB\nAnonPages: 65095404 kB\nMapped: 17876 kB\nSlab: 1381100 kB\nPageTables: 278860 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141209012 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32144343040}, {buffered_memory,290721792}, {free_memory,531054592}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3441525642,1}}, {context_switches,{751440988,0}}, {garbage_collection,{185722572,674944851846,0}}, {io,{{input,58255581068},{output,130518874771}}}, {reductions,{324646310363,1397789}}, {run_queue,0}, {runtime,{37312230,170}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 06:24:07 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 06:24:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,168686,408514}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102304400}, {processes,31967640}, {processes_used,29293368}, {system,70336760}, {atom,903241}, {atom_used,880290}, {binary,1370312}, {code,8604866}, {ets,38358768}]}, {system_stats, [{cpu_utilization_rate,0.37546933667083854}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3441615}, {memory_data,{101368545280,100266717184,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1075636 kB\nBuffers: 283620 kB\nCached: 32674080 kB\nSwapCached: 5096 kB\nActive: 82503472 kB\nInactive: 13860472 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1075636 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 0 kB\nWriteback: 196 kB\nAnonPages: 63398616 kB\nMapped: 16728 kB\nSlab: 1346092 kB\nPageTables: 171512 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86131812 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33458257920}, {buffered_memory,290426880}, {free_memory,1101451264}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3441600476,1}}, {context_switches,{1912824295,0}}, {garbage_collection,{247971872,3900823698320,0}}, {io,{{input,69545117620},{output,131941443144}}}, {reductions,{1585821980339,2438477}}, {run_queue,0}, {runtime,{130348880,200}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,168685,405264}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,83229064}, {processes,13943272}, {processes_used,13907576}, {system,69285792}, {atom,828337}, {atom_used,824728}, {binary,1425632}, {code,7914132}, {ets,38186488}]}, {system_stats, [{cpu_utilization_rate,0.625}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3441595}, {memory_data,{101368545280,100836859904,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 519076 kB\nBuffers: 283908 kB\nCached: 31391732 kB\nSwapCached: 3836 kB\nActive: 82924596 kB\nInactive: 13853716 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 519076 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 196 kB\nWriteback: 0 kB\nAnonPages: 65094472 kB\nMapped: 17876 kB\nSlab: 1381044 kB\nPageTables: 278864 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141209424 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32145133568}, {buffered_memory,290721792}, {free_memory,531533824}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3441585641,0}}, {context_switches,{751453513,0}}, {garbage_collection,{185725812,674956024061,0}}, {io,{{input,58255782348},{output,130519750395}}}, {reductions,{324651799078,1404146}}, {run_queue,1}, {runtime,{37312820,150}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 06:25:04 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 06:25:17 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.111.192> 2011-12-29 06:25:26 =============================================================================== ns_1@192.168.24.61:<5883.111.192>:stats_collector:83: Stats for bucket "JDPCachePrimary": accepting_conns 1 auth_cmds 10369 auth_errors 0 bucket_active_conns 1 bucket_conns 1 bytes_read 53441483230 bytes_written 45323369741 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 29975715 cmd_set 445274018 conn_yields 61205 connection_structures 1931 curr_connections 11 curr_items 297280049 curr_items_tot 297280049 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 466 delete_misses 174808 ep_bg_fetched 1661206 ep_bg_load 4365803470 ep_bg_load_avg 2628 ep_bg_max_load 2020165 ep_bg_max_wait 2929571 ep_bg_min_load 1 ep_bg_min_wait 10 ep_bg_num_samples 1661198 ep_bg_wait 48864908595 ep_bg_wait_avg 29415 ep_commit_num 818500 ep_commit_time 0 ep_commit_time_total 179775 ep_data_age 0 ep_data_age_highwat 2571 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 1 ep_dbname /opt/membase/var/lib/membase/data/JDPCachePrimary-data/JDPCachePrimary ep_dbshards 4 ep_diskqueue_drain 358215763 ep_diskqueue_fill 358215763 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 1256 ep_flush_duration_total 251908 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 1661206 ep_io_num_write 357979281 ep_io_read_bytes 145605718 ep_io_write_bytes 6012819006520 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 356086874 ep_kv_size 57356020515 ep_latency_arith_cmd 46093 ep_latency_get_cmd 101813590 ep_latency_store_cmd 515449025 ep_max_data_size 81788928000 ep_max_txn_size 1000 ep_mem_high_wat 61341696000 ep_mem_low_wat 49073356800 ep_min_data_age 0 ep_num_active_non_resident 94237869 ep_num_checkpoint_remover_runs354614 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 490 ep_num_non_resident 94237869 ep_num_not_my_vbuckets 0 ep_num_pager_runs 2 ep_num_value_ejects 95895079 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 1611440768 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 3597 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 62797498 ep_too_young 0 ep_total_cache_size 58930442359 ep_total_del_items 416 ep_total_enqueued 358215763 ep_total_new_items 297280465 ep_total_persisted 357979697 ep_uncommitted_items 0 ep_value_size 15808691965 ep_vb_total 512 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.1.1_39_g4bd26a3 ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 8853 get_hits 7311306 get_misses 22664409 incr_hits 44431 incr_misses 1662 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 58967461283 pid 17177 pointer_size 64 rejected_conns 0 rusage_system 91968.497663 rusage_user 249502.213898 threads 4 time 1325168726 total_connections 30202 uptime 3441657 vb_active_curr_items 297280049 vb_active_eject 95895079 vb_active_ht_memory 1610809344 vb_active_itm_memory 53788643543 vb_active_num 512 vb_active_num_non_resident 94237869 vb_active_ops_create 297280465 vb_active_ops_delete 416 vb_active_ops_reject 0 vb_active_ops_update 60698816 vb_active_perc_mem_resident 68 vb_active_queue_age 0 vb_active_queue_drain 358215763 vb_active_queue_fill 358215763 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version UNKNOWN INFO REPORT <5883.136.0> 2011-12-29 06:25:44 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 06:25:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,168746,408353}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102161584}, {processes,31751176}, {processes_used,29076904}, {system,70410408}, {atom,903241}, {atom_used,880290}, {binary,1410264}, {code,8604866}, {ets,38392152}]}, {system_stats, [{cpu_utilization_rate,0.375}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3441675}, {memory_data,{101368545280,100266315776,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1076288 kB\nBuffers: 283620 kB\nCached: 32674892 kB\nSwapCached: 5096 kB\nActive: 82502320 kB\nInactive: 13861208 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1076288 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 484 kB\nWriteback: 0 kB\nAnonPages: 63397180 kB\nMapped: 16728 kB\nSlab: 1346100 kB\nPageTables: 171512 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86133632 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33459089408}, {buffered_memory,290426880}, {free_memory,1102118912}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3441660475,0}}, {context_switches,{1912841104,0}}, {garbage_collection,{247975391,3900836199251,0}}, {io,{{input,69545335526},{output,131942317172}}}, {reductions,{1585834480994,2448925}}, {run_queue,0}, {runtime,{130349660,170}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,168745,405200}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,83756056}, {processes,14461048}, {processes_used,14425496}, {system,69295008}, {atom,828337}, {atom_used,824728}, {binary,1395640}, {code,7914132}, {ets,38225736}]}, {system_stats, [{cpu_utilization_rate,0.5006257822277848}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3441655}, {memory_data,{101368545280,100837781504,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 516576 kB\nBuffers: 283908 kB\nCached: 31392508 kB\nSwapCached: 3836 kB\nActive: 82926224 kB\nInactive: 13854492 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 516576 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 484 kB\nWriteback: 0 kB\nAnonPages: 65096652 kB\nMapped: 17876 kB\nSlab: 1381008 kB\nPageTables: 278864 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141207200 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32145928192}, {buffered_memory,290721792}, {free_memory,528973824}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3441645641,1}}, {context_switches,{751466362,0}}, {garbage_collection,{185729141,674967287001,0}}, {io,{{input,58255938828},{output,130520660353}}}, {reductions,{324657518257,1400198}}, {run_queue,0}, {runtime,{37313430,140}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 06:26:41 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 06:26:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,168806,408569}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102225096}, {processes,31698920}, {processes_used,29024648}, {system,70526176}, {atom,903241}, {atom_used,880290}, {binary,1377024}, {code,8604866}, {ets,38541408}]}, {system_stats, [{cpu_utilization_rate,12.907268170426065}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3441735}, {memory_data,{101368545280,100305903616,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1013468 kB\nBuffers: 283632 kB\nCached: 32674564 kB\nSwapCached: 5096 kB\nActive: 82562316 kB\nInactive: 13860824 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1013468 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 3360 kB\nWriteback: 0 kB\nAnonPages: 63457272 kB\nMapped: 23652 kB\nSlab: 1346912 kB\nPageTables: 172196 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86192176 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33458753536}, {buffered_memory,290439168}, {free_memory,1037791232}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3441720478,0}}, {context_switches,{1912858166,0}}, {garbage_collection,{247978740,3900848741950,0}}, {io,{{input,69548004344},{output,131945130024}}}, {reductions,{1585846578218,2447367}}, {run_queue,0}, {runtime,{130350520,180}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,168805,405371}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,84177264}, {processes,14710976}, {processes_used,14675280}, {system,69466288}, {atom,828337}, {atom_used,824728}, {binary,1422528}, {code,7914132}, {ets,38370128}]}, {system_stats, [{cpu_utilization_rate,13.0}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3441715}, {memory_data,{101368545280,100840583168,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 455040 kB\nBuffers: 283908 kB\nCached: 31392172 kB\nSwapCached: 3836 kB\nActive: 82984784 kB\nInactive: 13854068 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 455040 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 2720 kB\nWriteback: 0 kB\nAnonPages: 65154740 kB\nMapped: 25360 kB\nSlab: 1381656 kB\nPageTables: 280316 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141269288 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32145584128}, {buffered_memory,290721792}, {free_memory,465960960}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3441705642,2}}, {context_switches,{751479646,0}}, {garbage_collection,{185732527,674978994433,0}}, {io,{{input,58258582712},{output,130523438473}}}, {reductions,{324663115031,1406188}}, {run_queue,0}, {runtime,{37314090,150}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 06:27:05 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.111.192> 2011-12-29 06:27:06 =============================================================================== ns_1@192.168.24.61:<5883.111.192>:stats_collector:83: Stats for bucket "JDPCachePrimary": accepting_conns 1 auth_cmds 10369 auth_errors 0 bucket_active_conns 1 bucket_conns 1 bytes_read 53441490390 bytes_written 45325037881 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 29975715 cmd_set 445274018 conn_yields 61205 connection_structures 1931 curr_connections 11 curr_items 297280049 curr_items_tot 297280049 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 466 delete_misses 174808 ep_bg_fetched 1661206 ep_bg_load 4365803470 ep_bg_load_avg 2628 ep_bg_max_load 2020165 ep_bg_max_wait 2929571 ep_bg_min_load 1 ep_bg_min_wait 10 ep_bg_num_samples 1661198 ep_bg_wait 48864908595 ep_bg_wait_avg 29415 ep_commit_num 818500 ep_commit_time 0 ep_commit_time_total 179775 ep_data_age 0 ep_data_age_highwat 2571 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 1 ep_dbname /opt/membase/var/lib/membase/data/JDPCachePrimary-data/JDPCachePrimary ep_dbshards 4 ep_diskqueue_drain 358215763 ep_diskqueue_fill 358215763 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 1256 ep_flush_duration_total 251908 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 1661206 ep_io_num_write 357979281 ep_io_read_bytes 145605718 ep_io_write_bytes 6012819006520 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 356086874 ep_kv_size 57356020515 ep_latency_arith_cmd 46093 ep_latency_get_cmd 101813590 ep_latency_store_cmd 515449025 ep_max_data_size 81788928000 ep_max_txn_size 1000 ep_mem_high_wat 61341696000 ep_mem_low_wat 49073356800 ep_min_data_age 0 ep_num_active_non_resident 94237869 ep_num_checkpoint_remover_runs354634 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 490 ep_num_non_resident 94237869 ep_num_not_my_vbuckets 0 ep_num_pager_runs 2 ep_num_value_ejects 95895079 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 1611440768 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 3597 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 62797498 ep_too_young 0 ep_total_cache_size 58930442359 ep_total_del_items 416 ep_total_enqueued 358215763 ep_total_new_items 297280465 ep_total_persisted 357979697 ep_uncommitted_items 0 ep_value_size 15808691965 ep_vb_total 512 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.1.1_39_g4bd26a3 ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 8853 get_hits 7311306 get_misses 22664409 incr_hits 44431 incr_misses 1662 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 58967461283 pid 17177 pointer_size 64 rejected_conns 0 rusage_system 91968.527658 rusage_user 249502.308883 threads 4 time 1325168826 total_connections 30202 uptime 3441757 vb_active_curr_items 297280049 vb_active_eject 95895079 vb_active_ht_memory 1610809344 vb_active_itm_memory 53788643543 vb_active_num 512 vb_active_num_non_resident 94237869 vb_active_ops_create 297280465 vb_active_ops_delete 416 vb_active_ops_reject 0 vb_active_ops_update 60698816 vb_active_perc_mem_resident 68 vb_active_queue_age 0 vb_active_queue_drain 358215763 vb_active_queue_fill 358215763 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version UNKNOWN INFO REPORT <5883.1095.0> 2011-12-29 06:27:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,168866,408543}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102199512}, {processes,31542344}, {processes_used,28868072}, {system,70657168}, {atom,903241}, {atom_used,880290}, {binary,1475032}, {code,8604866}, {ets,38573952}]}, {system_stats, [{cpu_utilization_rate,0.2503128911138924}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3441795}, {memory_data,{101368545280,100269076480,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1073940 kB\nBuffers: 283632 kB\nCached: 32675392 kB\nSwapCached: 5096 kB\nActive: 82503832 kB\nInactive: 13861584 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1073940 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 120 kB\nWriteback: 20 kB\nAnonPages: 63398700 kB\nMapped: 16728 kB\nSlab: 1346348 kB\nPageTables: 171516 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86136268 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33459601408}, {buffered_memory,290439168}, {free_memory,1099714560}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3441780474,0}}, {context_switches,{1912874474,0}}, {garbage_collection,{247982026,3900860658226,0}}, {io,{{input,69548229139},{output,131946019696}}}, {reductions,{1585858595687,2471848}}, {run_queue,0}, {runtime,{130351310,170}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,168865,405769}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,84111288}, {processes,14669616}, {processes_used,14634000}, {system,69441672}, {atom,828337}, {atom_used,824728}, {binary,1356192}, {code,7914132}, {ets,38411472}]}, {system_stats, [{cpu_utilization_rate,0.37546933667083854}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3441775}, {memory_data,{101368545280,100893028352,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 516800 kB\nBuffers: 283908 kB\nCached: 31393016 kB\nSwapCached: 3836 kB\nActive: 82925588 kB\nInactive: 13854756 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 516800 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 312 kB\nWriteback: 0 kB\nAnonPages: 65095580 kB\nMapped: 17876 kB\nSlab: 1381156 kB\nPageTables: 278864 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141211340 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32146448384}, {buffered_memory,290721792}, {free_memory,529203200}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3441765642,1}}, {context_switches,{751492272,0}}, {garbage_collection,{185735625,674990282845,0}}, {io,{{input,58258778058},{output,130524334719}}}, {reductions,{324668613312,1411402}}, {run_queue,0}, {runtime,{37314680,150}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 06:28:02 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.111.192> 2011-12-29 06:28:46 =============================================================================== ns_1@192.168.24.61:<5883.111.192>:stats_collector:83: Stats for bucket "JDPCachePrimary": accepting_conns 1 auth_cmds 10369 auth_errors 0 bucket_active_conns 1 bucket_conns 1 bytes_read 53441497550 bytes_written 45326706021 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 29975715 cmd_set 445274018 conn_yields 61205 connection_structures 1931 curr_connections 11 curr_items 297280049 curr_items_tot 297280049 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 466 delete_misses 174808 ep_bg_fetched 1661206 ep_bg_load 4365803470 ep_bg_load_avg 2628 ep_bg_max_load 2020165 ep_bg_max_wait 2929571 ep_bg_min_load 1 ep_bg_min_wait 10 ep_bg_num_samples 1661198 ep_bg_wait 48864908595 ep_bg_wait_avg 29415 ep_commit_num 818500 ep_commit_time 0 ep_commit_time_total 179775 ep_data_age 0 ep_data_age_highwat 2571 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 1 ep_dbname /opt/membase/var/lib/membase/data/JDPCachePrimary-data/JDPCachePrimary ep_dbshards 4 ep_diskqueue_drain 358215763 ep_diskqueue_fill 358215763 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 1256 ep_flush_duration_total 251908 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 1661206 ep_io_num_write 357979281 ep_io_read_bytes 145605718 ep_io_write_bytes 6012819006520 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 356086874 ep_kv_size 57356020515 ep_latency_arith_cmd 46093 ep_latency_get_cmd 101813590 ep_latency_store_cmd 515449025 ep_max_data_size 81788928000 ep_max_txn_size 1000 ep_mem_high_wat 61341696000 ep_mem_low_wat 49073356800 ep_min_data_age 0 ep_num_active_non_resident 94237869 ep_num_checkpoint_remover_runs354654 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 490 ep_num_non_resident 94237869 ep_num_not_my_vbuckets 0 ep_num_pager_runs 2 ep_num_value_ejects 95895079 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 1611440768 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 3597 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 62797498 ep_too_young 0 ep_total_cache_size 58930442359 ep_total_del_items 416 ep_total_enqueued 358215763 ep_total_new_items 297280465 ep_total_persisted 357979697 ep_uncommitted_items 0 ep_value_size 15808691965 ep_vb_total 512 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.1.1_39_g4bd26a3 ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 8853 get_hits 7311306 get_misses 22664409 incr_hits 44431 incr_misses 1662 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 58967461283 pid 17177 pointer_size 64 rejected_conns 0 rusage_system 91968.559653 rusage_user 249502.402869 threads 4 time 1325168926 total_connections 30202 uptime 3441857 vb_active_curr_items 297280049 vb_active_eject 95895079 vb_active_ht_memory 1610809344 vb_active_itm_memory 53788643543 vb_active_num 512 vb_active_num_non_resident 94237869 vb_active_ops_create 297280465 vb_active_ops_delete 416 vb_active_ops_reject 0 vb_active_ops_update 60698816 vb_active_perc_mem_resident 68 vb_active_queue_age 0 vb_active_queue_drain 358215763 vb_active_queue_fill 358215763 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version UNKNOWN INFO REPORT <5883.136.0> 2011-12-29 06:28:49 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 06:28:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,168926,409041}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102663112}, {processes,31900760}, {processes_used,29226488}, {system,70762352}, {atom,903241}, {atom_used,880290}, {binary,1433880}, {code,8604866}, {ets,38720344}]}, {system_stats, [{cpu_utilization_rate,0.5012531328320802}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3441855}, {memory_data,{101368545280,100270067712,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1072980 kB\nBuffers: 283632 kB\nCached: 32676152 kB\nSwapCached: 5096 kB\nActive: 82504568 kB\nInactive: 13862256 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1072980 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 460 kB\nWriteback: 0 kB\nAnonPages: 63399292 kB\nMapped: 16728 kB\nSlab: 1346044 kB\nPageTables: 171516 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86136756 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33460379648}, {buffered_memory,290439168}, {free_memory,1098731520}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3441840474,0}}, {context_switches,{1912890766,0}}, {garbage_collection,{247985325,3900872734629,0}}, {io,{{input,69548404027},{output,131946891902}}}, {reductions,{1585870541348,2454547}}, {run_queue,0}, {runtime,{130352080,180}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,168925,404599}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,84629248}, {processes,15042208}, {processes_used,15006512}, {system,69587040}, {atom,828337}, {atom_used,824728}, {binary,1363016}, {code,7914132}, {ets,38549760}]}, {system_stats, [{cpu_utilization_rate,0.375}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3441835}, {memory_data,{101368545280,100838596608,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 516092 kB\nBuffers: 283908 kB\nCached: 31393792 kB\nSwapCached: 3836 kB\nActive: 82925964 kB\nInactive: 13855484 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 516092 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 480 kB\nWriteback: 0 kB\nAnonPages: 65096068 kB\nMapped: 17876 kB\nSlab: 1380972 kB\nPageTables: 278860 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141209492 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32147243008}, {buffered_memory,290721792}, {free_memory,528605184}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3441825641,1}}, {context_switches,{751504998,0}}, {garbage_collection,{185738683,675001368963,0}}, {io,{{input,58258973287},{output,130525161628}}}, {reductions,{324674067771,1378487}}, {run_queue,0}, {runtime,{37315280,140}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 06:29:20 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 06:29:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,168986,408522}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102745888}, {processes,31759008}, {processes_used,29084736}, {system,70986880}, {atom,903241}, {atom_used,880290}, {binary,1510568}, {code,8604866}, {ets,38868464}]}, {system_stats, [{cpu_utilization_rate,0.37453183520599254}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3441915}, {memory_data,{101368545280,100267700224,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1074676 kB\nBuffers: 283632 kB\nCached: 32676468 kB\nSwapCached: 5096 kB\nActive: 82502552 kB\nInactive: 13862624 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1074676 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 296 kB\nWriteback: 0 kB\nAnonPages: 63397080 kB\nMapped: 16728 kB\nSlab: 1346068 kB\nPageTables: 171516 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86132920 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33460703232}, {buffered_memory,290439168}, {free_memory,1100468224}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3441900474,0}}, {context_switches,{1912907760,0}}, {garbage_collection,{247988850,3900885542112,0}}, {io,{{input,69551103108},{output,131950403276}}}, {reductions,{1585882671752,2478802}}, {run_queue,0}, {runtime,{130353000,170}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,168985,405709}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,84164304}, {processes,14353928}, {processes_used,14318376}, {system,69810376}, {atom,828337}, {atom_used,824728}, {binary,1492320}, {code,7914132}, {ets,38644496}]}, {system_stats, [{cpu_utilization_rate,0.375}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3441895}, {memory_data,{101368545280,100840570880,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 514704 kB\nBuffers: 283912 kB\nCached: 31394088 kB\nSwapCached: 3836 kB\nActive: 82926584 kB\nInactive: 13855856 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 514704 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 164 kB\nWriteback: 0 kB\nAnonPages: 65096664 kB\nMapped: 17876 kB\nSlab: 1380980 kB\nPageTables: 278860 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141211888 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32147546112}, {buffered_memory,290725888}, {free_memory,527056896}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3441885642,1}}, {context_switches,{751518719,0}}, {garbage_collection,{185742173,675013305363,0}}, {io,{{input,58261654451},{output,130528688987}}}, {reductions,{324679942986,1410868}}, {run_queue,0}, {runtime,{37315990,150}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 06:30:12 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 06:30:26 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.111.192> 2011-12-29 06:30:26 =============================================================================== ns_1@192.168.24.61:<5883.111.192>:stats_collector:83: Stats for bucket "JDPCachePrimary": accepting_conns 1 auth_cmds 10369 auth_errors 0 bucket_active_conns 1 bucket_conns 1 bytes_read 53441504710 bytes_written 45328374161 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 29975715 cmd_set 445274018 conn_yields 61205 connection_structures 1931 curr_connections 11 curr_items 297280049 curr_items_tot 297280049 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 466 delete_misses 174808 ep_bg_fetched 1661206 ep_bg_load 4365803470 ep_bg_load_avg 2628 ep_bg_max_load 2020165 ep_bg_max_wait 2929571 ep_bg_min_load 1 ep_bg_min_wait 10 ep_bg_num_samples 1661198 ep_bg_wait 48864908595 ep_bg_wait_avg 29415 ep_commit_num 818500 ep_commit_time 0 ep_commit_time_total 179775 ep_data_age 0 ep_data_age_highwat 2571 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 1 ep_dbname /opt/membase/var/lib/membase/data/JDPCachePrimary-data/JDPCachePrimary ep_dbshards 4 ep_diskqueue_drain 358215763 ep_diskqueue_fill 358215763 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 1256 ep_flush_duration_total 251908 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 1661206 ep_io_num_write 357979281 ep_io_read_bytes 145605718 ep_io_write_bytes 6012819006520 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 356086874 ep_kv_size 57356020515 ep_latency_arith_cmd 46093 ep_latency_get_cmd 101813590 ep_latency_store_cmd 515449025 ep_max_data_size 81788928000 ep_max_txn_size 1000 ep_mem_high_wat 61341696000 ep_mem_low_wat 49073356800 ep_min_data_age 0 ep_num_active_non_resident 94237869 ep_num_checkpoint_remover_runs354674 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 490 ep_num_non_resident 94237869 ep_num_not_my_vbuckets 0 ep_num_pager_runs 2 ep_num_value_ejects 95895079 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 1611440768 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 3597 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 62797498 ep_too_young 0 ep_total_cache_size 58930442359 ep_total_del_items 416 ep_total_enqueued 358215763 ep_total_new_items 297280465 ep_total_persisted 357979697 ep_uncommitted_items 0 ep_value_size 15808691965 ep_vb_total 512 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.1.1_39_g4bd26a3 ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 8853 get_hits 7311306 get_misses 22664409 incr_hits 44431 incr_misses 1662 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 58967461283 pid 17177 pointer_size 64 rejected_conns 0 rusage_system 91968.584649 rusage_user 249502.503854 threads 4 time 1325169026 total_connections 30202 uptime 3441957 vb_active_curr_items 297280049 vb_active_eject 95895079 vb_active_ht_memory 1610809344 vb_active_itm_memory 53788643543 vb_active_num 512 vb_active_num_non_resident 94237869 vb_active_ops_create 297280465 vb_active_ops_delete 416 vb_active_ops_reject 0 vb_active_ops_update 60698816 vb_active_perc_mem_resident 68 vb_active_queue_age 0 vb_active_queue_drain 358215763 vb_active_queue_fill 358215763 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version UNKNOWN INFO REPORT <5883.136.0> 2011-12-29 06:30:37 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 06:30:43 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 06:30:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,169046,408651}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102083424}, {processes,31571096}, {processes_used,28896824}, {system,70512328}, {atom,903241}, {atom_used,880290}, {binary,1491056}, {code,8604866}, {ets,38413408}]}, {system_stats, [{cpu_utilization_rate,0.5}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3441975}, {memory_data,{101368545280,100269383680,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1073284 kB\nBuffers: 283632 kB\nCached: 32677232 kB\nSwapCached: 5096 kB\nActive: 82502564 kB\nInactive: 13863408 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1073284 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 48 kB\nWriteback: 0 kB\nAnonPages: 63397300 kB\nMapped: 16728 kB\nSlab: 1346028 kB\nPageTables: 171516 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86136492 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33461485568}, {buffered_memory,290439168}, {free_memory,1099042816}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3441960475,1}}, {context_switches,{1912924029,0}}, {garbage_collection,{247992141,3900897673231,0}}, {io,{{input,69551316343},{output,131951262855}}}, {reductions,{1585894602771,2439607}}, {run_queue,0}, {runtime,{130353760,170}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,169045,404738}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,84130256}, {processes,14729536}, {processes_used,14693840}, {system,69400720}, {atom,828337}, {atom_used,824728}, {binary,1485560}, {code,7914132}, {ets,38241216}]}, {system_stats, [{cpu_utilization_rate,0.75}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3441955}, {memory_data,{101368545280,100841607168,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 515280 kB\nBuffers: 283912 kB\nCached: 31394860 kB\nSwapCached: 3836 kB\nActive: 82926944 kB\nInactive: 13856536 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 515280 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 200 kB\nWriteback: 0 kB\nAnonPages: 65097104 kB\nMapped: 17876 kB\nSlab: 1380900 kB\nPageTables: 278860 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141211284 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32148336640}, {buffered_memory,290725888}, {free_memory,527646720}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3441945641,0}}, {context_switches,{751531384,0}}, {garbage_collection,{185745342,675024393704,0}}, {io,{{input,58261815161},{output,130529558155}}}, {reductions,{324685408912,1393637}}, {run_queue,0}, {runtime,{37316580,160}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 06:31:04 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 06:31:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,169106,409200}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102758408}, {processes,32139744}, {processes_used,29465472}, {system,70618664}, {atom,903241}, {atom_used,880290}, {binary,1564048}, {code,8604866}, {ets,38446408}]}, {system_stats, [{cpu_utilization_rate,14.768460575719649}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3442035}, {memory_data,{101368545280,100299296768,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1011588 kB\nBuffers: 283632 kB\nCached: 32678036 kB\nSwapCached: 5096 kB\nActive: 82561668 kB\nInactive: 13864176 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1011588 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 2684 kB\nWriteback: 140 kB\nAnonPages: 63456472 kB\nMapped: 23716 kB\nSlab: 1346492 kB\nPageTables: 172332 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86191748 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33462308864}, {buffered_memory,290439168}, {free_memory,1035866112}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3442020475,0}}, {context_switches,{1912941039,0}}, {garbage_collection,{247995670,3900909893951,0}}, {io,{{input,69551575532},{output,131952167203}}}, {reductions,{1585907133672,2462350}}, {run_queue,0}, {runtime,{130354580,170}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,169105,405759}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,83942592}, {processes,14610600}, {processes_used,14574904}, {system,69331992}, {atom,828337}, {atom_used,824728}, {binary,1383720}, {code,7914132}, {ets,38274016}]}, {system_stats, [{cpu_utilization_rate,15.0}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3442015}, {memory_data,{101368545280,100842278912,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 417908 kB\nBuffers: 283912 kB\nCached: 31395684 kB\nSwapCached: 3836 kB\nActive: 83019776 kB\nInactive: 13857268 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 417908 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 3652 kB\nWriteback: 0 kB\nAnonPages: 65189428 kB\nMapped: 25144 kB\nSlab: 1381504 kB\nPageTables: 279764 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141305384 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32149180416}, {buffered_memory,290725888}, {free_memory,427937792}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3442005642,1}}, {context_switches,{751544361,0}}, {garbage_collection,{185748761,675035711755,0}}, {io,{{input,58262009073},{output,130530506298}}}, {reductions,{324691148998,1377074}}, {run_queue,0}, {runtime,{37317200,170}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 06:31:50 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 06:31:55 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.111.192> 2011-12-29 06:32:06 =============================================================================== ns_1@192.168.24.61:<5883.111.192>:stats_collector:83: Stats for bucket "JDPCachePrimary": accepting_conns 1 auth_cmds 10369 auth_errors 0 bucket_active_conns 1 bucket_conns 1 bytes_read 53441511870 bytes_written 45330042301 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 29975715 cmd_set 445274018 conn_yields 61205 connection_structures 1931 curr_connections 11 curr_items 297280049 curr_items_tot 297280049 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 466 delete_misses 174808 ep_bg_fetched 1661206 ep_bg_load 4365803470 ep_bg_load_avg 2628 ep_bg_max_load 2020165 ep_bg_max_wait 2929571 ep_bg_min_load 1 ep_bg_min_wait 10 ep_bg_num_samples 1661198 ep_bg_wait 48864908595 ep_bg_wait_avg 29415 ep_commit_num 818500 ep_commit_time 0 ep_commit_time_total 179775 ep_data_age 0 ep_data_age_highwat 2571 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 1 ep_dbname /opt/membase/var/lib/membase/data/JDPCachePrimary-data/JDPCachePrimary ep_dbshards 4 ep_diskqueue_drain 358215763 ep_diskqueue_fill 358215763 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 1256 ep_flush_duration_total 251908 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 1661206 ep_io_num_write 357979281 ep_io_read_bytes 145605718 ep_io_write_bytes 6012819006520 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 356086874 ep_kv_size 57356020515 ep_latency_arith_cmd 46093 ep_latency_get_cmd 101813590 ep_latency_store_cmd 515449025 ep_max_data_size 81788928000 ep_max_txn_size 1000 ep_mem_high_wat 61341696000 ep_mem_low_wat 49073356800 ep_min_data_age 0 ep_num_active_non_resident 94237869 ep_num_checkpoint_remover_runs354694 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 490 ep_num_non_resident 94237869 ep_num_not_my_vbuckets 0 ep_num_pager_runs 2 ep_num_value_ejects 95895079 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 1611440768 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 3597 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 62797498 ep_too_young 0 ep_total_cache_size 58930442359 ep_total_del_items 416 ep_total_enqueued 358215763 ep_total_new_items 297280465 ep_total_persisted 357979697 ep_uncommitted_items 0 ep_value_size 15808691965 ep_vb_total 512 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.1.1_39_g4bd26a3 ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 8853 get_hits 7311306 get_misses 22664409 incr_hits 44431 incr_misses 1662 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 58967461283 pid 17177 pointer_size 64 rejected_conns 0 rusage_system 91968.866607 rusage_user 249502.606838 threads 4 time 1325169126 total_connections 30202 uptime 3442057 vb_active_curr_items 297280049 vb_active_eject 95895079 vb_active_ht_memory 1610809344 vb_active_itm_memory 53788643543 vb_active_num 512 vb_active_num_non_resident 94237869 vb_active_ops_create 297280465 vb_active_ops_delete 416 vb_active_ops_reject 0 vb_active_ops_update 60698816 vb_active_perc_mem_resident 68 vb_active_queue_age 0 vb_active_queue_drain 358215763 vb_active_queue_fill 358215763 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version UNKNOWN INFO REPORT <5883.136.0> 2011-12-29 06:32:08 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 06:32:26 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 06:32:46 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 06:32:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,169166,408742}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102578472}, {processes,31778544}, {processes_used,29104272}, {system,70799928}, {atom,903241}, {atom_used,880290}, {binary,1596760}, {code,8604866}, {ets,38595360}]}, {system_stats, [{cpu_utilization_rate,0.2503128911138924}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3442095}, {memory_data,{101368545280,100268720128,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1074496 kB\nBuffers: 283640 kB\nCached: 32673136 kB\nSwapCached: 5096 kB\nActive: 82504104 kB\nInactive: 13859216 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1074496 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 2164 kB\nWriteback: 0 kB\nAnonPages: 63398444 kB\nMapped: 16728 kB\nSlab: 1346624 kB\nPageTables: 171520 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86136940 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33457291264}, {buffered_memory,290447360}, {free_memory,1100283904}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3442080475,0}}, {context_switches,{1912958141,0}}, {garbage_collection,{247999119,3900923307764,0}}, {io,{{input,69554369363},{output,131960029694}}}, {reductions,{1585919240397,2435321}}, {run_queue,0}, {runtime,{130355610,180}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,169165,405537}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,83827480}, {processes,14360720}, {processes_used,14325024}, {system,69466760}, {atom,828337}, {atom_used,824728}, {binary,1369176}, {code,7914132}, {ets,38423424}]}, {system_stats, [{cpu_utilization_rate,0.37546933667083854}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3442075}, {memory_data,{101368545280,100953243648,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 517992 kB\nBuffers: 283928 kB\nCached: 31389992 kB\nSwapCached: 3836 kB\nActive: 82926012 kB\nInactive: 13852656 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 517992 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 1944 kB\nWriteback: 0 kB\nAnonPages: 65096732 kB\nMapped: 17876 kB\nSlab: 1381372 kB\nPageTables: 278868 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141213364 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32143351808}, {buffered_memory,290742272}, {free_memory,530423808}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3442065641,0}}, {context_switches,{751557804,0}}, {garbage_collection,{185752337,675048307307,0}}, {io,{{input,58264723085},{output,130537623563}}}, {reductions,{324696791578,1413618}}, {run_queue,0}, {runtime,{37317970,150}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 06:33:36 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.111.192> 2011-12-29 06:33:46 =============================================================================== ns_1@192.168.24.61:<5883.111.192>:stats_collector:83: Stats for bucket "JDPCachePrimary": accepting_conns 1 auth_cmds 10369 auth_errors 0 bucket_active_conns 1 bucket_conns 1 bytes_read 53441519030 bytes_written 45331710441 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 29975715 cmd_set 445274018 conn_yields 61205 connection_structures 1931 curr_connections 11 curr_items 297280049 curr_items_tot 297280049 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 466 delete_misses 174808 ep_bg_fetched 1661206 ep_bg_load 4365803470 ep_bg_load_avg 2628 ep_bg_max_load 2020165 ep_bg_max_wait 2929571 ep_bg_min_load 1 ep_bg_min_wait 10 ep_bg_num_samples 1661198 ep_bg_wait 48864908595 ep_bg_wait_avg 29415 ep_commit_num 818500 ep_commit_time 0 ep_commit_time_total 179775 ep_data_age 0 ep_data_age_highwat 2571 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 1 ep_dbname /opt/membase/var/lib/membase/data/JDPCachePrimary-data/JDPCachePrimary ep_dbshards 4 ep_diskqueue_drain 358215763 ep_diskqueue_fill 358215763 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 1256 ep_flush_duration_total 251908 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 1661206 ep_io_num_write 357979281 ep_io_read_bytes 145605718 ep_io_write_bytes 6012819006520 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 356086874 ep_kv_size 57356020515 ep_latency_arith_cmd 46093 ep_latency_get_cmd 101813590 ep_latency_store_cmd 515449025 ep_max_data_size 81788928000 ep_max_txn_size 1000 ep_mem_high_wat 61341696000 ep_mem_low_wat 49073356800 ep_min_data_age 0 ep_num_active_non_resident 94237869 ep_num_checkpoint_remover_runs354714 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 490 ep_num_non_resident 94237869 ep_num_not_my_vbuckets 0 ep_num_pager_runs 2 ep_num_value_ejects 95895079 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 1611440768 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 3597 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 62797498 ep_too_young 0 ep_total_cache_size 58930442359 ep_total_del_items 416 ep_total_enqueued 358215763 ep_total_new_items 297280465 ep_total_persisted 357979697 ep_uncommitted_items 0 ep_value_size 15808691965 ep_vb_total 512 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.1.1_39_g4bd26a3 ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 8853 get_hits 7311306 get_misses 22664409 incr_hits 44431 incr_misses 1662 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 58967461283 pid 17177 pointer_size 64 rejected_conns 0 rusage_system 91968.900601 rusage_user 249502.700824 threads 4 time 1325169226 total_connections 30202 uptime 3442157 vb_active_curr_items 297280049 vb_active_eject 95895079 vb_active_ht_memory 1610809344 vb_active_itm_memory 53788643543 vb_active_num 512 vb_active_num_non_resident 94237869 vb_active_ops_create 297280465 vb_active_ops_delete 416 vb_active_ops_reject 0 vb_active_ops_update 60698816 vb_active_perc_mem_resident 68 vb_active_queue_age 0 vb_active_queue_drain 358215763 vb_active_queue_fill 358215763 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version UNKNOWN INFO REPORT <5883.1095.0> 2011-12-29 06:33:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,169226,409110}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102870800}, {processes,32005904}, {processes_used,29331632}, {system,70864896}, {atom,903241}, {atom_used,880290}, {binary,1628016}, {code,8604866}, {ets,38628736}]}, {system_stats, [{cpu_utilization_rate,1.375}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3442155}, {memory_data,{101368545280,100267458560,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1075332 kB\nBuffers: 283640 kB\nCached: 32673896 kB\nSwapCached: 5096 kB\nActive: 82504468 kB\nInactive: 13859944 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1075332 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 144 kB\nWriteback: 0 kB\nAnonPages: 63399116 kB\nMapped: 16728 kB\nSlab: 1346100 kB\nPageTables: 171516 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86139156 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33458069504}, {buffered_memory,290447360}, {free_memory,1101139968}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3442140475,1}}, {context_switches,{1912974426,0}}, {garbage_collection,{248002305,3900935372143,0}}, {io,{{input,69554559650},{output,131961019733}}}, {reductions,{1585931267325,2494186}}, {run_queue,0}, {runtime,{130356390,190}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,169225,405883}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,84234248}, {processes,14623408}, {processes_used,14587712}, {system,69610840}, {atom,828337}, {atom_used,824728}, {binary,1475104}, {code,7914132}, {ets,38461432}]}, {system_stats, [{cpu_utilization_rate,0.37546933667083854}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3442135}, {memory_data,{101368545280,100838293504,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 516460 kB\nBuffers: 283928 kB\nCached: 31390760 kB\nSwapCached: 3836 kB\nActive: 82927104 kB\nInactive: 13853300 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 516460 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 244 kB\nWriteback: 0 kB\nAnonPages: 65098196 kB\nMapped: 17876 kB\nSlab: 1381116 kB\nPageTables: 278868 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141211440 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32144138240}, {buffered_memory,290742272}, {free_memory,528855040}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3442125640,0}}, {context_switches,{751570529,0}}, {garbage_collection,{185755713,675059399699,0}}, {io,{{input,58265023311},{output,130538478527}}}, {reductions,{324702230223,1381196}}, {run_queue,1}, {runtime,{37318580,150}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 06:34:08 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 06:34:16 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 06:34:35 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 06:34:48 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 06:34:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,169286,408616}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102662672}, {processes,31724408}, {processes_used,29050136}, {system,70938264}, {atom,903241}, {atom_used,880290}, {binary,1555088}, {code,8604866}, {ets,38775048}]}, {system_stats, [{cpu_utilization_rate,0.25}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3442215}, {memory_data,{101368545280,100266491904,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1076188 kB\nBuffers: 283640 kB\nCached: 32674652 kB\nSwapCached: 5096 kB\nActive: 82502972 kB\nInactive: 13860728 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1076188 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 528 kB\nWriteback: 0 kB\nAnonPages: 63397368 kB\nMapped: 16728 kB\nSlab: 1345932 kB\nPageTables: 171512 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86137028 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33458843648}, {buffered_memory,290447360}, {free_memory,1102016512}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3442200475,1}}, {context_switches,{1912990742,0}}, {garbage_collection,{248005648,3900947671881,0}}, {io,{{input,69554811195},{output,131961907763}}}, {reductions,{1585943213875,2444646}}, {run_queue,0}, {runtime,{130357200,190}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,169285,405620}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,83999840}, {processes,14322344}, {processes_used,14286648}, {system,69677496}, {atom,828337}, {atom_used,824728}, {binary,1395320}, {code,7914132}, {ets,38607848}]}, {system_stats, [{cpu_utilization_rate,0.375}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3442195}, {memory_data,{101368545280,100840165376,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 516160 kB\nBuffers: 283928 kB\nCached: 31391508 kB\nSwapCached: 3836 kB\nActive: 82926852 kB\nInactive: 13854120 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 516160 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 492 kB\nWriteback: 0 kB\nAnonPages: 65097400 kB\nMapped: 17876 kB\nSlab: 1380948 kB\nPageTables: 278864 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141210672 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32144904192}, {buffered_memory,290742272}, {free_memory,528547840}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3442185641,0}}, {context_switches,{751583230,0}}, {garbage_collection,{185758813,675070681832,0}}, {io,{{input,58265228012},{output,130539391686}}}, {reductions,{324707687094,1412436}}, {run_queue,0}, {runtime,{37319170,150}}]}]}] INFO REPORT <5883.111.192> 2011-12-29 06:35:26 =============================================================================== ns_1@192.168.24.61:<5883.111.192>:stats_collector:83: Stats for bucket "JDPCachePrimary": accepting_conns 1 auth_cmds 10369 auth_errors 0 bucket_active_conns 1 bucket_conns 1 bytes_read 53441526190 bytes_written 45333378581 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 29975715 cmd_set 445274018 conn_yields 61205 connection_structures 1931 curr_connections 11 curr_items 297280049 curr_items_tot 297280049 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 466 delete_misses 174808 ep_bg_fetched 1661206 ep_bg_load 4365803470 ep_bg_load_avg 2628 ep_bg_max_load 2020165 ep_bg_max_wait 2929571 ep_bg_min_load 1 ep_bg_min_wait 10 ep_bg_num_samples 1661198 ep_bg_wait 48864908595 ep_bg_wait_avg 29415 ep_commit_num 818500 ep_commit_time 0 ep_commit_time_total 179775 ep_data_age 0 ep_data_age_highwat 2571 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 1 ep_dbname /opt/membase/var/lib/membase/data/JDPCachePrimary-data/JDPCachePrimary ep_dbshards 4 ep_diskqueue_drain 358215763 ep_diskqueue_fill 358215763 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 1256 ep_flush_duration_total 251908 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 1661206 ep_io_num_write 357979281 ep_io_read_bytes 145605718 ep_io_write_bytes 6012819006520 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 356086874 ep_kv_size 57356020515 ep_latency_arith_cmd 46093 ep_latency_get_cmd 101813590 ep_latency_store_cmd 515449025 ep_max_data_size 81788928000 ep_max_txn_size 1000 ep_mem_high_wat 61341696000 ep_mem_low_wat 49073356800 ep_min_data_age 0 ep_num_active_non_resident 94237869 ep_num_checkpoint_remover_runs354734 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 490 ep_num_non_resident 94237869 ep_num_not_my_vbuckets 0 ep_num_pager_runs 2 ep_num_value_ejects 95895079 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 1611440768 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 3597 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 62797498 ep_too_young 0 ep_total_cache_size 58930442359 ep_total_del_items 416 ep_total_enqueued 358215763 ep_total_new_items 297280465 ep_total_persisted 357979697 ep_uncommitted_items 0 ep_value_size 15808691965 ep_vb_total 512 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.1.1_39_g4bd26a3 ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 8853 get_hits 7311306 get_misses 22664409 incr_hits 44431 incr_misses 1662 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 58967461283 pid 17177 pointer_size 64 rejected_conns 0 rusage_system 91968.925598 rusage_user 249502.814806 threads 4 time 1325169326 total_connections 30202 uptime 3442257 vb_active_curr_items 297280049 vb_active_eject 95895079 vb_active_ht_memory 1610809344 vb_active_itm_memory 53788643543 vb_active_num 512 vb_active_num_non_resident 94237869 vb_active_ops_create 297280465 vb_active_ops_delete 416 vb_active_ops_reject 0 vb_active_ops_update 60698816 vb_active_perc_mem_resident 68 vb_active_queue_age 0 vb_active_queue_drain 358215763 vb_active_queue_fill 358215763 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version UNKNOWN INFO REPORT <5883.136.0> 2011-12-29 06:35:26 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 06:35:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,169346,409158}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,103087680}, {processes,31857048}, {processes_used,29182776}, {system,71230632}, {atom,903241}, {atom_used,880290}, {binary,1699704}, {code,8604866}, {ets,38923160}]}, {system_stats, [{cpu_utilization_rate,0.375}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3442275}, {memory_data,{101368545280,100266934272,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1075492 kB\nBuffers: 283644 kB\nCached: 32674964 kB\nSwapCached: 5096 kB\nActive: 82503580 kB\nInactive: 13861080 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1075492 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 348 kB\nWriteback: 0 kB\nAnonPages: 63398208 kB\nMapped: 16728 kB\nSlab: 1346064 kB\nPageTables: 171512 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86135960 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33459163136}, {buffered_memory,290451456}, {free_memory,1101303808}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3442260477,1}}, {context_switches,{1913007780,0}}, {garbage_collection,{248009231,3900960349373,0}}, {io,{{input,69557552661},{output,131965444867}}}, {reductions,{1585955361053,2503077}}, {run_queue,2}, {runtime,{130358090,190}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,169345,405152}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,84434880}, {processes,14552736}, {processes_used,14517040}, {system,69882144}, {atom,828337}, {atom_used,824728}, {binary,1514120}, {code,7914132}, {ets,38694328}]}, {system_stats, [{cpu_utilization_rate,13.125}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3442255}, {memory_data,{101368545280,100840751104,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 446332 kB\nBuffers: 283928 kB\nCached: 31391848 kB\nSwapCached: 3836 kB\nActive: 82996188 kB\nInactive: 13854360 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 446332 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 3324 kB\nWriteback: 0 kB\nAnonPages: 65166668 kB\nMapped: 25144 kB\nSlab: 1381280 kB\nPageTables: 279704 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141279672 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32145252352}, {buffered_memory,290742272}, {free_memory,457170944}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3442245641,0}}, {context_switches,{751596748,0}}, {garbage_collection,{185762321,675082471062,0}}, {io,{{input,58267926652},{output,130542958808}}}, {reductions,{324713495976,1415806}}, {run_queue,4}, {runtime,{37319890,160}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 06:36:09 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 06:36:20 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 06:36:42 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 06:36:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,169406,408439}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102455616}, {processes,31739488}, {processes_used,29065216}, {system,70716128}, {atom,903241}, {atom_used,880290}, {binary,1653544}, {code,8604866}, {ets,38454752}]}, {system_stats, [{cpu_utilization_rate,7.133917396745932}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3442335}, {memory_data,{101368545280,100343005184,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 967580 kB\nBuffers: 283644 kB\nCached: 32675780 kB\nSwapCached: 5096 kB\nActive: 82608220 kB\nInactive: 13861748 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 967580 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 3244 kB\nWriteback: 0 kB\nAnonPages: 63502772 kB\nMapped: 24076 kB\nSlab: 1346960 kB\nPageTables: 172424 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86235496 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33459998720}, {buffered_memory,290451456}, {free_memory,990801920}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3442320476,1}}, {context_switches,{1913023891,0}}, {garbage_collection,{248012503,3900972443172,0}}, {io,{{input,69557774591},{output,131966283359}}}, {reductions,{1585967395025,2551368}}, {run_queue,0}, {runtime,{130358880,170}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,169405,404759}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,84554344}, {processes,15001504}, {processes_used,14965808}, {system,69552840}, {atom,828337}, {atom_used,824728}, {binary,1595328}, {code,7914132}, {ets,38283800}]}, {system_stats, [{cpu_utilization_rate,0.6257822277847309}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3442315}, {memory_data,{101368545280,100938776576,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 513732 kB\nBuffers: 283932 kB\nCached: 31392696 kB\nSwapCached: 3836 kB\nActive: 82926764 kB\nInactive: 13854984 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 513732 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 688 kB\nWriteback: 0 kB\nAnonPages: 65097072 kB\nMapped: 17876 kB\nSlab: 1381412 kB\nPageTables: 278868 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141213696 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32146120704}, {buffered_memory,290746368}, {free_memory,526061568}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3442305641,1}}, {context_switches,{751609273,0}}, {garbage_collection,{185765551,675093596703,0}}, {io,{{input,58268084938},{output,130543831479}}}, {reductions,{324718975397,1383411}}, {run_queue,2}, {runtime,{37320480,130}}]}]}] INFO REPORT <5883.111.192> 2011-12-29 06:37:06 =============================================================================== ns_1@192.168.24.61:<5883.111.192>:stats_collector:83: Stats for bucket "JDPCachePrimary": accepting_conns 1 auth_cmds 10369 auth_errors 0 bucket_active_conns 1 bucket_conns 1 bytes_read 53441533350 bytes_written 45335046721 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 29975715 cmd_set 445274018 conn_yields 61205 connection_structures 1931 curr_connections 11 curr_items 297280049 curr_items_tot 297280049 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 466 delete_misses 174808 ep_bg_fetched 1661206 ep_bg_load 4365803470 ep_bg_load_avg 2628 ep_bg_max_load 2020165 ep_bg_max_wait 2929571 ep_bg_min_load 1 ep_bg_min_wait 10 ep_bg_num_samples 1661198 ep_bg_wait 48864908595 ep_bg_wait_avg 29415 ep_commit_num 818500 ep_commit_time 0 ep_commit_time_total 179775 ep_data_age 0 ep_data_age_highwat 2571 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 1 ep_dbname /opt/membase/var/lib/membase/data/JDPCachePrimary-data/JDPCachePrimary ep_dbshards 4 ep_diskqueue_drain 358215763 ep_diskqueue_fill 358215763 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 1256 ep_flush_duration_total 251908 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 1661206 ep_io_num_write 357979281 ep_io_read_bytes 145605718 ep_io_write_bytes 6012819006520 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 356086874 ep_kv_size 57356020515 ep_latency_arith_cmd 46093 ep_latency_get_cmd 101813590 ep_latency_store_cmd 515449025 ep_max_data_size 81788928000 ep_max_txn_size 1000 ep_mem_high_wat 61341696000 ep_mem_low_wat 49073356800 ep_min_data_age 0 ep_num_active_non_resident 94237869 ep_num_checkpoint_remover_runs354754 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 490 ep_num_non_resident 94237869 ep_num_not_my_vbuckets 0 ep_num_pager_runs 2 ep_num_value_ejects 95895079 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 1611440768 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 3597 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 62797498 ep_too_young 0 ep_total_cache_size 58930442359 ep_total_del_items 416 ep_total_enqueued 358215763 ep_total_new_items 297280465 ep_total_persisted 357979697 ep_uncommitted_items 0 ep_value_size 15808691965 ep_vb_total 512 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.1.1_39_g4bd26a3 ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 8853 get_hits 7311306 get_misses 22664409 incr_hits 44431 incr_misses 1662 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 58967461283 pid 17177 pointer_size 64 rejected_conns 0 rusage_system 91968.957593 rusage_user 249502.919790 threads 4 time 1325169426 total_connections 30202 uptime 3442357 vb_active_curr_items 297280049 vb_active_eject 95895079 vb_active_ht_memory 1610809344 vb_active_itm_memory 53788643543 vb_active_num 512 vb_active_num_non_resident 94237869 vb_active_ops_create 297280465 vb_active_ops_delete 416 vb_active_ops_reject 0 vb_active_ops_update 60698816 vb_active_perc_mem_resident 68 vb_active_queue_age 0 vb_active_queue_drain 358215763 vb_active_queue_fill 358215763 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version UNKNOWN INFO REPORT <5883.136.0> 2011-12-29 06:37:25 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 06:37:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,169466,409244}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102116128}, {processes,31394128}, {processes_used,28719856}, {system,70722000}, {atom,903241}, {atom_used,880290}, {binary,1627000}, {code,8604866}, {ets,38486880}]}, {system_stats, [{cpu_utilization_rate,0.37453183520599254}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3442395}, {memory_data,{101368545280,100270608384,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1072572 kB\nBuffers: 283644 kB\nCached: 32676640 kB\nSwapCached: 5096 kB\nActive: 82504448 kB\nInactive: 13862516 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1072572 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 360 kB\nWriteback: 0 kB\nAnonPages: 63398956 kB\nMapped: 16728 kB\nSlab: 1346236 kB\nPageTables: 171512 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86137048 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33460879360}, {buffered_memory,290451456}, {free_memory,1098313728}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3442380475,0}}, {context_switches,{1913040712,0}}, {garbage_collection,{248015879,3900984620631,0}}, {io,{{input,69558004294},{output,131967262398}}}, {reductions,{1585979789945,2462843}}, {run_queue,0}, {runtime,{130359670,170}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,169465,404851}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,84251984}, {processes,14640320}, {processes_used,14604624}, {system,69611664}, {atom,828337}, {atom_used,824728}, {binary,1621128}, {code,7914132}, {ets,38316544}]}, {system_stats, [{cpu_utilization_rate,0.2506265664160401}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3442375}, {memory_data,{101368545280,100842962944,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 513016 kB\nBuffers: 283932 kB\nCached: 31393484 kB\nSwapCached: 3836 kB\nActive: 82928508 kB\nInactive: 13855724 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 513016 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 668 kB\nWriteback: 0 kB\nAnonPages: 65098684 kB\nMapped: 17876 kB\nSlab: 1381028 kB\nPageTables: 278872 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141214268 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32146927616}, {buffered_memory,290746368}, {free_memory,525328384}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3442365641,0}}, {context_switches,{751622099,0}}, {garbage_collection,{185768809,675105091149,0}}, {io,{{input,58268358226},{output,130544753042}}}, {reductions,{324724768534,1429240}}, {run_queue,0}, {runtime,{37321090,140}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 06:37:57 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 06:38:27 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.111.192> 2011-12-29 06:38:46 =============================================================================== ns_1@192.168.24.61:<5883.111.192>:stats_collector:83: Stats for bucket "JDPCachePrimary": accepting_conns 1 auth_cmds 10369 auth_errors 0 bucket_active_conns 1 bucket_conns 1 bytes_read 53441540510 bytes_written 45336714861 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 29975715 cmd_set 445274018 conn_yields 61205 connection_structures 1931 curr_connections 11 curr_items 297280049 curr_items_tot 297280049 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 466 delete_misses 174808 ep_bg_fetched 1661206 ep_bg_load 4365803470 ep_bg_load_avg 2628 ep_bg_max_load 2020165 ep_bg_max_wait 2929571 ep_bg_min_load 1 ep_bg_min_wait 10 ep_bg_num_samples 1661198 ep_bg_wait 48864908595 ep_bg_wait_avg 29415 ep_commit_num 818500 ep_commit_time 0 ep_commit_time_total 179775 ep_data_age 0 ep_data_age_highwat 2571 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 1 ep_dbname /opt/membase/var/lib/membase/data/JDPCachePrimary-data/JDPCachePrimary ep_dbshards 4 ep_diskqueue_drain 358215763 ep_diskqueue_fill 358215763 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 1256 ep_flush_duration_total 251908 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 1661206 ep_io_num_write 357979281 ep_io_read_bytes 145605718 ep_io_write_bytes 6012819006520 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 356086874 ep_kv_size 57356020515 ep_latency_arith_cmd 46093 ep_latency_get_cmd 101813590 ep_latency_store_cmd 515449025 ep_max_data_size 81788928000 ep_max_txn_size 1000 ep_mem_high_wat 61341696000 ep_mem_low_wat 49073356800 ep_min_data_age 0 ep_num_active_non_resident 94237869 ep_num_checkpoint_remover_runs354774 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 490 ep_num_non_resident 94237869 ep_num_not_my_vbuckets 0 ep_num_pager_runs 2 ep_num_value_ejects 95895079 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 1611440768 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 3597 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 62797498 ep_too_young 0 ep_total_cache_size 58930442359 ep_total_del_items 416 ep_total_enqueued 358215763 ep_total_new_items 297280465 ep_total_persisted 357979697 ep_uncommitted_items 0 ep_value_size 15808691965 ep_vb_total 512 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.1.1_39_g4bd26a3 ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 8853 get_hits 7311306 get_misses 22664409 incr_hits 44431 incr_misses 1662 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 58967461283 pid 17177 pointer_size 64 rejected_conns 0 rusage_system 91968.986588 rusage_user 249503.019775 threads 4 time 1325169525 total_connections 30202 uptime 3442456 vb_active_curr_items 297280049 vb_active_eject 95895079 vb_active_ht_memory 1610809344 vb_active_itm_memory 53788643543 vb_active_num 512 vb_active_num_non_resident 94237869 vb_active_ops_create 297280465 vb_active_ops_delete 416 vb_active_ops_reject 0 vb_active_ops_update 60698816 vb_active_perc_mem_resident 68 vb_active_queue_age 0 vb_active_queue_drain 358215763 vb_active_queue_fill 358215763 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version UNKNOWN INFO REPORT <5883.1095.0> 2011-12-29 06:38:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,169526,408650}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102923632}, {processes,32107448}, {processes_used,29433176}, {system,70816184}, {atom,903241}, {atom_used,880290}, {binary,1571752}, {code,8604866}, {ets,38636648}]}, {system_stats, [{cpu_utilization_rate,1.6229712858926342}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3442455}, {memory_data,{101368545280,100267126784,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1075364 kB\nBuffers: 283648 kB\nCached: 32676268 kB\nSwapCached: 5096 kB\nActive: 82503256 kB\nInactive: 13862124 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1075364 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 316 kB\nWriteback: 0 kB\nAnonPages: 63397612 kB\nMapped: 16728 kB\nSlab: 1346028 kB\nPageTables: 171512 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86133764 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33460498432}, {buffered_memory,290455552}, {free_memory,1101172736}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3442440477,1}}, {context_switches,{1913057521,0}}, {garbage_collection,{248019319,3900997335437,0}}, {io,{{input,69560720277},{output,131970076275}}}, {reductions,{1585991893508,2491949}}, {run_queue,0}, {runtime,{130360560,180}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,169525,404966}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,83992520}, {processes,14356728}, {processes_used,14321032}, {system,69635792}, {atom,828337}, {atom_used,824728}, {binary,1495952}, {code,7914132}, {ets,38465888}]}, {system_stats, [{cpu_utilization_rate,0.6242197253433208}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3442435}, {memory_data,{101368545280,100843200512,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 513868 kB\nBuffers: 283936 kB\nCached: 31393116 kB\nSwapCached: 3836 kB\nActive: 82928160 kB\nInactive: 13855588 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 513868 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 292 kB\nWriteback: 0 kB\nAnonPages: 65099052 kB\nMapped: 17876 kB\nSlab: 1381008 kB\nPageTables: 278872 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141213912 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32146550784}, {buffered_memory,290750464}, {free_memory,526200832}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3442425641,1}}, {context_switches,{751635497,0}}, {garbage_collection,{185772153,675116716087,0}}, {io,{{input,58271012539},{output,130547562776}}}, {reductions,{324730319533,1382792}}, {run_queue,1}, {runtime,{37321720,140}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 06:39:22 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 06:39:49 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 06:39:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,169586,409133}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102793264}, {processes,31951848}, {processes_used,29277576}, {system,70841416}, {atom,903241}, {atom_used,880290}, {binary,1564456}, {code,8604866}, {ets,38668888}]}, {system_stats, [{cpu_utilization_rate,0.2503128911138924}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3442515}, {memory_data,{101368545280,100268269568,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1074144 kB\nBuffers: 283648 kB\nCached: 32677036 kB\nSwapCached: 5096 kB\nActive: 82503856 kB\nInactive: 13862860 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1074144 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 136 kB\nWriteback: 0 kB\nAnonPages: 63398740 kB\nMapped: 16728 kB\nSlab: 1345992 kB\nPageTables: 171516 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86136924 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33461284864}, {buffered_memory,290455552}, {free_memory,1099923456}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3442500475,1}}, {context_switches,{1913073782,0}}, {garbage_collection,{248022582,3901009395644,0}}, {io,{{input,69560904974},{output,131970975019}}}, {reductions,{1586003785176,2446325}}, {run_queue,0}, {runtime,{130361350,160}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,169585,406131}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,83945592}, {processes,14231048}, {processes_used,14195352}, {system,69714544}, {atom,828337}, {atom_used,824728}, {binary,1536544}, {code,7914132}, {ets,38503592}]}, {system_stats, [{cpu_utilization_rate,0.375}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3442495}, {memory_data,{101368545280,100842348544,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 512844 kB\nBuffers: 283936 kB\nCached: 31393876 kB\nSwapCached: 3836 kB\nActive: 82927132 kB\nInactive: 13856292 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 512844 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 36 kB\nWriteback: 0 kB\nAnonPages: 65097940 kB\nMapped: 17876 kB\nSlab: 1380924 kB\nPageTables: 278868 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141211640 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32147329024}, {buffered_memory,290750464}, {free_memory,525152256}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3442485641,0}}, {context_switches,{751648111,0}}, {garbage_collection,{185775406,675127991385,0}}, {io,{{input,58271208735},{output,130548430136}}}, {reductions,{324735806410,1397862}}, {run_queue,0}, {runtime,{37322340,180}}]}]}] INFO REPORT <5883.111.192> 2011-12-29 06:40:26 =============================================================================== ns_1@192.168.24.61:<5883.111.192>:stats_collector:83: Stats for bucket "JDPCachePrimary": accepting_conns 1 auth_cmds 10369 auth_errors 0 bucket_active_conns 1 bucket_conns 1 bytes_read 53441547670 bytes_written 45338383001 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 29975715 cmd_set 445274018 conn_yields 61205 connection_structures 1931 curr_connections 11 curr_items 297280049 curr_items_tot 297280049 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 466 delete_misses 174808 ep_bg_fetched 1661206 ep_bg_load 4365803470 ep_bg_load_avg 2628 ep_bg_max_load 2020165 ep_bg_max_wait 2929571 ep_bg_min_load 1 ep_bg_min_wait 10 ep_bg_num_samples 1661198 ep_bg_wait 48864908595 ep_bg_wait_avg 29415 ep_commit_num 818500 ep_commit_time 0 ep_commit_time_total 179775 ep_data_age 0 ep_data_age_highwat 2571 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 1 ep_dbname /opt/membase/var/lib/membase/data/JDPCachePrimary-data/JDPCachePrimary ep_dbshards 4 ep_diskqueue_drain 358215763 ep_diskqueue_fill 358215763 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 1256 ep_flush_duration_total 251908 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 1661206 ep_io_num_write 357979281 ep_io_read_bytes 145605718 ep_io_write_bytes 6012819006520 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 356086874 ep_kv_size 57356020515 ep_latency_arith_cmd 46093 ep_latency_get_cmd 101813590 ep_latency_store_cmd 515449025 ep_max_data_size 81788928000 ep_max_txn_size 1000 ep_mem_high_wat 61341696000 ep_mem_low_wat 49073356800 ep_min_data_age 0 ep_num_active_non_resident 94237869 ep_num_checkpoint_remover_runs354794 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 490 ep_num_non_resident 94237869 ep_num_not_my_vbuckets 0 ep_num_pager_runs 2 ep_num_value_ejects 95895079 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 1611440768 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 3597 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 62797498 ep_too_young 0 ep_total_cache_size 58930442359 ep_total_del_items 416 ep_total_enqueued 358215763 ep_total_new_items 297280465 ep_total_persisted 357979697 ep_uncommitted_items 0 ep_value_size 15808691965 ep_vb_total 512 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.1.1_39_g4bd26a3 ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 8853 get_hits 7311306 get_misses 22664409 incr_hits 44431 incr_misses 1662 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 58967461283 pid 17177 pointer_size 64 rejected_conns 0 rusage_system 91969.031582 rusage_user 249503.106762 threads 4 time 1325169625 total_connections 30202 uptime 3442556 vb_active_curr_items 297280049 vb_active_eject 95895079 vb_active_ht_memory 1610809344 vb_active_itm_memory 53788643543 vb_active_num 512 vb_active_num_non_resident 94237869 vb_active_ops_create 297280465 vb_active_ops_delete 416 vb_active_ops_reject 0 vb_active_ops_update 60698816 vb_active_perc_mem_resident 68 vb_active_queue_age 0 vb_active_queue_drain 358215763 vb_active_queue_fill 358215763 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version UNKNOWN INFO REPORT <5883.136.0> 2011-12-29 06:40:28 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 06:40:44 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 06:40:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,169646,408585}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102898416}, {processes,31779728}, {processes_used,29105456}, {system,71118688}, {atom,903241}, {atom_used,880290}, {binary,1687608}, {code,8604866}, {ets,38823088}]}, {system_stats, [{cpu_utilization_rate,0.2503128911138924}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3442575}, {memory_data,{101368545280,100270059520,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1073012 kB\nBuffers: 283648 kB\nCached: 32677812 kB\nSwapCached: 5096 kB\nActive: 82503888 kB\nInactive: 13863592 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1073012 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 424 kB\nWriteback: 0 kB\nAnonPages: 63397908 kB\nMapped: 16728 kB\nSlab: 1345976 kB\nPageTables: 171516 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86135984 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33462079488}, {buffered_memory,290455552}, {free_memory,1098764288}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3442560475,1}}, {context_switches,{1913090056,0}}, {garbage_collection,{248025662,3901021332357,0}}, {io,{{input,69561124878},{output,131971856943}}}, {reductions,{1586015722936,2436637}}, {run_queue,0}, {runtime,{130362130,170}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,169645,405142}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,84439992}, {processes,14599040}, {processes_used,14563344}, {system,69840952}, {atom,828337}, {atom_used,824728}, {binary,1515520}, {code,7914132}, {ets,38651144}]}, {system_stats, [{cpu_utilization_rate,12.875}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3442555}, {memory_data,{101368545280,100843294720,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 480184 kB\nBuffers: 283936 kB\nCached: 31394652 kB\nSwapCached: 3836 kB\nActive: 82958628 kB\nInactive: 13857032 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 480184 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 3316 kB\nWriteback: 0 kB\nAnonPages: 65128936 kB\nMapped: 24644 kB\nSlab: 1381152 kB\nPageTables: 279780 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141242704 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32148123648}, {buffered_memory,290750464}, {free_memory,491708416}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3442545641,1}}, {context_switches,{751660508,0}}, {garbage_collection,{185778580,675138999762,0}}, {io,{{input,58271405543},{output,130549305756}}}, {reductions,{324741241643,1402111}}, {run_queue,0}, {runtime,{37322940,160}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 06:41:30 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 06:41:35 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 06:41:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,169706,409348}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,103114848}, {processes,31908640}, {processes_used,29234368}, {system,71206208}, {atom,903241}, {atom_used,880290}, {binary,1627200}, {code,8604866}, {ets,38971648}]}, {system_stats, [{cpu_utilization_rate,12.76595744680851}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3442635}, {memory_data,{101368545280,100347138048,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1044936 kB\nBuffers: 283664 kB\nCached: 32674276 kB\nSwapCached: 5096 kB\nActive: 82532296 kB\nInactive: 13860144 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1044936 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 3572 kB\nWriteback: 0 kB\nAnonPages: 63427356 kB\nMapped: 21004 kB\nSlab: 1346324 kB\nPageTables: 171672 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86161640 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33458458624}, {buffered_memory,290471936}, {free_memory,1070014464}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3442620476,1}}, {context_switches,{1913107062,0}}, {garbage_collection,{248029085,3901034720225,0}}, {io,{{input,69563827492},{output,131979845658}}}, {reductions,{1586027836007,2464298}}, {run_queue,0}, {runtime,{130363110,180}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,169705,405935}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,84215080}, {processes,14205560}, {processes_used,14170008}, {system,70009520}, {atom,828337}, {atom_used,824728}, {binary,1588688}, {code,7914132}, {ets,38747344}]}, {system_stats, [{cpu_utilization_rate,0.37546933667083854}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3442615}, {memory_data,{101368545280,100885008384,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 512644 kB\nBuffers: 283936 kB\nCached: 31391228 kB\nSwapCached: 3836 kB\nActive: 82927824 kB\nInactive: 13853556 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 512644 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 3464 kB\nWriteback: 0 kB\nAnonPages: 65098268 kB\nMapped: 17876 kB\nSlab: 1381836 kB\nPageTables: 278872 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141213276 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32144617472}, {buffered_memory,290750464}, {free_memory,524947456}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3442605640,0}}, {context_switches,{751673956,0}}, {garbage_collection,{185782048,675151599838,0}}, {io,{{input,58274059139},{output,130557366378}}}, {reductions,{324747101324,1386887}}, {run_queue,0}, {runtime,{37323730,170}}]}]}] INFO REPORT <5883.111.192> 2011-12-29 06:42:06 =============================================================================== ns_1@192.168.24.61:<5883.111.192>:stats_collector:83: Stats for bucket "JDPCachePrimary": accepting_conns 1 auth_cmds 10369 auth_errors 0 bucket_active_conns 1 bucket_conns 1 bytes_read 53441554830 bytes_written 45340051141 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 29975715 cmd_set 445274018 conn_yields 61205 connection_structures 1931 curr_connections 11 curr_items 297280049 curr_items_tot 297280049 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 466 delete_misses 174808 ep_bg_fetched 1661206 ep_bg_load 4365803470 ep_bg_load_avg 2628 ep_bg_max_load 2020165 ep_bg_max_wait 2929571 ep_bg_min_load 1 ep_bg_min_wait 10 ep_bg_num_samples 1661198 ep_bg_wait 48864908595 ep_bg_wait_avg 29415 ep_commit_num 818500 ep_commit_time 0 ep_commit_time_total 179775 ep_data_age 0 ep_data_age_highwat 2571 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 1 ep_dbname /opt/membase/var/lib/membase/data/JDPCachePrimary-data/JDPCachePrimary ep_dbshards 4 ep_diskqueue_drain 358215763 ep_diskqueue_fill 358215763 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 1256 ep_flush_duration_total 251908 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 1661206 ep_io_num_write 357979281 ep_io_read_bytes 145605718 ep_io_write_bytes 6012819006520 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 356086874 ep_kv_size 57356020515 ep_latency_arith_cmd 46093 ep_latency_get_cmd 101813590 ep_latency_store_cmd 515449025 ep_max_data_size 81788928000 ep_max_txn_size 1000 ep_mem_high_wat 61341696000 ep_mem_low_wat 49073356800 ep_min_data_age 0 ep_num_active_non_resident 94237869 ep_num_checkpoint_remover_runs354814 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 490 ep_num_non_resident 94237869 ep_num_not_my_vbuckets 0 ep_num_pager_runs 2 ep_num_value_ejects 95895079 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 1611440768 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 3597 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 62797498 ep_too_young 0 ep_total_cache_size 58930442359 ep_total_del_items 416 ep_total_enqueued 358215763 ep_total_new_items 297280465 ep_total_persisted 357979697 ep_uncommitted_items 0 ep_value_size 15808691965 ep_vb_total 512 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.1.1_39_g4bd26a3 ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 8853 get_hits 7311306 get_misses 22664409 incr_hits 44431 incr_misses 1662 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 58967461283 pid 17177 pointer_size 64 rejected_conns 0 rusage_system 91969.073575 rusage_user 249503.193749 threads 4 time 1325169725 total_connections 30202 uptime 3442656 vb_active_curr_items 297280049 vb_active_eject 95895079 vb_active_ht_memory 1610809344 vb_active_itm_memory 53788643543 vb_active_num 512 vb_active_num_non_resident 94237869 vb_active_ops_create 297280465 vb_active_ops_delete 416 vb_active_ops_reject 0 vb_active_ops_update 60698816 vb_active_perc_mem_resident 68 vb_active_queue_age 0 vb_active_queue_drain 358215763 vb_active_queue_fill 358215763 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version UNKNOWN INFO REPORT <5883.136.0> 2011-12-29 06:42:23 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 06:42:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,169766,409441}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102302168}, {processes,31596648}, {processes_used,28922376}, {system,70705520}, {atom,903241}, {atom_used,880290}, {binary,1595032}, {code,8604866}, {ets,38502792}]}, {system_stats, [{cpu_utilization_rate,0.625}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3442695}, {memory_data,{101368545280,100269797376,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1074084 kB\nBuffers: 283672 kB\nCached: 32674128 kB\nSwapCached: 5096 kB\nActive: 82503684 kB\nInactive: 13860756 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1074084 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 1308 kB\nWriteback: 0 kB\nAnonPages: 63398580 kB\nMapped: 16728 kB\nSlab: 1346440 kB\nPageTables: 171512 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86138232 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33458307072}, {buffered_memory,290480128}, {free_memory,1099862016}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3442680476,1}}, {context_switches,{1913123324,0}}, {garbage_collection,{248032355,3901046875414,0}}, {io,{{input,69564050588},{output,131980791047}}}, {reductions,{1586039821203,2489563}}, {run_queue,0}, {runtime,{130363890,160}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,169765,404744}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,83943768}, {processes,14396440}, {processes_used,14360744}, {system,69547328}, {atom,828337}, {atom_used,824728}, {binary,1541488}, {code,7914132}, {ets,38332208}]}, {system_stats, [{cpu_utilization_rate,0.625}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3442675}, {memory_data,{101368545280,100840108032,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 514704 kB\nBuffers: 283936 kB\nCached: 31391996 kB\nSwapCached: 3836 kB\nActive: 82927576 kB\nInactive: 13854260 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 514704 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 108 kB\nWriteback: 0 kB\nAnonPages: 65098144 kB\nMapped: 17876 kB\nSlab: 1381080 kB\nPageTables: 278868 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141215608 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32145403904}, {buffered_memory,290750464}, {free_memory,527183872}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3442665641,1}}, {context_switches,{751686596,0}}, {garbage_collection,{185785266,675162977769,0}}, {io,{{input,58274320055},{output,130558226097}}}, {reductions,{324752616244,1408364}}, {run_queue,0}, {runtime,{37324330,150}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 06:43:21 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.111.192> 2011-12-29 06:43:46 =============================================================================== ns_1@192.168.24.61:<5883.111.192>:stats_collector:83: Stats for bucket "JDPCachePrimary": accepting_conns 1 auth_cmds 10369 auth_errors 0 bucket_active_conns 1 bucket_conns 1 bytes_read 53441561990 bytes_written 45341719281 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 29975715 cmd_set 445274018 conn_yields 61205 connection_structures 1931 curr_connections 11 curr_items 297280049 curr_items_tot 297280049 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 466 delete_misses 174808 ep_bg_fetched 1661206 ep_bg_load 4365803470 ep_bg_load_avg 2628 ep_bg_max_load 2020165 ep_bg_max_wait 2929571 ep_bg_min_load 1 ep_bg_min_wait 10 ep_bg_num_samples 1661198 ep_bg_wait 48864908595 ep_bg_wait_avg 29415 ep_commit_num 818500 ep_commit_time 0 ep_commit_time_total 179775 ep_data_age 0 ep_data_age_highwat 2571 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 1 ep_dbname /opt/membase/var/lib/membase/data/JDPCachePrimary-data/JDPCachePrimary ep_dbshards 4 ep_diskqueue_drain 358215763 ep_diskqueue_fill 358215763 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 1256 ep_flush_duration_total 251908 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 1661206 ep_io_num_write 357979281 ep_io_read_bytes 145605718 ep_io_write_bytes 6012819006520 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 356086874 ep_kv_size 57356020515 ep_latency_arith_cmd 46093 ep_latency_get_cmd 101813590 ep_latency_store_cmd 515449025 ep_max_data_size 81788928000 ep_max_txn_size 1000 ep_mem_high_wat 61341696000 ep_mem_low_wat 49073356800 ep_min_data_age 0 ep_num_active_non_resident 94237869 ep_num_checkpoint_remover_runs354834 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 490 ep_num_non_resident 94237869 ep_num_not_my_vbuckets 0 ep_num_pager_runs 2 ep_num_value_ejects 95895079 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 1611440768 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 3597 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 62797498 ep_too_young 0 ep_total_cache_size 58930442359 ep_total_del_items 416 ep_total_enqueued 358215763 ep_total_new_items 297280465 ep_total_persisted 357979697 ep_uncommitted_items 0 ep_value_size 15808691965 ep_vb_total 512 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.1.1_39_g4bd26a3 ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 8853 get_hits 7311306 get_misses 22664409 incr_hits 44431 incr_misses 1662 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 58967461283 pid 17177 pointer_size 64 rejected_conns 0 rusage_system 91969.103571 rusage_user 249503.288734 threads 4 time 1325169825 total_connections 30202 uptime 3442756 vb_active_curr_items 297280049 vb_active_eject 95895079 vb_active_ht_memory 1610809344 vb_active_itm_memory 53788643543 vb_active_num 512 vb_active_num_non_resident 94237869 vb_active_ops_create 297280465 vb_active_ops_delete 416 vb_active_ops_reject 0 vb_active_ops_update 60698816 vb_active_perc_mem_resident 68 vb_active_queue_age 0 vb_active_queue_drain 358215763 vb_active_queue_fill 358215763 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version UNKNOWN INFO REPORT <5883.1095.0> 2011-12-29 06:43:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,169826,409062}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102753376}, {processes,31893952}, {processes_used,29219680}, {system,70859424}, {atom,903241}, {atom_used,880290}, {binary,1714888}, {code,8604866}, {ets,38536448}]}, {system_stats, [{cpu_utilization_rate,1.3784461152882206}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3442755}, {memory_data,{101368545280,100267569152,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1075192 kB\nBuffers: 283672 kB\nCached: 32674908 kB\nSwapCached: 5096 kB\nActive: 82502712 kB\nInactive: 13861524 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1075192 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 452 kB\nWriteback: 0 kB\nAnonPages: 63397836 kB\nMapped: 16728 kB\nSlab: 1346164 kB\nPageTables: 171512 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86136328 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33459105792}, {buffered_memory,290480128}, {free_memory,1100996608}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3442740475,1}}, {context_switches,{1913140192,0}}, {garbage_collection,{248035650,3901059041300,0}}, {io,{{input,69564228360},{output,131981659235}}}, {reductions,{1586052242653,2423981}}, {run_queue,0}, {runtime,{130364690,170}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,169825,405592}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,84183888}, {processes,14615952}, {processes_used,14580256}, {system,69567936}, {atom,828337}, {atom_used,824728}, {binary,1530064}, {code,7914132}, {ets,38364032}]}, {system_stats, [{cpu_utilization_rate,0.5}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3442735}, {memory_data,{101368545280,100840493056,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 514084 kB\nBuffers: 283936 kB\nCached: 31392784 kB\nSwapCached: 3836 kB\nActive: 82928044 kB\nInactive: 13854988 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 514084 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 528 kB\nWriteback: 0 kB\nAnonPages: 65098208 kB\nMapped: 17876 kB\nSlab: 1381008 kB\nPageTables: 278868 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141213244 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32146210816}, {buffered_memory,290750464}, {free_memory,526422016}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3442725641,0}}, {context_switches,{751699514,0}}, {garbage_collection,{185788412,675174106474,0}}, {io,{{input,58274479597},{output,130559081155}}}, {reductions,{324758297470,1380373}}, {run_queue,0}, {runtime,{37324950,150}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 06:43:57 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 06:44:18 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 06:44:46 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 06:44:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,169886,408423}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102536056}, {processes,31787592}, {processes_used,29113320}, {system,70748464}, {atom,903241}, {atom_used,880290}, {binary,1456040}, {code,8604866}, {ets,38685040}]}, {system_stats, [{cpu_utilization_rate,0.6242197253433208}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3442815}, {memory_data,{101368545280,100266799104,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1076168 kB\nBuffers: 283672 kB\nCached: 32674540 kB\nSwapCached: 5096 kB\nActive: 82502280 kB\nInactive: 13861188 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1076168 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 340 kB\nWriteback: 0 kB\nAnonPages: 63397648 kB\nMapped: 16728 kB\nSlab: 1346132 kB\nPageTables: 171512 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86133340 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33458728960}, {buffered_memory,290480128}, {free_memory,1101996032}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3442800476,1}}, {context_switches,{1913157748,0}}, {garbage_collection,{248039051,3901071778042,0}}, {io,{{input,69566941234},{output,131984456168}}}, {reductions,{1586064374033,2450242}}, {run_queue,0}, {runtime,{130365530,160}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,169885,405779}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,84013728}, {processes,14395320}, {processes_used,14359768}, {system,69618408}, {atom,828337}, {atom_used,824728}, {binary,1427720}, {code,7914132}, {ets,38517768}]}, {system_stats, [{cpu_utilization_rate,0.5}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3442795}, {memory_data,{101368545280,100842381312,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 513768 kB\nBuffers: 283936 kB\nCached: 31392420 kB\nSwapCached: 3836 kB\nActive: 82928212 kB\nInactive: 13854740 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 513768 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 328 kB\nWriteback: 0 kB\nAnonPages: 65098452 kB\nMapped: 17876 kB\nSlab: 1380960 kB\nPageTables: 278868 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141216668 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32145838080}, {buffered_memory,290750464}, {free_memory,526098432}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3442785642,1}}, {context_switches,{751713063,0}}, {garbage_collection,{185791819,675185803090,0}}, {io,{{input,58277094973},{output,130561903199}}}, {reductions,{324763868516,1412349}}, {run_queue,0}, {runtime,{37325590,130}}]}]}] INFO REPORT <5883.111.192> 2011-12-29 06:45:26 =============================================================================== ns_1@192.168.24.61:<5883.111.192>:stats_collector:83: Stats for bucket "JDPCachePrimary": accepting_conns 1 auth_cmds 10369 auth_errors 0 bucket_active_conns 1 bucket_conns 1 bytes_read 53441569150 bytes_written 45343387421 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 29975715 cmd_set 445274018 conn_yields 61205 connection_structures 1931 curr_connections 11 curr_items 297280049 curr_items_tot 297280049 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 466 delete_misses 174808 ep_bg_fetched 1661206 ep_bg_load 4365803470 ep_bg_load_avg 2628 ep_bg_max_load 2020165 ep_bg_max_wait 2929571 ep_bg_min_load 1 ep_bg_min_wait 10 ep_bg_num_samples 1661198 ep_bg_wait 48864908595 ep_bg_wait_avg 29415 ep_commit_num 818500 ep_commit_time 0 ep_commit_time_total 179775 ep_data_age 0 ep_data_age_highwat 2571 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 1 ep_dbname /opt/membase/var/lib/membase/data/JDPCachePrimary-data/JDPCachePrimary ep_dbshards 4 ep_diskqueue_drain 358215763 ep_diskqueue_fill 358215763 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 1256 ep_flush_duration_total 251908 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 1661206 ep_io_num_write 357979281 ep_io_read_bytes 145605718 ep_io_write_bytes 6012819006520 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 356086874 ep_kv_size 57356020515 ep_latency_arith_cmd 46093 ep_latency_get_cmd 101813590 ep_latency_store_cmd 515449025 ep_max_data_size 81788928000 ep_max_txn_size 1000 ep_mem_high_wat 61341696000 ep_mem_low_wat 49073356800 ep_min_data_age 0 ep_num_active_non_resident 94237869 ep_num_checkpoint_remover_runs354854 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 490 ep_num_non_resident 94237869 ep_num_not_my_vbuckets 0 ep_num_pager_runs 2 ep_num_value_ejects 95895079 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 1611440768 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 3597 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 62797498 ep_too_young 0 ep_total_cache_size 58930442359 ep_total_del_items 416 ep_total_enqueued 358215763 ep_total_new_items 297280465 ep_total_persisted 357979697 ep_uncommitted_items 0 ep_value_size 15808691965 ep_vb_total 512 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.1.1_39_g4bd26a3 ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 8853 get_hits 7311306 get_misses 22664409 incr_hits 44431 incr_misses 1662 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 58967461283 pid 17177 pointer_size 64 rejected_conns 0 rusage_system 91969.138565 rusage_user 249503.385719 threads 4 time 1325169925 total_connections 30202 uptime 3442856 vb_active_curr_items 297280049 vb_active_eject 95895079 vb_active_ht_memory 1610809344 vb_active_itm_memory 53788643543 vb_active_num 512 vb_active_num_non_resident 94237869 vb_active_ops_create 297280465 vb_active_ops_delete 416 vb_active_ops_reject 0 vb_active_ops_update 60698816 vb_active_perc_mem_resident 68 vb_active_queue_age 0 vb_active_queue_drain 358215763 vb_active_queue_fill 358215763 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version UNKNOWN INFO REPORT <5883.136.0> 2011-12-29 06:45:38 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 06:45:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,169946,408767}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102620960}, {processes,31951832}, {processes_used,29277560}, {system,70669128}, {atom,903241}, {atom_used,880290}, {binary,1343424}, {code,8604866}, {ets,38718000}]}, {system_stats, [{cpu_utilization_rate,13.25}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3442875}, {memory_data,{101368545280,100267532288,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1074972 kB\nBuffers: 283672 kB\nCached: 32675316 kB\nSwapCached: 5096 kB\nActive: 82502892 kB\nInactive: 13861828 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1074972 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 180 kB\nWriteback: 0 kB\nAnonPages: 63398140 kB\nMapped: 16728 kB\nSlab: 1346088 kB\nPageTables: 171512 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86132972 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33459523584}, {buffered_memory,290480128}, {free_memory,1100771328}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3442860475,1}}, {context_switches,{1913174021,0}}, {garbage_collection,{248042286,3901083778127,0}}, {io,{{input,69567130593},{output,131985344143}}}, {reductions,{1586076419131,2516550}}, {run_queue,0}, {runtime,{130366320,140}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,169945,404941}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,84182128}, {processes,14506296}, {processes_used,14470600}, {system,69675832}, {atom,828337}, {atom_used,824728}, {binary,1455472}, {code,7914132}, {ets,38546928}]}, {system_stats, [{cpu_utilization_rate,12.76595744680851}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3442855}, {memory_data,{101368545280,100842471424,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 511700 kB\nBuffers: 283936 kB\nCached: 31393204 kB\nSwapCached: 3836 kB\nActive: 82929528 kB\nInactive: 13855472 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 511700 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 172 kB\nWriteback: 0 kB\nAnonPages: 65099384 kB\nMapped: 18844 kB\nSlab: 1380940 kB\nPageTables: 278864 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141216420 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32146640896}, {buffered_memory,290750464}, {free_memory,523980800}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3442845641,1}}, {context_switches,{751725553,0}}, {garbage_collection,{185794902,675196881729,0}}, {io,{{input,58277289083},{output,130562762848}}}, {reductions,{324769317364,1414070}}, {run_queue,1}, {runtime,{37326180,140}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 06:46:11 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 06:46:25 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 06:46:43 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 06:46:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,170006,408572}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102967416}, {processes,32008720}, {processes_used,29334448}, {system,70958696}, {atom,903241}, {atom_used,880290}, {binary,1485560}, {code,8604866}, {ets,38865064}]}, {system_stats, [{cpu_utilization_rate,0.5}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3442935}, {memory_data,{101368545280,100378537984,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1021892 kB\nBuffers: 283672 kB\nCached: 32676128 kB\nSwapCached: 5096 kB\nActive: 82551180 kB\nInactive: 13862456 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1021892 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 3392 kB\nWriteback: 0 kB\nAnonPages: 63446352 kB\nMapped: 18720 kB\nSlab: 1346864 kB\nPageTables: 172168 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86231988 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33460355072}, {buffered_memory,290480128}, {free_memory,1046417408}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3442920477,0}}, {context_switches,{1913190272,0}}, {garbage_collection,{248045537,3901095790766,0}}, {io,{{input,69567385093},{output,131986217009}}}, {reductions,{1586088296212,2458366}}, {run_queue,0}, {runtime,{130367150,200}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,170005,405671}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,84293160}, {processes,14572288}, {processes_used,14536880}, {system,69720872}, {atom,828337}, {atom_used,824728}, {binary,1349072}, {code,7914132}, {ets,38698712}]}, {system_stats, [{cpu_utilization_rate,0.5006257822277848}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3442915}, {memory_data,{101368545280,100872290304,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 508580 kB\nBuffers: 283936 kB\nCached: 31394056 kB\nSwapCached: 3836 kB\nActive: 82928004 kB\nInactive: 13856088 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 508580 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 4424 kB\nWriteback: 0 kB\nAnonPages: 65098848 kB\nMapped: 17572 kB\nSlab: 1382112 kB\nPageTables: 278692 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141216572 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32147513344}, {buffered_memory,290750464}, {free_memory,520785920}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3442905641,0}}, {context_switches,{751738102,0}}, {garbage_collection,{185798004,675208061373,0}}, {io,{{input,58277482168},{output,130563677278}}}, {reductions,{324774775383,1380720}}, {run_queue,0}, {runtime,{37326750,130}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 06:46:55 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.111.192> 2011-12-29 06:47:06 =============================================================================== ns_1@192.168.24.61:<5883.111.192>:stats_collector:83: Stats for bucket "JDPCachePrimary": accepting_conns 1 auth_cmds 10369 auth_errors 0 bucket_active_conns 1 bucket_conns 1 bytes_read 53441576310 bytes_written 45345055561 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 29975715 cmd_set 445274018 conn_yields 61205 connection_structures 1931 curr_connections 11 curr_items 297280049 curr_items_tot 297280049 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 466 delete_misses 174808 ep_bg_fetched 1661206 ep_bg_load 4365803470 ep_bg_load_avg 2628 ep_bg_max_load 2020165 ep_bg_max_wait 2929571 ep_bg_min_load 1 ep_bg_min_wait 10 ep_bg_num_samples 1661198 ep_bg_wait 48864908595 ep_bg_wait_avg 29415 ep_commit_num 818500 ep_commit_time 0 ep_commit_time_total 179775 ep_data_age 0 ep_data_age_highwat 2571 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 1 ep_dbname /opt/membase/var/lib/membase/data/JDPCachePrimary-data/JDPCachePrimary ep_dbshards 4 ep_diskqueue_drain 358215763 ep_diskqueue_fill 358215763 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 1256 ep_flush_duration_total 251908 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 1661206 ep_io_num_write 357979281 ep_io_read_bytes 145605718 ep_io_write_bytes 6012819006520 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 356086874 ep_kv_size 57356020515 ep_latency_arith_cmd 46093 ep_latency_get_cmd 101813590 ep_latency_store_cmd 515449025 ep_max_data_size 81788928000 ep_max_txn_size 1000 ep_mem_high_wat 61341696000 ep_mem_low_wat 49073356800 ep_min_data_age 0 ep_num_active_non_resident 94237869 ep_num_checkpoint_remover_runs354874 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 490 ep_num_non_resident 94237869 ep_num_not_my_vbuckets 0 ep_num_pager_runs 2 ep_num_value_ejects 95895079 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 1611440768 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 3597 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 62797498 ep_too_young 0 ep_total_cache_size 58930442359 ep_total_del_items 416 ep_total_enqueued 358215763 ep_total_new_items 297280465 ep_total_persisted 357979697 ep_uncommitted_items 0 ep_value_size 15808691965 ep_vb_total 512 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.1.1_39_g4bd26a3 ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 8853 get_hits 7311306 get_misses 22664409 incr_hits 44431 incr_misses 1662 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 58967461283 pid 17177 pointer_size 64 rejected_conns 0 rusage_system 91969.169561 rusage_user 249503.491703 threads 4 time 1325170025 total_connections 30202 uptime 3442956 vb_active_curr_items 297280049 vb_active_eject 95895079 vb_active_ht_memory 1610809344 vb_active_itm_memory 53788643543 vb_active_num 512 vb_active_num_non_resident 94237869 vb_active_ops_create 297280465 vb_active_ops_delete 416 vb_active_ops_reject 0 vb_active_ops_update 60698816 vb_active_perc_mem_resident 68 vb_active_queue_age 0 vb_active_queue_drain 358215763 vb_active_queue_fill 358215763 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version UNKNOWN INFO REPORT <5883.1095.0> 2011-12-29 06:47:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,170066,408376}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102654304}, {processes,31599664}, {processes_used,28925392}, {system,71054640}, {atom,903241}, {atom_used,880290}, {binary,1434504}, {code,8604866}, {ets,39013032}]}, {system_stats, [{cpu_utilization_rate,0.625}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3442995}, {memory_data,{101368545280,100269568000,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1072728 kB\nBuffers: 283676 kB\nCached: 32676484 kB\nSwapCached: 5096 kB\nActive: 82502468 kB\nInactive: 13862984 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1072728 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 360 kB\nWriteback: 0 kB\nAnonPages: 63397480 kB\nMapped: 16728 kB\nSlab: 1346272 kB\nPageTables: 171508 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86135984 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33460719616}, {buffered_memory,290484224}, {free_memory,1098473472}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3442980476,1}}, {context_switches,{1913207486,0}}, {garbage_collection,{248048846,3901108530052,0}}, {io,{{input,69570076518},{output,131989668810}}}, {reductions,{1586100386384,2432309}}, {run_queue,1}, {runtime,{130368000,150}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,170065,404877}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,84814856}, {processes,14894912}, {processes_used,14859216}, {system,69919944}, {atom,828337}, {atom_used,824728}, {binary,1462360}, {code,7914132}, {ets,38784432}]}, {system_stats, [{cpu_utilization_rate,0.37453183520599254}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3442975}, {memory_data,{101368545280,100845887488,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 513052 kB\nBuffers: 283936 kB\nCached: 31394368 kB\nSwapCached: 3836 kB\nActive: 82927428 kB\nInactive: 13856500 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 513052 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 436 kB\nWriteback: 0 kB\nAnonPages: 65097468 kB\nMapped: 17876 kB\nSlab: 1381120 kB\nPageTables: 278868 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141218660 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32147832832}, {buffered_memory,290750464}, {free_memory,525365248}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3442965641,0}}, {context_switches,{751751626,0}}, {garbage_collection,{185801562,675220008414,0}}, {io,{{input,58280133979},{output,130567185089}}}, {reductions,{324780637093,1424616}}, {run_queue,0}, {runtime,{37327480,160}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 06:47:51 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 06:48:29 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.111.192> 2011-12-29 06:48:46 =============================================================================== ns_1@192.168.24.61:<5883.111.192>:stats_collector:83: Stats for bucket "JDPCachePrimary": accepting_conns 1 auth_cmds 10369 auth_errors 0 bucket_active_conns 1 bucket_conns 1 bytes_read 53441583470 bytes_written 45346723701 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 29975715 cmd_set 445274018 conn_yields 61205 connection_structures 1931 curr_connections 11 curr_items 297280049 curr_items_tot 297280049 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 466 delete_misses 174808 ep_bg_fetched 1661206 ep_bg_load 4365803470 ep_bg_load_avg 2628 ep_bg_max_load 2020165 ep_bg_max_wait 2929571 ep_bg_min_load 1 ep_bg_min_wait 10 ep_bg_num_samples 1661198 ep_bg_wait 48864908595 ep_bg_wait_avg 29415 ep_commit_num 818500 ep_commit_time 0 ep_commit_time_total 179775 ep_data_age 0 ep_data_age_highwat 2571 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 1 ep_dbname /opt/membase/var/lib/membase/data/JDPCachePrimary-data/JDPCachePrimary ep_dbshards 4 ep_diskqueue_drain 358215763 ep_diskqueue_fill 358215763 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 1256 ep_flush_duration_total 251908 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 1661206 ep_io_num_write 357979281 ep_io_read_bytes 145605718 ep_io_write_bytes 6012819006520 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 356086874 ep_kv_size 57356020515 ep_latency_arith_cmd 46093 ep_latency_get_cmd 101813590 ep_latency_store_cmd 515449025 ep_max_data_size 81788928000 ep_max_txn_size 1000 ep_mem_high_wat 61341696000 ep_mem_low_wat 49073356800 ep_min_data_age 0 ep_num_active_non_resident 94237869 ep_num_checkpoint_remover_runs354894 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 490 ep_num_non_resident 94237869 ep_num_not_my_vbuckets 0 ep_num_pager_runs 2 ep_num_value_ejects 95895079 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 1611440768 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 3597 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 62797498 ep_too_young 0 ep_total_cache_size 58930442359 ep_total_del_items 416 ep_total_enqueued 358215763 ep_total_new_items 297280465 ep_total_persisted 357979697 ep_uncommitted_items 0 ep_value_size 15808691965 ep_vb_total 512 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.1.1_39_g4bd26a3 ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 8853 get_hits 7311306 get_misses 22664409 incr_hits 44431 incr_misses 1662 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 58967461283 pid 17177 pointer_size 64 rejected_conns 0 rusage_system 91969.201556 rusage_user 249503.580690 threads 4 time 1325170125 total_connections 30202 uptime 3443056 vb_active_curr_items 297280049 vb_active_eject 95895079 vb_active_ht_memory 1610809344 vb_active_itm_memory 53788643543 vb_active_num 512 vb_active_num_non_resident 94237869 vb_active_ops_create 297280465 vb_active_ops_delete 416 vb_active_ops_reject 0 vb_active_ops_update 60698816 vb_active_perc_mem_resident 68 vb_active_queue_age 0 vb_active_queue_drain 358215763 vb_active_queue_fill 358215763 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version UNKNOWN INFO REPORT <5883.1095.0> 2011-12-29 06:48:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,170126,409575}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102942632}, {processes,32281384}, {processes_used,29607112}, {system,70661248}, {atom,903241}, {atom_used,880290}, {binary,1509952}, {code,8604866}, {ets,38544048}]}, {system_stats, [{cpu_utilization_rate,2.625}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3443055}, {memory_data,{101368545280,100270686208,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1072132 kB\nBuffers: 283676 kB\nCached: 32677248 kB\nSwapCached: 5096 kB\nActive: 82502996 kB\nInactive: 13863712 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1072132 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 120 kB\nWriteback: 0 kB\nAnonPages: 63398020 kB\nMapped: 16728 kB\nSlab: 1346132 kB\nPageTables: 171508 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86134972 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33461501952}, {buffered_memory,290484224}, {free_memory,1097863168}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3443040475,1}}, {context_switches,{1913223763,0}}, {garbage_collection,{248052047,3901120552812,0}}, {io,{{input,69570297580},{output,131990580811}}}, {reductions,{1586112428647,2498180}}, {run_queue,2}, {runtime,{130368760,160}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,170125,404847}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,84103728}, {processes,14562512}, {processes_used,14526816}, {system,69541216}, {atom,828337}, {atom_used,824728}, {binary,1494408}, {code,7914132}, {ets,38373768}]}, {system_stats, [{cpu_utilization_rate,0.625}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3443035}, {memory_data,{101368545280,100843831296,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 510860 kB\nBuffers: 283936 kB\nCached: 31395136 kB\nSwapCached: 3836 kB\nActive: 82928792 kB\nInactive: 13857180 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 510860 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 144 kB\nWriteback: 0 kB\nAnonPages: 65099276 kB\nMapped: 17876 kB\nSlab: 1380968 kB\nPageTables: 278872 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141216968 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32148619264}, {buffered_memory,290750464}, {free_memory,523120640}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3443025641,1}}, {context_switches,{751764211,0}}, {garbage_collection,{185804636,675231079610,0}}, {io,{{input,58280327416},{output,130568070620}}}, {reductions,{324786094037,1402505}}, {run_queue,1}, {runtime,{37328090,150}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 06:49:23 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 06:49:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,170186,408540}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102741200}, {processes,32122056}, {processes_used,29447784}, {system,70619144}, {atom,903241}, {atom_used,880290}, {binary,1435056}, {code,8604866}, {ets,38576560}]}, {system_stats, [{cpu_utilization_rate,0.2503128911138924}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3443115}, {memory_data,{101368545280,100270526464,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1071412 kB\nBuffers: 283676 kB\nCached: 32678048 kB\nSwapCached: 5096 kB\nActive: 82502692 kB\nInactive: 13864504 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1071412 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 328 kB\nWriteback: 0 kB\nAnonPages: 63397904 kB\nMapped: 16728 kB\nSlab: 1346100 kB\nPageTables: 171508 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86136112 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33462321152}, {buffered_memory,290484224}, {free_memory,1097125888}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3443100475,1}}, {context_switches,{1913240636,0}}, {garbage_collection,{248055277,3901132827801,0}}, {io,{{input,69570478015},{output,131991496938}}}, {reductions,{1586124904583,2496424}}, {run_queue,0}, {runtime,{130369550,170}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,170185,404781}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,83743504}, {processes,14261328}, {processes_used,14225632}, {system,69482176}, {atom,828337}, {atom_used,824728}, {binary,1403544}, {code,7914132}, {ets,38405216}]}, {system_stats, [{cpu_utilization_rate,0.5}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3443095}, {memory_data,{101368545280,100845535232,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 511788 kB\nBuffers: 283936 kB\nCached: 31395916 kB\nSwapCached: 3836 kB\nActive: 82927088 kB\nInactive: 13857936 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 511788 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 488 kB\nWriteback: 0 kB\nAnonPages: 65096908 kB\nMapped: 17876 kB\nSlab: 1380892 kB\nPageTables: 278868 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141213676 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32149417984}, {buffered_memory,290750464}, {free_memory,523816960}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3443085641,1}}, {context_switches,{751777007,0}}, {garbage_collection,{185807925,675242478283,0}}, {io,{{input,58280525021},{output,130568925082}}}, {reductions,{324791823998,1411915}}, {run_queue,0}, {runtime,{37328730,160}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 06:50:18 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.111.192> 2011-12-29 06:50:26 =============================================================================== ns_1@192.168.24.61:<5883.111.192>:stats_collector:83: Stats for bucket "JDPCachePrimary": accepting_conns 1 auth_cmds 10369 auth_errors 0 bucket_active_conns 1 bucket_conns 1 bytes_read 53441590630 bytes_written 45348391841 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 29975715 cmd_set 445274018 conn_yields 61205 connection_structures 1931 curr_connections 11 curr_items 297280049 curr_items_tot 297280049 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 466 delete_misses 174808 ep_bg_fetched 1661206 ep_bg_load 4365803470 ep_bg_load_avg 2628 ep_bg_max_load 2020165 ep_bg_max_wait 2929571 ep_bg_min_load 1 ep_bg_min_wait 10 ep_bg_num_samples 1661198 ep_bg_wait 48864908595 ep_bg_wait_avg 29415 ep_commit_num 818500 ep_commit_time 0 ep_commit_time_total 179775 ep_data_age 0 ep_data_age_highwat 2571 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 1 ep_dbname /opt/membase/var/lib/membase/data/JDPCachePrimary-data/JDPCachePrimary ep_dbshards 4 ep_diskqueue_drain 358215763 ep_diskqueue_fill 358215763 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 1256 ep_flush_duration_total 251908 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 1661206 ep_io_num_write 357979281 ep_io_read_bytes 145605718 ep_io_write_bytes 6012819006520 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 356086874 ep_kv_size 57356020515 ep_latency_arith_cmd 46093 ep_latency_get_cmd 101813590 ep_latency_store_cmd 515449025 ep_max_data_size 81788928000 ep_max_txn_size 1000 ep_mem_high_wat 61341696000 ep_mem_low_wat 49073356800 ep_min_data_age 0 ep_num_active_non_resident 94237869 ep_num_checkpoint_remover_runs354914 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 490 ep_num_non_resident 94237869 ep_num_not_my_vbuckets 0 ep_num_pager_runs 2 ep_num_value_ejects 95895079 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 1611440768 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 3597 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 62797498 ep_too_young 0 ep_total_cache_size 58930442359 ep_total_del_items 416 ep_total_enqueued 358215763 ep_total_new_items 297280465 ep_total_persisted 357979697 ep_uncommitted_items 0 ep_value_size 15808691965 ep_vb_total 512 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.1.1_39_g4bd26a3 ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 8853 get_hits 7311306 get_misses 22664409 incr_hits 44431 incr_misses 1662 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 58967461283 pid 17177 pointer_size 64 rejected_conns 0 rusage_system 91969.228552 rusage_user 249503.689673 threads 4 time 1325170225 total_connections 30202 uptime 3443156 vb_active_curr_items 297280049 vb_active_eject 95895079 vb_active_ht_memory 1610809344 vb_active_itm_memory 53788643543 vb_active_num 512 vb_active_num_non_resident 94237869 vb_active_ops_create 297280465 vb_active_ops_delete 416 vb_active_ops_reject 0 vb_active_ops_update 60698816 vb_active_perc_mem_resident 68 vb_active_queue_age 0 vb_active_queue_drain 358215763 vb_active_queue_fill 358215763 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version UNKNOWN INFO REPORT <5883.1095.0> 2011-12-29 06:50:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,170246,408382}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102774312}, {processes,31907440}, {processes_used,29233168}, {system,70866872}, {atom,903241}, {atom_used,880290}, {binary,1526656}, {code,8604866}, {ets,38733016}]}, {system_stats, [{cpu_utilization_rate,0.375}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3443175}, {memory_data,{101368545280,100266999808,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1075112 kB\nBuffers: 283688 kB\nCached: 32673296 kB\nSwapCached: 5096 kB\nActive: 82503056 kB\nInactive: 13859864 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1075112 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 448 kB\nWriteback: 0 kB\nAnonPages: 63398160 kB\nMapped: 16728 kB\nSlab: 1346024 kB\nPageTables: 171512 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86138076 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33457455104}, {buffered_memory,290496512}, {free_memory,1100914688}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3443160475,0}}, {context_switches,{1913257589,0}}, {garbage_collection,{248058742,3901146113987,0}}, {io,{{input,69573150960},{output,131998504552}}}, {reductions,{1586136921128,2456045}}, {run_queue,0}, {runtime,{130370510,160}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,170245,404818}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,84100256}, {processes,14416704}, {processes_used,14381008}, {system,69683552}, {atom,828337}, {atom_used,824728}, {binary,1449960}, {code,7914132}, {ets,38560912}]}, {system_stats, [{cpu_utilization_rate,5.375}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3443155}, {memory_data,{101368545280,100843978752,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 513768 kB\nBuffers: 283940 kB\nCached: 31391160 kB\nSwapCached: 3836 kB\nActive: 82929708 kB\nInactive: 13853240 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 513768 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 368 kB\nWriteback: 0 kB\nAnonPages: 65100152 kB\nMapped: 17876 kB\nSlab: 1381028 kB\nPageTables: 278872 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141217708 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32144547840}, {buffered_memory,290754560}, {free_memory,526098432}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3443145641,0}}, {context_switches,{751790305,0}}, {garbage_collection,{185811295,675254640898,0}}, {io,{{input,58283136156},{output,130575951913}}}, {reductions,{324797386088,1413140}}, {run_queue,1}, {runtime,{37329540,180}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 06:50:55 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 06:51:12 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 06:51:26 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 06:51:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,170306,408512}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,103089928}, {processes,32101384}, {processes_used,29427112}, {system,70988544}, {atom,903241}, {atom_used,880290}, {binary,1613808}, {code,8604866}, {ets,38767600}]}, {system_stats, [{cpu_utilization_rate,13.0}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3443235}, {memory_data,{101368545280,100311199744,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1003212 kB\nBuffers: 283688 kB\nCached: 32674088 kB\nSwapCached: 5096 kB\nActive: 82574520 kB\nInactive: 13860600 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1003212 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 3432 kB\nWriteback: 0 kB\nAnonPages: 63469352 kB\nMapped: 24072 kB\nSlab: 1346376 kB\nPageTables: 172376 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86204512 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33458266112}, {buffered_memory,290496512}, {free_memory,1027289088}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3443220476,2}}, {context_switches,{1913273820,0}}, {garbage_collection,{248061969,3901158229362,0}}, {io,{{input,69573404284},{output,131999355197}}}, {reductions,{1586148850385,2426702}}, {run_queue,0}, {runtime,{130371300,190}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,170305,404840}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,84053312}, {processes,14352576}, {processes_used,14316880}, {system,69700736}, {atom,828337}, {atom_used,824728}, {binary,1432816}, {code,7914132}, {ets,38595240}]}, {system_stats, [{cpu_utilization_rate,0.625}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3443215}, {memory_data,{101368545280,100856885248,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 461408 kB\nBuffers: 283940 kB\nCached: 31391984 kB\nSwapCached: 3836 kB\nActive: 82977648 kB\nInactive: 13853948 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 461408 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 3160 kB\nWriteback: 0 kB\nAnonPages: 65148260 kB\nMapped: 19464 kB\nSlab: 1381820 kB\nPageTables: 279544 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141265540 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32145391616}, {buffered_memory,290754560}, {free_memory,472481792}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3443205641,1}}, {context_switches,{751802888,0}}, {garbage_collection,{185814549,675265837924,0}}, {io,{{input,58283299487},{output,130576857858}}}, {reductions,{324802845560,1377126}}, {run_queue,0}, {runtime,{37330180,190}}]}]}] INFO REPORT <5883.111.192> 2011-12-29 06:52:06 =============================================================================== ns_1@192.168.24.61:<5883.111.192>:stats_collector:83: Stats for bucket "JDPCachePrimary": accepting_conns 1 auth_cmds 10369 auth_errors 0 bucket_active_conns 1 bucket_conns 1 bytes_read 53441597790 bytes_written 45350059981 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 29975715 cmd_set 445274018 conn_yields 61205 connection_structures 1931 curr_connections 11 curr_items 297280049 curr_items_tot 297280049 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 466 delete_misses 174808 ep_bg_fetched 1661206 ep_bg_load 4365803470 ep_bg_load_avg 2628 ep_bg_max_load 2020165 ep_bg_max_wait 2929571 ep_bg_min_load 1 ep_bg_min_wait 10 ep_bg_num_samples 1661198 ep_bg_wait 48864908595 ep_bg_wait_avg 29415 ep_commit_num 818500 ep_commit_time 0 ep_commit_time_total 179775 ep_data_age 0 ep_data_age_highwat 2571 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 1 ep_dbname /opt/membase/var/lib/membase/data/JDPCachePrimary-data/JDPCachePrimary ep_dbshards 4 ep_diskqueue_drain 358215763 ep_diskqueue_fill 358215763 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 1256 ep_flush_duration_total 251908 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 1661206 ep_io_num_write 357979281 ep_io_read_bytes 145605718 ep_io_write_bytes 6012819006520 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 356086874 ep_kv_size 57356020515 ep_latency_arith_cmd 46093 ep_latency_get_cmd 101813590 ep_latency_store_cmd 515449025 ep_max_data_size 81788928000 ep_max_txn_size 1000 ep_mem_high_wat 61341696000 ep_mem_low_wat 49073356800 ep_min_data_age 0 ep_num_active_non_resident 94237869 ep_num_checkpoint_remover_runs354934 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 490 ep_num_non_resident 94237869 ep_num_not_my_vbuckets 0 ep_num_pager_runs 2 ep_num_value_ejects 95895079 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 1611440768 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 3597 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 62797498 ep_too_young 0 ep_total_cache_size 58930442359 ep_total_del_items 416 ep_total_enqueued 358215763 ep_total_new_items 297280465 ep_total_persisted 357979697 ep_uncommitted_items 0 ep_value_size 15808691965 ep_vb_total 512 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.1.1_39_g4bd26a3 ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 8853 get_hits 7311306 get_misses 22664409 incr_hits 44431 incr_misses 1662 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 58967461283 pid 17177 pointer_size 64 rejected_conns 0 rusage_system 91969.263546 rusage_user 249503.783659 threads 4 time 1325170325 total_connections 30202 uptime 3443256 vb_active_curr_items 297280049 vb_active_eject 95895079 vb_active_ht_memory 1610809344 vb_active_itm_memory 53788643543 vb_active_num 512 vb_active_num_non_resident 94237869 vb_active_ops_create 297280465 vb_active_ops_delete 416 vb_active_ops_reject 0 vb_active_ops_update 60698816 vb_active_perc_mem_resident 68 vb_active_queue_age 0 vb_active_queue_drain 358215763 vb_active_queue_fill 358215763 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version UNKNOWN INFO REPORT <5883.136.0> 2011-12-29 06:52:17 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 06:52:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,170366,408518}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102710456}, {processes,31570864}, {processes_used,28896592}, {system,71139592}, {atom,903241}, {atom_used,880290}, {binary,1618880}, {code,8604866}, {ets,38913280}]}, {system_stats, [{cpu_utilization_rate,0.2503128911138924}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3443295}, {memory_data,{101368545280,100269092864,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1074076 kB\nBuffers: 283688 kB\nCached: 32674932 kB\nSwapCached: 5096 kB\nActive: 82503388 kB\nInactive: 13861276 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1074076 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 1096 kB\nWriteback: 0 kB\nAnonPages: 63398208 kB\nMapped: 16728 kB\nSlab: 1346420 kB\nPageTables: 171512 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86134356 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33459130368}, {buffered_memory,290496512}, {free_memory,1099853824}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3443280475,1}}, {context_switches,{1913289940,0}}, {garbage_collection,{248065089,3901170094051,0}}, {io,{{input,69573589910},{output,132000264542}}}, {reductions,{1586160736828,2411056}}, {run_queue,0}, {runtime,{130372070,160}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,170365,406009}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,84366168}, {processes,14441528}, {processes_used,14406120}, {system,69924640}, {atom,828337}, {atom_used,824728}, {binary,1505648}, {code,7914132}, {ets,38746736}]}, {system_stats, [{cpu_utilization_rate,0.625}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3443275}, {memory_data,{101368545280,100846395392,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 512676 kB\nBuffers: 283940 kB\nCached: 31392812 kB\nSwapCached: 3836 kB\nActive: 82929728 kB\nInactive: 13854644 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 512676 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 704 kB\nWriteback: 0 kB\nAnonPages: 65099432 kB\nMapped: 17876 kB\nSlab: 1381172 kB\nPageTables: 278876 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141216180 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32146239488}, {buffered_memory,290754560}, {free_memory,524980224}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3443265641,1}}, {context_switches,{751815413,0}}, {garbage_collection,{185817631,675277068078,0}}, {io,{{input,58283524958},{output,130577715885}}}, {reductions,{324808315766,1397178}}, {run_queue,0}, {runtime,{37330790,140}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 06:52:51 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 06:53:29 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.111.192> 2011-12-29 06:53:46 =============================================================================== ns_1@192.168.24.61:<5883.111.192>:stats_collector:83: Stats for bucket "JDPCachePrimary": accepting_conns 1 auth_cmds 10369 auth_errors 0 bucket_active_conns 1 bucket_conns 1 bytes_read 53441604950 bytes_written 45351728121 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 29975715 cmd_set 445274018 conn_yields 61205 connection_structures 1931 curr_connections 11 curr_items 297280049 curr_items_tot 297280049 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 466 delete_misses 174808 ep_bg_fetched 1661206 ep_bg_load 4365803470 ep_bg_load_avg 2628 ep_bg_max_load 2020165 ep_bg_max_wait 2929571 ep_bg_min_load 1 ep_bg_min_wait 10 ep_bg_num_samples 1661198 ep_bg_wait 48864908595 ep_bg_wait_avg 29415 ep_commit_num 818500 ep_commit_time 0 ep_commit_time_total 179775 ep_data_age 0 ep_data_age_highwat 2571 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 1 ep_dbname /opt/membase/var/lib/membase/data/JDPCachePrimary-data/JDPCachePrimary ep_dbshards 4 ep_diskqueue_drain 358215763 ep_diskqueue_fill 358215763 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 1256 ep_flush_duration_total 251908 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 1661206 ep_io_num_write 357979281 ep_io_read_bytes 145605718 ep_io_write_bytes 6012819006520 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 356086874 ep_kv_size 57356020515 ep_latency_arith_cmd 46093 ep_latency_get_cmd 101813590 ep_latency_store_cmd 515449025 ep_max_data_size 81788928000 ep_max_txn_size 1000 ep_mem_high_wat 61341696000 ep_mem_low_wat 49073356800 ep_min_data_age 0 ep_num_active_non_resident 94237869 ep_num_checkpoint_remover_runs354954 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 490 ep_num_non_resident 94237869 ep_num_not_my_vbuckets 0 ep_num_pager_runs 2 ep_num_value_ejects 95895079 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 1611440768 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 3597 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 62797498 ep_too_young 0 ep_total_cache_size 58930442359 ep_total_del_items 416 ep_total_enqueued 358215763 ep_total_new_items 297280465 ep_total_persisted 357979697 ep_uncommitted_items 0 ep_value_size 15808691965 ep_vb_total 512 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.1.1_39_g4bd26a3 ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 8853 get_hits 7311306 get_misses 22664409 incr_hits 44431 incr_misses 1662 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 58967461283 pid 17177 pointer_size 64 rejected_conns 0 rusage_system 91969.305540 rusage_user 249503.864647 threads 4 time 1325170425 total_connections 30202 uptime 3443356 vb_active_curr_items 297280049 vb_active_eject 95895079 vb_active_ht_memory 1610809344 vb_active_itm_memory 53788643543 vb_active_num 512 vb_active_num_non_resident 94237869 vb_active_ops_create 297280465 vb_active_ops_delete 416 vb_active_ops_reject 0 vb_active_ops_update 60698816 vb_active_perc_mem_resident 68 vb_active_queue_age 0 vb_active_queue_drain 358215763 vb_active_queue_fill 358215763 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version UNKNOWN INFO REPORT <5883.136.0> 2011-12-29 06:53:46 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 06:53:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,170426,408437}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,103221680}, {processes,31895832}, {processes_used,29221560}, {system,71325848}, {atom,903241}, {atom_used,880290}, {binary,1657448}, {code,8604866}, {ets,39061656}]}, {system_stats, [{cpu_utilization_rate,1.4981273408239701}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3443355}, {memory_data,{101368545280,100267978752,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1073904 kB\nBuffers: 283696 kB\nCached: 32675224 kB\nSwapCached: 5096 kB\nActive: 82503696 kB\nInactive: 13861644 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1073904 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 328 kB\nWriteback: 0 kB\nAnonPages: 63398560 kB\nMapped: 16728 kB\nSlab: 1346084 kB\nPageTables: 171512 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86136320 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33459429376}, {buffered_memory,290504704}, {free_memory,1099677696}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3443340476,1}}, {context_switches,{1913306894,0}}, {garbage_collection,{248068481,3901182737104,0}}, {io,{{input,69576277146},{output,132003744732}}}, {reductions,{1586172863478,2449320}}, {run_queue,0}, {runtime,{130372980,180}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,170425,404679}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,84608576}, {processes,14613432}, {processes_used,14577736}, {system,69995144}, {atom,828337}, {atom_used,824728}, {binary,1490696}, {code,7914132}, {ets,38832704}]}, {system_stats, [{cpu_utilization_rate,0.625}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3443335}, {memory_data,{101368545280,100842553344,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 512508 kB\nBuffers: 283940 kB\nCached: 31393112 kB\nSwapCached: 3836 kB\nActive: 82929544 kB\nInactive: 13855108 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 512508 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 292 kB\nWriteback: 0 kB\nAnonPages: 65099932 kB\nMapped: 17876 kB\nSlab: 1381040 kB\nPageTables: 278880 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141213408 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32146546688}, {buffered_memory,290754560}, {free_memory,524808192}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3443325641,0}}, {context_switches,{751828954,0}}, {garbage_collection,{185821241,675288848119,0}}, {io,{{input,58286170596},{output,130581211893}}}, {reductions,{324814135964,1380200}}, {run_queue,0}, {runtime,{37331470,160}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 06:54:04 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 06:54:35 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 06:54:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,170486,408626}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,103039216}, {processes,32128816}, {processes_used,29454544}, {system,70910400}, {atom,903241}, {atom_used,880290}, {binary,1710608}, {code,8604866}, {ets,38593056}]}, {system_stats, [{cpu_utilization_rate,0.625}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3443415}, {memory_data,{101368545280,100268761088,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1073796 kB\nBuffers: 283696 kB\nCached: 32676008 kB\nSwapCached: 5096 kB\nActive: 82502996 kB\nInactive: 13862376 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1073796 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 168 kB\nWriteback: 0 kB\nAnonPages: 63397904 kB\nMapped: 16728 kB\nSlab: 1346044 kB\nPageTables: 171512 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86135324 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33460232192}, {buffered_memory,290504704}, {free_memory,1099567104}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3443400476,2}}, {context_switches,{1913323106,0}}, {garbage_collection,{248071703,3901194849236,0}}, {io,{{input,69576495892},{output,132004587180}}}, {reductions,{1586184869933,2490234}}, {run_queue,0}, {runtime,{130373790,170}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,170485,405088}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,83929376}, {processes,14322368}, {processes_used,14286672}, {system,69607008}, {atom,828337}, {atom_used,824728}, {binary,1513808}, {code,7914132}, {ets,38421904}]}, {system_stats, [{cpu_utilization_rate,0.8739076154806492}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3443395}, {memory_data,{101368545280,100843704320,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 512452 kB\nBuffers: 283940 kB\nCached: 31393880 kB\nSwapCached: 3836 kB\nActive: 82929424 kB\nInactive: 13855820 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 512452 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 40 kB\nWriteback: 0 kB\nAnonPages: 65099604 kB\nMapped: 17876 kB\nSlab: 1380984 kB\nPageTables: 278884 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141215436 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32147333120}, {buffered_memory,290754560}, {free_memory,524750848}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3443385641,1}}, {context_switches,{751841591,0}}, {garbage_collection,{185824433,675300007531,0}}, {io,{{input,58286336820},{output,130582100505}}}, {reductions,{324819588302,1395898}}, {run_queue,0}, {runtime,{37332050,140}}]}]}] INFO REPORT <5883.111.192> 2011-12-29 06:55:26 =============================================================================== ns_1@192.168.24.61:<5883.111.192>:stats_collector:83: Stats for bucket "JDPCachePrimary": accepting_conns 1 auth_cmds 10369 auth_errors 0 bucket_active_conns 1 bucket_conns 1 bytes_read 53441612110 bytes_written 45353396261 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 29975715 cmd_set 445274018 conn_yields 61205 connection_structures 1931 curr_connections 11 curr_items 297280049 curr_items_tot 297280049 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 466 delete_misses 174808 ep_bg_fetched 1661206 ep_bg_load 4365803470 ep_bg_load_avg 2628 ep_bg_max_load 2020165 ep_bg_max_wait 2929571 ep_bg_min_load 1 ep_bg_min_wait 10 ep_bg_num_samples 1661198 ep_bg_wait 48864908595 ep_bg_wait_avg 29415 ep_commit_num 818500 ep_commit_time 0 ep_commit_time_total 179775 ep_data_age 0 ep_data_age_highwat 2571 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 1 ep_dbname /opt/membase/var/lib/membase/data/JDPCachePrimary-data/JDPCachePrimary ep_dbshards 4 ep_diskqueue_drain 358215763 ep_diskqueue_fill 358215763 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 1256 ep_flush_duration_total 251908 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 1661206 ep_io_num_write 357979281 ep_io_read_bytes 145605718 ep_io_write_bytes 6012819006520 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 356086874 ep_kv_size 57356020515 ep_latency_arith_cmd 46093 ep_latency_get_cmd 101813590 ep_latency_store_cmd 515449025 ep_max_data_size 81788928000 ep_max_txn_size 1000 ep_mem_high_wat 61341696000 ep_mem_low_wat 49073356800 ep_min_data_age 0 ep_num_active_non_resident 94237869 ep_num_checkpoint_remover_runs354974 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 490 ep_num_non_resident 94237869 ep_num_not_my_vbuckets 0 ep_num_pager_runs 2 ep_num_value_ejects 95895079 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 1611440768 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 3597 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 62797498 ep_too_young 0 ep_total_cache_size 58930442359 ep_total_del_items 416 ep_total_enqueued 358215763 ep_total_new_items 297280465 ep_total_persisted 357979697 ep_uncommitted_items 0 ep_value_size 15808691965 ep_vb_total 512 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.1.1_39_g4bd26a3 ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 8853 get_hits 7311306 get_misses 22664409 incr_hits 44431 incr_misses 1662 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 58967461283 pid 17177 pointer_size 64 rejected_conns 0 rusage_system 91969.328536 rusage_user 249503.965631 threads 4 time 1325170525 total_connections 30202 uptime 3443456 vb_active_curr_items 297280049 vb_active_eject 95895079 vb_active_ht_memory 1610809344 vb_active_itm_memory 53788643543 vb_active_num 512 vb_active_num_non_resident 94237869 vb_active_ops_create 297280465 vb_active_ops_delete 416 vb_active_ops_reject 0 vb_active_ops_update 60698816 vb_active_perc_mem_resident 68 vb_active_queue_age 0 vb_active_queue_drain 358215763 vb_active_queue_fill 358215763 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version UNKNOWN INFO REPORT <5883.136.0> 2011-12-29 06:55:33 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 06:55:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,170546,408452}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,103026064}, {processes,32104416}, {processes_used,29430144}, {system,70921648}, {atom,903241}, {atom_used,880290}, {binary,1688368}, {code,8604866}, {ets,38626608}]}, {system_stats, [{cpu_utilization_rate,12.875}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3443475}, {memory_data,{101368545280,100269608960,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1073108 kB\nBuffers: 283696 kB\nCached: 32676800 kB\nSwapCached: 5096 kB\nActive: 82503872 kB\nInactive: 13863136 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1073108 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 552 kB\nWriteback: 0 kB\nAnonPages: 63398624 kB\nMapped: 16728 kB\nSlab: 1346032 kB\nPageTables: 171512 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86134724 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33461043200}, {buffered_memory,290504704}, {free_memory,1098862592}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3443460474,0}}, {context_switches,{1913339975,0}}, {garbage_collection,{248074869,3901206966163,0}}, {io,{{input,69576684420},{output,132005507146}}}, {reductions,{1586197299036,2487682}}, {run_queue,0}, {runtime,{130374610,190}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,170545,405253}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,84264816}, {processes,14569048}, {processes_used,14533352}, {system,69695768}, {atom,828337}, {atom_used,824728}, {binary,1570592}, {code,7914132}, {ets,38453912}]}, {system_stats, [{cpu_utilization_rate,0.6257822277847309}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3443455}, {memory_data,{101368545280,100844331008,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 509844 kB\nBuffers: 283940 kB\nCached: 31394664 kB\nSwapCached: 3836 kB\nActive: 82930996 kB\nInactive: 13856504 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 509844 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 548 kB\nWriteback: 0 kB\nAnonPages: 65100860 kB\nMapped: 17876 kB\nSlab: 1380964 kB\nPageTables: 278884 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141215840 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32148135936}, {buffered_memory,290754560}, {free_memory,522080256}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3443445641,1}}, {context_switches,{751854545,0}}, {garbage_collection,{185827580,675311140808,0}}, {io,{{input,58286529534},{output,130582998446}}}, {reductions,{324825289641,1408233}}, {run_queue,1}, {runtime,{37332670,150}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 06:56:29 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 06:56:47 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 06:56:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,170606,408597}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,103147344}, {processes,32184088}, {processes_used,29509816}, {system,70963256}, {atom,903241}, {atom_used,880290}, {binary,1581320}, {code,8604866}, {ets,38775224}]}, {system_stats, [{cpu_utilization_rate,0.4993757802746567}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3443535}, {memory_data,{101368545280,100319469568,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 973316 kB\nBuffers: 283700 kB\nCached: 32676508 kB\nSwapCached: 5096 kB\nActive: 82601392 kB\nInactive: 13862700 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 973316 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 3064 kB\nWriteback: 0 kB\nAnonPages: 63496428 kB\nMapped: 24072 kB\nSlab: 1347288 kB\nPageTables: 172428 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86231356 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33460744192}, {buffered_memory,290508800}, {free_memory,996675584}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3443520478,3}}, {context_switches,{1913357251,0}}, {garbage_collection,{248078154,3901219779615,0}}, {io,{{input,69579389952},{output,132008318397}}}, {reductions,{1586209376268,2484965}}, {run_queue,0}, {runtime,{130375510,190}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,170605,405145}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,84248696}, {processes,14484720}, {processes_used,14449024}, {system,69763976}, {atom,828337}, {atom_used,824728}, {binary,1489656}, {code,7914132}, {ets,38603424}]}, {system_stats, [{cpu_utilization_rate,13.391739674593241}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3443515}, {memory_data,{101368545280,100846862336,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 405632 kB\nBuffers: 283944 kB\nCached: 31394192 kB\nSwapCached: 3836 kB\nActive: 83032812 kB\nInactive: 13856084 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 405632 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 1876 kB\nWriteback: 0 kB\nAnonPages: 65202600 kB\nMapped: 25144 kB\nSlab: 1381944 kB\nPageTables: 279828 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141315328 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32147652608}, {buffered_memory,290758656}, {free_memory,415367168}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3443505641,0}}, {context_switches,{751868034,0}}, {garbage_collection,{185830934,675323101218,0}}, {io,{{input,58289176228},{output,130586577043}}}, {reductions,{324830888865,1400162}}, {run_queue,0}, {runtime,{37333400,140}}]}]}] INFO REPORT <5883.111.192> 2011-12-29 06:57:06 =============================================================================== ns_1@192.168.24.61:<5883.111.192>:stats_collector:83: Stats for bucket "JDPCachePrimary": accepting_conns 1 auth_cmds 10369 auth_errors 0 bucket_active_conns 1 bucket_conns 1 bytes_read 53441619270 bytes_written 45355064401 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 29975715 cmd_set 445274018 conn_yields 61205 connection_structures 1931 curr_connections 11 curr_items 297280049 curr_items_tot 297280049 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 466 delete_misses 174808 ep_bg_fetched 1661206 ep_bg_load 4365803470 ep_bg_load_avg 2628 ep_bg_max_load 2020165 ep_bg_max_wait 2929571 ep_bg_min_load 1 ep_bg_min_wait 10 ep_bg_num_samples 1661198 ep_bg_wait 48864908595 ep_bg_wait_avg 29415 ep_commit_num 818500 ep_commit_time 0 ep_commit_time_total 179775 ep_data_age 0 ep_data_age_highwat 2571 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 1 ep_dbname /opt/membase/var/lib/membase/data/JDPCachePrimary-data/JDPCachePrimary ep_dbshards 4 ep_diskqueue_drain 358215763 ep_diskqueue_fill 358215763 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 1256 ep_flush_duration_total 251908 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 1661206 ep_io_num_write 357979281 ep_io_read_bytes 145605718 ep_io_write_bytes 6012819006520 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 356086874 ep_kv_size 57356020515 ep_latency_arith_cmd 46093 ep_latency_get_cmd 101813590 ep_latency_store_cmd 515449025 ep_max_data_size 81788928000 ep_max_txn_size 1000 ep_mem_high_wat 61341696000 ep_mem_low_wat 49073356800 ep_min_data_age 0 ep_num_active_non_resident 94237869 ep_num_checkpoint_remover_runs354994 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 490 ep_num_non_resident 94237869 ep_num_not_my_vbuckets 0 ep_num_pager_runs 2 ep_num_value_ejects 95895079 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 1611440768 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 3597 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 62797498 ep_too_young 0 ep_total_cache_size 58930442359 ep_total_del_items 416 ep_total_enqueued 358215763 ep_total_new_items 297280465 ep_total_persisted 357979697 ep_uncommitted_items 0 ep_value_size 15808691965 ep_vb_total 512 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.1.1_39_g4bd26a3 ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 8853 get_hits 7311306 get_misses 22664409 incr_hits 44431 incr_misses 1662 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 58967461283 pid 17177 pointer_size 64 rejected_conns 0 rusage_system 91969.360532 rusage_user 249504.065616 threads 4 time 1325170625 total_connections 30202 uptime 3443556 vb_active_curr_items 297280049 vb_active_eject 95895079 vb_active_ht_memory 1610809344 vb_active_itm_memory 53788643543 vb_active_num 512 vb_active_num_non_resident 94237869 vb_active_ops_create 297280465 vb_active_ops_delete 416 vb_active_ops_reject 0 vb_active_ops_update 60698816 vb_active_perc_mem_resident 68 vb_active_queue_age 0 vb_active_queue_drain 358215763 vb_active_queue_fill 358215763 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version UNKNOWN INFO REPORT <5883.136.0> 2011-12-29 06:57:33 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 06:57:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,170666,409192}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102824568}, {processes,31633144}, {processes_used,28958872}, {system,71191424}, {atom,903241}, {atom_used,880290}, {binary,1777136}, {code,8604866}, {ets,38807336}]}, {system_stats, [{cpu_utilization_rate,0.375}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3443595}, {memory_data,{101368545280,100270989312,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1071708 kB\nBuffers: 283700 kB\nCached: 32677308 kB\nSwapCached: 5096 kB\nActive: 82504428 kB\nInactive: 13863408 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1071708 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 304 kB\nWriteback: 0 kB\nAnonPages: 63399084 kB\nMapped: 16728 kB\nSlab: 1346188 kB\nPageTables: 171512 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86137556 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33461563392}, {buffered_memory,290508800}, {free_memory,1097428992}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3443580476,1}}, {context_switches,{1913373516,0}}, {garbage_collection,{248081300,3901231642330,0}}, {io,{{input,69579576436},{output,132009229100}}}, {reductions,{1586221300016,2461843}}, {run_queue,0}, {runtime,{130376290,180}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,170665,404904}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,84392968}, {processes,14523016}, {processes_used,14487320}, {system,69869952}, {atom,828337}, {atom_used,824728}, {binary,1563264}, {code,7914132}, {ets,38635488}]}, {system_stats, [{cpu_utilization_rate,0.625}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3443575}, {memory_data,{101368545280,100900368384,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 509672 kB\nBuffers: 283944 kB\nCached: 31395020 kB\nSwapCached: 3836 kB\nActive: 82930664 kB\nInactive: 13856788 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 509672 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 128 kB\nWriteback: 0 kB\nAnonPages: 65100504 kB\nMapped: 17876 kB\nSlab: 1381256 kB\nPageTables: 278880 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141216484 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32148500480}, {buffered_memory,290758656}, {free_memory,521904128}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3443565641,0}}, {context_switches,{751880625,0}}, {garbage_collection,{185834157,675334382386,0}}, {io,{{input,58289408597},{output,130587468391}}}, {reductions,{324836383461,1411271}}, {run_queue,0}, {runtime,{37334010,160}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 06:58:13 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 06:58:35 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.111.192> 2011-12-29 06:58:46 =============================================================================== ns_1@192.168.24.61:<5883.111.192>:stats_collector:83: Stats for bucket "JDPCachePrimary": accepting_conns 1 auth_cmds 10369 auth_errors 0 bucket_active_conns 1 bucket_conns 1 bytes_read 53441626430 bytes_written 45356732541 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 29975715 cmd_set 445274018 conn_yields 61205 connection_structures 1931 curr_connections 11 curr_items 297280049 curr_items_tot 297280049 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 466 delete_misses 174808 ep_bg_fetched 1661206 ep_bg_load 4365803470 ep_bg_load_avg 2628 ep_bg_max_load 2020165 ep_bg_max_wait 2929571 ep_bg_min_load 1 ep_bg_min_wait 10 ep_bg_num_samples 1661198 ep_bg_wait 48864908595 ep_bg_wait_avg 29415 ep_commit_num 818500 ep_commit_time 0 ep_commit_time_total 179775 ep_data_age 0 ep_data_age_highwat 2571 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 1 ep_dbname /opt/membase/var/lib/membase/data/JDPCachePrimary-data/JDPCachePrimary ep_dbshards 4 ep_diskqueue_drain 358215763 ep_diskqueue_fill 358215763 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 1256 ep_flush_duration_total 251908 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 1661206 ep_io_num_write 357979281 ep_io_read_bytes 145605718 ep_io_write_bytes 6012819006520 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 356086874 ep_kv_size 57356020515 ep_latency_arith_cmd 46093 ep_latency_get_cmd 101813590 ep_latency_store_cmd 515449025 ep_max_data_size 81788928000 ep_max_txn_size 1000 ep_mem_high_wat 61341696000 ep_mem_low_wat 49073356800 ep_min_data_age 0 ep_num_active_non_resident 94237869 ep_num_checkpoint_remover_runs355014 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 490 ep_num_non_resident 94237869 ep_num_not_my_vbuckets 0 ep_num_pager_runs 2 ep_num_value_ejects 95895079 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 1611440768 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 3597 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 62797498 ep_too_young 0 ep_total_cache_size 58930442359 ep_total_del_items 416 ep_total_enqueued 358215763 ep_total_new_items 297280465 ep_total_persisted 357979697 ep_uncommitted_items 0 ep_value_size 15808691965 ep_vb_total 512 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.1.1_39_g4bd26a3 ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 8853 get_hits 7311306 get_misses 22664409 incr_hits 44431 incr_misses 1662 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 58967461283 pid 17177 pointer_size 64 rejected_conns 0 rusage_system 91969.388527 rusage_user 249504.170600 threads 4 time 1325170725 total_connections 30202 uptime 3443656 vb_active_curr_items 297280049 vb_active_eject 95895079 vb_active_ht_memory 1610809344 vb_active_itm_memory 53788643543 vb_active_num 512 vb_active_num_non_resident 94237869 vb_active_ops_create 297280465 vb_active_ops_delete 416 vb_active_ops_reject 0 vb_active_ops_update 60698816 vb_active_perc_mem_resident 68 vb_active_queue_age 0 vb_active_queue_drain 358215763 vb_active_queue_fill 358215763 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version UNKNOWN INFO REPORT <5883.1095.0> 2011-12-29 06:58:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,170726,408457}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,103047600}, {processes,31759008}, {processes_used,29084736}, {system,71288592}, {atom,903241}, {atom_used,880290}, {binary,1727664}, {code,8604866}, {ets,38954000}]}, {system_stats, [{cpu_utilization_rate,0.25}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3443655}, {memory_data,{101368545280,100271808512,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1071276 kB\nBuffers: 283700 kB\nCached: 32678068 kB\nSwapCached: 5096 kB\nActive: 82504436 kB\nInactive: 13864160 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1071276 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 524 kB\nWriteback: 0 kB\nAnonPages: 63399236 kB\nMapped: 16728 kB\nSlab: 1346024 kB\nPageTables: 171512 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86136248 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33462341632}, {buffered_memory,290508800}, {free_memory,1096986624}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3443640475,1}}, {context_switches,{1913389790,0}}, {garbage_collection,{248084578,3901243752348,0}}, {io,{{input,69579793484},{output,132010077102}}}, {reductions,{1586233281914,2451840}}, {run_queue,0}, {runtime,{130377070,180}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,170725,404695}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,84182072}, {processes,14298472}, {processes_used,14262776}, {system,69883600}, {atom,828337}, {atom_used,824728}, {binary,1430144}, {code,7914132}, {ets,38782312}]}, {system_stats, [{cpu_utilization_rate,0.625}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3443635}, {memory_data,{101368545280,100845887488,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 509120 kB\nBuffers: 283944 kB\nCached: 31395788 kB\nSwapCached: 3836 kB\nActive: 82930860 kB\nInactive: 13857456 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 509120 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 508 kB\nWriteback: 0 kB\nAnonPages: 65100984 kB\nMapped: 17876 kB\nSlab: 1381048 kB\nPageTables: 278884 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141216868 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32149286912}, {buffered_memory,290758656}, {free_memory,521338880}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3443625641,0}}, {context_switches,{751893233,0}}, {garbage_collection,{185837233,675345400769,0}}, {io,{{input,58289603204},{output,130588302491}}}, {reductions,{324841819621,1367207}}, {run_queue,0}, {runtime,{37334610,170}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 06:59:28 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 06:59:37 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 06:59:44 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 06:59:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,170786,408475}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,103507488}, {processes,32174328}, {processes_used,29500056}, {system,71333160}, {atom,903241}, {atom_used,880290}, {binary,1623704}, {code,8604866}, {ets,39102808}]}, {system_stats, [{cpu_utilization_rate,0.2506265664160401}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3443715}, {memory_data,{101368545280,100268216320,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1074432 kB\nBuffers: 283708 kB\nCached: 32674528 kB\nSwapCached: 5096 kB\nActive: 82504100 kB\nInactive: 13860768 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1074432 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 152 kB\nWriteback: 0 kB\nAnonPages: 63398908 kB\nMapped: 16728 kB\nSlab: 1346016 kB\nPageTables: 171512 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86137808 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33458716672}, {buffered_memory,290516992}, {free_memory,1100218368}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3443700475,1}}, {context_switches,{1913406736,0}}, {garbage_collection,{248088020,3901257103700,0}}, {io,{{input,69582476983},{output,132018077779}}}, {reductions,{1586245406990,2436250}}, {run_queue,0}, {runtime,{130377990,180}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,170785,405082}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,84325320}, {processes,14158136}, {processes_used,14122440}, {system,70167184}, {atom,828337}, {atom_used,824728}, {binary,1623344}, {code,7914132}, {ets,38873232}]}, {system_stats, [{cpu_utilization_rate,0.37453183520599254}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3443695}, {memory_data,{101368545280,100847198208,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 512668 kB\nBuffers: 283960 kB\nCached: 31392256 kB\nSwapCached: 3836 kB\nActive: 82930576 kB\nInactive: 13854184 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 512668 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 304 kB\nWriteback: 0 kB\nAnonPages: 65100364 kB\nMapped: 17876 kB\nSlab: 1381040 kB\nPageTables: 278880 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141216916 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32145670144}, {buffered_memory,290775040}, {free_memory,524972032}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3443685641,1}}, {context_switches,{751906910,0}}, {garbage_collection,{185840843,675358079847,0}}, {io,{{input,58292213370},{output,130596334305}}}, {reductions,{324847719080,1408324}}, {run_queue,0}, {runtime,{37335380,140}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 07:00:05 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.111.192> 2011-12-29 07:00:26 =============================================================================== ns_1@192.168.24.61:<5883.111.192>:stats_collector:83: Stats for bucket "JDPCachePrimary": accepting_conns 1 auth_cmds 10369 auth_errors 0 bucket_active_conns 1 bucket_conns 1 bytes_read 53441633590 bytes_written 45358400681 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 29975715 cmd_set 445274018 conn_yields 61205 connection_structures 1931 curr_connections 11 curr_items 297280049 curr_items_tot 297280049 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 466 delete_misses 174808 ep_bg_fetched 1661206 ep_bg_load 4365803470 ep_bg_load_avg 2628 ep_bg_max_load 2020165 ep_bg_max_wait 2929571 ep_bg_min_load 1 ep_bg_min_wait 10 ep_bg_num_samples 1661198 ep_bg_wait 48864908595 ep_bg_wait_avg 29415 ep_commit_num 818500 ep_commit_time 0 ep_commit_time_total 179775 ep_data_age 0 ep_data_age_highwat 2571 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 1 ep_dbname /opt/membase/var/lib/membase/data/JDPCachePrimary-data/JDPCachePrimary ep_dbshards 4 ep_diskqueue_drain 358215763 ep_diskqueue_fill 358215763 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 1256 ep_flush_duration_total 251908 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 1661206 ep_io_num_write 357979281 ep_io_read_bytes 145605718 ep_io_write_bytes 6012819006520 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 356086874 ep_kv_size 57356020515 ep_latency_arith_cmd 46093 ep_latency_get_cmd 101813590 ep_latency_store_cmd 515449025 ep_max_data_size 81788928000 ep_max_txn_size 1000 ep_mem_high_wat 61341696000 ep_mem_low_wat 49073356800 ep_min_data_age 0 ep_num_active_non_resident 94237869 ep_num_checkpoint_remover_runs355034 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 490 ep_num_non_resident 94237869 ep_num_not_my_vbuckets 0 ep_num_pager_runs 2 ep_num_value_ejects 95895079 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 1611440768 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 3597 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 62797498 ep_too_young 0 ep_total_cache_size 58930442359 ep_total_del_items 416 ep_total_enqueued 358215763 ep_total_new_items 297280465 ep_total_persisted 357979697 ep_uncommitted_items 0 ep_value_size 15808691965 ep_vb_total 512 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.1.1_39_g4bd26a3 ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 8853 get_hits 7311306 get_misses 22664409 incr_hits 44431 incr_misses 1662 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 58967461283 pid 17177 pointer_size 64 rejected_conns 0 rusage_system 91969.410524 rusage_user 249504.276584 threads 4 time 1325170825 total_connections 30202 uptime 3443756 vb_active_curr_items 297280049 vb_active_eject 95895079 vb_active_ht_memory 1610809344 vb_active_itm_memory 53788643543 vb_active_num 512 vb_active_num_non_resident 94237869 vb_active_ops_create 297280465 vb_active_ops_delete 416 vb_active_ops_reject 0 vb_active_ops_update 60698816 vb_active_perc_mem_resident 68 vb_active_queue_age 0 vb_active_queue_drain 358215763 vb_active_queue_fill 358215763 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version UNKNOWN INFO REPORT <5883.136.0> 2011-12-29 07:00:41 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 07:00:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,170846,408647}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102777248}, {processes,31929312}, {processes_used,29255040}, {system,70847936}, {atom,903241}, {atom_used,880290}, {binary,1593872}, {code,8604866}, {ets,38647272}]}, {system_stats, [{cpu_utilization_rate,0.625}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3443775}, {memory_data,{101368545280,100267810816,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1075656 kB\nBuffers: 283708 kB\nCached: 32675316 kB\nSwapCached: 5096 kB\nActive: 82502984 kB\nInactive: 13861472 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1075656 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 280 kB\nWriteback: 0 kB\nAnonPages: 63397784 kB\nMapped: 16728 kB\nSlab: 1345928 kB\nPageTables: 171516 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86136768 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33459523584}, {buffered_memory,290516992}, {free_memory,1101471744}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3443760475,0}}, {context_switches,{1913423063,0}}, {garbage_collection,{248091155,3901269412579,0}}, {io,{{input,69582738170},{output,132018981392}}}, {reductions,{1586257475099,2527500}}, {run_queue,0}, {runtime,{130378800,190}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,170845,405090}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,84470920}, {processes,14769040}, {processes_used,14733344}, {system,69701880}, {atom,828337}, {atom_used,824728}, {binary,1556880}, {code,7914132}, {ets,38474416}]}, {system_stats, [{cpu_utilization_rate,0.5006257822277848}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3443755}, {memory_data,{101368545280,100842303488,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 512588 kB\nBuffers: 283960 kB\nCached: 31393032 kB\nSwapCached: 3836 kB\nActive: 82930192 kB\nInactive: 13854916 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 512588 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 176 kB\nWriteback: 0 kB\nAnonPages: 65099964 kB\nMapped: 17876 kB\nSlab: 1381004 kB\nPageTables: 278880 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141214680 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32146464768}, {buffered_memory,290775040}, {free_memory,524890112}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3443745642,1}}, {context_switches,{751919632,0}}, {garbage_collection,{185844043,675369211278,0}}, {io,{{input,58292449431},{output,130597245042}}}, {reductions,{324853189884,1417802}}, {run_queue,0}, {runtime,{37335950,160}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 07:01:16 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 07:01:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,170906,408388}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102792776}, {processes,31790656}, {processes_used,29116384}, {system,71002120}, {atom,903241}, {atom_used,880290}, {binary,1715728}, {code,8604866}, {ets,38679440}]}, {system_stats, [{cpu_utilization_rate,12.875}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3443835}, {memory_data,{101368545280,100354207744,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1020340 kB\nBuffers: 283708 kB\nCached: 32676132 kB\nSwapCached: 5096 kB\nActive: 82554856 kB\nInactive: 13862216 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1020340 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 3916 kB\nWriteback: 0 kB\nAnonPages: 63449208 kB\nMapped: 23068 kB\nSlab: 1346584 kB\nPageTables: 172416 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86183776 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33460359168}, {buffered_memory,290516992}, {free_memory,1044955136}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3443820475,1}}, {context_switches,{1913439854,0}}, {garbage_collection,{248094257,3901281530837,0}}, {io,{{input,69582958303},{output,132019900290}}}, {reductions,{1586269789220,2410952}}, {run_queue,0}, {runtime,{130379600,180}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,170905,405255}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,84406704}, {processes,15003416}, {processes_used,14967720}, {system,69403288}, {atom,828337}, {atom_used,824728}, {binary,1223216}, {code,7914132}, {ets,38508872}]}, {system_stats, [{cpu_utilization_rate,15.125}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3443815}, {memory_data,{101368545280,100844130304,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 478320 kB\nBuffers: 283960 kB\nCached: 31393848 kB\nSwapCached: 3836 kB\nActive: 82961588 kB\nInactive: 13855616 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 478320 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 2884 kB\nWriteback: 0 kB\nAnonPages: 65131864 kB\nMapped: 19336 kB\nSlab: 1381524 kB\nPageTables: 279344 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141245636 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32147300352}, {buffered_memory,290775040}, {free_memory,489799680}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3443805641,1}}, {context_switches,{751932464,0}}, {garbage_collection,{185847257,675380589494,0}}, {io,{{input,58292642176},{output,130598153932}}}, {reductions,{324858945662,1403428}}, {run_queue,0}, {runtime,{37336570,170}}]}]}] INFO REPORT <5883.111.192> 2011-12-29 07:02:06 =============================================================================== ns_1@192.168.24.61:<5883.111.192>:stats_collector:83: Stats for bucket "JDPCachePrimary": accepting_conns 1 auth_cmds 10369 auth_errors 0 bucket_active_conns 1 bucket_conns 1 bytes_read 53441640750 bytes_written 45360068821 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 29975715 cmd_set 445274018 conn_yields 61205 connection_structures 1931 curr_connections 11 curr_items 297280049 curr_items_tot 297280049 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 466 delete_misses 174808 ep_bg_fetched 1661206 ep_bg_load 4365803470 ep_bg_load_avg 2628 ep_bg_max_load 2020165 ep_bg_max_wait 2929571 ep_bg_min_load 1 ep_bg_min_wait 10 ep_bg_num_samples 1661198 ep_bg_wait 48864908595 ep_bg_wait_avg 29415 ep_commit_num 818500 ep_commit_time 0 ep_commit_time_total 179775 ep_data_age 0 ep_data_age_highwat 2571 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 1 ep_dbname /opt/membase/var/lib/membase/data/JDPCachePrimary-data/JDPCachePrimary ep_dbshards 4 ep_diskqueue_drain 358215763 ep_diskqueue_fill 358215763 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 1256 ep_flush_duration_total 251908 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 1661206 ep_io_num_write 357979281 ep_io_read_bytes 145605718 ep_io_write_bytes 6012819006520 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 356086874 ep_kv_size 57356020515 ep_latency_arith_cmd 46093 ep_latency_get_cmd 101813590 ep_latency_store_cmd 515449025 ep_max_data_size 81788928000 ep_max_txn_size 1000 ep_mem_high_wat 61341696000 ep_mem_low_wat 49073356800 ep_min_data_age 0 ep_num_active_non_resident 94237869 ep_num_checkpoint_remover_runs355054 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 490 ep_num_non_resident 94237869 ep_num_not_my_vbuckets 0 ep_num_pager_runs 2 ep_num_value_ejects 95895079 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 1611440768 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 3597 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 62797498 ep_too_young 0 ep_total_cache_size 58930442359 ep_total_del_items 416 ep_total_enqueued 358215763 ep_total_new_items 297280465 ep_total_persisted 357979697 ep_uncommitted_items 0 ep_value_size 15808691965 ep_vb_total 512 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.1.1_39_g4bd26a3 ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 8853 get_hits 7311306 get_misses 22664409 incr_hits 44431 incr_misses 1662 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 58967461283 pid 17177 pointer_size 64 rejected_conns 0 rusage_system 91969.444519 rusage_user 249504.381568 threads 4 time 1325170925 total_connections 30202 uptime 3443856 vb_active_curr_items 297280049 vb_active_eject 95895079 vb_active_ht_memory 1610809344 vb_active_itm_memory 53788643543 vb_active_num 512 vb_active_num_non_resident 94237869 vb_active_ops_create 297280465 vb_active_ops_delete 416 vb_active_ops_reject 0 vb_active_ops_update 60698816 vb_active_perc_mem_resident 68 vb_active_queue_age 0 vb_active_queue_drain 358215763 vb_active_queue_fill 358215763 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version UNKNOWN INFO REPORT <5883.136.0> 2011-12-29 07:02:11 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 07:02:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,170966,409198}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102680552}, {processes,31569064}, {processes_used,28894792}, {system,71111488}, {atom,903241}, {atom_used,880290}, {binary,1675160}, {code,8604866}, {ets,38828936}]}, {system_stats, [{cpu_utilization_rate,0.375}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3443895}, {memory_data,{101368545280,100268183552,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1075048 kB\nBuffers: 283716 kB\nCached: 32673936 kB\nSwapCached: 5096 kB\nActive: 82503880 kB\nInactive: 13860200 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1075048 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 880 kB\nWriteback: 0 kB\nAnonPages: 63398268 kB\nMapped: 16728 kB\nSlab: 1346424 kB\nPageTables: 171512 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86139760 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33458110464}, {buffered_memory,290525184}, {free_memory,1100849152}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3443880476,2}}, {context_switches,{1913457112,0}}, {garbage_collection,{248097599,3901295214936,0}}, {io,{{input,69585630686},{output,132032072066}}}, {reductions,{1586281984460,2527516}}, {run_queue,1}, {runtime,{130380670,200}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,170965,405229}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,83924376}, {processes,14272880}, {processes_used,14237184}, {system,69651496}, {atom,828337}, {atom_used,824728}, {binary,1323048}, {code,7914132}, {ets,38657160}]}, {system_stats, [{cpu_utilization_rate,0.375}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3443875}, {memory_data,{101368545280,100903178240,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 515528 kB\nBuffers: 283972 kB\nCached: 31391672 kB\nSwapCached: 3836 kB\nActive: 82926708 kB\nInactive: 13853468 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 515528 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 484 kB\nWriteback: 0 kB\nAnonPages: 65096600 kB\nMapped: 17876 kB\nSlab: 1381376 kB\nPageTables: 278864 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141208356 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32145072128}, {buffered_memory,290787328}, {free_memory,527900672}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3443865641,0}}, {context_switches,{751945988,0}}, {garbage_collection,{185850709,675393491834,0}}, {io,{{input,58295251050},{output,130610315598}}}, {reductions,{324864571433,1425949}}, {run_queue,0}, {runtime,{37337450,160}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 07:03:01 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 07:03:18 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 07:03:25 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.111.192> 2011-12-29 07:03:46 =============================================================================== ns_1@192.168.24.61:<5883.111.192>:stats_collector:83: Stats for bucket "JDPCachePrimary": accepting_conns 1 auth_cmds 10369 auth_errors 0 bucket_active_conns 1 bucket_conns 1 bytes_read 53441647910 bytes_written 45361736961 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 29975715 cmd_set 445274018 conn_yields 61205 connection_structures 1931 curr_connections 11 curr_items 297280049 curr_items_tot 297280049 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 466 delete_misses 174808 ep_bg_fetched 1661206 ep_bg_load 4365803470 ep_bg_load_avg 2628 ep_bg_max_load 2020165 ep_bg_max_wait 2929571 ep_bg_min_load 1 ep_bg_min_wait 10 ep_bg_num_samples 1661198 ep_bg_wait 48864908595 ep_bg_wait_avg 29415 ep_commit_num 818500 ep_commit_time 0 ep_commit_time_total 179775 ep_data_age 0 ep_data_age_highwat 2571 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 1 ep_dbname /opt/membase/var/lib/membase/data/JDPCachePrimary-data/JDPCachePrimary ep_dbshards 4 ep_diskqueue_drain 358215763 ep_diskqueue_fill 358215763 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 1256 ep_flush_duration_total 251908 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 1661206 ep_io_num_write 357979281 ep_io_read_bytes 145605718 ep_io_write_bytes 6012819006520 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 356086874 ep_kv_size 57356020515 ep_latency_arith_cmd 46093 ep_latency_get_cmd 101813590 ep_latency_store_cmd 515449025 ep_max_data_size 81788928000 ep_max_txn_size 1000 ep_mem_high_wat 61341696000 ep_mem_low_wat 49073356800 ep_min_data_age 0 ep_num_active_non_resident 94237869 ep_num_checkpoint_remover_runs355074 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 490 ep_num_non_resident 94237869 ep_num_not_my_vbuckets 0 ep_num_pager_runs 2 ep_num_value_ejects 95895079 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 1611440768 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 3597 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 62797498 ep_too_young 0 ep_total_cache_size 58930442359 ep_total_del_items 416 ep_total_enqueued 358215763 ep_total_new_items 297280465 ep_total_persisted 357979697 ep_uncommitted_items 0 ep_value_size 15808691965 ep_vb_total 512 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.1.1_39_g4bd26a3 ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 8853 get_hits 7311306 get_misses 22664409 incr_hits 44431 incr_misses 1662 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 58967461283 pid 17177 pointer_size 64 rejected_conns 0 rusage_system 91969.478514 rusage_user 249504.481553 threads 4 time 1325171025 total_connections 30202 uptime 3443956 vb_active_curr_items 297280049 vb_active_eject 95895079 vb_active_ht_memory 1610809344 vb_active_itm_memory 53788643543 vb_active_num 512 vb_active_num_non_resident 94237869 vb_active_ops_create 297280465 vb_active_ops_delete 416 vb_active_ops_reject 0 vb_active_ops_update 60698816 vb_active_perc_mem_resident 68 vb_active_queue_age 0 vb_active_queue_drain 358215763 vb_active_queue_fill 358215763 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version UNKNOWN INFO REPORT <5883.1095.0> 2011-12-29 07:03:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,171026,409326}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,103173992}, {processes,32012000}, {processes_used,29337728}, {system,71161992}, {atom,903241}, {atom_used,880290}, {binary,1692008}, {code,8604866}, {ets,38862248}]}, {system_stats, [{cpu_utilization_rate,0.37546933667083854}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3443955}, {memory_data,{101368545280,100268191744,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1074680 kB\nBuffers: 283716 kB\nCached: 32674712 kB\nSwapCached: 5096 kB\nActive: 82504396 kB\nInactive: 13860936 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1074680 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 16 kB\nWriteback: 188 kB\nAnonPages: 63399192 kB\nMapped: 16728 kB\nSlab: 1346180 kB\nPageTables: 171516 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86136236 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33458905088}, {buffered_memory,290525184}, {free_memory,1100472320}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3443940475,0}}, {context_switches,{1913473432,0}}, {garbage_collection,{248100714,3901307250806,0}}, {io,{{input,69585880188},{output,132032919808}}}, {reductions,{1586293843989,2441835}}, {run_queue,0}, {runtime,{130381460,180}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,171025,404841}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,84353912}, {processes,14739520}, {processes_used,14703824}, {system,69614392}, {atom,828337}, {atom_used,824728}, {binary,1252552}, {code,7914132}, {ets,38690256}]}, {system_stats, [{cpu_utilization_rate,0.4993757802746567}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3443935}, {memory_data,{101368545280,100839735296,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 515540 kB\nBuffers: 283972 kB\nCached: 31392436 kB\nSwapCached: 3836 kB\nActive: 82926388 kB\nInactive: 13854180 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 515540 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 172 kB\nWriteback: 0 kB\nAnonPages: 65096536 kB\nMapped: 17876 kB\nSlab: 1380968 kB\nPageTables: 278864 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141212700 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32145854464}, {buffered_memory,290787328}, {free_memory,527912960}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3443925641,1}}, {context_switches,{751958673,0}}, {garbage_collection,{185853938,675404696430,0}}, {io,{{input,58295420139},{output,130611244509}}}, {reductions,{324870054706,1387585}}, {run_queue,0}, {runtime,{37338060,150}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 07:04:09 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 07:04:37 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 07:04:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,171086,409283}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,103354968}, {processes,31989464}, {processes_used,29315192}, {system,71365504}, {atom,903241}, {atom_used,880290}, {binary,1749240}, {code,8604866}, {ets,39008408}]}, {system_stats, [{cpu_utilization_rate,0.375}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3444015}, {memory_data,{101368545280,100268269568,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1074112 kB\nBuffers: 283716 kB\nCached: 32675484 kB\nSwapCached: 5096 kB\nActive: 82504116 kB\nInactive: 13861676 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1074112 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 488 kB\nWriteback: 0 kB\nAnonPages: 63398532 kB\nMapped: 16728 kB\nSlab: 1346108 kB\nPageTables: 171512 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86137260 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33459695616}, {buffered_memory,290525184}, {free_memory,1099890688}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3444000475,0}}, {context_switches,{1913489659,0}}, {garbage_collection,{248103847,3901319255939,0}}, {io,{{input,69586067302},{output,132033763050}}}, {reductions,{1586305780224,2392456}}, {run_queue,0}, {runtime,{130382280,190}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,171085,405555}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,84131400}, {processes,14473792}, {processes_used,14438240}, {system,69657608}, {atom,828337}, {atom_used,824728}, {binary,1143816}, {code,7914132}, {ets,38842456}]}, {system_stats, [{cpu_utilization_rate,0.6242197253433208}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3443995}, {memory_data,{101368545280,100841267200,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 514740 kB\nBuffers: 283972 kB\nCached: 31393208 kB\nSwapCached: 3836 kB\nActive: 82926980 kB\nInactive: 13854912 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 514740 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 484 kB\nWriteback: 0 kB\nAnonPages: 65096888 kB\nMapped: 17876 kB\nSlab: 1380920 kB\nPageTables: 278864 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141214240 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32146644992}, {buffered_memory,290787328}, {free_memory,527093760}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3443985641,0}}, {context_switches,{751971336,0}}, {garbage_collection,{185857156,675415871556,0}}, {io,{{input,58295579447},{output,130612083676}}}, {reductions,{324875557160,1405115}}, {run_queue,0}, {runtime,{37338670,150}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 07:05:15 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.111.192> 2011-12-29 07:05:26 =============================================================================== ns_1@192.168.24.61:<5883.111.192>:stats_collector:83: Stats for bucket "JDPCachePrimary": accepting_conns 1 auth_cmds 10369 auth_errors 0 bucket_active_conns 1 bucket_conns 1 bytes_read 53441655070 bytes_written 45363405101 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 29975715 cmd_set 445274018 conn_yields 61205 connection_structures 1931 curr_connections 11 curr_items 297280049 curr_items_tot 297280049 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 466 delete_misses 174808 ep_bg_fetched 1661206 ep_bg_load 4365803470 ep_bg_load_avg 2628 ep_bg_max_load 2020165 ep_bg_max_wait 2929571 ep_bg_min_load 1 ep_bg_min_wait 10 ep_bg_num_samples 1661198 ep_bg_wait 48864908595 ep_bg_wait_avg 29415 ep_commit_num 818500 ep_commit_time 0 ep_commit_time_total 179775 ep_data_age 0 ep_data_age_highwat 2571 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 1 ep_dbname /opt/membase/var/lib/membase/data/JDPCachePrimary-data/JDPCachePrimary ep_dbshards 4 ep_diskqueue_drain 358215763 ep_diskqueue_fill 358215763 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 1256 ep_flush_duration_total 251908 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 1661206 ep_io_num_write 357979281 ep_io_read_bytes 145605718 ep_io_write_bytes 6012819006520 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 356086874 ep_kv_size 57356020515 ep_latency_arith_cmd 46093 ep_latency_get_cmd 101813590 ep_latency_store_cmd 515449025 ep_max_data_size 81788928000 ep_max_txn_size 1000 ep_mem_high_wat 61341696000 ep_mem_low_wat 49073356800 ep_min_data_age 0 ep_num_active_non_resident 94237869 ep_num_checkpoint_remover_runs355094 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 490 ep_num_non_resident 94237869 ep_num_not_my_vbuckets 0 ep_num_pager_runs 2 ep_num_value_ejects 95895079 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 1611440768 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 3597 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 62797498 ep_too_young 0 ep_total_cache_size 58930442359 ep_total_del_items 416 ep_total_enqueued 358215763 ep_total_new_items 297280465 ep_total_persisted 357979697 ep_uncommitted_items 0 ep_value_size 15808691965 ep_vb_total 512 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.1.1_39_g4bd26a3 ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 8853 get_hits 7311306 get_misses 22664409 incr_hits 44431 incr_misses 1662 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 58967461283 pid 17177 pointer_size 64 rejected_conns 0 rusage_system 91969.507509 rusage_user 249504.583537 threads 4 time 1325171125 total_connections 30202 uptime 3444056 vb_active_curr_items 297280049 vb_active_eject 95895079 vb_active_ht_memory 1610809344 vb_active_itm_memory 53788643543 vb_active_num 512 vb_active_num_non_resident 94237869 vb_active_ops_create 297280465 vb_active_ops_delete 416 vb_active_ops_reject 0 vb_active_ops_update 60698816 vb_active_perc_mem_resident 68 vb_active_queue_age 0 vb_active_queue_drain 358215763 vb_active_queue_fill 358215763 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version UNKNOWN INFO REPORT <5883.136.0> 2011-12-29 07:05:30 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 07:05:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,171146,409187}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,103395008}, {processes,31932280}, {processes_used,29258008}, {system,71462728}, {atom,903241}, {atom_used,880290}, {binary,1697720}, {code,8604866}, {ets,39157568}]}, {system_stats, [{cpu_utilization_rate,15.625}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3444075}, {memory_data,{101368545280,100268621824,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1060220 kB\nBuffers: 283720 kB\nCached: 32675780 kB\nSwapCached: 5096 kB\nActive: 82516384 kB\nInactive: 13861816 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1060220 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 608 kB\nWriteback: 0 kB\nAnonPages: 63410964 kB\nMapped: 18180 kB\nSlab: 1346260 kB\nPageTables: 171840 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86151452 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33459998720}, {buffered_memory,290529280}, {free_memory,1085665280}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3444060475,0}}, {context_switches,{1913506701,0}}, {garbage_collection,{248107186,3901332089573,0}}, {io,{{input,69588792876},{output,132037230613}}}, {reductions,{1586317868960,2429025}}, {run_queue,0}, {runtime,{130383160,200}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,171145,405778}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,84690416}, {processes,14779032}, {processes_used,14743336}, {system,69911384}, {atom,828337}, {atom_used,824728}, {binary,1312296}, {code,7914132}, {ets,38927744}]}, {system_stats, [{cpu_utilization_rate,12.781954887218046}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3444055}, {memory_data,{101368545280,100841439232,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 448592 kB\nBuffers: 283976 kB\nCached: 31393524 kB\nSwapCached: 3836 kB\nActive: 82992152 kB\nInactive: 13855328 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 448592 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 3492 kB\nWriteback: 0 kB\nAnonPages: 65162092 kB\nMapped: 25068 kB\nSlab: 1381232 kB\nPageTables: 279728 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141274300 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32146968576}, {buffered_memory,290791424}, {free_memory,459358208}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3444045641,1}}, {context_switches,{751985125,0}}, {garbage_collection,{185860581,675427655723,0}}, {io,{{input,58298191948},{output,130615633142}}}, {reductions,{324881410803,1408351}}, {run_queue,1}, {runtime,{37339400,140}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 07:05:56 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 07:06:18 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 07:06:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,171206,408695}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,103207888}, {processes,32104200}, {processes_used,29429928}, {system,71103688}, {atom,903241}, {atom_used,880290}, {binary,1807976}, {code,8604866}, {ets,38687648}]}, {system_stats, [{cpu_utilization_rate,0.625}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3444135}, {memory_data,{101368545280,100377894912,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1069276 kB\nBuffers: 283720 kB\nCached: 32676640 kB\nSwapCached: 5096 kB\nActive: 82504108 kB\nInactive: 13862468 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1069276 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 4204 kB\nWriteback: 24 kB\nAnonPages: 63398588 kB\nMapped: 16728 kB\nSlab: 1347304 kB\nPageTables: 171520 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86136360 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33460879360}, {buffered_memory,290529280}, {free_memory,1094938624}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3444120474,0}}, {context_switches,{1913522872,0}}, {garbage_collection,{248110300,3901343965155,0}}, {io,{{input,69589008958},{output,132038109363}}}, {reductions,{1586329766970,2411840}}, {run_queue,0}, {runtime,{130383990,210}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,171205,405283}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,83902816}, {processes,14441360}, {processes_used,14405664}, {system,69461456}, {atom,828337}, {atom_used,824728}, {binary,1272880}, {code,7914132}, {ets,38517304}]}, {system_stats, [{cpu_utilization_rate,0.5}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3444115}, {memory_data,{101368545280,100943572992,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 513252 kB\nBuffers: 283976 kB\nCached: 31394376 kB\nSwapCached: 3836 kB\nActive: 82927176 kB\nInactive: 13855992 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 513252 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 824 kB\nWriteback: 0 kB\nAnonPages: 65097184 kB\nMapped: 17876 kB\nSlab: 1381352 kB\nPageTables: 278864 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141211600 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32147841024}, {buffered_memory,290791424}, {free_memory,525570048}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3444105641,0}}, {context_switches,{751997715,0}}, {garbage_collection,{185863799,675438862254,0}}, {io,{{input,58298393570},{output,130616501618}}}, {reductions,{324886890623,1386536}}, {run_queue,0}, {runtime,{37339980,140}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 07:06:56 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.111.192> 2011-12-29 07:07:06 =============================================================================== ns_1@192.168.24.61:<5883.111.192>:stats_collector:83: Stats for bucket "JDPCachePrimary": accepting_conns 1 auth_cmds 10369 auth_errors 0 bucket_active_conns 1 bucket_conns 1 bytes_read 53441662230 bytes_written 45365073241 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 29975715 cmd_set 445274018 conn_yields 61205 connection_structures 1931 curr_connections 11 curr_items 297280049 curr_items_tot 297280049 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 466 delete_misses 174808 ep_bg_fetched 1661206 ep_bg_load 4365803470 ep_bg_load_avg 2628 ep_bg_max_load 2020165 ep_bg_max_wait 2929571 ep_bg_min_load 1 ep_bg_min_wait 10 ep_bg_num_samples 1661198 ep_bg_wait 48864908595 ep_bg_wait_avg 29415 ep_commit_num 818500 ep_commit_time 0 ep_commit_time_total 179775 ep_data_age 0 ep_data_age_highwat 2571 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 1 ep_dbname /opt/membase/var/lib/membase/data/JDPCachePrimary-data/JDPCachePrimary ep_dbshards 4 ep_diskqueue_drain 358215763 ep_diskqueue_fill 358215763 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 1256 ep_flush_duration_total 251908 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 1661206 ep_io_num_write 357979281 ep_io_read_bytes 145605718 ep_io_write_bytes 6012819006520 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 356086874 ep_kv_size 57356020515 ep_latency_arith_cmd 46093 ep_latency_get_cmd 101813590 ep_latency_store_cmd 515449025 ep_max_data_size 81788928000 ep_max_txn_size 1000 ep_mem_high_wat 61341696000 ep_mem_low_wat 49073356800 ep_min_data_age 0 ep_num_active_non_resident 94237869 ep_num_checkpoint_remover_runs355114 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 490 ep_num_non_resident 94237869 ep_num_not_my_vbuckets 0 ep_num_pager_runs 2 ep_num_value_ejects 95895079 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 1611440768 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 3597 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 62797498 ep_too_young 0 ep_total_cache_size 58930442359 ep_total_del_items 416 ep_total_enqueued 358215763 ep_total_new_items 297280465 ep_total_persisted 357979697 ep_uncommitted_items 0 ep_value_size 15808691965 ep_vb_total 512 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.1.1_39_g4bd26a3 ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 8853 get_hits 7311306 get_misses 22664409 incr_hits 44431 incr_misses 1662 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 58967461283 pid 17177 pointer_size 64 rejected_conns 0 rusage_system 91969.544504 rusage_user 249504.678523 threads 4 time 1325171225 total_connections 30202 uptime 3444156 vb_active_curr_items 297280049 vb_active_eject 95895079 vb_active_ht_memory 1610809344 vb_active_itm_memory 53788643543 vb_active_num 512 vb_active_num_non_resident 94237869 vb_active_ops_create 297280465 vb_active_ops_delete 416 vb_active_ops_reject 0 vb_active_ops_update 60698816 vb_active_perc_mem_resident 68 vb_active_queue_age 0 vb_active_queue_drain 358215763 vb_active_queue_fill 358215763 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version UNKNOWN INFO REPORT <5883.136.0> 2011-12-29 07:07:07 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 07:07:32 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 07:07:49 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 07:07:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,171266,409258}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102833976}, {processes,31946720}, {processes_used,29272448}, {system,70887256}, {atom,903241}, {atom_used,880290}, {binary,1557424}, {code,8604866}, {ets,38721336}]}, {system_stats, [{cpu_utilization_rate,0.375}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3444195}, {memory_data,{101368545280,100271906816,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1070580 kB\nBuffers: 283720 kB\nCached: 32677456 kB\nSwapCached: 5096 kB\nActive: 82504776 kB\nInactive: 13863244 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1070580 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 576 kB\nWriteback: 0 kB\nAnonPages: 63399088 kB\nMapped: 16728 kB\nSlab: 1346208 kB\nPageTables: 171520 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86135916 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33461714944}, {buffered_memory,290529280}, {free_memory,1096273920}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3444180475,1}}, {context_switches,{1913539733,0}}, {garbage_collection,{248113479,3901356097592,0}}, {io,{{input,69589221124},{output,132038985445}}}, {reductions,{1586342212473,2419258}}, {run_queue,0}, {runtime,{130384800,180}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,171265,405128}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,84088504}, {processes,14516328}, {processes_used,14480632}, {system,69572176}, {atom,828337}, {atom_used,824728}, {binary,1351928}, {code,7914132}, {ets,38548616}]}, {system_stats, [{cpu_utilization_rate,0.4993757802746567}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3444175}, {memory_data,{101368545280,100842115072,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 514204 kB\nBuffers: 283976 kB\nCached: 31395156 kB\nSwapCached: 3836 kB\nActive: 82926400 kB\nInactive: 13856840 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 514204 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 544 kB\nWriteback: 0 kB\nAnonPages: 65095988 kB\nMapped: 17876 kB\nSlab: 1381000 kB\nPageTables: 278860 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141210712 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32148639744}, {buffered_memory,290791424}, {free_memory,526671872}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3444165641,0}}, {context_switches,{752010609,0}}, {garbage_collection,{185866893,675450037050,0}}, {io,{{input,58298551215},{output,130617441105}}}, {reductions,{324892577225,1384813}}, {run_queue,0}, {runtime,{37340580,170}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 07:08:42 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.111.192> 2011-12-29 07:08:46 =============================================================================== ns_1@192.168.24.61:<5883.111.192>:stats_collector:83: Stats for bucket "JDPCachePrimary": accepting_conns 1 auth_cmds 10369 auth_errors 0 bucket_active_conns 1 bucket_conns 1 bytes_read 53441669390 bytes_written 45366741381 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 29975715 cmd_set 445274018 conn_yields 61205 connection_structures 1931 curr_connections 11 curr_items 297280049 curr_items_tot 297280049 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 466 delete_misses 174808 ep_bg_fetched 1661206 ep_bg_load 4365803470 ep_bg_load_avg 2628 ep_bg_max_load 2020165 ep_bg_max_wait 2929571 ep_bg_min_load 1 ep_bg_min_wait 10 ep_bg_num_samples 1661198 ep_bg_wait 48864908595 ep_bg_wait_avg 29415 ep_commit_num 818500 ep_commit_time 0 ep_commit_time_total 179775 ep_data_age 0 ep_data_age_highwat 2571 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 1 ep_dbname /opt/membase/var/lib/membase/data/JDPCachePrimary-data/JDPCachePrimary ep_dbshards 4 ep_diskqueue_drain 358215763 ep_diskqueue_fill 358215763 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 1256 ep_flush_duration_total 251908 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 1661206 ep_io_num_write 357979281 ep_io_read_bytes 145605718 ep_io_write_bytes 6012819006520 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 356086874 ep_kv_size 57356020515 ep_latency_arith_cmd 46093 ep_latency_get_cmd 101813590 ep_latency_store_cmd 515449025 ep_max_data_size 81788928000 ep_max_txn_size 1000 ep_mem_high_wat 61341696000 ep_mem_low_wat 49073356800 ep_min_data_age 0 ep_num_active_non_resident 94237869 ep_num_checkpoint_remover_runs355134 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 490 ep_num_non_resident 94237869 ep_num_not_my_vbuckets 0 ep_num_pager_runs 2 ep_num_value_ejects 95895079 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 1611440768 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 3597 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 62797498 ep_too_young 0 ep_total_cache_size 58930442359 ep_total_del_items 416 ep_total_enqueued 358215763 ep_total_new_items 297280465 ep_total_persisted 357979697 ep_uncommitted_items 0 ep_value_size 15808691965 ep_vb_total 512 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.1.1_39_g4bd26a3 ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 8853 get_hits 7311306 get_misses 22664409 incr_hits 44431 incr_misses 1662 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 58967461283 pid 17177 pointer_size 64 rejected_conns 0 rusage_system 91969.568500 rusage_user 249504.776508 threads 4 time 1325171325 total_connections 30202 uptime 3444256 vb_active_curr_items 297280049 vb_active_eject 95895079 vb_active_ht_memory 1610809344 vb_active_itm_memory 53788643543 vb_active_num 512 vb_active_num_non_resident 94237869 vb_active_ops_create 297280465 vb_active_ops_delete 416 vb_active_ops_reject 0 vb_active_ops_update 60698816 vb_active_perc_mem_resident 68 vb_active_queue_age 0 vb_active_queue_drain 358215763 vb_active_queue_fill 358215763 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version UNKNOWN INFO REPORT <5883.1095.0> 2011-12-29 07:08:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,171326,409007}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102762592}, {processes,32039152}, {processes_used,29364880}, {system,70723440}, {atom,903241}, {atom_used,880290}, {binary,1245000}, {code,8604866}, {ets,38870192}]}, {system_stats, [{cpu_utilization_rate,0.2503128911138924}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3444255}, {memory_data,{101368545280,100265418752,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1076816 kB\nBuffers: 283728 kB\nCached: 32672700 kB\nSwapCached: 5096 kB\nActive: 82504584 kB\nInactive: 13858736 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1076816 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 332 kB\nWriteback: 0 kB\nAnonPages: 63399304 kB\nMapped: 16728 kB\nSlab: 1346128 kB\nPageTables: 171520 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86136352 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33456844800}, {buffered_memory,290537472}, {free_memory,1102659584}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3444240476,1}}, {context_switches,{1913556802,0}}, {garbage_collection,{248116905,3901369471044,0}}, {io,{{input,69591928949},{output,132045986150}}}, {reductions,{1586354366217,2488569}}, {run_queue,0}, {runtime,{130385770,190}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,171325,405904}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,84339704}, {processes,14654744}, {processes_used,14619048}, {system,69684960}, {atom,828337}, {atom_used,824728}, {binary,1303760}, {code,7914132}, {ets,38710104}]}, {system_stats, [{cpu_utilization_rate,0.2503128911138924}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3444235}, {memory_data,{101368545280,100841426944,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 519344 kB\nBuffers: 283976 kB\nCached: 31390392 kB\nSwapCached: 3836 kB\nActive: 82925780 kB\nInactive: 13852120 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 519344 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 336 kB\nWriteback: 0 kB\nAnonPages: 65095460 kB\nMapped: 17876 kB\nSlab: 1381004 kB\nPageTables: 278860 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141210128 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32143761408}, {buffered_memory,290791424}, {free_memory,531808256}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3444225642,1}}, {context_switches,{752024040,0}}, {garbage_collection,{185870238,675462476107,0}}, {io,{{input,58301168081},{output,130624436645}}}, {reductions,{324898160726,1393651}}, {run_queue,0}, {runtime,{37341330,160}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 07:09:06 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 07:09:27 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 07:09:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,171386,409206}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102479440}, {processes,31744168}, {processes_used,29069896}, {system,70735272}, {atom,903241}, {atom_used,880290}, {binary,1223768}, {code,8604866}, {ets,38902920}]}, {system_stats, [{cpu_utilization_rate,0.37546933667083854}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3444315}, {memory_data,{101368545280,100264570880,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1078004 kB\nBuffers: 283728 kB\nCached: 32673472 kB\nSwapCached: 5096 kB\nActive: 82502592 kB\nInactive: 13859452 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1078004 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 188 kB\nWriteback: 20 kB\nAnonPages: 63397220 kB\nMapped: 16728 kB\nSlab: 1346124 kB\nPageTables: 171512 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86136320 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33457635328}, {buffered_memory,290537472}, {free_memory,1103876096}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3444300475,0}}, {context_switches,{1913573149,0}}, {garbage_collection,{248120499,3901381826753,0}}, {io,{{input,69592180274},{output,132046880640}}}, {reductions,{1586366336509,2437770}}, {run_queue,0}, {runtime,{130386590,190}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,171385,405017}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,84029456}, {processes,14225032}, {processes_used,14189336}, {system,69804424}, {atom,828337}, {atom_used,824728}, {binary,1401024}, {code,7914132}, {ets,38731688}]}, {system_stats, [{cpu_utilization_rate,0.37546933667083854}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3444295}, {memory_data,{101368545280,100836384768,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 518044 kB\nBuffers: 283976 kB\nCached: 31391164 kB\nSwapCached: 3836 kB\nActive: 82926988 kB\nInactive: 13852912 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 518044 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 168 kB\nWriteback: 0 kB\nAnonPages: 65097144 kB\nMapped: 17876 kB\nSlab: 1380988 kB\nPageTables: 278864 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141211908 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32144551936}, {buffered_memory,290791424}, {free_memory,530477056}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3444285641,0}}, {context_switches,{752036736,0}}, {garbage_collection,{185873320,675473652277,0}}, {io,{{input,58301364845},{output,130625365827}}}, {reductions,{324903618855,1388099}}, {run_queue,0}, {runtime,{37341910,160}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 07:10:00 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.111.192> 2011-12-29 07:10:26 =============================================================================== ns_1@192.168.24.61:<5883.111.192>:stats_collector:83: Stats for bucket "JDPCachePrimary": accepting_conns 1 auth_cmds 10369 auth_errors 0 bucket_active_conns 1 bucket_conns 1 bytes_read 53441676550 bytes_written 45368409521 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 29975715 cmd_set 445274018 conn_yields 61205 connection_structures 1931 curr_connections 11 curr_items 297280049 curr_items_tot 297280049 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 466 delete_misses 174808 ep_bg_fetched 1661206 ep_bg_load 4365803470 ep_bg_load_avg 2628 ep_bg_max_load 2020165 ep_bg_max_wait 2929571 ep_bg_min_load 1 ep_bg_min_wait 10 ep_bg_num_samples 1661198 ep_bg_wait 48864908595 ep_bg_wait_avg 29415 ep_commit_num 818500 ep_commit_time 0 ep_commit_time_total 179775 ep_data_age 0 ep_data_age_highwat 2571 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 1 ep_dbname /opt/membase/var/lib/membase/data/JDPCachePrimary-data/JDPCachePrimary ep_dbshards 4 ep_diskqueue_drain 358215763 ep_diskqueue_fill 358215763 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 1256 ep_flush_duration_total 251908 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 1661206 ep_io_num_write 357979281 ep_io_read_bytes 145605718 ep_io_write_bytes 6012819006520 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 356086874 ep_kv_size 57356020515 ep_latency_arith_cmd 46093 ep_latency_get_cmd 101813590 ep_latency_store_cmd 515449025 ep_max_data_size 81788928000 ep_max_txn_size 1000 ep_mem_high_wat 61341696000 ep_mem_low_wat 49073356800 ep_min_data_age 0 ep_num_active_non_resident 94237869 ep_num_checkpoint_remover_runs355154 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 490 ep_num_non_resident 94237869 ep_num_not_my_vbuckets 0 ep_num_pager_runs 2 ep_num_value_ejects 95895079 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 1611440768 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 3597 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 62797498 ep_too_young 0 ep_total_cache_size 58930442359 ep_total_del_items 416 ep_total_enqueued 358215763 ep_total_new_items 297280465 ep_total_persisted 357979697 ep_uncommitted_items 0 ep_value_size 15808691965 ep_vb_total 512 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.1.1_39_g4bd26a3 ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 8853 get_hits 7311306 get_misses 22664409 incr_hits 44431 incr_misses 1662 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 58967461283 pid 17177 pointer_size 64 rejected_conns 0 rusage_system 91969.590497 rusage_user 249504.868494 threads 4 time 1325171425 total_connections 30202 uptime 3444356 vb_active_curr_items 297280049 vb_active_eject 95895079 vb_active_ht_memory 1610809344 vb_active_itm_memory 53788643543 vb_active_num 512 vb_active_num_non_resident 94237869 vb_active_ops_create 297280465 vb_active_ops_delete 416 vb_active_ops_reject 0 vb_active_ops_update 60698816 vb_active_perc_mem_resident 68 vb_active_queue_age 0 vb_active_queue_drain 358215763 vb_active_queue_fill 358215763 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version UNKNOWN INFO REPORT <5883.136.0> 2011-12-29 07:10:37 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 07:10:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,171446,409060}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102675584}, {processes,31802016}, {processes_used,29127744}, {system,70873568}, {atom,903241}, {atom_used,880290}, {binary,1209336}, {code,8604866}, {ets,39055656}]}, {system_stats, [{cpu_utilization_rate,0.5}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3444375}, {memory_data,{101368545280,100265955328,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1076760 kB\nBuffers: 283728 kB\nCached: 32674248 kB\nSwapCached: 5096 kB\nActive: 82503216 kB\nInactive: 13860168 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1076760 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 400 kB\nWriteback: 300 kB\nAnonPages: 63397820 kB\nMapped: 16728 kB\nSlab: 1346060 kB\nPageTables: 171512 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86133856 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33458429952}, {buffered_memory,290537472}, {free_memory,1102602240}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3444360475,0}}, {context_switches,{1913589278,0}}, {garbage_collection,{248123766,3901393831921,0}}, {io,{{input,69592364742},{output,132047760867}}}, {reductions,{1586378326501,2452325}}, {run_queue,0}, {runtime,{130387360,180}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,171445,408637}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,84624560}, {processes,14766232}, {processes_used,14730536}, {system,69858328}, {atom,828337}, {atom_used,824728}, {binary,1302448}, {code,7914132}, {ets,38884136}]}, {system_stats, [{cpu_utilization_rate,12.8428927680798}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3444355}, {memory_data,{101368545280,100837830656,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 484648 kB\nBuffers: 283976 kB\nCached: 31391956 kB\nSwapCached: 3836 kB\nActive: 82957296 kB\nInactive: 13853640 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 484648 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 3252 kB\nWriteback: 0 kB\nAnonPages: 65126932 kB\nMapped: 23216 kB\nSlab: 1381332 kB\nPageTables: 279356 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141237168 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32145362944}, {buffered_memory,290791424}, {free_memory,496279552}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3444345641,0}}, {context_switches,{752049427,0}}, {garbage_collection,{185876480,675484707355,0}}, {io,{{input,58301564975},{output,130626219007}}}, {reductions,{324909066104,1393357}}, {run_queue,0}, {runtime,{37342540,170}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 07:11:27 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 07:11:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,171506,409203}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102981576}, {processes,31972256}, {processes_used,29297984}, {system,71009320}, {atom,903241}, {atom_used,880290}, {binary,1196616}, {code,8604866}, {ets,39204456}]}, {system_stats, [{cpu_utilization_rate,12.875}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3444435}, {memory_data,{101368545280,100320882688,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1000992 kB\nBuffers: 283728 kB\nCached: 32674564 kB\nSwapCached: 5096 kB\nActive: 82575928 kB\nInactive: 13860544 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1000992 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 3120 kB\nWriteback: 92 kB\nAnonPages: 63470656 kB\nMapped: 23668 kB\nSlab: 1346828 kB\nPageTables: 172164 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86207676 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33458753536}, {buffered_memory,290537472}, {free_memory,1025015808}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3444420475,1}}, {context_switches,{1913605989,0}}, {garbage_collection,{248127306,3901406456838,0}}, {io,{{input,69595048165},{output,132051245054}}}, {reductions,{1586390409144,2442140}}, {run_queue,0}, {runtime,{130388250,190}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,171505,405427}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,84412736}, {processes,14608752}, {processes_used,14573056}, {system,69803984}, {atom,828337}, {atom_used,824728}, {binary,1157992}, {code,7914132}, {ets,38974112}]}, {system_stats, [{cpu_utilization_rate,0.625}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3444415}, {memory_data,{101368545280,100913618944,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 515348 kB\nBuffers: 283980 kB\nCached: 31392368 kB\nSwapCached: 3836 kB\nActive: 82926352 kB\nInactive: 13853904 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 515348 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 1492 kB\nWriteback: 0 kB\nAnonPages: 65096172 kB\nMapped: 17876 kB\nSlab: 1381640 kB\nPageTables: 278860 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141209116 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32145784832}, {buffered_memory,290795520}, {free_memory,527716352}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3444405640,0}}, {context_switches,{752063120,0}}, {garbage_collection,{185879955,675496611301,0}}, {io,{{input,58304210145},{output,130629708227}}}, {reductions,{324914926474,1384122}}, {run_queue,0}, {runtime,{37343220,140}}]}]}] INFO REPORT <5883.111.192> 2011-12-29 07:12:06 =============================================================================== ns_1@192.168.24.61:<5883.111.192>:stats_collector:83: Stats for bucket "JDPCachePrimary": accepting_conns 1 auth_cmds 10369 auth_errors 0 bucket_active_conns 1 bucket_conns 1 bytes_read 53441683710 bytes_written 45370077661 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 29975715 cmd_set 445274018 conn_yields 61205 connection_structures 1931 curr_connections 11 curr_items 297280049 curr_items_tot 297280049 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 466 delete_misses 174808 ep_bg_fetched 1661206 ep_bg_load 4365803470 ep_bg_load_avg 2628 ep_bg_max_load 2020165 ep_bg_max_wait 2929571 ep_bg_min_load 1 ep_bg_min_wait 10 ep_bg_num_samples 1661198 ep_bg_wait 48864908595 ep_bg_wait_avg 29415 ep_commit_num 818500 ep_commit_time 0 ep_commit_time_total 179775 ep_data_age 0 ep_data_age_highwat 2571 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 1 ep_dbname /opt/membase/var/lib/membase/data/JDPCachePrimary-data/JDPCachePrimary ep_dbshards 4 ep_diskqueue_drain 358215763 ep_diskqueue_fill 358215763 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 1256 ep_flush_duration_total 251908 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 1661206 ep_io_num_write 357979281 ep_io_read_bytes 145605718 ep_io_write_bytes 6012819006520 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 356086874 ep_kv_size 57356020515 ep_latency_arith_cmd 46093 ep_latency_get_cmd 101813590 ep_latency_store_cmd 515449025 ep_max_data_size 81788928000 ep_max_txn_size 1000 ep_mem_high_wat 61341696000 ep_mem_low_wat 49073356800 ep_min_data_age 0 ep_num_active_non_resident 94237869 ep_num_checkpoint_remover_runs355174 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 490 ep_num_non_resident 94237869 ep_num_not_my_vbuckets 0 ep_num_pager_runs 2 ep_num_value_ejects 95895079 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 1611440768 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 3597 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 62797498 ep_too_young 0 ep_total_cache_size 58930442359 ep_total_del_items 416 ep_total_enqueued 358215763 ep_total_new_items 297280465 ep_total_persisted 357979697 ep_uncommitted_items 0 ep_value_size 15808691965 ep_vb_total 512 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.1.1_39_g4bd26a3 ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 8853 get_hits 7311306 get_misses 22664409 incr_hits 44431 incr_misses 1662 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 58967461283 pid 17177 pointer_size 64 rejected_conns 0 rusage_system 91969.618492 rusage_user 249504.979477 threads 4 time 1325171525 total_connections 30202 uptime 3444456 vb_active_curr_items 297280049 vb_active_eject 95895079 vb_active_ht_memory 1610809344 vb_active_itm_memory 53788643543 vb_active_num 512 vb_active_num_non_resident 94237869 vb_active_ops_create 297280465 vb_active_ops_delete 416 vb_active_ops_reject 0 vb_active_ops_update 60698816 vb_active_perc_mem_resident 68 vb_active_queue_age 0 vb_active_queue_drain 358215763 vb_active_queue_fill 358215763 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version UNKNOWN INFO REPORT <5883.136.0> 2011-12-29 07:12:13 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 07:12:44 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 07:12:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,171566,409217}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102049368}, {processes,31542528}, {processes_used,28868256}, {system,70506840}, {atom,903241}, {atom_used,880290}, {binary,1163424}, {code,8604866}, {ets,38735088}]}, {system_stats, [{cpu_utilization_rate,0.5006257822277848}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3444495}, {memory_data,{101368545280,100270198784,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1073132 kB\nBuffers: 283728 kB\nCached: 32675412 kB\nSwapCached: 5096 kB\nActive: 82504220 kB\nInactive: 13861248 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1073132 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 160 kB\nWriteback: 128 kB\nAnonPages: 63398296 kB\nMapped: 16728 kB\nSlab: 1346484 kB\nPageTables: 171512 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86133756 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33459621888}, {buffered_memory,290537472}, {free_memory,1098887168}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3444480475,1}}, {context_switches,{1913622127,0}}, {garbage_collection,{248130761,3901418319692,0}}, {io,{{input,69595233262},{output,132052121529}}}, {reductions,{1586402413412,2456554}}, {run_queue,0}, {runtime,{130389010,190}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,171565,406017}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,84251424}, {processes,14863576}, {processes_used,14828024}, {system,69387848}, {atom,828337}, {atom_used,824728}, {binary,1147200}, {code,7914132}, {ets,38569104}]}, {system_stats, [{cpu_utilization_rate,0.625}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3444475}, {memory_data,{101368545280,100839788544,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 516008 kB\nBuffers: 283980 kB\nCached: 31393128 kB\nSwapCached: 3836 kB\nActive: 82926836 kB\nInactive: 13854608 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 516008 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 332 kB\nWriteback: 0 kB\nAnonPages: 65096356 kB\nMapped: 17876 kB\nSlab: 1381124 kB\nPageTables: 278860 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141212328 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32146563072}, {buffered_memory,290795520}, {free_memory,528392192}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3444465641,1}}, {context_switches,{752075837,0}}, {garbage_collection,{185883255,675507729487,0}}, {io,{{input,58304444394},{output,130630566583}}}, {reductions,{324920391274,1403960}}, {run_queue,0}, {runtime,{37343810,140}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 07:13:13 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.111.192> 2011-12-29 07:13:46 =============================================================================== ns_1@192.168.24.61:<5883.111.192>:stats_collector:83: Stats for bucket "JDPCachePrimary": accepting_conns 1 auth_cmds 10369 auth_errors 0 bucket_active_conns 1 bucket_conns 1 bytes_read 53441690870 bytes_written 45371745801 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 29975715 cmd_set 445274018 conn_yields 61205 connection_structures 1931 curr_connections 11 curr_items 297280049 curr_items_tot 297280049 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 466 delete_misses 174808 ep_bg_fetched 1661206 ep_bg_load 4365803470 ep_bg_load_avg 2628 ep_bg_max_load 2020165 ep_bg_max_wait 2929571 ep_bg_min_load 1 ep_bg_min_wait 10 ep_bg_num_samples 1661198 ep_bg_wait 48864908595 ep_bg_wait_avg 29415 ep_commit_num 818500 ep_commit_time 0 ep_commit_time_total 179775 ep_data_age 0 ep_data_age_highwat 2571 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 1 ep_dbname /opt/membase/var/lib/membase/data/JDPCachePrimary-data/JDPCachePrimary ep_dbshards 4 ep_diskqueue_drain 358215763 ep_diskqueue_fill 358215763 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 1256 ep_flush_duration_total 251908 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 1661206 ep_io_num_write 357979281 ep_io_read_bytes 145605718 ep_io_write_bytes 6012819006520 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 356086874 ep_kv_size 57356020515 ep_latency_arith_cmd 46093 ep_latency_get_cmd 101813590 ep_latency_store_cmd 515449025 ep_max_data_size 81788928000 ep_max_txn_size 1000 ep_mem_high_wat 61341696000 ep_mem_low_wat 49073356800 ep_min_data_age 0 ep_num_active_non_resident 94237869 ep_num_checkpoint_remover_runs355194 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 490 ep_num_non_resident 94237869 ep_num_not_my_vbuckets 0 ep_num_pager_runs 2 ep_num_value_ejects 95895079 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 1611440768 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 3597 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 62797498 ep_too_young 0 ep_total_cache_size 58930442359 ep_total_del_items 416 ep_total_enqueued 358215763 ep_total_new_items 297280465 ep_total_persisted 357979697 ep_uncommitted_items 0 ep_value_size 15808691965 ep_vb_total 512 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.1.1_39_g4bd26a3 ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 8853 get_hits 7311306 get_misses 22664409 incr_hits 44431 incr_misses 1662 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 58967461283 pid 17177 pointer_size 64 rejected_conns 0 rusage_system 91969.650487 rusage_user 249505.086461 threads 4 time 1325171625 total_connections 30202 uptime 3444556 vb_active_curr_items 297280049 vb_active_eject 95895079 vb_active_ht_memory 1610809344 vb_active_itm_memory 53788643543 vb_active_num 512 vb_active_num_non_resident 94237869 vb_active_ops_create 297280465 vb_active_ops_delete 416 vb_active_ops_reject 0 vb_active_ops_update 60698816 vb_active_perc_mem_resident 68 vb_active_queue_age 0 vb_active_queue_drain 358215763 vb_active_queue_fill 358215763 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version UNKNOWN INFO REPORT <5883.136.0> 2011-12-29 07:13:48 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 07:13:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,171626,408269}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102618664}, {processes,32001024}, {processes_used,29326752}, {system,70617640}, {atom,903241}, {atom_used,880290}, {binary,1241648}, {code,8604866}, {ets,38767312}]}, {system_stats, [{cpu_utilization_rate,0.375}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3444555}, {memory_data,{101368545280,100270428160,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1072244 kB\nBuffers: 283728 kB\nCached: 32676204 kB\nSwapCached: 5096 kB\nActive: 82504480 kB\nInactive: 13862008 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1072244 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 324 kB\nWriteback: 8 kB\nAnonPages: 63399012 kB\nMapped: 16728 kB\nSlab: 1346260 kB\nPageTables: 171512 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86134592 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33460432896}, {buffered_memory,290537472}, {free_memory,1097977856}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3444540475,0}}, {context_switches,{1913638985,0}}, {garbage_collection,{248134215,3901430618623,0}}, {io,{{input,69595457217},{output,132053025663}}}, {reductions,{1586414858426,2437231}}, {run_queue,0}, {runtime,{130389840,190}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,171625,405088}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,84098336}, {processes,14668736}, {processes_used,14633040}, {system,69429600}, {atom,828337}, {atom_used,824728}, {binary,1160624}, {code,7914132}, {ets,38596864}]}, {system_stats, [{cpu_utilization_rate,0.5006257822277848}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3444535}, {memory_data,{101368545280,100839649280,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 516604 kB\nBuffers: 283980 kB\nCached: 31393908 kB\nSwapCached: 3836 kB\nActive: 82926192 kB\nInactive: 13855356 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 516604 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 476 kB\nWriteback: 0 kB\nAnonPages: 65095576 kB\nMapped: 17876 kB\nSlab: 1381024 kB\nPageTables: 278860 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141212328 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32147361792}, {buffered_memory,290795520}, {free_memory,529002496}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3444525642,1}}, {context_switches,{752088853,0}}, {garbage_collection,{185886462,675518995250,0}}, {io,{{input,58304601957},{output,130631458306}}}, {reductions,{324926090649,1385690}}, {run_queue,0}, {runtime,{37344450,150}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 07:14:41 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 07:14:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,171686,408346}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102812520}, {processes,31959776}, {processes_used,29285504}, {system,70852744}, {atom,903241}, {atom_used,880290}, {binary,1326512}, {code,8604866}, {ets,38917624}]}, {system_stats, [{cpu_utilization_rate,0.37546933667083854}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3444615}, {memory_data,{101368545280,100267634688,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1074740 kB\nBuffers: 283732 kB\nCached: 32675840 kB\nSwapCached: 5096 kB\nActive: 82502936 kB\nInactive: 13861664 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1074740 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 344 kB\nWriteback: 8 kB\nAnonPages: 63396868 kB\nMapped: 16728 kB\nSlab: 1346268 kB\nPageTables: 171512 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86134956 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33460060160}, {buffered_memory,290541568}, {free_memory,1100660736}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3444600476,1}}, {context_switches,{1913655932,0}}, {garbage_collection,{248137594,3901443391264,0}}, {io,{{input,69598132033},{output,132055839759}}}, {reductions,{1586426986425,2438636}}, {run_queue,0}, {runtime,{130390670,180}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,171685,405105}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,84171328}, {processes,14514448}, {processes_used,14478752}, {system,69656880}, {atom,828337}, {atom_used,824728}, {binary,1238552}, {code,7914132}, {ets,38746560}]}, {system_stats, [{cpu_utilization_rate,0.5}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3444595}, {memory_data,{101368545280,100840284160,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 515612 kB\nBuffers: 283980 kB\nCached: 31393540 kB\nSwapCached: 3836 kB\nActive: 82926888 kB\nInactive: 13855044 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 515612 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 356 kB\nWriteback: 0 kB\nAnonPages: 65096816 kB\nMapped: 17876 kB\nSlab: 1381004 kB\nPageTables: 278860 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141213068 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32146984960}, {buffered_memory,290795520}, {free_memory,527986688}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3444585642,1}}, {context_switches,{752102382,0}}, {garbage_collection,{185889901,675530786923,0}}, {io,{{input,58307249324},{output,130634258978}}}, {reductions,{324931668890,1411295}}, {run_queue,0}, {runtime,{37345140,160}}]}]}] INFO REPORT <5883.111.192> 2011-12-29 07:15:26 =============================================================================== ns_1@192.168.24.61:<5883.111.192>:stats_collector:83: Stats for bucket "JDPCachePrimary": accepting_conns 1 auth_cmds 10369 auth_errors 0 bucket_active_conns 1 bucket_conns 1 bytes_read 53441698030 bytes_written 45373413941 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 29975715 cmd_set 445274018 conn_yields 61205 connection_structures 1931 curr_connections 11 curr_items 297280049 curr_items_tot 297280049 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 466 delete_misses 174808 ep_bg_fetched 1661206 ep_bg_load 4365803470 ep_bg_load_avg 2628 ep_bg_max_load 2020165 ep_bg_max_wait 2929571 ep_bg_min_load 1 ep_bg_min_wait 10 ep_bg_num_samples 1661198 ep_bg_wait 48864908595 ep_bg_wait_avg 29415 ep_commit_num 818500 ep_commit_time 0 ep_commit_time_total 179775 ep_data_age 0 ep_data_age_highwat 2571 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 1 ep_dbname /opt/membase/var/lib/membase/data/JDPCachePrimary-data/JDPCachePrimary ep_dbshards 4 ep_diskqueue_drain 358215763 ep_diskqueue_fill 358215763 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 1256 ep_flush_duration_total 251908 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 1661206 ep_io_num_write 357979281 ep_io_read_bytes 145605718 ep_io_write_bytes 6012819006520 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 356086874 ep_kv_size 57356020515 ep_latency_arith_cmd 46093 ep_latency_get_cmd 101813590 ep_latency_store_cmd 515449025 ep_max_data_size 81788928000 ep_max_txn_size 1000 ep_mem_high_wat 61341696000 ep_mem_low_wat 49073356800 ep_min_data_age 0 ep_num_active_non_resident 94237869 ep_num_checkpoint_remover_runs355214 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 490 ep_num_non_resident 94237869 ep_num_not_my_vbuckets 0 ep_num_pager_runs 2 ep_num_value_ejects 95895079 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 1611440768 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 3597 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 62797498 ep_too_young 0 ep_total_cache_size 58930442359 ep_total_del_items 416 ep_total_enqueued 358215763 ep_total_new_items 297280465 ep_total_persisted 357979697 ep_uncommitted_items 0 ep_value_size 15808691965 ep_vb_total 512 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.1.1_39_g4bd26a3 ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 8853 get_hits 7311306 get_misses 22664409 incr_hits 44431 incr_misses 1662 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 58967461283 pid 17177 pointer_size 64 rejected_conns 0 rusage_system 91969.687482 rusage_user 249505.175447 threads 4 time 1325171725 total_connections 30202 uptime 3444656 vb_active_curr_items 297280049 vb_active_eject 95895079 vb_active_ht_memory 1610809344 vb_active_itm_memory 53788643543 vb_active_num 512 vb_active_num_non_resident 94237869 vb_active_ops_create 297280465 vb_active_ops_delete 416 vb_active_ops_reject 0 vb_active_ops_update 60698816 vb_active_perc_mem_resident 68 vb_active_queue_age 0 vb_active_queue_drain 358215763 vb_active_queue_fill 358215763 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version UNKNOWN INFO REPORT <5883.136.0> 2011-12-29 07:15:32 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 07:15:39 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 07:15:44 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 07:15:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,171746,408337}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102882424}, {processes,31959528}, {processes_used,29285256}, {system,70922896}, {atom,903241}, {atom_used,880290}, {binary,1363240}, {code,8604866}, {ets,38950856}]}, {system_stats, [{cpu_utilization_rate,0.4993757802746567}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3444675}, {memory_data,{101368545280,100268204032,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1074544 kB\nBuffers: 283732 kB\nCached: 32676596 kB\nSwapCached: 5096 kB\nActive: 82502560 kB\nInactive: 13862424 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1074544 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 212 kB\nWriteback: 36 kB\nAnonPages: 63397064 kB\nMapped: 16728 kB\nSlab: 1346216 kB\nPageTables: 171512 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86134860 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33460834304}, {buffered_memory,290541568}, {free_memory,1100333056}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3444660475,1}}, {context_switches,{1913672238,0}}, {garbage_collection,{248140784,3901455365955,0}}, {io,{{input,69598310770},{output,132056732399}}}, {reductions,{1586438949849,2454290}}, {run_queue,0}, {runtime,{130391510,200}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,171745,406467}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,84333800}, {processes,14667752}, {processes_used,14632056}, {system,69666048}, {atom,828337}, {atom_used,824728}, {binary,1215792}, {code,7914132}, {ets,38778360}]}, {system_stats, [{cpu_utilization_rate,12.875}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3444655}, {memory_data,{101368545280,100839665664,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 501260 kB\nBuffers: 283980 kB\nCached: 31394308 kB\nSwapCached: 3836 kB\nActive: 82939620 kB\nInactive: 13855772 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 501260 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 1236 kB\nWriteback: 0 kB\nAnonPages: 65108908 kB\nMapped: 19272 kB\nSlab: 1380992 kB\nPageTables: 278708 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141224684 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32147771392}, {buffered_memory,290795520}, {free_memory,513290240}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3444645641,0}}, {context_switches,{752115054,0}}, {garbage_collection,{185893022,675541859675,0}}, {io,{{input,58307440011},{output,130635129287}}}, {reductions,{324937140185,1407292}}, {run_queue,1}, {runtime,{37345700,140}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 07:16:37 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 07:16:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,171806,408541}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102728360}, {processes,31638056}, {processes_used,28963784}, {system,71090304}, {atom,903241}, {atom_used,880290}, {binary,1384240}, {code,8604866}, {ets,39097584}]}, {system_stats, [{cpu_utilization_rate,12.875}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3444735}, {memory_data,{101368545280,100282331136,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1057008 kB\nBuffers: 283732 kB\nCached: 32677364 kB\nSwapCached: 5096 kB\nActive: 82517656 kB\nInactive: 13863164 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1057008 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 3356 kB\nWriteback: 12 kB\nAnonPages: 63411376 kB\nMapped: 18736 kB\nSlab: 1346332 kB\nPageTables: 171912 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86150268 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33461620736}, {buffered_memory,290541568}, {free_memory,1082376192}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3444720475,1}}, {context_switches,{1913688533,0}}, {garbage_collection,{248144108,3901467382912,0}}, {io,{{input,69598568994},{output,132057576995}}}, {reductions,{1586450880570,2458126}}, {run_queue,0}, {runtime,{130392320,180}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,171805,405889}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,84556736}, {processes,14670800}, {processes_used,14635104}, {system,69885936}, {atom,828337}, {atom_used,824728}, {binary,1288504}, {code,7914132}, {ets,38925208}]}, {system_stats, [{cpu_utilization_rate,0.2503128911138924}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3444715}, {memory_data,{101368545280,100884516864,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 511000 kB\nBuffers: 283980 kB\nCached: 31395184 kB\nSwapCached: 3836 kB\nActive: 82926560 kB\nInactive: 13856444 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 511000 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 4052 kB\nWriteback: 0 kB\nAnonPages: 65095572 kB\nMapped: 17876 kB\nSlab: 1382000 kB\nPageTables: 278860 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141209468 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32148668416}, {buffered_memory,290795520}, {free_memory,523264000}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3444705641,0}}, {context_switches,{752127498,0}}, {garbage_collection,{185896406,675553055862,0}}, {io,{{input,58307602661},{output,130636009450}}}, {reductions,{324942643435,1399004}}, {run_queue,1}, {runtime,{37346310,170}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 07:17:03 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.111.192> 2011-12-29 07:17:06 =============================================================================== ns_1@192.168.24.61:<5883.111.192>:stats_collector:83: Stats for bucket "JDPCachePrimary": accepting_conns 1 auth_cmds 10369 auth_errors 0 bucket_active_conns 1 bucket_conns 1 bytes_read 53441705190 bytes_written 45375082081 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 29975715 cmd_set 445274018 conn_yields 61205 connection_structures 1931 curr_connections 11 curr_items 297280049 curr_items_tot 297280049 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 466 delete_misses 174808 ep_bg_fetched 1661206 ep_bg_load 4365803470 ep_bg_load_avg 2628 ep_bg_max_load 2020165 ep_bg_max_wait 2929571 ep_bg_min_load 1 ep_bg_min_wait 10 ep_bg_num_samples 1661198 ep_bg_wait 48864908595 ep_bg_wait_avg 29415 ep_commit_num 818500 ep_commit_time 0 ep_commit_time_total 179775 ep_data_age 0 ep_data_age_highwat 2571 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 1 ep_dbname /opt/membase/var/lib/membase/data/JDPCachePrimary-data/JDPCachePrimary ep_dbshards 4 ep_diskqueue_drain 358215763 ep_diskqueue_fill 358215763 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 1256 ep_flush_duration_total 251908 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 1661206 ep_io_num_write 357979281 ep_io_read_bytes 145605718 ep_io_write_bytes 6012819006520 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 356086874 ep_kv_size 57356020515 ep_latency_arith_cmd 46093 ep_latency_get_cmd 101813590 ep_latency_store_cmd 515449025 ep_max_data_size 81788928000 ep_max_txn_size 1000 ep_mem_high_wat 61341696000 ep_mem_low_wat 49073356800 ep_min_data_age 0 ep_num_active_non_resident 94237869 ep_num_checkpoint_remover_runs355234 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 490 ep_num_non_resident 94237869 ep_num_not_my_vbuckets 0 ep_num_pager_runs 2 ep_num_value_ejects 95895079 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 1611440768 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 3597 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 62797498 ep_too_young 0 ep_total_cache_size 58930442359 ep_total_del_items 416 ep_total_enqueued 358215763 ep_total_new_items 297280465 ep_total_persisted 357979697 ep_uncommitted_items 0 ep_value_size 15808691965 ep_vb_total 512 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.1.1_39_g4bd26a3 ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 8853 get_hits 7311306 get_misses 22664409 incr_hits 44431 incr_misses 1662 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 58967461283 pid 17177 pointer_size 64 rejected_conns 0 rusage_system 91969.717477 rusage_user 249505.284431 threads 4 time 1325171825 total_connections 30202 uptime 3444756 vb_active_curr_items 297280049 vb_active_eject 95895079 vb_active_ht_memory 1610809344 vb_active_itm_memory 53788643543 vb_active_num 512 vb_active_num_non_resident 94237869 vb_active_ops_create 297280465 vb_active_ops_delete 416 vb_active_ops_reject 0 vb_active_ops_update 60698816 vb_active_perc_mem_resident 68 vb_active_queue_age 0 vb_active_queue_drain 358215763 vb_active_queue_fill 358215763 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version UNKNOWN INFO REPORT <5883.136.0> 2011-12-29 07:17:13 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 07:17:31 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 07:17:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,171866,408435}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102833688}, {processes,31671504}, {processes_used,28997232}, {system,71162184}, {atom,903241}, {atom_used,880290}, {binary,1308232}, {code,8604866}, {ets,39245640}]}, {system_stats, [{cpu_utilization_rate,0.375}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3444795}, {memory_data,{101368545280,100270047232,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1074436 kB\nBuffers: 283736 kB\nCached: 32673936 kB\nSwapCached: 5096 kB\nActive: 82503744 kB\nInactive: 13859620 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1074436 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 3072 kB\nWriteback: 0 kB\nAnonPages: 63397680 kB\nMapped: 16728 kB\nSlab: 1347040 kB\nPageTables: 171512 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86132964 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33458110464}, {buffered_memory,290545664}, {free_memory,1100222464}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3444780475,1}}, {context_switches,{1913705473,0}}, {garbage_collection,{248147627,3901480665154,0}}, {io,{{input,69601291751},{output,132065576603}}}, {reductions,{1586462956528,2411606}}, {run_queue,1}, {runtime,{130393330,180}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,171865,404981}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,84672552}, {processes,14739576}, {processes_used,14703880}, {system,69932976}, {atom,828337}, {atom_used,824728}, {binary,1245320}, {code,7914132}, {ets,39015776}]}, {system_stats, [{cpu_utilization_rate,0.375}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3444775}, {memory_data,{101368545280,100843962368,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 516476 kB\nBuffers: 283992 kB\nCached: 31391652 kB\nSwapCached: 3836 kB\nActive: 82926732 kB\nInactive: 13853064 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 516476 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 248 kB\nWriteback: 0 kB\nAnonPages: 65096176 kB\nMapped: 17876 kB\nSlab: 1381224 kB\nPageTables: 278860 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141212512 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32145051648}, {buffered_memory,290807808}, {free_memory,528871424}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3444765641,1}}, {context_switches,{752140963,0}}, {garbage_collection,{185899773,675565688063,0}}, {io,{{input,58310251917},{output,130644110345}}}, {reductions,{324948521871,1404746}}, {run_queue,1}, {runtime,{37347110,160}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 07:17:57 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 07:18:35 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.111.192> 2011-12-29 07:18:46 =============================================================================== ns_1@192.168.24.61:<5883.111.192>:stats_collector:83: Stats for bucket "JDPCachePrimary": accepting_conns 1 auth_cmds 10369 auth_errors 0 bucket_active_conns 1 bucket_conns 1 bytes_read 53441712350 bytes_written 45376750221 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 29975715 cmd_set 445274018 conn_yields 61205 connection_structures 1931 curr_connections 11 curr_items 297280049 curr_items_tot 297280049 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 466 delete_misses 174808 ep_bg_fetched 1661206 ep_bg_load 4365803470 ep_bg_load_avg 2628 ep_bg_max_load 2020165 ep_bg_max_wait 2929571 ep_bg_min_load 1 ep_bg_min_wait 10 ep_bg_num_samples 1661198 ep_bg_wait 48864908595 ep_bg_wait_avg 29415 ep_commit_num 818500 ep_commit_time 0 ep_commit_time_total 179775 ep_data_age 0 ep_data_age_highwat 2571 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 1 ep_dbname /opt/membase/var/lib/membase/data/JDPCachePrimary-data/JDPCachePrimary ep_dbshards 4 ep_diskqueue_drain 358215763 ep_diskqueue_fill 358215763 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 1256 ep_flush_duration_total 251908 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 1661206 ep_io_num_write 357979281 ep_io_read_bytes 145605718 ep_io_write_bytes 6012819006520 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 356086874 ep_kv_size 57356020515 ep_latency_arith_cmd 46093 ep_latency_get_cmd 101813590 ep_latency_store_cmd 515449025 ep_max_data_size 81788928000 ep_max_txn_size 1000 ep_mem_high_wat 61341696000 ep_mem_low_wat 49073356800 ep_min_data_age 0 ep_num_active_non_resident 94237869 ep_num_checkpoint_remover_runs355254 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 490 ep_num_non_resident 94237869 ep_num_not_my_vbuckets 0 ep_num_pager_runs 2 ep_num_value_ejects 95895079 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 1611440768 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 3597 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 62797498 ep_too_young 0 ep_total_cache_size 58930442359 ep_total_del_items 416 ep_total_enqueued 358215763 ep_total_new_items 297280465 ep_total_persisted 357979697 ep_uncommitted_items 0 ep_value_size 15808691965 ep_vb_total 512 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.1.1_39_g4bd26a3 ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 8853 get_hits 7311306 get_misses 22664409 incr_hits 44431 incr_misses 1662 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 58967461283 pid 17177 pointer_size 64 rejected_conns 0 rusage_system 91969.748473 rusage_user 249505.381416 threads 4 time 1325171925 total_connections 30202 uptime 3444856 vb_active_curr_items 297280049 vb_active_eject 95895079 vb_active_ht_memory 1610809344 vb_active_itm_memory 53788643543 vb_active_num 512 vb_active_num_non_resident 94237869 vb_active_ops_create 297280465 vb_active_ops_delete 416 vb_active_ops_reject 0 vb_active_ops_update 60698816 vb_active_perc_mem_resident 68 vb_active_queue_age 0 vb_active_queue_drain 358215763 vb_active_queue_fill 358215763 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version UNKNOWN INFO REPORT <5883.1095.0> 2011-12-29 07:18:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,171926,409230}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102327560}, {processes,31496304}, {processes_used,28822032}, {system,70831256}, {atom,903241}, {atom_used,880290}, {binary,1446144}, {code,8604866}, {ets,38776688}]}, {system_stats, [{cpu_utilization_rate,0.6242197253433208}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3444855}, {memory_data,{101368545280,100265164800,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1077056 kB\nBuffers: 283736 kB\nCached: 32674696 kB\nSwapCached: 5096 kB\nActive: 82502368 kB\nInactive: 13860364 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1077056 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 144 kB\nWriteback: 0 kB\nAnonPages: 63396672 kB\nMapped: 16728 kB\nSlab: 1346340 kB\nPageTables: 171512 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86133880 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33458888704}, {buffered_memory,290545664}, {free_memory,1102905344}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3444840474,0}}, {context_switches,{1913721782,0}}, {garbage_collection,{248150922,3901492911352,0}}, {io,{{input,69601508781},{output,132066414245}}}, {reductions,{1586474985687,2461089}}, {run_queue,0}, {runtime,{130394100,190}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,171925,404875}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,84086672}, {processes,14721704}, {processes_used,14686008}, {system,69364968}, {atom,828337}, {atom_used,824728}, {binary,1088032}, {code,7914132}, {ets,38605192}]}, {system_stats, [{cpu_utilization_rate,0.5}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3444835}, {memory_data,{101368545280,100839043072,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 516544 kB\nBuffers: 283992 kB\nCached: 31392420 kB\nSwapCached: 3836 kB\nActive: 82926788 kB\nInactive: 13853780 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 516544 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 196 kB\nWriteback: 0 kB\nAnonPages: 65096504 kB\nMapped: 17876 kB\nSlab: 1381072 kB\nPageTables: 278864 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141212820 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32145838080}, {buffered_memory,290807808}, {free_memory,528941056}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3444825641,1}}, {context_switches,{752153371,0}}, {garbage_collection,{185902918,675576837413,0}}, {io,{{input,58310411393},{output,130644942352}}}, {reductions,{324954007303,1381838}}, {run_queue,0}, {runtime,{37347690,150}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 07:18:55 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 07:19:42 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 07:19:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,171986,408908}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102670744}, {processes,31815944}, {processes_used,29141672}, {system,70854800}, {atom,903241}, {atom_used,880290}, {binary,1435912}, {code,8604866}, {ets,38810472}]}, {system_stats, [{cpu_utilization_rate,12.76595744680851}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3444915}, {memory_data,{101368545280,100266844160,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1075500 kB\nBuffers: 283736 kB\nCached: 32675500 kB\nSwapCached: 5096 kB\nActive: 82502676 kB\nInactive: 13861136 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1075500 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 472 kB\nWriteback: 0 kB\nAnonPages: 63396660 kB\nMapped: 16728 kB\nSlab: 1346208 kB\nPageTables: 171512 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86133192 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33459712000}, {buffered_memory,290545664}, {free_memory,1101312000}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3444900476,0}}, {context_switches,{1913738639,0}}, {garbage_collection,{248154253,3901505182980,0}}, {io,{{input,69601722874},{output,132067298759}}}, {reductions,{1586487418062,2438156}}, {run_queue,0}, {runtime,{130394880,170}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,171985,407968}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,83919280}, {processes,14376480}, {processes_used,14340784}, {system,69542800}, {atom,828337}, {atom_used,824728}, {binary,1232024}, {code,7914132}, {ets,38638688}]}, {system_stats, [{cpu_utilization_rate,12.875}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3444895}, {memory_data,{101368545280,100839321600,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 514824 kB\nBuffers: 283992 kB\nCached: 31393220 kB\nSwapCached: 3836 kB\nActive: 82928148 kB\nInactive: 13854520 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 514824 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 432 kB\nWriteback: 0 kB\nAnonPages: 65097364 kB\nMapped: 17876 kB\nSlab: 1380924 kB\nPageTables: 278864 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141212832 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32146657280}, {buffered_memory,290807808}, {free_memory,527179776}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3444885641,0}}, {context_switches,{752166077,0}}, {garbage_collection,{185906084,675588186898,0}}, {io,{{input,58310572978},{output,130645853172}}}, {reductions,{324959762717,1415080}}, {run_queue,0}, {runtime,{37348250,150}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 07:20:07 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 07:20:21 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.111.192> 2011-12-29 07:20:26 =============================================================================== ns_1@192.168.24.61:<5883.111.192>:stats_collector:83: Stats for bucket "JDPCachePrimary": accepting_conns 1 auth_cmds 10369 auth_errors 0 bucket_active_conns 1 bucket_conns 1 bytes_read 53441719510 bytes_written 45378418361 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 29975715 cmd_set 445274018 conn_yields 61205 connection_structures 1931 curr_connections 11 curr_items 297280049 curr_items_tot 297280049 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 466 delete_misses 174808 ep_bg_fetched 1661206 ep_bg_load 4365803470 ep_bg_load_avg 2628 ep_bg_max_load 2020165 ep_bg_max_wait 2929571 ep_bg_min_load 1 ep_bg_min_wait 10 ep_bg_num_samples 1661198 ep_bg_wait 48864908595 ep_bg_wait_avg 29415 ep_commit_num 818500 ep_commit_time 0 ep_commit_time_total 179775 ep_data_age 0 ep_data_age_highwat 2571 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 1 ep_dbname /opt/membase/var/lib/membase/data/JDPCachePrimary-data/JDPCachePrimary ep_dbshards 4 ep_diskqueue_drain 358215763 ep_diskqueue_fill 358215763 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 1256 ep_flush_duration_total 251908 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 1661206 ep_io_num_write 357979281 ep_io_read_bytes 145605718 ep_io_write_bytes 6012819006520 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 356086874 ep_kv_size 57356020515 ep_latency_arith_cmd 46093 ep_latency_get_cmd 101813590 ep_latency_store_cmd 515449025 ep_max_data_size 81788928000 ep_max_txn_size 1000 ep_mem_high_wat 61341696000 ep_mem_low_wat 49073356800 ep_min_data_age 0 ep_num_active_non_resident 94237869 ep_num_checkpoint_remover_runs355273 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 491 ep_num_non_resident 94237869 ep_num_not_my_vbuckets 0 ep_num_pager_runs 2 ep_num_value_ejects 95895079 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 1611440768 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 3597 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 62797498 ep_too_young 0 ep_total_cache_size 58930442359 ep_total_del_items 416 ep_total_enqueued 358215763 ep_total_new_items 297280465 ep_total_persisted 357979697 ep_uncommitted_items 0 ep_value_size 15808691965 ep_vb_total 512 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.1.1_39_g4bd26a3 ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 8853 get_hits 7311306 get_misses 22664409 incr_hits 44431 incr_misses 1662 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 58967461283 pid 17177 pointer_size 64 rejected_conns 0 rusage_system 91969.820462 rusage_user 249604.677321 threads 4 time 1325172025 total_connections 30202 uptime 3444956 vb_active_curr_items 297280049 vb_active_eject 95895079 vb_active_ht_memory 1610809344 vb_active_itm_memory 53788643543 vb_active_num 512 vb_active_num_non_resident 94237869 vb_active_ops_create 297280465 vb_active_ops_delete 416 vb_active_ops_reject 0 vb_active_ops_update 60698816 vb_active_perc_mem_resident 68 vb_active_queue_age 0 vb_active_queue_drain 358215763 vb_active_queue_fill 358215763 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version UNKNOWN INFO REPORT <5883.1095.0> 2011-12-29 07:20:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,172046,409215}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,103134712}, {processes,32137864}, {processes_used,29463592}, {system,70996848}, {atom,903241}, {atom_used,880290}, {binary,1421768}, {code,8604866}, {ets,38966768}]}, {system_stats, [{cpu_utilization_rate,0.49875311720698257}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3444975}, {memory_data,{101368545280,100266889216,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1076076 kB\nBuffers: 283736 kB\nCached: 32675132 kB\nSwapCached: 5096 kB\nActive: 82503208 kB\nInactive: 13860820 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1076076 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 284 kB\nWriteback: 0 kB\nAnonPages: 63397568 kB\nMapped: 16728 kB\nSlab: 1346228 kB\nPageTables: 171512 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86135592 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33459335168}, {buffered_memory,290545664}, {free_memory,1101901824}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3444960475,0}}, {context_switches,{1913755630,0}}, {garbage_collection,{248157789,3901517830096,0}}, {io,{{input,69604468951},{output,132070116781}}}, {reductions,{1586499556483,2470551}}, {run_queue,0}, {runtime,{130395700,180}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,172045,405796}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,84283456}, {processes,14664752}, {processes_used,14629136}, {system,69618704}, {atom,828337}, {atom_used,824728}, {binary,1149904}, {code,7914132}, {ets,38797320}]}, {system_stats, [{cpu_utilization_rate,13.5}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3444955}, {memory_data,{101368545280,100840751104,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 515620 kB\nBuffers: 283996 kB\nCached: 31392840 kB\nSwapCached: 3836 kB\nActive: 82927680 kB\nInactive: 13854216 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 515620 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 324 kB\nWriteback: 0 kB\nAnonPages: 65096956 kB\nMapped: 17876 kB\nSlab: 1380980 kB\nPageTables: 278860 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141210364 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32146268160}, {buffered_memory,290811904}, {free_memory,527994880}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3444945641,0}}, {context_switches,{752179328,0}}, {garbage_collection,{185909443,675599879654,0}}, {io,{{input,58313232438},{output,130648711007}}}, {reductions,{324965363995,1403825}}, {run_queue,0}, {runtime,{37348910,140}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 07:20:54 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 07:21:20 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 07:21:26 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 07:21:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,172106,408386}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,103179184}, {processes,32104432}, {processes_used,29430160}, {system,71074752}, {atom,903241}, {atom_used,880290}, {binary,1466624}, {code,8604866}, {ets,39000096}]}, {system_stats, [{cpu_utilization_rate,12.85892634207241}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3445035}, {memory_data,{101368545280,100341067776,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 979788 kB\nBuffers: 283736 kB\nCached: 32675936 kB\nSwapCached: 5096 kB\nActive: 82596896 kB\nInactive: 13861516 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 979788 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 2872 kB\nWriteback: 0 kB\nAnonPages: 63491104 kB\nMapped: 24076 kB\nSlab: 1346824 kB\nPageTables: 172444 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86226472 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33460158464}, {buffered_memory,290545664}, {free_memory,1003302912}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3445020476,1}}, {context_switches,{1913771931,0}}, {garbage_collection,{248160994,3901529768256,0}}, {io,{{input,69604724291},{output,132071008260}}}, {reductions,{1586511510979,2488704}}, {run_queue,0}, {runtime,{130396490,210}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,172105,405895}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,84498712}, {processes,14735584}, {processes_used,14699888}, {system,69763128}, {atom,828337}, {atom_used,824728}, {binary,1262416}, {code,7914132}, {ets,38828784}]}, {system_stats, [{cpu_utilization_rate,0.625}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3445015}, {memory_data,{101368545280,100919275520,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 462664 kB\nBuffers: 284004 kB\nCached: 31392668 kB\nSwapCached: 3836 kB\nActive: 82973764 kB\nInactive: 13854920 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 462664 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 4440 kB\nWriteback: 0 kB\nAnonPages: 65144328 kB\nMapped: 19656 kB\nSlab: 1382152 kB\nPageTables: 279684 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141257808 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32146092032}, {buffered_memory,290820096}, {free_memory,473698304}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3445005641,0}}, {context_switches,{752191930,0}}, {garbage_collection,{185912696,675611095394,0}}, {io,{{input,58313426332},{output,130649614753}}}, {reductions,{324970842631,1385280}}, {run_queue,0}, {runtime,{37349480,140}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 07:21:56 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.111.192> 2011-12-29 07:22:06 =============================================================================== ns_1@192.168.24.61:<5883.111.192>:stats_collector:83: Stats for bucket "JDPCachePrimary": accepting_conns 1 auth_cmds 10369 auth_errors 0 bucket_active_conns 1 bucket_conns 1 bytes_read 53441726670 bytes_written 45380086501 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 29975715 cmd_set 445274018 conn_yields 61205 connection_structures 1931 curr_connections 11 curr_items 297280049 curr_items_tot 297280049 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 466 delete_misses 174808 ep_bg_fetched 1661206 ep_bg_load 4365803470 ep_bg_load_avg 2628 ep_bg_max_load 2020165 ep_bg_max_wait 2929571 ep_bg_min_load 1 ep_bg_min_wait 10 ep_bg_num_samples 1661198 ep_bg_wait 48864908595 ep_bg_wait_avg 29415 ep_commit_num 818500 ep_commit_time 0 ep_commit_time_total 179775 ep_data_age 0 ep_data_age_highwat 2571 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 1 ep_dbname /opt/membase/var/lib/membase/data/JDPCachePrimary-data/JDPCachePrimary ep_dbshards 4 ep_diskqueue_drain 358215763 ep_diskqueue_fill 358215763 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 1256 ep_flush_duration_total 251908 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 1661206 ep_io_num_write 357979281 ep_io_read_bytes 145605718 ep_io_write_bytes 6012819006520 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 356086874 ep_kv_size 57356020515 ep_latency_arith_cmd 46093 ep_latency_get_cmd 101813590 ep_latency_store_cmd 515449025 ep_max_data_size 81788928000 ep_max_txn_size 1000 ep_mem_high_wat 61341696000 ep_mem_low_wat 49073356800 ep_min_data_age 0 ep_num_active_non_resident 94237869 ep_num_checkpoint_remover_runs355293 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 491 ep_num_non_resident 94237869 ep_num_not_my_vbuckets 0 ep_num_pager_runs 2 ep_num_value_ejects 95895079 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 1611440768 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 3597 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 62797498 ep_too_young 0 ep_total_cache_size 58930442359 ep_total_del_items 416 ep_total_enqueued 358215763 ep_total_new_items 297280465 ep_total_persisted 357979697 ep_uncommitted_items 0 ep_value_size 15808691965 ep_vb_total 512 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.1.1_39_g4bd26a3 ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 8853 get_hits 7311306 get_misses 22664409 incr_hits 44431 incr_misses 1662 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 58967461283 pid 17177 pointer_size 64 rejected_conns 0 rusage_system 91969.852457 rusage_user 249615.055743 threads 4 time 1325172125 total_connections 30202 uptime 3445056 vb_active_curr_items 297280049 vb_active_eject 95895079 vb_active_ht_memory 1610809344 vb_active_itm_memory 53788643543 vb_active_num 512 vb_active_num_non_resident 94237869 vb_active_ops_create 297280465 vb_active_ops_delete 416 vb_active_ops_reject 0 vb_active_ops_update 60698816 vb_active_perc_mem_resident 68 vb_active_queue_age 0 vb_active_queue_drain 358215763 vb_active_queue_fill 358215763 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version UNKNOWN INFO REPORT <5883.136.0> 2011-12-29 07:22:07 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 07:22:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,172166,408347}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102721296}, {processes,31631264}, {processes_used,28956992}, {system,71090032}, {atom,903241}, {atom_used,880290}, {binary,1335568}, {code,8604866}, {ets,39145776}]}, {system_stats, [{cpu_utilization_rate,0.375}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3445095}, {memory_data,{101368545280,100270575616,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1072704 kB\nBuffers: 283736 kB\nCached: 32676776 kB\nSwapCached: 5096 kB\nActive: 82504444 kB\nInactive: 13862220 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1072704 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 1752 kB\nWriteback: 0 kB\nAnonPages: 63398192 kB\nMapped: 16728 kB\nSlab: 1346380 kB\nPageTables: 171512 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86139940 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33461018624}, {buffered_memory,290545664}, {free_memory,1098448896}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3445080476,0}}, {context_switches,{1913788214,0}}, {garbage_collection,{248164410,3901541794337,0}}, {io,{{input,69604941329},{output,132071923271}}}, {reductions,{1586523479605,2456428}}, {run_queue,0}, {runtime,{130397290,200}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,172165,405666}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,84708640}, {processes,14791784}, {processes_used,14756088}, {system,69916856}, {atom,828337}, {atom_used,824728}, {binary,1270968}, {code,7914132}, {ets,38974000}]}, {system_stats, [{cpu_utilization_rate,0.625}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3445075}, {memory_data,{101368545280,100844109824,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 514524 kB\nBuffers: 284004 kB\nCached: 31393476 kB\nSwapCached: 3836 kB\nActive: 82926504 kB\nInactive: 13855700 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 514524 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 524 kB\nWriteback: 0 kB\nAnonPages: 65096716 kB\nMapped: 17876 kB\nSlab: 1381228 kB\nPageTables: 278860 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141211072 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32146919424}, {buffered_memory,290820096}, {free_memory,526872576}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3445065641,1}}, {context_switches,{752204363,0}}, {garbage_collection,{185915774,675622330170,0}}, {io,{{input,58313657744},{output,130650506198}}}, {reductions,{324976337166,1409409}}, {run_queue,0}, {runtime,{37350050,150}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 07:23:07 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.111.192> 2011-12-29 07:23:46 =============================================================================== ns_1@192.168.24.61:<5883.111.192>:stats_collector:83: Stats for bucket "JDPCachePrimary": accepting_conns 1 auth_cmds 10369 auth_errors 0 bucket_active_conns 1 bucket_conns 1 bytes_read 53441733830 bytes_written 45381754641 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 29975715 cmd_set 445274018 conn_yields 61205 connection_structures 1931 curr_connections 11 curr_items 297280049 curr_items_tot 297280049 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 466 delete_misses 174808 ep_bg_fetched 1661206 ep_bg_load 4365803470 ep_bg_load_avg 2628 ep_bg_max_load 2020165 ep_bg_max_wait 2929571 ep_bg_min_load 1 ep_bg_min_wait 10 ep_bg_num_samples 1661198 ep_bg_wait 48864908595 ep_bg_wait_avg 29415 ep_commit_num 818500 ep_commit_time 0 ep_commit_time_total 179775 ep_data_age 0 ep_data_age_highwat 2571 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 1 ep_dbname /opt/membase/var/lib/membase/data/JDPCachePrimary-data/JDPCachePrimary ep_dbshards 4 ep_diskqueue_drain 358215763 ep_diskqueue_fill 358215763 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 1256 ep_flush_duration_total 251908 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 1661206 ep_io_num_write 357979281 ep_io_read_bytes 145605718 ep_io_write_bytes 6012819006520 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 356086874 ep_kv_size 57356020515 ep_latency_arith_cmd 46093 ep_latency_get_cmd 101813590 ep_latency_store_cmd 515449025 ep_max_data_size 81788928000 ep_max_txn_size 1000 ep_mem_high_wat 61341696000 ep_mem_low_wat 49073356800 ep_min_data_age 0 ep_num_active_non_resident 94237869 ep_num_checkpoint_remover_runs355313 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 491 ep_num_non_resident 94237869 ep_num_not_my_vbuckets 0 ep_num_pager_runs 2 ep_num_value_ejects 95895079 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 1611440768 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 3597 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 62797498 ep_too_young 0 ep_total_cache_size 58930442359 ep_total_del_items 416 ep_total_enqueued 358215763 ep_total_new_items 297280465 ep_total_persisted 357979697 ep_uncommitted_items 0 ep_value_size 15808691965 ep_vb_total 512 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.1.1_39_g4bd26a3 ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 8853 get_hits 7311306 get_misses 22664409 incr_hits 44431 incr_misses 1662 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 58967461283 pid 17177 pointer_size 64 rejected_conns 0 rusage_system 91969.873454 rusage_user 249615.139730 threads 4 time 1325172225 total_connections 30202 uptime 3445156 vb_active_curr_items 297280049 vb_active_eject 95895079 vb_active_ht_memory 1610809344 vb_active_itm_memory 53788643543 vb_active_num 512 vb_active_num_non_resident 94237869 vb_active_ops_create 297280465 vb_active_ops_delete 416 vb_active_ops_reject 0 vb_active_ops_update 60698816 vb_active_perc_mem_resident 68 vb_active_queue_age 0 vb_active_queue_drain 358215763 vb_active_queue_fill 358215763 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version UNKNOWN INFO REPORT <5883.1095.0> 2011-12-29 07:23:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,172226,408989}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,103220112}, {processes,32013800}, {processes_used,29339528}, {system,71206312}, {atom,903241}, {atom_used,880290}, {binary,1303232}, {code,8604866}, {ets,39294296}]}, {system_stats, [{cpu_utilization_rate,0.37453183520599254}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3445155}, {memory_data,{101368545280,100268584960,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1074336 kB\nBuffers: 283736 kB\nCached: 32677072 kB\nSwapCached: 5096 kB\nActive: 82502688 kB\nInactive: 13862540 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1074336 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 344 kB\nWriteback: 0 kB\nAnonPages: 63396652 kB\nMapped: 16728 kB\nSlab: 1346304 kB\nPageTables: 171508 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86133296 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33461321728}, {buffered_memory,290545664}, {free_memory,1100120064}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3445140476,0}}, {context_switches,{1913805150,0}}, {garbage_collection,{248167867,3901554609969,0}}, {io,{{input,69607595076},{output,132075402033}}}, {reductions,{1586535581691,2479539}}, {run_queue,0}, {runtime,{130398160,190}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,172225,405269}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,84804216}, {processes,14809424}, {processes_used,14773728}, {system,69994792}, {atom,828337}, {atom_used,824728}, {binary,1259640}, {code,7914132}, {ets,39063720}]}, {system_stats, [{cpu_utilization_rate,0.6257822277847309}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3445135}, {memory_data,{101368545280,100841644032,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 515088 kB\nBuffers: 284008 kB\nCached: 31393772 kB\nSwapCached: 3836 kB\nActive: 82925956 kB\nInactive: 13856008 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 515088 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 284 kB\nWriteback: 0 kB\nAnonPages: 65096520 kB\nMapped: 17876 kB\nSlab: 1381084 kB\nPageTables: 278860 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141208756 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32147222528}, {buffered_memory,290824192}, {free_memory,527450112}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3445125641,0}}, {context_switches,{752217990,0}}, {garbage_collection,{185918952,675634174085,0}}, {io,{{input,58316312137},{output,130653963716}}}, {reductions,{324982170019,1392890}}, {run_queue,0}, {runtime,{37350690,130}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 07:23:54 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 07:24:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,172286,408358}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102764792}, {processes,31861216}, {processes_used,29186944}, {system,70903576}, {atom,903241}, {atom_used,880290}, {binary,1470264}, {code,8604866}, {ets,38824424}]}, {system_stats, [{cpu_utilization_rate,0.5006257822277848}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3445215}, {memory_data,{101368545280,100269031424,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1073924 kB\nBuffers: 283736 kB\nCached: 32677848 kB\nSwapCached: 5096 kB\nActive: 82502716 kB\nInactive: 13863280 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1073924 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 172 kB\nWriteback: 0 kB\nAnonPages: 63396300 kB\nMapped: 16728 kB\nSlab: 1346132 kB\nPageTables: 171508 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86134692 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33462116352}, {buffered_memory,290545664}, {free_memory,1099698176}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3445200475,1}}, {context_switches,{1913821349,0}}, {garbage_collection,{248171163,3901566640411,0}}, {io,{{input,69607782629},{output,132076320630}}}, {reductions,{1586547536237,2403900}}, {run_queue,0}, {runtime,{130398950,190}}]}]}, {'ns_1@192.168.24.62', [{last_heard,{1325,172285,405864}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,84067064}, {processes,14389240}, {processes_used,14353688}, {system,69677824}, {atom,828337}, {atom_used,824728}, {binary,1348648}, {code,7914132}, {ets,38658032}]}, {system_stats, [{cpu_utilization_rate,0.8739076154806492}, {swap_total,8589926400}, {swap_used,10342400}]}, {interesting_stats, [{curr_items,297310020}, {curr_items_tot,297310020}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3445195}, {memory_data,{101368545280,100839555072,{<5981.16810.159>,4714968}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 514396 kB\nBuffers: 284008 kB\nCached: 31394548 kB\nSwapCached: 3836 kB\nActive: 82925708 kB\nInactive: 13856744 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 514396 kB\nSwapTotal: 8388600 kB\nSwapFree: 8378500 kB\nDirty: 160 kB\nWriteback: 0 kB\nAnonPages: 65095988 kB\nMapped: 17876 kB\nSlab: 1381028 kB\nPageTables: 278860 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 141212900 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267496 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8579584000}, {total_swap,8589926400}, {cached_memory,32148017152}, {buffered_memory,290824192}, {free_memory,526741504}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3445185642,1}}, {context_switches,{752230521,0}}, {garbage_collection,{185922072,675645337127,0}}, {io,{{input,58316548093},{output,130654833524}}}, {reductions,{324987678217,1420235}}, {run_queue,0}, {runtime,{37351290,150}}]}]}] INFO REPORT <5883.136.0> 2011-12-29 07:24:54 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.136.0> 2011-12-29 07:25:04 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.111.192> 2011-12-29 07:25:26 =============================================================================== ns_1@192.168.24.61:<5883.111.192>:stats_collector:83: Stats for bucket "JDPCachePrimary": accepting_conns 1 auth_cmds 10369 auth_errors 0 bucket_active_conns 1 bucket_conns 1 bytes_read 53441740990 bytes_written 45383422781 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 29975715 cmd_set 445274018 conn_yields 61205 connection_structures 1931 curr_connections 11 curr_items 297280049 curr_items_tot 297280049 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 466 delete_misses 174808 ep_bg_fetched 1661206 ep_bg_load 4365803470 ep_bg_load_avg 2628 ep_bg_max_load 2020165 ep_bg_max_wait 2929571 ep_bg_min_load 1 ep_bg_min_wait 10 ep_bg_num_samples 1661198 ep_bg_wait 48864908595 ep_bg_wait_avg 29415 ep_commit_num 818500 ep_commit_time 0 ep_commit_time_total 179775 ep_data_age 0 ep_data_age_highwat 2571 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 1 ep_dbname /opt/membase/var/lib/membase/data/JDPCachePrimary-data/JDPCachePrimary ep_dbshards 4 ep_diskqueue_drain 358215763 ep_diskqueue_fill 358215763 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_exp_pager_stime 3600 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 1256 ep_flush_duration_total 251908 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 1661206 ep_io_num_write 357979281 ep_io_read_bytes 145605718 ep_io_write_bytes 6012819006520 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 356086874 ep_kv_size 57356020515 ep_latency_arith_cmd 46093 ep_latency_get_cmd 101813590 ep_latency_store_cmd 515449025 ep_max_data_size 81788928000 ep_max_txn_size 1000 ep_mem_high_wat 61341696000 ep_mem_low_wat 49073356800 ep_min_data_age 0 ep_num_active_non_resident 94237869 ep_num_checkpoint_remover_runs355333 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 491 ep_num_non_resident 94237869 ep_num_not_my_vbuckets 0 ep_num_pager_runs 2 ep_num_value_ejects 95895079 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 1611440768 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 3597 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 62797498 ep_too_young 0 ep_total_cache_size 58930442359 ep_total_del_items 416 ep_total_enqueued 358215763 ep_total_new_items 297280465 ep_total_persisted 357979697 ep_uncommitted_items 0 ep_value_size 15808691965 ep_vb_total 512 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.1.1_39_g4bd26a3 ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 8853 get_hits 7311306 get_misses 22664409 incr_hits 44431 incr_misses 1662 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 58967461283 pid 17177 pointer_size 64 rejected_conns 0 rusage_system 91969.894450 rusage_user 249615.218718 threads 4 time 1325172325 total_connections 30202 uptime 3445256 vb_active_curr_items 297280049 vb_active_eject 95895079 vb_active_ht_memory 1610809344 vb_active_itm_memory 53788643543 vb_active_num 512 vb_active_num_non_resident 94237869 vb_active_ops_create 297280465 vb_active_ops_delete 416 vb_active_ops_reject 0 vb_active_ops_update 60698816 vb_active_perc_mem_resident 68 vb_active_queue_age 0 vb_active_queue_drain 358215763 vb_active_queue_fill 358215763 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version UNKNOWN INFO REPORT <5883.136.0> 2011-12-29 07:25:28 =============================================================================== ns_1@192.168.24.61:<5883.136.0>:ns_config_rep:257: Pulling config from: 'ns_1@192.168.24.62' INFO REPORT <5883.1095.0> 2011-12-29 07:25:50 =============================================================================== ns_1@192.168.24.61:<5883.1095.0>:ns_doctor:86: Current node statuses: [{'ns_1@192.168.24.61', [{last_heard,{1325,172346,408354}}, {active_buckets,["JDPCachePrimary"]}, {ready_buckets,["JDPCachePrimary"]}, {replication,[{"JDPCachePrimary",1.0}]}, {memory, [{total,102705528}, {processes,31920232}, {processes_used,29245960}, {system,70785296}, {atom,903241}, {atom_used,880290}, {binary,1324264}, {code,8604866}, {ets,38851800}]}, {system_stats, [{cpu_utilization_rate,12.983770287141073}, {swap_total,8589926400}, {swap_used,12918784}]}, {interesting_stats, [{curr_items,297280049}, {curr_items_tot,297280049}, {vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.6"}, {mnesia,"4.4.19"}, {inets,"5.6"}, {kernel,"2.14.4"}, {sasl,"2.1.9.4"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.4"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,3445275}, {memory_data,{101368545280,100269993984,{<5883.113.192>,12342432}}}, {disk_data, [{"/",130014692,37},{"/boot",101086,14},{"/dev/shm",49496360,0}]}, {meminfo, <<"MemTotal: 98992720 kB\nMemFree: 1070068 kB\nBuffers: 283736 kB\nCached: 32678648 kB\nSwapCached: 5096 kB\nActive: 82505296 kB\nInactive: 13864056 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 98992720 kB\nLowFree: 1070068 kB\nSwapTotal: 8388600 kB\nSwapFree: 8375984 kB\nDirty: 652 kB\nWriteback: 0 kB\nAnonPages: 63398652 kB\nMapped: 18212 kB\nSlab: 1346100 kB\nPageTables: 171940 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 57884960 kB\nCommitted_AS: 86136080 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 267516 kB\nVmallocChunk: 34359470763 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {system_memory_data, [{system_total_memory,101368545280}, {free_swap,8577007616}, {total_swap,8589926400}, {cached_memory,33462935552}, {buffered_memory,290545664}, {free_memory,1095749632}, {total_memory,101368545280}]}, {statistics, [{wall_clock,{3445260476,1}}, {context_switches,{1913838370,0}}, {garbage_collection,{248174552,390157892