Performance drop to very near 0 when memcachetest runs on the cluster with more than one node

Steve Yen added a comment - 09/Oct/10 3:23 AM stats proxy output from Chiyoung over the course of a reproduction on 2 node linux physical boxes, embedded moxi. [ root@localhost ~]# echo stats proxy | nc 10.2.1.51 11211 | grep -v NULL STAT basic:version 1.6.0beta4_37_g036e93d STAT basic:nthreads 5 STAT basic:hostname localhost.localdomain STAT memcached:settings:maxbytes 67108864 STAT memcached:settings:maxconns 1024 STAT memcached:settings:tcpport 0 STAT memcached:settings:udpport -2 STAT memcached:settings:verbosity 0 STAT memcached:settings:oldest 0 STAT memcached:settings:evictions on STAT memcached:settings:umask 700 STAT memcached:settings:growth_factor 1.25 STAT memcached:settings:chunk_size 48 STAT memcached:settings:num_threads 5 STAT memcached:settings:stat_key_prefix : STAT memcached:settings:detail_enabled no STAT memcached:settings:reqs_per_event 20 STAT memcached:settings:cas_enabled yes STAT memcached:settings:tcp_backlog 1024 STAT memcached:settings:binding_protocol auto-negotiate STAT memcached:stats:pid 2308 STAT memcached:stats:uptime 4473 STAT memcached:stats:time 1286561902 STAT memcached:stats:version 1.6.0beta4_37_g036e93d STAT memcached:stats:pointer_size 64 STAT memcached:stats:rusage_user 491.340304 STAT memcached:stats:rusage_system 1403.897575 STAT memcached:stats:curr_connections 54 STAT memcached:stats:total_connections 244 STAT memcached:stats:connection_structures 68 STAT memcached:stats:cmd_get 0 STAT memcached:stats:cmd_set 0 STAT memcached:stats:cmd_flush 0 STAT memcached:stats:get_hits 0 STAT memcached:stats:get_misses 0 STAT memcached:stats:delete_misses 0 STAT memcached:stats:delete_hits 0 STAT memcached:stats:incr_misses 0 STAT memcached:stats:incr_hits 0 STAT memcached:stats:decr_misses 0 STAT memcached:stats:decr_hits 0 STAT memcached:stats:cas_misses 0 STAT memcached:stats:cas_hits 0 STAT memcached:stats:cas_badval 0 STAT memcached:stats:bytes_read 144109066187 STAT memcached:stats:bytes_written 75412408452 STAT memcached:stats:limit_maxbytes 67108864 STAT memcached:stats:accepting_conns 1 STAT memcached:stats:listen_disabled_num 0 STAT memcached:stats:threads 5 STAT memcached:stats:conn_yields 52 STAT proxy_main:conf_type dynamic STAT proxy_main:behavior:cycle 0 STAT proxy_main:behavior:downstream_max 4 STAT proxy_main:behavior:downstream_conn_max 0 STAT proxy_main:behavior:downstream_weight 0 STAT proxy_main:behavior:downstream_retry 1 STAT proxy_main:behavior:downstream_protocol 8 STAT proxy_main:behavior:downstream_timeout 0 STAT proxy_main:behavior:wait_queue_timeout 0 STAT proxy_main:behavior:time_stats 0 STAT proxy_main:behavior:connect_max_errors 0 STAT proxy_main:behavior:connect_retry_interval 0 STAT proxy_main:behavior:front_cache_max 200 STAT proxy_main:behavior:front_cache_lifespan 0 STAT proxy_main:behavior:front_cache_spec STAT proxy_main:behavior:front_cache_unspec STAT proxy_main:behavior:key_stats_max 4000 STAT proxy_main:behavior:key_stats_lifespan 0 STAT proxy_main:behavior:key_stats_spec STAT proxy_main:behavior:key_stats_unspec STAT proxy_main:behavior:optimize_set STAT proxy_main:behavior:usr Administrator STAT proxy_main:behavior:host STAT proxy_main:behavior:port 0 STAT proxy_main:behavior:bucket STAT proxy_main:behavior:port_listen 11211 STAT proxy_main:behavior:default_bucket_name default STAT proxy_main:stats:stat_configs 0 STAT proxy_main:stats:stat_config_fails 0 STAT proxy_main:stats:stat_proxy_starts 0 STAT proxy_main:stats:stat_proxy_start_fails 0 STAT proxy_main:stats:stat_proxy_existings 0 STAT proxy_main:stats:stat_proxy_shutdowns 0 STAT 11211:default:info:port 11211 STAT 11211:default:info:name default STAT 11211:default:info:config { "name": "default", "nodeLocator": "vbucket", "saslPassword": "", "nodes": [{ "clusterMembership": "active", "status":"h STAT 11211:default:info:config_ver 22 STAT 11211:default:info:behaviors_num 2 STAT 11211:default:behavior:downstream_max 4 STAT 11211:default:behavior:downstream_conn_max 0 STAT 11211:default:behavior:downstream_weight 0 STAT 11211:default:behavior:downstream_retry 1 STAT 11211:default:behavior:downstream_protocol 8 STAT 11211:default:behavior:downstream_timeout 0 STAT 11211:default:behavior:wait_queue_timeout 0 STAT 11211:default:behavior:time_stats 0 STAT 11211:default:behavior:connect_max_errors 0 STAT 11211:default:behavior:connect_retry_interval 0 STAT 11211:default:behavior:front_cache_max 200 STAT 11211:default:behavior:front_cache_lifespan 0 STAT 11211:default:behavior:front_cache_spec STAT 11211:default:behavior:front_cache_unspec STAT 11211:default:behavior:key_stats_max 4000 STAT 11211:default:behavior:key_stats_lifespan 0 STAT 11211:default:behavior:key_stats_spec STAT 11211:default:behavior:key_stats_unspec STAT 11211:default:behavior:optimize_set STAT 11211:default:behavior:usr default STAT 11211:default:behavior:host STAT 11211:default:behavior:port 0 STAT 11211:default:behavior:bucket STAT 11211:default:behavior:port_listen 11211 STAT 11211:default:behavior:default_bucket_name default STAT 11211:default:behavior-0:downstream_weight 0 STAT 11211:default:behavior-0:downstream_retry 1 STAT 11211:default:behavior-0:downstream_protocol 8 STAT 11211:default:behavior-0:downstream_timeout 0 STAT 11211:default:behavior-0:usr default STAT 11211:default:behavior-0:host 10.2.1.51 STAT 11211:default:behavior-0:port 11210 STAT 11211:default:behavior-0:bucket STAT 11211:default:behavior-1:downstream_weight 0 STAT 11211:default:behavior-1:downstream_retry 1 STAT 11211:default:behavior-1:downstream_protocol 8 STAT 11211:default:behavior-1:downstream_timeout 0 STAT 11211:default:behavior-1:usr default STAT 11211:default:behavior-1:host 10.2.1.54 STAT 11211:default:behavior-1:port 11210 STAT 11211:default:behavior-1:bucket STAT 11211:default:stats:listening 2 STAT 11211:default:stats:listening_failed 0 STAT 11211:default:frontcache:max 0 STAT 11211:default:frontcache:oldest_live 0 STAT 11211:default:frontcache:tot_get_hits 0 STAT 11211:default:frontcache:tot_get_expires 0 STAT 11211:default:frontcache:tot_get_misses 0 STAT 11211:default:frontcache:tot_get_bytes 0 STAT 11211:default:frontcache:tot_adds 0 STAT 11211:default:frontcache:tot_add_skips 0 STAT 11211:default:frontcache:tot_add_fails 0 STAT 11211:default:frontcache:tot_add_bytes 0 STAT 11211:default:frontcache:tot_deletes 0 STAT 11211:default:frontcache:tot_evictions 0 STAT 11211:default:pstd_stats:num_upstream 19 STAT 11211:default:pstd_stats:tot_upstream 1 STAT 11211:default:pstd_stats:num_downstream_conn 94 STAT 11211:default:pstd_stats:tot_downstream_conn 0 STAT 11211:default:pstd_stats:tot_downstream_conn_acquired 133541 STAT 11211:default:pstd_stats:tot_downstream_conn_released 133551 STAT 11211:default:pstd_stats:tot_downstream_released 133551 STAT 11211:default:pstd_stats:tot_downstream_reserved 133541 STAT 11211:default:pstd_stats:tot_downstream_reserved_time 0 STAT 11211:default:pstd_stats:max_downstream_reserved_time 0 STAT 11211:default:pstd_stats:tot_downstream_freed 0 STAT 11211:default:pstd_stats:tot_downstream_quit_server 0 STAT 11211:default:pstd_stats:tot_downstream_max_reached 0 STAT 11211:default:pstd_stats:tot_downstream_create_failed 0 STAT 11211:default:pstd_stats:tot_downstream_connect 0 STAT 11211:default:pstd_stats:tot_downstream_connect_failed 0 STAT 11211:default:pstd_stats:tot_downstream_connect_timeout 0 STAT 11211:default:pstd_stats:tot_downstream_connect_interval 0 STAT 11211:default:pstd_stats:tot_downstream_connect_max_reached 0 STAT 11211:default:pstd_stats:tot_downstream_waiting_errors 0 STAT 11211:default:pstd_stats:tot_downstream_auth 0 STAT 11211:default:pstd_stats:tot_downstream_auth_failed 0 STAT 11211:default:pstd_stats:tot_downstream_bucket 0 STAT 11211:default:pstd_stats:tot_downstream_bucket_failed 0 STAT 11211:default:pstd_stats:tot_downstream_propagate_failed 0 STAT 11211:default:pstd_stats:tot_downstream_close_on_upstream_close 0 STAT 11211:default:pstd_stats:tot_downstream_timeout 0 STAT 11211:default:pstd_stats:tot_wait_queue_timeout 0 STAT 11211:default:pstd_stats:tot_assign_downstream 133541 STAT 11211:default:pstd_stats:tot_assign_upstream 133541 STAT 11211:default:pstd_stats:tot_assign_recursion 0 STAT 11211:default:pstd_stats:tot_reset_upstream_avail 0 STAT 11211:default:pstd_stats:tot_multiget_keys 0 STAT 11211:default:pstd_stats:tot_multiget_keys_dedupe 0 STAT 11211:default:pstd_stats:tot_multiget_bytes_dedupe 0 STAT 11211:default:pstd_stats:tot_optimize_sets 0 STAT 11211:default:pstd_stats:tot_optimize_self 0 STAT 11211:default:pstd_stats:tot_retry 0 STAT 11211:default:pstd_stats:tot_retry_time 0 STAT 11211:default:pstd_stats:max_retry_time 0 STAT 11211:default:pstd_stats:tot_retry_vbucket 0 STAT 11211:default:pstd_stats:tot_upstream_paused 133541 STAT 11211:default:pstd_stats:tot_upstream_unpaused 133551 STAT 11211:default:pstd_stats:err_oom 0 STAT 11211:default:pstd_stats:err_upstream_write_prep 0 STAT 11211:default:pstd_stats:err_downstream_write_prep 0 STAT 11211:default:pstd_stats_cmd:regular_get:seen 89467 STAT 11211:default:pstd_stats_cmd:regular_get_key:hits 89473 STAT 11211:default:pstd_stats_cmd:regular_get_key:write_bytes 183756910 STAT 11211:default:pstd_stats_cmd:regular_set:seen 44074 STAT 11211:default:pstd_stats_cmd:regular_set:read_bytes 90774438 END [ root@localhost ~]# echo after rebalanced in 2nd node, and reset stats after rebalanced in 2nd node, and reset stats [ root@localhost ~]# echo stats proxy | nc 10.2.1.51 11211 | grep -v NULL STAT basic:version 1.6.0beta4_37_g036e93d STAT basic:nthreads 5 STAT basic:hostname localhost.localdomain STAT memcached:settings:maxbytes 67108864 STAT memcached:settings:maxconns 1024 STAT memcached:settings:tcpport 0 STAT memcached:settings:udpport -2 STAT memcached:settings:verbosity 0 STAT memcached:settings:oldest 0 STAT memcached:settings:evictions on STAT memcached:settings:umask 700 STAT memcached:settings:growth_factor 1.25 STAT memcached:settings:chunk_size 48 STAT memcached:settings:num_threads 5 STAT memcached:settings:stat_key_prefix : STAT memcached:settings:detail_enabled no STAT memcached:settings:reqs_per_event 20 STAT memcached:settings:cas_enabled yes STAT memcached:settings:tcp_backlog 1024 STAT memcached:settings:binding_protocol auto-negotiate STAT memcached:stats:pid 2308 STAT memcached:stats:uptime 4566 STAT memcached:stats:time 1286561995 STAT memcached:stats:version 1.6.0beta4_37_g036e93d STAT memcached:stats:pointer_size 64 STAT memcached:stats:rusage_user 523.836364 STAT memcached:stats:rusage_system 1498.278227 STAT memcached:stats:curr_connections 54 STAT memcached:stats:total_connections 245 STAT memcached:stats:connection_structures 68 STAT memcached:stats:cmd_get 0 STAT memcached:stats:cmd_set 0 STAT memcached:stats:cmd_flush 0 STAT memcached:stats:get_hits 0 STAT memcached:stats:get_misses 0 STAT memcached:stats:delete_misses 0 STAT memcached:stats:delete_hits 0 STAT memcached:stats:incr_misses 0 STAT memcached:stats:incr_hits 0 STAT memcached:stats:decr_misses 0 STAT memcached:stats:decr_hits 0 STAT memcached:stats:cas_misses 0 STAT memcached:stats:cas_hits 0 STAT memcached:stats:cas_badval 0 STAT memcached:stats:bytes_read 153558925599 STAT memcached:stats:bytes_written 80381334366 STAT memcached:stats:limit_maxbytes 67108864 STAT memcached:stats:accepting_conns 1 STAT memcached:stats:listen_disabled_num 0 STAT memcached:stats:threads 5 STAT memcached:stats:conn_yields 52 STAT proxy_main:conf_type dynamic STAT proxy_main:behavior:cycle 0 STAT proxy_main:behavior:downstream_max 4 STAT proxy_main:behavior:downstream_conn_max 0 STAT proxy_main:behavior:downstream_weight 0 STAT proxy_main:behavior:downstream_retry 1 STAT proxy_main:behavior:downstream_protocol 8 STAT proxy_main:behavior:downstream_timeout 0 STAT proxy_main:behavior:wait_queue_timeout 0 STAT proxy_main:behavior:time_stats 0 STAT proxy_main:behavior:connect_max_errors 0 STAT proxy_main:behavior:connect_retry_interval 0 STAT proxy_main:behavior:front_cache_max 200 STAT proxy_main:behavior:front_cache_lifespan 0 STAT proxy_main:behavior:front_cache_spec STAT proxy_main:behavior:front_cache_unspec STAT proxy_main:behavior:key_stats_max 4000 STAT proxy_main:behavior:key_stats_lifespan 0 STAT proxy_main:behavior:key_stats_spec STAT proxy_main:behavior:key_stats_unspec STAT proxy_main:behavior:optimize_set STAT proxy_main:behavior:usr Administrator STAT proxy_main:behavior:host STAT proxy_main:behavior:port 0 STAT proxy_main:behavior:bucket STAT proxy_main:behavior:port_listen 11211 STAT proxy_main:behavior:default_bucket_name default STAT proxy_main:stats:stat_configs 0 STAT proxy_main:stats:stat_config_fails 0 STAT proxy_main:stats:stat_proxy_starts 0 STAT proxy_main:stats:stat_proxy_start_fails 0 STAT proxy_main:stats:stat_proxy_existings 0 STAT proxy_main:stats:stat_proxy_shutdowns 0 STAT 11211:default:info:port 11211 STAT 11211:default:info:name default STAT 11211:default:info:config { "name": "default", "nodeLocator": "vbucket", "saslPassword": "", "nodes": [{ "clusterMembership": "active", "status":"h STAT 11211:default:info:config_ver 22 STAT 11211:default:info:behaviors_num 2 STAT 11211:default:behavior:downstream_max 4 STAT 11211:default:behavior:downstream_conn_max 0 STAT 11211:default:behavior:downstream_weight 0 STAT 11211:default:behavior:downstream_retry 1 STAT 11211:default:behavior:downstream_protocol 8 STAT 11211:default:behavior:downstream_timeout 0 STAT 11211:default:behavior:wait_queue_timeout 0 STAT 11211:default:behavior:time_stats 0 STAT 11211:default:behavior:connect_max_errors 0 STAT 11211:default:behavior:connect_retry_interval 0 STAT 11211:default:behavior:front_cache_max 200 STAT 11211:default:behavior:front_cache_lifespan 0 STAT 11211:default:behavior:front_cache_spec STAT 11211:default:behavior:front_cache_unspec STAT 11211:default:behavior:key_stats_max 4000 STAT 11211:default:behavior:key_stats_lifespan 0 STAT 11211:default:behavior:key_stats_spec STAT 11211:default:behavior:key_stats_unspec STAT 11211:default:behavior:optimize_set STAT 11211:default:behavior:usr default STAT 11211:default:behavior:host STAT 11211:default:behavior:port 0 STAT 11211:default:behavior:bucket STAT 11211:default:behavior:port_listen 11211 STAT 11211:default:behavior:default_bucket_name default STAT 11211:default:behavior-0:downstream_weight 0 STAT 11211:default:behavior-0:downstream_retry 1 STAT 11211:default:behavior-0:downstream_protocol 8 STAT 11211:default:behavior-0:downstream_timeout 0 STAT 11211:default:behavior-0:usr default STAT 11211:default:behavior-0:host 10.2.1.51 STAT 11211:default:behavior-0:port 11210 STAT 11211:default:behavior-0:bucket STAT 11211:default:behavior-1:downstream_weight 0 STAT 11211:default:behavior-1:downstream_retry 1 STAT 11211:default:behavior-1:downstream_protocol 8 STAT 11211:default:behavior-1:downstream_timeout 0 STAT 11211:default:behavior-1:usr default STAT 11211:default:behavior-1:host 10.2.1.54 STAT 11211:default:behavior-1:port 11210 STAT 11211:default:behavior-1:bucket STAT 11211:default:stats:listening 2 STAT 11211:default:stats:listening_failed 0 STAT 11211:default:frontcache:max 0 STAT 11211:default:frontcache:oldest_live 0 STAT 11211:default:frontcache:tot_get_hits 0 STAT 11211:default:frontcache:tot_get_expires 0 STAT 11211:default:frontcache:tot_get_misses 0 STAT 11211:default:frontcache:tot_get_bytes 0 STAT 11211:default:frontcache:tot_adds 0 STAT 11211:default:frontcache:tot_add_skips 0 STAT 11211:default:frontcache:tot_add_fails 0 STAT 11211:default:frontcache:tot_add_bytes 0 STAT 11211:default:frontcache:tot_deletes 0 STAT 11211:default:frontcache:tot_evictions 0 STAT 11211:default:pstd_stats:num_upstream 19 STAT 11211:default:pstd_stats:tot_upstream 2 STAT 11211:default:pstd_stats:num_downstream_conn 94 STAT 11211:default:pstd_stats:tot_downstream_conn 0 STAT 11211:default:pstd_stats:tot_downstream_conn_acquired 2495772 STAT 11211:default:pstd_stats:tot_downstream_conn_released 2495770 STAT 11211:default:pstd_stats:tot_downstream_released 2495770 STAT 11211:default:pstd_stats:tot_downstream_reserved 2495772 STAT 11211:default:pstd_stats:tot_downstream_reserved_time 0 STAT 11211:default:pstd_stats:max_downstream_reserved_time 0 STAT 11211:default:pstd_stats:tot_downstream_freed 0 STAT 11211:default:pstd_stats:tot_downstream_quit_server 0 STAT 11211:default:pstd_stats:tot_downstream_max_reached 0 STAT 11211:default:pstd_stats:tot_downstream_create_failed 0 STAT 11211:default:pstd_stats:tot_downstream_connect 0 STAT 11211:default:pstd_stats:tot_downstream_connect_failed 0 STAT 11211:default:pstd_stats:tot_downstream_connect_timeout 0 STAT 11211:default:pstd_stats:tot_downstream_connect_interval 0 STAT 11211:default:pstd_stats:tot_downstream_connect_max_reached 0 STAT 11211:default:pstd_stats:tot_downstream_waiting_errors 0 STAT 11211:default:pstd_stats:tot_downstream_auth 0 STAT 11211:default:pstd_stats:tot_downstream_auth_failed 0 STAT 11211:default:pstd_stats:tot_downstream_bucket 0 STAT 11211:default:pstd_stats:tot_downstream_bucket_failed 0 STAT 11211:default:pstd_stats:tot_downstream_propagate_failed 0 STAT 11211:default:pstd_stats:tot_downstream_close_on_upstream_close 0 STAT 11211:default:pstd_stats:tot_downstream_timeout 0 STAT 11211:default:pstd_stats:tot_wait_queue_timeout 0 STAT 11211:default:pstd_stats:tot_assign_downstream 2495772 STAT 11211:default:pstd_stats:tot_assign_upstream 2495772 STAT 11211:default:pstd_stats:tot_assign_recursion 0 STAT 11211:default:pstd_stats:tot_reset_upstream_avail 0 STAT 11211:default:pstd_stats:tot_multiget_keys 0 STAT 11211:default:pstd_stats:tot_multiget_keys_dedupe 0 STAT 11211:default:pstd_stats:tot_multiget_bytes_dedupe 0 STAT 11211:default:pstd_stats:tot_optimize_sets 0 STAT 11211:default:pstd_stats:tot_optimize_self 0 STAT 11211:default:pstd_stats:tot_retry 0 STAT 11211:default:pstd_stats:tot_retry_time 0 STAT 11211:default:pstd_stats:max_retry_time 0 STAT 11211:default:pstd_stats:tot_retry_vbucket 0 STAT 11211:default:pstd_stats:tot_upstream_paused 2495772 STAT 11211:default:pstd_stats:tot_upstream_unpaused 2495770 STAT 11211:default:pstd_stats:err_oom 0 STAT 11211:default:pstd_stats:err_upstream_write_prep 0 STAT 11211:default:pstd_stats:err_downstream_write_prep 0 STAT 11211:default:pstd_stats_cmd:regular_get:seen 1672160 STAT 11211:default:pstd_stats_cmd:regular_get_key:hits 1672153 STAT 11211:default:pstd_stats_cmd:regular_get_key:write_bytes 3432516902 STAT 11211:default:pstd_stats_cmd:regular_set:seen 823612 STAT 11211:default:pstd_stats_cmd:regular_set:read_bytes 1703725606 STAT 11211:default:pstd_stats_cmd:regular_stats:seen 1 STAT 11211:default:pstd_stats_cmd:regular_stats:read_bytes 11 END [ root@localhost ~]# echo bad perf started in last sample bad perf started in last sample [ root@localhost ~]# [ root@localhost ~]# [ root@localhost ~]# [ root@localhost ~]# [ root@localhost ~]# [ root@localhost ~]# [ root@localhost ~]# [ root@localhost ~]# [ root@localhost ~]# echo stats proxy | nc 10.2.1.51 11211 | grep -v NULL STAT basic:version 1.6.0beta4_37_g036e93d STAT basic:nthreads 5 STAT basic:hostname localhost.localdomain STAT memcached:settings:maxbytes 67108864 STAT memcached:settings:maxconns 1024 STAT memcached:settings:tcpport 0 STAT memcached:settings:udpport -2 STAT memcached:settings:verbosity 0 STAT memcached:settings:oldest 0 STAT memcached:settings:evictions on STAT memcached:settings:umask 700 STAT memcached:settings:growth_factor 1.25 STAT memcached:settings:chunk_size 48 STAT memcached:settings:num_threads 5 STAT memcached:settings:stat_key_prefix : STAT memcached:settings:detail_enabled no STAT memcached:settings:reqs_per_event 20 STAT memcached:settings:cas_enabled yes STAT memcached:settings:tcp_backlog 1024 STAT memcached:settings:binding_protocol auto-negotiate STAT memcached:stats:pid 2308 STAT memcached:stats:uptime 4587 STAT memcached:stats:time 1286562016 STAT memcached:stats:version 1.6.0beta4_37_g036e93d STAT memcached:stats:pointer_size 64 STAT memcached:stats:rusage_user 524.554255 STAT memcached:stats:rusage_system 1500.171939 STAT memcached:stats:curr_connections 54 STAT memcached:stats:total_connections 246 STAT memcached:stats:connection_structures 68 STAT memcached:stats:cmd_get 0 STAT memcached:stats:cmd_set 0 STAT memcached:stats:cmd_flush 0 STAT memcached:stats:get_hits 0 STAT memcached:stats:get_misses 0 STAT memcached:stats:delete_misses 0 STAT memcached:stats:delete_hits 0 STAT memcached:stats:incr_misses 0 STAT memcached:stats:incr_hits 0 STAT memcached:stats:decr_misses 0 STAT memcached:stats:decr_hits 0 STAT memcached:stats:cas_misses 0 STAT memcached:stats:cas_hits 0 STAT memcached:stats:cas_badval 0 STAT memcached:stats:bytes_read 153752278417 STAT memcached:stats:bytes_written 80482394512 STAT memcached:stats:limit_maxbytes 67108864 STAT memcached:stats:accepting_conns 1 STAT memcached:stats:listen_disabled_num 0 STAT memcached:stats:threads 5 STAT memcached:stats:conn_yields 52 STAT proxy_main:conf_type dynamic STAT proxy_main:behavior:cycle 0 STAT proxy_main:behavior:downstream_max 4 STAT proxy_main:behavior:downstream_conn_max 0 STAT proxy_main:behavior:downstream_weight 0 STAT proxy_main:behavior:downstream_retry 1 STAT proxy_main:behavior:downstream_protocol 8 STAT proxy_main:behavior:downstream_timeout 0 STAT proxy_main:behavior:wait_queue_timeout 0 STAT proxy_main:behavior:time_stats 0 STAT proxy_main:behavior:connect_max_errors 0 STAT proxy_main:behavior:connect_retry_interval 0 STAT proxy_main:behavior:front_cache_max 200 STAT proxy_main:behavior:front_cache_lifespan 0 STAT proxy_main:behavior:front_cache_spec STAT proxy_main:behavior:front_cache_unspec STAT proxy_main:behavior:key_stats_max 4000 STAT proxy_main:behavior:key_stats_lifespan 0 STAT proxy_main:behavior:key_stats_spec STAT proxy_main:behavior:key_stats_unspec STAT proxy_main:behavior:optimize_set STAT proxy_main:behavior:usr Administrator STAT proxy_main:behavior:host STAT proxy_main:behavior:port 0 STAT proxy_main:behavior:bucket STAT proxy_main:behavior:port_listen 11211 STAT proxy_main:behavior:default_bucket_name default STAT proxy_main:stats:stat_configs 0 STAT proxy_main:stats:stat_config_fails 0 STAT proxy_main:stats:stat_proxy_starts 0 STAT proxy_main:stats:stat_proxy_start_fails 0 STAT proxy_main:stats:stat_proxy_existings 0 STAT proxy_main:stats:stat_proxy_shutdowns 0 STAT 11211:default:info:port 11211 STAT 11211:default:info:name default STAT 11211:default:info:config { "name": "default", "nodeLocator": "vbucket", "saslPassword": "", "nodes": [{ "clusterMembership": "active", "status":"h STAT 11211:default:info:config_ver 22 STAT 11211:default:info:behaviors_num 2 STAT 11211:default:behavior:downstream_max 4 STAT 11211:default:behavior:downstream_conn_max 0 STAT 11211:default:behavior:downstream_weight 0 STAT 11211:default:behavior:downstream_retry 1 STAT 11211:default:behavior:downstream_protocol 8 STAT 11211:default:behavior:downstream_timeout 0 STAT 11211:default:behavior:wait_queue_timeout 0 STAT 11211:default:behavior:time_stats 0 STAT 11211:default:behavior:connect_max_errors 0 STAT 11211:default:behavior:connect_retry_interval 0 STAT 11211:default:behavior:front_cache_max 200 STAT 11211:default:behavior:front_cache_lifespan 0 STAT 11211:default:behavior:front_cache_spec STAT 11211:default:behavior:front_cache_unspec STAT 11211:default:behavior:key_stats_max 4000 STAT 11211:default:behavior:key_stats_lifespan 0 STAT 11211:default:behavior:key_stats_spec STAT 11211:default:behavior:key_stats_unspec STAT 11211:default:behavior:optimize_set STAT 11211:default:behavior:usr default STAT 11211:default:behavior:host STAT 11211:default:behavior:port 0 STAT 11211:default:behavior:bucket STAT 11211:default:behavior:port_listen 11211 STAT 11211:default:behavior:default_bucket_name default STAT 11211:default:behavior-0:downstream_weight 0 STAT 11211:default:behavior-0:downstream_retry 1 STAT 11211:default:behavior-0:downstream_protocol 8 STAT 11211:default:behavior-0:downstream_timeout 0 STAT 11211:default:behavior-0:usr default STAT 11211:default:behavior-0:host 10.2.1.51 STAT 11211:default:behavior-0:port 11210 STAT 11211:default:behavior-0:bucket STAT 11211:default:behavior-1:downstream_weight 0 STAT 11211:default:behavior-1:downstream_retry 1 STAT 11211:default:behavior-1:downstream_protocol 8 STAT 11211:default:behavior-1:downstream_timeout 0 STAT 11211:default:behavior-1:usr default STAT 11211:default:behavior-1:host 10.2.1.54 STAT 11211:default:behavior-1:port 11210 STAT 11211:default:behavior-1:bucket STAT 11211:default:stats:listening 2 STAT 11211:default:stats:listening_failed 0 STAT 11211:default:frontcache:max 0 STAT 11211:default:frontcache:oldest_live 0 STAT 11211:default:frontcache:tot_get_hits 0 STAT 11211:default:frontcache:tot_get_expires 0 STAT 11211:default:frontcache:tot_get_misses 0 STAT 11211:default:frontcache:tot_get_bytes 0 STAT 11211:default:frontcache:tot_adds 0 STAT 11211:default:frontcache:tot_add_skips 0 STAT 11211:default:frontcache:tot_add_fails 0 STAT 11211:default:frontcache:tot_add_bytes 0 STAT 11211:default:frontcache:tot_deletes 0 STAT 11211:default:frontcache:tot_evictions 0 STAT 11211:default:pstd_stats:num_upstream 19 STAT 11211:default:pstd_stats:tot_upstream 3 STAT 11211:default:pstd_stats:num_downstream_conn 94 STAT 11211:default:pstd_stats:tot_downstream_conn 0 STAT 11211:default:pstd_stats:tot_downstream_conn_acquired 2544059 STAT 11211:default:pstd_stats:tot_downstream_conn_released 2544057 STAT 11211:default:pstd_stats:tot_downstream_released 2544057 STAT 11211:default:pstd_stats:tot_downstream_reserved 2544059 STAT 11211:default:pstd_stats:tot_downstream_reserved_time 0 STAT 11211:default:pstd_stats:max_downstream_reserved_time 0 STAT 11211:default:pstd_stats:tot_downstream_freed 0 STAT 11211:default:pstd_stats:tot_downstream_quit_server 0 STAT 11211:default:pstd_stats:tot_downstream_max_reached 0 STAT 11211:default:pstd_stats:tot_downstream_create_failed 0 STAT 11211:default:pstd_stats:tot_downstream_connect 0 STAT 11211:default:pstd_stats:tot_downstream_connect_failed 0 STAT 11211:default:pstd_stats:tot_downstream_connect_timeout 0 STAT 11211:default:pstd_stats:tot_downstream_connect_interval 0 STAT 11211:default:pstd_stats:tot_downstream_connect_max_reached 0 STAT 11211:default:pstd_stats:tot_downstream_waiting_errors 0 STAT 11211:default:pstd_stats:tot_downstream_auth 0 STAT 11211:default:pstd_stats:tot_downstream_auth_failed 0 STAT 11211:default:pstd_stats:tot_downstream_bucket 0 STAT 11211:default:pstd_stats:tot_downstream_bucket_failed 0 STAT 11211:default:pstd_stats:tot_downstream_propagate_failed 0 STAT 11211:default:pstd_stats:tot_downstream_close_on_upstream_close 0 STAT 11211:default:pstd_stats:tot_downstream_timeout 0 STAT 11211:default:pstd_stats:tot_wait_queue_timeout 0 STAT 11211:default:pstd_stats:tot_assign_downstream 2544059 STAT 11211:default:pstd_stats:tot_assign_upstream 2544059 STAT 11211:default:pstd_stats:tot_assign_recursion 0 STAT 11211:default:pstd_stats:tot_reset_upstream_avail 0 STAT 11211:default:pstd_stats:tot_multiget_keys 0 STAT 11211:default:pstd_stats:tot_multiget_keys_dedupe 0 STAT 11211:default:pstd_stats:tot_multiget_bytes_dedupe 0 STAT 11211:default:pstd_stats:tot_optimize_sets 0 STAT 11211:default:pstd_stats:tot_optimize_self 0 STAT 11211:default:pstd_stats:tot_retry 0 STAT 11211:default:pstd_stats:tot_retry_time 0 STAT 11211:default:pstd_stats:max_retry_time 0 STAT 11211:default:pstd_stats:tot_retry_vbucket 0 STAT 11211:default:pstd_stats:tot_upstream_paused 2544059 STAT 11211:default:pstd_stats:tot_upstream_unpaused 2544057 STAT 11211:default:pstd_stats:err_oom 0 STAT 11211:default:pstd_stats:err_upstream_write_prep 0 STAT 11211:default:pstd_stats:err_downstream_write_prep 0 STAT 11211:default:pstd_stats_cmd:regular_get:seen 1704533 STAT 11211:default:pstd_stats_cmd:regular_get_key:hits 1704526 STAT 11211:default:pstd_stats_cmd:regular_get_key:write_bytes 3498592982 STAT 11211:default:pstd_stats_cmd:regular_set:seen 839526 STAT 11211:default:pstd_stats_cmd:regular_set:read_bytes 1736502640 STAT 11211:default:pstd_stats_cmd:regular_stats:seen 2 STAT 11211:default:pstd_stats_cmd:regular_stats:read_bytes 22 END [ root@localhost ~]# echo disk fetches per sec popped up disk fetches per sec popped up [ root@localhost ~]# [ root@localhost ~]# [ root@localhost ~]# [ root@localhost ~]# echo stats proxy | nc 10.2.1.51 11211 | grep -v NULL STAT basic:version 1.6.0beta4_37_g036e93d STAT basic:nthreads 5 STAT basic:hostname localhost.localdomain STAT memcached:settings:maxbytes 67108864 STAT memcached:settings:maxconns 1024 STAT memcached:settings:tcpport 0 STAT memcached:settings:udpport -2 STAT memcached:settings:verbosity 0 STAT memcached:settings:oldest 0 STAT memcached:settings:evictions on STAT memcached:settings:umask 700 STAT memcached:settings:growth_factor 1.25 STAT memcached:settings:chunk_size 48 STAT memcached:settings:num_threads 5 STAT memcached:settings:stat_key_prefix : STAT memcached:settings:detail_enabled no STAT memcached:settings:reqs_per_event 20 STAT memcached:settings:cas_enabled yes STAT memcached:settings:tcp_backlog 1024 STAT memcached:settings:binding_protocol auto-negotiate STAT memcached:stats:pid 2308 STAT memcached:stats:uptime 4661 STAT memcached:stats:time 1286562090 STAT memcached:stats:version 1.6.0beta4_37_g036e93d STAT memcached:stats:pointer_size 64 STAT memcached:stats:rusage_user 524.937197 STAT memcached:stats:rusage_system 1501.221779 STAT memcached:stats:curr_connections 54 STAT memcached:stats:total_connections 247 STAT memcached:stats:connection_structures 68 STAT memcached:stats:cmd_get 0 STAT memcached:stats:cmd_set 0 STAT memcached:stats:cmd_flush 0 STAT memcached:stats:get_hits 0 STAT memcached:stats:get_misses 0 STAT memcached:stats:delete_misses 0 STAT memcached:stats:delete_hits 0 STAT memcached:stats:incr_misses 0 STAT memcached:stats:incr_hits 0 STAT memcached:stats:decr_misses 0 STAT memcached:stats:decr_hits 0 STAT memcached:stats:cas_misses 0 STAT memcached:stats:cas_hits 0 STAT memcached:stats:cas_badval 0 STAT memcached:stats:bytes_read 153853121070 STAT memcached:stats:bytes_written 80535312691 STAT memcached:stats:limit_maxbytes 67108864 STAT memcached:stats:accepting_conns 1 STAT memcached:stats:listen_disabled_num 0 STAT memcached:stats:threads 5 STAT memcached:stats:conn_yields 52 STAT proxy_main:conf_type dynamic STAT proxy_main:behavior:cycle 0 STAT proxy_main:behavior:downstream_max 4 STAT proxy_main:behavior:downstream_conn_max 0 STAT proxy_main:behavior:downstream_weight 0 STAT proxy_main:behavior:downstream_retry 1 STAT proxy_main:behavior:downstream_protocol 8 STAT proxy_main:behavior:downstream_timeout 0 STAT proxy_main:behavior:wait_queue_timeout 0 STAT proxy_main:behavior:time_stats 0 STAT proxy_main:behavior:connect_max_errors 0 STAT proxy_main:behavior:connect_retry_interval 0 STAT proxy_main:behavior:front_cache_max 200 STAT proxy_main:behavior:front_cache_lifespan 0 STAT proxy_main:behavior:front_cache_spec STAT proxy_main:behavior:front_cache_unspec STAT proxy_main:behavior:key_stats_max 4000 STAT proxy_main:behavior:key_stats_lifespan 0 STAT proxy_main:behavior:key_stats_spec STAT proxy_main:behavior:key_stats_unspec STAT proxy_main:behavior:optimize_set STAT proxy_main:behavior:usr Administrator STAT proxy_main:behavior:host STAT proxy_main:behavior:port 0 STAT proxy_main:behavior:bucket STAT proxy_main:behavior:port_listen 11211 STAT proxy_main:behavior:default_bucket_name default STAT proxy_main:stats:stat_configs 0 STAT proxy_main:stats:stat_config_fails 0 STAT proxy_main:stats:stat_proxy_starts 0 STAT proxy_main:stats:stat_proxy_start_fails 0 STAT proxy_main:stats:stat_proxy_existings 0 STAT proxy_main:stats:stat_proxy_shutdowns 0 STAT 11211:default:info:port 11211 STAT 11211:default:info:name default STAT 11211:default:info:config { "name": "default", "nodeLocator": "vbucket", "saslPassword": "", "nodes": [{ "clusterMembership": "active", "status":"h STAT 11211:default:info:config_ver 22 STAT 11211:default:info:behaviors_num 2 STAT 11211:default:behavior:downstream_max 4 STAT 11211:default:behavior:downstream_conn_max 0 STAT 11211:default:behavior:downstream_weight 0 STAT 11211:default:behavior:downstream_retry 1 STAT 11211:default:behavior:downstream_protocol 8 STAT 11211:default:behavior:downstream_timeout 0 STAT 11211:default:behavior:wait_queue_timeout 0 STAT 11211:default:behavior:time_stats 0 STAT 11211:default:behavior:connect_max_errors 0 STAT 11211:default:behavior:connect_retry_interval 0 STAT 11211:default:behavior:front_cache_max 200 STAT 11211:default:behavior:front_cache_lifespan 0 STAT 11211:default:behavior:front_cache_spec STAT 11211:default:behavior:front_cache_unspec STAT 11211:default:behavior:key_stats_max 4000 STAT 11211:default:behavior:key_stats_lifespan 0 STAT 11211:default:behavior:key_stats_spec STAT 11211:default:behavior:key_stats_unspec STAT 11211:default:behavior:optimize_set STAT 11211:default:behavior:usr default STAT 11211:default:behavior:host STAT 11211:default:behavior:port 0 STAT 11211:default:behavior:bucket STAT 11211:default:behavior:port_listen 11211 STAT 11211:default:behavior:default_bucket_name default STAT 11211:default:behavior-0:downstream_weight 0 STAT 11211:default:behavior-0:downstream_retry 1 STAT 11211:default:behavior-0:downstream_protocol 8 STAT 11211:default:behavior-0:downstream_timeout 0 STAT 11211:default:behavior-0:usr default STAT 11211:default:behavior-0:host 10.2.1.51 STAT 11211:default:behavior-0:port 11210 STAT 11211:default:behavior-0:bucket STAT 11211:default:behavior-1:downstream_weight 0 STAT 11211:default:behavior-1:downstream_retry 1 STAT 11211:default:behavior-1:downstream_protocol 8 STAT 11211:default:behavior-1:downstream_timeout 0 STAT 11211:default:behavior-1:usr default STAT 11211:default:behavior-1:host 10.2.1.54 STAT 11211:default:behavior-1:port 11210 STAT 11211:default:behavior-1:bucket STAT 11211:default:stats:listening 2 STAT 11211:default:stats:listening_failed 0 STAT 11211:default:frontcache:max 0 STAT 11211:default:frontcache:oldest_live 0 STAT 11211:default:frontcache:tot_get_hits 0 STAT 11211:default:frontcache:tot_get_expires 0 STAT 11211:default:frontcache:tot_get_misses 0 STAT 11211:default:frontcache:tot_get_bytes 0 STAT 11211:default:frontcache:tot_adds 0 STAT 11211:default:frontcache:tot_add_skips 0 STAT 11211:default:frontcache:tot_add_fails 0 STAT 11211:default:frontcache:tot_add_bytes 0 STAT 11211:default:frontcache:tot_deletes 0 STAT 11211:default:frontcache:tot_evictions 0 STAT 11211:default:pstd_stats:num_upstream 19 STAT 11211:default:pstd_stats:tot_upstream 4 STAT 11211:default:pstd_stats:num_downstream_conn 94 STAT 11211:default:pstd_stats:tot_downstream_conn 0 STAT 11211:default:pstd_stats:tot_downstream_conn_acquired 2569122 STAT 11211:default:pstd_stats:tot_downstream_conn_released 2569120 STAT 11211:default:pstd_stats:tot_downstream_released 2569120 STAT 11211:default:pstd_stats:tot_downstream_reserved 2569122 STAT 11211:default:pstd_stats:tot_downstream_reserved_time 0 STAT 11211:default:pstd_stats:max_downstream_reserved_time 0 STAT 11211:default:pstd_stats:tot_downstream_freed 0 STAT 11211:default:pstd_stats:tot_downstream_quit_server 0 STAT 11211:default:pstd_stats:tot_downstream_max_reached 0 STAT 11211:default:pstd_stats:tot_downstream_create_failed 0 STAT 11211:default:pstd_stats:tot_downstream_connect 0 STAT 11211:default:pstd_stats:tot_downstream_connect_failed 0 STAT 11211:default:pstd_stats:tot_downstream_connect_timeout 0 STAT 11211:default:pstd_stats:tot_downstream_connect_interval 0 STAT 11211:default:pstd_stats:tot_downstream_connect_max_reached 0 STAT 11211:default:pstd_stats:tot_downstream_waiting_errors 0 STAT 11211:default:pstd_stats:tot_downstream_auth 0 STAT 11211:default:pstd_stats:tot_downstream_auth_failed 0 STAT 11211:default:pstd_stats:tot_downstream_bucket 0 STAT 11211:default:pstd_stats:tot_downstream_bucket_failed 0 STAT 11211:default:pstd_stats:tot_downstream_propagate_failed 0 STAT 11211:default:pstd_stats:tot_downstream_close_on_upstream_close 0 STAT 11211:default:pstd_stats:tot_downstream_timeout 0 STAT 11211:default:pstd_stats:tot_wait_queue_timeout 0 STAT 11211:default:pstd_stats:tot_assign_downstream 2569122 STAT 11211:default:pstd_stats:tot_assign_upstream 2569122 STAT 11211:default:pstd_stats:tot_assign_recursion 0 STAT 11211:default:pstd_stats:tot_reset_upstream_avail 0 STAT 11211:default:pstd_stats:tot_multiget_keys 0 STAT 11211:default:pstd_stats:tot_multiget_keys_dedupe 0 STAT 11211:default:pstd_stats:tot_multiget_bytes_dedupe 0 STAT 11211:default:pstd_stats:tot_optimize_sets 0 STAT 11211:default:pstd_stats:tot_optimize_self 0 STAT 11211:default:pstd_stats:tot_retry 0 STAT 11211:default:pstd_stats:tot_retry_time 0 STAT 11211:default:pstd_stats:max_retry_time 0 STAT 11211:default:pstd_stats:tot_retry_vbucket 0 STAT 11211:default:pstd_stats:tot_upstream_paused 2569122 STAT 11211:default:pstd_stats:tot_upstream_unpaused 2569120 STAT 11211:default:pstd_stats:err_oom 0 STAT 11211:default:pstd_stats:err_upstream_write_prep 0 STAT 11211:default:pstd_stats:err_downstream_write_prep 0 STAT 11211:default:pstd_stats_cmd:regular_get:seen 1721438 STAT 11211:default:pstd_stats_cmd:regular_get_key:hits 1721431 STAT 11211:default:pstd_stats_cmd:regular_get_key:write_bytes 3533481881 STAT 11211:default:pstd_stats_cmd:regular_set:seen 847684 STAT 11211:default:pstd_stats_cmd:regular_set:read_bytes 1753375920 STAT 11211:default:pstd_stats_cmd:regular_stats:seen 3 STAT 11211:default:pstd_stats_cmd:regular_stats:read_bytes 33 END

Steve Yen added a comment - 09/Oct/10 8:44 PM more context... - ./memcachetest -h 10.2.1.51:11211 -i 500000 -c 500000 -l -t 16 - tried but could not reproduce on linux labmgr vm's, on windows physical boxes, on osx

Steve Yen added a comment - 09/Oct/10 8:57 PM More notes: (continual) rebalancing not necessary to reproduce this, so whatever is going wrong is probably not in moxi's dynamic reconfiguration codepaths. The other observation is that moxi became incredibly slow, but eventually comes back with an answer to requests -- unlike the other bug where moxi locked up a worker thread and ops/sec dropped to true and forever 0. --------------------------- Yes, you're right. Rebalancing didn't cause this problem. Chiyoung On Oct 9, 2010, at 1:43 PM, Steve Yen wrote: > Thanks. Double-checking something -- I've been somewhat assuming Rebalance mostly isn't a factor. That is, with a stable 2 node cluster (not adding/removing servers), it's been easily reproducible on boxes 51 & 54 . > > Let me know if that's incorrect.

Steve Yen added a comment - 09/Oct/10 9:32 PM this does not seem to reproduce (on the physical linux boxes) when # of items is small...   # ./memcachetest -h 10.2.1.54:11211 -i 500 -c 5000 -l -t 16

Steve Yen added a comment - 09/Oct/10 9:54 PM more reproductions... especially when moxi just has 1 worker thread... with a memcachetest using 500K number of items driving load from a separate box, against a stable 2-node linux physical cluster (1.6.0beta4-84), and a standalone moxi running on the same box as membase... reproduced when standalone moxi has 4 worker threads...   # ./memcachetest -h 10.2.1.54:11511 -i 500000 -c 5000000 -l -t 16   # ./membase/bin/moxi/moxi -Z port_listen=11511,default_bucket_name=default,downstream_max=4,downstream_conn_max=0,usr=Administrator,pwd=password -u root -t 4 -v http://10.2.1.54:8091/pools/default/saslBucketsStreaming reproduced when standalone moxi has 1 worker thread...   # ./memcachetest -h 10.2.1.54:11511 -i 500000 -c 5000000 -l -t 16   # ./membase/bin/moxi/moxi -Z port_listen=11511,default_bucket_name=default,downstream_max=4,downstream_conn_max=0,usr=Administrator,pwd=password -u root -t 1 -v http://10.2.1.54:8091/pools/default/saslBucketsStreaming unlike a previous case yesterday, the single-threaded moxi is _not_ slow to new requests from other clients...    steve@ubuntu-server-9-1-32bit :~$ time "echo incr a 1 | nc 10.2.1.54 11211"   -bash: echo incr a 1 | nc 10.2.1.54 11211: command not found   real 0m0.001s   user 0m0.000s   sys 0m0.000s    steve@ubuntu-server-9-1-32bit :~$ time "echo incr a 1 | nc 10.2.1.54 11211"   -bash: echo incr a 1 | nc 10.2.1.54 11211: command not found   real 0m0.001s   user 0m0.000s   sys 0m0.000s from the single-threaded moxi (after reproduction of the slowness), it has stats proxy output of... $ echo stats proxy | nc 10.2.1.54 11511 | grep -v NULL STAT basic:version 1.6.0beta4_37_g036e93d STAT basic:nthreads 2 STAT basic:hostname localhost.localdomain STAT memcached:settings:maxbytes 67108864 STAT memcached:settings:maxconns 1024 STAT memcached:settings:tcpport -2 STAT memcached:settings:udpport -2 STAT memcached:settings:verbosity 1 STAT memcached:settings:oldest 0 STAT memcached:settings:evictions on STAT memcached:settings:umask 700 STAT memcached:settings:growth_factor 1.25 STAT memcached:settings:chunk_size 48 STAT memcached:settings:num_threads 2 STAT memcached:settings:stat_key_prefix : STAT memcached:settings:detail_enabled no STAT memcached:settings:reqs_per_event 20 STAT memcached:settings:cas_enabled yes STAT memcached:settings:tcp_backlog 1024 STAT memcached:settings:binding_protocol auto-negotiate STAT memcached:stats:pid 30040 STAT memcached:stats:uptime 350 STAT memcached:stats:time 1286660894 STAT memcached:stats:version 1.6.0beta4_37_g036e93d STAT memcached:stats:pointer_size 64 STAT memcached:stats:rusage_user 24.598260 STAT memcached:stats:rusage_system 85.561992 STAT memcached:stats:curr_connections 26 STAT memcached:stats:total_connections 30 STAT memcached:stats:connection_structures 27 STAT memcached:stats:cmd_get 0 STAT memcached:stats:cmd_set 0 STAT memcached:stats:cmd_flush 0 STAT memcached:stats:get_hits 0 STAT memcached:stats:get_misses 0 STAT memcached:stats:delete_misses 0 STAT memcached:stats:delete_hits 0 STAT memcached:stats:incr_misses 0 STAT memcached:stats:incr_hits 0 STAT memcached:stats:decr_misses 0 STAT memcached:stats:decr_hits 0 STAT memcached:stats:cas_misses 0 STAT memcached:stats:cas_hits 0 STAT memcached:stats:cas_badval 0 STAT memcached:stats:bytes_read 8234500996 STAT memcached:stats:bytes_written 4195098813 STAT memcached:stats:limit_maxbytes 67108864 STAT memcached:stats:accepting_conns 1 STAT memcached:stats:listen_disabled_num 0 STAT memcached:stats:threads 2 STAT memcached:stats:conn_yields 10 STAT proxy_main:conf_type dynamic STAT proxy_main:behavior:cycle 0 STAT proxy_main:behavior:downstream_max 4 STAT proxy_main:behavior:downstream_conn_max 0 STAT proxy_main:behavior:downstream_weight 0 STAT proxy_main:behavior:downstream_retry 1 STAT proxy_main:behavior:downstream_protocol 8 STAT proxy_main:behavior:downstream_timeout 0 STAT proxy_main:behavior:wait_queue_timeout 0 STAT proxy_main:behavior:time_stats 0 STAT proxy_main:behavior:connect_max_errors 0 STAT proxy_main:behavior:connect_retry_interval 0 STAT proxy_main:behavior:front_cache_max 200 STAT proxy_main:behavior:front_cache_lifespan 0 STAT proxy_main:behavior:front_cache_spec STAT proxy_main:behavior:front_cache_unspec STAT proxy_main:behavior:key_stats_max 4000 STAT proxy_main:behavior:key_stats_lifespan 0 STAT proxy_main:behavior:key_stats_spec STAT proxy_main:behavior:key_stats_unspec STAT proxy_main:behavior:optimize_set STAT proxy_main:behavior:usr Administrator STAT proxy_main:behavior:host STAT proxy_main:behavior:port 0 STAT proxy_main:behavior:bucket STAT proxy_main:behavior:port_listen 11511 STAT proxy_main:behavior:default_bucket_name default STAT proxy_main:stats:stat_configs 1 STAT proxy_main:stats:stat_config_fails 0 STAT proxy_main:stats:stat_proxy_starts 2 STAT proxy_main:stats:stat_proxy_start_fails 0 STAT proxy_main:stats:stat_proxy_existings 0 STAT proxy_main:stats:stat_proxy_shutdowns 0 STAT 11511:default:info:port 11511 STAT 11511:default:info:name default STAT 11511:default:info:config { "name": "default", "nodeLocator": "vbucket", "saslPassword": "", "nodes": [{ "clusterMembership": "active", "status": "h STAT 11511:default:info:config_ver 1 STAT 11511:default:info:behaviors_num 2 STAT 11511:default:behavior:downstream_max 4 STAT 11511:default:behavior:downstream_conn_max 0 STAT 11511:default:behavior:downstream_weight 0 STAT 11511:default:behavior:downstream_retry 1 STAT 11511:default:behavior:downstream_protocol 8 STAT 11511:default:behavior:downstream_timeout 0 STAT 11511:default:behavior:wait_queue_timeout 0 STAT 11511:default:behavior:time_stats 0 STAT 11511:default:behavior:connect_max_errors 0 STAT 11511:default:behavior:connect_retry_interval 0 STAT 11511:default:behavior:front_cache_max 200 STAT 11511:default:behavior:front_cache_lifespan 0 STAT 11511:default:behavior:front_cache_spec STAT 11511:default:behavior:front_cache_unspec STAT 11511:default:behavior:key_stats_max 4000 STAT 11511:default:behavior:key_stats_lifespan 0 STAT 11511:default:behavior:key_stats_spec STAT 11511:default:behavior:key_stats_unspec STAT 11511:default:behavior:optimize_set STAT 11511:default:behavior:usr default STAT 11511:default:behavior:host STAT 11511:default:behavior:port 0 STAT 11511:default:behavior:bucket STAT 11511:default:behavior:port_listen 11511 STAT 11511:default:behavior:default_bucket_name default STAT 11511:default:behavior-0:downstream_weight 0 STAT 11511:default:behavior-0:downstream_retry 1 STAT 11511:default:behavior-0:downstream_protocol 8 STAT 11511:default:behavior-0:downstream_timeout 0 STAT 11511:default:behavior-0:usr default STAT 11511:default:behavior-0:host 10.2.1.51 STAT 11511:default:behavior-0:port 11210 STAT 11511:default:behavior-0:bucket STAT 11511:default:behavior-1:downstream_weight 0 STAT 11511:default:behavior-1:downstream_retry 1 STAT 11511:default:behavior-1:downstream_protocol 8 STAT 11511:default:behavior-1:downstream_timeout 0 STAT 11511:default:behavior-1:usr default STAT 11511:default:behavior-1:host 10.2.1.54 STAT 11511:default:behavior-1:port 11210 STAT 11511:default:behavior-1:bucket STAT 11511:default:stats:listening 2 STAT 11511:default:stats:listening_failed 0 STAT 11511:default:frontcache:max 0 STAT 11511:default:frontcache:oldest_live 0 STAT 11511:default:frontcache:tot_get_hits 0 STAT 11511:default:frontcache:tot_get_expires 0 STAT 11511:default:frontcache:tot_get_misses 0 STAT 11511:default:frontcache:tot_get_bytes 0 STAT 11511:default:frontcache:tot_adds 0 STAT 11511:default:frontcache:tot_add_skips 0 STAT 11511:default:frontcache:tot_add_fails 0 STAT 11511:default:frontcache:tot_add_bytes 0 STAT 11511:default:frontcache:tot_deletes 0 STAT 11511:default:frontcache:tot_evictions 0 STAT 11511:default:pstd_stats:num_upstream 17 STAT 11511:default:pstd_stats:tot_upstream 20 STAT 11511:default:pstd_stats:num_downstream_conn 8 STAT 11511:default:pstd_stats:tot_downstream_conn 8 STAT 11511:default:pstd_stats:tot_downstream_conn_acquired 1996902 STAT 11511:default:pstd_stats:tot_downstream_conn_released 1996898 STAT 11511:default:pstd_stats:tot_downstream_released 1996901 STAT 11511:default:pstd_stats:tot_downstream_reserved 1996901 STAT 11511:default:pstd_stats:tot_downstream_reserved_time 0 STAT 11511:default:pstd_stats:max_downstream_reserved_time 0 STAT 11511:default:pstd_stats:tot_downstream_freed 0 STAT 11511:default:pstd_stats:tot_downstream_quit_server 0 STAT 11511:default:pstd_stats:tot_downstream_max_reached 3962552 STAT 11511:default:pstd_stats:tot_downstream_create_failed 0 STAT 11511:default:pstd_stats:tot_downstream_connect 8 STAT 11511:default:pstd_stats:tot_downstream_connect_failed 0 STAT 11511:default:pstd_stats:tot_downstream_connect_timeout 0 STAT 11511:default:pstd_stats:tot_downstream_connect_interval 0 STAT 11511:default:pstd_stats:tot_downstream_connect_max_reached 0 STAT 11511:default:pstd_stats:tot_downstream_waiting_errors 0 STAT 11511:default:pstd_stats:tot_downstream_auth 8 STAT 11511:default:pstd_stats:tot_downstream_auth_failed 0 STAT 11511:default:pstd_stats:tot_downstream_bucket 8 STAT 11511:default:pstd_stats:tot_downstream_bucket_failed 0 STAT 11511:default:pstd_stats:tot_downstream_propagate_failed 0 STAT 11511:default:pstd_stats:tot_downstream_close_on_upstream_close 0 STAT 11511:default:pstd_stats:tot_downstream_timeout 0 STAT 11511:default:pstd_stats:tot_wait_queue_timeout 0 STAT 11511:default:pstd_stats:tot_assign_downstream 1996901 STAT 11511:default:pstd_stats:tot_assign_upstream 1996901 STAT 11511:default:pstd_stats:tot_assign_recursion 0 STAT 11511:default:pstd_stats:tot_reset_upstream_avail 0 STAT 11511:default:pstd_stats:tot_multiget_keys 0 STAT 11511:default:pstd_stats:tot_multiget_keys_dedupe 0 STAT 11511:default:pstd_stats:tot_multiget_bytes_dedupe 0 STAT 11511:default:pstd_stats:tot_optimize_sets 0 STAT 11511:default:pstd_stats:tot_optimize_self 0 STAT 11511:default:pstd_stats:tot_retry 0 STAT 11511:default:pstd_stats:tot_retry_time 0 STAT 11511:default:pstd_stats:max_retry_time 0 STAT 11511:default:pstd_stats:tot_retry_vbucket 0 STAT 11511:default:pstd_stats:tot_upstream_paused 1996913 STAT 11511:default:pstd_stats:tot_upstream_unpaused 1996897 STAT 11511:default:pstd_stats:err_oom 0 STAT 11511:default:pstd_stats:err_upstream_write_prep 0 STAT 11511:default:pstd_stats:err_downstream_write_prep 0 STAT 11511:default:pstd_stats_cmd:regular_get:seen 1002921 STAT 11511:default:pstd_stats_cmd:regular_get_key:hits 1002907 STAT 11511:default:pstd_stats_cmd:regular_get_key:write_bytes 2055633524 STAT 11511:default:pstd_stats_cmd:regular_set:seen 993991 STAT 11511:default:pstd_stats_cmd:regular_set:read_bytes 2058874289 STAT 11511:default:pstd_stats_cmd:regular_stats:seen 3 STAT 11511:default:pstd_stats_cmd:regular_stats:read_bytes 27 END

Steve Yen added a comment - 09/Oct/10 10:01 PM heh, error on those incr cmds.

Steve Yen added a comment - 09/Oct/10 10:46 PM All upstream connections (even new ones) that need downstream connections seem to be gated. Reconfirmed that after getting into slow mode, single-threaded moxi is _indeed_ slow to new requests from other clients. This is 5 seconds of slow...   # echo incr b 1 | nc 10.2.1.54 11511   NOT_FOUND But this is instantaneous (becase it doesn't need a downstream conn)...   # echo version | nc 10.2.1.54 11511   VERSION 1.6.0beta4_37_g036e93d

James Phillips added a comment - 10/Oct/10 12:12 AM

James Phillips added a comment - 10/Oct/10 12:13 AM Created an attachment (id=802) screen killing moxi on the server and allowing ns_server to restart it seemed to "fix" the situation. See ops/sec graph attached.

Sharon Barr added a comment - 19/Nov/10 6:12 PM Issue was resolved on multiple places (moxi, ep-engine).