============================================================================== membase logs mbbrowse_logs ============================================================================== rb: reading report...done. rb: reading report...done. INFO REPORT <0.54.0> 2011-11-22 17:48:00 =============================================================================== nonode@nohost:<0.54.0>:log_os_info:25: OS type: {unix,linux} Version: {3,0,0} Runtime info: [{otp_release,"R14B02"}, {erl_version,"5.8.3"}, {erl_version_long, "Erlang R14B02 (erts-5.8.3) [source] [64-bit] [smp:2:2] [rq:2] [async-threads:16] [hipe] [kernel-poll:false]\n"}, {system_arch_raw,"x86_64-unknown-linux-gnu"}, {system_arch,"x86_64-unknown-linux-gnu"}, {localtime,{{2011,11,22},{17,48,0}}}, {memory, [{total,15839088}, {processes,1671912}, {processes_used,1656824}, {system,14167176}, {atom,452097}, {atom_used,423209}, {binary,698584}, {code,3363556}, {ets,279472}]}, {loaded, [ns_info,log_os_info,misc,ns_log_mf_h, ns_server_cluster_sup,ns_server,timer,io_lib_fread, cpu_sup,memsup,disksup,os_mon,io_lib_pretty,io_lib_format, io_lib,io,sasl_report,release_handler,calendar,overload, alarm_handler,sasl_report_tty_h,sasl,ns_bootstrap, file_io_server,orddict,erl_eval,c,error_logger_tty_h, queue,kernel_config,user,user_sup,supervisor_bridge, standard_error,ram_file,file,beam_lib,unicode,binary,ets, gb_sets,hipe_unified_loader,packages,code_server,code, file_server,net_kernel,global_group,erl_distribution, filename,inet_gethost_native,inet_parse,inet,inet_udp,os, inet_config,inet_db,global,gb_trees,rpc,supervisor,kernel, application_master,sys,application,gen_server,erl_parse, proplists,erl_scan,lists,application_controller,proc_lib, gen,gen_event,error_logger,heart,error_handler,erlang, erl_prim_loader,prim_zip,zlib,prim_file,prim_inet,init, otp_ring0]}, {applications, [{os_mon,"CPO CXC 138 46","2.2.5"}, {kernel,"ERTS CXC 138 10","2.14.3"}, {sasl,"SASL CXC 138 11","2.1.9.3"}, {ns_server,"Membase server","1.7.2r-20-g6604356"}, {stdlib,"ERTS CXC 138 10","1.17.3"}]}, {pre_loaded, [erlang,erl_prim_loader,prim_zip,zlib,prim_file,prim_inet, init,otp_ring0]}, {process_count,44}, {node,nonode@nohost}, {nodes,[]}, {registered, [rex,release_handler,inet_db,kernel_sup,code_server, global_name_server,overload,file_server_2, application_controller,alarm_handler,os_mon_sup, standard_error,init,cpu_sup,global_group,erl_prim_loader, sasl_sup,memsup,disksup,timer_server,standard_error_sup, user,sasl_safe_sup,error_logger,kernel_safe_sup, ns_server_cluster_sup]}, {cookie,nocookie}, {wordsize,8}, {wall_clock,0}] INFO REPORT <0.54.0> 2011-11-22 17:48:00 =============================================================================== nonode@nohost:<0.54.0>:log_os_info:27: Manifest: ["bucket_engine 1.7.0-0-g721dff0 Linux-x86_64", "ep-engine 1.7.1.1-39-g4bd26a3 Linux-x86_64", "geocouch couchbase_1.1_geo-0-gd09068e Linux-x86_64", "icu4c 2635939 Linux-x86_64","libconflate 1.7.0-0-gfee8f94 Linux-x86_64", "libmemcached 2cd40dc Linux-x86_64", "libvbucket 1.7.0-0-g4bd0aba Linux-x86_64", "manifest 1.7.0-30-gd83dca9 Linux-x86_64", "membase-cli 1.7.2-0-g1d9c77e Linux-x86_64", "membase-server 1.7.2r-20-g6604356 Linux-x86_64", "membasex 1.7.0-3-g30d5dba Linux-x86_64", "memcached membase-1.7.1-0-gf99c147 Linux-x86_64", "memcachetest 0.8.3-0-g88ae3b3 Linux-x86_64", "moxi 1.7.2-0-gd5076d9 Linux-x86_64", "ns_server 1.7.2-0-g4925ee0 Linux-x86_64", "otp OTP_R14B03-0-g4a5a758 Linux-x86_64", "portsigar 1.7.0-0-ga191e6c Linux-x86_64", "sigar sigar-1.6.4-406-ge1dcf32 Linux-x86_64", "spidermonkey a3c48c1 Linux-x86_64","tlm 1.7.1-0-g535dadc Linux-x86_64", "vbucketmigrator 1.7.0-0-g0fdc96c Linux-x86_64"] PROGRESS REPORT <0.54.0> 2011-11-22 17:48:00 =============================================================================== supervisor {local,ns_server_cluster_sup} started [{pid,<0.56.0>}, {name,timeout_diag_logger}, {mfargs,{timeout_diag_logger,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <0.57.0> 2011-11-22 17:48:00 =============================================================================== reading ip config from "/opt/membase/var/lib/membase/ip" ERROR REPORT <0.57.0> 2011-11-22 17:48:00 =============================================================================== Got error:einval. Ignoring bad address:[] INFO REPORT <0.57.0> 2011-11-22 17:48:00 =============================================================================== nonode@nohost:<0.57.0>:dist_manager:105: Attempting to bring up net_kernel with name 'ns_1@127.0.0.1' PROGRESS REPORT <0.58.0> 2011-11-22 17:48:00 =============================================================================== supervisor {local,net_sup} started [{pid,<0.59.0>}, {name,erl_epmd}, {mfargs,{erl_epmd,start_link,[]}}, {restart_type,permanent}, {shutdown,2000}, {child_type,worker}] PROGRESS REPORT <0.58.0> 2011-11-22 17:48:00 =============================================================================== supervisor {local,net_sup} started [{pid,<0.60.0>}, {name,auth}, {mfargs,{auth,start_link,[]}}, {restart_type,permanent}, {shutdown,2000}, {child_type,worker}] PROGRESS REPORT <5901.58.0> 2011-11-22 17:48:00 =============================================================================== supervisor {local,net_sup} started [{pid,<5901.61.0>}, {name,net_kernel}, {mfargs,{net_kernel,start_link,[['ns_1@127.0.0.1',longnames]]}}, {restart_type,permanent}, {shutdown,2000}, {child_type,worker}] PROGRESS REPORT <5901.11.0> 2011-11-22 17:48:00 =============================================================================== supervisor {local,kernel_sup} started [{pid,<5901.58.0>}, {name,net_sup_dynamic}, {mfargs,{erl_distribution,start_link, [['ns_1@127.0.0.1',longnames]]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,supervisor}] PROGRESS REPORT <5901.54.0> 2011-11-22 17:48:00 =============================================================================== supervisor {local,ns_server_cluster_sup} started [{pid,<5901.57.0>}, {name,dist_manager}, {mfargs,{dist_manager,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.54.0> 2011-11-22 17:48:00 =============================================================================== supervisor {local,ns_server_cluster_sup} started [{pid,<5901.64.0>}, {name,ns_cluster}, {mfargs,{ns_cluster,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] PROGRESS REPORT <5901.65.0> 2011-11-22 17:48:00 =============================================================================== supervisor {local,mb_mnesia_sup} started [{pid,<5901.66.0>}, {name,mb_mnesia_events}, {mfargs,{gen_event,start_link,[{local,mb_mnesia_events}]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <5901.72.0> 2011-11-22 17:48:00 =============================================================================== supervisor {local,mnesia_sup} started [{pid,<5901.73.0>}, {name,mnesia_event}, {mfargs,{mnesia_sup,start_event,[]}}, {restart_type,permanent}, {shutdown,30000}, {child_type,worker}] PROGRESS REPORT <5901.74.0> 2011-11-22 17:48:00 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<5901.75.0>}, {name,mnesia_monitor}, {mfargs,{mnesia_monitor,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] PROGRESS REPORT <5901.74.0> 2011-11-22 17:48:00 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<5901.76.0>}, {name,mnesia_subscr}, {mfargs,{mnesia_subscr,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] PROGRESS REPORT <5901.74.0> 2011-11-22 17:48:01 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<5901.77.0>}, {name,mnesia_locker}, {mfargs,{mnesia_locker,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] PROGRESS REPORT <5901.74.0> 2011-11-22 17:48:01 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<5901.78.0>}, {name,mnesia_recover}, {mfargs,{mnesia_recover,start,[]}}, {restart_type,permanent}, {shutdown,180000}, {child_type,worker}] PROGRESS REPORT <5901.74.0> 2011-11-22 17:48:01 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<5901.79.0>}, {name,mnesia_tm}, {mfargs,{mnesia_tm,start,[]}}, {restart_type,permanent}, {shutdown,30000}, {child_type,worker}] PROGRESS REPORT <5901.74.0> 2011-11-22 17:48:01 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<5901.80.0>}, {name,mnesia_checkpoint_sup}, {mfargs,{mnesia_checkpoint_sup,start,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <5901.74.0> 2011-11-22 17:48:01 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<5901.81.0>}, {name,mnesia_snmp_sup}, {mfargs,{mnesia_snmp_sup,start,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <5901.74.0> 2011-11-22 17:48:01 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<5901.82.0>}, {name,mnesia_controller}, {mfargs,{mnesia_controller,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] PROGRESS REPORT <5901.74.0> 2011-11-22 17:48:01 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<5901.83.0>}, {name,mnesia_late_loader}, {mfargs,{mnesia_late_loader,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] PROGRESS REPORT <5901.72.0> 2011-11-22 17:48:01 =============================================================================== supervisor {local,mnesia_sup} started [{pid,<5901.74.0>}, {name,mnesia_kernel_sup}, {mfargs,{mnesia_kernel_sup,start,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <5901.7.0> 2011-11-22 17:48:01 =============================================================================== application mnesia started_at 'ns_1@127.0.0.1' PROGRESS REPORT <5901.26.0> 2011-11-22 17:48:01 =============================================================================== supervisor {local,kernel_safe_sup} started [{pid,<5901.87.0>}, {name,disk_log_sup}, {mfargs,{disk_log_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,supervisor}] PROGRESS REPORT <5901.26.0> 2011-11-22 17:48:01 =============================================================================== supervisor {local,kernel_safe_sup} started [{pid,<5901.88.0>}, {name,disk_log_server}, {mfargs,{disk_log_server,start_link,[]}}, {restart_type,permanent}, {shutdown,2000}, {child_type,worker}] PROGRESS REPORT <5901.26.0> 2011-11-22 17:48:01 =============================================================================== supervisor {local,kernel_safe_sup} started [{pid,<5901.91.0>}, {name,dets_sup}, {mfargs,{dets_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,supervisor}] PROGRESS REPORT <5901.26.0> 2011-11-22 17:48:01 =============================================================================== supervisor {local,kernel_safe_sup} started [{pid,<5901.92.0>}, {name,dets}, {mfargs,{dets_server,start_link,[]}}, {restart_type,permanent}, {shutdown,2000}, {child_type,worker}] INFO REPORT <5901.67.0> 2011-11-22 17:48:01 =============================================================================== ns_1@127.0.0.1:<5901.67.0>:mb_mnesia:409: Committed schema to disk. INFO REPORT <5901.67.0> 2011-11-22 17:48:01 =============================================================================== ns_1@127.0.0.1:<5901.67.0>:mb_mnesia:245: Current config: [{access_module,mnesia}, {auto_repair,true}, {backup_module,mnesia_backup}, {checkpoints,[]}, {db_nodes,['ns_1@127.0.0.1']}, {debug,verbose}, {directory,"/opt/membase/var/lib/membase/mnesia"}, {dump_log_load_regulation,false}, {dump_log_time_threshold,180000}, {dump_log_update_in_place,true}, {dump_log_write_threshold,1000}, {embedded_mnemosyne,false}, {event_module,mnesia_event}, {extra_db_nodes,[]}, {fallback_activated,false}, {held_locks,[]}, {ignore_fallback_at_startup,false}, {fallback_error_function,{mnesia,lkill}}, {is_running,yes}, {local_tables,[local_config,cluster,schema]}, {lock_queue,[]}, {log_version,"4.3"}, {master_node_tables,[]}, {max_wait_for_decision,10000}, {protocol_version,{7,6}}, {running_db_nodes,['ns_1@127.0.0.1']}, {schema_location,opt_disc}, {schema_version,{3,0}}, {subscribers,[<5901.73.0>,<5901.67.0>]}, {tables,[local_config,cluster,schema]}, {transaction_commits,5}, {transaction_failures,0}, {transaction_log_writes,3}, {transaction_restarts,0}, {transactions,[]}, {use_dir,true}, {core_dir,false}, {no_table_loaders,2}, {dc_dump_limit,4}, {send_compressed,0}, {version,"4.4.17"}] Peers: ['ns_1@127.0.0.1'] INFO REPORT <5901.67.0> 2011-11-22 17:48:01 =============================================================================== ns_1@127.0.0.1:<5901.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,schema, [{name,schema}, {type,set}, {ram_copies,[]}, {disc_copies,['ns_1@127.0.0.1']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,false}, {record_name,schema}, {attributes,[table,cstruct]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1321,984081,13188},'ns_1@127.0.0.1'}}, {version,{{3,0},{'ns_1@127.0.0.1',{1321,984081,34779}}}}]}, {tid,3,<5901.86.0>}} INFO REPORT <5901.67.0> 2011-11-22 17:48:01 =============================================================================== ns_1@127.0.0.1:<5901.67.0>:mb_mnesia:194: Peers: ['ns_1@127.0.0.1'] INFO REPORT <5901.67.0> 2011-11-22 17:48:01 =============================================================================== ns_1@127.0.0.1:<5901.67.0>:mb_mnesia:194: Peers: ['ns_1@127.0.0.1'] PROGRESS REPORT <5901.65.0> 2011-11-22 17:48:01 =============================================================================== supervisor {local,mb_mnesia_sup} started [{pid,<5901.67.0>}, {name,mb_mnesia}, {mfargs,{mb_mnesia,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <5901.67.0> 2011-11-22 17:48:01 =============================================================================== ns_1@127.0.0.1:<5901.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,local_config, [{name,local_config}, {type,set}, {ram_copies,[]}, {disc_copies,['ns_1@127.0.0.1']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,local_config}, {attributes,[key,val]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1321,984081,299640},'ns_1@127.0.0.1'}}, {version,{{2,0},[]}}]}, {tid,5,<5901.111.0>}} INFO REPORT <5901.67.0> 2011-11-22 17:48:01 =============================================================================== ns_1@127.0.0.1:<5901.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,local_config, [{name,local_config}, {type,set}, {ram_copies,[]}, {disc_copies,['ns_1@127.0.0.1']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,local_config}, {attributes,[key,val]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1321,984081,299640},'ns_1@127.0.0.1'}}, {version,{{2,0},[]}}]}, {tid,5,<5901.111.0>}} PROGRESS REPORT <5901.54.0> 2011-11-22 17:48:01 =============================================================================== supervisor {local,ns_server_cluster_sup} started [{pid,<5901.65.0>}, {name,mb_mnesia_sup}, {mfargs,{mb_mnesia_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] INFO REPORT <5901.117.0> 2011-11-22 17:48:01 =============================================================================== loading static ns_config from "/opt/membase/etc/membase/config" PROGRESS REPORT <5901.117.0> 2011-11-22 17:48:01 =============================================================================== supervisor {local,ns_config_sup} started [{pid,<5901.118.0>}, {name,ns_config_events}, {mfargs,{gen_event,start_link,[{local,ns_config_events}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.119.0> 2011-11-22 17:48:01 =============================================================================== ns_1@127.0.0.1:<5901.119.0>:ns_config_default:226: Upgrading config from 1.6 to 1.7 INFO REPORT <5901.119.0> 2011-11-22 17:48:01 =============================================================================== ns_1@127.0.0.1:<5901.119.0>:ns_config:413: Upgrading config by changes: [{set,{node,'ns_1@127.0.0.1',config_version},{1,7}}, {set, {node,'ns_1@127.0.0.1',memcached}, [{'_vclock',[{'ns_1@127.0.0.1',{1,63489203281}}]}, {port,11210}, {dbdir,"/opt/membase/var/lib/membase/data"}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine,"/opt/membase/lib/memcached/bucket_engine.so"}, {engines, [{membase, [{engine,"/opt/membase/lib/memcached/ep.so"}, {initfile,"/opt/membase/etc/membase/init.sql"}, {static_config_string, "vb0=false;waitforwarmup=false;failpartialwarmup=false;shardpattern=%d/%b-%i.mb;db_strategy=multiMTVBDB"}]}, {memcached, [{engine,"/opt/membase/lib/memcached/default_engine.so"}, {static_config_string,"vb0=true"}]}]}, {verbosity,[]}]}, {set, {node,'ns_1@127.0.0.1',ns_log}, [{'_vclock',[{'ns_1@127.0.0.1',{1,63489203281}}]}, {filename,"/opt/membase/var/lib/membase/data/ns_log"}]}, {set,port_servers, [{moxi,"/opt/membase/bin/moxi", ["-Z", {"port_listen=~B,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", [port]}, "-z", {"url=http://127.0.0.1:~B/pools/default/saslBucketsStreaming", [{rest,port}]}, "-p","0","-Y","y","-O","stderr", {"~s",[verbosity]}], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR",{"~s",[{ns_moxi_sup,rest_user,[]}]}}, {"MOXI_SASL_PLAIN_PWD",{"~s",[{ns_moxi_sup,rest_pass,[]}]}}]}, use_stdio,exit_status,port_server_send_eol,stderr_to_stdout, stream]}, {memcached,"/opt/membase/bin/memcached", ["-X","/opt/membase/lib/memcached/stdin_term_handler.so","-p", {"~B",[port]}, "-E","/opt/membase/lib/memcached/bucket_engine.so","-B","binary", "-r","-c","10000","-e", {"admin=~s;default_bucket_name=default;auto_create=false", [admin_user]}, {"~s",[verbosity]}], [{env, [{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE",{"~s",[{isasl,path}]}}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,exit_status,port_server_send_eol, stream]}]}, {set, {node,'ns_1@127.0.0.1',isasl}, [{'_vclock',[{'ns_1@127.0.0.1',{1,63489203281}}]}, {path,"/opt/membase/var/lib/membase/data/isasl.pw"}]}, {set,directory,"/opt/membase/var/lib/membase/config"}] INFO REPORT <5901.119.0> 2011-11-22 17:48:01 =============================================================================== ns_1@127.0.0.1:<5901.119.0>:ns_config_default:262: Upgrading config from 1.7 to 1.7.1 INFO REPORT <5901.119.0> 2011-11-22 17:48:01 =============================================================================== ns_1@127.0.0.1:<5901.119.0>:ns_config:413: Upgrading config by changes: [{set,{node,'ns_1@127.0.0.1',config_version},{1,7,1}}, {set,email_alerts, [{recipients,["root@localhost"]}, {sender,"membase@localhost"}, {enabled,true}, {email_server,[{user,[]}, {pass,[]}, {host,"localhost"}, {port,25}, {encrypt,false}]}, {alerts,[auto_failover_node,auto_failover_maximum_reached, auto_failover_other_nodes_down, auto_failover_cluster_too_small]}]}, {set,auto_failover_cfg, [{enabled,false},{timeout,30},{max_nodes,1},{count,0}]}] INFO REPORT <5901.119.0> 2011-11-22 17:48:01 =============================================================================== ns_1@127.0.0.1:<5901.119.0>:ns_config_default:273: Upgrading config from 1.7.1 to 1.7.2 INFO REPORT <5901.119.0> 2011-11-22 17:48:01 =============================================================================== ns_1@127.0.0.1:<5901.119.0>:ns_config_default:300: Setting global and per-node rest port to 8091 INFO REPORT <5901.119.0> 2011-11-22 17:48:01 =============================================================================== ns_1@127.0.0.1:<5901.119.0>:ns_config:413: Upgrading config by changes: [{set,{node,'ns_1@127.0.0.1',config_version},{1,7,2}}, {set,rest,[{port,8091}]}, {set,{node,'ns_1@127.0.0.1',rest},[{port,8091},{port_meta,global}]}] INFO REPORT <5901.119.0> 2011-11-22 17:48:01 =============================================================================== ns_1@127.0.0.1:<5901.119.0>:ns_config:447: Upgraded initial config: {config, {full,"/opt/membase/etc/membase/config",undefined,ns_config_default}, [[], [{directory,"/opt/membase/var/lib/membase/config"}, {nodes_wanted,['ns_1@127.0.0.1']}, {{node,'ns_1@127.0.0.1',membership},active}, {rest,[{port,8091}]}, {{node,'ns_1@127.0.0.1',rest},[{port,8091},{port_meta,global}]}, {rest_creds,[{creds,[]}]}, {{node,'ns_1@127.0.0.1',isasl}, [{path,"/opt/membase/var/lib/membase/data/isasl.pw"}]}, {{node,'ns_1@127.0.0.1',memcached}, [{port,11210}, {dbdir,"/opt/membase/var/lib/membase/data"}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine,"/opt/membase/lib/memcached/bucket_engine.so"}, {engines, [{membase, [{engine,"/opt/membase/lib/memcached/ep.so"}, {initfile,"/opt/membase/etc/membase/init.sql"}, {static_config_string, "vb0=false;waitforwarmup=false;failpartialwarmup=false;shardpattern=%d/%b-%i.mb;db_strategy=multiMTVBDB"}]}, {memcached, [{engine,"/opt/membase/lib/memcached/default_engine.so"}, {static_config_string,"vb0=true"}]}]}, {verbosity,[]}]}, {memory_quota,16090}, {buckets,[{configs,[]}]}, {{node,'ns_1@127.0.0.1',moxi},[{port,11211},{verbosity,[]}]}, {port_servers, [{moxi,"/opt/membase/bin/moxi", ["-Z", {"port_listen=~B,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", [port]}, "-z", {"url=http://127.0.0.1:~B/pools/default/saslBucketsStreaming", [{rest,port}]}, "-p","0","-Y","y","-O","stderr", {"~s",[verbosity]}], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR", {"~s",[{ns_moxi_sup,rest_user,[]}]}}, {"MOXI_SASL_PLAIN_PWD", {"~s",[{ns_moxi_sup,rest_pass,[]}]}}]}, use_stdio,exit_status,port_server_send_eol,stderr_to_stdout, stream]}, {memcached,"/opt/membase/bin/memcached", ["-X","/opt/membase/lib/memcached/stdin_term_handler.so","-p", {"~B",[port]}, "-E","/opt/membase/lib/memcached/bucket_engine.so","-B", "binary","-r","-c","10000","-e", {"admin=~s;default_bucket_name=default;auto_create=false", [admin_user]}, {"~s",[verbosity]}], [{env, [{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE",{"~s",[{isasl,path}]}}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,exit_status,port_server_send_eol, stream]}]}, {{node,'ns_1@127.0.0.1',ns_log}, [{filename,"/opt/membase/var/lib/membase/data/ns_log"}]}, {email_alerts, [{recipients,["root@localhost"]}, {sender,"membase@localhost"}, {enabled,true}, {email_server, [{user,[]}, {pass,[]}, {host,"localhost"}, {port,25}, {encrypt,false}]}, {alerts, [auto_failover_node,auto_failover_maximum_reached, auto_failover_other_nodes_down, auto_failover_cluster_too_small]}]}, {replication,[{enabled,true}]}, {auto_failover_cfg, [{enabled,false},{timeout,30},{max_nodes,1},{count,0}]}]], [[{directory,"/opt/membase/var/lib/membase/config"}, {{node,'ns_1@127.0.0.1',config_version},{1,7,2}}, {auto_failover_cfg, [{enabled,false},{timeout,30},{max_nodes,1},{count,0}]}, {buckets,[{configs,[]}]}, {email_alerts, [{recipients,["root@localhost"]}, {sender,"membase@localhost"}, {enabled,true}, {email_server, [{user,[]}, {pass,[]}, {host,"localhost"}, {port,25}, {encrypt,false}]}, {alerts, [auto_failover_node,auto_failover_maximum_reached, auto_failover_other_nodes_down, auto_failover_cluster_too_small]}]}, {memory_quota,16090}, {nodes_wanted,['ns_1@127.0.0.1']}, {port_servers, [{moxi,"/opt/membase/bin/moxi", ["-Z", {"port_listen=~B,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", [port]}, "-z", {"url=http://127.0.0.1:~B/pools/default/saslBucketsStreaming", [{rest,port}]}, "-p","0","-Y","y","-O","stderr", {"~s",[verbosity]}], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR", {"~s",[{ns_moxi_sup,rest_user,[]}]}}, {"MOXI_SASL_PLAIN_PWD", {"~s",[{ns_moxi_sup,rest_pass,[]}]}}]}, use_stdio,exit_status,port_server_send_eol,stderr_to_stdout, stream]}, {memcached,"/opt/membase/bin/memcached", ["-X","/opt/membase/lib/memcached/stdin_term_handler.so","-p", {"~B",[port]}, "-E","/opt/membase/lib/memcached/bucket_engine.so","-B", "binary","-r","-c","10000","-e", {"admin=~s;default_bucket_name=default;auto_create=false", [admin_user]}, {"~s",[verbosity]}], [{env, [{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE",{"~s",[{isasl,path}]}}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,exit_status,port_server_send_eol, stream]}]}, {replication,[{enabled,true}]}, {rest,[{port,8091}]}, {rest_creds,[{creds,[]}]}, {{node,'ns_1@127.0.0.1',isasl}, [{'_vclock',[{'ns_1@127.0.0.1',{1,63489203281}}]}, {path,"/opt/membase/var/lib/membase/data/isasl.pw"}]}, {{node,'ns_1@127.0.0.1',membership},active}, {{node,'ns_1@127.0.0.1',memcached}, [{'_vclock',[{'ns_1@127.0.0.1',{1,63489203281}}]}, {port,11210}, {dbdir,"/opt/membase/var/lib/membase/data"}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine,"/opt/membase/lib/memcached/bucket_engine.so"}, {engines, [{membase, [{engine,"/opt/membase/lib/memcached/ep.so"}, {initfile,"/opt/membase/etc/membase/init.sql"}, {static_config_string, "vb0=false;waitforwarmup=false;failpartialwarmup=false;shardpattern=%d/%b-%i.mb;db_strategy=multiMTVBDB"}]}, {memcached, [{engine,"/opt/membase/lib/memcached/default_engine.so"}, {static_config_string,"vb0=true"}]}]}, {verbosity,[]}]}, {{node,'ns_1@127.0.0.1',moxi},[{port,11211},{verbosity,[]}]}, {{node,'ns_1@127.0.0.1',ns_log}, [{'_vclock',[{'ns_1@127.0.0.1',{1,63489203281}}]}, {filename,"/opt/membase/var/lib/membase/data/ns_log"}]}, {{node,'ns_1@127.0.0.1',rest},[{port,8091},{port_meta,global}]}]], ns_config_default, {ns_config,save_config_sync,[]}, undefined,false} PROGRESS REPORT <5901.117.0> 2011-11-22 17:48:01 =============================================================================== supervisor {local,ns_config_sup} started [{pid,<5901.119.0>}, {name,ns_config}, {mfargs,{ns_config,start_link, ["/opt/membase/etc/membase/config", ns_config_default]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.117.0> 2011-11-22 17:48:01 =============================================================================== supervisor {local,ns_config_sup} started [{pid,<5901.122.0>}, {name,ns_config_remote}, {mfargs,{ns_config_replica,start_link,[{local,ns_config_remote}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.123.0> 2011-11-22 17:48:01 =============================================================================== isasl_sync init: ["/opt/membase/var/lib/membase/data/isasl.pw","_admin", "_admin"] INFO REPORT <5901.123.0> 2011-11-22 17:48:01 =============================================================================== isasl_sync init buckets: [] INFO REPORT <5901.123.0> 2011-11-22 17:48:01 =============================================================================== Writing isasl passwd file: "/opt/membase/var/lib/membase/data/isasl.pw" PROGRESS REPORT <5901.117.0> 2011-11-22 17:48:01 =============================================================================== supervisor {local,ns_config_sup} started [{pid,<5901.123.0>}, {name,ns_config_isasl_sync}, {mfargs,{ns_config_isasl_sync,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.117.0> 2011-11-22 17:48:01 =============================================================================== supervisor {local,ns_config_sup} started [{pid,<5901.125.0>}, {name,ns_config_log}, {mfargs,{ns_config_log,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.54.0> 2011-11-22 17:48:01 =============================================================================== supervisor {local,ns_server_cluster_sup} started [{pid,<5901.117.0>}, {name,ns_config_sup}, {mfargs,{ns_config_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] INFO REPORT <5901.127.0> 2011-11-22 17:48:01 =============================================================================== ns_log:init(): Couldn't load logs from "/opt/membase/var/lib/membase/data/ns_log": {error, enoent} PROGRESS REPORT <5901.126.0> 2011-11-22 17:48:01 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.127.0>}, {name,ns_log}, {mfargs,{ns_log,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.126.0> 2011-11-22 17:48:01 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.128.0>}, {name,ns_log_events}, {mfargs,{gen_event,start_link,[{local,ns_log_events}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.129.0> 2011-11-22 17:48:01 =============================================================================== supervisor {local,ns_node_disco_sup} started [{pid,<5901.130.0>}, {name,ns_node_disco_events}, {mfargs,{gen_event,start_link,[{local,ns_node_disco_events}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.131.0> 2011-11-22 17:48:01 =============================================================================== Initting ns_node_disco with [] INFO REPORT <5901.132.0> 2011-11-22 17:48:01 =============================================================================== ns_node_disco cookie_sync INFO REPORT <5901.132.0> 2011-11-22 17:48:01 =============================================================================== ns_log: logging ns_node_disco:3:Initial otp cookie generated: sgzcoaesvptwzitb INFO REPORT <5901.118.0> 2011-11-22 17:48:01 =============================================================================== config change: otp -> [{cookie,sgzcoaesvptwzitb}] INFO REPORT <5901.132.0> 2011-11-22 17:48:01 =============================================================================== ns_node_disco: nodes_wanted updated: ['ns_1@127.0.0.1'], with cookie: sgzcoaesvptwzitb INFO REPORT <5901.132.0> 2011-11-22 17:48:01 =============================================================================== ns_node_disco: nodes_wanted pong: ['ns_1@127.0.0.1'], with cookie: sgzcoaesvptwzitb PROGRESS REPORT <5901.129.0> 2011-11-22 17:48:01 =============================================================================== supervisor {local,ns_node_disco_sup} started [{pid,<5901.131.0>}, {name,ns_node_disco}, {mfargs,{ns_node_disco,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.129.0> 2011-11-22 17:48:01 =============================================================================== supervisor {local,ns_node_disco_sup} started [{pid,<5901.135.0>}, {name,ns_node_disco_log}, {mfargs,{ns_node_disco_log,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.129.0> 2011-11-22 17:48:01 =============================================================================== supervisor {local,ns_node_disco_sup} started [{pid,<5901.136.0>}, {name,ns_node_disco_conf_events}, {mfargs,{ns_node_disco_conf_events,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.137.0> 2011-11-22 17:48:01 =============================================================================== ns_1@127.0.0.1:<5901.137.0>:ns_config_rep:56: init pulling INFO REPORT <5901.137.0> 2011-11-22 17:48:01 =============================================================================== ns_1@127.0.0.1:<5901.137.0>:ns_config_rep:58: init pushing INFO REPORT <5901.137.0> 2011-11-22 17:48:01 =============================================================================== ns_1@127.0.0.1:<5901.137.0>:ns_config_rep:62: init reannouncing INFO REPORT <5901.118.0> 2011-11-22 17:48:01 =============================================================================== ns_node_disco_conf_events config on otp INFO REPORT <5901.118.0> 2011-11-22 17:48:01 =============================================================================== config change: otp -> [{cookie,sgzcoaesvptwzitb}] INFO REPORT <5901.118.0> 2011-11-22 17:48:01 =============================================================================== config change: directory -> "/opt/membase/var/lib/membase/config" INFO REPORT <5901.118.0> 2011-11-22 17:48:01 =============================================================================== config change: {node,'ns_1@127.0.0.1',config_version} -> {1,7,2} INFO REPORT <5901.118.0> 2011-11-22 17:48:01 =============================================================================== config change: auto_failover_cfg -> [{enabled,false},{timeout,30},{max_nodes,1},{count,0}] INFO REPORT <5901.139.0> 2011-11-22 17:48:01 =============================================================================== ns_node_disco cookie_sync INFO REPORT <5901.118.0> 2011-11-22 17:48:01 =============================================================================== config change: buckets -> [{configs,[]}] INFO REPORT <5901.139.0> 2011-11-22 17:48:01 =============================================================================== ns_node_disco: nodes_wanted updated: ['ns_1@127.0.0.1'], with cookie: sgzcoaesvptwzitb INFO REPORT <5901.118.0> 2011-11-22 17:48:01 =============================================================================== config change: email_alerts -> [{recipients,["root@localhost"]}, {sender,"membase@localhost"}, {enabled,true}, {email_server,[{user,[]}, {pass,[]}, {host,"localhost"}, {port,25}, {encrypt,false}]}, {alerts,[auto_failover_node,auto_failover_maximum_reached, auto_failover_other_nodes_down,auto_failover_cluster_too_small]}] INFO REPORT <5901.118.0> 2011-11-22 17:48:01 =============================================================================== config change: memory_quota -> 16090 INFO REPORT <5901.118.0> 2011-11-22 17:48:01 =============================================================================== ns_node_disco_conf_events config on nodes_wanted INFO REPORT <5901.118.0> 2011-11-22 17:48:01 =============================================================================== config change: nodes_wanted -> ['ns_1@127.0.0.1'] INFO REPORT <5901.139.0> 2011-11-22 17:48:01 =============================================================================== ns_node_disco: nodes_wanted pong: ['ns_1@127.0.0.1'], with cookie: sgzcoaesvptwzitb INFO REPORT <5901.140.0> 2011-11-22 17:48:01 =============================================================================== ns_node_disco cookie_sync INFO REPORT <5901.118.0> 2011-11-22 17:48:01 =============================================================================== config change: port_servers -> [{moxi,"/opt/membase/bin/moxi", ["-Z", {"port_listen=~B,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", [port]}, "-z", {"url=http://127.0.0.1:~B/pools/default/saslBucketsStreaming", [{rest,port}]}, "-p","0","-Y","y","-O","stderr", {"~s",[verbosity]}], [{env,[{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR",{"~s",[{ns_moxi_sup,rest_user,[]}]}}, {"MOXI_SASL_PLAIN_PWD",{"~s",[{ns_moxi_sup,rest_pass,[]}]}}]}, use_stdio,exit_status,port_server_send_eol,stderr_to_stdout,stream]}, {memcached,"/opt/membase/bin/memcached", ["-X","/opt/membase/lib/memcached/stdin_term_handler.so","-p", {"~B",[port]}, "-E","/opt/membase/lib/memcached/bucket_engine.so","-B","binary", "-r","-c","10000","-e", {"admin=~s;default_bucket_name=default;auto_create=false", [admin_user]}, {"~s",[verbosity]}], [{env,[{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE",{"~s",[{isasl,path}]}}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,exit_status,port_server_send_eol, stream]}] INFO REPORT <5901.118.0> 2011-11-22 17:48:01 =============================================================================== config change: replication -> [{enabled,true}] INFO REPORT <5901.118.0> 2011-11-22 17:48:01 =============================================================================== config change: rest -> [{port,8091}] INFO REPORT <5901.118.0> 2011-11-22 17:48:01 =============================================================================== config change: rest_creds -> ******** INFO REPORT <5901.118.0> 2011-11-22 17:48:01 =============================================================================== config change: {node,'ns_1@127.0.0.1',isasl} -> [{path,"/opt/membase/var/lib/membase/data/isasl.pw"}] INFO REPORT <5901.118.0> 2011-11-22 17:48:01 =============================================================================== config change: {node,'ns_1@127.0.0.1',membership} -> active INFO REPORT <5901.118.0> 2011-11-22 17:48:01 =============================================================================== config change: {node,'ns_1@127.0.0.1',memcached} -> [{port,11210}, {dbdir,"/opt/membase/var/lib/membase/data"}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine,"/opt/membase/lib/memcached/bucket_engine.so"}, {engines, [{membase, [{engine,"/opt/membase/lib/memcached/ep.so"}, {initfile,"/opt/membase/etc/membase/init.sql"}, {static_config_string, "vb0=false;waitforwarmup=false;failpartialwarmup=false;shardpattern=%d/%b-%i.mb;db_strategy=multiMTVBDB"}]}, {memcached, [{engine,"/opt/membase/lib/memcached/default_engine.so"}, {static_config_string,"vb0=true"}]}]}, {verbosity,[]}] INFO REPORT <5901.118.0> 2011-11-22 17:48:01 =============================================================================== config change: {node,'ns_1@127.0.0.1',moxi} -> [{port,11211},{verbosity,[]}] INFO REPORT <5901.140.0> 2011-11-22 17:48:01 =============================================================================== ns_node_disco: nodes_wanted updated: ['ns_1@127.0.0.1'], with cookie: sgzcoaesvptwzitb INFO REPORT <5901.140.0> 2011-11-22 17:48:01 =============================================================================== ns_node_disco: nodes_wanted pong: ['ns_1@127.0.0.1'], with cookie: sgzcoaesvptwzitb INFO REPORT <5901.118.0> 2011-11-22 17:48:01 =============================================================================== config change: {node,'ns_1@127.0.0.1',ns_log} -> [{filename,"/opt/membase/var/lib/membase/data/ns_log"}] INFO REPORT <5901.118.0> 2011-11-22 17:48:01 =============================================================================== config change: {node,'ns_1@127.0.0.1',rest} -> [{port,8091},{port_meta,global}] INFO REPORT <5901.118.0> 2011-11-22 17:48:01 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <5901.137.0> 2011-11-22 17:48:01 =============================================================================== Pushing config PROGRESS REPORT <5901.129.0> 2011-11-22 17:48:01 =============================================================================== supervisor {local,ns_node_disco_sup} started [{pid,<5901.137.0>}, {name,ns_config_rep}, {mfargs,{ns_config_rep,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.126.0> 2011-11-22 17:48:01 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.129.0>}, {name,ns_node_disco_sup}, {mfargs,{ns_node_disco_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <5901.126.0> 2011-11-22 17:48:01 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.142.0>}, {name,ns_tick_event}, {mfargs,{gen_event,start_link,[{local,ns_tick_event}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.137.0> 2011-11-22 17:48:01 =============================================================================== Pushing config done INFO REPORT <5901.144.0> 2011-11-22 17:48:01 =============================================================================== ns_1@127.0.0.1:<5901.144.0>:mb_master:97: I'm the only node, so I'm the master. INFO REPORT <5901.145.0> 2011-11-22 17:48:01 =============================================================================== ns_1@127.0.0.1:<5901.145.0>:misc:785: start_singleton(gen_fsm, ns_orchestrator, [], []): started as <5901.146.0> on 'ns_1@127.0.0.1' PROGRESS REPORT <5901.145.0> 2011-11-22 17:48:01 =============================================================================== supervisor {local,mb_master_sup} started [{pid,<5901.146.0>}, {name,ns_orchestrator}, {mfargs,{ns_orchestrator,start_link,[]}}, {restart_type,permanent}, {shutdown,20}, {child_type,worker}] INFO REPORT <5901.145.0> 2011-11-22 17:48:01 =============================================================================== ns_1@127.0.0.1:<5901.145.0>:misc:785: start_singleton(gen_server, ns_tick, [], []): started as <5901.147.0> on 'ns_1@127.0.0.1' PROGRESS REPORT <5901.145.0> 2011-11-22 17:48:01 =============================================================================== supervisor {local,mb_master_sup} started [{pid,<5901.147.0>}, {name,ns_tick}, {mfargs,{ns_tick,start_link,[]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] INFO REPORT <5901.148.0> 2011-11-22 17:48:01 =============================================================================== ns_1@127.0.0.1:<5901.148.0>:auto_failover:120: init auto_failover. INFO REPORT <5901.145.0> 2011-11-22 17:48:01 =============================================================================== ns_1@127.0.0.1:<5901.145.0>:misc:785: start_singleton(gen_server, auto_failover, [], []): started as <5901.148.0> on 'ns_1@127.0.0.1' PROGRESS REPORT <5901.145.0> 2011-11-22 17:48:01 =============================================================================== supervisor {local,mb_master_sup} started [{pid,<5901.148.0>}, {name,auto_failover}, {mfargs,{auto_failover,start_link,[]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <5901.126.0> 2011-11-22 17:48:01 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.144.0>}, {name,mb_master}, {mfargs,{mb_master,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <5901.126.0> 2011-11-22 17:48:01 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.149.0>}, {name,buckets_events}, {mfargs,{gen_event,start_link,[{local,buckets_events}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.150.0> 2011-11-22 17:48:01 =============================================================================== supervisor {local,ns_mail_sup} started [{pid,<5901.151.0>}, {name,ns_mail}, {mfargs,{ns_mail,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.150.0> 2011-11-22 17:48:01 =============================================================================== supervisor {local,ns_mail_sup} started [{pid,<5901.152.0>}, {name,ns_mail_log}, {mfargs,{ns_mail_log,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.126.0> 2011-11-22 17:48:01 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.150.0>}, {name,ns_mail_sup}, {mfargs,{ns_mail_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] INFO REPORT <5901.128.0> 2011-11-22 17:48:01 =============================================================================== ns_1@127.0.0.1:<5901.128.0>:ns_mail_log:45: ns_mail_log started up PROGRESS REPORT <5901.126.0> 2011-11-22 17:48:01 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.153.0>}, {name,ns_heart}, {mfargs,{ns_heart,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.126.0> 2011-11-22 17:48:01 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.155.0>}, {name,ns_doctor}, {mfargs,{ns_doctor,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] ERROR REPORT <5901.153.0> 2011-11-22 17:48:01 =============================================================================== ns_1@127.0.0.1:<5901.153.0>:ns_heart:137: Failed to grab system stats: {'EXIT',{noproc,{gen_server,call, [{'stats_reader-@system','ns_1@127.0.0.1'}, {latest,"minute"}]}}} INFO REPORT <5901.155.0> 2011-11-22 17:48:01 =============================================================================== ns_1@127.0.0.1:<5901.155.0>:ns_doctor:82: Got initial status [{'ns_1@127.0.0.1', [{last_heard, {1321,984081, 374108}}, {active_buckets, []}, {ready_buckets, []}, {replication, []}, {memory, [{total, 21796320}, {processes, 3523144}, {processes_used, 3510496}, {system, 18273176}, {atom, 761969}, {atom_used, 733937}, {binary, 695168}, {code, 6891478}, {ets, 470280}]}, {system_stats, [{cpu_utilization_rate, 0}, {swap_total, 0}, {swap_used, 0}]}, {interesting_stats, []}, {cluster_compatibility_version, 1}, {version, [{os_mon, "2.2.5"}, {mnesia, "4.4.17"}, {kernel, "2.14.3"}, {sasl, "2.1.9.3"}, {ns_server, "1.7.2r-20-g6604356"}, {stdlib, "1.17.3"}]}, {system_arch, "x86_64-unknown-linux-gnu"}, {wall_clock,0}, {memory_data, {17946165248, 959442944, {<5901.7.0>, 513760}}}, {disk_data, [{"/", 8256952,15}, {"/dev", 8754936,1}, {"/run", 3505112,1}, {"/run/lock", 5120,0}, {"/run/shm", 8762776,0}, {"/mnt", 423135208, 1}]}, {meminfo, <<"MemTotal: 17525552 kB\nMemFree: 16583488 kB\nBuffers: 27972 kB\nCached: 600604 kB\nSwapCached: 0 kB\nActive: 362920 kB\nInactive: 313352 kB\nActive(anon): 47544 kB\nInactive(anon): 208 kB\nActive(file): 315376 kB\nInactive(file): 313144 kB\nUnevictable: 0 kB\nMlocked: 0 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 5412 kB\nWriteback: 0 kB\nAnonPages: 47508 kB\nMapped: 7908 kB\nShmem: 324 kB\nSlab: 53908 kB\nSReclaimable: 47584 kB\nSUnreclaim: 6324 kB\nKernelStack: 648 kB\nPageTables: 1848 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nWritebackTmp: 0 kB\nCommitLimit: 8762776 kB\nCommitted_AS: 126356 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 64000 kB\nVmallocChunk: 34359674252 kB\nHardwareCorrupted: 0 kB\nAnonHugePages: 0 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugePages_Surp: 0\nHugepagesize: 2048 kB\nDirectMap4k: 17928192 kB\nDirectMap2M: 0 kB\n">>}, {system_memory_data, [{system_total_memory, 17946165248}, {free_swap, 0}, {total_swap, 0}, {cached_memory, 615018496}, {buffered_memory, 28643328}, {free_memory, 16981491712}, {total_memory, 17946165248}]}, {statistics, [{wall_clock, {630,433}}, {context_switches, {20770,0}}, {garbage_collection, {1251, 2596016, 0}}, {io, {{input, 6326854}, {output, 1462748}}}, {reductions, {1345982, 1345982}}, {run_queue, 0}, {runtime, {330, 330}}]}]}] PROGRESS REPORT <5901.157.0> 2011-11-22 17:48:01 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<5901.158.0>}, {name,menelaus_web}, {mfargs,{menelaus_web,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] PROGRESS REPORT <5901.157.0> 2011-11-22 17:48:01 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<5901.160.0>}, {name,menelaus_event}, {mfargs,{menelaus_event,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] PROGRESS REPORT <5901.157.0> 2011-11-22 17:48:01 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<5901.161.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] PROGRESS REPORT <5901.157.0> 2011-11-22 17:48:01 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<5901.162.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <5901.126.0> 2011-11-22 17:48:01 =============================================================================== ns_log: logging menelaus_sup:1:Membase Server has started on web port 8091 on node 'ns_1@127.0.0.1'. PROGRESS REPORT <5901.126.0> 2011-11-22 17:48:01 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.157.0>}, {name,menelaus}, {mfargs,{menelaus_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <5901.163.0> 2011-11-22 17:48:01 =============================================================================== supervisor {local,ns_port_sup} started [{pid,<5901.164.0>}, {name,ns_port_init}, {mfargs,{ns_port_init,start_link,[]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] INFO REPORT <5901.165.0> 2011-11-22 17:48:01 =============================================================================== starting ns_port_server with delay of 5000 PROGRESS REPORT <5901.163.0> 2011-11-22 17:48:01 =============================================================================== supervisor {local,ns_port_sup} started [{pid,<5901.165.0>}, {name, {moxi,"/opt/membase/bin/moxi", ["-Z", "port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", "-z", "url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming", "-p","0","-Y","y","-O","stderr",[]], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR",[]}, {"MOXI_SASL_PLAIN_PWD",[]}]}, use_stdio,exit_status,port_server_send_eol, stderr_to_stdout,stream]}}, {mfargs, {supervisor_cushion,start_link, [moxi,5000,ns_port_server,start_link, [moxi,"/opt/membase/bin/moxi", ["-Z", "port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", "-z", "url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming", "-p","0","-Y","y","-O","stderr",[]], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR",[]}, {"MOXI_SASL_PLAIN_PWD",[]}]}, use_stdio,exit_status,port_server_send_eol, stderr_to_stdout,stream]]]}}, {restart_type,permanent}, {shutdown,10000}, {child_type,worker}] INFO REPORT <5901.167.0> 2011-11-22 17:48:01 =============================================================================== starting ns_port_server with delay of 5000 PROGRESS REPORT <5901.163.0> 2011-11-22 17:48:01 =============================================================================== supervisor {local,ns_port_sup} started [{pid,<5901.167.0>}, {name, {memcached,"/opt/membase/bin/memcached", ["-X","/opt/membase/lib/memcached/stdin_term_handler.so", "-p","11210","-E", "/opt/membase/lib/memcached/bucket_engine.so","-B", "binary","-r","-c","10000","-e", "admin=_admin;default_bucket_name=default;auto_create=false", []], [{env, [{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE", "/opt/membase/var/lib/membase/data/isasl.pw"}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,exit_status, port_server_send_eol,stream]}}, {mfargs, {supervisor_cushion,start_link, [memcached,5000,ns_port_server,start_link, [memcached,"/opt/membase/bin/memcached", ["-X","/opt/membase/lib/memcached/stdin_term_handler.so", "-p","11210","-E", "/opt/membase/lib/memcached/bucket_engine.so","-B", "binary","-r","-c","10000","-e", "admin=_admin;default_bucket_name=default;auto_create=false", []], [{env, [{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE", "/opt/membase/var/lib/membase/data/isasl.pw"}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,exit_status, port_server_send_eol,stream]]]}}, {restart_type,permanent}, {shutdown,10000}, {child_type,worker}] PROGRESS REPORT <5901.126.0> 2011-11-22 17:48:01 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.163.0>}, {name,ns_port_sup}, {mfargs,{ns_port_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,60000}, {child_type,worker}] PROGRESS REPORT <5901.126.0> 2011-11-22 17:48:01 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.169.0>}, {name,ns_stats_event}, {mfargs,{gen_event,start_link,[{local,ns_stats_event}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.126.0> 2011-11-22 17:48:01 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.170.0>}, {name,ns_bucket_worker}, {mfargs,{work_queue,start_link,[ns_bucket_worker]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.126.0> 2011-11-22 17:48:01 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.171.0>}, {name,ns_bucket_sup}, {mfargs,{ns_bucket_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <5901.126.0> 2011-11-22 17:48:01 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.172.0>}, {name,system_stats_collector}, {mfargs,{system_stats_collector,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.126.0> 2011-11-22 17:48:01 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.173.0>}, {name,{stats_archiver,"@system"}}, {mfargs,{stats_archiver,start_link,["@system"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.67.0> 2011-11-22 17:48:01 =============================================================================== ns_1@127.0.0.1:<5901.67.0>:mb_mnesia:109: Creating table 'stats_archiver-@system-minute' PROGRESS REPORT <5901.126.0> 2011-11-22 17:48:01 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.175.0>}, {name,{stats_reader,"@system"}}, {mfargs,{stats_reader,start_link,["@system"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.126.0> 2011-11-22 17:48:01 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.178.0>}, {name,ns_moxi_sup_work_queue}, {mfargs,{work_queue,start_link,[ns_moxi_sup_work_queue]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.126.0> 2011-11-22 17:48:01 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.179.0>}, {name,ns_moxi_sup}, {mfargs,{ns_moxi_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <5901.54.0> 2011-11-22 17:48:01 =============================================================================== supervisor {local,ns_server_cluster_sup} started [{pid,<5901.126.0>}, {name,ns_server_sup}, {mfargs,{ns_server_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <5901.7.0> 2011-11-22 17:48:01 =============================================================================== application ns_server started_at 'ns_1@127.0.0.1' INFO REPORT <5901.67.0> 2011-11-22 17:48:01 =============================================================================== ns_1@127.0.0.1:<5901.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-@system-minute', [{name,'stats_archiver-@system-minute'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@127.0.0.1']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1321,984081,398506},'ns_1@127.0.0.1'}}, {version,{{2,0},[]}}]}, {tid,6,<5901.177.0>}} INFO REPORT <5901.67.0> 2011-11-22 17:48:01 =============================================================================== ns_1@127.0.0.1:<5901.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-@system-minute', [{name,'stats_archiver-@system-minute'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@127.0.0.1']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1321,984081,398506},'ns_1@127.0.0.1'}}, {version,{{2,0},[]}}]}, {tid,6,<5901.177.0>}} INFO REPORT <5901.67.0> 2011-11-22 17:48:01 =============================================================================== ns_1@127.0.0.1:<5901.67.0>:mb_mnesia:109: Creating table 'stats_archiver-@system-hour' INFO REPORT <5901.67.0> 2011-11-22 17:48:01 =============================================================================== ns_1@127.0.0.1:<5901.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-@system-hour', [{name,'stats_archiver-@system-hour'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@127.0.0.1']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1321,984081,412208},'ns_1@127.0.0.1'}}, {version,{{2,0},[]}}]}, {tid,7,<5901.186.0>}} INFO REPORT <5901.67.0> 2011-11-22 17:48:01 =============================================================================== ns_1@127.0.0.1:<5901.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-@system-hour', [{name,'stats_archiver-@system-hour'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@127.0.0.1']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1321,984081,412208},'ns_1@127.0.0.1'}}, {version,{{2,0},[]}}]}, {tid,7,<5901.186.0>}} INFO REPORT <5901.67.0> 2011-11-22 17:48:01 =============================================================================== ns_1@127.0.0.1:<5901.67.0>:mb_mnesia:109: Creating table 'stats_archiver-@system-day' INFO REPORT <5901.67.0> 2011-11-22 17:48:01 =============================================================================== ns_1@127.0.0.1:<5901.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-@system-day', [{name,'stats_archiver-@system-day'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@127.0.0.1']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1321,984081,451634},'ns_1@127.0.0.1'}}, {version,{{2,0},[]}}]}, {tid,8,<5901.193.0>}} INFO REPORT <5901.67.0> 2011-11-22 17:48:01 =============================================================================== ns_1@127.0.0.1:<5901.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-@system-day', [{name,'stats_archiver-@system-day'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@127.0.0.1']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1321,984081,451634},'ns_1@127.0.0.1'}}, {version,{{2,0},[]}}]}, {tid,8,<5901.193.0>}} INFO REPORT <5901.67.0> 2011-11-22 17:48:01 =============================================================================== ns_1@127.0.0.1:<5901.67.0>:mb_mnesia:109: Creating table 'stats_archiver-@system-week' INFO REPORT <5901.67.0> 2011-11-22 17:48:01 =============================================================================== ns_1@127.0.0.1:<5901.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-@system-week', [{name,'stats_archiver-@system-week'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@127.0.0.1']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1321,984081,463122},'ns_1@127.0.0.1'}}, {version,{{2,0},[]}}]}, {tid,9,<5901.200.0>}} INFO REPORT <5901.67.0> 2011-11-22 17:48:01 =============================================================================== ns_1@127.0.0.1:<5901.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-@system-week', [{name,'stats_archiver-@system-week'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@127.0.0.1']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1321,984081,463122},'ns_1@127.0.0.1'}}, {version,{{2,0},[]}}]}, {tid,9,<5901.200.0>}} INFO REPORT <5901.67.0> 2011-11-22 17:48:01 =============================================================================== ns_1@127.0.0.1:<5901.67.0>:mb_mnesia:109: Creating table 'stats_archiver-@system-month' INFO REPORT <5901.67.0> 2011-11-22 17:48:01 =============================================================================== ns_1@127.0.0.1:<5901.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-@system-month', [{name,'stats_archiver-@system-month'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@127.0.0.1']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1321,984081,501780},'ns_1@127.0.0.1'}}, {version,{{2,0},[]}}]}, {tid,10,<5901.207.0>}} INFO REPORT <5901.67.0> 2011-11-22 17:48:01 =============================================================================== ns_1@127.0.0.1:<5901.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-@system-month', [{name,'stats_archiver-@system-month'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@127.0.0.1']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1321,984081,501780},'ns_1@127.0.0.1'}}, {version,{{2,0},[]}}]}, {tid,10,<5901.207.0>}} INFO REPORT <5901.67.0> 2011-11-22 17:48:01 =============================================================================== ns_1@127.0.0.1:<5901.67.0>:mb_mnesia:109: Creating table 'stats_archiver-@system-year' INFO REPORT <5901.67.0> 2011-11-22 17:48:01 =============================================================================== ns_1@127.0.0.1:<5901.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-@system-year', [{name,'stats_archiver-@system-year'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@127.0.0.1']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1321,984081,511107},'ns_1@127.0.0.1'}}, {version,{{2,0},[]}}]}, {tid,11,<5901.214.0>}} INFO REPORT <5901.67.0> 2011-11-22 17:48:01 =============================================================================== ns_1@127.0.0.1:<5901.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-@system-year', [{name,'stats_archiver-@system-year'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@127.0.0.1']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1321,984081,511107},'ns_1@127.0.0.1'}}, {version,{{2,0},[]}}]}, {tid,11,<5901.214.0>}} PROGRESS REPORT <5901.223.0> 2011-11-22 17:48:04 =============================================================================== supervisor {local,inet_gethost_native_sup} started [{pid,<5901.224.0>},{mfa,{inet_gethost_native,init,[[]]}}] PROGRESS REPORT <5901.26.0> 2011-11-22 17:48:04 =============================================================================== supervisor {local,kernel_safe_sup} started [{pid,<5901.223.0>}, {name,inet_gethost_native_sup}, {mfargs,{inet_gethost_native,start_link,[]}}, {restart_type,temporary}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.155.0> 2011-11-22 17:49:01 =============================================================================== ns_1@127.0.0.1:<5901.155.0>:ns_doctor:86: Current node statuses: [{'ns_1@127.0.0.1', [{last_heard,{1321,984136,360322}}, {active_buckets,[]}, {ready_buckets,[]}, {replication,[]}, {memory, [{total,23946024}, {processes,4910232}, {processes_used,4894792}, {system,19035792}, {atom,785401}, {atom_used,766564}, {binary,746376}, {code,7415109}, {ets,610232}]}, {system_stats, [{cpu_utilization_rate,0.25906735751295334}, {swap_total,0}, {swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.5"}, {mnesia,"4.4.17"}, {kernel,"2.14.3"}, {sasl,"2.1.9.3"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,55}, {memory_data,{17946165248,959442944,{<5901.7.0>,513760}}}, {disk_data, [{"/",8256952,15}, {"/dev",8754936,1}, {"/run",3505112,1}, {"/run/lock",5120,0}, {"/run/shm",8762776,0}, {"/mnt",423135208,1}]}, {meminfo, <<"MemTotal: 17525552 kB\nMemFree: 16591000 kB\nBuffers: 28160 kB\nCached: 600836 kB\nSwapCached: 0 kB\nActive: 354772 kB\nInactive: 313548 kB\nActive(anon): 39460 kB\nInactive(anon): 208 kB\nActive(file): 315312 kB\nInactive(file): 313340 kB\nUnevictable: 0 kB\nMlocked: 0 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 56 kB\nWriteback: 0 kB\nAnonPages: 39396 kB\nMapped: 8836 kB\nShmem: 324 kB\nSlab: 54144 kB\nSReclaimable: 47396 kB\nSUnreclaim: 6748 kB\nKernelStack: 896 kB\nPageTables: 1928 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nWritebackTmp: 0 kB\nCommitLimit: 8762776 kB\nCommitted_AS: 235440 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 64000 kB\nVmallocChunk: 34359674252 kB\nHardwareCorrupted: 0 kB\nAnonHugePages: 0 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugePages_Surp: 0\nHugepagesize: 2048 kB\nDirectMap4k: 17928192 kB\nDirectMap2M: 0 kB\n">>}, {system_memory_data, [{system_total_memory,17946165248}, {free_swap,0}, {total_swap,0}, {cached_memory,615256064}, {buffered_memory,28835840}, {free_memory,16989184000}, {total_memory,17946165248}]}, {statistics, [{wall_clock,{45631,1}}, {context_switches,{26635,0}}, {garbage_collection,{2353,4344909,0}}, {io,{{input,6839903},{output,1573639}}}, {reductions,{1997128,173692}}, {run_queue,0}, {runtime,{480,40}}]}]}] INFO REPORT <0.54.0> 2011-11-30 18:47:41 =============================================================================== nonode@nohost:<0.54.0>:log_os_info:25: OS type: {unix,linux} Version: {3,0,0} Runtime info: [{otp_release,"R14B02"}, {erl_version,"5.8.3"}, {erl_version_long, "Erlang R14B02 (erts-5.8.3) [source] [64-bit] [smp:2:2] [rq:2] [async-threads:16] [hipe] [kernel-poll:false]\n"}, {system_arch_raw,"x86_64-unknown-linux-gnu"}, {system_arch,"x86_64-unknown-linux-gnu"}, {localtime,{{2011,11,30},{18,47,41}}}, {memory, [{total,15894240}, {processes,1726016}, {processes_used,1711072}, {system,14168224}, {atom,452097}, {atom_used,423209}, {binary,698960}, {code,3363956}, {ets,279440}]}, {loaded, [ns_info,log_os_info,misc,ns_log_mf_h, ns_server_cluster_sup,timer,ns_server,io_lib_fread, cpu_sup,memsup,disksup,os_mon,io_lib_pretty,io_lib_format, io_lib,io,sasl_report,release_handler,calendar,overload, alarm_handler,sasl_report_tty_h,sasl,ns_bootstrap, file_io_server,orddict,erl_eval,c,error_logger_tty_h, queue,kernel_config,user,user_sup,supervisor_bridge, standard_error,ram_file,file,beam_lib,unicode,binary,ets, gb_sets,hipe_unified_loader,packages,code_server,code, file_server,net_kernel,global_group,erl_distribution, filename,inet_gethost_native,inet_parse,inet,inet_udp,os, inet_config,inet_db,global,gb_trees,rpc,supervisor,kernel, application_master,sys,application,gen_server,erl_parse, proplists,erl_scan,lists,application_controller,proc_lib, gen,gen_event,error_logger,heart,error_handler,erlang, erl_prim_loader,prim_zip,zlib,prim_file,prim_inet,init, otp_ring0]}, {applications, [{os_mon,"CPO CXC 138 46","2.2.5"}, {kernel,"ERTS CXC 138 10","2.14.3"}, {sasl,"SASL CXC 138 11","2.1.9.3"}, {ns_server,"Membase server","1.7.2r-20-g6604356"}, {stdlib,"ERTS CXC 138 10","1.17.3"}]}, {pre_loaded, [erlang,erl_prim_loader,prim_zip,zlib,prim_file,prim_inet, init,otp_ring0]}, {process_count,44}, {node,nonode@nohost}, {nodes,[]}, {registered, [rex,release_handler,inet_db,kernel_sup, ns_server_cluster_sup,code_server,global_name_server, overload,file_server_2,application_controller, alarm_handler,os_mon_sup,standard_error,init,cpu_sup, global_group,erl_prim_loader,sasl_sup,memsup,disksup, timer_server,standard_error_sup,user,sasl_safe_sup, error_logger,kernel_safe_sup]}, {cookie,nocookie}, {wordsize,8}, {wall_clock,0}] INFO REPORT <0.54.0> 2011-11-30 18:47:41 =============================================================================== nonode@nohost:<0.54.0>:log_os_info:27: Manifest: ["bucket_engine 1.7.0-0-g721dff0 Linux-x86_64", "ep-engine 1.7.1.1-39-g4bd26a3 Linux-x86_64", "geocouch couchbase_1.1_geo-0-gd09068e Linux-x86_64", "icu4c 2635939 Linux-x86_64","libconflate 1.7.0-0-gfee8f94 Linux-x86_64", "libmemcached 2cd40dc Linux-x86_64", "libvbucket 1.7.0-0-g4bd0aba Linux-x86_64", "manifest 1.7.0-30-gd83dca9 Linux-x86_64", "membase-cli 1.7.2-0-g1d9c77e Linux-x86_64", "membase-server 1.7.2r-20-g6604356 Linux-x86_64", "membasex 1.7.0-3-g30d5dba Linux-x86_64", "memcached membase-1.7.1-0-gf99c147 Linux-x86_64", "memcachetest 0.8.3-0-g88ae3b3 Linux-x86_64", "moxi 1.7.2-0-gd5076d9 Linux-x86_64", "ns_server 1.7.2-0-g4925ee0 Linux-x86_64", "otp OTP_R14B03-0-g4a5a758 Linux-x86_64", "portsigar 1.7.0-0-ga191e6c Linux-x86_64", "sigar sigar-1.6.4-406-ge1dcf32 Linux-x86_64", "spidermonkey a3c48c1 Linux-x86_64","tlm 1.7.1-0-g535dadc Linux-x86_64", "vbucketmigrator 1.7.0-0-g0fdc96c Linux-x86_64"] PROGRESS REPORT <0.54.0> 2011-11-30 18:47:41 =============================================================================== supervisor {local,ns_server_cluster_sup} started [{pid,<0.56.0>}, {name,timeout_diag_logger}, {mfargs,{timeout_diag_logger,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <0.57.0> 2011-11-30 18:47:41 =============================================================================== reading ip config from "/opt/membase/var/lib/membase/ip" ERROR REPORT <0.57.0> 2011-11-30 18:47:41 =============================================================================== Got error:einval. Ignoring bad address:[] INFO REPORT <0.57.0> 2011-11-30 18:47:41 =============================================================================== nonode@nohost:<0.57.0>:dist_manager:105: Attempting to bring up net_kernel with name 'ns_1@127.0.0.1' PROGRESS REPORT <0.58.0> 2011-11-30 18:47:41 =============================================================================== supervisor {local,net_sup} started [{pid,<0.59.0>}, {name,erl_epmd}, {mfargs,{erl_epmd,start_link,[]}}, {restart_type,permanent}, {shutdown,2000}, {child_type,worker}] PROGRESS REPORT <0.58.0> 2011-11-30 18:47:41 =============================================================================== supervisor {local,net_sup} started [{pid,<0.60.0>}, {name,auth}, {mfargs,{auth,start_link,[]}}, {restart_type,permanent}, {shutdown,2000}, {child_type,worker}] PROGRESS REPORT <5901.58.0> 2011-11-30 18:47:41 =============================================================================== supervisor {local,net_sup} started [{pid,<5901.61.0>}, {name,net_kernel}, {mfargs,{net_kernel,start_link,[['ns_1@127.0.0.1',longnames]]}}, {restart_type,permanent}, {shutdown,2000}, {child_type,worker}] PROGRESS REPORT <5901.11.0> 2011-11-30 18:47:41 =============================================================================== supervisor {local,kernel_sup} started [{pid,<5901.58.0>}, {name,net_sup_dynamic}, {mfargs,{erl_distribution,start_link, [['ns_1@127.0.0.1',longnames]]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,supervisor}] PROGRESS REPORT <5901.54.0> 2011-11-30 18:47:41 =============================================================================== supervisor {local,ns_server_cluster_sup} started [{pid,<5901.57.0>}, {name,dist_manager}, {mfargs,{dist_manager,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.54.0> 2011-11-30 18:47:41 =============================================================================== supervisor {local,ns_server_cluster_sup} started [{pid,<5901.64.0>}, {name,ns_cluster}, {mfargs,{ns_cluster,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] PROGRESS REPORT <5901.65.0> 2011-11-30 18:47:41 =============================================================================== supervisor {local,mb_mnesia_sup} started [{pid,<5901.66.0>}, {name,mb_mnesia_events}, {mfargs,{gen_event,start_link,[{local,mb_mnesia_events}]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <5901.72.0> 2011-11-30 18:47:42 =============================================================================== supervisor {local,mnesia_sup} started [{pid,<5901.73.0>}, {name,mnesia_event}, {mfargs,{mnesia_sup,start_event,[]}}, {restart_type,permanent}, {shutdown,30000}, {child_type,worker}] PROGRESS REPORT <5901.74.0> 2011-11-30 18:47:42 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<5901.75.0>}, {name,mnesia_monitor}, {mfargs,{mnesia_monitor,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] PROGRESS REPORT <5901.74.0> 2011-11-30 18:47:42 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<5901.76.0>}, {name,mnesia_subscr}, {mfargs,{mnesia_subscr,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] PROGRESS REPORT <5901.74.0> 2011-11-30 18:47:42 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<5901.77.0>}, {name,mnesia_locker}, {mfargs,{mnesia_locker,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] PROGRESS REPORT <5901.74.0> 2011-11-30 18:47:42 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<5901.78.0>}, {name,mnesia_recover}, {mfargs,{mnesia_recover,start,[]}}, {restart_type,permanent}, {shutdown,180000}, {child_type,worker}] PROGRESS REPORT <5901.26.0> 2011-11-30 18:47:42 =============================================================================== supervisor {local,kernel_safe_sup} started [{pid,<5901.80.0>}, {name,dets_sup}, {mfargs,{dets_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,supervisor}] PROGRESS REPORT <5901.26.0> 2011-11-30 18:47:42 =============================================================================== supervisor {local,kernel_safe_sup} started [{pid,<5901.81.0>}, {name,dets}, {mfargs,{dets_server,start_link,[]}}, {restart_type,permanent}, {shutdown,2000}, {child_type,worker}] PROGRESS REPORT <5901.26.0> 2011-11-30 18:47:42 =============================================================================== supervisor {local,kernel_safe_sup} started [{pid,<5901.85.0>}, {name,disk_log_sup}, {mfargs,{disk_log_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,supervisor}] PROGRESS REPORT <5901.26.0> 2011-11-30 18:47:42 =============================================================================== supervisor {local,kernel_safe_sup} started [{pid,<5901.86.0>}, {name,disk_log_server}, {mfargs,{disk_log_server,start_link,[]}}, {restart_type,permanent}, {shutdown,2000}, {child_type,worker}] PROGRESS REPORT <5901.74.0> 2011-11-30 18:47:42 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<5901.79.0>}, {name,mnesia_tm}, {mfargs,{mnesia_tm,start,[]}}, {restart_type,permanent}, {shutdown,30000}, {child_type,worker}] PROGRESS REPORT <5901.74.0> 2011-11-30 18:47:42 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<5901.106.0>}, {name,mnesia_checkpoint_sup}, {mfargs,{mnesia_checkpoint_sup,start,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <5901.74.0> 2011-11-30 18:47:42 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<5901.107.0>}, {name,mnesia_snmp_sup}, {mfargs,{mnesia_snmp_sup,start,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <5901.74.0> 2011-11-30 18:47:42 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<5901.108.0>}, {name,mnesia_controller}, {mfargs,{mnesia_controller,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] PROGRESS REPORT <5901.74.0> 2011-11-30 18:47:42 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<5901.109.0>}, {name,mnesia_late_loader}, {mfargs,{mnesia_late_loader,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] PROGRESS REPORT <5901.72.0> 2011-11-30 18:47:42 =============================================================================== supervisor {local,mnesia_sup} started [{pid,<5901.74.0>}, {name,mnesia_kernel_sup}, {mfargs,{mnesia_kernel_sup,start,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <5901.7.0> 2011-11-30 18:47:42 =============================================================================== application mnesia started_at 'ns_1@127.0.0.1' INFO REPORT <5901.67.0> 2011-11-30 18:47:42 =============================================================================== ns_1@127.0.0.1:<5901.67.0>:mb_mnesia:417: Using existing disk schema on ['ns_1@127.0.0.1']. INFO REPORT <5901.67.0> 2011-11-30 18:47:42 =============================================================================== ns_1@127.0.0.1:<5901.67.0>:mb_mnesia:427: Have local copy of 'stats_archiver-@system-year' INFO REPORT <5901.67.0> 2011-11-30 18:47:42 =============================================================================== ns_1@127.0.0.1:<5901.67.0>:mb_mnesia:427: Have local copy of 'stats_archiver-@system-month' INFO REPORT <5901.67.0> 2011-11-30 18:47:42 =============================================================================== ns_1@127.0.0.1:<5901.67.0>:mb_mnesia:427: Have local copy of 'stats_archiver-@system-week' INFO REPORT <5901.67.0> 2011-11-30 18:47:42 =============================================================================== ns_1@127.0.0.1:<5901.67.0>:mb_mnesia:427: Have local copy of 'stats_archiver-@system-day' INFO REPORT <5901.67.0> 2011-11-30 18:47:42 =============================================================================== ns_1@127.0.0.1:<5901.67.0>:mb_mnesia:427: Have local copy of 'stats_archiver-@system-hour' INFO REPORT <5901.67.0> 2011-11-30 18:47:42 =============================================================================== ns_1@127.0.0.1:<5901.67.0>:mb_mnesia:427: Have local copy of 'stats_archiver-@system-minute' INFO REPORT <5901.67.0> 2011-11-30 18:47:42 =============================================================================== ns_1@127.0.0.1:<5901.67.0>:mb_mnesia:427: Have local copy of local_config INFO REPORT <5901.67.0> 2011-11-30 18:47:42 =============================================================================== ns_1@127.0.0.1:<5901.67.0>:mb_mnesia:427: Have local copy of cluster INFO REPORT <5901.67.0> 2011-11-30 18:47:42 =============================================================================== ns_1@127.0.0.1:<5901.67.0>:mb_mnesia:245: Current config: [{access_module,mnesia}, {auto_repair,true}, {backup_module,mnesia_backup}, {checkpoints,[]}, {db_nodes,['ns_1@127.0.0.1']}, {debug,verbose}, {directory,"/opt/membase/var/lib/membase/mnesia"}, {dump_log_load_regulation,false}, {dump_log_time_threshold,180000}, {dump_log_update_in_place,true}, {dump_log_write_threshold,1000}, {embedded_mnemosyne,false}, {event_module,mnesia_event}, {extra_db_nodes,[]}, {fallback_activated,false}, {held_locks,[]}, {ignore_fallback_at_startup,false}, {fallback_error_function,{mnesia,lkill}}, {is_running,yes}, {local_tables,['stats_archiver-@system-minute','stats_archiver-@system-year', 'stats_archiver-@system-week','stats_archiver-@system-month', 'stats_archiver-@system-day','stats_archiver-@system-hour', local_config,cluster,schema]}, {lock_queue,[]}, {log_version,"4.3"}, {master_node_tables,[]}, {max_wait_for_decision,10000}, {protocol_version,{7,6}}, {running_db_nodes,['ns_1@127.0.0.1']}, {schema_location,opt_disc}, {schema_version,{3,0}}, {subscribers,[<5901.73.0>,<5901.67.0>]}, {tables,['stats_archiver-@system-year','stats_archiver-@system-month', 'stats_archiver-@system-week','stats_archiver-@system-day', 'stats_archiver-@system-hour','stats_archiver-@system-minute', local_config,cluster,schema]}, {transaction_commits,2}, {transaction_failures,8}, {transaction_log_writes,0}, {transaction_restarts,0}, {transactions,[]}, {use_dir,true}, {core_dir,false}, {no_table_loaders,2}, {dc_dump_limit,4}, {send_compressed,0}, {version,"4.4.17"}] Peers: ['ns_1@127.0.0.1'] PROGRESS REPORT <5901.65.0> 2011-11-30 18:47:42 =============================================================================== supervisor {local,mb_mnesia_sup} started [{pid,<5901.67.0>}, {name,mb_mnesia}, {mfargs,{mb_mnesia,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] PROGRESS REPORT <5901.54.0> 2011-11-30 18:47:42 =============================================================================== supervisor {local,ns_server_cluster_sup} started [{pid,<5901.65.0>}, {name,mb_mnesia_sup}, {mfargs,{mb_mnesia_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] INFO REPORT <5901.157.0> 2011-11-30 18:47:42 =============================================================================== loading static ns_config from "/opt/membase/etc/membase/config" PROGRESS REPORT <5901.157.0> 2011-11-30 18:47:42 =============================================================================== supervisor {local,ns_config_sup} started [{pid,<5901.158.0>}, {name,ns_config_events}, {mfargs,{gen_event,start_link,[{local,ns_config_events}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.157.0> 2011-11-30 18:47:42 =============================================================================== supervisor {local,ns_config_sup} started [{pid,<5901.159.0>}, {name,ns_config}, {mfargs,{ns_config,start_link, ["/opt/membase/etc/membase/config", ns_config_default]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.157.0> 2011-11-30 18:47:42 =============================================================================== supervisor {local,ns_config_sup} started [{pid,<5901.161.0>}, {name,ns_config_remote}, {mfargs,{ns_config_replica,start_link,[{local,ns_config_remote}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.162.0> 2011-11-30 18:47:42 =============================================================================== isasl_sync init: ["/opt/membase/var/lib/membase/data/isasl.pw","_admin", "_admin"] INFO REPORT <5901.162.0> 2011-11-30 18:47:42 =============================================================================== isasl_sync init buckets: [] INFO REPORT <5901.162.0> 2011-11-30 18:47:42 =============================================================================== Writing isasl passwd file: "/opt/membase/var/lib/membase/data/isasl.pw" PROGRESS REPORT <5901.157.0> 2011-11-30 18:47:42 =============================================================================== supervisor {local,ns_config_sup} started [{pid,<5901.162.0>}, {name,ns_config_isasl_sync}, {mfargs,{ns_config_isasl_sync,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.157.0> 2011-11-30 18:47:42 =============================================================================== supervisor {local,ns_config_sup} started [{pid,<5901.164.0>}, {name,ns_config_log}, {mfargs,{ns_config_log,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.54.0> 2011-11-30 18:47:42 =============================================================================== supervisor {local,ns_server_cluster_sup} started [{pid,<5901.157.0>}, {name,ns_config_sup}, {mfargs,{ns_config_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <5901.165.0> 2011-11-30 18:47:42 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.166.0>}, {name,ns_log}, {mfargs,{ns_log,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.165.0> 2011-11-30 18:47:42 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.167.0>}, {name,ns_log_events}, {mfargs,{gen_event,start_link,[{local,ns_log_events}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.168.0> 2011-11-30 18:47:42 =============================================================================== supervisor {local,ns_node_disco_sup} started [{pid,<5901.169.0>}, {name,ns_node_disco_events}, {mfargs,{gen_event,start_link,[{local,ns_node_disco_events}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.170.0> 2011-11-30 18:47:42 =============================================================================== Initting ns_node_disco with [] INFO REPORT <5901.171.0> 2011-11-30 18:47:42 =============================================================================== ns_node_disco cookie_sync INFO REPORT <5901.171.0> 2011-11-30 18:47:42 =============================================================================== ns_log: logging ns_node_disco:2:Node 'ns_1@127.0.0.1' synchronized otp cookie sgzcoaesvptwzitb from cluster INFO REPORT <5901.171.0> 2011-11-30 18:47:42 =============================================================================== ns_node_disco: nodes_wanted updated: ['ns_1@127.0.0.1'], with cookie: sgzcoaesvptwzitb INFO REPORT <5901.171.0> 2011-11-30 18:47:42 =============================================================================== ns_node_disco: nodes_wanted pong: ['ns_1@127.0.0.1'], with cookie: sgzcoaesvptwzitb PROGRESS REPORT <5901.168.0> 2011-11-30 18:47:42 =============================================================================== supervisor {local,ns_node_disco_sup} started [{pid,<5901.170.0>}, {name,ns_node_disco}, {mfargs,{ns_node_disco,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.168.0> 2011-11-30 18:47:42 =============================================================================== supervisor {local,ns_node_disco_sup} started [{pid,<5901.173.0>}, {name,ns_node_disco_log}, {mfargs,{ns_node_disco_log,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.168.0> 2011-11-30 18:47:42 =============================================================================== supervisor {local,ns_node_disco_sup} started [{pid,<5901.174.0>}, {name,ns_node_disco_conf_events}, {mfargs,{ns_node_disco_conf_events,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.175.0> 2011-11-30 18:47:42 =============================================================================== ns_1@127.0.0.1:<5901.175.0>:ns_config_rep:56: init pulling INFO REPORT <5901.175.0> 2011-11-30 18:47:42 =============================================================================== ns_1@127.0.0.1:<5901.175.0>:ns_config_rep:58: init pushing INFO REPORT <5901.175.0> 2011-11-30 18:47:42 =============================================================================== ns_1@127.0.0.1:<5901.175.0>:ns_config_rep:62: init reannouncing INFO REPORT <5901.158.0> 2011-11-30 18:47:42 =============================================================================== config change: auto_failover_cfg -> [{enabled,false},{timeout,30},{max_nodes,1},{count,0}] INFO REPORT <5901.158.0> 2011-11-30 18:47:42 =============================================================================== config change: buckets -> [{configs,[]}] INFO REPORT <5901.158.0> 2011-11-30 18:47:42 =============================================================================== config change: email_alerts -> [{recipients,["root@localhost"]}, {sender,"membase@localhost"}, {enabled,true}, {email_server,[{user,[]}, {pass,[]}, {host,"localhost"}, {port,25}, {encrypt,false}]}, {alerts,[auto_failover_node,auto_failover_maximum_reached, auto_failover_other_nodes_down,auto_failover_cluster_too_small]}] INFO REPORT <5901.158.0> 2011-11-30 18:47:42 =============================================================================== config change: memory_quota -> 16090 INFO REPORT <5901.158.0> 2011-11-30 18:47:42 =============================================================================== ns_node_disco_conf_events config on nodes_wanted INFO REPORT <5901.158.0> 2011-11-30 18:47:42 =============================================================================== config change: nodes_wanted -> ['ns_1@127.0.0.1'] INFO REPORT <5901.158.0> 2011-11-30 18:47:42 =============================================================================== ns_node_disco_conf_events config on otp INFO REPORT <5901.158.0> 2011-11-30 18:47:42 =============================================================================== config change: otp -> [{cookie,sgzcoaesvptwzitb}] INFO REPORT <5901.158.0> 2011-11-30 18:47:42 =============================================================================== config change: port_servers -> [{moxi,"/opt/membase/bin/moxi", ["-Z", {"port_listen=~B,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", [port]}, "-z", {"url=http://127.0.0.1:~B/pools/default/saslBucketsStreaming", [{rest,port}]}, "-p","0","-Y","y","-O","stderr", {"~s",[verbosity]}], [{env,[{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR",{"~s",[{ns_moxi_sup,rest_user,[]}]}}, {"MOXI_SASL_PLAIN_PWD",{"~s",[{ns_moxi_sup,rest_pass,[]}]}}]}, use_stdio,exit_status,port_server_send_eol,stderr_to_stdout,stream]}, {memcached,"/opt/membase/bin/memcached", ["-X","/opt/membase/lib/memcached/stdin_term_handler.so","-p", {"~B",[port]}, "-E","/opt/membase/lib/memcached/bucket_engine.so","-B","binary", "-r","-c","10000","-e", {"admin=~s;default_bucket_name=default;auto_create=false", [admin_user]}, {"~s",[verbosity]}], [{env,[{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE",{"~s",[{isasl,path}]}}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,exit_status,port_server_send_eol, stream]}] INFO REPORT <5901.158.0> 2011-11-30 18:47:42 =============================================================================== config change: replication -> [{enabled,true}] INFO REPORT <5901.158.0> 2011-11-30 18:47:42 =============================================================================== config change: rest -> [{port,8091}] INFO REPORT <5901.158.0> 2011-11-30 18:47:42 =============================================================================== config change: rest_creds -> ******** INFO REPORT <5901.158.0> 2011-11-30 18:47:42 =============================================================================== config change: {node,'ns_1@127.0.0.1',config_version} -> {1,7,2} INFO REPORT <5901.158.0> 2011-11-30 18:47:42 =============================================================================== config change: {node,'ns_1@127.0.0.1',isasl} -> [{path,"/opt/membase/var/lib/membase/data/isasl.pw"}] INFO REPORT <5901.158.0> 2011-11-30 18:47:42 =============================================================================== config change: {node,'ns_1@127.0.0.1',membership} -> active INFO REPORT <5901.158.0> 2011-11-30 18:47:42 =============================================================================== config change: {node,'ns_1@127.0.0.1',memcached} -> [{port,11210}, {dbdir,"/opt/membase/var/lib/membase/data"}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine,"/opt/membase/lib/memcached/bucket_engine.so"}, {engines, [{membase, [{engine,"/opt/membase/lib/memcached/ep.so"}, {initfile,"/opt/membase/etc/membase/init.sql"}, {static_config_string, "vb0=false;waitforwarmup=false;failpartialwarmup=false;shardpattern=%d/%b-%i.mb;db_strategy=multiMTVBDB"}]}, {memcached, [{engine,"/opt/membase/lib/memcached/default_engine.so"}, {static_config_string,"vb0=true"}]}]}, {verbosity,[]}] INFO REPORT <5901.158.0> 2011-11-30 18:47:42 =============================================================================== config change: {node,'ns_1@127.0.0.1',moxi} -> [{port,11211},{verbosity,[]}] INFO REPORT <5901.158.0> 2011-11-30 18:47:42 =============================================================================== config change: {node,'ns_1@127.0.0.1',ns_log} -> [{filename,"/opt/membase/var/lib/membase/data/ns_log"}] INFO REPORT <5901.158.0> 2011-11-30 18:47:42 =============================================================================== config change: {node,'ns_1@127.0.0.1',rest} -> [{port,8091},{port_meta,global}] INFO REPORT <5901.158.0> 2011-11-30 18:47:42 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <5901.177.0> 2011-11-30 18:47:42 =============================================================================== ns_node_disco cookie_sync INFO REPORT <5901.178.0> 2011-11-30 18:47:42 =============================================================================== ns_node_disco cookie_sync INFO REPORT <5901.177.0> 2011-11-30 18:47:42 =============================================================================== ns_node_disco: nodes_wanted updated: ['ns_1@127.0.0.1'], with cookie: sgzcoaesvptwzitb INFO REPORT <5901.178.0> 2011-11-30 18:47:42 =============================================================================== ns_node_disco: nodes_wanted updated: ['ns_1@127.0.0.1'], with cookie: sgzcoaesvptwzitb INFO REPORT <5901.177.0> 2011-11-30 18:47:42 =============================================================================== ns_node_disco: nodes_wanted pong: ['ns_1@127.0.0.1'], with cookie: sgzcoaesvptwzitb INFO REPORT <5901.178.0> 2011-11-30 18:47:42 =============================================================================== ns_node_disco: nodes_wanted pong: ['ns_1@127.0.0.1'], with cookie: sgzcoaesvptwzitb INFO REPORT <5901.175.0> 2011-11-30 18:47:42 =============================================================================== Pushing config PROGRESS REPORT <5901.168.0> 2011-11-30 18:47:42 =============================================================================== supervisor {local,ns_node_disco_sup} started [{pid,<5901.175.0>}, {name,ns_config_rep}, {mfargs,{ns_config_rep,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.165.0> 2011-11-30 18:47:42 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.168.0>}, {name,ns_node_disco_sup}, {mfargs,{ns_node_disco_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <5901.165.0> 2011-11-30 18:47:42 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.180.0>}, {name,ns_tick_event}, {mfargs,{gen_event,start_link,[{local,ns_tick_event}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.175.0> 2011-11-30 18:47:42 =============================================================================== Pushing config done INFO REPORT <5901.182.0> 2011-11-30 18:47:42 =============================================================================== ns_1@127.0.0.1:<5901.182.0>:mb_master:97: I'm the only node, so I'm the master. INFO REPORT <5901.183.0> 2011-11-30 18:47:42 =============================================================================== ns_1@127.0.0.1:<5901.183.0>:misc:785: start_singleton(gen_fsm, ns_orchestrator, [], []): started as <5901.184.0> on 'ns_1@127.0.0.1' PROGRESS REPORT <5901.183.0> 2011-11-30 18:47:42 =============================================================================== supervisor {local,mb_master_sup} started [{pid,<5901.184.0>}, {name,ns_orchestrator}, {mfargs,{ns_orchestrator,start_link,[]}}, {restart_type,permanent}, {shutdown,20}, {child_type,worker}] INFO REPORT <5901.183.0> 2011-11-30 18:47:42 =============================================================================== ns_1@127.0.0.1:<5901.183.0>:misc:785: start_singleton(gen_server, ns_tick, [], []): started as <5901.185.0> on 'ns_1@127.0.0.1' PROGRESS REPORT <5901.183.0> 2011-11-30 18:47:42 =============================================================================== supervisor {local,mb_master_sup} started [{pid,<5901.185.0>}, {name,ns_tick}, {mfargs,{ns_tick,start_link,[]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] INFO REPORT <5901.186.0> 2011-11-30 18:47:42 =============================================================================== ns_1@127.0.0.1:<5901.186.0>:auto_failover:120: init auto_failover. INFO REPORT <5901.183.0> 2011-11-30 18:47:42 =============================================================================== ns_1@127.0.0.1:<5901.183.0>:misc:785: start_singleton(gen_server, auto_failover, [], []): started as <5901.186.0> on 'ns_1@127.0.0.1' PROGRESS REPORT <5901.183.0> 2011-11-30 18:47:42 =============================================================================== supervisor {local,mb_master_sup} started [{pid,<5901.186.0>}, {name,auto_failover}, {mfargs,{auto_failover,start_link,[]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <5901.165.0> 2011-11-30 18:47:42 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.182.0>}, {name,mb_master}, {mfargs,{mb_master,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <5901.165.0> 2011-11-30 18:47:42 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.187.0>}, {name,buckets_events}, {mfargs,{gen_event,start_link,[{local,buckets_events}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.188.0> 2011-11-30 18:47:42 =============================================================================== supervisor {local,ns_mail_sup} started [{pid,<5901.189.0>}, {name,ns_mail}, {mfargs,{ns_mail,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.188.0> 2011-11-30 18:47:42 =============================================================================== supervisor {local,ns_mail_sup} started [{pid,<5901.190.0>}, {name,ns_mail_log}, {mfargs,{ns_mail_log,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.165.0> 2011-11-30 18:47:42 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.188.0>}, {name,ns_mail_sup}, {mfargs,{ns_mail_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] INFO REPORT <5901.167.0> 2011-11-30 18:47:42 =============================================================================== ns_1@127.0.0.1:<5901.167.0>:ns_mail_log:45: ns_mail_log started up PROGRESS REPORT <5901.165.0> 2011-11-30 18:47:42 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.191.0>}, {name,ns_heart}, {mfargs,{ns_heart,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.165.0> 2011-11-30 18:47:42 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.193.0>}, {name,ns_doctor}, {mfargs,{ns_doctor,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] ERROR REPORT <5901.191.0> 2011-11-30 18:47:42 =============================================================================== ns_1@127.0.0.1:<5901.191.0>:ns_heart:137: Failed to grab system stats: {'EXIT',{noproc,{gen_server,call, [{'stats_reader-@system','ns_1@127.0.0.1'}, {latest,"minute"}]}}} INFO REPORT <5901.193.0> 2011-11-30 18:47:43 =============================================================================== ns_1@127.0.0.1:<5901.193.0>:ns_doctor:82: Got initial status [{'ns_1@127.0.0.1', [{last_heard, {1322,678863, 129121}}, {active_buckets, []}, {ready_buckets, []}, {replication, []}, {memory, [{total, 22769744}, {processes, 4375840}, {processes_used, 4363192}, {system, 18393904}, {atom, 728921}, {atom_used, 699170}, {binary, 915152}, {code, 6734298}, {ets, 555384}]}, {system_stats, [{cpu_utilization_rate, 0}, {swap_total, 0}, {swap_used, 0}]}, {interesting_stats, []}, {cluster_compatibility_version, 1}, {version, [{os_mon, "2.2.5"}, {mnesia, "4.4.17"}, {kernel, "2.14.3"}, {sasl, "2.1.9.3"}, {ns_server, "1.7.2r-20-g6604356"}, {stdlib, "1.17.3"}]}, {system_arch, "x86_64-unknown-linux-gnu"}, {wall_clock,2}, {memory_data, {17946181632, 406937600, {<5901.7.0>, 426272}}}, {disk_data, [{"/", 8256952,15}, {"/dev", 8754940,1}, {"/run", 3505116,1}, {"/run/lock", 5120,0}, {"/run/shm", 8762784,0}, {"/mnt", 423135208, 1}]}, {meminfo, <<"MemTotal: 17525568 kB\nMemFree: 17117756 kB\nBuffers: 98232 kB\nCached: 50692 kB\nSwapCached: 0 kB\nActive: 138728 kB\nInactive: 41448 kB\nActive(anon): 31556 kB\nInactive(anon): 140 kB\nActive(file): 107172 kB\nInactive(file): 41308 kB\nUnevictable: 0 kB\nMlocked: 0 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 2264 kB\nWriteback: 0 kB\nAnonPages: 31720 kB\nMapped: 7304 kB\nShmem: 156 kB\nSlab: 17808 kB\nSReclaimable: 12260 kB\nSUnreclaim: 5548 kB\nKernelStack: 720 kB\nPageTables: 1412 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nWritebackTmp: 0 kB\nCommitLimit: 8762784 kB\nCommitted_AS: 105984 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 64000 kB\nVmallocChunk: 34359674252 kB\nHardwareCorrupted: 0 kB\nAnonHugePages: 0 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugePages_Surp: 0\nHugepagesize: 2048 kB\nDirectMap4k: 17920000 kB\nDirectMap2M: 0 kB\n">>}, {system_memory_data, [{system_total_memory, 17946181632}, {free_swap, 0}, {total_swap, 0}, {cached_memory, 51908608}, {buffered_memory, 100589568}, {free_memory, 17528582144}, {total_memory, 17946181632}]}, {statistics, [{wall_clock, {2044, 1304}}, {context_switches, {36645,0}}, {garbage_collection, {1225, 2650294, 0}}, {io, {{input, 6510238}, {output, 1536283}}}, {reductions, {1411799, 1411799}}, {run_queue, 0}, {runtime, {330, 330}}]}]}] PROGRESS REPORT <5901.195.0> 2011-11-30 18:47:43 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<5901.196.0>}, {name,menelaus_web}, {mfargs,{menelaus_web,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] PROGRESS REPORT <5901.195.0> 2011-11-30 18:47:43 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<5901.198.0>}, {name,menelaus_event}, {mfargs,{menelaus_event,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] PROGRESS REPORT <5901.195.0> 2011-11-30 18:47:43 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<5901.199.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] PROGRESS REPORT <5901.195.0> 2011-11-30 18:47:43 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<5901.200.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <5901.165.0> 2011-11-30 18:47:43 =============================================================================== ns_log: logging menelaus_sup:1:Membase Server has started on web port 8091 on node 'ns_1@127.0.0.1'. PROGRESS REPORT <5901.165.0> 2011-11-30 18:47:43 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.195.0>}, {name,menelaus}, {mfargs,{menelaus_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <5901.201.0> 2011-11-30 18:47:43 =============================================================================== supervisor {local,ns_port_sup} started [{pid,<5901.202.0>}, {name,ns_port_init}, {mfargs,{ns_port_init,start_link,[]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] INFO REPORT <5901.203.0> 2011-11-30 18:47:43 =============================================================================== starting ns_port_server with delay of 5000 PROGRESS REPORT <5901.201.0> 2011-11-30 18:47:43 =============================================================================== supervisor {local,ns_port_sup} started [{pid,<5901.203.0>}, {name, {moxi,"/opt/membase/bin/moxi", ["-Z", "port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", "-z", "url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming", "-p","0","-Y","y","-O","stderr",[]], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR",[]}, {"MOXI_SASL_PLAIN_PWD",[]}]}, use_stdio,exit_status,port_server_send_eol, stderr_to_stdout,stream]}}, {mfargs, {supervisor_cushion,start_link, [moxi,5000,ns_port_server,start_link, [moxi,"/opt/membase/bin/moxi", ["-Z", "port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", "-z", "url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming", "-p","0","-Y","y","-O","stderr",[]], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR",[]}, {"MOXI_SASL_PLAIN_PWD",[]}]}, use_stdio,exit_status,port_server_send_eol, stderr_to_stdout,stream]]]}}, {restart_type,permanent}, {shutdown,10000}, {child_type,worker}] INFO REPORT <5901.205.0> 2011-11-30 18:47:43 =============================================================================== starting ns_port_server with delay of 5000 PROGRESS REPORT <5901.201.0> 2011-11-30 18:47:43 =============================================================================== supervisor {local,ns_port_sup} started [{pid,<5901.205.0>}, {name, {memcached,"/opt/membase/bin/memcached", ["-X","/opt/membase/lib/memcached/stdin_term_handler.so", "-p","11210","-E", "/opt/membase/lib/memcached/bucket_engine.so","-B", "binary","-r","-c","10000","-e", "admin=_admin;default_bucket_name=default;auto_create=false", []], [{env, [{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE", "/opt/membase/var/lib/membase/data/isasl.pw"}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,exit_status, port_server_send_eol,stream]}}, {mfargs, {supervisor_cushion,start_link, [memcached,5000,ns_port_server,start_link, [memcached,"/opt/membase/bin/memcached", ["-X","/opt/membase/lib/memcached/stdin_term_handler.so", "-p","11210","-E", "/opt/membase/lib/memcached/bucket_engine.so","-B", "binary","-r","-c","10000","-e", "admin=_admin;default_bucket_name=default;auto_create=false", []], [{env, [{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE", "/opt/membase/var/lib/membase/data/isasl.pw"}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,exit_status, port_server_send_eol,stream]]]}}, {restart_type,permanent}, {shutdown,10000}, {child_type,worker}] PROGRESS REPORT <5901.165.0> 2011-11-30 18:47:43 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.201.0>}, {name,ns_port_sup}, {mfargs,{ns_port_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,60000}, {child_type,worker}] PROGRESS REPORT <5901.165.0> 2011-11-30 18:47:43 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.207.0>}, {name,ns_stats_event}, {mfargs,{gen_event,start_link,[{local,ns_stats_event}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.165.0> 2011-11-30 18:47:43 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.208.0>}, {name,ns_bucket_worker}, {mfargs,{work_queue,start_link,[ns_bucket_worker]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.165.0> 2011-11-30 18:47:43 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.209.0>}, {name,ns_bucket_sup}, {mfargs,{ns_bucket_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <5901.165.0> 2011-11-30 18:47:43 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.210.0>}, {name,system_stats_collector}, {mfargs,{system_stats_collector,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.165.0> 2011-11-30 18:47:43 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.211.0>}, {name,{stats_archiver,"@system"}}, {mfargs,{stats_archiver,start_link,["@system"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.165.0> 2011-11-30 18:47:43 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.212.0>}, {name,{stats_reader,"@system"}}, {mfargs,{stats_reader,start_link,["@system"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.165.0> 2011-11-30 18:47:43 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.213.0>}, {name,ns_moxi_sup_work_queue}, {mfargs,{work_queue,start_link,[ns_moxi_sup_work_queue]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.165.0> 2011-11-30 18:47:43 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.214.0>}, {name,ns_moxi_sup}, {mfargs,{ns_moxi_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <5901.54.0> 2011-11-30 18:47:43 =============================================================================== supervisor {local,ns_server_cluster_sup} started [{pid,<5901.165.0>}, {name,ns_server_sup}, {mfargs,{ns_server_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <5901.7.0> 2011-11-30 18:47:43 =============================================================================== application ns_server started_at 'ns_1@127.0.0.1' PROGRESS REPORT <5901.219.0> 2011-11-30 18:47:46 =============================================================================== supervisor {local,inet_gethost_native_sup} started [{pid,<5901.220.0>},{mfa,{inet_gethost_native,init,[[]]}}] PROGRESS REPORT <5901.26.0> 2011-11-30 18:47:46 =============================================================================== supervisor {local,kernel_safe_sup} started [{pid,<5901.219.0>}, {name,inet_gethost_native_sup}, {mfargs,{inet_gethost_native,start_link,[]}}, {restart_type,temporary}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.193.0> 2011-11-30 18:48:42 =============================================================================== ns_1@127.0.0.1:<5901.193.0>:ns_doctor:86: Current node statuses: [{'ns_1@127.0.0.1', [{last_heard,{1322,678922,892098}}, {active_buckets,[]}, {ready_buckets,[]}, {replication,[]}, {memory, [{total,24541184}, {processes,5455568}, {processes_used,5440128}, {system,19085616}, {atom,779745}, {atom_used,759280}, {binary,917248}, {code,7280616}, {ets,632392}]}, {system_stats,[{cpu_utilization_rate,0.0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.5"}, {mnesia,"4.4.17"}, {kernel,"2.14.3"}, {sasl,"2.1.9.3"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,62}, {memory_data,{17946181632,423759872,{<5901.7.0>,601336}}}, {disk_data, [{"/",8256952,15}, {"/dev",8754940,1}, {"/run",3505116,1}, {"/run/lock",5120,0}, {"/run/shm",8762784,0}, {"/mnt",423135208,1}]}, {meminfo, <<"MemTotal: 17525568 kB\nMemFree: 17111740 kB\nBuffers: 98252 kB\nCached: 53800 kB\nSwapCached: 0 kB\nActive: 141780 kB\nInactive: 43624 kB\nActive(anon): 33380 kB\nInactive(anon): 140 kB\nActive(file): 108400 kB\nInactive(file): 43484 kB\nUnevictable: 0 kB\nMlocked: 0 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 16 kB\nWriteback: 0 kB\nAnonPages: 33380 kB\nMapped: 7356 kB\nShmem: 156 kB\nSlab: 18380 kB\nSReclaimable: 12608 kB\nSUnreclaim: 5772 kB\nKernelStack: 856 kB\nPageTables: 1600 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nWritebackTmp: 0 kB\nCommitLimit: 8762784 kB\nCommitted_AS: 223528 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 64000 kB\nVmallocChunk: 34359674252 kB\nHardwareCorrupted: 0 kB\nAnonHugePages: 0 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugePages_Surp: 0\nHugepagesize: 2048 kB\nDirectMap4k: 17920000 kB\nDirectMap2M: 0 kB\n">>}, {system_memory_data, [{system_total_memory,17946181632}, {free_swap,0}, {total_swap,0}, {cached_memory,55091200}, {buffered_memory,100610048}, {free_memory,17522421760}, {total_memory,17946181632}]}, {statistics, [{wall_clock,{47044,0}}, {context_switches,{42069,0}}, {garbage_collection,{2185,4389034,0}}, {io,{{input,7005515},{output,1625393}}}, {reductions,{2097478,207510}}, {run_queue,0}, {runtime,{520,60}}]}]}] INFO REPORT <5901.193.0> 2011-11-30 18:49:42 =============================================================================== ns_1@127.0.0.1:<5901.193.0>:ns_doctor:86: Current node statuses: [{'ns_1@127.0.0.1', [{last_heard,{1322,678982,892533}}, {active_buckets,[]}, {ready_buckets,[]}, {replication,[]}, {memory, [{total,24579448}, {processes,5447456}, {processes_used,5432016}, {system,19131992}, {atom,779745}, {atom_used,759280}, {binary,934008}, {code,7280616}, {ets,660680}]}, {system_stats,[{cpu_utilization_rate,0.0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.5"}, {mnesia,"4.4.17"}, {kernel,"2.14.3"}, {sasl,"2.1.9.3"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,122}, {memory_data,{17946181632,423870464,{<5901.7.0>,601336}}}, {disk_data, [{"/",8256952,15}, {"/dev",8754940,1}, {"/run",3505116,1}, {"/run/lock",5120,0}, {"/run/shm",8762784,0}, {"/mnt",423135208,1}]}, {meminfo, <<"MemTotal: 17525568 kB\nMemFree: 17111632 kB\nBuffers: 98252 kB\nCached: 53876 kB\nSwapCached: 0 kB\nActive: 141672 kB\nInactive: 43620 kB\nActive(anon): 33188 kB\nInactive(anon): 140 kB\nActive(file): 108484 kB\nInactive(file): 43480 kB\nUnevictable: 0 kB\nMlocked: 0 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 12 kB\nWriteback: 0 kB\nAnonPages: 33176 kB\nMapped: 7324 kB\nShmem: 156 kB\nSlab: 18380 kB\nSReclaimable: 12608 kB\nSUnreclaim: 5772 kB\nKernelStack: 840 kB\nPageTables: 1536 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nWritebackTmp: 0 kB\nCommitLimit: 8762784 kB\nCommitted_AS: 223324 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 64000 kB\nVmallocChunk: 34359674252 kB\nHardwareCorrupted: 0 kB\nAnonHugePages: 0 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugePages_Surp: 0\nHugepagesize: 2048 kB\nDirectMap4k: 17920000 kB\nDirectMap2M: 0 kB\n">>}, {system_memory_data, [{system_total_memory,17946181632}, {free_swap,0}, {total_swap,0}, {cached_memory,55169024}, {buffered_memory,100610048}, {free_memory,17522311168}, {total_memory,17946181632}]}, {statistics, [{wall_clock,{107044,0}}, {context_switches,{48260,0}}, {garbage_collection,{3424,6540603,0}}, {io,{{input,7053299},{output,1714496}}}, {reductions,{2965962,215810}}, {run_queue,0}, {runtime,{690,50}}]}]}] INFO REPORT <5901.193.0> 2011-11-30 18:50:42 =============================================================================== ns_1@127.0.0.1:<5901.193.0>:ns_doctor:86: Current node statuses: [{'ns_1@127.0.0.1', [{last_heard,{1322,679042,892030}}, {active_buckets,[]}, {ready_buckets,[]}, {replication,[]}, {memory, [{total,24778776}, {processes,5621240}, {processes_used,5604200}, {system,19157536}, {atom,779745}, {atom_used,759280}, {binary,924608}, {code,7280616}, {ets,693504}]}, {system_stats, [{cpu_utilization_rate,0.5025125628140703}, {swap_total,0}, {swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.5"}, {mnesia,"4.4.17"}, {kernel,"2.14.3"}, {sasl,"2.1.9.3"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,182}, {memory_data,{17946181632,424759296,{<5901.7.0>,601336}}}, {disk_data, [{"/",8256952,15}, {"/dev",8754940,1}, {"/run",3505116,1}, {"/run/lock",5120,0}, {"/run/shm",8762784,0}, {"/mnt",423135208,1}]}, {meminfo, <<"MemTotal: 17525568 kB\nMemFree: 17111012 kB\nBuffers: 98252 kB\nCached: 53960 kB\nSwapCached: 0 kB\nActive: 142288 kB\nInactive: 43668 kB\nActive(anon): 33764 kB\nInactive(anon): 140 kB\nActive(file): 108524 kB\nInactive(file): 43528 kB\nUnevictable: 0 kB\nMlocked: 0 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 52 kB\nWriteback: 0 kB\nAnonPages: 33756 kB\nMapped: 7324 kB\nShmem: 156 kB\nSlab: 18380 kB\nSReclaimable: 12608 kB\nSUnreclaim: 5772 kB\nKernelStack: 840 kB\nPageTables: 1536 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nWritebackTmp: 0 kB\nCommitLimit: 8762784 kB\nCommitted_AS: 224760 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 64000 kB\nVmallocChunk: 34359674252 kB\nHardwareCorrupted: 0 kB\nAnonHugePages: 0 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugePages_Surp: 0\nHugepagesize: 2048 kB\nDirectMap4k: 17920000 kB\nDirectMap2M: 0 kB\n">>}, {system_memory_data, [{system_total_memory,17946181632}, {free_swap,0}, {total_swap,0}, {cached_memory,55255040}, {buffered_memory,100610048}, {free_memory,17521676288}, {total_memory,17946181632}]}, {statistics, [{wall_clock,{167044,0}}, {context_switches,{54437,0}}, {garbage_collection,{4676,8678822,0}}, {io,{{input,7101083},{output,1805257}}}, {reductions,{3844642,237831}}, {run_queue,0}, {runtime,{850,40}}]}]}] INFO REPORT <5901.193.0> 2011-11-30 18:51:42 =============================================================================== ns_1@127.0.0.1:<5901.193.0>:ns_doctor:86: Current node statuses: [{'ns_1@127.0.0.1', [{last_heard,{1322,679097,891988}}, {active_buckets,[]}, {ready_buckets,[]}, {replication,[]}, {memory, [{total,24899440}, {processes,5702264}, {processes_used,5685224}, {system,19197176}, {atom,779745}, {atom_used,759280}, {binary,938864}, {code,7280616}, {ets,717552}]}, {system_stats, [{cpu_utilization_rate,0.25},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.5"}, {mnesia,"4.4.17"}, {kernel,"2.14.3"}, {sasl,"2.1.9.3"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,237}, {memory_data,{17946181632,424759296,{<5901.7.0>,601336}}}, {disk_data, [{"/",8256952,15}, {"/dev",8754940,1}, {"/run",3505116,1}, {"/run/lock",5120,0}, {"/run/shm",8762784,0}, {"/mnt",423135208,1}]}, {meminfo, <<"MemTotal: 17525568 kB\nMemFree: 17110764 kB\nBuffers: 98256 kB\nCached: 54028 kB\nSwapCached: 0 kB\nActive: 142296 kB\nInactive: 43748 kB\nActive(anon): 33780 kB\nInactive(anon): 140 kB\nActive(file): 108516 kB\nInactive(file): 43608 kB\nUnevictable: 0 kB\nMlocked: 0 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 16 kB\nWriteback: 0 kB\nAnonPages: 33768 kB\nMapped: 7324 kB\nShmem: 156 kB\nSlab: 18384 kB\nSReclaimable: 12612 kB\nSUnreclaim: 5772 kB\nKernelStack: 840 kB\nPageTables: 1536 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nWritebackTmp: 0 kB\nCommitLimit: 8762784 kB\nCommitted_AS: 223324 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 64000 kB\nVmallocChunk: 34359674252 kB\nHardwareCorrupted: 0 kB\nAnonHugePages: 0 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugePages_Surp: 0\nHugepagesize: 2048 kB\nDirectMap4k: 17920000 kB\nDirectMap2M: 0 kB\n">>}, {system_memory_data, [{system_total_memory,17946181632}, {free_swap,0}, {total_swap,0}, {cached_memory,55324672}, {buffered_memory,100614144}, {free_memory,17521422336}, {total_memory,17946181632}]}, {statistics, [{wall_clock,{227044,0}}, {context_switches,{60916,0}}, {garbage_collection,{5827,10795275,0}}, {io,{{input,7390867},{output,2115716}}}, {reductions,{4678680,194607}}, {run_queue,0}, {runtime,{1020,30}}]}]}] INFO REPORT <5901.193.0> 2011-11-30 18:52:42 =============================================================================== ns_1@127.0.0.1:<5901.193.0>:ns_doctor:86: Current node statuses: [{'ns_1@127.0.0.1', [{last_heard,{1322,679157,891982}}, {active_buckets,[]}, {ready_buckets,[]}, {replication,[]}, {memory, [{total,24932032}, {processes,5675544}, {processes_used,5658504}, {system,19256488}, {atom,779745}, {atom_used,759280}, {binary,967792}, {code,7280616}, {ets,746512}]}, {system_stats,[{cpu_utilization_rate,0.0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.5"}, {mnesia,"4.4.17"}, {kernel,"2.14.3"}, {sasl,"2.1.9.3"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,297}, {memory_data,{17946181632,424759296,{<5901.7.0>,601336}}}, {disk_data, [{"/",8256952,15}, {"/dev",8754940,1}, {"/run",3505116,1}, {"/run/lock",5120,0}, {"/run/shm",8762784,0}, {"/mnt",423135208,1}]}, {meminfo, <<"MemTotal: 17525568 kB\nMemFree: 17110516 kB\nBuffers: 98256 kB\nCached: 54108 kB\nSwapCached: 0 kB\nActive: 142776 kB\nInactive: 43780 kB\nActive(anon): 34220 kB\nInactive(anon): 140 kB\nActive(file): 108556 kB\nInactive(file): 43640 kB\nUnevictable: 0 kB\nMlocked: 0 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 4 kB\nWriteback: 0 kB\nAnonPages: 34204 kB\nMapped: 7324 kB\nShmem: 156 kB\nSlab: 18380 kB\nSReclaimable: 12608 kB\nSUnreclaim: 5772 kB\nKernelStack: 840 kB\nPageTables: 1536 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nWritebackTmp: 0 kB\nCommitLimit: 8762784 kB\nCommitted_AS: 223324 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 64000 kB\nVmallocChunk: 34359674252 kB\nHardwareCorrupted: 0 kB\nAnonHugePages: 0 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugePages_Surp: 0\nHugepagesize: 2048 kB\nDirectMap4k: 17920000 kB\nDirectMap2M: 0 kB\n">>}, {system_memory_data, [{system_total_memory,17946181632}, {free_swap,0}, {total_swap,0}, {cached_memory,55406592}, {buffered_memory,100614144}, {free_memory,17521168384}, {total_memory,17946181632}]}, {statistics, [{wall_clock,{287044,0}}, {context_switches,{67076,0}}, {garbage_collection,{7012,12916723,0}}, {io,{{input,7438660},{output,2207898}}}, {reductions,{5541387,206487}}, {run_queue,0}, {runtime,{1160,40}}]}]}] INFO REPORT <5901.64.0> 2011-11-30 18:53:28 =============================================================================== rb: ERROR! A report on bad form was encountered. It can not be printed to the log. Details: "2011-11-30 18:53:28" {{{2011,11,30},{18,53,28}}, {info_msg,<5901.50.0>, {<5901.64.0>, "~s:~p:~s:~B: handling engage_cluster(..)~n~n", ['ns_1@127.0.0.1',<5901.64.0>,ns_cluster,96, [{<<"availableStorage">>, {struct, [{<<"hdd">>, [{struct, [{<<"path">>,<<"/">>}, {<<"sizeKBytes">>,8256952}, {<<"usagePercent">>,15}]}, {struct, [{<<"path">>,<<"/dev">>}, {<<"sizeKBytes">>,8754940}, {<<"usagePercent">>,1}]}, {struct, [{<<"path">>,<<"/run">>}, {<<"sizeKBytes">>,3505116}, {<<"usagePercent">>,1}]}, {struct, [{<<"path">>,<<"/run/lock">>}, {<<"sizeKBytes">>,5120}, {<<"usagePercent">>,0}]}, {struct, [{<<"path">>,<<"/run/shm">>}, {<<"sizeKBytes">>,8762784}, {<<"usagePercent">>,0}]}, {struct, [{<<"path">>,<<"/mnt">>}, {<<"sizeKBytes">>,423135208}, {<<"usagePercent">>,1}]}]}]}}, {<<"memoryQuota">>,15000}, {<<"storageTotals">>, {struct, [{<<"ram">>, {struct, [{<<"usedByData">>,0}, {<<"total">>,17946181632.0}, {<<"quotaTotal">>,1.572864e10}, {<<"used">>,508162048}]}}, {<<"hdd">>, {struct, [{<<"usedByData">>,1808192}, {<<"total">>,8455118848.0}, {<<"quotaTotal">>,8455118848.0}, {<<"used">>,1268267827}, {<<"free">>,7186851021.0}]}}]}}, {<<"storage">>, {struct, [{<<"ssd">>,[]}, {<<"hdd">>, [{struct, [{<<"path">>, <<"/opt/membase/var/lib/membase/data">>}, {<<"quotaMb">>,<<"none">>}, {<<"state">>,<<"ok">>}]}]}]}}, {<<"systemStats">>, {struct, [{<<"cpu_utilization_rate">>,0}, {<<"swap_total">>,0}, {<<"swap_used">>,0}]}}, {<<"interestingStats">>,{struct,[]}}, {<<"uptime">>,<<"337">>}, {<<"memoryTotal">>,17946181632.0}, {<<"memoryFree">>,17438019584.0}, {<<"mcdMemoryReserved">>,13691}, {<<"mcdMemoryAllocated">>,13691}, {<<"otpNode">>,<<"ns_1@10.13.41.69">>}, {<<"otpCookie">>,<<"sgzcoaesvptwzitb">>}, {<<"clusterMembership">>,<<"active">>}, {<<"status">>,<<"healthy">>}, {<<"hostname">>,<<"10.13.41.69:8091">>}, {<<"clusterCompatibility">>,1}, {<<"version">>,<<"1.7.2r-20-g6604356">>}, {<<"os">>,<<"x86_64-unknown-linux-gnu">>}, {<<"ports">>, {struct, [{<<"proxy">>,11211},{<<"direct">>,11210}]}}]]}}} rb: Logging resumed... INFO REPORT <5901.64.0> 2011-11-30 18:53:31 =============================================================================== ns_1@127.0.0.1:<5901.64.0>:ns_cluster:253: Decided to change address to "10.13.41.47" INFO REPORT <0.170.0> 2011-11-30 18:53:31 =============================================================================== ns_log: logging ns_node_disco:5:Node nonode@nohost saw that node 'ns_1@127.0.0.1' went down. INFO REPORT <0.57.0> 2011-11-30 18:53:31 =============================================================================== Adjusted IP to "10.13.41.47" INFO REPORT <0.57.0> 2011-11-30 18:53:31 =============================================================================== nonode@nohost:<0.57.0>:dist_manager:105: Attempting to bring up net_kernel with name 'ns_1@10.13.41.47' PROGRESS REPORT <0.1007.0> 2011-11-30 18:53:31 =============================================================================== supervisor {local,net_sup} started [{pid,<0.1008.0>}, {name,erl_epmd}, {mfargs,{erl_epmd,start_link,[]}}, {restart_type,permanent}, {shutdown,2000}, {child_type,worker}] PROGRESS REPORT <0.1007.0> 2011-11-30 18:53:31 =============================================================================== supervisor {local,net_sup} started [{pid,<0.1009.0>}, {name,auth}, {mfargs,{auth,start_link,[]}}, {restart_type,permanent}, {shutdown,2000}, {child_type,worker}] INFO REPORT <0.169.0> 2011-11-30 18:53:31 =============================================================================== ns_node_disco_log: nodes changed: [] PROGRESS REPORT <6058.1007.0> 2011-11-30 18:53:31 =============================================================================== supervisor {local,net_sup} started [{pid,<6058.1010.0>}, {name,net_kernel}, {mfargs,{net_kernel,start_link,[['ns_1@10.13.41.47',longnames]]}}, {restart_type,permanent}, {shutdown,2000}, {child_type,worker}] PROGRESS REPORT <6058.11.0> 2011-11-30 18:53:31 =============================================================================== supervisor {local,kernel_sup} started [{pid,<6058.1007.0>}, {name,net_sup_dynamic}, {mfargs,{erl_distribution,start_link, [['ns_1@10.13.41.47',longnames]]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,supervisor}] INFO REPORT <6058.170.0> 2011-11-30 18:53:31 =============================================================================== ns_log: logging ns_node_disco:4:Node 'ns_1@10.13.41.47' saw that node 'ns_1@10.13.41.47' came up. INFO REPORT <6058.57.0> 2011-11-30 18:53:31 =============================================================================== Re-setting cookie {sgzcoaesvptwzitb,'ns_1@10.13.41.47'} INFO REPORT <6058.57.0> 2011-11-30 18:53:31 =============================================================================== saving ip config to "/opt/membase/var/lib/membase/ip" INFO REPORT <6058.57.0> 2011-11-30 18:53:31 =============================================================================== save_address_config: ok INFO REPORT <6058.67.0> 2011-11-30 18:53:31 =============================================================================== ns_1@10.13.41.47:<6058.67.0>:mb_mnesia:132: Renaming node from 'ns_1@127.0.0.1' to 'ns_1@10.13.41.47'. INFO REPORT <6058.7.0> 2011-11-30 18:53:31 =============================================================================== application mnesia exited stopped type temporary ERROR REPORT <6058.211.0> 2011-11-30 18:53:31 =============================================================================== ** Generic server 'stats_archiver-@system' terminating ** Last message in was {truncate,minute,60} ** When Server state == {state,"@system"} ** Reason for termination == ** {{badmatch,{aborted,{node_not_running,'ns_1@10.13.41.47'}}}, [{mb_mnesia,truncate,2}, {stats_archiver,do_handle_info,2}, {stats_archiver,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <6058.211.0> 2011-11-30 18:53:31 =============================================================================== Crashing process initial_call {stats_archiver,init,['Argument__1']} pid <6058.211.0> registered_name 'stats_archiver-@system' error_info {exit,{{badmatch,{aborted,{node_not_running,'ns_1@10.13.41.47'}}}, [{mb_mnesia,truncate,2}, {stats_archiver,do_handle_info,2}, {stats_archiver,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors [ns_server_sup,ns_server_cluster_sup,<6058.51.0>] messages [] links [<6058.165.0>,<6058.207.0>,<6058.53.0>] dictionary [] trap_exit false status running heap_size 1597 stack_size 24 reductions 1706048 SUPERVISOR REPORT <6058.165.0> 2011-11-30 18:53:31 =============================================================================== Reporting supervisor {local,ns_server_sup} Child process errorContext child_terminated reason {{badmatch,{aborted,{node_not_running,'ns_1@10.13.41.47'}}}, [{mb_mnesia,truncate,2}, {stats_archiver,do_handle_info,2}, {stats_archiver,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <6058.211.0> name {stats_archiver,"@system"} mfargs {stats_archiver,start_link,["@system"]} restart_type permanent shutdown 1000 child_type worker PROGRESS REPORT <6058.165.0> 2011-11-30 18:53:31 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<6058.1018.0>}, {name,{stats_archiver,"@system"}}, {mfargs,{stats_archiver,start_link,["@system"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <6058.67.0> 2011-11-30 18:53:31 =============================================================================== ns_1@10.13.41.47:<6058.67.0>:mb_mnesia:232: Found backup. Restoring Mnesia database. PROGRESS REPORT <6058.1031.0> 2011-11-30 18:53:31 =============================================================================== supervisor {local,mnesia_sup} started [{pid,<6058.1032.0>}, {name,mnesia_event}, {mfargs,{mnesia_sup,start_event,[]}}, {restart_type,permanent}, {shutdown,30000}, {child_type,worker}] PROGRESS REPORT <6058.1033.0> 2011-11-30 18:53:31 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<6058.1034.0>}, {name,mnesia_monitor}, {mfargs,{mnesia_monitor,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] PROGRESS REPORT <6058.1033.0> 2011-11-30 18:53:31 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<6058.1035.0>}, {name,mnesia_subscr}, {mfargs,{mnesia_subscr,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] PROGRESS REPORT <6058.1033.0> 2011-11-30 18:53:31 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<6058.1036.0>}, {name,mnesia_locker}, {mfargs,{mnesia_locker,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] PROGRESS REPORT <6058.1033.0> 2011-11-30 18:53:31 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<6058.1037.0>}, {name,mnesia_recover}, {mfargs,{mnesia_recover,start,[]}}, {restart_type,permanent}, {shutdown,180000}, {child_type,worker}] PROGRESS REPORT <6058.1033.0> 2011-11-30 18:53:31 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<6058.1038.0>}, {name,mnesia_tm}, {mfargs,{mnesia_tm,start,[]}}, {restart_type,permanent}, {shutdown,30000}, {child_type,worker}] PROGRESS REPORT <6058.1033.0> 2011-11-30 18:53:31 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<6058.1081.0>}, {name,mnesia_checkpoint_sup}, {mfargs,{mnesia_checkpoint_sup,start,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <6058.1033.0> 2011-11-30 18:53:31 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<6058.1082.0>}, {name,mnesia_snmp_sup}, {mfargs,{mnesia_snmp_sup,start,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <6058.1033.0> 2011-11-30 18:53:31 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<6058.1083.0>}, {name,mnesia_controller}, {mfargs,{mnesia_controller,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] PROGRESS REPORT <6058.1033.0> 2011-11-30 18:53:31 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<6058.1084.0>}, {name,mnesia_late_loader}, {mfargs,{mnesia_late_loader,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] PROGRESS REPORT <6058.1031.0> 2011-11-30 18:53:31 =============================================================================== supervisor {local,mnesia_sup} started [{pid,<6058.1033.0>}, {name,mnesia_kernel_sup}, {mfargs,{mnesia_kernel_sup,start,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <6058.7.0> 2011-11-30 18:53:31 =============================================================================== application mnesia started_at 'ns_1@10.13.41.47' INFO REPORT <6058.67.0> 2011-11-30 18:53:31 =============================================================================== ns_1@10.13.41.47:<6058.67.0>:mb_mnesia:245: Current config: [{access_module,mnesia}, {auto_repair,true}, {backup_module,mnesia_backup}, {checkpoints,[]}, {db_nodes,['ns_1@10.13.41.47']}, {debug,verbose}, {directory,"/opt/membase/var/lib/membase/mnesia"}, {dump_log_load_regulation,false}, {dump_log_time_threshold,180000}, {dump_log_update_in_place,true}, {dump_log_write_threshold,1000}, {embedded_mnemosyne,false}, {event_module,mnesia_event}, {extra_db_nodes,[]}, {fallback_activated,false}, {held_locks,[]}, {ignore_fallback_at_startup,false}, {fallback_error_function,{mnesia,lkill}}, {is_running,yes}, {local_tables,['stats_archiver-@system-week','stats_archiver-@system-day', 'stats_archiver-@system-month',local_config, 'stats_archiver-@system-minute',cluster, 'stats_archiver-@system-year','stats_archiver-@system-hour', schema]}, {lock_queue,[]}, {log_version,"4.3"}, {master_node_tables,[]}, {max_wait_for_decision,10000}, {protocol_version,{7,6}}, {running_db_nodes,['ns_1@10.13.41.47']}, {schema_location,opt_disc}, {schema_version,{3,0}}, {subscribers,[<6058.1032.0>,<6058.67.0>]}, {tables,['stats_archiver-@system-week','stats_archiver-@system-day', 'stats_archiver-@system-month',local_config, 'stats_archiver-@system-minute',cluster, 'stats_archiver-@system-year','stats_archiver-@system-hour',schema]}, {transaction_commits,2}, {transaction_failures,0}, {transaction_log_writes,0}, {transaction_restarts,0}, {transactions,[]}, {use_dir,true}, {core_dir,false}, {no_table_loaders,2}, {dc_dump_limit,4}, {send_compressed,0}, {version,"4.4.17"}] Peers: ['ns_1@10.13.41.47'] INFO REPORT <6058.67.0> 2011-11-30 18:53:31 =============================================================================== ns_1@10.13.41.47:<6058.67.0>:mb_mnesia:166: Info from Mnesia: {mnesia_checkpoint,{{1322,679211,531226},'ns_1@127.0.0.1'}} starting: <6058.1001.0> INFO REPORT <6058.67.0> 2011-11-30 18:53:31 =============================================================================== ns_1@10.13.41.47:<6058.67.0>:mb_mnesia:166: Info from Mnesia: {mnesia_checkpoint,{{1322,679211,531226},'ns_1@127.0.0.1'}} terminated: shutdown INFO REPORT <6058.67.0> 2011-11-30 18:53:31 =============================================================================== ns_1@10.13.41.47:<6058.67.0>:mb_mnesia:166: Info from Mnesia: mnesia_controller terminated: shutdown INFO REPORT <6058.67.0> 2011-11-30 18:53:31 =============================================================================== ns_1@10.13.41.47:<6058.67.0>:mb_mnesia:166: Info from Mnesia: mnesia_tm terminated: shutdown INFO REPORT <6058.67.0> 2011-11-30 18:53:31 =============================================================================== ns_1@10.13.41.47:<6058.67.0>:mb_mnesia:166: Info from Mnesia: mnesia_recover terminated: shutdown INFO REPORT <6058.67.0> 2011-11-30 18:53:31 =============================================================================== ns_1@10.13.41.47:<6058.67.0>:mb_mnesia:166: Info from Mnesia: mnesia_locker terminated: shutdown INFO REPORT <6058.67.0> 2011-11-30 18:53:31 =============================================================================== ns_1@10.13.41.47:<6058.67.0>:mb_mnesia:169: Saw Mnesia go down on 'ns_1@10.13.41.47' INFO REPORT <6058.159.0> 2011-11-30 18:53:31 =============================================================================== renaming node conf nodes_wanted -> nodes_wanted: ['ns_1@127.0.0.1'] -> ['ns_1@10.13.41.47'] ERROR REPORT <6058.1018.0> 2011-11-30 18:53:31 =============================================================================== ** Generic server 'stats_archiver-@system' terminating ** Last message in was {stats,"@system", {stat_entry,1322679211885, [{cpu_idle_ms,4020}, {cpu_local_ms,4050}, {cpu_utilization_rate,0.7407407407407407}, {mem_actual_free,17676730368}, {mem_actual_used,269451264}, {mem_free,17520898048}, {mem_total,17946181632}, {mem_used,425283584}, {swap_total,0}, {swap_used,0}]}} ** When Server state == {state,"@system"} ** Reason for termination == ** {{badmatch,{aborted,{no_exists,'stats_archiver-@system-minute'}}}, [{stats_archiver,do_handle_info,2}, {stats_archiver,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <6058.1018.0> 2011-11-30 18:53:31 =============================================================================== Crashing process initial_call {stats_archiver,init,['Argument__1']} pid <6058.1018.0> registered_name 'stats_archiver-@system' error_info {exit, {{badmatch,{aborted,{no_exists,'stats_archiver-@system-minute'}}}, [{stats_archiver,do_handle_info,2}, {stats_archiver,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors [ns_server_sup,ns_server_cluster_sup,<6058.51.0>] messages [] links [<6058.165.0>,<6058.207.0>,<6058.53.0>] dictionary [] trap_exit false status running heap_size 610 stack_size 24 reductions 1335 SUPERVISOR REPORT <6058.165.0> 2011-11-30 18:53:31 =============================================================================== Reporting supervisor {local,ns_server_sup} Child process errorContext child_terminated reason {{badmatch,{aborted,{no_exists,'stats_archiver-@system-minute'}}}, [{stats_archiver,do_handle_info,2}, {stats_archiver,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <6058.1018.0> name {stats_archiver,"@system"} mfargs {stats_archiver,start_link,["@system"]} restart_type permanent shutdown 1000 child_type worker PROGRESS REPORT <6058.165.0> 2011-11-30 18:53:31 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<6058.1097.0>}, {name,{stats_archiver,"@system"}}, {mfargs,{stats_archiver,start_link,["@system"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <6058.159.0> 2011-11-30 18:53:31 =============================================================================== renaming node conf {node,'ns_1@127.0.0.1',config_version} -> {node, 'ns_1@10.13.41.47', config_version}: {1,7,2} -> {1,7,2} INFO REPORT <6058.159.0> 2011-11-30 18:53:31 =============================================================================== renaming node conf {node,'ns_1@127.0.0.1',isasl} -> {node,'ns_1@10.13.41.47', isasl}: [{path,"/opt/membase/var/lib/membase/data/isasl.pw"}] -> [{path,"/opt/membase/var/lib/membase/data/isasl.pw"}] INFO REPORT <6058.159.0> 2011-11-30 18:53:31 =============================================================================== renaming node conf {node,'ns_1@127.0.0.1',membership} -> {node, 'ns_1@10.13.41.47', membership}: active -> active INFO REPORT <6058.159.0> 2011-11-30 18:53:31 =============================================================================== renaming node conf {node,'ns_1@127.0.0.1',memcached} -> {node, 'ns_1@10.13.41.47', memcached}: [{port,11210}, {dbdir,"/opt/membase/var/lib/membase/data"}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine,"/opt/membase/lib/memcached/bucket_engine.so"}, {engines, [{membase, [{engine,"/opt/membase/lib/memcached/ep.so"}, {initfile,"/opt/membase/etc/membase/init.sql"}, {static_config_string, "vb0=false;waitforwarmup=false;failpartialwarmup=false;shardpattern=%d/%b-%i.mb;db_strategy=multiMTVBDB"}]}, {memcached, [{engine,"/opt/membase/lib/memcached/default_engine.so"}, {static_config_string,"vb0=true"}]}]}, {verbosity,[]}] -> [{port,11210}, {dbdir,"/opt/membase/var/lib/membase/data"}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine,"/opt/membase/lib/memcached/bucket_engine.so"}, {engines, [{membase, [{engine,"/opt/membase/lib/memcached/ep.so"}, {initfile,"/opt/membase/etc/membase/init.sql"}, {static_config_string, "vb0=false;waitforwarmup=false;failpartialwarmup=false;shardpattern=%d/%b-%i.mb;db_strategy=multiMTVBDB"}]}, {memcached, [{engine,"/opt/membase/lib/memcached/default_engine.so"}, {static_config_string,"vb0=true"}]}]}, {verbosity,[]}] INFO REPORT <6058.159.0> 2011-11-30 18:53:31 =============================================================================== renaming node conf {node,'ns_1@127.0.0.1',moxi} -> {node,'ns_1@10.13.41.47', moxi}: [{port,11211},{verbosity,[]}] -> [{port,11211},{verbosity,[]}] INFO REPORT <6058.159.0> 2011-11-30 18:53:31 =============================================================================== renaming node conf {node,'ns_1@127.0.0.1',ns_log} -> {node, 'ns_1@10.13.41.47', ns_log}: [{filename,"/opt/membase/var/lib/membase/data/ns_log"}] -> [{filename,"/opt/membase/var/lib/membase/data/ns_log"}] INFO REPORT <6058.159.0> 2011-11-30 18:53:31 =============================================================================== renaming node conf {node,'ns_1@127.0.0.1',rest} -> {node,'ns_1@10.13.41.47', rest}: [{port,8091},{port_meta,global}] -> [{port,8091},{port_meta,global}] PROGRESS REPORT <6058.165.0> 2011-11-30 18:53:31 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<6058.1104.0>}, {name,ns_doctor}, {mfargs,{ns_doctor,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <6058.64.0> 2011-11-30 18:53:31 =============================================================================== ns_1@10.13.41.47:<6058.64.0>:ns_cluster:260: Renamed node. New name is 'ns_1@10.13.41.47'. INFO REPORT <6058.64.0> 2011-11-30 18:53:31 =============================================================================== ns_1@10.13.41.47:<6058.64.0>:ns_cluster:98: engage_cluster(..) -> {ok,ok} ERROR REPORT <6058.191.0> 2011-11-30 18:53:31 =============================================================================== ns_1@10.13.41.47:<6058.191.0>:ns_heart:137: Failed to grab system stats: {error,{exit,{aborted,{no_exists,['stats_archiver-@system-minute']}}}} INFO REPORT <6058.1104.0> 2011-11-30 18:53:31 =============================================================================== ns_1@10.13.41.47:<6058.1104.0>:ns_doctor:82: Got initial status [{'ns_1@10.13.41.47', [{last_heard, {1322, 679211, 941740}}, {active_buckets, []}, {ready_buckets, []}, {replication, []}, {memory, [{total, 25302840}, {processes, 6367400}, {processes_used, 6355336}, {system, 18935440}, {atom, 781361}, {atom_used, 761554}, {binary, 719016}, {code, 7320008}, {ets, 615808}]}, {system_stats, [{cpu_utilization_rate, 0}, {swap_total, 0}, {swap_used, 0}]}, {interesting_stats, []}, {cluster_compatibility_version, 1}, {version, [{os_mon, "2.2.5"}, {mnesia, "4.4.17"}, {kernel, "2.14.3"}, {sasl, "2.1.9.3"}, {ns_server, "1.7.2r-20-g6604356"}, {stdlib, "1.17.3"}]}, {system_arch, "x86_64-unknown-linux-gnu"}, {wall_clock, 351}, {memory_data, {17946181632, 425013248, {<6058.7.0>, 601336}}}, {disk_data, [{"/", 8256952, 15}, {"/dev", 8754940, 1}, {"/run", 3505116, 1}, {"/run/lock", 5120,0}, {"/run/shm", 8762784, 0}, {"/mnt", 423135208, 1}]}, {meminfo, <<"MemTotal: 17525568 kB\nMemFree: 17110392 kB\nBuffers: 98256 kB\nCached: 54192 kB\nSwapCached: 0 kB\nActive: 142840 kB\nInactive: 43832 kB\nActive(anon): 34252 kB\nInactive(anon): 140 kB\nActive(file): 108588 kB\nInactive(file): 43692 kB\nUnevictable: 0 kB\nMlocked: 0 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 48 kB\nWriteback: 0 kB\nAnonPages: 34240 kB\nMapped: 7324 kB\nShmem: 156 kB\nSlab: 18380 kB\nSReclaimable: 12608 kB\nSUnreclaim: 5772 kB\nKernelStack: 840 kB\nPageTables: 1536 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nWritebackTmp: 0 kB\nCommitLimit: 8762784 kB\nCommitted_AS: 223324 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 64000 kB\nVmallocChunk: 34359674252 kB\nHardwareCorrupted: 0 kB\nAnonHugePages: 0 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugePages_Surp: 0\nHugepagesize: 2048 kB\nDirectMap4k: 17920000 kB\nDirectMap2M: 0 kB\n">>}, {system_memory_data, [{system_total_memory, 17946181632}, {free_swap, 0}, {total_swap, 0}, {cached_memory, 55492608}, {buffered_memory, 100614144}, {free_memory, 17521041408}, {total_memory, 17946181632}]}, {statistics, [{wall_clock, {347044, 0}}, {context_switches, {73209, 0}}, {garbage_collection, {8159, 15016649, 0}}, {io, {{input, 7486444}, {output, 2297004}}}, {reductions, {6381342, 241815}}, {run_queue, 0}, {runtime, {1290, 30}}]}]}] INFO REPORT <6058.158.0> 2011-11-30 18:53:31 =============================================================================== config change: {node,'ns_1@10.13.41.47',rest} -> [{port,8091},{port_meta,global}] INFO REPORT <6058.158.0> 2011-11-30 18:53:31 =============================================================================== config change: {node,'ns_1@10.13.41.47',ns_log} -> [{filename,"/opt/membase/var/lib/membase/data/ns_log"}] INFO REPORT <6058.158.0> 2011-11-30 18:53:31 =============================================================================== config change: {node,'ns_1@10.13.41.47',moxi} -> [{port,11211},{verbosity,[]}] INFO REPORT <6058.158.0> 2011-11-30 18:53:31 =============================================================================== config change: {node,'ns_1@10.13.41.47',memcached} -> [{port,11210}, {dbdir,"/opt/membase/var/lib/membase/data"}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine,"/opt/membase/lib/memcached/bucket_engine.so"}, {engines, [{membase, [{engine,"/opt/membase/lib/memcached/ep.so"}, {initfile,"/opt/membase/etc/membase/init.sql"}, {static_config_string, "vb0=false;waitforwarmup=false;failpartialwarmup=false;shardpattern=%d/%b-%i.mb;db_strategy=multiMTVBDB"}]}, {memcached, [{engine,"/opt/membase/lib/memcached/default_engine.so"}, {static_config_string,"vb0=true"}]}]}, {verbosity,[]}] INFO REPORT <6058.158.0> 2011-11-30 18:53:31 =============================================================================== config change: {node,'ns_1@10.13.41.47',membership} -> active INFO REPORT <6058.158.0> 2011-11-30 18:53:31 =============================================================================== config change: {node,'ns_1@10.13.41.47',isasl} -> [{path,"/opt/membase/var/lib/membase/data/isasl.pw"}] INFO REPORT <6058.158.0> 2011-11-30 18:53:31 =============================================================================== config change: {node,'ns_1@10.13.41.47',config_version} -> {1,7,2} INFO REPORT <6058.158.0> 2011-11-30 18:53:31 =============================================================================== ns_node_disco_conf_events config on nodes_wanted INFO REPORT <6058.158.0> 2011-11-30 18:53:31 =============================================================================== config change: nodes_wanted -> ['ns_1@10.13.41.47'] INFO REPORT <6058.182.0> 2011-11-30 18:53:31 =============================================================================== ns_1@10.13.41.47:<6058.182.0>:mb_master:374: List of peers has changed from ['ns_1@127.0.0.1'] to ['ns_1@10.13.41.47'] INFO REPORT <6058.182.0> 2011-11-30 18:53:31 =============================================================================== ns_1@10.13.41.47:<6058.182.0>:mb_master:378: List of new peers has changed from ['ns_1@127.0.0.1'] to [] INFO REPORT <6058.1125.0> 2011-11-30 18:53:31 =============================================================================== ns_node_disco cookie_sync INFO REPORT <6058.1125.0> 2011-11-30 18:53:31 =============================================================================== ns_node_disco: nodes_wanted updated: ['ns_1@10.13.41.47'], with cookie: sgzcoaesvptwzitb INFO REPORT <6058.1125.0> 2011-11-30 18:53:31 =============================================================================== ns_node_disco: nodes_wanted pong: ['ns_1@10.13.41.47'], with cookie: sgzcoaesvptwzitb INFO REPORT <6058.158.0> 2011-11-30 18:53:31 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <6058.175.0> 2011-11-30 18:53:31 =============================================================================== Pushing config INFO REPORT <6058.175.0> 2011-11-30 18:53:31 =============================================================================== Pushing config done INFO REPORT <6058.170.0> 2011-11-30 18:53:31 =============================================================================== ns_log: logging ns_node_disco:4:Node 'ns_1@10.13.41.47' saw that node 'ns_1@10.13.41.69' came up. INFO REPORT <6058.169.0> 2011-11-30 18:53:31 =============================================================================== Detected a new nodes (['ns_1@10.13.41.47']). Moving config around. INFO REPORT <6058.169.0> 2011-11-30 18:53:31 =============================================================================== ns_node_disco_log: nodes changed: ['ns_1@10.13.41.47'] INFO REPORT <6058.64.0> 2011-11-30 18:53:31 =============================================================================== ns_1@10.13.41.47:<6058.64.0>:ns_cluster:102: handling complete_join([{<<"targetNode">>, <<"ns_1@10.13.41.47">>}, {<<"availableStorage">>, {struct, [{<<"hdd">>, [{struct, [{<<"path">>, <<"/">>}, {<<"sizeKBytes">>, 8256952}, {<<"usagePercent">>, 15}]}, {struct, [{<<"path">>, <<"/dev">>}, {<<"sizeKBytes">>, 8754940}, {<<"usagePercent">>, 1}]}, {struct, [{<<"path">>, <<"/run">>}, {<<"sizeKBytes">>, 3505116}, {<<"usagePercent">>, 1}]}, {struct, [{<<"path">>, <<"/run/lock">>}, {<<"sizeKBytes">>, 5120}, {<<"usagePercent">>, 0}]}, {struct, [{<<"path">>, <<"/run/shm">>}, {<<"sizeKBytes">>, 8762784}, {<<"usagePercent">>, 0}]}, {struct, [{<<"path">>, <<"/mnt">>}, {<<"sizeKBytes">>, 423135208}, {<<"usagePercent">>, 1}]}]}]}}, {<<"memoryQuota">>, 15000}, {<<"storageTotals">>, {struct, [{<<"ram">>, {struct, [{<<"usedByData">>, 26968704}, {<<"total">>, 17946181632.0}, {<<"quotaTotal">>, 1.572864e10}, {<<"used">>, 508162048}]}}, {<<"hdd">>, {struct, [{<<"usedByData">>, 1808192}, {<<"total">>, 8455118848.0}, {<<"quotaTotal">>, 8455118848.0}, {<<"used">>, 1268267827}, {<<"free">>, 7186851021.0}]}}]}}, {<<"storage">>, {struct, [{<<"ssd">>, []}, {<<"hdd">>, [{struct, [{<<"path">>, <<"/opt/membase/var/lib/membase/data">>}, {<<"quotaMb">>, <<"none">>}, {<<"state">>, <<"ok">>}]}]}]}}, {<<"systemStats">>, {struct, [{<<"cpu_utilization_rate">>, 0}, {<<"swap_total">>, 0}, {<<"swap_used">>, 0}]}}, {<<"interestingStats">>, {struct, []}}, {<<"uptime">>, <<"337">>}, {<<"memoryTotal">>, 17946181632.0}, {<<"memoryFree">>, 17438019584.0}, {<<"mcdMemoryReserved">>, 13691}, {<<"mcdMemoryAllocated">>, 13691}, {<<"otpNode">>, <<"ns_1@10.13.41.69">>}, {<<"otpCookie">>, <<"sgzcoaesvptwzitb">>}, {<<"clusterMembership">>, <<"active">>}, {<<"status">>, <<"healthy">>}, {<<"hostname">>, <<"10.13.41.69:8091">>}, {<<"clusterCompatibility">>, 1}, {<<"version">>, <<"1.7.2r-20-g6604356">>}, {<<"os">>, <<"x86_64-unknown-linux-gnu">>}, {<<"ports">>, {struct, [{<<"proxy">>, 11211}, {<<"direct">>, 11210}]}}]) INFO REPORT <6058.64.0> 2011-11-30 18:53:31 =============================================================================== ns_log: logging ns_cluster:2:Node 'ns_1@10.13.41.47' is joining cluster via node 'ns_1@10.13.41.69'. INFO REPORT <6058.158.0> 2011-11-30 18:53:31 =============================================================================== config change: i_am_a_dead_man -> true INFO REPORT <6058.182.0> 2011-11-30 18:53:32 =============================================================================== ns_1@10.13.41.47:<6058.182.0>:mb_master:160: Node changed name from 'ns_1@127.0.0.1' to 'ns_1@10.13.41.47'. Updating state. ERROR REPORT <6058.191.0> 2011-11-30 18:53:32 =============================================================================== ns_1@10.13.41.47:<6058.191.0>:ns_heart:137: Failed to grab system stats: {'EXIT',{noproc,{gen_server,call, [{'stats_reader-@system','ns_1@10.13.41.47'}, {latest,"minute"}]}}} INFO REPORT <6058.206.0> 2011-11-30 18:53:32 =============================================================================== memcached<0.206.0>: EOL on stdin. Exiting INFO REPORT <6058.197.0> 2011-11-30 18:53:32 =============================================================================== menelaus_web streaming socket closed by client INFO REPORT <6058.204.0> 2011-11-30 18:53:32 =============================================================================== moxi<0.204.0>: EOL on stdin. Exiting ERROR REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== ** gen_event handler ns_port_init crashed. ** Was installed in ns_config_events ** Last event was: [{i_am_a_dead_man,true}] ** When handler state == {state} ** Reason == {shutdown,{gen_server,call, [ns_port_sup,which_children,infinity]}} INFO REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <6058.175.0> 2011-11-30 18:53:32 =============================================================================== Pushing config INFO REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== config change: buckets -> [{configs,[{"default", [{sasl_password,[]}, {num_replicas,1}, {ram_quota,104857600}, {auth_type,sasl}, {type,membase}, {num_vbuckets,1024}, {servers,['ns_1@10.13.41.69']}, {map,[['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69'|...], [...]|...]}]}]}] INFO REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== config change: memory_quota -> 15000 INFO REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== ns_node_disco_conf_events config on nodes_wanted INFO REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== config change: nodes_wanted -> ['ns_1@10.13.41.47'] INFO REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== config change: rest_creds -> ******** INFO REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== config change: settings -> [{stats,[{send_stats,true}]}] INFO REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== config change: {node,'ns_1@10.13.41.69',config_version} -> {1,7,2} INFO REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== config change: {node,'ns_1@10.13.41.69',isasl} -> [{path,"/opt/membase/var/lib/membase/data/isasl.pw"}] INFO REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== config change: {node,'ns_1@10.13.41.69',membership} -> active INFO REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== config change: {node,'ns_1@10.13.41.69',memcached} -> [{dbdir,"/opt/membase/var/lib/membase/data"}, {port,11210}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine,"/opt/membase/lib/memcached/bucket_engine.so"}, {engines, [{membase, [{engine,"/opt/membase/lib/memcached/ep.so"}, {initfile,"/opt/membase/etc/membase/init.sql"}, {static_config_string, "vb0=false;waitforwarmup=false;failpartialwarmup=false;shardpattern=%d/%b-%i.mb;db_strategy=multiMTVBDB"}]}, {memcached, [{engine,"/opt/membase/lib/memcached/default_engine.so"}, {static_config_string,"vb0=true"}]}]}, {verbosity,[]}] INFO REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== config change: {node,'ns_1@10.13.41.69',moxi} -> [{port,11211},{verbosity,[]}] INFO REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== config change: {node,'ns_1@10.13.41.69',ns_log} -> [{filename,"/opt/membase/var/lib/membase/data/ns_log"}] INFO REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== config change: {node,'ns_1@10.13.41.69',rest} -> [{port,8091},{port_meta,global}] INFO REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <6058.64.0> 2011-11-30 18:53:32 =============================================================================== ns_cluster: joining cluster. Child has exited. INFO REPORT <6058.162.0> 2011-11-30 18:53:32 =============================================================================== Writing isasl passwd file: "/opt/membase/var/lib/membase/data/isasl.pw" INFO REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== config change: nodes_wanted -> ['ns_1@10.13.41.47','ns_1@10.13.41.69'] INFO REPORT <6058.64.0> 2011-11-30 18:53:32 =============================================================================== pre-join cleaned config is: {config,{}, [[], [{directory,"/opt/membase/var/lib/membase/config"}, {nodes_wanted,['ns_1@127.0.0.1']}, {{node,'ns_1@127.0.0.1',membership},active}, {rest,[{port,8091}]}, {{node,'ns_1@127.0.0.1',rest},[{port,8091},{port_meta,global}]}, {rest_creds,[{creds,[]}]}, {{node,'ns_1@127.0.0.1',isasl}, [{path,"/opt/membase/var/lib/membase/data/isasl.pw"}]}, {{node,'ns_1@127.0.0.1',memcached}, [{port,11210}, {dbdir,"/opt/membase/var/lib/membase/data"}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine,"/opt/membase/lib/memcached/bucket_engine.so"}, {engines, [{membase, [{engine,"/opt/membase/lib/memcached/ep.so"}, {initfile,"/opt/membase/etc/membase/init.sql"}, {static_config_string, "vb0=false;waitforwarmup=false;failpartialwarmup=false;shardpattern=%d/%b-%i.mb;db_strategy=multiMTVBDB"}]}, {memcached, [{engine,"/opt/membase/lib/memcached/default_engine.so"}, {static_config_string,"vb0=true"}]}]}, {verbosity,[]}]}, {memory_quota,16090}, {buckets,[{configs,[]}]}, {{node,'ns_1@127.0.0.1',moxi},[{port,11211},{verbosity,[]}]}, {port_servers, [{moxi,"/opt/membase/bin/moxi", ["-Z", {"port_listen=~B,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", [port]}, "-z", {"url=http://127.0.0.1:~B/pools/default/saslBucketsStreaming", [{rest,port}]}, "-p","0","-Y","y","-O","stderr", {"~s",[verbosity]}], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR", {"~s",[{ns_moxi_sup,rest_user,[]}]}}, {"MOXI_SASL_PLAIN_PWD", {"~s",[{ns_moxi_sup,rest_pass,[]}]}}]}, use_stdio,exit_status,port_server_send_eol,stderr_to_stdout, stream]}, {memcached,"/opt/membase/bin/memcached", ["-X","/opt/membase/lib/memcached/stdin_term_handler.so","-p", {"~B",[port]}, "-E","/opt/membase/lib/memcached/bucket_engine.so","-B", "binary","-r","-c","10000","-e", {"admin=~s;default_bucket_name=default;auto_create=false", [admin_user]}, {"~s",[verbosity]}], [{env, [{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE",{"~s",[{isasl,path}]}}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,exit_status,port_server_send_eol, stream]}]}, {{node,'ns_1@127.0.0.1',ns_log}, [{filename,"/opt/membase/var/lib/membase/data/ns_log"}]}, {email_alerts, [{recipients,["root@localhost"]}, {sender,"membase@localhost"}, {enabled,true}, {email_server, [{user,[]}, {pass,[]}, {host,"localhost"}, {port,25}, {encrypt,false}]}, {alerts, [auto_failover_node,auto_failover_maximum_reached, auto_failover_other_nodes_down, auto_failover_cluster_too_small]}]}, {replication,[{enabled,true}]}, {auto_failover_cfg, [{enabled,false},{timeout,30},{max_nodes,1},{count,0}]}]], [[{nodes_wanted,['ns_1@10.13.41.47','ns_1@10.13.41.69']}, {{node,'ns_1@10.13.41.47',rest}, [{'_vclock',[{'ns_1@10.13.41.47',{1,63489898411}}]}, {port,8091}, {port_meta,global}]}, {{node,'ns_1@10.13.41.47',ns_log}, [{'_vclock', [{'ns_1@10.13.41.47',{1,63489898411}}, {'ns_1@127.0.0.1',{1,63489203281}}]}, {filename,"/opt/membase/var/lib/membase/data/ns_log"}]}, {{node,'ns_1@10.13.41.47',moxi}, [{'_vclock',[{'ns_1@10.13.41.47',{1,63489898411}}]}, {port,11211}, {verbosity,[]}]}, {{node,'ns_1@10.13.41.47',memcached}, [{'_vclock', [{'ns_1@10.13.41.47',{1,63489898411}}, {'ns_1@127.0.0.1',{1,63489203281}}]}, {port,11210}, {dbdir,"/opt/membase/var/lib/membase/data"}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine,"/opt/membase/lib/memcached/bucket_engine.so"}, {engines, [{membase, [{engine,"/opt/membase/lib/memcached/ep.so"}, {initfile,"/opt/membase/etc/membase/init.sql"}, {static_config_string, "vb0=false;waitforwarmup=false;failpartialwarmup=false;shardpattern=%d/%b-%i.mb;db_strategy=multiMTVBDB"}]}, {memcached, [{engine,"/opt/membase/lib/memcached/default_engine.so"}, {static_config_string,"vb0=true"}]}]}, {verbosity,[]}]}, {{node,'ns_1@10.13.41.47',isasl}, [{'_vclock', [{'ns_1@10.13.41.47',{1,63489898411}}, {'ns_1@127.0.0.1',{1,63489203281}}]}, {path,"/opt/membase/var/lib/membase/data/isasl.pw"}]}, {{node,'ns_1@10.13.41.47',config_version}, [{'_vclock',[{'ns_1@10.13.41.47',{1,63489898411}}]}|{1,7,2}]}, {otp,[{cookie,sgzcoaesvptwzitb}]}]], ns_config_default} INFO REPORT <6058.64.0> 2011-11-30 18:53:32 =============================================================================== ns_1@10.13.41.47:<6058.64.0>:ns_cluster:597: Connection from 'ns_1@10.13.41.47' to 'ns_1@10.13.41.69': true INFO REPORT <6058.64.0> 2011-11-30 18:53:32 =============================================================================== ns_1@10.13.41.47:<6058.64.0>:ns_cluster:606: Join status: {ok,ok}, starting ns_server_cluster back INFO REPORT <6058.1160.0> 2011-11-30 18:53:32 =============================================================================== ns_log:init(): Couldn't load logs from "/opt/membase/var/lib/membase/data/ns_log": {error, enoent} PROGRESS REPORT <6058.1159.0> 2011-11-30 18:53:32 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<6058.1160.0>}, {name,ns_log}, {mfargs,{ns_log,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <6058.1159.0> 2011-11-30 18:53:32 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<6058.1161.0>}, {name,ns_log_events}, {mfargs,{gen_event,start_link,[{local,ns_log_events}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <6058.1162.0> 2011-11-30 18:53:32 =============================================================================== supervisor {local,ns_node_disco_sup} started [{pid,<6058.1163.0>}, {name,ns_node_disco_events}, {mfargs,{gen_event,start_link,[{local,ns_node_disco_events}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <6058.1164.0> 2011-11-30 18:53:32 =============================================================================== Initting ns_node_disco with ['ns_1@10.13.41.69'] INFO REPORT <6058.1165.0> 2011-11-30 18:53:32 =============================================================================== ns_node_disco cookie_sync INFO REPORT <6058.1165.0> 2011-11-30 18:53:32 =============================================================================== ns_node_disco: nodes_wanted updated: ['ns_1@10.13.41.47','ns_1@10.13.41.69'], with cookie: sgzcoaesvptwzitb INFO REPORT <6058.1165.0> 2011-11-30 18:53:32 =============================================================================== ns_node_disco: nodes_wanted pong: ['ns_1@10.13.41.47','ns_1@10.13.41.69'], with cookie: sgzcoaesvptwzitb PROGRESS REPORT <6058.1162.0> 2011-11-30 18:53:32 =============================================================================== supervisor {local,ns_node_disco_sup} started [{pid,<6058.1164.0>}, {name,ns_node_disco}, {mfargs,{ns_node_disco,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <6058.1162.0> 2011-11-30 18:53:32 =============================================================================== supervisor {local,ns_node_disco_sup} started [{pid,<6058.1167.0>}, {name,ns_node_disco_log}, {mfargs,{ns_node_disco_log,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <6058.1162.0> 2011-11-30 18:53:32 =============================================================================== supervisor {local,ns_node_disco_sup} started [{pid,<6058.1168.0>}, {name,ns_node_disco_conf_events}, {mfargs,{ns_node_disco_conf_events,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <6058.1169.0> 2011-11-30 18:53:32 =============================================================================== ns_1@10.13.41.47:<6058.1169.0>:ns_config_rep:56: init pulling INFO REPORT <6058.1163.0> 2011-11-30 18:53:32 =============================================================================== ns_node_disco_log: nodes changed: ['ns_1@10.13.41.47','ns_1@10.13.41.69'] INFO REPORT <6058.1169.0> 2011-11-30 18:53:32 =============================================================================== ns_1@10.13.41.47:<6058.1169.0>:ns_config_rep:257: Pulling config from: 'ns_1@10.13.41.69' INFO REPORT <6058.1169.0> 2011-11-30 18:53:32 =============================================================================== ns_1@10.13.41.47:<6058.1169.0>:ns_config_rep:58: init pushing INFO REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== config change: auto_failover_cfg -> [{enabled,false},{timeout,30},{max_nodes,1},{count,0}] INFO REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== config change: buckets -> [{configs,[{"default", [{sasl_password,[]}, {num_replicas,1}, {ram_quota,104857600}, {auth_type,sasl}, {type,membase}, {num_vbuckets,1024}, {servers,['ns_1@10.13.41.69']}, {map,[['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69'|...], [...]|...]}]}]}] INFO REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== config change: email_alerts -> [{recipients,["root@localhost"]}, {sender,"membase@localhost"}, {enabled,true}, {email_server,[{user,[]}, {pass,[]}, {host,"localhost"}, {port,25}, {encrypt,false}]}, {alerts,[auto_failover_node,auto_failover_maximum_reached, auto_failover_other_nodes_down,auto_failover_cluster_too_small]}] INFO REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== config change: memory_quota -> 15000 INFO REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== ns_node_disco_conf_events config on nodes_wanted INFO REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== config change: nodes_wanted -> ['ns_1@10.13.41.47'] INFO REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== config change: port_servers -> [{moxi,"/opt/membase/bin/moxi", ["-Z", {"port_listen=~B,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", [port]}, "-z", {"url=http://127.0.0.1:~B/pools/default/saslBucketsStreaming", [{rest,port}]}, "-p","0","-Y","y","-O","stderr", {"~s",[verbosity]}], [{env,[{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR",{"~s",[{ns_moxi_sup,rest_user,[]}]}}, {"MOXI_SASL_PLAIN_PWD",{"~s",[{ns_moxi_sup,rest_pass,[]}]}}]}, use_stdio,exit_status,port_server_send_eol,stderr_to_stdout,stream]}, {memcached,"/opt/membase/bin/memcached", ["-X","/opt/membase/lib/memcached/stdin_term_handler.so","-p", {"~B",[port]}, "-E","/opt/membase/lib/memcached/bucket_engine.so","-B","binary", "-r","-c","10000","-e", {"admin=~s;default_bucket_name=default;auto_create=false", [admin_user]}, {"~s",[verbosity]}], [{env,[{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE",{"~s",[{isasl,path}]}}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,exit_status,port_server_send_eol, stream]}] INFO REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== config change: replication -> [{enabled,true}] INFO REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== config change: rest -> [{port,8091}] INFO REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== config change: rest_creds -> ******** INFO REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== config change: settings -> [{stats,[{send_stats,true}]}] INFO REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== config change: {node,'ns_1@10.13.41.47',membership} -> active INFO REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== config change: {node,'ns_1@10.13.41.69',config_version} -> {1,7,2} INFO REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== config change: {node,'ns_1@10.13.41.69',isasl} -> [{path,"/opt/membase/var/lib/membase/data/isasl.pw"}] INFO REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== config change: {node,'ns_1@10.13.41.69',membership} -> active INFO REPORT <6058.1169.0> 2011-11-30 18:53:32 =============================================================================== ns_1@10.13.41.47:<6058.1169.0>:ns_config_rep:62: init reannouncing INFO REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== config change: {node,'ns_1@10.13.41.69',memcached} -> [{dbdir,"/opt/membase/var/lib/membase/data"}, {port,11210}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine,"/opt/membase/lib/memcached/bucket_engine.so"}, {engines, [{membase, [{engine,"/opt/membase/lib/memcached/ep.so"}, {initfile,"/opt/membase/etc/membase/init.sql"}, {static_config_string, "vb0=false;waitforwarmup=false;failpartialwarmup=false;shardpattern=%d/%b-%i.mb;db_strategy=multiMTVBDB"}]}, {memcached, [{engine,"/opt/membase/lib/memcached/default_engine.so"}, {static_config_string,"vb0=true"}]}]}, {verbosity,[]}] INFO REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== config change: {node,'ns_1@10.13.41.69',moxi} -> [{port,11211},{verbosity,[]}] INFO REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== config change: {node,'ns_1@10.13.41.69',ns_log} -> [{filename,"/opt/membase/var/lib/membase/data/ns_log"}] INFO REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== config change: {node,'ns_1@10.13.41.69',rest} -> [{port,8091},{port_meta,global}] INFO REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <6058.1173.0> 2011-11-30 18:53:32 =============================================================================== ns_node_disco cookie_sync PROGRESS REPORT <6058.1162.0> 2011-11-30 18:53:32 =============================================================================== supervisor {local,ns_node_disco_sup} started [{pid,<6058.1169.0>}, {name,ns_config_rep}, {mfargs,{ns_config_rep,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <6058.1159.0> 2011-11-30 18:53:32 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<6058.1162.0>}, {name,ns_node_disco_sup}, {mfargs,{ns_node_disco_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <6058.1159.0> 2011-11-30 18:53:32 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<6058.1175.0>}, {name,ns_tick_event}, {mfargs,{gen_event,start_link,[{local,ns_tick_event}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <6058.1173.0> 2011-11-30 18:53:32 =============================================================================== ns_node_disco: nodes_wanted updated: ['ns_1@10.13.41.47'], with cookie: sgzcoaesvptwzitb INFO REPORT <6058.1173.0> 2011-11-30 18:53:32 =============================================================================== ns_node_disco: nodes_wanted pong: ['ns_1@10.13.41.47'], with cookie: sgzcoaesvptwzitb INFO REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== config change: auto_failover_cfg -> [{enabled,false},{timeout,30},{max_nodes,1},{count,0}] INFO REPORT <6058.1169.0> 2011-11-30 18:53:32 =============================================================================== Pushing config INFO REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== config change: buckets -> [{configs,[{"default", [{sasl_password,[]}, {num_replicas,1}, {ram_quota,104857600}, {auth_type,sasl}, {type,membase}, {num_vbuckets,1024}, {servers,['ns_1@10.13.41.69']}, {map,[['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69',undefined], ['ns_1@10.13.41.69'|...], [...]|...]}]}]}] INFO REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== config change: email_alerts -> [{recipients,["root@localhost"]}, {sender,"membase@localhost"}, {enabled,true}, {email_server,[{user,[]}, {pass,[]}, {host,"localhost"}, {port,25}, {encrypt,false}]}, {alerts,[auto_failover_node,auto_failover_maximum_reached, auto_failover_other_nodes_down,auto_failover_cluster_too_small]}] INFO REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== config change: memory_quota -> 15000 INFO REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== ns_node_disco_conf_events config on nodes_wanted INFO REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== config change: nodes_wanted -> ['ns_1@10.13.41.47'] INFO REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== ns_node_disco_conf_events config on otp INFO REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== config change: otp -> [{cookie,sgzcoaesvptwzitb}] INFO REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== config change: port_servers -> [{moxi,"/opt/membase/bin/moxi", ["-Z", {"port_listen=~B,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", [port]}, "-z", {"url=http://127.0.0.1:~B/pools/default/saslBucketsStreaming", [{rest,port}]}, "-p","0","-Y","y","-O","stderr", {"~s",[verbosity]}], [{env,[{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR",{"~s",[{ns_moxi_sup,rest_user,[]}]}}, {"MOXI_SASL_PLAIN_PWD",{"~s",[{ns_moxi_sup,rest_pass,[]}]}}]}, use_stdio,exit_status,port_server_send_eol,stderr_to_stdout,stream]}, {memcached,"/opt/membase/bin/memcached", ["-X","/opt/membase/lib/memcached/stdin_term_handler.so","-p", {"~B",[port]}, "-E","/opt/membase/lib/memcached/bucket_engine.so","-B","binary", "-r","-c","10000","-e", {"admin=~s;default_bucket_name=default;auto_create=false", [admin_user]}, {"~s",[verbosity]}], [{env,[{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE",{"~s",[{isasl,path}]}}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,exit_status,port_server_send_eol, stream]}] INFO REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== config change: replication -> [{enabled,true}] INFO REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== config change: rest -> [{port,8091}] INFO REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== config change: rest_creds -> ******** INFO REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== config change: settings -> [{stats,[{send_stats,true}]}] INFO REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== config change: {node,'ns_1@10.13.41.47',config_version} -> {1,7,2} INFO REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== config change: {node,'ns_1@10.13.41.47',isasl} -> [{path,"/opt/membase/var/lib/membase/data/isasl.pw"}] INFO REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== config change: {node,'ns_1@10.13.41.47',membership} -> active INFO REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== config change: {node,'ns_1@10.13.41.47',memcached} -> [{port,11210}, {dbdir,"/opt/membase/var/lib/membase/data"}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine,"/opt/membase/lib/memcached/bucket_engine.so"}, {engines, [{membase, [{engine,"/opt/membase/lib/memcached/ep.so"}, {initfile,"/opt/membase/etc/membase/init.sql"}, {static_config_string, "vb0=false;waitforwarmup=false;failpartialwarmup=false;shardpattern=%d/%b-%i.mb;db_strategy=multiMTVBDB"}]}, {memcached, [{engine,"/opt/membase/lib/memcached/default_engine.so"}, {static_config_string,"vb0=true"}]}]}, {verbosity,[]}] INFO REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== config change: {node,'ns_1@10.13.41.47',moxi} -> [{port,11211},{verbosity,[]}] INFO REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== config change: {node,'ns_1@10.13.41.47',ns_log} -> [{filename,"/opt/membase/var/lib/membase/data/ns_log"}] INFO REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== config change: {node,'ns_1@10.13.41.47',rest} -> [{port,8091},{port_meta,global}] INFO REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== config change: {node,'ns_1@10.13.41.69',config_version} -> {1,7,2} INFO REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== config change: {node,'ns_1@10.13.41.69',isasl} -> [{path,"/opt/membase/var/lib/membase/data/isasl.pw"}] INFO REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== config change: {node,'ns_1@10.13.41.69',membership} -> active INFO REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== config change: {node,'ns_1@10.13.41.69',memcached} -> [{dbdir,"/opt/membase/var/lib/membase/data"}, {port,11210}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine,"/opt/membase/lib/memcached/bucket_engine.so"}, {engines, [{membase, [{engine,"/opt/membase/lib/memcached/ep.so"}, {initfile,"/opt/membase/etc/membase/init.sql"}, {static_config_string, "vb0=false;waitforwarmup=false;failpartialwarmup=false;shardpattern=%d/%b-%i.mb;db_strategy=multiMTVBDB"}]}, {memcached, [{engine,"/opt/membase/lib/memcached/default_engine.so"}, {static_config_string,"vb0=true"}]}]}, {verbosity,[]}] INFO REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== config change: {node,'ns_1@10.13.41.69',moxi} -> [{port,11211},{verbosity,[]}] INFO REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== config change: {node,'ns_1@10.13.41.69',ns_log} -> [{filename,"/opt/membase/var/lib/membase/data/ns_log"}] INFO REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== config change: {node,'ns_1@10.13.41.69',rest} -> [{port,8091},{port_meta,global}] INFO REPORT <6058.158.0> 2011-11-30 18:53:32 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <6058.1169.0> 2011-11-30 18:53:32 =============================================================================== Pushing config done INFO REPORT <6058.1169.0> 2011-11-30 18:53:32 =============================================================================== Pushing config INFO REPORT <6058.1179.0> 2011-11-30 18:53:32 =============================================================================== ns_node_disco cookie_sync INFO REPORT <6058.1180.0> 2011-11-30 18:53:32 =============================================================================== ns_node_disco cookie_sync INFO REPORT <6058.1180.0> 2011-11-30 18:53:32 =============================================================================== ns_node_disco: nodes_wanted updated: ['ns_1@10.13.41.47'], with cookie: sgzcoaesvptwzitb INFO REPORT <6058.1180.0> 2011-11-30 18:53:32 =============================================================================== ns_node_disco: nodes_wanted pong: ['ns_1@10.13.41.47'], with cookie: sgzcoaesvptwzitb INFO REPORT <6058.1179.0> 2011-11-30 18:53:32 =============================================================================== ns_node_disco: nodes_wanted updated: ['ns_1@10.13.41.47'], with cookie: sgzcoaesvptwzitb INFO REPORT <6058.1179.0> 2011-11-30 18:53:32 =============================================================================== ns_node_disco: nodes_wanted pong: ['ns_1@10.13.41.47'], with cookie: sgzcoaesvptwzitb INFO REPORT <6058.1176.0> 2011-11-30 18:53:32 =============================================================================== ns_1@10.13.41.47:<6058.1176.0>:mb_master:97: I'm the only node, so I'm the master. INFO REPORT <6058.1169.0> 2011-11-30 18:53:32 =============================================================================== Pushing config done INFO REPORT <6058.1181.0> 2011-11-30 18:53:32 =============================================================================== ns_1@10.13.41.47:<6058.1181.0>:misc:785: start_singleton(gen_fsm, ns_orchestrator, [], []): started as <6058.1182.0> on 'ns_1@10.13.41.47' PROGRESS REPORT <6058.1181.0> 2011-11-30 18:53:32 =============================================================================== supervisor {local,mb_master_sup} started [{pid,<6058.1182.0>}, {name,ns_orchestrator}, {mfargs,{ns_orchestrator,start_link,[]}}, {restart_type,permanent}, {shutdown,20}, {child_type,worker}] INFO REPORT <6058.1181.0> 2011-11-30 18:53:32 =============================================================================== ns_1@10.13.41.47:<6058.1181.0>:misc:785: start_singleton(gen_server, ns_tick, [], []): started as <6058.1184.0> on 'ns_1@10.13.41.47' PROGRESS REPORT <6058.1181.0> 2011-11-30 18:53:32 =============================================================================== supervisor {local,mb_master_sup} started [{pid,<6058.1184.0>}, {name,ns_tick}, {mfargs,{ns_tick,start_link,[]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] INFO REPORT <6058.1185.0> 2011-11-30 18:53:32 =============================================================================== ns_1@10.13.41.47:<6058.1185.0>:auto_failover:120: init auto_failover. INFO REPORT <6058.1181.0> 2011-11-30 18:53:32 =============================================================================== ns_1@10.13.41.47:<6058.1181.0>:misc:785: start_singleton(gen_server, auto_failover, [], []): started as <6058.1185.0> on 'ns_1@10.13.41.47' PROGRESS REPORT <6058.1181.0> 2011-11-30 18:53:32 =============================================================================== supervisor {local,mb_master_sup} started [{pid,<6058.1185.0>}, {name,auto_failover}, {mfargs,{auto_failover,start_link,[]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <6058.1159.0> 2011-11-30 18:53:32 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<6058.1176.0>}, {name,mb_master}, {mfargs,{mb_master,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <6058.1159.0> 2011-11-30 18:53:32 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<6058.1186.0>}, {name,buckets_events}, {mfargs,{gen_event,start_link,[{local,buckets_events}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <6058.1187.0> 2011-11-30 18:53:32 =============================================================================== supervisor {local,ns_mail_sup} started [{pid,<6058.1188.0>}, {name,ns_mail}, {mfargs,{ns_mail,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <6058.1187.0> 2011-11-30 18:53:32 =============================================================================== supervisor {local,ns_mail_sup} started [{pid,<6058.1189.0>}, {name,ns_mail_log}, {mfargs,{ns_mail_log,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <6058.1159.0> 2011-11-30 18:53:32 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<6058.1187.0>}, {name,ns_mail_sup}, {mfargs,{ns_mail_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] INFO REPORT <6058.1161.0> 2011-11-30 18:53:32 =============================================================================== ns_1@10.13.41.47:<6058.1161.0>:ns_mail_log:45: ns_mail_log started up PROGRESS REPORT <6058.1159.0> 2011-11-30 18:53:32 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<6058.1190.0>}, {name,ns_heart}, {mfargs,{ns_heart,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <6058.1159.0> 2011-11-30 18:53:32 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<6058.1191.0>}, {name,ns_doctor}, {mfargs,{ns_doctor,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <6058.1193.0> 2011-11-30 18:53:32 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<6058.1195.0>}, {name,menelaus_web}, {mfargs,{menelaus_web,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] PROGRESS REPORT <6058.1193.0> 2011-11-30 18:53:32 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<6058.1197.0>}, {name,menelaus_event}, {mfargs,{menelaus_event,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] PROGRESS REPORT <6058.1193.0> 2011-11-30 18:53:32 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<6058.1198.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] PROGRESS REPORT <6058.1193.0> 2011-11-30 18:53:32 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<6058.1199.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <6058.1159.0> 2011-11-30 18:53:32 =============================================================================== ns_log: logging menelaus_sup:1:Membase Server has started on web port 8091 on node 'ns_1@10.13.41.47'. PROGRESS REPORT <6058.1159.0> 2011-11-30 18:53:32 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<6058.1193.0>}, {name,menelaus}, {mfargs,{menelaus_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] ERROR REPORT <6058.1190.0> 2011-11-30 18:53:32 =============================================================================== ns_1@10.13.41.47:<6058.1190.0>:ns_heart:137: Failed to grab system stats: {'EXIT',{noproc,{gen_server,call, [{'stats_reader-@system','ns_1@10.13.41.47'}, {latest,"minute"}]}}} INFO REPORT <6058.1191.0> 2011-11-30 18:53:32 =============================================================================== ns_1@10.13.41.47:<6058.1191.0>:ns_doctor:82: Got initial status [{'ns_1@10.13.41.47', [{last_heard, {1322, 679212, 926465}}, {active_buckets, []}, {ready_buckets, []}, {replication, [{"default", 1.0}]}, {memory, [{total, 27598728}, {processes, 8290648}, {processes_used, 8261888}, {system, 19308080}, {atom, 784593}, {atom_used, 766942}, {binary, 901048}, {code, 7427684}, {ets, 663288}]}, {system_stats, [{cpu_utilization_rate, 0}, {swap_total, 0}, {swap_used, 0}]}, {interesting_stats, []}, {cluster_compatibility_version, 1}, {version, [{os_mon, "2.2.5"}, {mnesia, "4.4.17"}, {kernel, "2.14.3"}, {sasl, "2.1.9.3"}, {ns_server, "1.7.2r-20-g6604356"}, {stdlib, "1.17.3"}]}, {system_arch, "x86_64-unknown-linux-gnu"}, {wall_clock, 352}, {memory_data, {17946181632, 425013248, {<6058.7.0>, 601336}}}, {disk_data, [{"/", 8256952, 15}, {"/dev", 8754940, 1}, {"/run", 3505116, 1}, {"/run/lock", 5120,0}, {"/run/shm", 8762784, 0}, {"/mnt", 423135208, 1}]}, {meminfo, <<"MemTotal: 17525568 kB\nMemFree: 17109424 kB\nBuffers: 98256 kB\nCached: 53984 kB\nSwapCached: 0 kB\nActive: 144268 kB\nInactive: 43648 kB\nActive(anon): 35712 kB\nInactive(anon): 140 kB\nActive(file): 108556 kB\nInactive(file): 43508 kB\nUnevictable: 0 kB\nMlocked: 0 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 184 kB\nWriteback: 0 kB\nAnonPages: 35756 kB\nMapped: 6172 kB\nShmem: 156 kB\nSlab: 18408 kB\nSReclaimable: 12636 kB\nSUnreclaim: 5772 kB\nKernelStack: 824 kB\nPageTables: 1324 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nWritebackTmp: 0 kB\nCommitLimit: 8762784 kB\nCommitted_AS: 107888 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 64000 kB\nVmallocChunk: 34359674252 kB\nHardwareCorrupted: 0 kB\nAnonHugePages: 0 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugePages_Surp: 0\nHugepagesize: 2048 kB\nDirectMap4k: 17920000 kB\nDirectMap2M: 0 kB\n">>}, {system_memory_data, [{system_total_memory, 17946181632}, {free_swap, 0}, {total_swap, 0}, {cached_memory, 55279616}, {buffered_memory, 100614144}, {free_memory, 17520050176}, {total_memory, 17946181632}]}, {statistics, [{wall_clock, {352076, 31}}, {context_switches, {77825, 0}}, {garbage_collection, {9045, 16866557, 0}}, {io, {{input, 8067644}, {output, 3101684}}}, {reductions, {7107563, 337294}}, {run_queue, 0}, {runtime, {1410, 40}}]}]}, {'ns_1@10.13.41.69', [{last_heard, {1322, 679212, 926472}}, {active_buckets, ["default"]}, {ready_buckets, ["default"]}, {replication, [{"default", 0.0}]}, {memory, [{total, 33491960}, {processes, 11957920}, {processes_used, 11935848}, {system, 21534040}, {atom, 896777}, {atom_used, 871341}, {binary, 1170408}, {code, 8509446}, {ets, 1351128}]}, {system_stats, [{cpu_utilization_rate, 0.7849293563579278}, {swap_total, 0}, {swap_used, 0}]}, {interesting_stats, [{curr_items, 0}, {curr_items_tot, 0}, {vb_replica_curr_items, 0}]}, {cluster_compatibility_version, 1}, {version, [{os_mon, "2.2.5"}, {mnesia, "4.4.17"}, {inets, "5.5.2"}, {kernel, "2.14.3"}, {sasl, "2.1.9.3"}, {ns_server, "1.7.2r-20-g6604356"}, {stdlib, "1.17.3"}]}, {system_arch, "x86_64-unknown-linux-gnu"}, {wall_clock, 342}, {memory_data, {17946181632, 508162048, {<6087.545.0>, 2172736}}}, {disk_data, [{"/", 8256952, 15}, {"/dev", 8754940, 1}, {"/run", 3505116, 1}, {"/run/lock", 5120,0}, {"/run/shm", 8762784, 0}, {"/mnt", 423135208, 1}]}, {meminfo, <<"MemTotal: 17525568 kB\nMemFree: 17030068 kB\nBuffers: 98268 kB\nCached: 61580 kB\nSwapCached: 0 kB\nActive: 216820 kB\nInactive: 49272 kB\nActive(anon): 106304 kB\nInactive(anon): 140 kB\nActive(file): 110516 kB\nInactive(file): 49132 kB\nUnevictable: 0 kB\nMlocked: 0 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 164 kB\nWriteback: 0 kB\nAnonPages: 106232 kB\nMapped: 8908 kB\nShmem: 156 kB\nSlab: 18828 kB\nSReclaimable: 12976 kB\nSUnreclaim: 5852 kB\nKernelStack: 872 kB\nPageTables: 1756 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nWritebackTmp: 0 kB\nCommitLimit: 8762784 kB\nCommitted_AS: 332148 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 64000 kB\nVmallocChunk: 34359674252 kB\nHardwareCorrupted: 0 kB\nAnonHugePages: 0 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugePages_Surp: 0\nHugepagesize: 2048 kB\nDirectMap4k: 17920000 kB\nDirectMap2M: 0 kB\n">>}, {system_memory_data, [{system_total_memory, 17946181632}, {free_swap, 0}, {total_swap, 0}, {cached_memory, 63057920}, {buffered_memory, 100626432}, {free_memory, 17438789632}, {total_memory, 17946181632}]}, {statistics, [{wall_clock, {331624, 1}}, {context_switches, {89358, 0}}, {garbage_collection, {15555, 63704093, 0}}, {io, {{input, 9486622}, {output, 6290172}}}, {reductions, {38243947, 2601327}}, {run_queue, 1}, {runtime, {3100, 160}}]}]}] PROGRESS REPORT <6058.1200.0> 2011-11-30 18:53:32 =============================================================================== supervisor {local,ns_port_sup} started [{pid,<6058.1201.0>}, {name,ns_port_init}, {mfargs,{ns_port_init,start_link,[]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] INFO REPORT <6058.1202.0> 2011-11-30 18:53:32 =============================================================================== starting ns_port_server with delay of 5000 PROGRESS REPORT <6058.1200.0> 2011-11-30 18:53:32 =============================================================================== supervisor {local,ns_port_sup} started [{pid,<6058.1202.0>}, {name, {moxi,"/opt/membase/bin/moxi", ["-Z", "port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", "-z", "url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming", "-p","0","-Y","y","-O","stderr",[]], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR","Administrator"}, {"MOXI_SASL_PLAIN_PWD","test1234"}]}, use_stdio,exit_status,port_server_send_eol, stderr_to_stdout,stream]}}, {mfargs, {supervisor_cushion,start_link, [moxi,5000,ns_port_server,start_link, [moxi,"/opt/membase/bin/moxi", ["-Z", "port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", "-z", "url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming", "-p","0","-Y","y","-O","stderr",[]], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR","Administrator"}, {"MOXI_SASL_PLAIN_PWD","test1234"}]}, use_stdio,exit_status,port_server_send_eol, stderr_to_stdout,stream]]]}}, {restart_type,permanent}, {shutdown,10000}, {child_type,worker}] INFO REPORT <6058.1204.0> 2011-11-30 18:53:32 =============================================================================== starting ns_port_server with delay of 5000 PROGRESS REPORT <6058.1200.0> 2011-11-30 18:53:32 =============================================================================== supervisor {local,ns_port_sup} started [{pid,<6058.1204.0>}, {name, {memcached,"/opt/membase/bin/memcached", ["-X","/opt/membase/lib/memcached/stdin_term_handler.so", "-p","11210","-E", "/opt/membase/lib/memcached/bucket_engine.so","-B", "binary","-r","-c","10000","-e", "admin=_admin;default_bucket_name=default;auto_create=false", []], [{env, [{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE", "/opt/membase/var/lib/membase/data/isasl.pw"}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,exit_status, port_server_send_eol,stream]}}, {mfargs, {supervisor_cushion,start_link, [memcached,5000,ns_port_server,start_link, [memcached,"/opt/membase/bin/memcached", ["-X","/opt/membase/lib/memcached/stdin_term_handler.so", "-p","11210","-E", "/opt/membase/lib/memcached/bucket_engine.so","-B", "binary","-r","-c","10000","-e", "admin=_admin;default_bucket_name=default;auto_create=false", []], [{env, [{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE", "/opt/membase/var/lib/membase/data/isasl.pw"}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,exit_status, port_server_send_eol,stream]]]}}, {restart_type,permanent}, {shutdown,10000}, {child_type,worker}] PROGRESS REPORT <6058.1159.0> 2011-11-30 18:53:32 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<6058.1200.0>}, {name,ns_port_sup}, {mfargs,{ns_port_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,60000}, {child_type,worker}] PROGRESS REPORT <6058.1159.0> 2011-11-30 18:53:32 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<6058.1206.0>}, {name,ns_stats_event}, {mfargs,{gen_event,start_link,[{local,ns_stats_event}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <6058.1159.0> 2011-11-30 18:53:32 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<6058.1207.0>}, {name,ns_bucket_worker}, {mfargs,{work_queue,start_link,[ns_bucket_worker]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <6058.1159.0> 2011-11-30 18:53:32 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<6058.1208.0>}, {name,ns_bucket_sup}, {mfargs,{ns_bucket_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <6058.1159.0> 2011-11-30 18:53:32 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<6058.1209.0>}, {name,system_stats_collector}, {mfargs,{system_stats_collector,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <6058.1159.0> 2011-11-30 18:53:32 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<6058.1211.0>}, {name,{stats_archiver,"@system"}}, {mfargs,{stats_archiver,start_link,["@system"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <6058.1159.0> 2011-11-30 18:53:32 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<6058.1212.0>}, {name,{stats_reader,"@system"}}, {mfargs,{stats_reader,start_link,["@system"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <6058.1159.0> 2011-11-30 18:53:32 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<6058.1213.0>}, {name,ns_moxi_sup_work_queue}, {mfargs,{work_queue,start_link,[ns_moxi_sup_work_queue]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <6058.1159.0> 2011-11-30 18:53:32 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<6058.1214.0>}, {name,ns_moxi_sup}, {mfargs,{ns_moxi_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <6058.54.0> 2011-11-30 18:53:32 =============================================================================== supervisor {local,ns_server_cluster_sup} started [{pid,<6058.1159.0>}, {name,ns_server_sup}, {mfargs,{ns_server_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] INFO REPORT <6058.64.0> 2011-11-30 18:53:32 =============================================================================== ns_log: logging ns_cluster:3:Node ns_1@10.13.41.47 joined cluster INFO REPORT <6058.64.0> 2011-11-30 18:53:32 =============================================================================== ns_1@10.13.41.47:<6058.64.0>:ns_cluster:104: complete_join([{<<"targetNode">>, <<"ns_1@10.13.41.47">>}, {<<"availableStorage">>, {struct, [{<<"hdd">>, [{struct, [{<<"path">>, <<"/">>}, {<<"sizeKBytes">>, 8256952}, {<<"usagePercent">>, 15}]}, {struct, [{<<"path">>, <<"/dev">>}, {<<"sizeKBytes">>, 8754940}, {<<"usagePercent">>, 1}]}, {struct, [{<<"path">>, <<"/run">>}, {<<"sizeKBytes">>, 3505116}, {<<"usagePercent">>, 1}]}, {struct, [{<<"path">>, <<"/run/lock">>}, {<<"sizeKBytes">>, 5120}, {<<"usagePercent">>, 0}]}, {struct, [{<<"path">>, <<"/run/shm">>}, {<<"sizeKBytes">>, 8762784}, {<<"usagePercent">>, 0}]}, {struct, [{<<"path">>, <<"/mnt">>}, {<<"sizeKBytes">>, 423135208}, {<<"usagePercent">>, 1}]}]}]}}, {<<"memoryQuota">>, 15000}, {<<"storageTotals">>, {struct, [{<<"ram">>, {struct, [{<<"usedByData">>, 26968704}, {<<"total">>, 17946181632.0}, {<<"quotaTotal">>, 1.572864e10}, {<<"used">>, 508162048}]}}, {<<"hdd">>, {struct, [{<<"usedByData">>, 1808192}, {<<"total">>, 8455118848.0}, {<<"quotaTotal">>, 8455118848.0}, {<<"used">>, 1268267827}, {<<"free">>, 7186851021.0}]}}]}}, {<<"storage">>, {struct, [{<<"ssd">>,[]}, {<<"hdd">>, [{struct, [{<<"path">>, <<"/opt/membase/var/lib/membase/data">>}, {<<"quotaMb">>, <<"none">>}, {<<"state">>, <<"ok">>}]}]}]}}, {<<"systemStats">>, {struct, [{<<"cpu_utilization_rate">>, 0}, {<<"swap_total">>, 0}, {<<"swap_used">>, 0}]}}, {<<"interestingStats">>, {struct,[]}}, {<<"uptime">>, <<"337">>}, {<<"memoryTotal">>, 17946181632.0}, {<<"memoryFree">>, 17438019584.0}, {<<"mcdMemoryReserved">>, 13691}, {<<"mcdMemoryAllocated">>, 13691}, {<<"otpNode">>, <<"ns_1@10.13.41.69">>}, {<<"otpCookie">>, <<"sgzcoaesvptwzitb">>}, {<<"clusterMembership">>, <<"active">>}, {<<"status">>, <<"healthy">>}, {<<"hostname">>, <<"10.13.41.69:8091">>}, {<<"clusterCompatibility">>, 1}, {<<"version">>, <<"1.7.2r-20-g6604356">>}, {<<"os">>, <<"x86_64-unknown-linux-gnu">>}, {<<"ports">>, {struct, [{<<"proxy">>, 11211}, {<<"direct">>, 11210}]}}]) -> {ok, ok} INFO REPORT <6058.1203.0> 2011-11-30 18:53:33 =============================================================================== moxi<0.1203.0>: 2011-11-30 18:53:32: (cproxy_config.c.317) env: MOXI_SASL_PLAIN_USR (13) moxi<0.1203.0>: 2011-11-30 18:53:32: (cproxy_config.c.326) env: MOXI_SASL_PLAIN_PWD (8) INFO REPORT <6058.1164.0> 2011-11-30 18:53:34 =============================================================================== ns_log: logging ns_node_disco:5:Node 'ns_1@10.13.41.47' saw that node 'ns_1@10.13.41.69' went down. INFO REPORT <6058.1163.0> 2011-11-30 18:53:34 =============================================================================== ns_node_disco_log: nodes changed: ['ns_1@10.13.41.47'] INFO REPORT <6058.1233.0> 2011-11-30 18:53:42 =============================================================================== ns_1@10.13.41.47:<6058.1233.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.1233.0> 2011-11-30 18:53:43 =============================================================================== ns_1@10.13.41.47:<6058.1233.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.1233.0> 2011-11-30 18:53:44 =============================================================================== ns_1@10.13.41.47:<6058.1233.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.1233.0> 2011-11-30 18:53:45 =============================================================================== ns_1@10.13.41.47:<6058.1233.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] ERROR REPORT <6058.1233.0> 2011-11-30 18:53:46 =============================================================================== ns_1@10.13.41.47:<6058.1233.0>:ns_janitor:68: Bucket "default" not yet ready on ['ns_1@10.13.41.69'] INFO REPORT <6058.1261.0> 2011-11-30 18:53:52 =============================================================================== ns_1@10.13.41.47:<6058.1261.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.1261.0> 2011-11-30 18:53:53 =============================================================================== ns_1@10.13.41.47:<6058.1261.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.1261.0> 2011-11-30 18:53:54 =============================================================================== ns_1@10.13.41.47:<6058.1261.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.1261.0> 2011-11-30 18:53:55 =============================================================================== ns_1@10.13.41.47:<6058.1261.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] ERROR REPORT <6058.1261.0> 2011-11-30 18:53:56 =============================================================================== ns_1@10.13.41.47:<6058.1261.0>:ns_janitor:68: Bucket "default" not yet ready on ['ns_1@10.13.41.69'] INFO REPORT <6058.1288.0> 2011-11-30 18:54:02 =============================================================================== ns_1@10.13.41.47:<6058.1288.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.1288.0> 2011-11-30 18:54:03 =============================================================================== ns_1@10.13.41.47:<6058.1288.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.1288.0> 2011-11-30 18:54:04 =============================================================================== ns_1@10.13.41.47:<6058.1288.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.1288.0> 2011-11-30 18:54:05 =============================================================================== ns_1@10.13.41.47:<6058.1288.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] ERROR REPORT <6058.1288.0> 2011-11-30 18:54:06 =============================================================================== ns_1@10.13.41.47:<6058.1288.0>:ns_janitor:68: Bucket "default" not yet ready on ['ns_1@10.13.41.69'] INFO REPORT <6058.1323.0> 2011-11-30 18:54:12 =============================================================================== ns_1@10.13.41.47:<6058.1323.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.1323.0> 2011-11-30 18:54:13 =============================================================================== ns_1@10.13.41.47:<6058.1323.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.1323.0> 2011-11-30 18:54:14 =============================================================================== ns_1@10.13.41.47:<6058.1323.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.1323.0> 2011-11-30 18:54:15 =============================================================================== ns_1@10.13.41.47:<6058.1323.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] ERROR REPORT <6058.1323.0> 2011-11-30 18:54:16 =============================================================================== ns_1@10.13.41.47:<6058.1323.0>:ns_janitor:68: Bucket "default" not yet ready on ['ns_1@10.13.41.69'] INFO REPORT <6058.1351.0> 2011-11-30 18:54:22 =============================================================================== ns_1@10.13.41.47:<6058.1351.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] ERROR REPORT <6058.1319.0> 2011-11-30 18:54:23 =============================================================================== ns_1@10.13.41.47:<6058.1319.0>:stats_reader:191: Some nodes didn't respond: ['ns_1@10.13.41.47'] INFO REPORT <6058.1351.0> 2011-11-30 18:54:23 =============================================================================== ns_1@10.13.41.47:<6058.1351.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.1351.0> 2011-11-30 18:54:24 =============================================================================== ns_1@10.13.41.47:<6058.1351.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.1351.0> 2011-11-30 18:54:25 =============================================================================== ns_1@10.13.41.47:<6058.1351.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] ERROR REPORT <6058.1313.0> 2011-11-30 18:54:26 =============================================================================== ns_1@10.13.41.47:<6058.1313.0>:stats_reader:191: Some nodes didn't respond: ['ns_1@10.13.41.47'] ERROR REPORT <6058.1351.0> 2011-11-30 18:54:26 =============================================================================== ns_1@10.13.41.47:<6058.1351.0>:ns_janitor:68: Bucket "default" not yet ready on ['ns_1@10.13.41.69'] ERROR REPORT <6058.1317.0> 2011-11-30 18:54:27 =============================================================================== ns_1@10.13.41.47:<6058.1317.0>:stats_reader:191: Some nodes didn't respond: ['ns_1@10.13.41.47'] ERROR REPORT <6058.1313.0> 2011-11-30 18:54:29 =============================================================================== ns_1@10.13.41.47:<6058.1313.0>:stats_reader:191: Some nodes didn't respond: ['ns_1@10.13.41.47'] PROGRESS REPORT <6058.33.0> 2011-11-30 18:54:32 =============================================================================== supervisor {local,sasl_sup} started [{pid,<6058.1399.0>}, {name,rb_server}, {mfargs,{rb,start_link, [[{report_dir,"/opt/membase/var/lib/membase/logs"}, {start_log,"/opt/membase/var/lib/membase/tmp/nslogs732795_705.log"}, {type,all}, {max,all}, {report_dir}]]}}, {restart_type,temporary}, {shutdown,brutal_kill}, {child_type,worker}] INFO REPORT <6058.1604.0> 2011-11-30 18:54:32 =============================================================================== ns_1@10.13.41.47:<6058.1604.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.1191.0> 2011-11-30 18:54:32 =============================================================================== ns_1@10.13.41.47:<6058.1191.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.13.41.47', [{last_heard,{1322,679267,924837}}, {active_buckets,[]}, {ready_buckets,[]}, {replication,[{"default",1.0}]}, {memory, [{total,32356512}, {processes,12733656}, {processes_used,12722336}, {system,19622856}, {atom,801673}, {atom_used,788011}, {binary,1046992}, {code,7567464}, {ets,693792}]}, {system_stats, [{cpu_utilization_rate,0.5050505050505051}, {swap_total,0}, {swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.5"}, {mnesia,"4.4.17"}, {kernel,"2.14.3"}, {sasl,"2.1.9.3"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,407}, {memory_data,{17946181632,428949504,{<6058.1196.0>,1943480}}}, {disk_data, [{"/",8256952,15}, {"/dev",8754940,1}, {"/run",3505116,1}, {"/run/lock",5120,0}, {"/run/shm",8762784,0}, {"/mnt",423135208,1}]}, {meminfo, <<"MemTotal: 17525568 kB\nMemFree: 17103816 kB\nBuffers: 98256 kB\nCached: 55476 kB\nSwapCached: 0 kB\nActive: 148028 kB\nInactive: 45024 kB\nActive(anon): 39352 kB\nInactive(anon): 140 kB\nActive(file): 108676 kB\nInactive(file): 44884 kB\nUnevictable: 0 kB\nMlocked: 0 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 76 kB\nWriteback: 0 kB\nAnonPages: 39464 kB\nMapped: 7404 kB\nShmem: 156 kB\nSlab: 18544 kB\nSReclaimable: 12764 kB\nSUnreclaim: 5780 kB\nKernelStack: 840 kB\nPageTables: 1556 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nWritebackTmp: 0 kB\nCommitLimit: 8762784 kB\nCommitted_AS: 232164 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 64000 kB\nVmallocChunk: 34359674252 kB\nHardwareCorrupted: 0 kB\nAnonHugePages: 0 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugePages_Surp: 0\nHugepagesize: 2048 kB\nDirectMap4k: 17920000 kB\nDirectMap2M: 0 kB\n">>}, {system_memory_data, [{system_total_memory,17946181632}, {free_swap,0}, {total_swap,0}, {cached_memory,56807424}, {buffered_memory,100614144}, {free_memory,17514307584}, {total_memory,17946181632}]}, {statistics, [{wall_clock,{397079,1}}, {context_switches,{84578,0}}, {garbage_collection,{10160,20107547,0}}, {io,{{input,9306315},{output,4420200}}}, {reductions,{8381439,579285}}, {run_queue,0}, {runtime,{1600,100}}]}]}, {'ns_1@10.13.41.69', [{last_heard,{1322,679213,262565}}, {active_buckets,[]}, {ready_buckets,[]}, {replication,[{"default",0.0}]}, {memory, [{total,30057456}, {processes,8527512}, {processes_used,8486992}, {system,21529944}, {atom,896777}, {atom_used,871341}, {binary,1179592}, {code,8509446}, {ets,1341304}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.5"}, {mnesia,"4.4.17"}, {inets,"5.5.2"}, {kernel,"2.14.3"}, {sasl,"2.1.9.3"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,342}, {memory_data,{17946181632,508162048,{<6087.545.0>,2172736}}}, {disk_data, [{"/",8256952,15}, {"/dev",8754940,1}, {"/run",3505116,1}, {"/run/lock",5120,0}, {"/run/shm",8762784,0}, {"/mnt",423135208,1}]}, {meminfo, <<"MemTotal: 17525568 kB\nMemFree: 17030068 kB\nBuffers: 98268 kB\nCached: 61580 kB\nSwapCached: 0 kB\nActive: 216820 kB\nInactive: 49272 kB\nActive(anon): 106304 kB\nInactive(anon): 140 kB\nActive(file): 110516 kB\nInactive(file): 49132 kB\nUnevictable: 0 kB\nMlocked: 0 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 164 kB\nWriteback: 0 kB\nAnonPages: 106232 kB\nMapped: 8908 kB\nShmem: 156 kB\nSlab: 18828 kB\nSReclaimable: 12976 kB\nSUnreclaim: 5852 kB\nKernelStack: 872 kB\nPageTables: 1756 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nWritebackTmp: 0 kB\nCommitLimit: 8762784 kB\nCommitted_AS: 332148 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 64000 kB\nVmallocChunk: 34359674252 kB\nHardwareCorrupted: 0 kB\nAnonHugePages: 0 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugePages_Surp: 0\nHugepagesize: 2048 kB\nDirectMap4k: 17920000 kB\nDirectMap2M: 0 kB\n">>}, {system_memory_data, [{system_total_memory,17946181632}, {free_swap,0}, {total_swap,0}, {cached_memory,63057920}, {buffered_memory,100626432}, {free_memory,17438789632}, {total_memory,17946181632}]}, {statistics, [{wall_clock,{331624,1}}, {context_switches,{89358,0}}, {garbage_collection,{15555,63704093,0}}, {io,{{input,9486622},{output,6290172}}}, {reductions,{38243947,2601327}}, {run_queue,1}, {runtime,{3100,160}}]}]}] INFO REPORT <6058.1604.0> 2011-11-30 18:54:33 =============================================================================== ns_1@10.13.41.47:<6058.1604.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.1604.0> 2011-11-30 18:54:34 =============================================================================== ns_1@10.13.41.47:<6058.1604.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.1604.0> 2011-11-30 18:54:35 =============================================================================== ns_1@10.13.41.47:<6058.1604.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] ERROR REPORT <6058.1604.0> 2011-11-30 18:54:36 =============================================================================== ns_1@10.13.41.47:<6058.1604.0>:ns_janitor:68: Bucket "default" not yet ready on ['ns_1@10.13.41.69'] INFO REPORT <6058.2006.0> 2011-11-30 18:54:42 =============================================================================== ns_1@10.13.41.47:<6058.2006.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2006.0> 2011-11-30 18:54:43 =============================================================================== ns_1@10.13.41.47:<6058.2006.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2006.0> 2011-11-30 18:54:44 =============================================================================== ns_1@10.13.41.47:<6058.2006.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2006.0> 2011-11-30 18:54:45 =============================================================================== ns_1@10.13.41.47:<6058.2006.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] ERROR REPORT <6058.2006.0> 2011-11-30 18:54:46 =============================================================================== ns_1@10.13.41.47:<6058.2006.0>:ns_janitor:68: Bucket "default" not yet ready on ['ns_1@10.13.41.69'] ERROR REPORT <6058.1312.0> 2011-11-30 18:54:48 =============================================================================== ns_1@10.13.41.47:<6058.1312.0>:stats_reader:191: Some nodes didn't respond: ['ns_1@10.13.41.47'] INFO REPORT <6058.2038.0> 2011-11-30 18:54:52 =============================================================================== ns_1@10.13.41.47:<6058.2038.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2038.0> 2011-11-30 18:54:53 =============================================================================== ns_1@10.13.41.47:<6058.2038.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2038.0> 2011-11-30 18:54:54 =============================================================================== ns_1@10.13.41.47:<6058.2038.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2038.0> 2011-11-30 18:54:55 =============================================================================== ns_1@10.13.41.47:<6058.2038.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] ERROR REPORT <6058.2038.0> 2011-11-30 18:54:56 =============================================================================== ns_1@10.13.41.47:<6058.2038.0>:ns_janitor:68: Bucket "default" not yet ready on ['ns_1@10.13.41.69'] INFO REPORT <6058.2065.0> 2011-11-30 18:55:02 =============================================================================== ns_1@10.13.41.47:<6058.2065.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2065.0> 2011-11-30 18:55:03 =============================================================================== ns_1@10.13.41.47:<6058.2065.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2065.0> 2011-11-30 18:55:04 =============================================================================== ns_1@10.13.41.47:<6058.2065.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2065.0> 2011-11-30 18:55:05 =============================================================================== ns_1@10.13.41.47:<6058.2065.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] ERROR REPORT <6058.2065.0> 2011-11-30 18:55:06 =============================================================================== ns_1@10.13.41.47:<6058.2065.0>:ns_janitor:68: Bucket "default" not yet ready on ['ns_1@10.13.41.69'] ERROR REPORT <6058.1312.0> 2011-11-30 18:55:08 =============================================================================== ns_1@10.13.41.47:<6058.1312.0>:stats_reader:191: Some nodes didn't respond: ['ns_1@10.13.41.47'] INFO REPORT <6058.2098.0> 2011-11-30 18:55:12 =============================================================================== ns_1@10.13.41.47:<6058.2098.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2098.0> 2011-11-30 18:55:13 =============================================================================== ns_1@10.13.41.47:<6058.2098.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2098.0> 2011-11-30 18:55:14 =============================================================================== ns_1@10.13.41.47:<6058.2098.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2098.0> 2011-11-30 18:55:15 =============================================================================== ns_1@10.13.41.47:<6058.2098.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] ERROR REPORT <6058.2098.0> 2011-11-30 18:55:16 =============================================================================== ns_1@10.13.41.47:<6058.2098.0>:ns_janitor:68: Bucket "default" not yet ready on ['ns_1@10.13.41.69'] INFO REPORT <6058.2126.0> 2011-11-30 18:55:22 =============================================================================== ns_1@10.13.41.47:<6058.2126.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2126.0> 2011-11-30 18:55:23 =============================================================================== ns_1@10.13.41.47:<6058.2126.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2126.0> 2011-11-30 18:55:24 =============================================================================== ns_1@10.13.41.47:<6058.2126.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2126.0> 2011-11-30 18:55:25 =============================================================================== ns_1@10.13.41.47:<6058.2126.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] ERROR REPORT <6058.2126.0> 2011-11-30 18:55:26 =============================================================================== ns_1@10.13.41.47:<6058.2126.0>:ns_janitor:68: Bucket "default" not yet ready on ['ns_1@10.13.41.69'] ERROR REPORT <6058.1312.0> 2011-11-30 18:55:28 =============================================================================== ns_1@10.13.41.47:<6058.1312.0>:stats_reader:191: Some nodes didn't respond: ['ns_1@10.13.41.47'] INFO REPORT <6058.2159.0> 2011-11-30 18:55:32 =============================================================================== ns_1@10.13.41.47:<6058.2159.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.1191.0> 2011-11-30 18:55:32 =============================================================================== ns_1@10.13.41.47:<6058.1191.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.13.41.47', [{last_heard,{1322,679327,925167}}, {active_buckets,[]}, {ready_buckets,[]}, {replication,[{"default",1.0}]}, {memory, [{total,31371504}, {processes,11232552}, {processes_used,11217488}, {system,20138952}, {atom,807329}, {atom_used,795214}, {binary,1304376}, {code,7748964}, {ets,727984}]}, {system_stats,[{cpu_utilization_rate,0.0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.5"}, {mnesia,"4.4.17"}, {kernel,"2.14.3"}, {sasl,"2.1.9.3"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,467}, {memory_data,{17946181632,436477952,{<6058.1196.0>,2172736}}}, {disk_data, [{"/",8256952,15}, {"/dev",8754940,1}, {"/run",3505116,1}, {"/run/lock",5120,0}, {"/run/shm",8762784,0}, {"/mnt",423135208,1}]}, {meminfo, <<"MemTotal: 17525568 kB\nMemFree: 17099436 kB\nBuffers: 98256 kB\nCached: 55904 kB\nSwapCached: 0 kB\nActive: 151656 kB\nInactive: 45260 kB\nActive(anon): 42772 kB\nInactive(anon): 140 kB\nActive(file): 108884 kB\nInactive(file): 45120 kB\nUnevictable: 0 kB\nMlocked: 0 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 28 kB\nWriteback: 0 kB\nAnonPages: 42820 kB\nMapped: 7428 kB\nShmem: 156 kB\nSlab: 18612 kB\nSReclaimable: 12832 kB\nSUnreclaim: 5780 kB\nKernelStack: 840 kB\nPageTables: 1568 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nWritebackTmp: 0 kB\nCommitLimit: 8762784 kB\nCommitted_AS: 236520 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 64000 kB\nVmallocChunk: 34359674252 kB\nHardwareCorrupted: 0 kB\nAnonHugePages: 0 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugePages_Surp: 0\nHugepagesize: 2048 kB\nDirectMap4k: 17920000 kB\nDirectMap2M: 0 kB\n">>}, {system_memory_data, [{system_total_memory,17946181632}, {free_swap,0}, {total_swap,0}, {cached_memory,57245696}, {buffered_memory,100614144}, {free_memory,17509822464}, {total_memory,17946181632}]}, {statistics, [{wall_clock,{457077,0}}, {context_switches,{118273,0}}, {garbage_collection,{14140,36688842,0}}, {io,{{input,19100312},{output,5796125}}}, {reductions,{14948618,295144}}, {run_queue,0}, {runtime,{2230,30}}]}]}, {'ns_1@10.13.41.69', [{last_heard,{1322,679213,262565}}, {active_buckets,[]}, {ready_buckets,[]}, {replication,[{"default",0.0}]}, {memory, [{total,30057456}, {processes,8527512}, {processes_used,8486992}, {system,21529944}, {atom,896777}, {atom_used,871341}, {binary,1179592}, {code,8509446}, {ets,1341304}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.5"}, {mnesia,"4.4.17"}, {inets,"5.5.2"}, {kernel,"2.14.3"}, {sasl,"2.1.9.3"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,342}, {memory_data,{17946181632,508162048,{<6087.545.0>,2172736}}}, {disk_data, [{"/",8256952,15}, {"/dev",8754940,1}, {"/run",3505116,1}, {"/run/lock",5120,0}, {"/run/shm",8762784,0}, {"/mnt",423135208,1}]}, {meminfo, <<"MemTotal: 17525568 kB\nMemFree: 17030068 kB\nBuffers: 98268 kB\nCached: 61580 kB\nSwapCached: 0 kB\nActive: 216820 kB\nInactive: 49272 kB\nActive(anon): 106304 kB\nInactive(anon): 140 kB\nActive(file): 110516 kB\nInactive(file): 49132 kB\nUnevictable: 0 kB\nMlocked: 0 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 164 kB\nWriteback: 0 kB\nAnonPages: 106232 kB\nMapped: 8908 kB\nShmem: 156 kB\nSlab: 18828 kB\nSReclaimable: 12976 kB\nSUnreclaim: 5852 kB\nKernelStack: 872 kB\nPageTables: 1756 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nWritebackTmp: 0 kB\nCommitLimit: 8762784 kB\nCommitted_AS: 332148 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 64000 kB\nVmallocChunk: 34359674252 kB\nHardwareCorrupted: 0 kB\nAnonHugePages: 0 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugePages_Surp: 0\nHugepagesize: 2048 kB\nDirectMap4k: 17920000 kB\nDirectMap2M: 0 kB\n">>}, {system_memory_data, [{system_total_memory,17946181632}, {free_swap,0}, {total_swap,0}, {cached_memory,63057920}, {buffered_memory,100626432}, {free_memory,17438789632}, {total_memory,17946181632}]}, {statistics, [{wall_clock,{331624,1}}, {context_switches,{89358,0}}, {garbage_collection,{15555,63704093,0}}, {io,{{input,9486622},{output,6290172}}}, {reductions,{38243947,2601327}}, {run_queue,1}, {runtime,{3100,160}}]}]}] INFO REPORT <6058.2159.0> 2011-11-30 18:55:33 =============================================================================== ns_1@10.13.41.47:<6058.2159.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2159.0> 2011-11-30 18:55:34 =============================================================================== ns_1@10.13.41.47:<6058.2159.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2159.0> 2011-11-30 18:55:35 =============================================================================== ns_1@10.13.41.47:<6058.2159.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] ERROR REPORT <6058.2159.0> 2011-11-30 18:55:36 =============================================================================== ns_1@10.13.41.47:<6058.2159.0>:ns_janitor:68: Bucket "default" not yet ready on ['ns_1@10.13.41.69'] INFO REPORT <6058.2188.0> 2011-11-30 18:55:42 =============================================================================== ns_1@10.13.41.47:<6058.2188.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2188.0> 2011-11-30 18:55:43 =============================================================================== ns_1@10.13.41.47:<6058.2188.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2188.0> 2011-11-30 18:55:44 =============================================================================== ns_1@10.13.41.47:<6058.2188.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2188.0> 2011-11-30 18:55:45 =============================================================================== ns_1@10.13.41.47:<6058.2188.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] ERROR REPORT <6058.2188.0> 2011-11-30 18:55:46 =============================================================================== ns_1@10.13.41.47:<6058.2188.0>:ns_janitor:68: Bucket "default" not yet ready on ['ns_1@10.13.41.69'] ERROR REPORT <6058.1312.0> 2011-11-30 18:55:49 =============================================================================== ns_1@10.13.41.47:<6058.1312.0>:stats_reader:191: Some nodes didn't respond: ['ns_1@10.13.41.47'] INFO REPORT <6058.2220.0> 2011-11-30 18:55:52 =============================================================================== ns_1@10.13.41.47:<6058.2220.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2220.0> 2011-11-30 18:55:53 =============================================================================== ns_1@10.13.41.47:<6058.2220.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2220.0> 2011-11-30 18:55:54 =============================================================================== ns_1@10.13.41.47:<6058.2220.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2220.0> 2011-11-30 18:55:55 =============================================================================== ns_1@10.13.41.47:<6058.2220.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] ERROR REPORT <6058.2220.0> 2011-11-30 18:55:56 =============================================================================== ns_1@10.13.41.47:<6058.2220.0>:ns_janitor:68: Bucket "default" not yet ready on ['ns_1@10.13.41.69'] INFO REPORT <6058.2248.0> 2011-11-30 18:56:02 =============================================================================== ns_1@10.13.41.47:<6058.2248.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2248.0> 2011-11-30 18:56:03 =============================================================================== ns_1@10.13.41.47:<6058.2248.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2248.0> 2011-11-30 18:56:04 =============================================================================== ns_1@10.13.41.47:<6058.2248.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2248.0> 2011-11-30 18:56:05 =============================================================================== ns_1@10.13.41.47:<6058.2248.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] ERROR REPORT <6058.2248.0> 2011-11-30 18:56:06 =============================================================================== ns_1@10.13.41.47:<6058.2248.0>:ns_janitor:68: Bucket "default" not yet ready on ['ns_1@10.13.41.69'] ERROR REPORT <6058.1312.0> 2011-11-30 18:56:09 =============================================================================== ns_1@10.13.41.47:<6058.1312.0>:stats_reader:191: Some nodes didn't respond: ['ns_1@10.13.41.47'] INFO REPORT <6058.2281.0> 2011-11-30 18:56:12 =============================================================================== ns_1@10.13.41.47:<6058.2281.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2281.0> 2011-11-30 18:56:13 =============================================================================== ns_1@10.13.41.47:<6058.2281.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2281.0> 2011-11-30 18:56:14 =============================================================================== ns_1@10.13.41.47:<6058.2281.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2281.0> 2011-11-30 18:56:15 =============================================================================== ns_1@10.13.41.47:<6058.2281.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] ERROR REPORT <6058.2281.0> 2011-11-30 18:56:16 =============================================================================== ns_1@10.13.41.47:<6058.2281.0>:ns_janitor:68: Bucket "default" not yet ready on ['ns_1@10.13.41.69'] INFO REPORT <6058.2309.0> 2011-11-30 18:56:22 =============================================================================== ns_1@10.13.41.47:<6058.2309.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2309.0> 2011-11-30 18:56:23 =============================================================================== ns_1@10.13.41.47:<6058.2309.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2309.0> 2011-11-30 18:56:24 =============================================================================== ns_1@10.13.41.47:<6058.2309.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2309.0> 2011-11-30 18:56:25 =============================================================================== ns_1@10.13.41.47:<6058.2309.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] ERROR REPORT <6058.2309.0> 2011-11-30 18:56:26 =============================================================================== ns_1@10.13.41.47:<6058.2309.0>:ns_janitor:68: Bucket "default" not yet ready on ['ns_1@10.13.41.69'] ERROR REPORT <6058.1312.0> 2011-11-30 18:56:29 =============================================================================== ns_1@10.13.41.47:<6058.1312.0>:stats_reader:191: Some nodes didn't respond: ['ns_1@10.13.41.47'] INFO REPORT <6058.2355.0> 2011-11-30 18:56:32 =============================================================================== ns_1@10.13.41.47:<6058.2355.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.1191.0> 2011-11-30 18:56:32 =============================================================================== ns_1@10.13.41.47:<6058.1191.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.13.41.47', [{last_heard,{1322,679387,925567}}, {active_buckets,[]}, {ready_buckets,[]}, {replication,[{"default",1.0}]}, {memory, [{total,31612776}, {processes,11445984}, {processes_used,11430712}, {system,20166792}, {atom,807329}, {atom_used,795214}, {binary,1301688}, {code,7748964}, {ets,756288}]}, {system_stats, [{cpu_utilization_rate,0.47961630695443647}, {swap_total,0}, {swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.5"}, {mnesia,"4.4.17"}, {kernel,"2.14.3"}, {sasl,"2.1.9.3"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,527}, {memory_data,{17946181632,435843072,{<6058.1196.0>,2172736}}}, {disk_data, [{"/",8256952,15}, {"/dev",8754940,1}, {"/run",3505116,1}, {"/run/lock",5120,0}, {"/run/shm",8762784,0}, {"/mnt",423135208,1}]}, {meminfo, <<"MemTotal: 17525568 kB\nMemFree: 17099320 kB\nBuffers: 98256 kB\nCached: 55996 kB\nSwapCached: 0 kB\nActive: 152324 kB\nInactive: 45288 kB\nActive(anon): 43376 kB\nInactive(anon): 140 kB\nActive(file): 108948 kB\nInactive(file): 45148 kB\nUnevictable: 0 kB\nMlocked: 0 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 48 kB\nWriteback: 0 kB\nAnonPages: 43428 kB\nMapped: 7428 kB\nShmem: 156 kB\nSlab: 18604 kB\nSReclaimable: 12824 kB\nSUnreclaim: 5780 kB\nKernelStack: 840 kB\nPageTables: 1568 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nWritebackTmp: 0 kB\nCommitLimit: 8762784 kB\nCommitted_AS: 237108 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 64000 kB\nVmallocChunk: 34359674252 kB\nHardwareCorrupted: 0 kB\nAnonHugePages: 0 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugePages_Surp: 0\nHugepagesize: 2048 kB\nDirectMap4k: 17920000 kB\nDirectMap2M: 0 kB\n">>}, {system_memory_data, [{system_total_memory,17946181632}, {free_swap,0}, {total_swap,0}, {cached_memory,57339904}, {buffered_memory,100614144}, {free_memory,17509703680}, {total_memory,17946181632}]}, {statistics, [{wall_clock,{517078,1}}, {context_switches,{125639,0}}, {garbage_collection,{15164,40521086,0}}, {io,{{input,19170642},{output,5914566}}}, {reductions,{16055137,290923}}, {run_queue,0}, {runtime,{2370,20}}]}]}, {'ns_1@10.13.41.69', [{last_heard,{1322,679213,262565}}, {active_buckets,[]}, {ready_buckets,[]}, {replication,[{"default",0.0}]}, {memory, [{total,30057456}, {processes,8527512}, {processes_used,8486992}, {system,21529944}, {atom,896777}, {atom_used,871341}, {binary,1179592}, {code,8509446}, {ets,1341304}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.5"}, {mnesia,"4.4.17"}, {inets,"5.5.2"}, {kernel,"2.14.3"}, {sasl,"2.1.9.3"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,342}, {memory_data,{17946181632,508162048,{<6087.545.0>,2172736}}}, {disk_data, [{"/",8256952,15}, {"/dev",8754940,1}, {"/run",3505116,1}, {"/run/lock",5120,0}, {"/run/shm",8762784,0}, {"/mnt",423135208,1}]}, {meminfo, <<"MemTotal: 17525568 kB\nMemFree: 17030068 kB\nBuffers: 98268 kB\nCached: 61580 kB\nSwapCached: 0 kB\nActive: 216820 kB\nInactive: 49272 kB\nActive(anon): 106304 kB\nInactive(anon): 140 kB\nActive(file): 110516 kB\nInactive(file): 49132 kB\nUnevictable: 0 kB\nMlocked: 0 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 164 kB\nWriteback: 0 kB\nAnonPages: 106232 kB\nMapped: 8908 kB\nShmem: 156 kB\nSlab: 18828 kB\nSReclaimable: 12976 kB\nSUnreclaim: 5852 kB\nKernelStack: 872 kB\nPageTables: 1756 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nWritebackTmp: 0 kB\nCommitLimit: 8762784 kB\nCommitted_AS: 332148 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 64000 kB\nVmallocChunk: 34359674252 kB\nHardwareCorrupted: 0 kB\nAnonHugePages: 0 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugePages_Surp: 0\nHugepagesize: 2048 kB\nDirectMap4k: 17920000 kB\nDirectMap2M: 0 kB\n">>}, {system_memory_data, [{system_total_memory,17946181632}, {free_swap,0}, {total_swap,0}, {cached_memory,63057920}, {buffered_memory,100626432}, {free_memory,17438789632}, {total_memory,17946181632}]}, {statistics, [{wall_clock,{331624,1}}, {context_switches,{89358,0}}, {garbage_collection,{15555,63704093,0}}, {io,{{input,9486622},{output,6290172}}}, {reductions,{38243947,2601327}}, {run_queue,1}, {runtime,{3100,160}}]}]}] INFO REPORT <6058.2355.0> 2011-11-30 18:56:33 =============================================================================== ns_1@10.13.41.47:<6058.2355.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2355.0> 2011-11-30 18:56:34 =============================================================================== ns_1@10.13.41.47:<6058.2355.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2355.0> 2011-11-30 18:56:35 =============================================================================== ns_1@10.13.41.47:<6058.2355.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] ERROR REPORT <6058.2355.0> 2011-11-30 18:56:36 =============================================================================== ns_1@10.13.41.47:<6058.2355.0>:ns_janitor:68: Bucket "default" not yet ready on ['ns_1@10.13.41.69'] INFO REPORT <6058.2384.0> 2011-11-30 18:56:42 =============================================================================== ns_1@10.13.41.47:<6058.2384.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2384.0> 2011-11-30 18:56:43 =============================================================================== ns_1@10.13.41.47:<6058.2384.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2384.0> 2011-11-30 18:56:44 =============================================================================== ns_1@10.13.41.47:<6058.2384.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2384.0> 2011-11-30 18:56:45 =============================================================================== ns_1@10.13.41.47:<6058.2384.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] ERROR REPORT <6058.2384.0> 2011-11-30 18:56:46 =============================================================================== ns_1@10.13.41.47:<6058.2384.0>:ns_janitor:68: Bucket "default" not yet ready on ['ns_1@10.13.41.69'] ERROR REPORT <6058.1312.0> 2011-11-30 18:56:50 =============================================================================== ns_1@10.13.41.47:<6058.1312.0>:stats_reader:191: Some nodes didn't respond: ['ns_1@10.13.41.47'] INFO REPORT <6058.2416.0> 2011-11-30 18:56:52 =============================================================================== ns_1@10.13.41.47:<6058.2416.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2416.0> 2011-11-30 18:56:53 =============================================================================== ns_1@10.13.41.47:<6058.2416.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2416.0> 2011-11-30 18:56:54 =============================================================================== ns_1@10.13.41.47:<6058.2416.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2416.0> 2011-11-30 18:56:55 =============================================================================== ns_1@10.13.41.47:<6058.2416.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] ERROR REPORT <6058.2416.0> 2011-11-30 18:56:56 =============================================================================== ns_1@10.13.41.47:<6058.2416.0>:ns_janitor:68: Bucket "default" not yet ready on ['ns_1@10.13.41.69'] INFO REPORT <6058.2444.0> 2011-11-30 18:57:02 =============================================================================== ns_1@10.13.41.47:<6058.2444.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2444.0> 2011-11-30 18:57:03 =============================================================================== ns_1@10.13.41.47:<6058.2444.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2444.0> 2011-11-30 18:57:04 =============================================================================== ns_1@10.13.41.47:<6058.2444.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2444.0> 2011-11-30 18:57:05 =============================================================================== ns_1@10.13.41.47:<6058.2444.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] ERROR REPORT <6058.2444.0> 2011-11-30 18:57:06 =============================================================================== ns_1@10.13.41.47:<6058.2444.0>:ns_janitor:68: Bucket "default" not yet ready on ['ns_1@10.13.41.69'] ERROR REPORT <6058.1312.0> 2011-11-30 18:57:10 =============================================================================== ns_1@10.13.41.47:<6058.1312.0>:stats_reader:191: Some nodes didn't respond: ['ns_1@10.13.41.47'] INFO REPORT <6058.2477.0> 2011-11-30 18:57:12 =============================================================================== ns_1@10.13.41.47:<6058.2477.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2477.0> 2011-11-30 18:57:13 =============================================================================== ns_1@10.13.41.47:<6058.2477.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2477.0> 2011-11-30 18:57:14 =============================================================================== ns_1@10.13.41.47:<6058.2477.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2477.0> 2011-11-30 18:57:15 =============================================================================== ns_1@10.13.41.47:<6058.2477.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] ERROR REPORT <6058.2477.0> 2011-11-30 18:57:16 =============================================================================== ns_1@10.13.41.47:<6058.2477.0>:ns_janitor:68: Bucket "default" not yet ready on ['ns_1@10.13.41.69'] INFO REPORT <6058.2505.0> 2011-11-30 18:57:22 =============================================================================== ns_1@10.13.41.47:<6058.2505.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2505.0> 2011-11-30 18:57:23 =============================================================================== ns_1@10.13.41.47:<6058.2505.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2505.0> 2011-11-30 18:57:24 =============================================================================== ns_1@10.13.41.47:<6058.2505.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2505.0> 2011-11-30 18:57:25 =============================================================================== ns_1@10.13.41.47:<6058.2505.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] ERROR REPORT <6058.2505.0> 2011-11-30 18:57:26 =============================================================================== ns_1@10.13.41.47:<6058.2505.0>:ns_janitor:68: Bucket "default" not yet ready on ['ns_1@10.13.41.69'] ERROR REPORT <6058.1312.0> 2011-11-30 18:57:30 =============================================================================== ns_1@10.13.41.47:<6058.1312.0>:stats_reader:191: Some nodes didn't respond: ['ns_1@10.13.41.47'] INFO REPORT <6058.2538.0> 2011-11-30 18:57:32 =============================================================================== ns_1@10.13.41.47:<6058.2538.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.1191.0> 2011-11-30 18:57:32 =============================================================================== ns_1@10.13.41.47:<6058.1191.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.13.41.47', [{last_heard,{1322,679447,925628}}, {active_buckets,[]}, {ready_buckets,[]}, {replication,[{"default",1.0}]}, {memory, [{total,31760168}, {processes,11580960}, {processes_used,11565688}, {system,20179208}, {atom,807329}, {atom_used,795214}, {binary,1279584}, {code,7748964}, {ets,788520}]}, {system_stats,[{cpu_utilization_rate,0.0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.5"}, {mnesia,"4.4.17"}, {kernel,"2.14.3"}, {sasl,"2.1.9.3"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,587}, {memory_data,{17946181632,436756480,{<6058.1196.0>,2172736}}}, {disk_data, [{"/",8256952,15}, {"/dev",8754940,1}, {"/run",3505116,1}, {"/run/lock",5120,0}, {"/run/shm",8762784,0}, {"/mnt",423135208,1}]}, {meminfo, <<"MemTotal: 17525568 kB\nMemFree: 17099180 kB\nBuffers: 98256 kB\nCached: 56076 kB\nSwapCached: 0 kB\nActive: 152416 kB\nInactive: 45380 kB\nActive(anon): 43476 kB\nInactive(anon): 140 kB\nActive(file): 108940 kB\nInactive(file): 45240 kB\nUnevictable: 0 kB\nMlocked: 0 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 20 kB\nWriteback: 0 kB\nAnonPages: 43628 kB\nMapped: 7428 kB\nShmem: 156 kB\nSlab: 18624 kB\nSReclaimable: 12844 kB\nSUnreclaim: 5780 kB\nKernelStack: 840 kB\nPageTables: 1568 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nWritebackTmp: 0 kB\nCommitLimit: 8762784 kB\nCommitted_AS: 237108 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 64000 kB\nVmallocChunk: 34359674252 kB\nHardwareCorrupted: 0 kB\nAnonHugePages: 0 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugePages_Surp: 0\nHugepagesize: 2048 kB\nDirectMap4k: 17920000 kB\nDirectMap2M: 0 kB\n">>}, {system_memory_data, [{system_total_memory,17946181632}, {free_swap,0}, {total_swap,0}, {cached_memory,57421824}, {buffered_memory,100614144}, {free_memory,17509560320}, {total_memory,17946181632}]}, {statistics, [{wall_clock,{577078,1}}, {context_switches,{133337,0}}, {garbage_collection,{16319,44566031,0}}, {io,{{input,19483287},{output,6251959}}}, {reductions,{17230322,284842}}, {run_queue,0}, {runtime,{2540,50}}]}]}, {'ns_1@10.13.41.69', [{last_heard,{1322,679213,262565}}, {active_buckets,[]}, {ready_buckets,[]}, {replication,[{"default",0.0}]}, {memory, [{total,30057456}, {processes,8527512}, {processes_used,8486992}, {system,21529944}, {atom,896777}, {atom_used,871341}, {binary,1179592}, {code,8509446}, {ets,1341304}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.5"}, {mnesia,"4.4.17"}, {inets,"5.5.2"}, {kernel,"2.14.3"}, {sasl,"2.1.9.3"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,342}, {memory_data,{17946181632,508162048,{<6087.545.0>,2172736}}}, {disk_data, [{"/",8256952,15}, {"/dev",8754940,1}, {"/run",3505116,1}, {"/run/lock",5120,0}, {"/run/shm",8762784,0}, {"/mnt",423135208,1}]}, {meminfo, <<"MemTotal: 17525568 kB\nMemFree: 17030068 kB\nBuffers: 98268 kB\nCached: 61580 kB\nSwapCached: 0 kB\nActive: 216820 kB\nInactive: 49272 kB\nActive(anon): 106304 kB\nInactive(anon): 140 kB\nActive(file): 110516 kB\nInactive(file): 49132 kB\nUnevictable: 0 kB\nMlocked: 0 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 164 kB\nWriteback: 0 kB\nAnonPages: 106232 kB\nMapped: 8908 kB\nShmem: 156 kB\nSlab: 18828 kB\nSReclaimable: 12976 kB\nSUnreclaim: 5852 kB\nKernelStack: 872 kB\nPageTables: 1756 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nWritebackTmp: 0 kB\nCommitLimit: 8762784 kB\nCommitted_AS: 332148 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 64000 kB\nVmallocChunk: 34359674252 kB\nHardwareCorrupted: 0 kB\nAnonHugePages: 0 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugePages_Surp: 0\nHugepagesize: 2048 kB\nDirectMap4k: 17920000 kB\nDirectMap2M: 0 kB\n">>}, {system_memory_data, [{system_total_memory,17946181632}, {free_swap,0}, {total_swap,0}, {cached_memory,63057920}, {buffered_memory,100626432}, {free_memory,17438789632}, {total_memory,17946181632}]}, {statistics, [{wall_clock,{331624,1}}, {context_switches,{89358,0}}, {garbage_collection,{15555,63704093,0}}, {io,{{input,9486622},{output,6290172}}}, {reductions,{38243947,2601327}}, {run_queue,1}, {runtime,{3100,160}}]}]}] INFO REPORT <6058.2538.0> 2011-11-30 18:57:33 =============================================================================== ns_1@10.13.41.47:<6058.2538.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2538.0> 2011-11-30 18:57:34 =============================================================================== ns_1@10.13.41.47:<6058.2538.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2538.0> 2011-11-30 18:57:35 =============================================================================== ns_1@10.13.41.47:<6058.2538.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] ERROR REPORT <6058.2538.0> 2011-11-30 18:57:36 =============================================================================== ns_1@10.13.41.47:<6058.2538.0>:ns_janitor:68: Bucket "default" not yet ready on ['ns_1@10.13.41.69'] INFO REPORT <6058.2567.0> 2011-11-30 18:57:42 =============================================================================== ns_1@10.13.41.47:<6058.2567.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2567.0> 2011-11-30 18:57:43 =============================================================================== ns_1@10.13.41.47:<6058.2567.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2567.0> 2011-11-30 18:57:44 =============================================================================== ns_1@10.13.41.47:<6058.2567.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2567.0> 2011-11-30 18:57:45 =============================================================================== ns_1@10.13.41.47:<6058.2567.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] ERROR REPORT <6058.2567.0> 2011-11-30 18:57:46 =============================================================================== ns_1@10.13.41.47:<6058.2567.0>:ns_janitor:68: Bucket "default" not yet ready on ['ns_1@10.13.41.69'] ERROR REPORT <6058.1312.0> 2011-11-30 18:57:51 =============================================================================== ns_1@10.13.41.47:<6058.1312.0>:stats_reader:191: Some nodes didn't respond: ['ns_1@10.13.41.47'] INFO REPORT <6058.2599.0> 2011-11-30 18:57:52 =============================================================================== ns_1@10.13.41.47:<6058.2599.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2599.0> 2011-11-30 18:57:53 =============================================================================== ns_1@10.13.41.47:<6058.2599.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2599.0> 2011-11-30 18:57:54 =============================================================================== ns_1@10.13.41.47:<6058.2599.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2599.0> 2011-11-30 18:57:55 =============================================================================== ns_1@10.13.41.47:<6058.2599.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] ERROR REPORT <6058.2599.0> 2011-11-30 18:57:56 =============================================================================== ns_1@10.13.41.47:<6058.2599.0>:ns_janitor:68: Bucket "default" not yet ready on ['ns_1@10.13.41.69'] INFO REPORT <6058.2627.0> 2011-11-30 18:58:02 =============================================================================== ns_1@10.13.41.47:<6058.2627.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2627.0> 2011-11-30 18:58:03 =============================================================================== ns_1@10.13.41.47:<6058.2627.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2627.0> 2011-11-30 18:58:04 =============================================================================== ns_1@10.13.41.47:<6058.2627.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2627.0> 2011-11-30 18:58:05 =============================================================================== ns_1@10.13.41.47:<6058.2627.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] ERROR REPORT <6058.2627.0> 2011-11-30 18:58:06 =============================================================================== ns_1@10.13.41.47:<6058.2627.0>:ns_janitor:68: Bucket "default" not yet ready on ['ns_1@10.13.41.69'] ERROR REPORT <6058.1312.0> 2011-11-30 18:58:11 =============================================================================== ns_1@10.13.41.47:<6058.1312.0>:stats_reader:191: Some nodes didn't respond: ['ns_1@10.13.41.47'] INFO REPORT <6058.2660.0> 2011-11-30 18:58:12 =============================================================================== ns_1@10.13.41.47:<6058.2660.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2660.0> 2011-11-30 18:58:13 =============================================================================== ns_1@10.13.41.47:<6058.2660.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2660.0> 2011-11-30 18:58:14 =============================================================================== ns_1@10.13.41.47:<6058.2660.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2660.0> 2011-11-30 18:58:15 =============================================================================== ns_1@10.13.41.47:<6058.2660.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] ERROR REPORT <6058.2660.0> 2011-11-30 18:58:16 =============================================================================== ns_1@10.13.41.47:<6058.2660.0>:ns_janitor:68: Bucket "default" not yet ready on ['ns_1@10.13.41.69'] INFO REPORT <6058.2688.0> 2011-11-30 18:58:22 =============================================================================== ns_1@10.13.41.47:<6058.2688.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2688.0> 2011-11-30 18:58:23 =============================================================================== ns_1@10.13.41.47:<6058.2688.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2688.0> 2011-11-30 18:58:24 =============================================================================== ns_1@10.13.41.47:<6058.2688.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2688.0> 2011-11-30 18:58:25 =============================================================================== ns_1@10.13.41.47:<6058.2688.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] ERROR REPORT <6058.2688.0> 2011-11-30 18:58:26 =============================================================================== ns_1@10.13.41.47:<6058.2688.0>:ns_janitor:68: Bucket "default" not yet ready on ['ns_1@10.13.41.69'] ERROR REPORT <6058.1312.0> 2011-11-30 18:58:31 =============================================================================== ns_1@10.13.41.47:<6058.1312.0>:stats_reader:191: Some nodes didn't respond: ['ns_1@10.13.41.47'] INFO REPORT <6058.2721.0> 2011-11-30 18:58:32 =============================================================================== ns_1@10.13.41.47:<6058.2721.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.1191.0> 2011-11-30 18:58:32 =============================================================================== ns_1@10.13.41.47:<6058.1191.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.13.41.47', [{last_heard,{1322,679507,924717}}, {active_buckets,[]}, {ready_buckets,[]}, {replication,[{"default",1.0}]}, {memory, [{total,31821992}, {processes,11593736}, {processes_used,11578464}, {system,20228256}, {atom,807329}, {atom_used,795214}, {binary,1298464}, {code,7748964}, {ets,817176}]}, {system_stats, [{cpu_utilization_rate,0.2386634844868735}, {swap_total,0}, {swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.5"}, {mnesia,"4.4.17"}, {kernel,"2.14.3"}, {sasl,"2.1.9.3"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,647}, {memory_data,{17946181632,436748288,{<6058.1196.0>,2172736}}}, {disk_data, [{"/",8256952,15}, {"/dev",8754940,1}, {"/run",3505116,1}, {"/run/lock",5120,0}, {"/run/shm",8762784,0}, {"/mnt",423135208,1}]}, {meminfo, <<"MemTotal: 17525568 kB\nMemFree: 17098940 kB\nBuffers: 98256 kB\nCached: 56168 kB\nSwapCached: 0 kB\nActive: 152460 kB\nInactive: 45388 kB\nActive(anon): 43440 kB\nInactive(anon): 140 kB\nActive(file): 109020 kB\nInactive(file): 45248 kB\nUnevictable: 0 kB\nMlocked: 0 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 28 kB\nWriteback: 0 kB\nAnonPages: 43396 kB\nMapped: 7428 kB\nShmem: 156 kB\nSlab: 18624 kB\nSReclaimable: 12844 kB\nSUnreclaim: 5780 kB\nKernelStack: 832 kB\nPageTables: 1568 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nWritebackTmp: 0 kB\nCommitLimit: 8762784 kB\nCommitted_AS: 237108 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 64000 kB\nVmallocChunk: 34359674252 kB\nHardwareCorrupted: 0 kB\nAnonHugePages: 0 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugePages_Surp: 0\nHugepagesize: 2048 kB\nDirectMap4k: 17920000 kB\nDirectMap2M: 0 kB\n">>}, {system_memory_data, [{system_total_memory,17946181632}, {free_swap,0}, {total_swap,0}, {cached_memory,57516032}, {buffered_memory,100614144}, {free_memory,17509314560}, {total_memory,17946181632}]}, {statistics, [{wall_clock,{637077,0}}, {context_switches,{140723,0}}, {garbage_collection,{17389,48509389,0}}, {io,{{input,19553617},{output,6370430}}}, {reductions,{18390797,299836}}, {run_queue,1}, {runtime,{2690,40}}]}]}, {'ns_1@10.13.41.69', [{last_heard,{1322,679213,262565}}, {active_buckets,[]}, {ready_buckets,[]}, {replication,[{"default",0.0}]}, {memory, [{total,30057456}, {processes,8527512}, {processes_used,8486992}, {system,21529944}, {atom,896777}, {atom_used,871341}, {binary,1179592}, {code,8509446}, {ets,1341304}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.5"}, {mnesia,"4.4.17"}, {inets,"5.5.2"}, {kernel,"2.14.3"}, {sasl,"2.1.9.3"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,342}, {memory_data,{17946181632,508162048,{<6087.545.0>,2172736}}}, {disk_data, [{"/",8256952,15}, {"/dev",8754940,1}, {"/run",3505116,1}, {"/run/lock",5120,0}, {"/run/shm",8762784,0}, {"/mnt",423135208,1}]}, {meminfo, <<"MemTotal: 17525568 kB\nMemFree: 17030068 kB\nBuffers: 98268 kB\nCached: 61580 kB\nSwapCached: 0 kB\nActive: 216820 kB\nInactive: 49272 kB\nActive(anon): 106304 kB\nInactive(anon): 140 kB\nActive(file): 110516 kB\nInactive(file): 49132 kB\nUnevictable: 0 kB\nMlocked: 0 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 164 kB\nWriteback: 0 kB\nAnonPages: 106232 kB\nMapped: 8908 kB\nShmem: 156 kB\nSlab: 18828 kB\nSReclaimable: 12976 kB\nSUnreclaim: 5852 kB\nKernelStack: 872 kB\nPageTables: 1756 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nWritebackTmp: 0 kB\nCommitLimit: 8762784 kB\nCommitted_AS: 332148 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 64000 kB\nVmallocChunk: 34359674252 kB\nHardwareCorrupted: 0 kB\nAnonHugePages: 0 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugePages_Surp: 0\nHugepagesize: 2048 kB\nDirectMap4k: 17920000 kB\nDirectMap2M: 0 kB\n">>}, {system_memory_data, [{system_total_memory,17946181632}, {free_swap,0}, {total_swap,0}, {cached_memory,63057920}, {buffered_memory,100626432}, {free_memory,17438789632}, {total_memory,17946181632}]}, {statistics, [{wall_clock,{331624,1}}, {context_switches,{89358,0}}, {garbage_collection,{15555,63704093,0}}, {io,{{input,9486622},{output,6290172}}}, {reductions,{38243947,2601327}}, {run_queue,1}, {runtime,{3100,160}}]}]}] INFO REPORT <6058.2721.0> 2011-11-30 18:58:33 =============================================================================== ns_1@10.13.41.47:<6058.2721.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2721.0> 2011-11-30 18:58:34 =============================================================================== ns_1@10.13.41.47:<6058.2721.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2721.0> 2011-11-30 18:58:35 =============================================================================== ns_1@10.13.41.47:<6058.2721.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] ERROR REPORT <6058.2721.0> 2011-11-30 18:58:36 =============================================================================== ns_1@10.13.41.47:<6058.2721.0>:ns_janitor:68: Bucket "default" not yet ready on ['ns_1@10.13.41.69'] INFO REPORT <6058.2750.0> 2011-11-30 18:58:42 =============================================================================== ns_1@10.13.41.47:<6058.2750.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2750.0> 2011-11-30 18:58:43 =============================================================================== ns_1@10.13.41.47:<6058.2750.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2750.0> 2011-11-30 18:58:44 =============================================================================== ns_1@10.13.41.47:<6058.2750.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2750.0> 2011-11-30 18:58:45 =============================================================================== ns_1@10.13.41.47:<6058.2750.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] ERROR REPORT <6058.2750.0> 2011-11-30 18:58:46 =============================================================================== ns_1@10.13.41.47:<6058.2750.0>:ns_janitor:68: Bucket "default" not yet ready on ['ns_1@10.13.41.69'] ERROR REPORT <6058.1312.0> 2011-11-30 18:58:51 =============================================================================== ns_1@10.13.41.47:<6058.1312.0>:stats_reader:191: Some nodes didn't respond: ['ns_1@10.13.41.47'] INFO REPORT <6058.2782.0> 2011-11-30 18:58:52 =============================================================================== ns_1@10.13.41.47:<6058.2782.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2782.0> 2011-11-30 18:58:53 =============================================================================== ns_1@10.13.41.47:<6058.2782.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2782.0> 2011-11-30 18:58:54 =============================================================================== ns_1@10.13.41.47:<6058.2782.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2782.0> 2011-11-30 18:58:55 =============================================================================== ns_1@10.13.41.47:<6058.2782.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] ERROR REPORT <6058.2782.0> 2011-11-30 18:58:56 =============================================================================== ns_1@10.13.41.47:<6058.2782.0>:ns_janitor:68: Bucket "default" not yet ready on ['ns_1@10.13.41.69'] INFO REPORT <6058.2810.0> 2011-11-30 18:59:02 =============================================================================== ns_1@10.13.41.47:<6058.2810.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2810.0> 2011-11-30 18:59:03 =============================================================================== ns_1@10.13.41.47:<6058.2810.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2810.0> 2011-11-30 18:59:04 =============================================================================== ns_1@10.13.41.47:<6058.2810.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2810.0> 2011-11-30 18:59:05 =============================================================================== ns_1@10.13.41.47:<6058.2810.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] ERROR REPORT <6058.2810.0> 2011-11-30 18:59:06 =============================================================================== ns_1@10.13.41.47:<6058.2810.0>:ns_janitor:68: Bucket "default" not yet ready on ['ns_1@10.13.41.69'] ERROR REPORT <6058.1312.0> 2011-11-30 18:59:12 =============================================================================== ns_1@10.13.41.47:<6058.1312.0>:stats_reader:191: Some nodes didn't respond: ['ns_1@10.13.41.47'] INFO REPORT <6058.2843.0> 2011-11-30 18:59:12 =============================================================================== ns_1@10.13.41.47:<6058.2843.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2843.0> 2011-11-30 18:59:13 =============================================================================== ns_1@10.13.41.47:<6058.2843.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2843.0> 2011-11-30 18:59:14 =============================================================================== ns_1@10.13.41.47:<6058.2843.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2843.0> 2011-11-30 18:59:15 =============================================================================== ns_1@10.13.41.47:<6058.2843.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] ERROR REPORT <6058.2843.0> 2011-11-30 18:59:16 =============================================================================== ns_1@10.13.41.47:<6058.2843.0>:ns_janitor:68: Bucket "default" not yet ready on ['ns_1@10.13.41.69'] INFO REPORT <6058.2871.0> 2011-11-30 18:59:22 =============================================================================== ns_1@10.13.41.47:<6058.2871.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2871.0> 2011-11-30 18:59:23 =============================================================================== ns_1@10.13.41.47:<6058.2871.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2871.0> 2011-11-30 18:59:24 =============================================================================== ns_1@10.13.41.47:<6058.2871.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2871.0> 2011-11-30 18:59:25 =============================================================================== ns_1@10.13.41.47:<6058.2871.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] ERROR REPORT <6058.2871.0> 2011-11-30 18:59:26 =============================================================================== ns_1@10.13.41.47:<6058.2871.0>:ns_janitor:68: Bucket "default" not yet ready on ['ns_1@10.13.41.69'] ERROR REPORT <6058.1312.0> 2011-11-30 18:59:32 =============================================================================== ns_1@10.13.41.47:<6058.1312.0>:stats_reader:191: Some nodes didn't respond: ['ns_1@10.13.41.47'] INFO REPORT <6058.2918.0> 2011-11-30 18:59:32 =============================================================================== ns_1@10.13.41.47:<6058.2918.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.1191.0> 2011-11-30 18:59:32 =============================================================================== ns_1@10.13.41.47:<6058.1191.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.13.41.47', [{last_heard,{1322,679567,924904}}, {active_buckets,[]}, {ready_buckets,[]}, {replication,[{"default",1.0}]}, {memory, [{total,31848856}, {processes,11578688}, {processes_used,11564432}, {system,20270168}, {atom,807329}, {atom_used,795214}, {binary,1310336}, {code,7748964}, {ets,845728}]}, {system_stats, [{cpu_utilization_rate,0.49875311720698257}, {swap_total,0}, {swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.5"}, {mnesia,"4.4.17"}, {kernel,"2.14.3"}, {sasl,"2.1.9.3"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,707}, {memory_data,{17946181632,436748288,{<6058.1196.0>,2172736}}}, {disk_data, [{"/",8256952,15}, {"/dev",8754940,1}, {"/run",3505116,1}, {"/run/lock",5120,0}, {"/run/shm",8762784,0}, {"/mnt",423135208,1}]}, {meminfo, <<"MemTotal: 17525568 kB\nMemFree: 17088636 kB\nBuffers: 98300 kB\nCached: 57648 kB\nSwapCached: 0 kB\nActive: 163128 kB\nInactive: 43028 kB\nActive(anon): 50228 kB\nInactive(anon): 144 kB\nActive(file): 112900 kB\nInactive(file): 42884 kB\nUnevictable: 0 kB\nMlocked: 0 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 296 kB\nWriteback: 0 kB\nAnonPages: 50276 kB\nMapped: 8720 kB\nShmem: 160 kB\nSlab: 19804 kB\nSReclaimable: 13856 kB\nSUnreclaim: 5948 kB\nKernelStack: 856 kB\nPageTables: 1932 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nWritebackTmp: 0 kB\nCommitLimit: 8762784 kB\nCommitted_AS: 247972 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 64000 kB\nVmallocChunk: 34359674252 kB\nHardwareCorrupted: 0 kB\nAnonHugePages: 0 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugePages_Surp: 0\nHugepagesize: 2048 kB\nDirectMap4k: 17920000 kB\nDirectMap2M: 0 kB\n">>}, {system_memory_data, [{system_total_memory,17946181632}, {free_swap,0}, {total_swap,0}, {cached_memory,59031552}, {buffered_memory,100659200}, {free_memory,17498763264}, {total_memory,17946181632}]}, {statistics, [{wall_clock,{697077,0}}, {context_switches,{148087,0}}, {garbage_collection,{18464,52417117,0}}, {io,{{input,19623947},{output,6488267}}}, {reductions,{19522580,283631}}, {run_queue,0}, {runtime,{2850,30}}]}]}, {'ns_1@10.13.41.69', [{last_heard,{1322,679213,262565}}, {active_buckets,[]}, {ready_buckets,[]}, {replication,[{"default",0.0}]}, {memory, [{total,30057456}, {processes,8527512}, {processes_used,8486992}, {system,21529944}, {atom,896777}, {atom_used,871341}, {binary,1179592}, {code,8509446}, {ets,1341304}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.5"}, {mnesia,"4.4.17"}, {inets,"5.5.2"}, {kernel,"2.14.3"}, {sasl,"2.1.9.3"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,342}, {memory_data,{17946181632,508162048,{<6087.545.0>,2172736}}}, {disk_data, [{"/",8256952,15}, {"/dev",8754940,1}, {"/run",3505116,1}, {"/run/lock",5120,0}, {"/run/shm",8762784,0}, {"/mnt",423135208,1}]}, {meminfo, <<"MemTotal: 17525568 kB\nMemFree: 17030068 kB\nBuffers: 98268 kB\nCached: 61580 kB\nSwapCached: 0 kB\nActive: 216820 kB\nInactive: 49272 kB\nActive(anon): 106304 kB\nInactive(anon): 140 kB\nActive(file): 110516 kB\nInactive(file): 49132 kB\nUnevictable: 0 kB\nMlocked: 0 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 164 kB\nWriteback: 0 kB\nAnonPages: 106232 kB\nMapped: 8908 kB\nShmem: 156 kB\nSlab: 18828 kB\nSReclaimable: 12976 kB\nSUnreclaim: 5852 kB\nKernelStack: 872 kB\nPageTables: 1756 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nWritebackTmp: 0 kB\nCommitLimit: 8762784 kB\nCommitted_AS: 332148 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 64000 kB\nVmallocChunk: 34359674252 kB\nHardwareCorrupted: 0 kB\nAnonHugePages: 0 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugePages_Surp: 0\nHugepagesize: 2048 kB\nDirectMap4k: 17920000 kB\nDirectMap2M: 0 kB\n">>}, {system_memory_data, [{system_total_memory,17946181632}, {free_swap,0}, {total_swap,0}, {cached_memory,63057920}, {buffered_memory,100626432}, {free_memory,17438789632}, {total_memory,17946181632}]}, {statistics, [{wall_clock,{331624,1}}, {context_switches,{89358,0}}, {garbage_collection,{15555,63704093,0}}, {io,{{input,9486622},{output,6290172}}}, {reductions,{38243947,2601327}}, {run_queue,1}, {runtime,{3100,160}}]}]}] INFO REPORT <6058.2918.0> 2011-11-30 18:59:33 =============================================================================== ns_1@10.13.41.47:<6058.2918.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2918.0> 2011-11-30 18:59:34 =============================================================================== ns_1@10.13.41.47:<6058.2918.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2918.0> 2011-11-30 18:59:35 =============================================================================== ns_1@10.13.41.47:<6058.2918.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] ERROR REPORT <6058.2918.0> 2011-11-30 18:59:36 =============================================================================== ns_1@10.13.41.47:<6058.2918.0>:ns_janitor:68: Bucket "default" not yet ready on ['ns_1@10.13.41.69'] INFO REPORT <6058.2947.0> 2011-11-30 18:59:42 =============================================================================== ns_1@10.13.41.47:<6058.2947.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2947.0> 2011-11-30 18:59:43 =============================================================================== ns_1@10.13.41.47:<6058.2947.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2947.0> 2011-11-30 18:59:44 =============================================================================== ns_1@10.13.41.47:<6058.2947.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2947.0> 2011-11-30 18:59:45 =============================================================================== ns_1@10.13.41.47:<6058.2947.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] ERROR REPORT <6058.2947.0> 2011-11-30 18:59:46 =============================================================================== ns_1@10.13.41.47:<6058.2947.0>:ns_janitor:68: Bucket "default" not yet ready on ['ns_1@10.13.41.69'] ERROR REPORT <6058.2807.0> 2011-11-30 18:59:52 =============================================================================== ns_1@10.13.41.47:<6058.2807.0>:stats_reader:191: Some nodes didn't respond: ['ns_1@10.13.41.47'] INFO REPORT <6058.2978.0> 2011-11-30 18:59:52 =============================================================================== ns_1@10.13.41.47:<6058.2978.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2978.0> 2011-11-30 18:59:53 =============================================================================== ns_1@10.13.41.47:<6058.2978.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2978.0> 2011-11-30 18:59:54 =============================================================================== ns_1@10.13.41.47:<6058.2978.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.2978.0> 2011-11-30 18:59:55 =============================================================================== ns_1@10.13.41.47:<6058.2978.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] ERROR REPORT <6058.2978.0> 2011-11-30 18:59:56 =============================================================================== ns_1@10.13.41.47:<6058.2978.0>:ns_janitor:68: Bucket "default" not yet ready on ['ns_1@10.13.41.69'] INFO REPORT <6058.3006.0> 2011-11-30 19:00:02 =============================================================================== ns_1@10.13.41.47:<6058.3006.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.3006.0> 2011-11-30 19:00:03 =============================================================================== ns_1@10.13.41.47:<6058.3006.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.3006.0> 2011-11-30 19:00:04 =============================================================================== ns_1@10.13.41.47:<6058.3006.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.3006.0> 2011-11-30 19:00:05 =============================================================================== ns_1@10.13.41.47:<6058.3006.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] ERROR REPORT <6058.3006.0> 2011-11-30 19:00:06 =============================================================================== ns_1@10.13.41.47:<6058.3006.0>:ns_janitor:68: Bucket "default" not yet ready on ['ns_1@10.13.41.69'] INFO REPORT <6058.3036.0> 2011-11-30 19:00:12 =============================================================================== ns_1@10.13.41.47:<6058.3036.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] ERROR REPORT <6058.2807.0> 2011-11-30 19:00:12 =============================================================================== ns_1@10.13.41.47:<6058.2807.0>:stats_reader:191: Some nodes didn't respond: ['ns_1@10.13.41.47'] INFO REPORT <6058.3036.0> 2011-11-30 19:00:13 =============================================================================== ns_1@10.13.41.47:<6058.3036.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.3036.0> 2011-11-30 19:00:14 =============================================================================== ns_1@10.13.41.47:<6058.3036.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.3036.0> 2011-11-30 19:00:15 =============================================================================== ns_1@10.13.41.47:<6058.3036.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] ERROR REPORT <6058.3036.0> 2011-11-30 19:00:16 =============================================================================== ns_1@10.13.41.47:<6058.3036.0>:ns_janitor:68: Bucket "default" not yet ready on ['ns_1@10.13.41.69'] INFO REPORT <6058.3068.0> 2011-11-30 19:00:22 =============================================================================== ns_1@10.13.41.47:<6058.3068.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.3068.0> 2011-11-30 19:00:23 =============================================================================== ns_1@10.13.41.47:<6058.3068.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.3068.0> 2011-11-30 19:00:24 =============================================================================== ns_1@10.13.41.47:<6058.3068.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.3068.0> 2011-11-30 19:00:25 =============================================================================== ns_1@10.13.41.47:<6058.3068.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] ERROR REPORT <6058.3068.0> 2011-11-30 19:00:26 =============================================================================== ns_1@10.13.41.47:<6058.3068.0>:ns_janitor:68: Bucket "default" not yet ready on ['ns_1@10.13.41.69'] INFO REPORT <6058.3096.0> 2011-11-30 19:00:32 =============================================================================== ns_1@10.13.41.47:<6058.3096.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.1191.0> 2011-11-30 19:00:32 =============================================================================== ns_1@10.13.41.47:<6058.1191.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.13.41.47', [{last_heard,{1322,679627,925735}}, {active_buckets,[]}, {ready_buckets,[]}, {replication,[{"default",1.0}]}, {memory, [{total,32030936}, {processes,11703840}, {processes_used,11688568}, {system,20327096}, {atom,807329}, {atom_used,795214}, {binary,1332160}, {code,7748964}, {ets,878744}]}, {system_stats,[{cpu_utilization_rate,0.0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.5"}, {mnesia,"4.4.17"}, {kernel,"2.14.3"}, {sasl,"2.1.9.3"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,767}, {memory_data,{17946181632,446656512,{<6058.1196.0>,2172736}}}, {disk_data, [{"/",8256952,15}, {"/dev",8754940,1}, {"/run",3505116,1}, {"/run/lock",5120,0}, {"/run/shm",8762784,0}, {"/mnt",423135208,1}]}, {meminfo, <<"MemTotal: 17525568 kB\nMemFree: 17089256 kB\nBuffers: 98300 kB\nCached: 57336 kB\nSwapCached: 0 kB\nActive: 162752 kB\nInactive: 42836 kB\nActive(anon): 49968 kB\nInactive(anon): 144 kB\nActive(file): 112784 kB\nInactive(file): 42692 kB\nUnevictable: 0 kB\nMlocked: 0 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 24 kB\nWriteback: 0 kB\nAnonPages: 50080 kB\nMapped: 8724 kB\nShmem: 160 kB\nSlab: 19808 kB\nSReclaimable: 13860 kB\nSUnreclaim: 5948 kB\nKernelStack: 856 kB\nPageTables: 1932 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nWritebackTmp: 0 kB\nCommitLimit: 8762784 kB\nCommitted_AS: 247972 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 64000 kB\nVmallocChunk: 34359674252 kB\nHardwareCorrupted: 0 kB\nAnonHugePages: 0 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugePages_Surp: 0\nHugepagesize: 2048 kB\nDirectMap4k: 17920000 kB\nDirectMap2M: 0 kB\n">>}, {system_memory_data, [{system_total_memory,17946181632}, {free_swap,0}, {total_swap,0}, {cached_memory,58712064}, {buffered_memory,100659200}, {free_memory,17499398144}, {total_memory,17946181632}]}, {statistics, [{wall_clock,{757077,0}}, {context_switches,{155841,0}}, {garbage_collection,{19614,56491016,0}}, {io,{{input,19939965},{output,6741483}}}, {reductions,{20752428,310705}}, {run_queue,0}, {runtime,{3080,60}}]}]}, {'ns_1@10.13.41.69', [{last_heard,{1322,679213,262565}}, {active_buckets,[]}, {ready_buckets,[]}, {replication,[{"default",0.0}]}, {memory, [{total,30057456}, {processes,8527512}, {processes_used,8486992}, {system,21529944}, {atom,896777}, {atom_used,871341}, {binary,1179592}, {code,8509446}, {ets,1341304}]}, {system_stats,[{cpu_utilization_rate,0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,1}, {version, [{os_mon,"2.2.5"}, {mnesia,"4.4.17"}, {inets,"5.5.2"}, {kernel,"2.14.3"}, {sasl,"2.1.9.3"}, {ns_server,"1.7.2r-20-g6604356"}, {stdlib,"1.17.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,342}, {memory_data,{17946181632,508162048,{<6087.545.0>,2172736}}}, {disk_data, [{"/",8256952,15}, {"/dev",8754940,1}, {"/run",3505116,1}, {"/run/lock",5120,0}, {"/run/shm",8762784,0}, {"/mnt",423135208,1}]}, {meminfo, <<"MemTotal: 17525568 kB\nMemFree: 17030068 kB\nBuffers: 98268 kB\nCached: 61580 kB\nSwapCached: 0 kB\nActive: 216820 kB\nInactive: 49272 kB\nActive(anon): 106304 kB\nInactive(anon): 140 kB\nActive(file): 110516 kB\nInactive(file): 49132 kB\nUnevictable: 0 kB\nMlocked: 0 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 164 kB\nWriteback: 0 kB\nAnonPages: 106232 kB\nMapped: 8908 kB\nShmem: 156 kB\nSlab: 18828 kB\nSReclaimable: 12976 kB\nSUnreclaim: 5852 kB\nKernelStack: 872 kB\nPageTables: 1756 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nWritebackTmp: 0 kB\nCommitLimit: 8762784 kB\nCommitted_AS: 332148 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 64000 kB\nVmallocChunk: 34359674252 kB\nHardwareCorrupted: 0 kB\nAnonHugePages: 0 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugePages_Surp: 0\nHugepagesize: 2048 kB\nDirectMap4k: 17920000 kB\nDirectMap2M: 0 kB\n">>}, {system_memory_data, [{system_total_memory,17946181632}, {free_swap,0}, {total_swap,0}, {cached_memory,63057920}, {buffered_memory,100626432}, {free_memory,17438789632}, {total_memory,17946181632}]}, {statistics, [{wall_clock,{331624,1}}, {context_switches,{89358,0}}, {garbage_collection,{15555,63704093,0}}, {io,{{input,9486622},{output,6290172}}}, {reductions,{38243947,2601327}}, {run_queue,1}, {runtime,{3100,160}}]}]}] ERROR REPORT <6058.2807.0> 2011-11-30 19:00:33 =============================================================================== ns_1@10.13.41.47:<6058.2807.0>:stats_reader:191: Some nodes didn't respond: ['ns_1@10.13.41.47'] INFO REPORT <6058.3096.0> 2011-11-30 19:00:33 =============================================================================== ns_1@10.13.41.47:<6058.3096.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.3096.0> 2011-11-30 19:00:34 =============================================================================== ns_1@10.13.41.47:<6058.3096.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.3096.0> 2011-11-30 19:00:35 =============================================================================== ns_1@10.13.41.47:<6058.3096.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] ERROR REPORT <6058.3096.0> 2011-11-30 19:00:36 =============================================================================== ns_1@10.13.41.47:<6058.3096.0>:ns_janitor:68: Bucket "default" not yet ready on ['ns_1@10.13.41.69'] INFO REPORT <6058.3131.0> 2011-11-30 19:00:42 =============================================================================== ns_1@10.13.41.47:<6058.3131.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.3131.0> 2011-11-30 19:00:43 =============================================================================== ns_1@10.13.41.47:<6058.3131.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.3131.0> 2011-11-30 19:00:44 =============================================================================== ns_1@10.13.41.47:<6058.3131.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] INFO REPORT <6058.3131.0> 2011-11-30 19:00:45 =============================================================================== ns_1@10.13.41.47:<6058.3131.0>:ns_janitor:308: Waiting for "default" on ['ns_1@10.13.41.69'] ERROR REPORT <6058.3131.0> 2011-11-30 19:00:46 =============================================================================== ns_1@10.13.41.47:<6058.3131.0>:ns_janitor:68: Bucket "default" not yet ready on ['ns_1@10.13.41.69']