Details
-
Type:
Bug
-
Status:
Closed
-
Priority:
Major
-
Resolution: Won't Fix
-
Affects Version/s: 2.0-developer-preview-3
-
Fix Version/s: 2.0-beta
-
Component/s: storage-engine
-
Security Level: Public
-
Labels:
-
Environment:2.0.0r-610
centos 64
10 nodes
eperf write test
Description
Running the eperf write test, one node showed as down. the memcached never crashed, but it was not reachable.
Thread 12 (Thread 0x7f79b6dc3700 (LWP 25082)):
#0 0x00007f79b75b94dd in read () from /lib64/libc.so.6
#1 0x00007f79b7551fd8 in _IO_new_file_underflow () from /lib64/libc.so.6
#2 0x00007f79b7553ade in _IO_default_uflow_internal () from /lib64/libc.so.6
#3 0x00007f79b754efcb in getc () from /lib64/libc.so.6
#4 0x00007f79b6dc4875 in ?? () from /opt/couchbase/lib/memcached/stdin_term_handler.so
#5 0x00007f79b78777e1 in start_thread () from /lib64/libpthread.so.0
#6 0x00007f79b75c677d in clone () from /lib64/libc.so.6
Thread 11 (Thread 0x7f79b65c2700 (LWP 25083)):
#0 0x00007f79b758bbed in nanosleep () from /lib64/libc.so.6
#1 0x00007f79b758ba60 in sleep () from /lib64/libc.so.6
#2 0x0000000000414288 in ?? ()
#3 0x00007f79b78777e1 in start_thread () from /lib64/libpthread.so.0
#4 0x00007f79b75c677d in clone () from /lib64/libc.so.6
Thread 10 (Thread 0x7f79b5bb7700 (LWP 25084)):
#0 0x00007f79b75c6d73 in epoll_wait () from /lib64/libc.so.6
#1 0x00007f79b840bc28 in ?? () from /opt/couchbase/lib/libevent-2.0.so.5
#2 0x00007f79b83faa4c in event_base_loop () from /opt/couchbase/lib/libevent-2.0.so.5
#3 0x0000000000412b14 in ?? ()
#4 0x00007f79b78777e1 in start_thread () from /lib64/libpthread.so.0
#5 0x00007f79b75c677d in clone () from /lib64/libc.so.6
Thread 9 (Thread 0x7f79b53b6700 (LWP 25085)):
#0 0x00007f79b75c6d73 in epoll_wait () from /lib64/libc.so.6
#1 0x00007f79b840bc28 in ?? () from /opt/couchbase/lib/libevent-2.0.so.5
#2 0x00007f79b83faa4c in event_base_loop () from /opt/couchbase/lib/libevent-2.0.so.5
#3 0x0000000000412b14 in ?? ()
#4 0x00007f79b78777e1 in start_thread () from /lib64/libpthread.so.0
#5 0x00007f79b75c677d in clone () from /lib64/libc.so.6
Thread 8 (Thread 0x7f79b4bb5700 (LWP 25086)):
#0 0x00007f79b75c6d73 in epoll_wait () from /lib64/libc.so.6
#1 0x00007f79b840bc28 in ?? () from /opt/couchbase/lib/libevent-2.0.so.5
#2 0x00007f79b83faa4c in event_base_loop () from /opt/couchbase/lib/libevent-2.0.so.5
#3 0x0000000000412b14 in ?? ()
#4 0x00007f79b78777e1 in start_thread () from /lib64/libpthread.so.0
#5 0x00007f79b75c677d in clone () from /lib64/libc.so.6
Thread 7 (Thread 0x7f79b43b4700 (LWP 25087)):
#0 0x00007f79b75c6d73 in epoll_wait () from /lib64/libc.so.6
#1 0x00007f79b840bc28 in ?? () from /opt/couchbase/lib/libevent-2.0.so.5
#2 0x00007f79b83faa4c in event_base_loop () from /opt/couchbase/lib/libevent-2.0.so.5
#3 0x0000000000412b14 in ?? ()
#4 0x00007f79b78777e1 in start_thread () from /lib64/libpthread.so.0
#5 0x00007f79b75c677d in clone () from /lib64/libc.so.6
Thread 6 (Thread 0x7f79b3bb3700 (LWP 25088)):
#0 0x00007f79b75c6d73 in epoll_wait () from /lib64/libc.so.6
#1 0x00007f79b840bc28 in ?? () from /opt/couchbase/lib/libevent-2.0.so.5
#2 0x00007f79b83faa4c in event_base_loop () from /opt/couchbase/lib/libevent-2.0.so.5
#3 0x0000000000412b14 in ?? ()
#4 0x00007f79b78777e1 in start_thread () from /lib64/libpthread.so.0
#5 0x00007f79b75c677d in clone () from /lib64/libc.so.6
Thread 5 (Thread 0x7f79b2d98700 (LWP 25089)):
#0 0x00007f79b787b3cc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007f79b300dcb2 in Dispatcher::run() () from /opt/couchbase/lib/memcached/ep.so
#2 0x00007f79b300e6d3 in ?? () from /opt/couchbase/lib/memcached/ep.so
#3 0x00007f79b78777e1 in start_thread () from /lib64/libpthread.so.0
#4 0x00007f79b75c677d in clone () from /lib64/libc.so.6
Thread 4 (Thread 0x7f79b2597700 (LWP 25090)):
#0 0x00007f79b75bd123 in poll () from /lib64/libc.so.6
#1 0x00007f79b3096e03 in MemcachedEngine::waitForReadable() () from /opt/couchbase/lib/memcached/ep.so
#2 0x00007f79b3097189 in MemcachedEngine::wait() () from /opt/couchbase/lib/memcached/ep.so
#3 0x00007f79b30973c7 in MemcachedEngine::noop(Callback<bool>&) () from /opt/couchbase/lib/memcached/ep.so
#4 0x00007f79b309c819 in MCKVStore::commit() () from /opt/couchbase/lib/memcached/ep.so
#5 0x00007f79b30121d7 in TransactionContext::commit() () from /opt/couchbase/lib/memcached/ep.so
#6 0x00007f79b301cb86 in EventuallyPersistentStore::flushSome(std::queue<RCPtr<QueuedItem>, std::deque<RCPtr<QueuedItem>, std::allocator<RCPtr<QueuedItem> > > >*, std::queue<RCPtr<QueuedItem>, std::deque<RCPtr<QueuedItem>, std::allocator<RCPtr<QueuedItem> > > >*) () from /opt/couchbase/lib/memcached/ep.so
#7 0x00007f79b30517f7 in Flusher::doFlush() () from /opt/couchbase/lib/memcached/ep.so
#8 0x00007f79b305209a in Flusher::step(Dispatcher&, std::tr1::shared_ptr<Task>) () from /opt/couchbase/lib/memcached/ep.so
#9 0x00007f79b30529ee in FlusherStepper::callback(Dispatcher&, std::tr1::shared_ptr<Task>) () from /opt/couchbase/lib/memcached/ep.so
#10 0x00007f79b300ed2f in Task::run(Dispatcher&, std::tr1::shared_ptr<Task>) () from /opt/couchbase/lib/memcached/ep.so
#11 0x00007f79b300de81 in Dispatcher::run() () from /opt/couchbase/lib/memcached/ep.so
#12 0x00007f79b300e6d3 in ?? () from /opt/couchbase/lib/memcached/ep.so
#13 0x00007f79b78777e1 in start_thread () from /lib64/libpthread.so.0
#14 0x00007f79b75c677d in clone () from /lib64/libc.so.6
Thread 3 (Thread 0x7f79b1d96700 (LWP 25091)):
#0 0x00007f79b787b74b in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007f79b300c038 in IdleTask::run(Dispatcher&, std::tr1::shared_ptr<Task>) () from /opt/couchbase/lib/memcached/ep.so
#2 0x00007f79b300de81 in Dispatcher::run() () from /opt/couchbase/lib/memcached/ep.so
#3 0x00007f79b300e6d3 in ?? () from /opt/couchbase/lib/memcached/ep.so
#4 0x00007f79b78777e1 in start_thread () from /lib64/libpthread.so.0
#5 0x00007f79b75c677d in clone () from /lib64/libc.so.6
Thread 2 (Thread 0x7f79b0d94700 (LWP 16798)):
#0 0x00007f79b758bbed in nanosleep () from /lib64/libc.so.6
#1 0x00007f79b75bfbf4 in usleep () from /lib64/libc.so.6
#2 0x00007f79b3050f6a in Flusher::wait() () from /opt/couchbase/lib/memcached/ep.so
#3 0x00007f79b3020d10 in EventuallyPersistentStore::~EventuallyPersistentStore() () from /opt/couchbase/lib/memcached/ep.so
#4 0x00007f79b304d2f7 in EventuallyPersistentEngine::~EventuallyPersistentEngine() () from /opt/couchbase/lib/memcached/ep.so
#5 0x00007f79b303b6c9 in ?? () from /opt/couchbase/lib/memcached/ep.so
#6 0x00007f79b5bbc815 in ?? () from /opt/couchbase/lib/memcached/bucket_engine.so
#7 0x00007f79b78777e1 in start_thread () from /lib64/libpthread.so.0
#8 0x00007f79b75c677d in clone () from /lib64/libc.so.6
Thread 1 (Thread 0x7f79b8a3f720 (LWP 25073)):
#0 0x00007f79b75c6d73 in epoll_wait () from /lib64/libc.so.6
#1 0x00007f79b840bc28 in ?? () from /opt/couchbase/lib/libevent-2.0.so.5
#2 0x00007f79b83faa4c in event_base_loop () from /opt/couchbase/lib/libevent-2.0.so.5
#3 0x0000000000408e8b in ?? ()
#4 0x00007f79b74ffcdd in __libc_start_main () from /lib64/libc.so.6
#5 0x0000000000402969 in ?? ()
#6 0x00007fff768176e8 in ?? ()
#7 0x000000000000001c in ?? ()
#8 0x000000000000000f in ?? ()
#9 0x00007fff76818bab in ?? ()
#10 0x00007fff76818bc8 in ?? ()
#11 0x00007fff76818bcb in ?? ()
#12 0x00007fff76818bfe in ?? ()
#13 0x00007fff76818c01 in ?? ()
#14 0x00007fff76818c07 in ?? ()
#15 0x00007fff76818c0a in ?? ()
#16 0x00007fff76818c38 in ?? ()
#17 0x00007fff76818c3b in ?? ()
#18 0x00007fff76818c42 in ?? ()
#19 0x00007fff76818c45 in ?? ()
#20 0x00007fff76818c48 in ?? ()
#21 0x00007fff76818c4e in ?? ()
#22 0x00007fff76818c51 in ?? ()
#23 0x00007fff76818c8c in ?? ()
#24 0x0000000000000000 in ?? ()
Thread 12 (Thread 0x7f79b6dc3700 (LWP 25082)):
#0 0x00007f79b75b94dd in read () from /lib64/libc.so.6
#1 0x00007f79b7551fd8 in _IO_new_file_underflow () from /lib64/libc.so.6
#2 0x00007f79b7553ade in _IO_default_uflow_internal () from /lib64/libc.so.6
#3 0x00007f79b754efcb in getc () from /lib64/libc.so.6
#4 0x00007f79b6dc4875 in ?? () from /opt/couchbase/lib/memcached/stdin_term_handler.so
#5 0x00007f79b78777e1 in start_thread () from /lib64/libpthread.so.0
#6 0x00007f79b75c677d in clone () from /lib64/libc.so.6
Thread 11 (Thread 0x7f79b65c2700 (LWP 25083)):
#0 0x00007f79b758bbed in nanosleep () from /lib64/libc.so.6
#1 0x00007f79b758ba60 in sleep () from /lib64/libc.so.6
#2 0x0000000000414288 in ?? ()
#3 0x00007f79b78777e1 in start_thread () from /lib64/libpthread.so.0
#4 0x00007f79b75c677d in clone () from /lib64/libc.so.6
Thread 10 (Thread 0x7f79b5bb7700 (LWP 25084)):
#0 0x00007f79b75c6d73 in epoll_wait () from /lib64/libc.so.6
#1 0x00007f79b840bc28 in ?? () from /opt/couchbase/lib/libevent-2.0.so.5
#2 0x00007f79b83faa4c in event_base_loop () from /opt/couchbase/lib/libevent-2.0.so.5
#3 0x0000000000412b14 in ?? ()
#4 0x00007f79b78777e1 in start_thread () from /lib64/libpthread.so.0
#5 0x00007f79b75c677d in clone () from /lib64/libc.so.6
Thread 9 (Thread 0x7f79b53b6700 (LWP 25085)):
#0 0x00007f79b75c6d73 in epoll_wait () from /lib64/libc.so.6
#1 0x00007f79b840bc28 in ?? () from /opt/couchbase/lib/libevent-2.0.so.5
#2 0x00007f79b83faa4c in event_base_loop () from /opt/couchbase/lib/libevent-2.0.so.5
#3 0x0000000000412b14 in ?? ()
#4 0x00007f79b78777e1 in start_thread () from /lib64/libpthread.so.0
#5 0x00007f79b75c677d in clone () from /lib64/libc.so.6
Thread 8 (Thread 0x7f79b4bb5700 (LWP 25086)):
#0 0x00007f79b75c6d73 in epoll_wait () from /lib64/libc.so.6
#1 0x00007f79b840bc28 in ?? () from /opt/couchbase/lib/libevent-2.0.so.5
#2 0x00007f79b83faa4c in event_base_loop () from /opt/couchbase/lib/libevent-2.0.so.5
#3 0x0000000000412b14 in ?? ()
#4 0x00007f79b78777e1 in start_thread () from /lib64/libpthread.so.0
#5 0x00007f79b75c677d in clone () from /lib64/libc.so.6
Thread 7 (Thread 0x7f79b43b4700 (LWP 25087)):
#0 0x00007f79b75c6d73 in epoll_wait () from /lib64/libc.so.6
#1 0x00007f79b840bc28 in ?? () from /opt/couchbase/lib/libevent-2.0.so.5
#2 0x00007f79b83faa4c in event_base_loop () from /opt/couchbase/lib/libevent-2.0.so.5
#3 0x0000000000412b14 in ?? ()
#4 0x00007f79b78777e1 in start_thread () from /lib64/libpthread.so.0
#5 0x00007f79b75c677d in clone () from /lib64/libc.so.6
Thread 6 (Thread 0x7f79b3bb3700 (LWP 25088)):
#0 0x00007f79b75c6d73 in epoll_wait () from /lib64/libc.so.6
#1 0x00007f79b840bc28 in ?? () from /opt/couchbase/lib/libevent-2.0.so.5
#2 0x00007f79b83faa4c in event_base_loop () from /opt/couchbase/lib/libevent-2.0.so.5
#3 0x0000000000412b14 in ?? ()
#4 0x00007f79b78777e1 in start_thread () from /lib64/libpthread.so.0
#5 0x00007f79b75c677d in clone () from /lib64/libc.so.6
Thread 5 (Thread 0x7f79b2d98700 (LWP 25089)):
#0 0x00007f79b787b3cc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007f79b300dcb2 in Dispatcher::run() () from /opt/couchbase/lib/memcached/ep.so
#2 0x00007f79b300e6d3 in ?? () from /opt/couchbase/lib/memcached/ep.so
#3 0x00007f79b78777e1 in start_thread () from /lib64/libpthread.so.0
#4 0x00007f79b75c677d in clone () from /lib64/libc.so.6
Thread 4 (Thread 0x7f79b2597700 (LWP 25090)):
#0 0x00007f79b75bd123 in poll () from /lib64/libc.so.6
#1 0x00007f79b3096e03 in MemcachedEngine::waitForReadable() () from /opt/couchbase/lib/memcached/ep.so
#2 0x00007f79b3097189 in MemcachedEngine::wait() () from /opt/couchbase/lib/memcached/ep.so
#3 0x00007f79b30973c7 in MemcachedEngine::noop(Callback<bool>&) () from /opt/couchbase/lib/memcached/ep.so
#4 0x00007f79b309c819 in MCKVStore::commit() () from /opt/couchbase/lib/memcached/ep.so
#5 0x00007f79b30121d7 in TransactionContext::commit() () from /opt/couchbase/lib/memcached/ep.so
#6 0x00007f79b301cb86 in EventuallyPersistentStore::flushSome(std::queue<RCPtr<QueuedItem>, std::deque<RCPtr<QueuedItem>, std::allocator<RCPtr<QueuedItem> > > >*, std::queue<RCPtr<QueuedItem>, std::deque<RCPtr<QueuedItem>, std::allocator<RCPtr<QueuedItem> > > >*) () from /opt/couchbase/lib/memcached/ep.so
#7 0x00007f79b30517f7 in Flusher::doFlush() () from /opt/couchbase/lib/memcached/ep.so
#8 0x00007f79b305209a in Flusher::step(Dispatcher&, std::tr1::shared_ptr<Task>) () from /opt/couchbase/lib/memcached/ep.so
#9 0x00007f79b30529ee in FlusherStepper::callback(Dispatcher&, std::tr1::shared_ptr<Task>) () from /opt/couchbase/lib/memcached/ep.so
#10 0x00007f79b300ed2f in Task::run(Dispatcher&, std::tr1::shared_ptr<Task>) () from /opt/couchbase/lib/memcached/ep.so
#11 0x00007f79b300de81 in Dispatcher::run() () from /opt/couchbase/lib/memcached/ep.so
#12 0x00007f79b300e6d3 in ?? () from /opt/couchbase/lib/memcached/ep.so
#13 0x00007f79b78777e1 in start_thread () from /lib64/libpthread.so.0
#14 0x00007f79b75c677d in clone () from /lib64/libc.so.6
Thread 3 (Thread 0x7f79b1d96700 (LWP 25091)):
#0 0x00007f79b787b74b in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007f79b300c038 in IdleTask::run(Dispatcher&, std::tr1::shared_ptr<Task>) () from /opt/couchbase/lib/memcached/ep.so
#2 0x00007f79b300de81 in Dispatcher::run() () from /opt/couchbase/lib/memcached/ep.so
#3 0x00007f79b300e6d3 in ?? () from /opt/couchbase/lib/memcached/ep.so
#4 0x00007f79b78777e1 in start_thread () from /lib64/libpthread.so.0
#5 0x00007f79b75c677d in clone () from /lib64/libc.so.6
Thread 2 (Thread 0x7f79b0d94700 (LWP 16798)):
#0 0x00007f79b758bbed in nanosleep () from /lib64/libc.so.6
#1 0x00007f79b75bfbf4 in usleep () from /lib64/libc.so.6
#2 0x00007f79b3050f6a in Flusher::wait() () from /opt/couchbase/lib/memcached/ep.so
#3 0x00007f79b3020d10 in EventuallyPersistentStore::~EventuallyPersistentStore() () from /opt/couchbase/lib/memcached/ep.so
#4 0x00007f79b304d2f7 in EventuallyPersistentEngine::~EventuallyPersistentEngine() () from /opt/couchbase/lib/memcached/ep.so
#5 0x00007f79b303b6c9 in ?? () from /opt/couchbase/lib/memcached/ep.so
#6 0x00007f79b5bbc815 in ?? () from /opt/couchbase/lib/memcached/bucket_engine.so
#7 0x00007f79b78777e1 in start_thread () from /lib64/libpthread.so.0
#8 0x00007f79b75c677d in clone () from /lib64/libc.so.6
Thread 1 (Thread 0x7f79b8a3f720 (LWP 25073)):
#0 0x00007f79b75c6d73 in epoll_wait () from /lib64/libc.so.6
#1 0x00007f79b840bc28 in ?? () from /opt/couchbase/lib/libevent-2.0.so.5
#2 0x00007f79b83faa4c in event_base_loop () from /opt/couchbase/lib/libevent-2.0.so.5
#3 0x0000000000408e8b in ?? ()
#4 0x00007f79b74ffcdd in __libc_start_main () from /lib64/libc.so.6
#5 0x0000000000402969 in ?? ()
#6 0x00007fff768176e8 in ?? ()
#7 0x000000000000001c in ?? ()
#8 0x000000000000000f in ?? ()
#9 0x00007fff76818bab in ?? ()
#10 0x00007fff76818bc8 in ?? ()
#11 0x00007fff76818bcb in ?? ()
#12 0x00007fff76818bfe in ?? ()
#13 0x00007fff76818c01 in ?? ()
#14 0x00007fff76818c07 in ?? ()
#15 0x00007fff76818c0a in ?? ()
#16 0x00007fff76818c38 in ?? ()
#17 0x00007fff76818c3b in ?? ()
#18 0x00007fff76818c42 in ?? ()
#19 0x00007fff76818c45 in ?? ()
#20 0x00007fff76818c48 in ?? ()
#21 0x00007fff76818c4e in ?? ()
#22 0x00007fff76818c51 in ?? ()
#23 0x00007fff76818c8c in ?? ()
#24 0x0000000000000000 in ?? ()
[ec2-user@ip-10-68-179-163 ~]$ sudo strace -p 25073
Process 25073 attached - interrupt to quit
clock_gettime(CLOCK_MONOTONIC, {897600, 672201030}) = 0
gettimeofday({1328038071, 598039}, NULL) = 0
epoll_wait(3, {}, 32, 836) = 0
clock_gettime(CLOCK_MONOTONIC, {897601, 508339228}) = 0
gettimeofday({1328038072, 434155}, NULL) = 0
gettimeofday({1328038072, 434261}, NULL) = 0
epoll_wait(3, {}, 32, 1000) = 0
clock_gettime(CLOCK_MONOTONIC, {897602, 508274416}) = 0
gettimeofday({1328038073, 434092}, NULL) = 0
epoll_wait(3, {}, 32, 1) = 0
clock_gettime(CLOCK_MONOTONIC, {897602, 509271562}) = 0
gettimeofday({1328038073, 435099}, NULL) = 0
gettimeofday({1328038073, 435222}, NULL) = 0
epoll_wait(3, {}, 32, 1000) = 0
clock_gettime(CLOCK_MONOTONIC, {897603, 509156161}) = 0
gettimeofday({1328038074, 434973}, NULL) = 0
epoll_wait(3, {}, 32, 1) = 0
clock_gettime(CLOCK_MONOTONIC, {897603, 510275045}) = 0
gettimeofday({1328038074, 436106}, NULL) = 0
gettimeofday({1328038074, 436228}, NULL) = 0
epoll_wait(3, {}, 32, 1000) = 0
clock_gettime(CLOCK_MONOTONIC, {897604, 510170545}) = 0
gettimeofday({1328038075, 435989}, NULL) = 0
epoll_wait(3, {}, 32, 1) = 0
clock_gettime(CLOCK_MONOTONIC, {897604, 511271613}) = 0
gettimeofday({1328038075, 437103}, NULL) = 0
gettimeofday({1328038075, 437223}, NULL) = 0
epoll_wait(3, {}, 32, 1000) = 0
clock_gettime(CLOCK_MONOTONIC, {897605, 511163697}) = 0
gettimeofday({1328038076, 436980}, NULL) = 0
epoll_wait(3, {}, 32, 1) = 0
clock_gettime(CLOCK_MONOTONIC, {897605, 512272309}) = 0
gettimeofday({1328038076, 438103}, NULL) = 0
gettimeofday({1328038076, 438221}, NULL) = 0
epoll_wait(3, {}, 32, 1000) = 0
clock_gettime(CLOCK_MONOTONIC, {897606, 512162653}) = 0
gettimeofday({1328038077, 437979}, NULL) = 0
epoll_wait(3, {}, 32, 1) = 0
clock_gettime(CLOCK_MONOTONIC, {897606, 513266635}) = 0
gettimeofday({1328038077, 439095}, NULL) = 0
gettimeofday({1328038077, 439213}, NULL) = 0
epoll_wait(3, {}, 32, 1000) = 0
clock_gettime(CLOCK_MONOTONIC, {897607, 513266114}) = 0
gettimeofday({1328038078, 439083}, NULL) = 0
gettimeofday({1328038078, 439190}, NULL) = 0
epoll_wait(3, {}, 32, 1000) = 0
clock_gettime(CLOCK_MONOTONIC, {897608, 513259946}) = 0
gettimeofday({1328038079, 439076}, NULL) = 0
epoll_wait(3, {}, 32, 1) = 0
clock_gettime(CLOCK_MONOTONIC, {897608, 514270783}) = 0
gettimeofday({1328038079, 440101}, NULL) = 0
gettimeofday({1328038079, 440220}, NULL) = 0
epoll_wait(3, ^C <unfinished ...>
Process 25073 detached