Hide
added a comment -
As this is an internal crash from ep_engine, the log file won't be helpful.
I've looked at the core dump and saw that the crash was caused by the abort from the C++ hash table library. We didn't see this crash before from the QA regressions, but I need more investigations on which execution path would cause the abort on the hash table library.
Thread 1 (Thread 13379):
#0 0x00007fba5e9eea75 in raise () from /lib/libc.so.6
#1 0x00007fba5e9f25c0 in abort () from /lib/libc.so.6
#2 0x00007fba5e7738e5 in __gnu_cxx::__verbose_terminate_handler() () from /usr/lib/libstdc++.so.6
#3 0x00007fba5e771d16 in ?? () from /usr/lib/libstdc++.so.6
#4 0x00007fba5e771d43 in std::terminate() () from /usr/lib/libstdc++.so.6
#5 0x00007fba5e771dc6 in __cxa_rethrow () from /usr/lib/libstdc++.so.6
#6 0x00007fba5a52e652 in std::tr1::_Hashtable<std::string, std::pair<std::string const, index_entry>, std::allocator<std::pair<std::string const, index_entr
y> >, std::_Select1st<std::pair<std::string const, index_entry> >, std::equal_to<std::string>, std::tr1::hash<std::string>, std::tr1::__detail::_Mod_range_ha
shing, std::tr1::__detail::_Default_ranged_hash, std::tr1::__detail::_Prime_rehash_policy, false, false, true>::_M_insert_bucket(std::pair<std::string const,
index_entry> const&, unsigned long, unsigned long) ()
from /opt/couchbase/lib/memcached/ep.so
#7 0x00007fba5a52e777 in std::tr1::__detail::_Map_base<std::string, std::pair<std::string const, index_entry>, std::_Select1st<std::pair<std::string const,
index_entry> >, true, std::tr1::_Hashtable<std::string, std::pair<std::string const, index_entry>, std::allocator<std::pair<std::string const, index_entry> >
, std::_Select1st<std::pair<std::string const, index_entry> >, std::equal_to<std::string>, std::tr1::hash<std::string>, std::tr1::__detail::_Mod_range_hashin
g, std::tr1::__detail::_Default_ranged_hash, std::tr1::__detail::_Prime_rehash_policy, false, false, true> >::operator[](std::string const&) () from /opt/cou
chbase/lib/memcached/ep.so
#8 0x00007fba5a52aa36 in Checkpoint::queueDirty (this=0xed626280, item=..., checkpointManager=<value optimized out>) at checkpoint.cc:94
#9 0x00007fba5a52b2c6 in CheckpointManager::queueDirty (this=0x35aec80, item=..., vbucket=...) at checkpoint.cc:678
#10 0x00007fba5a53d02e in EventuallyPersistentStore::queueDirty (this=0x2579280, key=..., vbid=<value optimized out>, op=<value optimized out>, value=<value
optimized out>, flags=0, exptime=2147483647, cas=400591098, rowid=745497, tapBackfill=false) at ep.cc:2007
#11 0x00007fba5a53da62 in EventuallyPersistentStore::set (this=0x2579280, item=..., cookie=<value optimized out>, force=<value optimized out>) at ep.cc:606
#12 0x00007fba5a55ee12 in EventuallyPersistentEngine::store (this=0x15e1000, cookie=0x9040b608, itm=0x13b116c0, cas=<value optimized out>, operation=<value o
ptimized out>, vbucket=170) at ep_engine.cc:1692
#13 0x00007fba5d086fed in bucket_store (handle=<value optimized out>, cookie=0x9040b608, itm=0x13b116c0, cas=0x9040b860, operation=OPERATION_SET, vbucket=170
) at bucket_engine.c:1780
#14 0x000000000040eb51 in complete_update_bin (c=0x9040b608) at daemon/memcached.c:1573
#15 complete_nread_binary (c=0x9040b608) at daemon/memcached.c:3353
#16 0x000000000040f394 in complete_nread (c=0x9040b608) at daemon/memcached.c:3471
#17 conn_nread (c=0x9040b608) at daemon/memcached.c:5275
#18 0x0000000000406bb4 in event_handler (fd=<value optimized out>, which=<value optimized out>, arg=0x9040b608) at daemon/memcached.c:5576
#19 0x00007fba5f84926c in event_process_active_single_queue (base=0x150ba00, flags=<value optimized out>) at event.c:1308
#20 event_process_active (base=0x150ba00, flags=<value optimized out>) at event.c:1375
#21 event_base_loop (base=0x150ba00, flags=<value optimized out>) at event.c:1572
#22 0x0000000000414584 in worker_libevent (arg=0x14ed800) at daemon/thread.c:304
#23 0x00007fba5ed449ca in start_thread () from /lib/libpthread.so.0
#24 0x00007fba5eaa170d in clone () from /lib/libc.so.6
#25 0x0000000000000000 in ?? ()
Show
added a comment - As this is an internal crash from ep_engine, the log file won't be helpful.
I've looked at the core dump and saw that the crash was caused by the abort from the C++ hash table library. We didn't see this crash before from the QA regressions, but I need more investigations on which execution path would cause the abort on the hash table library.
Thread 1 (Thread 13379):
#0 0x00007fba5e9eea75 in raise () from /lib/libc.so.6
#1 0x00007fba5e9f25c0 in abort () from /lib/libc.so.6
#2 0x00007fba5e7738e5 in __gnu_cxx::__verbose_terminate_handler() () from /usr/lib/libstdc++.so.6
#3 0x00007fba5e771d16 in ?? () from /usr/lib/libstdc++.so.6
#4 0x00007fba5e771d43 in std::terminate() () from /usr/lib/libstdc++.so.6
#5 0x00007fba5e771dc6 in __cxa_rethrow () from /usr/lib/libstdc++.so.6
#6 0x00007fba5a52e652 in std::tr1::_Hashtable<std::string, std::pair<std::string const, index_entry>, std::allocator<std::pair<std::string const, index_entr
y> >, std::_Select1st<std::pair<std::string const, index_entry> >, std::equal_to<std::string>, std::tr1::hash<std::string>, std::tr1::__detail::_Mod_range_ha
shing, std::tr1::__detail::_Default_ranged_hash, std::tr1::__detail::_Prime_rehash_policy, false, false, true>::_M_insert_bucket(std::pair<std::string const,
index_entry> const&, unsigned long, unsigned long) ()
from /opt/couchbase/lib/memcached/ep.so
#7 0x00007fba5a52e777 in std::tr1::__detail::_Map_base<std::string, std::pair<std::string const, index_entry>, std::_Select1st<std::pair<std::string const,
index_entry> >, true, std::tr1::_Hashtable<std::string, std::pair<std::string const, index_entry>, std::allocator<std::pair<std::string const, index_entry> >
, std::_Select1st<std::pair<std::string const, index_entry> >, std::equal_to<std::string>, std::tr1::hash<std::string>, std::tr1::__detail::_Mod_range_hashin
g, std::tr1::__detail::_Default_ranged_hash, std::tr1::__detail::_Prime_rehash_policy, false, false, true> >::operator[](std::string const&) () from /opt/cou
chbase/lib/memcached/ep.so
#8 0x00007fba5a52aa36 in Checkpoint::queueDirty (this=0xed626280, item=..., checkpointManager=<value optimized out>) at checkpoint.cc:94
#9 0x00007fba5a52b2c6 in CheckpointManager::queueDirty (this=0x35aec80, item=..., vbucket=...) at checkpoint.cc:678
#10 0x00007fba5a53d02e in EventuallyPersistentStore::queueDirty (this=0x2579280, key=..., vbid=<value optimized out>, op=<value optimized out>, value=<value
optimized out>, flags=0, exptime=2147483647, cas=400591098, rowid=745497, tapBackfill=false) at ep.cc:2007
#11 0x00007fba5a53da62 in EventuallyPersistentStore::set (this=0x2579280, item=..., cookie=<value optimized out>, force=<value optimized out>) at ep.cc:606
#12 0x00007fba5a55ee12 in EventuallyPersistentEngine::store (this=0x15e1000, cookie=0x9040b608, itm=0x13b116c0, cas=<value optimized out>, operation=<value o
ptimized out>, vbucket=170) at ep_engine.cc:1692
#13 0x00007fba5d086fed in bucket_store (handle=<value optimized out>, cookie=0x9040b608, itm=0x13b116c0, cas=0x9040b860, operation=OPERATION_SET, vbucket=170
) at bucket_engine.c:1780
#14 0x000000000040eb51 in complete_update_bin (c=0x9040b608) at daemon/memcached.c:1573
#15 complete_nread_binary (c=0x9040b608) at daemon/memcached.c:3353
#16 0x000000000040f394 in complete_nread (c=0x9040b608) at daemon/memcached.c:3471
#17 conn_nread (c=0x9040b608) at daemon/memcached.c:5275
#18 0x0000000000406bb4 in event_handler (fd=<value optimized out>, which=<value optimized out>, arg=0x9040b608) at daemon/memcached.c:5576
#19 0x00007fba5f84926c in event_process_active_single_queue (base=0x150ba00, flags=<value optimized out>) at event.c:1308
#20 event_process_active (base=0x150ba00, flags=<value optimized out>) at event.c:1375
#21 event_base_loop (base=0x150ba00, flags=<value optimized out>) at event.c:1572
#22 0x0000000000414584 in worker_libevent (arg=0x14ed800) at daemon/thread.c:304
#23 0x00007fba5ed449ca in start_thread () from /lib/libpthread.so.0
#24 0x00007fba5eaa170d in clone () from /lib/libc.so.6
#25 0x0000000000000000 in ?? ()