Start multiple replicator can crash on android device

ming · February 2, 2021, 10:37am

when we start multiple replicator at the same time, it mush crash after run 2-3 minute，we use android sdk version is 2.8.1 and client/gateway/server is community version, here is our demoActivity and crash logs.

// demo
public class MultipleCouchTestActivity extends AppCompatActivity {

private static final String TAG = "MultipleCouchTestActivi";

private ActivityMultipleCouchTestBinding binding;
private List<String> testDocIdList = new ArrayList<>();
private int index;
private SyncGateway syncGateway;

@Override
protected void onCreate(@Nullable Bundle savedInstanceState) {
    super.onCreate(savedInstanceState);
    binding = DataBindingUtil.setContentView(this, R.layout.activity_multiple_couch_test);
    binding.start.setOnClickListener(v -> prepareStart());
}

private void prepareStart() {
    GetSyncGatewayRequest request = new GetSyncGatewayRequest(getKSyncBundle().getCloudManager());
    getKSyncBundle().enqueue(request, new RxCallback<GetSyncGatewayRequest>() {
        @Override
        public void onNext(@NonNull GetSyncGatewayRequest getSyncGatewayRequest) {
            syncGateway = getSyncGatewayRequest.getSyncGateway();
            start();
        }
    });
}

private void start(){
    int testDocCount = 200;
    reset();
    for (int i = 0; i < testDocCount; i++) {
        testDocIdList.add(UUID.randomUUID().toString());
    }
    for (int i = 0; i < 10; i++) {
        testDocSync();
    }
}

private void reset() {
    index = -1 ;
    testDocIdList.clear();
}

private void testDocSync() {
    if (CollectionUtils.isNullOrEmpty(testDocIdList)) {
        return;
    }
    index++;
    if (index >= testDocIdList.size()) {
        index = 0;
    }
    Log.d(TAG, "sync doc index: "+ index);
    String documentId = testDocIdList.get(index);
    try {
        startReplicator(documentId);
    } catch (Exception e) {
        e.printStackTrace();
    }
}

private void startReplicator(String documentId) throws CouchbaseLiteException, URISyntaxException {
    CouchbaseLite.init(getApplicationContext());
    DatabaseConfiguration config = new DatabaseConfiguration();
    config.setDirectory(getExternalFilesDir("files") + "/couch");
    Database database = new Database(documentId, config);
    Endpoint targetEndpoint = new URLEndpoint(new URI(syncGateway.getUrl()));
    ReplicatorConfiguration replConfig = new ReplicatorConfiguration(database, targetEndpoint);
    replConfig.setReplicatorType(ReplicatorConfiguration.ReplicatorType.PUSH_AND_PULL);
    replConfig.setContinuous(false);
    List<String> channels = new ArrayList<>();
    channels.add(documentId);
    replConfig.setChannels(channels);
    replConfig.setAuthenticator(new SessionAuthenticator(syncGateway.getSessionID(), syncGateway.getCookieName()));
    Replicator replicator = new Replicator(replConfig);
    replicator.addChangeListener(change -> {
        Log.d(TAG, "sync doc document: "+ documentId + ", state: " + change.getStatus().toString());
        if (change.getStatus().getActivityLevel() == AbstractReplicator.ActivityLevel.STOPPED) {
            testDocSync();
        }
    });
    replicator.start(false);
}

// crash
2021-01-07 11:02:42.232 8493-8493/? 2021-01-07 11:02:42.309 8493-8493/? 2021-01-07 11:02:42.309 8493-8493/? 2021-01-07 11:02:42.309 8493-8493/? 2021-01-07 11:02:42.310 8493-8493/? 2021-01-07 11:02:42.310 8493-8493/? 2021-01-07 11:02:42.310 8493-8493/? 2021-01-07 11:02:42.310 8493-8493/? 2021-01-07 11:02:42.310 8493-8493/? 2021-01-07 11:02:42.310 8493-8493/? 2021-01-07 11:02:42.310 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.310 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.310 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.311 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.543 1143-8496/? 2021-01-07 11:02:42.654 8493-8493/? 2021-01-07 11:02:42.654 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.654 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.654 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.655 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.655 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.655 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.655 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.655 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.655 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.655 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.655 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.655 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.655 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.655 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.655 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.655 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.655 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.655 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.656 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.656 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.656 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.656 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.656 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.656 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.656 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.656 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.656 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.657 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.657 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.658 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.658 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.658 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.658 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.658 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.658 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.658 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.658 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.658 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.658 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.658 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.659 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.659 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.659 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.659 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.659 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.659 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.659 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.659 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.659 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.659 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.659 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.660 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.660 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.660 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.660 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.660 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.660 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.661 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.661 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.662 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.662 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.662 8493-8493/? A/DEBUG: 2021-01-07 11:02:42.662 8493-8493/? A/DEBUG: E/DEBUG: failed to readlink /proc/8489/fd/122: No such file or directory
A/DEBUG: *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** ***
A/DEBUG: Build fingerprint: ‘ONYX/MaxLumi/MaxLumi:10/2021-01-07_00-41_dev_0e390660b/1526:userdebug/dev-keys’
A/DEBUG: Revision: ‘0’
A/DEBUG: ABI: ‘arm’
A/DEBUG: Timestamp: 2021-01-07 11:02:42+0800
A/DEBUG: pid: 8313, tid: 8489, name: nd2boox.com/… >>> com.onyx.android.ksync <<<
A/DEBUG: uid: 10087
A/DEBUG: signal 6 (SIGABRT), code -1 (SI_QUEUE), fault addr --------
A/DEBUG: Abort message: ‘FORTIFY: pthread_mutex_lock called on a destroyed mutex (0xb99c0688)’
r0 00000000 r1 00002129 r2 00000006 r3 b1d99bf8
r4 b1d99c0c r5 b1d99bf0 r6 00002079 r7 0000016b
r8 b1d99c08 r9 b1d99bf8 r10 b1d99c28 r11 b1d99c18
ip 00002129 sp b1d99bc8 lr e66814fb pc e668150e
E/ResolverController: No valid NAT64 prefix (100, /0)
A/DEBUG: backtrace:
#00 pc 0006050e /apex/com.android.runtime/lib/bionic/libc.so (abort+166) (BuildId: 138eaae04428341f0b54cafdabfa7102)
#01 pc 000aba57 /apex/com.android.runtime/lib/bionic/libc.so (__fortify_fatal(char const*, …)+26) (BuildId: 138eaae04428341f0b54cafdabfa7102)
#02 pc 000ab2b1 /apex/com.android.runtime/lib/bionic/libc.so (HandleUsingDestroyedMutex(pthread_mutex_t*, char const*)+20) (BuildId: 138eaae04428341f0b54cafdabfa7102)
#03 pc 000ab1e3 /apex/com.android.runtime/lib/bionic/libc.so (pthread_mutex_lock+210) (BuildId: 138eaae04428341f0b54cafdabfa7102)
#04 pc 00333f99 /system/app/ksync-release/ksync-release.apk!libLiteCoreJNI.so (offset 0x999000) (std::__ndk1::mutex::lock()+4) (BuildId: 4fb2389b824661224921482fc5a27cd5a915fc17)
#05 pc 0020fe89 /system/app/ksync-release/ksync-release.apk!libLiteCoreJNI.so (offset 0x999000) (litecore::websocket::WebSocketImpl::onClose(litecore::websocket::CloseStatus)+48) (BuildId: 4fb2389b824661224921482fc5a27cd5a915fc17)
#06 pc 001916df /system/app/ksync-release/ksync-release.apk!libLiteCoreJNI.so (offset 0x999000) (c4socket_closed+154) (BuildId: 4fb2389b824661224921482fc5a27cd5a915fc17)
#07 pc 00124993 /system/app/ksync-release/ksync-release.apk!libLiteCoreJNI.so (offset 0x999000) (Java_com_couchbase_lite_internal_core_C4Socket_closed+50) (BuildId: 4fb2389b824661224921482fc5a27cd5a915fc17)
#08 pc 000dc519 /apex/com.android.runtime/lib/libart.so (art_quick_generic_jni_trampoline+40) (BuildId: 449420dce782f3ed43e0aacfceaa8ac1)
#09 pc 000d7bc5 /apex/com.android.runtime/lib/libart.so (art_quick_invoke_stub_internal+68) (BuildId: 449420dce782f3ed43e0aacfceaa8ac1)
#10 pc 00436279 /apex/com.android.runtime/lib/libart.so (art_quick_invoke_static_stub+248) (BuildId: 449420dce782f3ed43e0aacfceaa8ac1)
#11 pc 000dffff /apex/com.android.runtime/lib/libart.so (art::ArtMethod::Invoke(art::Thread*, unsigned int*, unsigned int, art::JValue*, char const*)+198) (BuildId: 449420dce782f3ed43e0aacfceaa8ac1)
#12 pc 0021374d /apex/com.android.runtime/lib/libart.so (art::interpreter::ArtInterpreterToCompiledCodeBridge(art::Thread*, art::ArtMethod*, art::ShadowFrame*, unsigned short, art::JValue*)+280) (BuildId: 449420dce782f3ed43e0aacfceaa8ac1)
#13 pc 0020f065 /apex/com.android.runtime/lib/libart.so (bool art::interpreter::DoCall<false, false>(art::ArtMethod*, art::Thread*, art::ShadowFrame&, art::Instruction const*, unsigned short, art::JValue*)+716) (BuildId: 449420dce782f3ed43e0aacfceaa8ac1)
#14 pc 0042d93d /apex/com.android.runtime/lib/libart.so (MterpInvokeStatic+348) (BuildId: 449420dce782f3ed43e0aacfceaa8ac1)
#15 pc 000d2994 /apex/com.android.runtime/lib/libart.so (mterp_op_invoke_static+20) (BuildId: 449420dce782f3ed43e0aacfceaa8ac1)
#16 pc 0031a44a /data/dalvik-cache/arm/system@app@ksync-release@ksync-release.apk@classes.vdex (com.couchbase.lite.internal.core.C4Socket.closed+70)
#17 pc 0042b611 /apex/com.android.runtime/lib/libart.so (MterpInvokeVirtual+1192) (BuildId: 449420dce782f3ed43e0aacfceaa8ac1)
#18 pc 000d2814 /apex/com.android.runtime/lib/libart.so (mterp_op_invoke_virtual+20) (BuildId: 449420dce782f3ed43e0aacfceaa8ac1)
#19 pc 0031f1c0 /data/dalvik-cache/arm/system@app@ksync-release@ksync-release.apk@classes.vdex (com.couchbase.lite.internal.replicator.CBLWebSocket.handleClose+64)
#20 pc 0042b611 /apex/com.android.runtime/lib/libart.so (MterpInvokeVirtual+1192) (BuildId: 449420dce782f3ed43e0aacfceaa8ac1)
#21 pc 000d2814 /apex/com.android.runtime/lib/libart.so (mterp_op_invoke_virtual+20) (BuildId: 449420dce782f3ed43e0aacfceaa8ac1)
#22 pc 0031e84e /data/dalvik-cache/arm/system@app@ksync-release@ksync-release.apk@classes.vdex (com.couchbase.lite.internal.replicator.AbstractCBLWebSocket.didClose+18)
#23 pc 0042d443 /apex/com.android.runtime/lib/libart.so (MterpInvokeDirect+962) (BuildId: 449420dce782f3ed43e0aacfceaa8ac1)
#24 pc 000d2914 /apex/com.android.runtime/lib/libart.so (mterp_op_invoke_direct+20) (BuildId: 449420dce782f3ed43e0aacfceaa8ac1)
#25 pc 0031e788 /data/dalvik-cache/arm/system@app@ksync-release@ksync-release.apk@classes.vdex (com.couchbase.lite.internal.replicator.AbstractCBLWebSocket.access$800)
#26 pc 0042db85 /apex/com.android.runtime/lib/libart.so (MterpInvokeStatic+932) (BuildId: 449420dce782f3ed43e0aacfceaa8ac1)
#27 pc 000d2994 /apex/com.android.runtime/lib/libart.so (mterp_op_invoke_static+20) (BuildId: 449420dce782f3ed43e0aacfceaa8ac1)
#28 pc 0031dd34 /data/dalvik-cache/arm/system@app@ksync-release@ksync-release.apk@classes.vdex (com.couchbase.lite.internal.replicator.AbstractCBLWebSocket$CBLWebSocketListener.onFailure+56)
#29 pc 0042b611 /apex/com.android.runtime/lib/libart.so (MterpInvokeVirtual+1192) (BuildId: 449420dce782f3ed43e0aacfceaa8ac1)
#30 pc 000d2814 /apex/com.android.runtime/lib/libart.so (mterp_op_invoke_virtual+20) (BuildId: 449420dce782f3ed43e0aacfceaa8ac1)
#31 pc 011b34e2 /data/dalvik-cache/arm/system@app@ksync-release@ksync-release.apk@classes.vdex (okhttp3.internal.ws.RealWebSocket.failWebSocket+74)
#32 pc 001eea51 /apex/com.android.runtime/lib/libart.so (_ZN3art11interpreterL7ExecuteEPNS_6ThreadERKNS_20CodeItemDataAccessorERNS_11ShadowFrameENS_6JValueEbb.llvm.18077834472730452097+192) (BuildId: 449420dce782f3ed43e0aacfceaa8ac1)
#33 pc 001f3243 /apex/com.android.runtime/lib/libart.so (art::interpreter::EnterInterpreterFromEntryPoint(art::Thread*, art::CodeItemDataAccessor const&, art::ShadowFrame*)+126) (BuildId: 449420dce782f3ed43e0aacfceaa8ac1)
#34 pc 0041fc4d /apex/com.android.runtime/lib/libart.so (artQuickToInterpreterBridge+852) (BuildId: 449420dce782f3ed43e0aacfceaa8ac1)
#35 pc 000dc5a1 /apex/com.android.runtime/lib/libart.so (art_quick_to_interpreter_bridge+32) (BuildId: 449420dce782f3ed43e0aacfceaa8ac1)
#36 pc 0204efd9 /memfd:/jit-cache (deleted) (okhttp3.internal.ws.RealWebSocket.a+88)
#37 pc 000d7bc5 /apex/com.android.runtime/lib/libart.so (art_quick_invoke_stub_internal+68) (BuildId: 449420dce782f3ed43e0aacfceaa8ac1)
#38 pc 00436165 /apex/com.android.runtime/lib/libart.so (art_quick_invoke_stub+252) (BuildId: 449420dce782f3ed43e0aacfceaa8ac1)
#39 pc 000dffeb /apex/com.android.runtime/lib/libart.so (art::ArtMethod::Invoke(art::Thread*, unsigned int*, unsigned int, art::JValue*, char const*)+178) (BuildId: 449420dce782f3ed43e0aacfceaa8ac1)
#40 pc 0021374d /apex/com.android.runtime/lib/libart.so (art::interpreter::ArtInterpreterToCompiledCodeBridge(art::Thread*, art::ArtMethod*, art::ShadowFrame*, unsigned short, art::JValue*)+280) (BuildId: 449420dce782f3ed43e0aacfceaa8ac1)
#41 pc 0020f065 /apex/com.android.runtime/lib/libart.so (bool art::interpreter::DoCall<false, false>(art::ArtMethod*, art::Thread*, art::ShadowFrame&, art::Instruction const*, unsigned short, art::JValue*)+716) (BuildId: 449420dce782f3ed43e0aacfceaa8ac1)
#42 pc 0042b3c5 /apex/com.android.runtime/lib/libart.so (MterpInvokeVirtual+604) (BuildId: 449420dce782f3ed43e0aacfceaa8ac1)
#43 pc 000d2814 /apex/com.android.runtime/lib/libart.so (mterp_op_invoke_virtual+20) (BuildId: 449420dce782f3ed43e0aacfceaa8ac1)
#44 pc 011b29e4 /data/dalvik-cache/arm/system@app@ksync-release@ksync-release.apk@classes.vdex (okhttp3.internal.ws.a.run+4)
#45 pc 001eea51 /apex/com.android.runtime/lib/libart.so (_ZN3art11interpreterL7ExecuteEPNS_6ThreadERKNS_20CodeItemDataAccessorERNS_11ShadowFrameENS_6JValueEbb.llvm.18077834472730452097+192) (BuildId: 449420dce782f3ed43e0aacfceaa8ac1)
#46 pc 001f3243 /apex/com.android.runtime/lib/libart.so (art::interpreter::EnterInterpreterFromEntryPoint(art::Thread*, art::CodeItemDataAccessor const&, art::ShadowFrame*)+126) (BuildId: 449420dce782f3ed43e0aacfceaa8ac1)
#47 pc 0041fc4d /apex/com.android.runtime/lib/libart.so (artQuickToInterpreterBridge+852) (BuildId: 449420dce782f3ed43e0aacfceaa8ac1)
#48 pc 000dc5a1 /apex/com.android.runtime/lib/libart.so (art_quick_to_interpreter_bridge+32) (BuildId: 449420dce782f3ed43e0aacfceaa8ac1)
#49 pc 001c0189 /system/framework/arm/boot.oat (java.util.concurrent.Executors$RunnableAdapter.call+56) (BuildId: b992db819117955967a605e0e31d95fcf35fb276)
#50 pc 0023c755 /system/framework/arm/boot.oat (java.util.concurrent.FutureTask.run+196) (BuildId: b992db819117955967a605e0e31d95fcf35fb276)
#51 pc 002c0fef /system/framework/arm/boot.oat (java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run+118) (BuildId: b992db819117955967a605e0e31d95fcf35fb276)
#52 pc 0028db67 /system/framework/arm/boot.oat (java.util.concurrent.ThreadPoolExecutor.runWorker+1014) (BuildId: b992db819117955967a605e0e31d95fcf35fb276)
#53 pc 0028b7f7 /system/framework/arm/boot.oat (java.util.concurrent.ThreadPoolExecutor$Worker.run+54) (BuildId: b992db819117955967a605e0e31d95fcf35fb276)
#54 pc 00173641 /system/framework/arm/boot.oat (java.lang.Thread.run+64) (BuildId: b992db819117955967a605e0e31d95fcf35fb276)
#55 pc 000d7bc5 /apex/com.android.runtime/lib/libart.so (art_quick_invoke_stub_internal+68) (BuildId: 449420dce782f3ed43e0aacfceaa8ac1)
#56 pc 00436165 /apex/com.android.runtime/lib/libart.so (art_quick_invoke_stub+252) (BuildId: 449420dce782f3ed43e0aacfceaa8ac1)
#57 pc 000dffeb /apex/com.android.runtime/lib/libart.so (art::ArtMethod::Invoke(art::Thread*, unsigned int*, unsigned int, art::JValue*, char const*)+178) (BuildId: 449420dce782f3ed43e0aacfceaa8ac1)
#58 pc 003772bb /apex/com.android.runtime/lib/libart.so (art::(anonymous namespace)::InvokeWithArgArray(art::ScopedObjectAccessAlreadyRunnable const&, art::ArtMethod*, art::(anonymous namespace)::ArgArray*, art::JValue*, char const*)+54) (BuildId: 449420dce782f3ed43e0aacfceaa8ac1)
#59 pc 00378043 /apex/com.android.runtime/lib/libart.so (art::InvokeVirtualOrInterfaceWithJValues(art::ScopedObjectAccessAlreadyRunnable const&, _jobject*, _jmethodID*, jvalue const*)+306) (BuildId: 449420dce782f3ed43e0aacfceaa8ac1)
#60 pc 003a917f /apex/com.android.runtime/lib/libart.so (art::CreateCallback(void*)+986) (BuildId: 449420dce782f3ed43e0aacfceaa8ac1)
#61 pc 000aa8ab /apex/com.android.runtime/lib/bionic/libc.so (__pthread_start(void*)+20) (BuildId: 138eaae04428341f0b54cafdabfa7102)
#62 pc 000619b3 /apex/com.android.runtime/lib/bionic/libc.so (__start_thread+30) (BuildId: 138eaae04428341f0b54cafdabfa7102)

blake.meike · February 3, 2021, 5:35pm

It looks to me as if the connection is failing (probably because of a firewall/layer 4 router between the mobile device and the SGW). When the connection fails, the mobile app is incorrectly handling the failure, and crashing.

We will soon be releasing version 2.8.4. It addresses exactly some of these issues.

In the main time, you might want to fix whatever is causing the connection to fail.

ming · February 5, 2021, 1:56am

thanks，we waiting new veision to reslove crash issues.

daniel.petersen · March 12, 2021, 7:48pm

Couchbase Lite Android version 2.8.5 is now available.