Hello,
We are deploying couchbase 6.0.0 community version on aws using docker. We have a main server running. We then spin off a second EC2 is a second subset and run the bellow join command:
"/opt/couchbase/bin/couchbase-cli server-add -c "${hname1}" --username ${uname} --password ${password} --server-add "${hname}" --server-add-username {uname} --server-add-password ${password}
We get the following error:
ERROR: Join completion call failed. Got HTTP status 500 from REST call post to http://ip-10-77-60-123.ec2.internal:8091/completeJoin. Body was: "[\"Unexpected server error, request logged.\"]"
Looking into the error.log file, I see this error:
[cluster:error,2021-04-27T15:44:07.414Z,ns_1@ip-10-77-60-123.ec2.internal:ns_cluster<0.152.0>:ns_cluster:perform_actual_join:1114]Error during join: {error,
{badmatch,
{error,
[{'ns_1@172.17.0.2',
{'EXIT',
{{nodedown,'ns_1@172.17.0.2'},
{gen_server,call,
[{ns_config_remote,'ns_1@172.17.0.2'},
get_compressed,10000]}}}}]}},
[{ns_cluster,perform_actual_join,2,
[{file,"src/ns_cluster.erl"},{line,1106}]},
{ns_cluster,do_complete_join,1,
[{file,"src/ns_cluster.erl"},{line,1040}]},
{ns_cluster,handle_call,3,
[{file,"src/ns_cluster.erl"},{line,187}]},
{gen_server,handle_msg,5,
[{file,"gen_server.erl"},{line,585}]},
{proc_lib,init_p_do_apply,3,
[{file,"proc_lib.erl"},{line,239}]}]}
[ns_server:error,2021-04-27T15:44:08.816Z,ns_1@ip-10-77-60-123.ec2.internal:<0.1759.0>:menelaus_web:loop:143]Server error during processing: ["web request failed",
{path,"/completeJoin"},
{method,'POST'},
{type,exit},
{what,
{{{badmatch,
{error,
[{'ns_1@172.17.0.2',
{'EXIT',
{{nodedown,'ns_1@172.17.0.2'},
{gen_server,call,
[{ns_config_remote,
'ns_1@172.17.0.2'},
get_compressed,10000]}}}}]}},
[{ns_cluster,perform_actual_join,2,
[{file,"src/ns_cluster.erl"},
{line,1106}]},
{ns_cluster,do_complete_join,1,
[{file,"src/ns_cluster.erl"},
{line,1040}]},
{ns_cluster,handle_call,3,
[{file,"src/ns_cluster.erl"},
{line,187}]},
{gen_server,handle_msg,5,
[{file,"gen_server.erl"},{line,585}]},
{proc_lib,init_p_do_apply,3,
[{file,"proc_lib.erl"},{line,239}]}]},
{gen_server,call,
[ns_cluster,
{complete_join,
[{<<"targetNode">>,
<<"ns_1@[redacted]">>},
{<<"requestedServices">>,[<<"kv">>]},
{<<"availableStorage">>,
{struct,
[{<<"hdd">>,
[{struct,
[{<<"path">>,<<"/">>},
{<<"sizeKBytes">>,67096556},
{<<"usagePercent">>,6}]},
{struct,
[{<<"path">>,<<"/dev">>},
{<<"sizeKBytes">>,65536},
{<<"usagePercent">>,0}]},
{struct,
[{<<"path">>,
<<"/sys/fs/cgroup">>},
{<<"sizeKBytes">>,3932480},
{<<"usagePercent">>,0}]},
{struct,
[{<<"path">>,<<"/dev/shm">>},
{<<"sizeKBytes">>,65536},
{<<"usagePercent">>,0}]},
{struct,
[{<<"path">>,
<<"/etc/resolv.conf">>},
{<<"sizeKBytes">>,67096556},
{<<"usagePercent">>,6}]},
{struct,
[{<<"path">>,<<"/etc/hostname">>},
{<<"sizeKBytes">>,67096556},
{<<"usagePercent">>,6}]},
{struct,
[{<<"path">>,<<"/etc/hosts">>},
{<<"sizeKBytes">>,67096556},
{<<"usagePercent">>,6}]},
{struct,
[{<<"path">>,
<<"/opt/couchbase/var">>},
{<<"sizeKBytes">>,20511312},
{<<"usagePercent">>,4}]},
{struct,
[{<<"path">>,<<"/proc/acpi">>},
{<<"sizeKBytes">>,3932480},
{<<"usagePercent">>,0}]},
{struct,
[{<<"path">>,<<"/proc/kcore">>},
{<<"sizeKBytes">>,65536},
{<<"usagePercent">>,0}]},
{struct,
[{<<"path">>,<<"/proc/keys">>},
{<<"sizeKBytes">>,65536},
{<<"usagePercent">>,0}]},
{struct,
[{<<"path">>,
<<"/proc/latency_stats">>},
{<<"sizeKBytes">>,65536},
{<<"usagePercent">>,0}]},
{struct,
[{<<"path">>,
<<"/proc/timer_list">>},
{<<"sizeKBytes">>,65536},
{<<"usagePercent">>,0}]},
{struct,
[{<<"path">>,
<<"/proc/sched_debug">>},
{<<"sizeKBytes">>,65536},
{<<"usagePercent">>,0}]},
{struct,
[{<<"path">>,<<"/sys/firmware">>},
{<<"sizeKBytes">>,3932480},
{<<"usagePercent">>,0}]}]}]}},
{<<"storageTotals">>,
{struct,
[{<<"ram">>,
{struct,
[{<<"total">>,8053723136},
{<<"quotaTotal">>,4311744512},
{<<"quotaUsed">>,2151677952},
{<<"used">>,5611556864},
{<<"usedByData">>,62106936},
{<<"quotaUsedPerNode">>,
2151677952},
{<<"quotaTotalPerNode">>,
4311744512}]}},
{<<"hdd">>,
{struct,
[{<<"total">>,21003583488},
{<<"quotaTotal">>,21003583488},
{<<"used">>,840143339},
{<<"usedByData">>,78508018},
{<<"free">>,20163440149}]}}]}},
{<<"storage">>,
{struct,
[{<<"ssd">>,[]},
{<<"hdd">>,
[{struct,
[{<<"path">>,
<<"/opt/couchbase/var/lib/couchbase/data">>},
{<<"index_path">>,
<<"/opt/couchbase/var/lib/couchbase/data">>},
{<<"cbas_dirs">>,
[<<"/opt/couchbase/var/lib/couchbase/data">>]},
{<<"java_home">>,<<>>},
{<<"quotaMb">>,<<"none">>},
{<<"state">>,<<"ok">>}]}]}]}},
{<<"systemStats">>,
{struct,
[{<<"cpu_utilization_rate">>,
11.11111111111111},
{<<"swap_total">>,0},
{<<"swap_used">>,0},
{<<"mem_total">>,8053723136},
{<<"mem_free">>,5316153344}]}},
{<<"interestingStats">>,
{struct,
[{<<"cmd_get">>,0.0},
{<<"couch_docs_actual_disk_size">>,
78508018},
{<<"couch_docs_data_size">>,
42178560},
{<<"couch_spatial_data_size">>,0},
{<<"couch_spatial_disk_size">>,0},
{<<"couch_views_actual_disk_size">>,
0},
{<<"couch_views_data_size">>,0},
{<<"curr_items">>,0},
{<<"curr_items_tot">>,0},
{<<"ep_bg_fetched">>,0.0},
{<<"get_hits">>,0.0},
{<<"mem_used">>,62106936},
{<<"ops">>,0.0},
{<<"vb_active_num_non_resident">>,0},
{<<"vb_replica_curr_items">>,0}]}},
{<<"uptime">>,<<"63454">>},
{<<"memoryTotal">>,8053723136},
{<<"memoryFree">>,5316153344},
{<<"mcdMemoryReserved">>,6144},
{<<"mcdMemoryAllocated">>,6144},
{<<"couchApiBase">>,
<<"http://172.17.0.2:8092/">>},
{<<"otpCookie">>,
<<"f016ac28c49a6f83c4851fb7e5376153e57985dabe7fe3fdced32f1671b9718f">>},
{<<"clusterMembership">>,<<"active">>},
{<<"recoveryType">>,<<"none">>},
{<<"status">>,<<"healthy">>},
{<<"otpNode">>,<<"ns_1@172.17.0.2">>},
{<<"thisNode">>,true},
{<<"hostname">>,<<"172.17.0.2:8091">>},
{<<"clusterCompatibility">>,393216},
{<<"version">>,
<<"6.0.0-1693-community">>},
{<<"os">>,
<<"x86_64-unknown-linux-gnu">>},
{<<"cpuCount">>,2},
{<<"ports">>,
{struct,
[{<<"proxy">>,11211},
{<<"direct">>,11210}]}},
{<<"services">>,
[<<"index">>,<<"kv">>,<<"n1ql">>]},
{<<"memoryQuota">>,4112},
{<<"indexMemoryQuota">>,1028},
{<<"ftsMemoryQuota">>,425},
{<<"cbasMemoryQuota">>,1028},
{<<"eventingMemoryQuota">>,256}]},
120000]}}},
{trace,
[{gen_server,call,3,
[{file,"gen_server.erl"},{line,188}]},
{menelaus_web_cluster,
handle_complete_join,1,
[{file,"src/menelaus_web_cluster.erl"},
{line,131}]},
{request_throttler,do_request,3,
[{file,"src/request_throttler.erl"},
{line,59}]},
{menelaus_web,loop,2,
[{file,"src/menelaus_web.erl"},
{line,121}]},
{mochiweb_http,headers,5,
[{file,
"/home/couchbase/jenkins/workspace/couchbase-server-unix/couchdb/src/mochiweb/mochiweb_http.erl"},
{line,94}]},
{proc_lib,init_p_do_apply,3,
[{file,"proc_lib.erl"},{line,239}]}]}]