Fatal error: runtime: out of memory sync gateway

Hi,

I got the"fatal error: runtime: out of memory sync gateway " every time I running the batch job via sync gateway to get a huge amount of json to check and update, any idea how to resolve?

My program is a web application which need to do a batch job to check all the cases to update, currently it will via sync gateway to get the json to check and update (total request may be around 100k - 200k)

Below is my config.

Couchbase Server 5.5
Sync Gateway 2.0
Sync Gateway RAM: 16GB

Error details:

fatal error: runtime: out of memory

runtime stack:
runtime.throw(0xca34a4, 0x16)
/usr/local/go/1.8.5/go/src/runtime/panic.go:596 +0x95
runtime.sysMap(0xc770b90000, 0x100000, 0x7fe592ffcc00, 0x1276af8)
/usr/local/go/1.8.5/go/src/runtime/mem_linux.go:216 +0x1d0
runtime.(*mheap).sysAlloc(0x125dba0, 0x100000, 0x7fe54e7f9570)
/usr/local/go/1.8.5/go/src/runtime/malloc.go:440 +0x374
runtime.(*mheap).grow(0x125dba0, 0x1, 0x0)
/usr/local/go/1.8.5/go/src/runtime/mheap.go:774 +0x62
runtime.(*mheap).allocSpanLocked(0x125dba0, 0x1, 0x7fe592ffcce8)
/usr/local/go/1.8.5/go/src/runtime/mheap.go:678 +0x44f
runtime.(*mheap).alloc_m(0x125dba0, 0x1, 0x4, 0x7fe54e7f95b8)
/usr/local/go/1.8.5/go/src/runtime/mheap.go:562 +0xe2
runtime.(*mheap).alloc.func1()
/usr/local/go/1.8.5/go/src/runtime/mheap.go:627 +0x4b
runtime.systemstack(0x7fe592ffcd58)
/usr/local/go/1.8.5/go/src/runtime/asm_amd64.s:343 +0xab
runtime.(*mheap).alloc(0x125dba0, 0x1, 0x10000000004, 0x7fe54e7f9570)
/usr/local/go/1.8.5/go/src/runtime/mheap.go:628 +0xa0
runtime.(*mcentral).grow(0x125f090, 0x0)
/usr/local/go/1.8.5/go/src/runtime/mcentral.go:212 +0x8e
runtime.(*mcentral).cacheSpan(0x125f090, 0xc42001b828)
/usr/local/go/1.8.5/go/src/runtime/mcentral.go:93 +0x12a
runtime.(*mcache).refill(0x7fe59bfe7960, 0xc700000004, 0xc770b8e820)
/usr/local/go/1.8.5/go/src/runtime/mcache.go:122 +0xa0
runtime.(*mcache).nextFree.func1()
/usr/local/go/1.8.5/go/src/runtime/malloc.go:538 +0x32
runtime.systemstack(0xc4203053c0)
/usr/local/go/1.8.5/go/src/runtime/asm_amd64.s:327 +0x79
runtime.mstart()
/usr/local/go/1.8.5/go/src/runtime/proc.go:1132

goroutine 46879 [running]:
runtime.systemstack_switch()
/usr/local/go/1.8.5/go/src/runtime/asm_amd64.s:281 fp=0xc754f1df58 sp=0xc754f1df50
runtime.(*mcache).nextFree(0x7fe59bfe7960, 0x411f04, 0x20, 0x28, 0xc76eddbfb0)
/usr/local/go/1.8.5/go/src/runtime/malloc.go:539 +0xb9 fp=0xc754f1dfb0 sp=0xc754f1df58
runtime.mallocgc(0x30, 0xc25460, 0xbe4c01, 0xc76eddbfb0)
/usr/local/go/1.8.5/go/src/runtime/malloc.go:691 +0x827 fp=0xc754f1e050 sp=0xc754f1dfb0
runtime.newobject(0xc25460, 0xc76eddbfb0)
/usr/local/go/1.8.5/go/src/runtime/malloc.go:820 +0x38 fp=0xc754f1e080 sp=0xc754f1e050
reflect.Swapper(0xb63020, 0xc770b8a2a0, 0xb63020)
/usr/local/go/1.8.5/go/src/reflect/swapper.go:64 +0x171 fp=0xc754f1e1c0 sp=0xc754f1e080
sort.Slice(0xb63020, 0xc770b8a2a0, 0xc754f1e310)
/usr/local/go/1.8.5/go/src/sort/sort.go:249 +0x6f fp=0xc754f1e220 sp=0xc754f1e1c0
encoding/json.(*mapEncoder).encode(0xc420326048, 0xc75c20cc60, 0xbaf4e0, 0xc76fc09bf0, 0x15, 0xba0100)
/usr/local/go/1.8.5/go/src/encoding/json/encode.go:682 +0x409 fp=0xc754f1e390 sp=0xc754f1e220
encoding/json.(*mapEncoder).(encoding/json.encode)-fm(0xc75c20cc60, 0xbaf4e0, 0xc76fc09bf0, 0x15, 0xc76fc00100)
/usr/local/go/1.8.5/go/src/encoding/json/encode.go:706 +0x64 fp=0xc754f1e3d0 sp=0xc754f1e390
encoding/json.(*encodeState).reflectValue(0xc75c20cc60, 0xbaf4e0, 0xc76fc09bf0, 0x15, 0xc76fc00100)
/usr/local/go/1.8.5/go/src/encoding/json/encode.go:323 +0x82 fp=0xc754f1e408 sp=0xc754f1e3d0
encoding/json.interfaceEncoder(0xc75c20cc60, 0xb9c280, 0xc76fcedf30, 0x194, 0xb90100)
/usr/local/go/1.8.5/go/src/encoding/json/encode.go:617 +0xdb fp=0xc754f1e448 sp=0xc754f1e408
encoding/json.(*arrayEncoder).encode(0xc420326058, 0xc75c20cc60, 0xb658e0, 0xc76fae1ba0, 0x97, 0xc420300100)
/usr/local/go/1.8.5/go/src/encoding/json/encode.go:767 +0xf5 fp=0xc754f1e4a0 sp=0xc754f1e448
encoding/json.(*arrayEncoder).(encoding/json.encode)-fm(0xc75c20cc60, 0xb658e0, 0xc76fae1ba0, 0x97, 0x100)
/usr/local/go/1.8.5/go/src/encoding/json/encode.go:774 +0x64 fp=0xc754f1e4e0 sp=0xc754f1e4a0
encoding/json.(*sliceEncoder).encode(0xc4203260d0, 0xc75c20cc60, 0xb658e0, 0xc76fae1ba0, 0x97, 0xb60100)
/usr/local/go/1.8.5/go/src/encoding/json/encode.go:741 +0xc1 fp=0xc754f1e520 sp=0xc754f1e4e0
encoding/json.(*sliceEncoder).(encoding/json.encode)-fm(0xc75c20cc60, 0xb658e0, 0xc76fae1ba0, 0x97, 0xc76fae0100)
/usr/local/go/1.8.5/go/src/encoding/json/encode.go:753 +0x64 fp=0xc754f1e560 sp=0xc754f1e520
encoding/json.(*encodeState).reflectValue(0xc75c20cc60, 0xb658e0, 0xc76fae1ba0, 0x97, 0xc76fae0100)
/usr/local/go/1.8.5/go/src/encoding/json/encode.go:323 +0x82 fp=0xc754f1e598 sp=0xc754f1e560
encoding/json.interfaceEncoder(0xc75c20cc60, 0xb9c280, 0xc7704e0cd0, 0x94, 0xc7704e0100)
/usr/local/go/1.8.5/go/src/encoding/json/encode.go:617 +0xdb fp=0xc754f1e5d8 sp=0xc754f1e598
encoding/json.(*mapEncoder).encode(0xc420326048, 0xc75c20cc60, 0xbaf4e0, 0xc76f73f530, 0x15, 0xba0100)
/usr/local/go/1.8.5/go/src/encoding/json/encode.go:690 +0x589 fp=0xc754f1e748 sp=0xc754f1e5d8
encoding/json.(*mapEncoder).(encoding/json.encode)-fm(0xc75c20cc60, 0xbaf4e0, 0xc76f73f530, 0x15, 0xc76f730100)
/usr/local/go/1.8.5/go/src/encoding/json/encode.go:706 +0x64 fp=0xc754f1e788 sp=0xc754f1e748
encoding/json.(*encodeState).reflectValue(0xc75c20cc60, 0xbaf4e0, 0xc76f73f530, 0x15, 0xc76f730100)
/usr/local/go/1.8.5/go/src/encoding/json/encode.go:323 +0x82 fp=0xc754f1e7c0 sp=0xc754f1e788
encoding/json.interfaceEncoder(0xc75c20cc60, 0xb9c280, 0xc7704e0c50, 0x94, 0xc7704e0100)
/usr/local/go/1.8.5/go/src/encoding/json/encode.go:617 +0xdb fp=0xc754f1e800 sp=0xc754f1e7c0
encoding/json.(*mapEncoder).encode(0xc420326048, 0xc75c20cc60, 0xbaf4e0, 0xc76f73f500, 0x15, 0xba0100)
/usr/local/go/1.8.5/go/src/encoding/json/encode.go:690 +0x589 fp=0xc754f1e970 sp=0xc754f1e800
encoding/json.(*mapEncoder).(encoding/json.encode)-fm(0xc75c20cc60, 0xbaf4e0, 0xc76f73f500, 0x15, 0xc76f730100)
/usr/local/go/1.8.5/go/src/encoding/json/encode.go:706 +0x64 fp=0xc754f1e9b0 sp=0xc754f1e970
encoding/json.(*encodeState).reflectValue(0xc75c20cc60, 0xbaf4e0, 0xc76f73f500, 0x15, 0xc76f730100)
/usr/local/go/1.8.5/go/src/encoding/json/encode.go:323 +0x82 fp=0xc754f1e9e8 sp=0xc754f1e9b0
encoding/json.interfaceEncoder(0xc75c20cc60, 0xb9c280, 0xc7704e0c30, 0x94, 0xc7704e0100)
/usr/local/go/1.8.5/go/src/encoding/json/encode.go:617 +0xdb fp=0xc754f1ea28 sp=0xc754f1e9e8
encoding/json.(*mapEncoder).encode(0xc420326080, 0xc75c20cc60, 0xc188c0, 0xc76f73f4d0, 0x15, 0xc10100)
/usr/local/go/1.8.5/go/src/encoding/json/encode.go:690 +0x589 fp=0xc754f1eb98 sp=0xc754f1ea28
encoding/json.(*mapEncoder).(encoding/json.encode)-fm(0xc75c20cc60, 0xc188c0, 0xc76f73f4d0, 0x15, 0xc76f730100)
/usr/local/go/1.8.5/go/src/encoding/json/encode.go:706 +0x64 fp=0xc754f1ebd8 sp=0xc754f1eb98
encoding/json.(*encodeState).reflectValue(0xc75c20cc60, 0xc188c0, 0xc76f73f4d0, 0x15, 0x100)
/usr/local/go/1.8.5/go/src/encoding/json/encode.go:323 +0x82 fp=0xc754f1ec10 sp=0xc754f1ebd8
encoding/json.(*encodeState).marshal(0xc75c20cc60, 0xc188c0, 0xc76f73f4d0, 0x100, 0x0, 0x0)
/usr/local/go/1.8.5/go/src/encoding/json/encode.go:296 +0xb8 fp=0xc754f1ec48 sp=0xc754f1ec10
encoding/json.Marshal(0xc188c0, 0xc76f73f4d0, 0x7bdee891, 0xc9d51e10cd80c1ea, 0xc754f1ece8, 0xc90000000041204d, 0x2)
/usr/local/go/1.8.5/go/src/encoding/json/encode.go:161 +0x6e fp=0xc754f1ec90 sp=0xc754f1ec48
github.com/couchbase/sync_gateway/db.(*document).MarshalBody(0xc76f718000, 0xc76f73f4d0, 0xc76e3617a0, 0x26, 0xc76f7c5e00, 0x0)
/home/couchbase/jenkins/workspace/sgw-unix-build@2/2.0.0/enterprise/godeps/src/github.com/couchbase/sync_gateway/db/document.go:123 +0x49 fp=0xc754f1ecf8 sp=0xc754f1ec90
github.com/couchbase/sync_gateway/db.(*Database).updateAndReturnDoc.func1(0xc76f718000, 0xc76f710101, 0xc76f63c000, 0xdb02c, 0xdc000, 0xc76f62e000, 0xc02b, 0xe000)
/home/couchbase/jenkins/workspace/sgw-unix-build@2/2.0.0/enterprise/godeps/src/github.com/couchbase/sync_gateway/db/crud.go:849 +0x1cfb fp=0xc754f1f0a8 sp=0xc754f1ecf8
github.com/couchbase/sync_gateway/db.(*Database).updateAndReturnDoc.func3(0xc76f63c000, 0xdb02c, 0xdc000, 0xc76f62e000, 0xc02b, 0xe000, 0x15545037bd3b0000, 0x0, 0x0, 0x10d3c40, …)
/home/couchbase/jenkins/workspace/sgw-unix-build@2/2.0.0/enterprise/godeps/src/github.com/couchbase/sync_gateway/db/crud.go:1073 +0x138 fp=0xc754f1f1f0 sp=0xc754f1f0a8
github.com/couchbase/sync_gateway/base.CouchbaseBucketGoCB.WriteUpdateWithXattr(0xc420130870, 0xc4201b8e70, 0x2f, 0xc9533b, 0x7, 0xc4201b71a0, 0xc, 0x0, 0x0, 0x10d3c40, …)
/home/couchbase/jenkins/workspace/sgw-unix-build@2/2.0.0/enterprise/godeps/src/github.com/couchbase/sync_gateway/base/bucket_gocb.go:1548 +0x174 fp=0xc754f1f3b8 sp=0xc754f1f1f0
github.com/couchbase/sync_gateway/base.(*CouchbaseBucketGoCB).WriteUpdateWithXattr(0xc420278c80, 0xc76d7b4692, 0x1f, 0xc92cb3, 0x5, 0xc400000000, 0x0, 0xc76f539410, 0x0, 0x6600000001, …)
:472 +0xef fp=0xc754f1f4b0 sp=0xc754f1f3b8
github.com/couchbase/sync_gateway/db.(*Database).updateAndReturnDoc(0xc64f9fb0c0, 0xc76d7b4692, 0x1f, 0xe5174e01, 0x0, 0xc5e5174ec0, 0xc5e5174ef0, 0xc76f544890, 0xc3bb40, 0xb58501, …)
/home/couchbase/jenkins/workspace/sgw-unix-build@2/2.0.0/enterprise/godeps/src/github.com/couchbase/sync_gateway/db/crud.go:1090 +0x523 fp=0xc754f1f798 sp=0xc754f1f4b0
github.com/couchbase/sync_gateway/db.(*Database).updateDoc(0xc64f9fb0c0, 0xc76d7b4692, 0x1f, 0x1, 0xc5e5174ec0, 0xc96298, 0x8, 0x1275ec0, 0xc754f1f878)
/home/couchbase/jenkins/workspace/sgw-unix-build@2/2.0.0/enterprise/godeps/src/github.com/couchbase/sync_gateway/db/crud.go:767 +0x70 fp=0xc754f1f800 sp=0xc754f1f798
github.com/couchbase/sync_gateway/db.(*Database).Put(0xc64f9fb0c0, 0xc76d7b4692, 0x1f, 0xc64f9fecf0, 0x0, 0x7fe551a2b240, 0x86, 0x86)
/home/couchbase/jenkins/workspace/sgw-unix-build@2/2.0.0/enterprise/godeps/src/github.com/couchbase/sync_gateway/db/crud.go:636 +0x331 fp=0xc754f1f888 sp=0xc754f1f800
github.com/couchbase/sync_gateway/rest.(*handler).handlePutDoc(0xc76d7bc500, 0xc64f9fb0c0, 0x0)
/home/couchbase/jenkins/workspace/sgw-unix-build@2/2.0.0/enterprise/godeps/src/github.com/couchbase/sync_gateway/rest/doc_api.go:269 +0x1f0 fp=0xc754f1f9a8 sp=0xc754f1f888
github.com/couchbase/sync_gateway/rest.(*handler).invoke(0xc76d7bc500, 0xccb9b8, 0x0, 0x0)
/home/couchbase/jenkins/workspace/sgw-unix-build@2/2.0.0/enterprise/godeps/src/github.com/couchbase/sync_gateway/rest/handler.go:222 +0x2c9 fp=0xc754f1fa40 sp=0xc754f1f9a8
github.com/couchbase/sync_gateway/rest.makeHandler.func1(0x10dc800, 0xc76e36c0e0, 0xc76d7b8600)
/home/couchbase/jenkins/workspace/sgw-unix-build@2/2.0.0/enterprise/godeps/src/github.com/couchbase/sync_gateway/rest/handler.go:97 +0x80 fp=0xc754f1fa98 sp=0xc754f1fa40
net/http.HandlerFunc.ServeHTTP(0xc4208081a0, 0x10dc800, 0xc76e36c0e0, 0xc76d7b8600)
/usr/local/go/1.8.5/go/src/net/http/server.go:1942 +0x44 fp=0xc754f1fac0 sp=0xc754f1fa98
github.com/gorilla/mux.(*Router).ServeHTTP(0xc4204cb950, 0x10dc800, 0xc76e36c0e0, 0xc76d7b8600)
/home/couchbase/jenkins/workspace/sgw-unix-build@2/2.0.0/enterprise/godeps/src/github.com/gorilla/mux/mux.go:114 +0x10c fp=0xc754f1fbc0 sp=0xc754f1fac0
github.com/couchbase/sync_gateway/rest.wrapRouter.func1(0x10dc800, 0xc76e36c0e0, 0xc6aa774e00)
/home/couchbase/jenkins/workspace/sgw-unix-build@2/2.0.0/enterprise/godeps/src/github.com/couchbase/sync_gateway/rest/routing.go:304 +0x669 fp=0xc754f1fce8 sp=0xc754f1fbc0
net/http.HandlerFunc.ServeHTTP(0xc420dd3720, 0x10dc800, 0xc76e36c0e0, 0xc6aa774e00)
/usr/local/go/1.8.5/go/src/net/http/server.go:1942 +0x44 fp=0xc754f1fd10 sp=0xc754f1fce8
net/http.serverHandler.ServeHTTP(0xc420fd2840, 0x10dc800, 0xc76e36c0e0, 0xc6aa774e00)
/usr/local/go/1.8.5/go/src/net/http/server.go:2568 +0x92 fp=0xc754f1fd58 sp=0xc754f1fd10
net/http.(*conn).serve(0xc61bf7cbe0, 0x10dd300, 0xc4251b3240)
/usr/local/go/1.8.5/go/src/net/http/server.go:1825 +0x612 fp=0xc754f1ffc8 sp=0xc754f1fd58
runtime.goexit()
/usr/local/go/1.8.5/go/src/runtime/asm_amd64.s:2197 +0x1 fp=0xc754f1ffd0 sp=0xc754f1ffc8
created by net/http.(*Server).Serve
/usr/local/go/1.8.5/go/src/net/http/server.go:2668 +0x2ce