一则golang 内存溢出分析
- 2019 年 11 月 21 日
- 筆記
由于自己的服务器配置比较差,上传一篇较长博客后,进存kill,显示内存溢出。每次重启后不久,都会进存kill掉的现象出现。 问题 上传了一篇比较长的文章后,goblog后台直接被killed掉,查看日志,内存溢出:
2019/11/15 15:38:04.729 [I] [router.go:266] /root/Project/src/goblog/src/controller/admin no changed 2019/11/15 15:38:04.729 [I] [router.go:266] /root/Project/src/goblog/src/controller no changed [goblog] 2019/11/15 15:38:04.733401 user_mapper.go:57 [INFO] call GetByCondition rows:1 2019/11/15 15:38:05.109 [I] [asm_amd64.s:1333] http server Running on http://:9090 2019/11/15 15:38:05.127 [I] [asm_amd64.s:1333] Admin server Running on 127.0.0.1:8088 2019/11/15 15:38:08 Gse dictionary loaded finished. fatal error: runtime: out of memory runtime stack: runtime.throw(0xbeb588, 0x16) /root/go/src/runtime/panic.go:608 +0x72 runtime.sysMap(0xc050000000, 0x4000000, 0x138fb58) /root/go/src/runtime/mem_linux.go:156 +0xc7 runtime.(*mheap).sysAlloc(0x1376320, 0x4000000, 0x1376338, 0x7f4c067ddb48) /root/go/src/runtime/malloc.go:619 +0x1c7 runtime.(*mheap).grow(0x1376320, 0x2, 0x0) /root/go/src/runtime/mheap.go:920 +0x42 runtime.(*mheap).allocSpanLocked(0x1376320, 0x2, 0x138fb68, 0x800) /root/go/src/runtime/mheap.go:848 +0x337 runtime.(*mheap).alloc_m(0x1376320, 0x2, 0x75, 0x7f4c069e5fff) /root/go/src/runtime/mheap.go:692 +0x119 runtime.(*mheap).alloc.func1() /root/go/src/runtime/mheap.go:759 +0x4c runtime.(*mheap).alloc(0x1376320, 0x2, 0x7f4c06010075, 0x7f4c067ddab0) /root/go/src/runtime/mheap.go:758 +0x8a runtime.(*mcentral).grow(0x1379398, 0x0) /root/go/src/runtime/mcentral.go:232 +0x94 runtime.(*mcentral).cacheSpan(0x1379398, 0x7f4c067ddab0) /root/go/src/runtime/mcentral.go:106 +0x2f8 runtime.(*mcache).refill(0x7f4c1358d000, 0x7f4c0cbd1875) /root/go/src/runtime/mcache.go:122 +0x95 runtime.(*mcache).nextFree.func1() /root/go/src/runtime/malloc.go:749 +0x32 runtime.systemstack(0x45a029) /root/go/src/runtime/asm_amd64.s:351 +0x66 runtime.mstart() /root/go/src/runtime/proc.go:1229 goroutine 65 [running]: runtime.systemstack_switch() /root/go/src/runtime/asm_amd64.s:311 fp=0xc01af6b400 sp=0xc01af6b3f8 pc=0x45a120 runtime.(*mcache).nextFree(0x7f4c1358d000, 0x203075, 0xc04fffa000, 0x7f4c067ddab0, 0xc01af6b401) /root/go/src/runtime/malloc.go:748 +0xb6 fp=0xc01af6b458 sp=0xc01af6b400 pc=0x40d506 runtime.mallocgc(0x4000, 0x0, 0x7f4c0ced8b00, 0xc04fffa000) /root/go/src/runtime/malloc.go:903 +0x793 fp=0xc01af6b4f8 sp=0xc01af6b458 pc=0x40de53 runtime.rawstring(0x3fca, 0x0, 0x0, 0x0, 0x0, 0x0) /root/go/src/runtime/string.go:258 +0x4f fp=0xc01af6b528 sp=0xc01af6b4f8 pc=0x4491df runtime.rawstringtmp(0x0, 0x3fca, 0xc005a5b4f0, 0x0, 0x1370e00, 0x3fc9, 0x7f4c1358d000) /root/go/src/runtime/string.go:123 +0x72 fp=0xc01af6b568 sp=0xc01af6b528 pc=0x448bb2 runtime.concatstrings(0x0, 0xc01af6b648, 0x2, 0x2, 0x0, 0xc04ab3a830) /root/go/src/runtime/string.go:49 +0xfd fp=0xc01af6b600 sp=0xc01af6b568 pc=0x44868d runtime.concatstring2(0x0, 0xc04fffa000, 0x3fc9, 0xc045f674c9, 0x1, 0xc005a5b400, 0x0) /root/go/src/runtime/string.go:58 +0x47 fp=0xc01af6b640 sp=0xc01af6b600 pc=0x4488b7 github.com/go-ego/riot.(*Engine).ForSplitData(0x1370740, 0xc048a70000, 0x52cf, 0x52cf, 0x52cf, 0x52cf, 0x52cf) /root/Project/pkg/mod/github.com/go-ego/[email protected]/segment.go:53 +0x141 fp=0xc01af6b710 sp=0xc01af6b640 pc=0x9e4671 github.com/go-ego/riot.(*Engine).splitData(0x1370740, 0x12cf0d9, 0x1, 0x2ab8b572, 0xc045f48a80, 0x6321, 0x0, 0x0, 0x0, 0x0, ...) /root/Project/pkg/mod/github.com/go-ego/[email protected]/segment.go:108 +0x32d fp=0xc01af6b8e8 sp=0xc01af6b710 pc=0x9e4e0d github.com/go-ego/riot.(*Engine).segmenterData(0x1370740, 0x12cf0d9, 0x1, 0x2ab8b572, 0xc045f48a80, 0x6321, 0x0, 0x0, 0x0, 0x0, ...) /root/Project/pkg/mod/github.com/go-ego/[email protected]/segment.go:186 +0x2d2 fp=0xc01af6bb00 sp=0xc01af6b8e8 pc=0x9e54d2 github.com/go-ego/riot.(*Engine).makeTokensMap(0x1370740, 0x12cf0d9, 0x1, 0x2ab8b572, 0xc045f48a80, 0x6321, 0x0, 0x0, 0x0, 0x0, ...) /root/Project/pkg/mod/github.com/go-ego/[email protected]/segment.go:214 +0x4a4 fp=0xc01af6bc70 sp=0xc01af6bb00 pc=0x9e6284 github.com/go-ego/riot.(*Engine).segmenterWorker(0x1370740) /root/Project/pkg/mod/github.com/go-ego/[email protected]/segment.go:261 +0x1b5 fp=0xc01af6bfd8 sp=0xc01af6bc70 pc=0x9e6465 runtime.goexit() /root/go/src/runtime/asm_amd64.s:1333 +0x1 fp=0xc01af6bfe0 sp=0xc01af6bfd8 pc=0x45c201 created by github.com/go-ego/riot.(*Engine).Init /root/Project/pkg/mod/github.com/go-ego/[email protected]/engine.go:331 +0x52c goroutine 1 [chan receive]: github.com/astaxie/beego.(*App).Run(0xc000118130, 0x0, 0x0, 0x0) /root/Project/pkg/mod/github.com/astaxie/[email protected]/app.go:221 +0x262 github.com/astaxie/beego.Run(0x0, 0x0, 0x0) /root/Project/pkg/mod/github.com/astaxie/[email protected]/beego.go:67 +0x62 main.main() /root/Project/src/goblog/main.go:12 +0x32 goroutine 5 [syscall]: os/signal.signal_recv(0x0) /root/go/src/runtime/sigqueue.go:139 +0x9c os/signal.loop() /root/go/src/os/signal/signal_unix.go:23 +0x22 created by os/signal.init.0 /root/go/src/os/signal/signal_unix.go:29 +0x41 goroutine 6 [select]: goblog/src/logs.async() /root/Project/src/goblog/src/logs/logs.go:224 +0x108 created by goblog/src/logs.InitLogs /root/Project/src/goblog/src/logs/logs.go:89 +0x761 goroutine 7 [chan receive]: goblog/src/logs.(*logs).logCut(0xc000088550, 0xc00000e688) /root/Project/src/goblog/src/logs/logs.go:111 +0x2a7 created by goblog/src/logs.InitLogs /root/Project/src/goblog/src/logs/logs.go:91 +0x724 goroutine 8 [select]: database/sql.(*DB).connectionOpener(0xc0000f00c0, 0xcb23e0, 0xc000061d40) /root/go/src/database/sql/sql.go:1001 +0xe8 created by database/sql.OpenDB /root/go/src/database/sql/sql.go:671 +0x15d goroutine 9 [select]: database/sql.(*DB).connectionResetter(0xc0000f00c0, 0xcb23e0, 0xc000061d40) /root/go/src/database/sql/sql.go:1014 +0xfb created by database/sql.OpenDB /root/go/src/database/sql/sql.go:672 +0x193 goroutine 13 [select]: github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher.func1(0xc000062d20, 0xc0000f0180, 0xc00002a5a0) /root/Project/pkg/mod/github.com/go-sql-driver/[email protected]/connection_go18.go:179 +0xbf created by github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher /root/Project/pkg/mod/github.com/go-sql-driver/[email protected]/connection_go18.go:176 +0xbe goroutine 14 [select]: database/sql.(*DB).connectionCleaner(0xc0000f00c0, 0x1a3185c5000) /root/go/src/database/sql/sql.go:899 +0x37d created by database/sql.(*DB).startCleanerLocked /root/go/src/database/sql/sql.go:886 +0xa7 goroutine 19 [chan receive]: github.com/go-ego/riot.(*Engine).Store(0x1370740) /root/Project/pkg/mod/github.com/go-ego/[email protected]/engine.go:212 +0x48e github.com/go-ego/riot.(*Engine).Init(0x1370740, 0x0, 0x3, 0xbc5db6, 0x2, 0x0, 0x0, 0x0, 0x0, 0x1, ...) /root/Project/pkg/mod/github.com/go-ego/[email protected]/engine.go:351 +0x720 goblog/src/service.init.2.func1() /root/Project/src/goblog/src/service/search_service.go:55 +0xba created by goblog/src/component.GoRoutine /root/Project/src/goblog/src/component/routine_component.go:9 +0x33 goroutine 63 [IO wait]: internal/poll.runtime_pollWait(0x7f4c13531d60, 0x72, 0x0) /root/go/src/runtime/netpoll.go:173 +0x66 internal/poll.(*pollDesc).wait(0xc0000ef998, 0x72, 0xc000060000, 0x0, 0x0) /root/go/src/internal/poll/fd_poll_runtime.go:85 +0x9a internal/poll.(*pollDesc).waitRead(0xc0000ef998, 0xffffffffffffff00, 0x0, 0x0) /root/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Accept(0xc0000ef980, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /root/go/src/internal/poll/fd_unix.go:384 +0x1a0 net.(*netFD).accept(0xc0000ef980, 0x1370e00, 0x30, 0x30) /root/go/src/net/fd_unix.go:238 +0x42 net.(*TCPListener).accept(0xc0001c84b0, 0xc000052d18, 0x7f4c1358d000, 0xc000153500) /root/go/src/net/tcpsock_posix.go:139 +0x2e net.(*TCPListener).AcceptTCP(0xc0001c84b0, 0x40e238, 0x30, 0xb67640) /root/go/src/net/tcpsock.go:247 +0x47 net/http.tcpKeepAliveListener.Accept(0xc0001c84b0, 0xb67640, 0xc000df7da0, 0xaedd40, 0x135c100) /root/go/src/net/http/server.go:3232 +0x2f net/http.(*Server).Serve(0xc0000ff860, 0xcb21e0, 0xc0001c84b0, 0x0, 0x0) /root/go/src/net/http/server.go:2826 +0x22f net/http.(*Server).ListenAndServe(0xc0000ff860, 0xc0000ff860, 0x26) /root/go/src/net/http/server.go:2764 +0xb6 net/http.ListenAndServe(0xc000e00450, 0xe, 0x0, 0x0, 0x1, 0xc000e00450) /root/go/src/net/http/server.go:3004 +0x74 github.com/astaxie/beego.(*adminApp).Run(0xc00000e648) /root/Project/pkg/mod/github.com/astaxie/[email protected]/admin.go:399 +0x358 created by github.com/astaxie/beego.registerAdmin /root/Project/pkg/mod/github.com/astaxie/[email protected]/hooks.go:89 +0x67 ...
分析
猜测是线程数配置的问题,查看engine.go的Init方法中各参数的作用,如下:
func (engine *Engine) Init(options types.EngineOpts) { // 将线程数设置为CPU数 // runtime.GOMAXPROCS(runtime.NumCPU()) // runtime.GOMAXPROCS(128) // 初始化初始参数 if engine.initialized { log.Fatal("Do not re-initialize the engine.") } options = engine.initDef(options) options.Init() engine.initOptions = options engine.initialized = true if !options.NotUseGse { if !engine.loaded { // 载入分词器词典 engine.segmenter.LoadDict(options.GseDict) engine.loaded = true } // 初始化停用词 engine.stopTokens.Init(options.StopTokenFile) } // 初始化索引器和排序器 for shard := 0; shard < options.NumShards; shard++ { engine.indexers = append(engine.indexers, core.Indexer{}) engine.indexers[shard].Init(*options.IndexerOpts) engine.rankers = append(engine.rankers, core.Ranker{}) engine.rankers[shard].Init(options.IDOnly) } // 初始化分词器通道 engine.segmenterChan = make( chan segmenterReq, options.NumGseThreads) // 初始化索引器通道 engine.Indexer(options) // 初始化排序器通道 engine.Ranker(options) // engine.CheckMem(engine.initOptions.UseStore) engine.CheckMem() // 初始化持久化存储通道 if engine.initOptions.UseStore { engine.InitStore() } // 启动分词器 for iThread := 0; iThread < options.NumGseThreads; iThread++ { go engine.segmenterWorker() } // 启动索引器和排序器 for shard := 0; shard < options.NumShards; shard++ { go engine.indexerAddDoc(shard) go engine.indexerRemoveDoc(shard) go engine.rankerAddDoc(shard) go engine.rankerRemoveDoc(shard) for i := 0; i < options.NumIndexerThreads; i++ { go engine.indexerLookup(shard) } for i := 0; i < options.NumRankerThreads; i++ { go engine.rankerRank(shard) } } // 启动持久化存储工作协程 if engine.initOptions.UseStore { engine.Store() } atomic.AddUint64(&engine.numDocsStored, engine.numIndexingReqs) }
它的第331行代码为:
// 启动分词器 for iThread := 0; iThread < options.NumGseThreads; iThread++ { go engine.segmenterWorker() }
推测是riot组件的设置问题,于是查看search_service.go文件的init方法:
func init() { SearchBiz = searchService{} engineType = make(map[string]SearchEngine) //标签搜索 engineType[TAG] = TagSearchEngine{} //栏目搜索 engineType[CATEGORY] = CategorySearchEngine{} //博文搜索 engineType[ARTICLES_FULL_TEXT] = ArticlesSearchEngine{} //归档搜索 engineType[PLACE_OF_FILE] = PlaceOfFileSearchEngine{} runtime.GOMAXPROCS(runtime.NumCPU()) fmt.Println("start init search====================") component.GoRoutine(func() { gob.Register(ArticlesScoringFields{}) opts := types.EngineOpts{ Using: 3, GseDict: "zh", UseStore: true, StoreFolder: "./indexer", StoreShards: 4, StoreEngine: "", } fullTextSearcher.Init(opts) //data,_ := json.Marshal(opts) //fmt.Println(data) fullTextSearcher.Flush() }) fmt.Println("end init search====================") }
于是查看EngineOpts方件:
type EngineOpts struct { // 是否使用分词器 // 默认使用,否则在启动阶段跳过 GseDict 和 StopTokenFile 设置 // 如果你不需要在引擎内分词,可以将这个选项设为 true // 注意,如果你不用分词器,那么在调用 IndexDoc 时, // DocIndexData 中的 Content 会被忽略 // Not use the gse segment NotUseGse bool `toml:"not_use_gse"` // new, 分词规则 Using int `toml:"using"` // 半角逗号 "," 分隔的字典文件,具体用法见 // gse.Segmenter.LoadDict 函数的注释 GseDict string `toml:"gse_dict"` PinYin bool `toml:"pin_yin"` // 停用词文件 StopTokenFile string `toml:"stop_file"` // Gse search mode GseMode bool `toml:"gse_mode"` Hmm bool `toml:"hmm"` Model string `toml:"model"` // 分词器线程数 // NumSegmenterThreads int NumGseThreads int // 索引器和排序器的 shard 数目 // 被检索/排序的文档会被均匀分配到各个 shard 中 NumShards int // 索引器的信道缓冲长度 IndexerBufLen int // 索引器每个shard分配的线程数 NumIndexerThreads int // 排序器的信道缓冲长度 RankerBufLen int // 排序器每个 shard 分配的线程数 NumRankerThreads int // 索引器初始化选项 IndexerOpts *IndexerOpts // 默认的搜索选项 DefRankOpts *RankOpts // 是否使用持久数据库,以及数据库文件保存的目录和裂分数目 StoreOnly bool `toml:"store_only"` UseStore bool `toml:"use_store"` StoreFolder string `toml:"store_folder"` StoreShards int `toml:"store_shards"` StoreEngine string `toml:"store_engine"` IDOnly bool `toml:"id_only"` }
修改成:
component.GoRoutine(func() { gob.Register(ArticlesScoringFields{}) opts := types.EngineOpts{ NumGseThreads:2, NumIndexerThreads:2, NumRankerThreads:2, Using: 3, GseDict: "zh", UseStore: true, StoreFolder: "./indexer", StoreShards: 4, StoreEngine: "", } fullTextSearcher.Init(opts) //data,_ := json.Marshal(opts) //fmt.Println(data) fullTextSearcher.Flush() })
重启服务,问题解决。
总结
将相同的代码和配置放到一台性能较好的机器上不会复现上述问题,应该是riot组件默认设置对机器要求比较高,改低一些,问题解决。
另外还可以通过net/http/pprof查看golang内存情况,详见:https://segmentfault.com/a/1190000019929993