一则golang 内存溢出分析

• 2019 年 11 月 21 日
• 筆記

由于自己的服务器配置比较差，上传一篇较长博客后，进存kill，显示内存溢出。每次重启后不久，都会进存kill掉的现象出现。 问题 上传了一篇比较长的文章后，goblog后台直接被killed掉，查看日志，内存溢出:

2019/11/15 15:38:04.729 [I] [router.go:266] /root/Project/src/goblog/src/controller/admin no changed  2019/11/15 15:38:04.729 [I] [router.go:266] /root/Project/src/goblog/src/controller no changed  [goblog] 2019/11/15 15:38:04.733401 user_mapper.go:57 [INFO] call GetByCondition rows:1  2019/11/15 15:38:05.109 [I] [asm_amd64.s:1333] http server Running on http://:9090  2019/11/15 15:38:05.127 [I] [asm_amd64.s:1333] Admin server Running on 127.0.0.1:8088  2019/11/15 15:38:08 Gse dictionary loaded finished.  fatal error: runtime: out of memory    runtime stack:  runtime.throw(0xbeb588, 0x16)      /root/go/src/runtime/panic.go:608 +0x72  runtime.sysMap(0xc050000000, 0x4000000, 0x138fb58)      /root/go/src/runtime/mem_linux.go:156 +0xc7  runtime.(*mheap).sysAlloc(0x1376320, 0x4000000, 0x1376338, 0x7f4c067ddb48)      /root/go/src/runtime/malloc.go:619 +0x1c7  runtime.(*mheap).grow(0x1376320, 0x2, 0x0)      /root/go/src/runtime/mheap.go:920 +0x42  runtime.(*mheap).allocSpanLocked(0x1376320, 0x2, 0x138fb68, 0x800)      /root/go/src/runtime/mheap.go:848 +0x337  runtime.(*mheap).alloc_m(0x1376320, 0x2, 0x75, 0x7f4c069e5fff)      /root/go/src/runtime/mheap.go:692 +0x119  runtime.(*mheap).alloc.func1()      /root/go/src/runtime/mheap.go:759 +0x4c  runtime.(*mheap).alloc(0x1376320, 0x2, 0x7f4c06010075, 0x7f4c067ddab0)      /root/go/src/runtime/mheap.go:758 +0x8a  runtime.(*mcentral).grow(0x1379398, 0x0)      /root/go/src/runtime/mcentral.go:232 +0x94  runtime.(*mcentral).cacheSpan(0x1379398, 0x7f4c067ddab0)      /root/go/src/runtime/mcentral.go:106 +0x2f8  runtime.(*mcache).refill(0x7f4c1358d000, 0x7f4c0cbd1875)      /root/go/src/runtime/mcache.go:122 +0x95  runtime.(*mcache).nextFree.func1()      /root/go/src/runtime/malloc.go:749 +0x32  runtime.systemstack(0x45a029)      /root/go/src/runtime/asm_amd64.s:351 +0x66  runtime.mstart()      /root/go/src/runtime/proc.go:1229    goroutine 65 [running]:  runtime.systemstack_switch()      /root/go/src/runtime/asm_amd64.s:311 fp=0xc01af6b400 sp=0xc01af6b3f8 pc=0x45a120  runtime.(*mcache).nextFree(0x7f4c1358d000, 0x203075, 0xc04fffa000, 0x7f4c067ddab0, 0xc01af6b401)      /root/go/src/runtime/malloc.go:748 +0xb6 fp=0xc01af6b458 sp=0xc01af6b400 pc=0x40d506  runtime.mallocgc(0x4000, 0x0, 0x7f4c0ced8b00, 0xc04fffa000)      /root/go/src/runtime/malloc.go:903 +0x793 fp=0xc01af6b4f8 sp=0xc01af6b458 pc=0x40de53  runtime.rawstring(0x3fca, 0x0, 0x0, 0x0, 0x0, 0x0)      /root/go/src/runtime/string.go:258 +0x4f fp=0xc01af6b528 sp=0xc01af6b4f8 pc=0x4491df  runtime.rawstringtmp(0x0, 0x3fca, 0xc005a5b4f0, 0x0, 0x1370e00, 0x3fc9, 0x7f4c1358d000)      /root/go/src/runtime/string.go:123 +0x72 fp=0xc01af6b568 sp=0xc01af6b528 pc=0x448bb2  runtime.concatstrings(0x0, 0xc01af6b648, 0x2, 0x2, 0x0, 0xc04ab3a830)      /root/go/src/runtime/string.go:49 +0xfd fp=0xc01af6b600 sp=0xc01af6b568 pc=0x44868d  runtime.concatstring2(0x0, 0xc04fffa000, 0x3fc9, 0xc045f674c9, 0x1, 0xc005a5b400, 0x0)      /root/go/src/runtime/string.go:58 +0x47 fp=0xc01af6b640 sp=0xc01af6b600 pc=0x4488b7  github.com/go-ego/riot.(*Engine).ForSplitData(0x1370740, 0xc048a70000, 0x52cf, 0x52cf, 0x52cf, 0x52cf, 0x52cf)      /root/Project/pkg/mod/github.com/go-ego/riot@v0.0.0-20181222142112-75a2cab10779/segment.go:53 +0x141 fp=0xc01af6b710 sp=0xc01af6b640 pc=0x9e4671  github.com/go-ego/riot.(*Engine).splitData(0x1370740, 0x12cf0d9, 0x1, 0x2ab8b572, 0xc045f48a80, 0x6321, 0x0, 0x0, 0x0, 0x0, ...)      /root/Project/pkg/mod/github.com/go-ego/riot@v0.0.0-20181222142112-75a2cab10779/segment.go:108 +0x32d fp=0xc01af6b8e8 sp=0xc01af6b710 pc=0x9e4e0d  github.com/go-ego/riot.(*Engine).segmenterData(0x1370740, 0x12cf0d9, 0x1, 0x2ab8b572, 0xc045f48a80, 0x6321, 0x0, 0x0, 0x0, 0x0, ...)      /root/Project/pkg/mod/github.com/go-ego/riot@v0.0.0-20181222142112-75a2cab10779/segment.go:186 +0x2d2 fp=0xc01af6bb00 sp=0xc01af6b8e8 pc=0x9e54d2  github.com/go-ego/riot.(*Engine).makeTokensMap(0x1370740, 0x12cf0d9, 0x1, 0x2ab8b572, 0xc045f48a80, 0x6321, 0x0, 0x0, 0x0, 0x0, ...)      /root/Project/pkg/mod/github.com/go-ego/riot@v0.0.0-20181222142112-75a2cab10779/segment.go:214 +0x4a4 fp=0xc01af6bc70 sp=0xc01af6bb00 pc=0x9e6284  github.com/go-ego/riot.(*Engine).segmenterWorker(0x1370740)      /root/Project/pkg/mod/github.com/go-ego/riot@v0.0.0-20181222142112-75a2cab10779/segment.go:261 +0x1b5 fp=0xc01af6bfd8 sp=0xc01af6bc70 pc=0x9e6465  runtime.goexit()      /root/go/src/runtime/asm_amd64.s:1333 +0x1 fp=0xc01af6bfe0 sp=0xc01af6bfd8 pc=0x45c201  created by github.com/go-ego/riot.(*Engine).Init      /root/Project/pkg/mod/github.com/go-ego/riot@v0.0.0-20181222142112-75a2cab10779/engine.go:331 +0x52c    goroutine 1 [chan receive]:  github.com/astaxie/beego.(*App).Run(0xc000118130, 0x0, 0x0, 0x0)      /root/Project/pkg/mod/github.com/astaxie/beego@v1.9.2/app.go:221 +0x262  github.com/astaxie/beego.Run(0x0, 0x0, 0x0)      /root/Project/pkg/mod/github.com/astaxie/beego@v1.9.2/beego.go:67 +0x62  main.main()      /root/Project/src/goblog/main.go:12 +0x32    goroutine 5 [syscall]:  os/signal.signal_recv(0x0)      /root/go/src/runtime/sigqueue.go:139 +0x9c  os/signal.loop()      /root/go/src/os/signal/signal_unix.go:23 +0x22  created by os/signal.init.0      /root/go/src/os/signal/signal_unix.go:29 +0x41    goroutine 6 [select]:  goblog/src/logs.async()      /root/Project/src/goblog/src/logs/logs.go:224 +0x108  created by goblog/src/logs.InitLogs      /root/Project/src/goblog/src/logs/logs.go:89 +0x761    goroutine 7 [chan receive]:  goblog/src/logs.(*logs).logCut(0xc000088550, 0xc00000e688)      /root/Project/src/goblog/src/logs/logs.go:111 +0x2a7  created by goblog/src/logs.InitLogs      /root/Project/src/goblog/src/logs/logs.go:91 +0x724    goroutine 8 [select]:  database/sql.(*DB).connectionOpener(0xc0000f00c0, 0xcb23e0, 0xc000061d40)      /root/go/src/database/sql/sql.go:1001 +0xe8  created by database/sql.OpenDB      /root/go/src/database/sql/sql.go:671 +0x15d    goroutine 9 [select]:  database/sql.(*DB).connectionResetter(0xc0000f00c0, 0xcb23e0, 0xc000061d40)      /root/go/src/database/sql/sql.go:1014 +0xfb  created by database/sql.OpenDB      /root/go/src/database/sql/sql.go:672 +0x193    goroutine 13 [select]:  github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher.func1(0xc000062d20, 0xc0000f0180, 0xc00002a5a0)      /root/Project/pkg/mod/github.com/go-sql-driver/mysql@v1.4.0/connection_go18.go:179 +0xbf  created by github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher      /root/Project/pkg/mod/github.com/go-sql-driver/mysql@v1.4.0/connection_go18.go:176 +0xbe    goroutine 14 [select]:  database/sql.(*DB).connectionCleaner(0xc0000f00c0, 0x1a3185c5000)      /root/go/src/database/sql/sql.go:899 +0x37d  created by database/sql.(*DB).startCleanerLocked      /root/go/src/database/sql/sql.go:886 +0xa7    goroutine 19 [chan receive]:  github.com/go-ego/riot.(*Engine).Store(0x1370740)      /root/Project/pkg/mod/github.com/go-ego/riot@v0.0.0-20181222142112-75a2cab10779/engine.go:212 +0x48e  github.com/go-ego/riot.(*Engine).Init(0x1370740, 0x0, 0x3, 0xbc5db6, 0x2, 0x0, 0x0, 0x0, 0x0, 0x1, ...)      /root/Project/pkg/mod/github.com/go-ego/riot@v0.0.0-20181222142112-75a2cab10779/engine.go:351 +0x720  goblog/src/service.init.2.func1()      /root/Project/src/goblog/src/service/search_service.go:55 +0xba  created by goblog/src/component.GoRoutine      /root/Project/src/goblog/src/component/routine_component.go:9 +0x33    goroutine 63 [IO wait]:  internal/poll.runtime_pollWait(0x7f4c13531d60, 0x72, 0x0)      /root/go/src/runtime/netpoll.go:173 +0x66  internal/poll.(*pollDesc).wait(0xc0000ef998, 0x72, 0xc000060000, 0x0, 0x0)      /root/go/src/internal/poll/fd_poll_runtime.go:85 +0x9a  internal/poll.(*pollDesc).waitRead(0xc0000ef998, 0xffffffffffffff00, 0x0, 0x0)      /root/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d  internal/poll.(*FD).Accept(0xc0000ef980, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)      /root/go/src/internal/poll/fd_unix.go:384 +0x1a0  net.(*netFD).accept(0xc0000ef980, 0x1370e00, 0x30, 0x30)      /root/go/src/net/fd_unix.go:238 +0x42  net.(*TCPListener).accept(0xc0001c84b0, 0xc000052d18, 0x7f4c1358d000, 0xc000153500)      /root/go/src/net/tcpsock_posix.go:139 +0x2e  net.(*TCPListener).AcceptTCP(0xc0001c84b0, 0x40e238, 0x30, 0xb67640)      /root/go/src/net/tcpsock.go:247 +0x47  net/http.tcpKeepAliveListener.Accept(0xc0001c84b0, 0xb67640, 0xc000df7da0, 0xaedd40, 0x135c100)      /root/go/src/net/http/server.go:3232 +0x2f  net/http.(*Server).Serve(0xc0000ff860, 0xcb21e0, 0xc0001c84b0, 0x0, 0x0)      /root/go/src/net/http/server.go:2826 +0x22f  net/http.(*Server).ListenAndServe(0xc0000ff860, 0xc0000ff860, 0x26)      /root/go/src/net/http/server.go:2764 +0xb6  net/http.ListenAndServe(0xc000e00450, 0xe, 0x0, 0x0, 0x1, 0xc000e00450)      /root/go/src/net/http/server.go:3004 +0x74  github.com/astaxie/beego.(*adminApp).Run(0xc00000e648)      /root/Project/pkg/mod/github.com/astaxie/beego@v1.9.2/admin.go:399 +0x358  created by github.com/astaxie/beego.registerAdmin      /root/Project/pkg/mod/github.com/astaxie/beego@v1.9.2/hooks.go:89 +0x67  ...

分析

猜测是线程数配置的问题，查看engine.go的Init方法中各参数的作用，如下:

func (engine *Engine) Init(options types.EngineOpts) {      // 将线程数设置为CPU数      // runtime.GOMAXPROCS(runtime.NumCPU())      // runtime.GOMAXPROCS(128)        // 初始化初始参数      if engine.initialized {          log.Fatal("Do not re-initialize the engine.")      }      options = engine.initDef(options)        options.Init()      engine.initOptions = options      engine.initialized = true        if !options.NotUseGse {          if !engine.loaded {              // 载入分词器词典              engine.segmenter.LoadDict(options.GseDict)              engine.loaded = true          }            // 初始化停用词          engine.stopTokens.Init(options.StopTokenFile)      }        // 初始化索引器和排序器      for shard := 0; shard < options.NumShards; shard++ {          engine.indexers = append(engine.indexers, core.Indexer{})          engine.indexers[shard].Init(*options.IndexerOpts)            engine.rankers = append(engine.rankers, core.Ranker{})          engine.rankers[shard].Init(options.IDOnly)      }        // 初始化分词器通道      engine.segmenterChan = make(          chan segmenterReq, options.NumGseThreads)        // 初始化索引器通道      engine.Indexer(options)        // 初始化排序器通道      engine.Ranker(options)        // engine.CheckMem(engine.initOptions.UseStore)      engine.CheckMem()        // 初始化持久化存储通道      if engine.initOptions.UseStore {          engine.InitStore()      }        // 启动分词器      for iThread := 0; iThread < options.NumGseThreads; iThread++ {          go engine.segmenterWorker()      }        // 启动索引器和排序器      for shard := 0; shard < options.NumShards; shard++ {          go engine.indexerAddDoc(shard)          go engine.indexerRemoveDoc(shard)          go engine.rankerAddDoc(shard)          go engine.rankerRemoveDoc(shard)            for i := 0; i < options.NumIndexerThreads; i++ {              go engine.indexerLookup(shard)          }          for i := 0; i < options.NumRankerThreads; i++ {              go engine.rankerRank(shard)          }      }        // 启动持久化存储工作协程      if engine.initOptions.UseStore {          engine.Store()      }        atomic.AddUint64(&engine.numDocsStored, engine.numIndexingReqs)  }

它的第331行代码为:

// 启动分词器      for iThread := 0; iThread < options.NumGseThreads; iThread++ {          go engine.segmenterWorker()      }

推测是riot组件的设置问题，于是查看search_service.go文件的init方法:

func init() {      SearchBiz = searchService{}      engineType = make(map[string]SearchEngine)      //标签搜索      engineType[TAG] = TagSearchEngine{}      //栏目搜索      engineType[CATEGORY] = CategorySearchEngine{}      //博文搜索      engineType[ARTICLES_FULL_TEXT] = ArticlesSearchEngine{}      //归档搜索      engineType[PLACE_OF_FILE] = PlaceOfFileSearchEngine{}      runtime.GOMAXPROCS(runtime.NumCPU())      fmt.Println("start init search====================")      component.GoRoutine(func() {          gob.Register(ArticlesScoringFields{})          opts := types.EngineOpts{              Using:       3,              GseDict:     "zh",              UseStore:    true,              StoreFolder: "./indexer",              StoreShards: 4,              StoreEngine: "",          }          fullTextSearcher.Init(opts)          //data,_ := json.Marshal(opts)          //fmt.Println(data)          fullTextSearcher.Flush()      })      fmt.Println("end init search====================")  }

于是查看EngineOpts方件:

type EngineOpts struct {      // 是否使用分词器      // 默认使用，否则在启动阶段跳过 GseDict 和 StopTokenFile 设置      // 如果你不需要在引擎内分词，可以将这个选项设为 true      // 注意，如果你不用分词器，那么在调用 IndexDoc 时,      // DocIndexData 中的 Content 会被忽略      // Not use the gse segment      NotUseGse bool `toml:"not_use_gse"`        // new, 分词规则      Using int `toml:"using"`        // 半角逗号 "," 分隔的字典文件，具体用法见      // gse.Segmenter.LoadDict 函数的注释      GseDict string `toml:"gse_dict"`      PinYin  bool   `toml:"pin_yin"`        // 停用词文件      StopTokenFile string `toml:"stop_file"`      // Gse search mode      GseMode bool   `toml:"gse_mode"`      Hmm     bool   `toml:"hmm"`      Model   string `toml:"model"`        // 分词器线程数      // NumSegmenterThreads int      NumGseThreads int        // 索引器和排序器的 shard 数目      // 被检索/排序的文档会被均匀分配到各个 shard 中      NumShards int        // 索引器的信道缓冲长度      IndexerBufLen int        // 索引器每个shard分配的线程数      NumIndexerThreads int        // 排序器的信道缓冲长度      RankerBufLen int        // 排序器每个 shard 分配的线程数      NumRankerThreads int        // 索引器初始化选项      IndexerOpts *IndexerOpts        // 默认的搜索选项      DefRankOpts *RankOpts        // 是否使用持久数据库，以及数据库文件保存的目录和裂分数目      StoreOnly bool `toml:"store_only"`      UseStore  bool `toml:"use_store"`        StoreFolder string `toml:"store_folder"`      StoreShards int    `toml:"store_shards"`      StoreEngine string `toml:"store_engine"`        IDOnly bool `toml:"id_only"`  }

修改成:

component.GoRoutine(func() {          gob.Register(ArticlesScoringFields{})          opts := types.EngineOpts{              NumGseThreads:2,              NumIndexerThreads:2,              NumRankerThreads:2,              Using:       3,              GseDict:     "zh",              UseStore:    true,              StoreFolder: "./indexer",              StoreShards: 4,              StoreEngine: "",          }          fullTextSearcher.Init(opts)          //data,_ := json.Marshal(opts)          //fmt.Println(data)          fullTextSearcher.Flush()      })

重启服务，问题解决。

总结

将相同的代码和配置放到一台性能较好的机器上不会复现上述问题，应该是riot组件默认设置对机器要求比较高，改低一些，问题解决。

另外还可以通过net/http/pprof查看golang内存情况，详见:https://segmentfault.com/a/1190000019929993