From 82e79150ea5c194a7b0b0496981e1489e29fe91d Mon Sep 17 00:00:00 2001 From: Justin Azoff Date: Wed, 11 Apr 2018 09:10:09 -0400 Subject: [PATCH] Use default WriteBuffer size Using a write buffer of 64M was causing flow-indexer to use 7G of ram while indexing. Changing this back to the default of 4M cuts the peak usage in half. I'm still not sure how a 64M buffer ends up turning into 7G of usage, but I think it has something to do with GC and leveldb allocating a 64M buffer for each transaction. This is a partial fix for #13 --- cmd/version.go | 2 +- store/leveldb.go | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/cmd/version.go b/cmd/version.go index 53b66ce..a9e8869 100644 --- a/cmd/version.go +++ b/cmd/version.go @@ -7,7 +7,7 @@ import ( ) const ( - VERSION = "0.1.8" + VERSION = "0.1.9" ) var cmdVersion = &cobra.Command{ diff --git a/store/leveldb.go b/store/leveldb.go index f80270b..f141d48 100644 --- a/store/leveldb.go +++ b/store/leveldb.go @@ -27,7 +27,6 @@ func NewLevelDBStore(filename string) (IpStore, error) { //Options taken from ledisdb opts := &opt.Options{} opts.BlockSize = 32768 - opts.WriteBuffer = 67108864 opts.BlockCacheCapacity = 524288000 opts.OpenFilesCacheCapacity = 1024 opts.CompactionTableSize = 32 * 1024 * 1024