From 2c119a6e73557913e13182fbfa5351b53df7bbac Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Sun, 19 Feb 2023 15:43:04 +0000 Subject: [PATCH] Fixed race error. --- api/service/explorer/schema.go | 2 +- api/service/explorer/storage.go | 54 ++++++++++++++++++++++++++++++--- go.mod | 2 +- go.sum | 4 +-- scripts/go_executable_build.sh | 2 +- 5 files changed, 54 insertions(+), 10 deletions(-) diff --git a/api/service/explorer/schema.go b/api/service/explorer/schema.go index 09848573d..3104d75dc 100644 --- a/api/service/explorer/schema.go +++ b/api/service/explorer/schema.go @@ -41,7 +41,7 @@ func readCheckpointBitmap(db databaseReader) (*roaring64.Bitmap, error) { } // writeCheckpointBitmap write explorer checkpoint bitmap to storage -func writeCheckpointBitmap(db databaseWriter, rb *roaring64.Bitmap) error { +func writeCheckpointBitmap(db databaseWriter, rb Bitmap) error { bitmapByte, err := rb.MarshalBinary() if err != nil { return err diff --git a/api/service/explorer/storage.go b/api/service/explorer/storage.go index 72d028180..ebb4e2577 100644 --- a/api/service/explorer/storage.go +++ b/api/service/explorer/storage.go @@ -34,11 +34,54 @@ const ( // explorer db is doing migration and unavailable var ErrExplorerNotReady = errors.New("explorer db not ready") +type Bitmap interface { + Clone() *roaring64.Bitmap + MarshalBinary() ([]byte, error) + CheckedAdd(x uint64) bool + Contains(x uint64) bool +} + +type ThreadSafeBitmap struct { + b Bitmap + mu sync.Mutex +} + +func (a *ThreadSafeBitmap) Clone() *roaring64.Bitmap { + a.mu.Lock() + defer a.mu.Unlock() + return a.b.Clone() +} + +func (a *ThreadSafeBitmap) CheckedAdd(x uint64) bool { + a.mu.Lock() + defer a.mu.Unlock() + return a.b.CheckedAdd(x) +} + +func (a *ThreadSafeBitmap) Contains(x uint64) bool { + a.mu.Lock() + defer a.mu.Unlock() + return a.b.Contains(x) +} + +func (a *ThreadSafeBitmap) MarshalBinary() ([]byte, error) { + a.mu.Lock() + defer a.mu.Unlock() + return a.b.MarshalBinary() +} + +func NewThreadSafeBitmap(bitmap Bitmap) Bitmap { + return &ThreadSafeBitmap{ + b: bitmap, + mu: sync.Mutex{}, + } +} + type ( storage struct { db database bc core.BlockChain - rb *roaring64.Bitmap + rb Bitmap //*roaring64.Bitmap // TODO: optimize this with priority queue tm *taskManager @@ -89,11 +132,12 @@ func newStorage(hc *harmonyconfig.HarmonyConfig, bc core.BlockChain, dbPath stri return nil, err } + safeBitmap := NewThreadSafeBitmap(bitmap) return &storage{ db: db, bc: bc, - rb: bitmap, - tm: newTaskManager(bitmap), + rb: safeBitmap, + tm: newTaskManager(safeBitmap), resultC: make(chan blockResult, numWorker), resultT: make(chan *traceResult, numWorker), available: abool.New(), @@ -211,14 +255,14 @@ type taskManager struct { blocksLP []*types.Block // blocks with low priorities lock sync.Mutex - rb *roaring64.Bitmap + rb Bitmap rbChangedCount int C chan struct{} T chan *traceResult } -func newTaskManager(bitmap *roaring64.Bitmap) *taskManager { +func newTaskManager(bitmap Bitmap) *taskManager { return &taskManager{ rb: bitmap, C: make(chan struct{}, numWorker), diff --git a/go.mod b/go.mod index 4ebe35fa3..18e16a0c0 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/harmony-one/harmony go 1.19 require ( - github.com/RoaringBitmap/roaring v1.2.1 + github.com/RoaringBitmap/roaring v1.2.3 github.com/VictoriaMetrics/fastcache v1.5.7 github.com/Workiva/go-datastructures v1.0.50 github.com/allegro/bigcache v1.2.1 diff --git a/go.sum b/go.sum index 19d3d9e3d..2ebb5e94a 100644 --- a/go.sum +++ b/go.sum @@ -68,8 +68,8 @@ github.com/OneOfOne/xxhash v1.2.5 h1:zl/OfRA6nftbBK9qTohYBJ5xvw6C/oNKizR7cZGl3cI github.com/OneOfOne/xxhash v1.2.5/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= github.com/OpenPeeDeeP/depguard v1.0.1 h1:VlW4R6jmBIv3/u1JNlawEvJMM4J+dPORPaZasQee8Us= github.com/OpenPeeDeeP/depguard v1.0.1/go.mod h1:xsIw86fROiiwelg+jB2uM9PiKihMMmUx/1V+TNhjQvM= -github.com/RoaringBitmap/roaring v1.2.1 h1:58/LJlg/81wfEHd5L9qsHduznOIhyv4qb1yWcSvVq9A= -github.com/RoaringBitmap/roaring v1.2.1/go.mod h1:icnadbWcNyfEHlYdr+tDlOTih1Bf/h+rzPpv4sbomAA= +github.com/RoaringBitmap/roaring v1.2.3 h1:yqreLINqIrX22ErkKI0vY47/ivtJr6n+kMhVOVmhWBY= +github.com/RoaringBitmap/roaring v1.2.3/go.mod h1:plvDsJQpxOC5bw8LRteu/MLWHsHez/3y6cubLI4/1yE= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/VictoriaMetrics/fastcache v1.5.3/go.mod h1:+jv9Ckb+za/P1ZRg/sulP5Ni1v49daAVERr0H3CuscE= github.com/VictoriaMetrics/fastcache v1.5.7 h1:4y6y0G8PRzszQUYIQHHssv/jgPHAb5qQuuDNdCbyAgw= diff --git a/scripts/go_executable_build.sh b/scripts/go_executable_build.sh index 2c1cc99f0..6331aa522 100755 --- a/scripts/go_executable_build.sh +++ b/scripts/go_executable_build.sh @@ -21,7 +21,7 @@ VERBOSE= if [ "$(uname -s)" == "Darwin" ]; then GO_GCFLAGS="" else - GO_GCFLAGS="all=-c 2" + GO_GCFLAGS="" fi DEBUG=false STATIC=true