Merge pull request #4149 from LuttyYang/tikv

Add TiKV Storage On ExplorerNode

api/service/explorer/interface.go

@@ -1,11 +1,14 @@
 package explorer
 
 import (
+	"path"
+
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/ethereum/go-ethereum/ethdb"
-	"github.com/syndtr/goleveldb/leveldb/filter"
+	"github.com/ethereum/go-ethereum/ethdb/leveldb"
-	"github.com/syndtr/goleveldb/leveldb/opt"
+	tikvCommon "github.com/harmony-one/harmony/internal/tikv/common"
-	levelutil "github.com/syndtr/goleveldb/leveldb/util"
+	"github.com/harmony-one/harmony/internal/tikv/prefix"
+	"github.com/harmony-one/harmony/internal/tikv/remote"
 )
 
 // database is an adapter for *leveldb.DB
@@ -32,54 +35,56 @@ type batch interface {
 	ValueSize() int
 }
 
-// lvlDB is the adapter for leveldb.Database
+// explorerDB is the adapter for explorer node
-type lvlDB struct {
+type explorerDB struct {
-	db *leveldb.DB
+	db ethdb.KeyValueStore
 }
 
-func newLvlDB(dbPath string) (database, error) {
+// newExplorerLvlDB new explorer storage using leveldb
-	// https://github.com/ethereum/go-ethereum/blob/master/ethdb/leveldb/leveldb.go#L98 options.
+func newExplorerLvlDB(dbPath string) (database, error) {
-	// We had 0 for handles and cache params before, so set 0s for all of them. Filter opt is the same.
+	db, err := leveldb.New(dbPath, 16, 500, "explorer_db")
-	options := &opt.Options{
+	if err != nil {
-		OpenFilesCacheCapacity: 500,
+		return nil, err
-		BlockCacheCapacity:     8 * 1024 * 1024, // 8 MiB
-		WriteBuffer:            4 * 1024 * 1024, // 4 MiB
-		Filter:                 filter.NewBloomFilter(10),
 	}
-	db, err := leveldb.OpenFile(dbPath, options)
+	return &explorerDB{db}, nil
+}
+
+// newExplorerTiKv new explorer storage using leveldb
+func newExplorerTiKv(pdAddr []string, dbPath string, readOnly bool) (database, error) {
+	prefixStr := append([]byte(path.Base(dbPath)), '/')
+	db, err := remote.NewRemoteDatabase(pdAddr, readOnly)
 	if err != nil {
 		return nil, err
 	}
-	return &lvlDB{db}, nil
+	return &explorerDB{
+		db: tikvCommon.ToEthKeyValueStore(
+			prefix.NewPrefixDatabase(prefixStr, db),
+		),
+	}, nil
 }
 
-func (db *lvlDB) Put(key, val []byte) error {
+func (db *explorerDB) Put(key, val []byte) error {
-	return db.db.Put(key, val, nil)
+	return db.db.Put(key, val)
 }
 
-func (db *lvlDB) Get(key []byte) ([]byte, error) {
+func (db *explorerDB) Get(key []byte) ([]byte, error) {
-	return db.db.Get(key, nil)
+	return db.db.Get(key)
 }
 
-func (db *lvlDB) Has(key []byte) (bool, error) {
+func (db *explorerDB) Has(key []byte) (bool, error) {
-	return db.db.Has(key, nil)
+	return db.db.Has(key)
 }
 
-func (db *lvlDB) NewBatch() batch {
+func (db *explorerDB) NewBatch() batch {
-	batch := new(leveldb.Batch)
+	return db.db.NewBatch()
-	return &lvlBatch{
-		batch: batch,
-		db:    db.db,
-	}
 }
 
-func (db *lvlDB) NewPrefixIterator(prefix []byte) iterator {
+func (db *explorerDB) NewPrefixIterator(prefix []byte) iterator {
-	rng := levelutil.BytesPrefix(prefix)
+	it := db.db.NewIteratorWithPrefix(prefix)
-	it := db.db.NewIterator(rng, nil)
 	return it
 }
 
-func (db *lvlDB) NewSizedIterator(start []byte, size int) iterator {
+func (db *explorerDB) NewSizedIterator(start []byte, size int) iterator {
 	return db.newSizedIterator(start, size)
 }
 
@@ -89,9 +94,8 @@ type sizedIterator struct {
 	sizeLimit int
 }
 
-func (db *lvlDB) newSizedIterator(start []byte, size int) *sizedIterator {
+func (db *explorerDB) newSizedIterator(start []byte, size int) *sizedIterator {
-	rng := &levelutil.Range{Start: start, Limit: nil}
+	it := db.db.NewIteratorWithStart(start)
-	it := db.db.NewIterator(rng, nil)
 	return &sizedIterator{
 		it:        it,
 		curIndex:  0,
@@ -112,31 +116,6 @@ func (it *sizedIterator) Value() []byte { return it.it.Value() }
 func (it *sizedIterator) Release()      { it.it.Release() }
 func (it *sizedIterator) Error() error  { return it.it.Error() }
 
-// Note: lvlBatch is not thread safe
-type lvlBatch struct {
-	batch     *leveldb.Batch
-	db        *leveldb.DB
-	valueSize int
-}
-
-func (b *lvlBatch) Put(key, val []byte) error {
-	b.batch.Put(key, val)
-	b.valueSize += len(val)
-	return nil
-}
-
-func (b *lvlBatch) Write() error {
-	if err := b.db.Write(b.batch, nil); err != nil {
-		return err
-	}
-	b.valueSize = 0
-	return nil
-}
-
-func (b *lvlBatch) ValueSize() int {
-	return b.valueSize
-}
-
 type iterator interface {
 	Next() bool
 	Key() []byte

api/service/explorer/interface_test.go

@@ -73,7 +73,7 @@ func TestLevelDBPrefixIterator(t *testing.T) {
 
 func newTestLevelDB(t *testing.T, i int) database {
 	dbDir := tempTestDir(t, i)
-	db, err := newLvlDB(dbDir)
+	db, err := newExplorerLvlDB(dbDir)
 	if err != nil {
 		t.Fatal(err)
 	}

api/service/explorer/migration_test.go

@@ -40,11 +40,6 @@ type migrationDBFactory struct {
 
 func (f *migrationDBFactory) makeDB(numAddr int) database {
 	db := newTestLevelDB(f.t, 0)
-	for i := 0; i != 10000; i++ {
-		if err := writeCheckpoint(db, uint64(i)); err != nil {
-			f.t.Fatal(err)
-		}
-	}
 	for i := 0; i != numAddr; i++ {
 		addr := f.newAddress()
 		addrInfo := &Address{ID: string(addr)}

api/service/explorer/schema.go

@@ -3,8 +3,8 @@ package explorer
 import (
 	"encoding/binary"
 	"fmt"
-	"math/big"
 
+	"github.com/RoaringBitmap/roaring/roaring64"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/rlp"
 	goversion "github.com/hashicorp/go-version"
@@ -15,26 +15,39 @@ import (
 
 const (
 	LegAddressPrefix = "ad_"
-	CheckpointPrefix = "dc"
+	CheckpointBitmap = "checkpoint_bitmap"
 	TracePrefix      = "tr_"
 
 	oneAddrByteLen = 42 // byte size of string "one1..."
 )
 
-// Common schema
+// readCheckpointBitmap read explorer checkpoint bitmap from storage
-// GetCheckpointKey ...
+func readCheckpointBitmap(db databaseReader) (*roaring64.Bitmap, error) {
-func GetCheckpointKey(blockNum *big.Int) []byte {
+	bitmapByte, err := db.Get([]byte(CheckpointBitmap))
-	return []byte(fmt.Sprintf("%s_%x", CheckpointPrefix, blockNum))
+	if err != nil {
-}
+		if err == leveldb.ErrNotFound {
+			return roaring64.NewBitmap(), nil
+		}
+		return nil, err
+	}
 
-func isBlockComputedInDB(db databaseReader, bn uint64) (bool, error) {
+	rb := roaring64.NewBitmap()
-	key := GetCheckpointKey(new(big.Int).SetUint64(bn))
+	err = rb.UnmarshalBinary(bitmapByte)
-	return db.Has(key)
+	if err != nil {
+		return nil, err
+	}
+
+	return rb, nil
 }
 
-func writeCheckpoint(db databaseWriter, bn uint64) error {
+// writeCheckpointBitmap write explorer checkpoint bitmap to storage
-	blockCheckpoint := GetCheckpointKey(new(big.Int).SetUint64(bn))
+func writeCheckpointBitmap(db databaseWriter, rb *roaring64.Bitmap) error {
-	return db.Put(blockCheckpoint, []byte{})
+	bitmapByte, err := rb.MarshalBinary()
+	if err != nil {
+		return err
+	}
+
+	return db.Put([]byte(CheckpointBitmap), bitmapByte)
 }
 
 func getTraceResultKey(key []byte) []byte {

api/service/explorer/service.go

@@ -11,6 +11,9 @@ import (
 	"strconv"
 	"time"
 
+	"github.com/RoaringBitmap/roaring/roaring64"
+	harmonyconfig "github.com/harmony-one/harmony/internal/configs/harmony"
+
 	ethCommon "github.com/ethereum/go-ethereum/common"
 	"github.com/gorilla/mux"
 	msg_pb "github.com/harmony-one/harmony/api/proto/message"
@@ -42,20 +45,21 @@ type HTTPError struct {
 
 // Service is the struct for explorer service.
 type Service struct {
-	router      *mux.Router
+	router        *mux.Router
-	IP          string
+	IP            string
-	Port        string
+	Port          string
-	storage     *storage
+	storage       *storage
-	server      *http.Server
+	server        *http.Server
-	messageChan chan *msg_pb.Message
+	messageChan   chan *msg_pb.Message
-	blockchain  core.BlockChain
+	blockchain    core.BlockChain
-	backend     hmy.NodeAPI
+	backend       hmy.NodeAPI
+	harmonyConfig *harmonyconfig.HarmonyConfig
 }
 
 // New returns explorer service.
-func New(selfPeer *p2p.Peer, bc core.BlockChain, backend hmy.NodeAPI) *Service {
+func New(harmonyConfig *harmonyconfig.HarmonyConfig, selfPeer *p2p.Peer, bc core.BlockChain, backend hmy.NodeAPI) *Service {
 	dbPath := defaultDBPath(selfPeer.IP, selfPeer.Port)
-	storage, err := newStorage(bc, dbPath)
+	storage, err := newStorage(harmonyConfig, bc, dbPath)
 	if err != nil {
 		utils.Logger().Fatal().Err(err).Msg("cannot open explorer DB")
 	}
@@ -335,6 +339,11 @@ func (s *Service) SetMessageChan(messageChan chan *msg_pb.Message) {
 	s.messageChan = messageChan
 }
 
+// GetCheckpointBitmap get explorer checkpoint bitmap
+func (s *Service) GetCheckpointBitmap() *roaring64.Bitmap {
+	return s.storage.rb.Clone()
+}
+
 func defaultDBPath(ip, port string) string {
 	return path.Join(nodeconfig.GetDefaultConfig().DBDir, "explorer_storage_"+ip+"_"+port)
 }

api/service/explorer/storage.go

@@ -9,6 +9,10 @@ import (
 	"sync"
 	"time"
 
+	"github.com/RoaringBitmap/roaring/roaring64"
+	harmonyconfig "github.com/harmony-one/harmony/internal/configs/harmony"
+	"github.com/harmony-one/harmony/internal/tikv"
+
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/harmony-one/abool"
 	"github.com/harmony-one/harmony/core"
@@ -22,7 +26,8 @@ import (
 )
 
 const (
-	numWorker = 8
+	numWorker        = 8
+	changedSaveCount = 100
 )
 
 // ErrExplorerNotReady is the error when querying explorer db data when
@@ -33,6 +38,7 @@ type (
 	storage struct {
 		db database
 		bc core.BlockChain
+		rb *roaring64.Bitmap
 
 		// TODO: optimize this with priority queue
 		tm      *taskManager
@@ -55,17 +61,39 @@ type (
 	}
 )
 
-func newStorage(bc core.BlockChain, dbPath string) (*storage, error) {
+func newExplorerDB(hc *harmonyconfig.HarmonyConfig, dbPath string) (database, error) {
-	utils.Logger().Info().Msg("explorer storage folder: " + dbPath)
+	if hc.General.RunElasticMode {
-	db, err := newLvlDB(dbPath)
+		// init the storage using tikv
+		dbPath = fmt.Sprintf("explorer_tikv_%d", hc.General.ShardID)
+		readOnly := hc.TiKV.Role == tikv.RoleReader
+		utils.Logger().Info().Msg("explorer storage in tikv: " + dbPath)
+		return newExplorerTiKv(hc.TiKV.PDAddr, dbPath, readOnly)
+	} else {
+		// or leveldb
+		utils.Logger().Info().Msg("explorer storage folder: " + dbPath)
+		return newExplorerLvlDB(dbPath)
+	}
+}
+
+func newStorage(hc *harmonyconfig.HarmonyConfig, bc core.BlockChain, dbPath string) (*storage, error) {
+	db, err := newExplorerDB(hc, dbPath)
+	if err != nil {
+		utils.Logger().Error().Err(err).Msg("Failed to create new explorer database")
+		return nil, err
+	}
+
+	// load checkpoint roaring bitmap from storage
+	// roaring bitmap is a very high compression bitmap, in our scene, 1 million blocks use almost 1kb storage
+	bitmap, err := readCheckpointBitmap(db)
 	if err != nil {
-		utils.Logger().Error().Err(err).Msg("Failed to create new database")
 		return nil, err
 	}
+
 	return &storage{
 		db:        db,
 		bc:        bc,
-		tm:        newTaskManager(),
+		rb:        bitmap,
+		tm:        newTaskManager(bitmap),
 		resultC:   make(chan blockResult, numWorker),
 		resultT:   make(chan *traceResult, numWorker),
 		available: abool.New(),
@@ -79,6 +107,7 @@ func (s *storage) Start() {
 }
 
 func (s *storage) Close() {
+	_ = writeCheckpointBitmap(s.db, s.rb)
 	close(s.closeC)
 }
 
@@ -182,14 +211,18 @@ type taskManager struct {
 	blocksLP []*types.Block // blocks with low priorities
 	lock     sync.Mutex
 
+	rb             *roaring64.Bitmap
+	rbChangedCount int
+
 	C chan struct{}
 	T chan *traceResult
 }
 
-func newTaskManager() *taskManager {
+func newTaskManager(bitmap *roaring64.Bitmap) *taskManager {
 	return &taskManager{
-		C: make(chan struct{}, numWorker),
+		rb: bitmap,
-		T: make(chan *traceResult, numWorker),
+		C:  make(chan struct{}, numWorker),
+		T:  make(chan *traceResult, numWorker),
 	}
 }
 
@@ -245,6 +278,30 @@ func (tm *taskManager) PullTask() *types.Block {
 	return nil
 }
 
+// markBlockDone mark block processed done when explorer computed one block
+func (tm *taskManager) markBlockDone(btc batch, blockNum uint64) {
+	tm.lock.Lock()
+	defer tm.lock.Unlock()
+
+	if tm.rb.CheckedAdd(blockNum) {
+		tm.rbChangedCount++
+
+		// every 100 change write once
+		if tm.rbChangedCount == changedSaveCount {
+			tm.rbChangedCount = 0
+			_ = writeCheckpointBitmap(btc, tm.rb)
+		}
+	}
+}
+
+// markBlockDone check block is processed done
+func (tm *taskManager) hasBlockDone(blockNum uint64) bool {
+	tm.lock.Lock()
+	defer tm.lock.Unlock()
+
+	return tm.rb.Contains(blockNum)
+}
+
 func (s *storage) makeWorkersAndStart() {
 	workers := make([]*blockComputer, 0, numWorker)
 	for i := 0; i != numWorker; i++ {
@@ -321,9 +378,8 @@ LOOP:
 }
 
 func (bc *blockComputer) computeBlock(b *types.Block) (*blockResult, error) {
-	is, err := isBlockComputedInDB(bc.db, b.NumberU64())
+	if bc.tm.hasBlockDone(b.NumberU64()) {
-	if is || err != nil {
+		return nil, nil
-		return nil, err
 	}
 	btc := bc.db.NewBatch()
 
@@ -333,7 +389,7 @@ func (bc *blockComputer) computeBlock(b *types.Block) (*blockResult, error) {
 	for _, stk := range b.StakingTransactions() {
 		bc.computeStakingTx(btc, b, stk)
 	}
-	_ = writeCheckpoint(btc, b.NumberU64())
+	bc.tm.markBlockDone(btc, b.NumberU64())
 	return &blockResult{
 		btc: btc,
 		bn:  b.NumberU64(),

api/service/prometheus/service.go

@@ -8,6 +8,7 @@ import (
 	"net/http"
 	"runtime/debug"
 	"runtime/pprof"
+	"strings"
 	"sync"
 	"time"
 
@@ -30,13 +31,18 @@ type Config struct {
 	Legacy     bool   // legacy or not, legacy is harmony internal node
 	NodeType   string // node type, validator or exlorer node
 	Shard      uint32 // shard id, used as job suffix
-	Instance   string //identifier of the instance in prometheus metrics
+	Instance   string // identifier of the instance in prometheus metrics
+	TikvRole   string // use for tikv explorer node
 }
 
 func (p Config) String() string {
 	return fmt.Sprintf("%v, %v:%v, %v/%v, %v/%v/%v/%v:%v", p.Enabled, p.IP, p.Port, p.EnablePush, p.Gateway, p.Network, p.Legacy, p.NodeType, p.Shard, p.Instance)
 }
 
+func (p Config) IsUsedTiKV() bool {
+	return p.TikvRole != ""
+}
+
 // Service provides Prometheus metrics via the /metrics route. This route will
 // show all the metrics registered with the Prometheus DefaultRegisterer.
 type Service struct {
@@ -63,8 +69,9 @@ var (
 func (s *Service) getJobName() string {
 	var node string
 
-	// legacy nodes are harmony nodes: s0,s1,s2,s3
+	if s.config.IsUsedTiKV() { // tikv node must be explorer node, eg: te_reader0, te_writer0
-	if s.config.Legacy {
+		node = "te_" + strings.ToLower(s.config.TikvRole)
+	} else if s.config.Legacy { // legacy nodes are harmony nodes: s0,s1,s2,s3
 		node = "s"
 	} else {
 		if s.config.NodeType == "validator" {

cmd/harmony/main.go

@@ -15,6 +15,10 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/harmony-one/harmony/internal/shardchain/tikv_manage"
+	"github.com/harmony-one/harmony/internal/tikv/redis_helper"
+	"github.com/harmony-one/harmony/internal/tikv/statedb_cache"
+
 	"github.com/harmony-one/harmony/api/service/crosslink_sending"
 	rosetta_common "github.com/harmony-one/harmony/rosetta/common"
 
@@ -297,6 +301,11 @@ func setupNodeAndRun(hc harmonyconfig.HarmonyConfig) {
 		os.Exit(1)
 	}
 
+	if hc.General.RunElasticMode && hc.TiKV == nil {
+		fmt.Fprintf(os.Stderr, "Use TIKV MUST HAS TIKV CONFIG")
+		os.Exit(1)
+	}
+
 	// Update ethereum compatible chain ids
 	params.UpdateEthChainIDByShard(nodeConfig.ShardID)
 
@@ -427,6 +436,8 @@ func setupNodeAndRun(hc harmonyconfig.HarmonyConfig) {
 		currentNode.StartGRPCSyncClient()
 	}
 
+	currentNode.NodeSyncing()
+
 	if err := currentNode.StartServices(); err != nil {
 		fmt.Fprint(os.Stderr, err.Error())
 		os.Exit(-1)
@@ -680,7 +691,9 @@ func setupConsensusAndNode(hc harmonyconfig.HarmonyConfig, nodeConfig *nodeconfi
 
 	// Current node.
 	var chainDBFactory shardchain.DBFactory
-	if hc.ShardData.EnableShardData {
+	if hc.General.RunElasticMode {
+		chainDBFactory = setupTiKV(hc)
+	} else if hc.ShardData.EnableShardData {
 		chainDBFactory = &shardchain.LDBShardFactory{
 			RootDir:    nodeConfig.DBDir,
 			DiskCount:  hc.ShardData.DiskCount,
@@ -754,6 +767,28 @@ func setupConsensusAndNode(hc harmonyconfig.HarmonyConfig, nodeConfig *nodeconfi
 	return currentNode
 }
 
+func setupTiKV(hc harmonyconfig.HarmonyConfig) shardchain.DBFactory {
+	err := redis_helper.Init(hc.TiKV.StateDBRedisServerAddr)
+	if err != nil {
+		panic("can not connect to redis: " + err.Error())
+	}
+
+	factory := &shardchain.TiKvFactory{
+		PDAddr: hc.TiKV.PDAddr,
+		Role:   hc.TiKV.Role,
+		CacheConfig: statedb_cache.StateDBCacheConfig{
+			CacheSizeInMB:        hc.TiKV.StateDBCacheSizeInMB,
+			CachePersistencePath: hc.TiKV.StateDBCachePersistencePath,
+			RedisServerAddr:      hc.TiKV.StateDBRedisServerAddr,
+			RedisLRUTimeInDay:    hc.TiKV.StateDBRedisLRUTimeInDay,
+			DebugHitRate:         hc.TiKV.Debug,
+		},
+	}
+
+	tikv_manage.SetDefaultTiKVFactory(factory)
+	return factory
+}
+
 func processNodeType(hc harmonyconfig.HarmonyConfig, currentNode *node.Node, currentConsensus *consensus.Consensus) {
 	switch hc.General.NodeType {
 	case nodeTypeExplorer:
@@ -796,6 +831,11 @@ func setupPrometheusService(node *node.Node, hc harmonyconfig.HarmonyConfig, sid
 		Shard:      sid,
 		Instance:   myHost.GetID().Pretty(),
 	}
+
+	if hc.General.RunElasticMode {
+		prometheusConfig.TikvRole = hc.TiKV.Role
+	}
+
 	p := prometheus.NewService(prometheusConfig)
 	node.RegisterService(service.Prometheus, p)
 }

core/blockchain.go

@@ -13,7 +13,9 @@ import (
 	"github.com/harmony-one/harmony/core/state"
 	"github.com/harmony-one/harmony/core/types"
 	"github.com/harmony-one/harmony/core/vm"
+	harmonyconfig "github.com/harmony-one/harmony/internal/configs/harmony"
 	"github.com/harmony-one/harmony/internal/params"
+	"github.com/harmony-one/harmony/internal/tikv/redis_helper"
 	"github.com/harmony-one/harmony/shard"
 	"github.com/harmony-one/harmony/staking/slash"
 	types2 "github.com/harmony-one/harmony/staking/types"
@@ -331,4 +333,17 @@ type BlockChain interface {
 		payout reward.Reader,
 		state *state.DB,
 	) (status WriteStatus, err error)
+
+	// ========== Only For Tikv Start ==========
+
+	// return true if is tikv writer master
+	IsTikvWriterMaster() bool
+	// RedisPreempt used for tikv mode, get the redis preempt instance
+	RedisPreempt() *redis_helper.RedisPreempt
+	// SyncFromTiKVWriter used for tikv mode, all reader or follower writer used to sync block from master writer
+	SyncFromTiKVWriter(newBlkNum uint64, logs []*types.Log) error
+	// InitTiKV used for tikv mode, init the tikv mode
+	InitTiKV(conf *harmonyconfig.TiKVConfig)
+
+	// ========== Only For Tikv End ==========
 }

core/blockchain_impl.go

@@ -22,8 +22,10 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"log"
 	"math/big"
 	"os"
+	"strconv"
 	"strings"
 	"sync"
 	"sync/atomic"
@@ -45,7 +47,10 @@ import (
 	"github.com/harmony-one/harmony/core/state"
 	"github.com/harmony-one/harmony/core/types"
 	"github.com/harmony-one/harmony/core/vm"
+	harmonyconfig "github.com/harmony-one/harmony/internal/configs/harmony"
 	"github.com/harmony-one/harmony/internal/params"
+	"github.com/harmony-one/harmony/internal/tikv"
+	"github.com/harmony-one/harmony/internal/tikv/redis_helper"
 	"github.com/harmony-one/harmony/internal/utils"
 	"github.com/harmony-one/harmony/numeric"
 	"github.com/harmony-one/harmony/shard"
@@ -95,6 +100,7 @@ const (
 	maxTimeFutureBlocks                = 30
 	badBlockLimit                      = 10
 	triesInMemory                      = 128
+	triesInRedis                       = 1000
 	shardCacheLimit                    = 10
 	commitsCacheLimit                  = 10
 	epochCacheLimit                    = 10
@@ -129,6 +135,14 @@ type BlockChainImpl struct {
 	triegc *prque.Prque   // Priority queue mapping block numbers to tries to gc
 	gcproc time.Duration  // Accumulates canonical block processing for trie dumping
 
+	// The following two variables are used to clean up the cache of redis in tikv mode.
+	// This can improve the cache hit rate of redis
+	latestCleanCacheNum uint64      // The most recently cleaned cache of block num
+	cleanCacheChan      chan uint64 // Used to notify blocks that will be cleaned up
+
+	// redisPreempt used in tikv mode, write nodes preempt for write permissions and publish updates to reader nodes
+	redisPreempt *redis_helper.RedisPreempt
+
 	hc            *HeaderChain
 	trace         bool       // atomic?
 	traceFeed     event.Feed // send trace_block result to explorer
@@ -185,26 +199,26 @@ type BlockChainImpl struct {
 
 // NewBlockChainWithOptions same as NewBlockChain but can accept additional behaviour options.
 func NewBlockChainWithOptions(
-	db ethdb.Database, cacheConfig *CacheConfig, chainConfig *params.ChainConfig,
+	db ethdb.Database, stateCache state.Database, cacheConfig *CacheConfig, chainConfig *params.ChainConfig,
 	engine consensus_engine.Engine, vmConfig vm.Config,
 	shouldPreserve func(block *types.Block) bool, options Options,
 ) (*BlockChainImpl, error) {
-	return newBlockChainWithOptions(db, cacheConfig, chainConfig, engine, vmConfig, shouldPreserve, options)
+	return newBlockChainWithOptions(db, stateCache, cacheConfig, chainConfig, engine, vmConfig, shouldPreserve, options)
 }
 
 // NewBlockChain returns a fully initialised block chain using information
 // available in the database. It initialises the default Ethereum validator and
 // Processor.
 func NewBlockChain(
-	db ethdb.Database, cacheConfig *CacheConfig, chainConfig *params.ChainConfig,
+	db ethdb.Database, stateCache state.Database, cacheConfig *CacheConfig, chainConfig *params.ChainConfig,
 	engine consensus_engine.Engine, vmConfig vm.Config,
 	shouldPreserve func(block *types.Block) bool,
 ) (*BlockChainImpl, error) {
-	return newBlockChainWithOptions(db, cacheConfig, chainConfig, engine, vmConfig, shouldPreserve, Options{})
+	return newBlockChainWithOptions(db, stateCache, cacheConfig, chainConfig, engine, vmConfig, shouldPreserve, Options{})
 }
 
 func newBlockChainWithOptions(
-	db ethdb.Database, cacheConfig *CacheConfig, chainConfig *params.ChainConfig,
+	db ethdb.Database, stateCache state.Database, cacheConfig *CacheConfig, chainConfig *params.ChainConfig,
 	engine consensus_engine.Engine, vmConfig vm.Config,
 	shouldPreserve func(block *types.Block) bool, options Options) (*BlockChainImpl, error) {
 	if cacheConfig == nil {
@@ -235,7 +249,7 @@ func newBlockChainWithOptions(
 		cacheConfig:                   cacheConfig,
 		db:                            db,
 		triegc:                        prque.New(nil),
-		stateCache:                    state.NewDatabase(db),
+		stateCache:                    stateCache,
 		quit:                          make(chan struct{}),
 		shouldPreserve:                shouldPreserve,
 		bodyCache:                     bodyCache,
@@ -355,6 +369,7 @@ func (bc *BlockChainImpl) loadLastState() error {
 		}
 	}
 	// Everything seems to be fine, set as the head block
+	bc.latestCleanCacheNum = currentBlock.NumberU64() - triesInRedis
 	bc.currentBlock.Store(currentBlock)
 	headBlockGauge.Update(int64(currentBlock.NumberU64()))
 
@@ -689,6 +704,30 @@ func (bc *BlockChainImpl) writeHeadBlock(block *types.Block) error {
 	return nil
 }
 
+// tikvFastForward writes a new head block in tikv mode, used for reader node or follower writer node
+func (bc *BlockChainImpl) tikvFastForward(block *types.Block, logs []*types.Log) error {
+	bc.currentBlock.Store(block)
+	headBlockGauge.Update(int64(block.NumberU64()))
+
+	if err := bc.hc.SetCurrentHeader(block.Header()); err != nil {
+		return errors.Wrap(err, "HeaderChain SetCurrentHeader")
+	}
+
+	bc.currentFastBlock.Store(block)
+	headFastBlockGauge.Update(int64(block.NumberU64()))
+
+	var events []interface{}
+	events = append(events, ChainEvent{block, block.Hash(), logs})
+	events = append(events, ChainHeadEvent{block})
+
+	if block.NumberU64() > triesInRedis {
+		bc.latestCleanCacheNum = block.NumberU64() - triesInRedis
+	}
+
+	bc.PostChainEvents(events, logs)
+	return nil
+}
+
 // insert injects a new head block into the current block chain. This method
 // assumes that the block is indeed a true head. It will also reset the head
 // header and the head fast sync block to this very same block if they are older
@@ -846,6 +885,10 @@ func (bc *BlockChainImpl) TrieNode(hash common.Hash) ([]byte, error) {
 }
 
 func (bc *BlockChainImpl) Stop() {
+	if bc == nil {
+		return
+	}
+
 	if !atomic.CompareAndSwapInt32(&bc.running, 0, 1) {
 		return
 	}
@@ -1184,6 +1227,14 @@ func (bc *BlockChainImpl) WriteBlockWithState(
 			}
 			return NonStatTy, err
 		}
+
+		// clean block tire info in redis, used for tikv mode
+		if block.NumberU64() > triesInRedis {
+			select {
+			case bc.cleanCacheChan <- block.NumberU64() - triesInRedis:
+			default:
+			}
+		}
 	} else {
 		// Full but not archive node, do proper garbage collection
 		triedb.Reference(root, common.Hash{}) // metadata reference to keep trie alive
@@ -1300,8 +1351,17 @@ func (bc *BlockChainImpl) GetMaxGarbageCollectedBlockNumber() int64 {
 }
 
 func (bc *BlockChainImpl) InsertChain(chain types.Blocks, verifyHeaders bool) (int, error) {
+	// if in tikv mode, writer node need preempt master or come be a follower
+	if bc.isInitTiKV() && !bc.tikvPreemptMaster(bc.rangeBlock(chain)) {
+		return len(chain), nil
+	}
+
 	n, events, logs, err := bc.insertChain(chain, verifyHeaders)
 	bc.PostChainEvents(events, logs)
+	if bc.isInitTiKV() && err != nil {
+		// if has some error, master writer node will release the permission
+		_, _ = bc.redisPreempt.Unlock()
+	}
 	return n, err
 }
 
@@ -1542,6 +1602,14 @@ func (bc *BlockChainImpl) insertChain(chain types.Blocks, verifyHeaders bool) (i
 			events = append(events, ChainEvent{block, block.Hash(), logs})
 			lastCanon = block
 
+			// used for tikv mode, writer node will publish update to all reader node
+			if bc.isInitTiKV() {
+				err = redis_helper.PublishShardUpdate(bc.ShardID(), block.NumberU64(), logs)
+				if err != nil {
+					utils.Logger().Info().Err(err).Msg("redis publish shard update error")
+				}
+			}
+
 			// Only count canonical blocks for GC processing time
 			bc.gcproc += proctime
 		}
@@ -3034,6 +3102,172 @@ func (bc *BlockChainImpl) IsEnablePruneBeaconChainFeature() bool {
 	return bc.pruneBeaconChainEnable
 }
 
+// SyncFromTiKVWriter used for tikv mode, all reader or follower writer used to sync block from master writer
+func (bc *BlockChainImpl) SyncFromTiKVWriter(newBlkNum uint64, logs []*types.Log) error {
+	head := rawdb.ReadHeadBlockHash(bc.db)
+	dbBlock := bc.GetBlockByHash(head)
+	currentBlock := bc.CurrentBlock()
+
+	if dbBlock == nil || currentBlock == nil {
+		return nil
+	}
+
+	currentBlockNum := currentBlock.NumberU64()
+	if currentBlockNum < newBlkNum {
+		start := time.Now()
+		for i := currentBlockNum; i <= newBlkNum; i++ {
+			blk := bc.GetBlockByNumber(i)
+			if blk == nil {
+				// cluster synchronization may be in progress
+				utils.Logger().Warn().
+					Uint64("currentBlockNum", i).
+					Msg("[tikv sync] sync from writer got block nil, cluster synchronization may be in progress")
+				return nil
+			}
+
+			err := bc.tikvFastForward(blk, logs)
+			if err != nil {
+				return err
+			}
+		}
+		utils.Logger().Info().
+			Uint64("currentBlockNum", currentBlockNum).
+			Dur("usedTime", time.Now().Sub(start)).
+			Msg("[tikv sync] sync from writer")
+	}
+
+	return nil
+}
+
+// tikvCleanCache used for tikv mode, clean block tire data from redis
+func (bc *BlockChainImpl) tikvCleanCache() {
+	var count int
+	for to := range bc.cleanCacheChan {
+		for i := bc.latestCleanCacheNum + 1; i <= to; i++ {
+			// build previous block statedb
+			fromBlock := bc.GetBlockByNumber(i)
+			fromTrie, err := state.New(fromBlock.Root(), bc.stateCache)
+			if err != nil {
+				continue
+			}
+
+			// build current block statedb
+			toBlock := bc.GetBlockByNumber(i + 1)
+			toTrie, err := state.New(toBlock.Root(), bc.stateCache)
+			if err != nil {
+				continue
+			}
+
+			// diff two statedb and delete redis cache
+			start := time.Now()
+			count, err = fromTrie.DiffAndCleanCache(bc.ShardID(), toTrie)
+			if err != nil {
+				utils.Logger().Warn().
+					Err(err).
+					Msg("[tikv clean cache] error")
+				break
+			}
+
+			utils.Logger().Info().
+				Uint64("blockNum", i).
+				Int("removeCacheEntriesCount", count).
+				Dur("usedTime", time.Now().Sub(start)).
+				Msg("[tikv clean cache] success")
+			bc.latestCleanCacheNum = i
+		}
+	}
+}
+
+func (bc *BlockChainImpl) isInitTiKV() bool {
+	return bc.redisPreempt != nil
+}
+
+// tikvPreemptMaster used for tikv mode, writer node need preempt master or come be a follower
+func (bc *BlockChainImpl) tikvPreemptMaster(fromBlock, toBlock uint64) bool {
+	for {
+		// preempt master
+		if ok, _ := bc.redisPreempt.TryLock(60); ok {
+			return true
+		}
+
+		// follower
+		if bc.CurrentBlock().NumberU64() >= toBlock {
+			return false
+		}
+
+		time.Sleep(time.Second)
+	}
+}
+
+// rangeBlock get the block range of blocks
+func (bc *BlockChainImpl) rangeBlock(blocks types.Blocks) (uint64, uint64) {
+	if len(blocks) == 0 {
+		return 0, 0
+	}
+	max := blocks[0].NumberU64()
+	min := max
+	for _, tmpBlock := range blocks {
+		if tmpBlock.NumberU64() > max {
+			max = tmpBlock.NumberU64()
+		} else if tmpBlock.NumberU64() < min {
+			min = tmpBlock.NumberU64()
+		}
+	}
+
+	return min, max
+}
+
+// RedisPreempt used for tikv mode, get the redis preempt instance
+func (bc *BlockChainImpl) RedisPreempt() *redis_helper.RedisPreempt {
+	if bc == nil {
+		return nil
+	}
+	return bc.redisPreempt
+}
+
+func (bc *BlockChainImpl) IsTikvWriterMaster() bool {
+	if bc == nil || bc.redisPreempt == nil {
+		return false
+	}
+
+	return bc.redisPreempt.LastLockStatus()
+}
+
+// InitTiKV used for tikv mode, init the tikv mode
+func (bc *BlockChainImpl) InitTiKV(conf *harmonyconfig.TiKVConfig) {
+	bc.cleanCacheChan = make(chan uint64, 10)
+
+	if conf.Role == tikv.RoleWriter {
+		// only writer need preempt permission
+		bc.redisPreempt = redis_helper.CreatePreempt(fmt.Sprintf("shard_%d_preempt", bc.ShardID()))
+	}
+
+	if conf.Debug {
+		// used for init redis cache
+		// If redis is empty, the hit rate will be too low and the synchronization block speed will be slow
+		// set LOAD_PRE_FETCH is yes can significantly improve this.
+		if os.Getenv("LOAD_PRE_FETCH") == "yes" {
+			if trie, err := state.New(bc.CurrentBlock().Root(), bc.stateCache); err == nil {
+				trie.Prefetch(512)
+			} else {
+				log.Println("LOAD_PRE_FETCH ERR: ", err)
+			}
+		}
+
+		// If redis is empty, there is no need to clear the cache of nearly 1000 blocks
+		// set CLEAN_INIT is the latest block num can skip slow and unneeded cleanup process
+		if os.Getenv("CLEAN_INIT") != "" {
+			from, err := strconv.Atoi(os.Getenv("CLEAN_INIT"))
+			if err == nil {
+				bc.latestCleanCacheNum = uint64(from)
+			}
+		}
+	}
+
+	// start clean block tire data process
+	go bc.tikvCleanCache()
+}
+
 var (
 	leveldbErrSpec         = "leveldb"
 	tooManyOpenFilesErrStr = "Too many open files"

core/blockchain_stub.go

@@ -13,7 +13,9 @@ import (
 	"github.com/harmony-one/harmony/core/state"
 	"github.com/harmony-one/harmony/core/types"
 	"github.com/harmony-one/harmony/core/vm"
+	harmonyconfig "github.com/harmony-one/harmony/internal/configs/harmony"
 	"github.com/harmony-one/harmony/internal/params"
+	"github.com/harmony-one/harmony/internal/tikv/redis_helper"
 	"github.com/harmony-one/harmony/shard"
 	"github.com/harmony-one/harmony/staking/slash"
 	staking "github.com/harmony-one/harmony/staking/types"
@@ -400,3 +402,19 @@ func (a Stub) IsEnablePruneBeaconChainFeature() bool {
 func (a Stub) CommitOffChainData(batch rawdb.DatabaseWriter, block *types.Block, receipts []*types.Receipt, cxReceipts []*types.CXReceipt, stakeMsgs []staking.StakeMsg, payout reward.Reader, state *state.DB) (status WriteStatus, err error) {
 	return 0, errors.Errorf("method CommitOffChainData not implemented for %s", a.Name)
 }
+
+func (a Stub) IsTikvWriterMaster() bool {
+	return false
+}
+
+func (a Stub) RedisPreempt() *redis_helper.RedisPreempt {
+	return nil
+}
+
+func (a Stub) SyncFromTiKVWriter(newBlkNum uint64, logs []*types.Log) error {
+	return errors.Errorf("method SyncFromTiKVWriter not implemented for %s", a.Name)
+}
+
+func (a Stub) InitTiKV(conf *harmonyconfig.TiKVConfig) {
+	return
+}

core/evm_test.go

@@ -46,7 +46,7 @@ func getTestEnvironment(testBankKey ecdsa.PrivateKey) (*BlockChainImpl, *state.D
 	genesis := gspec.MustCommit(database)
 
 	// fake blockchain
-	chain, _ := NewBlockChain(database, nil, gspec.Config, engine, vm.Config{}, nil)
+	chain, _ := NewBlockChain(database, state.NewDatabase(database), nil, gspec.Config, engine, vm.Config{}, nil)
 	db, _ := chain.StateAt(genesis.Root())
 
 	// make a fake block header (use epoch 1 so that locked tokens can be tested)

core/state/prefeth.go

@@ -0,0 +1,167 @@
+package state
+
+import (
+	"bytes"
+	"log"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/trie"
+	"github.com/harmony-one/harmony/internal/utils"
+)
+
+type prefetchJob struct {
+	accountAddr []byte
+	account     *Account
+	start, end  []byte
+}
+
+// Prefetch If redis is empty, the hit rate will be too low and the synchronization block speed will be slow
+// this function will parallel load the latest block statedb to redis
+// this function used by debug or first time to init tikv cluster
+func (s *DB) Prefetch(parallel int) {
+	wg := sync.WaitGroup{}
+
+	jobChan := make(chan *prefetchJob, 10000)
+	waitWorker := int64(parallel)
+
+	// start parallel workers
+	for i := 0; i < parallel; i++ {
+		go func() {
+			defer wg.Done()
+			for job := range jobChan {
+				atomic.AddInt64(&waitWorker, -1)
+				s.prefetchWorker(job, jobChan)
+				atomic.AddInt64(&waitWorker, 1)
+			}
+		}()
+
+		wg.Add(1)
+	}
+
+	// add first jobs
+	for i := 0; i < 255; i++ {
+		start := []byte{byte(i)}
+		end := []byte{byte(i + 1)}
+		if i == parallel-1 {
+			end = nil
+		} else if i == 0 {
+			start = nil
+		}
+
+		jobChan <- &prefetchJob{
+			start: start,
+			end:   end,
+		}
+	}
+
+	// wait all worker done
+	start := time.Now()
+	log.Println("Prefetch start")
+	var sleepCount int
+	for sleepCount < 3 {
+		time.Sleep(time.Second)
+		waitWorkerCount := atomic.LoadInt64(&waitWorker)
+		if waitWorkerCount >= int64(parallel) {
+			sleepCount++
+		} else {
+			sleepCount = 0
+		}
+	}
+
+	close(jobChan)
+	wg.Wait()
+	end := time.Now()
+	log.Println("Prefetch end, use", end.Sub(start))
+}
+
+// prefetchWorker used to process one job
+func (s *DB) prefetchWorker(job *prefetchJob, jobs chan *prefetchJob) {
+	if job.account == nil {
+		// scan one account
+		nodeIterator := s.trie.NodeIterator(job.start)
+		it := trie.NewIterator(nodeIterator)
+
+		for it.Next() {
+			if job.end != nil && bytes.Compare(it.Key, job.end) >= 0 {
+				return
+			}
+
+			// build account data from main trie tree
+			var data Account
+			if err := rlp.DecodeBytes(it.Value, &data); err != nil {
+				panic(err)
+			}
+			addrBytes := s.trie.GetKey(it.Key)
+			addr := common.BytesToAddress(addrBytes)
+			obj := newObject(s, addr, data)
+			if data.CodeHash != nil {
+				obj.Code(s.db)
+			}
+
+			// build account trie tree
+			storageIt := trie.NewIterator(obj.getTrie(s.db).NodeIterator(nil))
+			storageJob := &prefetchJob{
+				accountAddr: addrBytes,
+				account:     &data,
+			}
+
+			// fetch data
+			s.prefetchAccountStorage(jobs, storageJob, storageIt)
+		}
+	} else {
+		// scan main trie tree
+		obj := newObject(s, common.BytesToAddress(job.accountAddr), *job.account)
+		storageIt := trie.NewIterator(obj.getTrie(s.db).NodeIterator(job.start))
+
+		// fetch data
+		s.prefetchAccountStorage(jobs, job, storageIt)
+	}
+}
+
+// prefetchAccountStorage used for fetch account storage
+func (s *DB) prefetchAccountStorage(jobs chan *prefetchJob, job *prefetchJob, it *trie.Iterator) {
+	start := time.Now()
+	count := 0
+
+	for it.Next() {
+		if job.end != nil && bytes.Compare(it.Key, job.end) >= 0 {
+			return
+		}
+
+		count++
+
+		// if fetch one account job used more than 15s, then split the job
+		if count%10000 == 0 && time.Now().Sub(start) > 15*time.Second {
+			middle := utils.BytesMiddle(it.Key, job.end)
+			startJob := &prefetchJob{
+				accountAddr: job.accountAddr,
+				account:     job.account,
+				start:       it.Key,
+				end:         middle,
+			}
+			endJob := &prefetchJob{
+				accountAddr: job.accountAddr,
+				account:     job.account,
+				start:       middle,
+				end:         job.end,
+			}
+
+			select {
+			case jobs <- endJob:
+				select {
+				case jobs <- startJob:
+					return
+				default:
+					job.end = startJob.end
+					start = time.Now()
+				}
+			default:
+				log.Println("job full, continue")
+			}
+		}
+	}
+}

core/state/tikv_clean.go

@@ -0,0 +1,80 @@
+package state
+
+import (
+	"bytes"
+	"sync"
+	"sync/atomic"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/trie"
+	"github.com/harmony-one/harmony/internal/shardchain/tikv_manage"
+)
+
+var secureKeyPrefix = []byte("secure-key-")
+
+// DiffAndCleanCache clean block tire data from redis, Used to reduce redis storage and increase hit rate
+func (s *DB) DiffAndCleanCache(shardId uint32, to *DB) (int, error) {
+	// create difference iterator
+	it, _ := trie.NewDifferenceIterator(to.trie.NodeIterator(nil), s.trie.NodeIterator(nil))
+	db, err := tikv_manage.GetDefaultTiKVFactory().NewCacheStateDB(shardId)
+	if err != nil {
+		return 0, err
+	}
+
+	batch := db.NewBatch()
+	wg := &sync.WaitGroup{}
+	count := uint64(0)
+	for it.Next(true) {
+		if !it.Leaf() {
+			// delete it if trie leaf node
+			atomic.AddUint64(&count, 1)
+			_ = batch.Delete(it.Hash().Bytes())
+		} else {
+
+			// build account data
+			addrBytes := s.trie.GetKey(it.LeafKey())
+			addr := common.BytesToAddress(addrBytes)
+
+			var fromAccount, toAccount Account
+			if err := rlp.DecodeBytes(it.LeafBlob(), &fromAccount); err != nil {
+				continue
+			}
+
+			if toByte, err := to.trie.TryGet(addrBytes); err != nil {
+				continue
+			} else if err := rlp.DecodeBytes(toByte, &toAccount); err != nil {
+				continue
+			}
+
+			// if account not changed, skip
+			if bytes.Compare(fromAccount.Root.Bytes(), toAccount.Root.Bytes()) == 0 {
+				continue
+			}
+
+			// create account difference iterator
+			fromAccountTrie := newObject(s, addr, fromAccount).getTrie(s.db)
+			toAccountTrie := newObject(to, addr, toAccount).getTrie(to.db)
+			accountIt, _ := trie.NewDifferenceIterator(toAccountTrie.NodeIterator(nil), fromAccountTrie.NodeIterator(nil))
+
+			// parallel to delete data
+			wg.Add(1)
+			go func() {
+				defer wg.Done()
+				for accountIt.Next(true) {
+					atomic.AddUint64(&count, 1)
+
+					if !accountIt.Leaf() {
+						_ = batch.Delete(accountIt.Hash().Bytes())
+					} else {
+						_ = batch.Delete(append(append([]byte{}, secureKeyPrefix...), accountIt.LeafKey()...))
+					}
+				}
+			}()
+		}
+	}
+
+	wg.Wait()
+
+	return int(count), db.ReplayCache(batch)
+}

core/tx_pool.go

@@ -170,6 +170,8 @@ type TxPoolConfig struct {
 
 	Lifetime time.Duration // Maximum amount of time non-executable transaction are queued
 
+	AddEvent func(tx types.PoolTransaction, local bool) // Fire add event
+
 	Blacklist  map[common.Address]struct{}      // Set of accounts that cannot be a part of any transaction
 	AllowedTxs map[common.Address]AllowedTxData // Set of allowed transactions can break the blocklist
 }
@@ -958,7 +960,14 @@ func (pool *TxPool) pendingEpoch() *big.Int {
 // If a newly added transaction is marked as local, its sending account will be
 // whitelisted, preventing any associated transaction from being dropped out of
 // the pool due to pricing constraints.
-func (pool *TxPool) add(tx types.PoolTransaction, local bool) (bool, error) {
+func (pool *TxPool) add(tx types.PoolTransaction, local bool) (replaced bool, err error) {
+	defer func() {
+		if err == nil && pool.config.AddEvent != nil {
+			// used for tikv mode, writer will publish txpool change to all reader, this makes the state consistent
+			pool.config.AddEvent(tx, local)
+		}
+	}()
+
 	logger := utils.Logger().With().Stack().Logger()
 	// If the transaction is in the error sink, remove it as it may succeed
 	if pool.txErrorSink.Contains(tx.Hash().String()) {

core/tx_pool_test.go

@@ -161,7 +161,7 @@ func createBlockChain() *BlockChainImpl {
 	genesis := gspec.MustCommit(database)
 	_ = genesis
 	engine := chain2.NewEngine()
-	blockchain, _ := NewBlockChain(database, nil, gspec.Config, engine, vm.Config{}, nil)
+	blockchain, _ := NewBlockChain(database, state.NewDatabase(database), nil, gspec.Config, engine, vm.Config{}, nil)
 	return blockchain
 }
 

go.mod

@@ -3,6 +3,8 @@ module github.com/harmony-one/harmony
 go 1.18
 
 require (
+	github.com/RoaringBitmap/roaring v1.1.0
+	github.com/VictoriaMetrics/fastcache v1.5.7
 	github.com/Workiva/go-datastructures v1.0.50
 	github.com/allegro/bigcache v1.2.1
 	github.com/aws/aws-sdk-go v1.30.1
@@ -13,6 +15,8 @@ require (
 	github.com/davecgh/go-spew v1.1.1
 	github.com/deckarep/golang-set v1.7.1
 	github.com/ethereum/go-ethereum v1.9.25
+	github.com/fjl/memsize v0.0.0-20180929194037-2a09253e352a // indirect
+	github.com/go-redis/redis/v8 v8.11.5
 	github.com/golang/mock v1.6.0
 	github.com/golang/protobuf v1.5.2
 	github.com/golangci/golangci-lint v1.22.2
@@ -38,7 +42,7 @@ require (
 	github.com/pborman/uuid v1.2.0
 	github.com/pelletier/go-toml v1.9.3
 	github.com/pkg/errors v0.9.1
-	github.com/prometheus/client_golang v1.10.0
+	github.com/prometheus/client_golang v1.11.0
 	github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0
 	github.com/rjeczalik/notify v0.9.2
 	github.com/rs/cors v1.7.0
@@ -48,13 +52,15 @@ require (
 	github.com/spf13/viper v1.6.1
 	github.com/stretchr/testify v1.7.0
 	github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca
+	github.com/tikv/client-go/v2 v2.0.1
 	github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee
 	go.uber.org/ratelimit v0.1.0
-	go.uber.org/zap v1.16.0
+	go.uber.org/zap v1.20.0
 	golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
 	golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba
-	google.golang.org/grpc v1.33.2
+	golang.org/x/tools v0.1.7 // indirect
+	google.golang.org/grpc v1.43.0
 	google.golang.org/protobuf v1.26.0
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c
 	gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce
@@ -67,21 +73,22 @@ require (
 	github.com/BurntSushi/toml v0.3.1 // indirect
 	github.com/OpenPeeDeeP/depguard v1.0.1 // indirect
 	github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 // indirect
-	github.com/VictoriaMetrics/fastcache v1.5.7 // indirect
 	github.com/aristanetworks/goarista v0.0.0-20190607111240-52c2a7864a08 // indirect
+	github.com/benbjohnson/clock v1.1.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/bits-and-blooms/bitset v1.2.0 // indirect
 	github.com/bombsimon/wsl/v2 v2.0.0 // indirect
 	github.com/btcsuite/btcd v0.21.0-beta // indirect
 	github.com/cespare/xxhash v1.1.0 // indirect
-	github.com/cespare/xxhash/v2 v2.1.1 // indirect
+	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect
 	github.com/dgraph-io/badger v1.6.2 // indirect
 	github.com/dgraph-io/ristretto v0.0.3 // indirect
+	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/dustin/go-humanize v1.0.0 // indirect
 	github.com/edsrzf/mmap-go v1.0.0 // indirect
 	github.com/elastic/gosigar v0.8.1-0.20180330100440-37f05ff46ffa // indirect
 	github.com/fatih/color v1.10.0 // indirect
-	github.com/fjl/memsize v0.0.0-20180929194037-2a09253e352a // indirect
 	github.com/flynn/noise v1.0.0 // indirect
 	github.com/fsnotify/fsnotify v1.4.9 // indirect
 	github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff // indirect
@@ -99,7 +106,7 @@ require (
 	github.com/gobwas/glob v0.2.3 // indirect
 	github.com/gofrs/flock v0.0.0-20190320160742-5135e617513b // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang/snappy v0.0.1 // indirect
+	github.com/golang/snappy v0.0.2-0.20190904063534-ff6b7dc882cf // indirect
 	github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2 // indirect
 	github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a // indirect
 	github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6 // indirect
@@ -114,9 +121,11 @@ require (
 	github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21 // indirect
 	github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0 // indirect
 	github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4 // indirect
+	github.com/google/btree v1.0.0 // indirect
 	github.com/google/gopacket v1.1.19 // indirect
 	github.com/google/uuid v1.1.2 // indirect
 	github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3 // indirect
+	github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 // indirect
 	github.com/hashicorp/errwrap v1.0.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
@@ -190,6 +199,7 @@ require (
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/mr-tron/base58 v1.2.0 // indirect
+	github.com/mschoch/smat v0.2.0 // indirect
 	github.com/multiformats/go-base32 v0.0.3 // indirect
 	github.com/multiformats/go-base36 v0.1.0 // indirect
 	github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect
@@ -200,9 +210,13 @@ require (
 	github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d // indirect
 	github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c // indirect
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
+	github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c // indirect
+	github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 // indirect
+	github.com/pingcap/kvproto v0.0.0-20220106070556-3fa8fa04f898 // indirect
+	github.com/pingcap/log v0.0.0-20211215031037-e024ba4eb0ee // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_model v0.2.0 // indirect
-	github.com/prometheus/common v0.18.0 // indirect
+	github.com/prometheus/common v0.26.0 // indirect
 	github.com/prometheus/procfs v0.6.0 // indirect
 	github.com/prometheus/tsdb v0.7.1 // indirect
 	github.com/securego/gosec v0.0.0-20191002120514-e680875ea14d // indirect
@@ -217,6 +231,7 @@ require (
 	github.com/steakknife/hamming v0.0.0-20180906055917-c99c65617cd3 // indirect
 	github.com/stretchr/objx v0.2.0 // indirect
 	github.com/subosito/gotenv v1.2.0 // indirect
+	github.com/tikv/pd/client v0.0.0-20220216070739-26c668271201 // indirect
 	github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e // indirect
 	github.com/tommy-muehle/go-mnd v1.1.1 // indirect
 	github.com/tyler-smith/go-bip39 v1.0.2 // indirect
@@ -227,20 +242,18 @@ require (
 	github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 // indirect
 	github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208 // indirect
 	go.opencensus.io v0.23.0 // indirect
-	go.uber.org/atomic v1.7.0 // indirect
+	go.uber.org/atomic v1.9.0 // indirect
-	go.uber.org/multierr v1.6.0 // indirect
+	go.uber.org/multierr v1.7.0 // indirect
-	golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect
 	golang.org/x/mod v0.4.2 // indirect
 	golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d // indirect
-	golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e // indirect
+	golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect
 	golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 // indirect
 	golang.org/x/text v0.3.6 // indirect
-	golang.org/x/tools v0.1.7 // indirect
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
-	google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 // indirect
+	google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c // indirect
 	gopkg.in/ini.v1 v1.51.0 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
 	honnef.co/go/tools v0.0.1-2020.1.5 // indirect
 	mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed // indirect
 	mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b // indirect

go.sum

@@ -35,6 +35,8 @@ github.com/OneOfOne/xxhash v1.2.5 h1:zl/OfRA6nftbBK9qTohYBJ5xvw6C/oNKizR7cZGl3cI
 github.com/OneOfOne/xxhash v1.2.5/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q=
 github.com/OpenPeeDeeP/depguard v1.0.1 h1:VlW4R6jmBIv3/u1JNlawEvJMM4J+dPORPaZasQee8Us=
 github.com/OpenPeeDeeP/depguard v1.0.1/go.mod h1:xsIw86fROiiwelg+jB2uM9PiKihMMmUx/1V+TNhjQvM=
+github.com/RoaringBitmap/roaring v1.1.0 h1:b10lZrZXaY6Q6EKIRrmOF519FIyQQ5anPgGr3niw2yY=
+github.com/RoaringBitmap/roaring v1.1.0/go.mod h1:icnadbWcNyfEHlYdr+tDlOTih1Bf/h+rzPpv4sbomAA=
 github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
 github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
 github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 h1:fLjPD/aNc3UIOA6tDi6QXUemppXK3P9BI7mr2hd6gx8=
@@ -57,6 +59,8 @@ github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax
 github.com/allegro/bigcache v1.2.1 h1:hg1sY1raCwic3Vnsvje6TT7/pnZba83LeFck5NrFKSc=
 github.com/allegro/bigcache v1.2.1/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
+github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q=
+github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
 github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
 github.com/aristanetworks/goarista v0.0.0-20170210015632-ea17b1a17847/go.mod h1:D/tb0zPVXnP7fmsLZjtdUhSsumbK/ij54UXjjVgMGxQ=
@@ -75,13 +79,16 @@ github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZw
 github.com/beevik/ntp v0.3.0 h1:xzVrPrE4ziasFXgBVBZJDP0Wg/KpMwk2KHJ4Ba8GrDw=
 github.com/beevik/ntp v0.3.0/go.mod h1:hIHWr+l3+/clUnF44zdK+CWW7fO8dR5cIylAQ76NRpg=
 github.com/benbjohnson/clock v1.0.2/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM=
-github.com/benbjohnson/clock v1.0.3 h1:vkLuvpK4fmtSCuo60+yC63p7y0BmQ8gm5ZXGuBCJyXg=
 github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM=
+github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
+github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
+github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjLyS07ChA=
+github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=
 github.com/bombsimon/wsl v1.2.5/go.mod h1:43lEF/i0kpXbLCeDXL9LMT8c92HyBywXb0AsgMHYngM=
 github.com/bombsimon/wsl/v2 v2.0.0 h1:+Vjcn+/T5lSrO8Bjzhk4v14Un/2UyCA1E3V5j9nwTkQ=
 github.com/bombsimon/wsl/v2 v2.0.0/go.mod h1:mf25kr/SqFEPhhcxW1+7pxzGlW+hIl/hYTKY95VwV8U=
@@ -117,14 +124,23 @@ github.com/cespare/cp v1.1.1/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW
 github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cespare/xxhash/v2 v2.0.1-0.20190104013014-3767db7a7e18/go.mod h1:HD5P3vAIAh+Y2GAxg0PrPN1P8WkepXGpjbUPDHJqqKM=
-github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
+github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE=
 github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ=
+github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
+github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
+github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
+github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI=
+github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
+github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
+github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
 github.com/coinbase/rosetta-sdk-go v0.7.0 h1:lmTO/JEpCvZgpbkOITL95rA80CPKb5CtMzLaqF2mCNg=
@@ -138,11 +154,13 @@ github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee
 github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
+github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM=
 github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@@ -170,6 +188,8 @@ github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUn
 github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
 github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y=
 github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
+github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
+github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
 github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
@@ -187,6 +207,9 @@ github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4s
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
+github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/ethereum/go-ethereum v1.9.9 h1:jnoBvjH8aMH++iH14XmiJdAsnRcmZUM+B5fsnEZBVE0=
 github.com/ethereum/go-ethereum v1.9.9/go.mod h1:a9TqabFudpDu1nucId+k9S8R9whYaHnGBLKFouA5EAo=
@@ -221,6 +244,7 @@ github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/kit v0.10.0 h1:dXFJfIHVvUcpSgDOV+Ne6t7jXri8Tfv2uOLHUZ2XNuo=
 github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o=
+github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
 github.com/go-lintpack/lintpack v0.5.2 h1:DI5mA3+eKdWeJ40nU4d6Wc26qmdG8RCi/btYq0TuRN0=
 github.com/go-lintpack/lintpack v0.5.2/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTDqfpXvXAN0sXM=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
@@ -229,6 +253,8 @@ github.com/go-logfmt/logfmt v0.5.0 h1:TrB8swr/68K7m9CcGut2g3UOihhbcbiMAYiuTXdEih
 github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
 github.com/go-ole/go-ole v1.2.1 h1:2lOsA72HgjxAuMlKpFiCbHTvu44PIVkZ5hqm3RSdI/E=
 github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
+github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI=
+github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo=
 github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
@@ -258,6 +284,7 @@ github.com/go-toolsmith/typep v1.0.0 h1:zKymWyA1TRYvqYrYDrfEMZULyrhcnGY3x7LDKU2X
 github.com/go-toolsmith/typep v1.0.0/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU=
 github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
 github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
+github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/gofrs/flock v0.0.0-20190320160742-5135e617513b h1:ekuhfTjngPhisSjOJ0QWKpPQE8/rbknHaes6WVJj5Hw=
 github.com/gofrs/flock v0.0.0-20190320160742-5135e617513b/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
 github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
@@ -300,8 +327,9 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS
 github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
-github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/golang/snappy v0.0.2-0.20190904063534-ff6b7dc882cf h1:gFVkHXmVAhEbxZVDln5V9GKrLaluNoFHDbrZwAWZgws=
+github.com/golang/snappy v0.0.2-0.20190904063534-ff6b7dc882cf/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2 h1:23T5iq8rbUYlhpt5DB4XJkc6BU31uODLD1o1gKvZmD0=
 github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2/go.mod h1:k9Qvh+8juN+UKMCS/3jFtGICgW8O96FVaZsaxdzDkR4=
 github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a h1:w8hkcTqaFpzKqonE9uMCefW1WDie15eSP/4MssdenaM=
@@ -334,6 +362,7 @@ github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0/go.mod h1:qOQCunE
 github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4 h1:zwtduBRr5SSWhqsYNgcuWO2kFlpdOZbP0+yRjmvPGys=
 github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg8RkN3rCIMLGE9CyYmU9pY2Jer6DgANEnZ/L/cQ=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
@@ -354,6 +383,7 @@ github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF
 github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@@ -380,10 +410,14 @@ github.com/graph-gophers/graphql-go v0.0.0-20191115155744-f33e81362277/go.mod h1
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
+github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 h1:THDBEeQ9xZ8JEaCLyLQqXMMdRqNr0QAUJTIkQAUtFjg=
+github.com/grpc-ecosystem/go-grpc-middleware v1.1.0/go.mod h1:f5nM7jw/oeRSadq3xCzHAvxcr8HZnzsqU6ILg/0NiiE=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
 github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=
 github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
 github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
+github.com/grpc-ecosystem/grpc-gateway v1.12.1/go.mod h1:8XEsbTttt/W+VvjtQhLACqCisSPWTxCZ7sBRjU6iH9c=
+github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
 github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU=
 github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48=
 github.com/harmony-one/abool v1.0.1 h1:SjXLmrr3W8h6lY37gRuWtLiRknUOchnUnsXJWK6Gbm4=
@@ -429,6 +463,7 @@ github.com/huin/goupnp v0.0.0-20161224104101-679507af18f3/go.mod h1:MZ2ZmwcBpvOo
 github.com/huin/goupnp v1.0.0 h1:wg75sLpL6DZqwHQN6E1Cfk6mtfzS45z8OV+ic+DtHRo=
 github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc=
 github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o=
+github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/influxdata/influxdb v1.2.3-0.20180221223340-01288bdb0883/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY=
@@ -508,6 +543,7 @@ github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCV
 github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
@@ -876,6 +912,8 @@ github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjW
 github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
 github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o=
 github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
+github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM=
+github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw=
 github.com/multiformats/go-base32 v0.0.3 h1:tw5+NhuwaOjJCC5Pp82QuXbrmLzWg7uxlMFp8Nq/kkI=
 github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA=
 github.com/multiformats/go-base36 v0.1.0 h1:JR6TyF7JjGd3m6FbLU2cOxhC0Li8z8dLNGQ89tUg4F4=
@@ -967,8 +1005,10 @@ github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
 github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
 github.com/onsi/ginkgo v1.16.2/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E=
-github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc=
 github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
+github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
+github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
+github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c=
 github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
@@ -976,8 +1016,10 @@ github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
-github.com/onsi/gomega v1.13.0 h1:7lLHu94wT9Ij0o6EWWclhu0aOh32VxhkwEJvzuWPeak=
 github.com/onsi/gomega v1.13.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY=
+github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
+github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE=
+github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs=
 github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
 github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis=
 github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74=
@@ -1002,6 +1044,22 @@ github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9
 github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0=
 github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc=
 github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
+github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ=
+github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0 h1:HVl5539r48eA+uDuX/ziBmQCxzT1pGrzWbKuXT46Bq0=
+github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0/go.mod h1:PYMCGwN0JHjoqGr3HrZoD+b8Tgx8bKnArhSq8YVzUMc=
+github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
+github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
+github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTmyFqUwr+jcCvpVkK7sumiz+ko5H9eq4=
+github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg=
+github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 h1:C3N3itkduZXDZFh4N3vQ5HEtld3S+Y+StULhWVvumU0=
+github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew=
+github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 h1:surzm05a8C9dN8dIUmo4Be2+pMRb6f55i+UIYrluu2E=
+github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw=
+github.com/pingcap/kvproto v0.0.0-20220106070556-3fa8fa04f898 h1:c0d/sMTeftJQF9O5OHyezWwPrzf2FXcEE5HWwnq/Ahs=
+github.com/pingcap/kvproto v0.0.0-20220106070556-3fa8fa04f898/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI=
+github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8=
+github.com/pingcap/log v0.0.0-20211215031037-e024ba4eb0ee h1:VO2t6IBpfvW34TdtD/G10VvnGqjLic1jzOuHjUb5VqM=
+github.com/pingcap/log v0.0.0-20211215031037-e024ba4eb0ee/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@@ -1018,8 +1076,9 @@ github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5Fsn
 github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og=
 github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
 github.com/prometheus/client_golang v1.9.0/go.mod h1:FqZLKOZnGdFAhOK4nqGHa7D66IdsO+O441Eve7ptJDU=
-github.com/prometheus/client_golang v1.10.0 h1:/o0BDeWzLWXNZ+4q5gXltUvaMpJqckTa+jTNoB+z4cg=
 github.com/prometheus/client_golang v1.10.0/go.mod h1:WJM3cc3yu7XKBKa/I8WeZm+V3eltZnBwfENSU7mdogU=
+github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ=
+github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
@@ -1035,8 +1094,9 @@ github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8
 github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA=
 github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
 github.com/prometheus/common v0.15.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s=
-github.com/prometheus/common v0.18.0 h1:WCVKW7aL6LEe1uryfI9dnEc2ZqNB1Fn0ok930v0iL1Y=
 github.com/prometheus/common v0.18.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s=
+github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ=
+github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
 github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
@@ -1054,11 +1114,13 @@ github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:
 github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 h1:MkV+77GLUNo5oJ0jf870itWm3D0Sjh7+Za9gazKc5LQ=
 github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
+github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho=
 github.com/rjeczalik/notify v0.9.2 h1:MiTWrPj55mNDHEiIX5YUSKefw/+lCQVoAFmD6oQm5w8=
 github.com/rjeczalik/notify v0.9.2/go.mod h1:aErll2f0sUX9PXZnVNyeiObbmTlk5jnMoCa4QEjJeqM=
 github.com/robertkrimen/otto v0.0.0-20170205013659-6a77b7cbc37d/go.mod h1:xvqspoSXJTIpemEonrMDFq6XzwHYYgToXWj5eRX1OtY=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
+github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rs/cors v0.0.0-20160617231935-a62a804a8a00/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
 github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik=
@@ -1076,6 +1138,7 @@ github.com/securego/gosec v0.0.0-20191002120514-e680875ea14d h1:BzRvVq1EHuIjxpij
 github.com/securego/gosec v0.0.0-20191002120514-e680875ea14d/go.mod h1:w5+eXa0mYznDkHaMCXA4XYffjlH+cy1oyKbfzJXa2Do=
 github.com/segmentio/fasthash v1.0.3/go.mod h1:waKX8l2N8yckOgmSsXJi7x1ZfdKZ4x7KRMzBtS3oedY=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
+github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
 github.com/shirou/gopsutil v0.0.0-20190901111213-e4ec7b275ada/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc=
 github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
 github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY=
@@ -1174,12 +1237,17 @@ github.com/tidwall/gjson v1.6.7/go.mod h1:zeFuBCIqD4sN/gmqBzZ4j7Jd6UcA2Fc56x7QFs
 github.com/tidwall/match v1.0.3/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
 github.com/tidwall/pretty v1.0.2/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
 github.com/tidwall/sjson v1.1.4/go.mod h1:wXpKXu8CtDjKAZ+3DrKY5ROCorDFahq8l0tey/Lx1fg=
+github.com/tikv/client-go/v2 v2.0.1 h1:+K/VvVOxEOXKMtR83bs5Aj3lrYdTdTZdvH0apfAWW10=
+github.com/tikv/client-go/v2 v2.0.1/go.mod h1:gaHSp8rnxZ0w36qb6QPPNPh9P0Mu5vAEwCQcc0Brni4=
+github.com/tikv/pd/client v0.0.0-20220216070739-26c668271201 h1:7h/Oi4Zw6eGCeXh4Q4ZvKI4k7nBJVUq0c29YCcLwKPM=
+github.com/tikv/pd/client v0.0.0-20220216070739-26c668271201/go.mod h1:fEvI5fhAuJn1Fn87VJF8ByE9Vc16EzWGoePZB21/nL8=
 github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e h1:RumXZ56IrCj4CL+g1b9OL/oH0QnsF976bC8xQFYUD5Q=
 github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tommy-muehle/go-mnd v1.1.1 h1:4D0wuPKjOTiK2garzuPGGvm4zZ/wLYDOH8TJSABC7KU=
 github.com/tommy-muehle/go-mnd v1.1.1/go.mod h1:dSUh0FtTP8VhvkL1S+gUR1OKd9ZnSaozuI6r3m6wOig=
+github.com/twmb/murmur3 v1.1.3/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ=
 github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs=
 github.com/tyler-smith/go-bip39 v1.0.2 h1:+t3w+KwLXO6154GNJY+qUtIxLTmFjfUmpguQT1OlOT8=
 github.com/tyler-smith/go-bip39 v1.0.2/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs=
@@ -1225,6 +1293,9 @@ github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxt
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=
+go.etcd.io/etcd/api/v3 v3.5.2/go.mod h1:5GB2vv4A4AOn3yk7MftYGHkUfGtDHnEraIjym4dYz5A=
+go.etcd.io/etcd/client/pkg/v3 v3.5.2/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
+go.etcd.io/etcd/client/v3 v3.5.2/go.mod h1:kOOaWFFgHygyT0WlSmL8TJiXmMysO/nNUlEsSsN6W4o=
 go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA=
 go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
 go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
@@ -1235,29 +1306,40 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M=
 go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
+go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
 go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
-go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
+go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/goleak v1.0.0/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
-go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0=
 go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
+go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
+go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA=
+go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
 go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
+go.uber.org/multierr v1.4.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
 go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU=
-go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
 go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
+go.uber.org/multierr v1.7.0 h1:zaiO/rmgFjbmCXdSYJWQcdvOCsthmdaHfr3Gm2Kx4Ec=
+go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=
 go.uber.org/ratelimit v0.1.0 h1:U2AruXqeTb4Eh9sYQSTrMhH8Cb7M0Ian2ibBOnBcnAw=
 go.uber.org/ratelimit v0.1.0/go.mod h1:2X8KaoNd1J0lZV+PxJk/5+DGbO/tpwLR1m++a7FnB/Y=
 go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=
+go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
+go.uber.org/zap v1.12.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM=
 go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM=
 go.uber.org/zap v1.14.1/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc=
 go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc=
-go.uber.org/zap v1.16.0 h1:uFRZXykJGK9lLY4HtgSw44DnIcAM+kRBP7x5m+NpAOM=
 go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ=
+go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
+go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI=
+go.uber.org/zap v1.20.0 h1:N4oPlghZwYG55MlU6LXk/Zp00FVNE9X9wrYO8CEs4lc=
+go.uber.org/zap v1.20.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw=
 go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE=
 golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw=
 golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
@@ -1297,7 +1379,6 @@ golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHl
 golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
-golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug=
 golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
@@ -1331,12 +1412,14 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
@@ -1352,6 +1435,7 @@ golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAG
 golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -1396,6 +1480,7 @@ golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -1418,12 +1503,14 @@ golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210317225723-c4fcb01b228e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210426080607-c94f62235c83/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210511113859-b0526f3d8744/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e h1:WUoyKPm6nCo1BnNUvPGnFG3T5DUVem42yDJZZ4CNxMA=
+golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@@ -1432,6 +1519,7 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -1469,6 +1557,7 @@ golang.org/x/tools v0.0.0-20190930201159-7c411dea38b0/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20191010075000-0337d82405ff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191107010934-f79515f33823/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191113232020-e2727e816f5a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
@@ -1480,6 +1569,8 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY
 golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.7 h1:6j8CgantCy3yc8JGBqkDLMKWqZ0RDU2g1HVgacojGWQ=
 golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -1504,8 +1595,11 @@ google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRn
 google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s=
 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
-google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY=
+google.golang.org/genproto v0.0.0-20190927181202-20e1ac93f88c/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
+google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c h1:wtujag7C+4D6KMoulW9YauvK2lgdvCMS260jsqqBXr0=
+google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0=
 google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
 google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio=
 google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
@@ -1516,13 +1610,18 @@ google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij
 google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA=
 google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
 google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
 google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
 google.golang.org/grpc v1.28.1/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
 google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
-google.golang.org/grpc v1.33.2 h1:EQyQC3sa8M+p6Ulc8yy9SWSS2GVwyRc83gAbG8lrl4o=
+google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
 google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
+google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
+google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
+google.golang.org/grpc v1.43.0 h1:Eeu7bZtDZ2DpRCsLhUlcrLnvYaMK1Gz86a+hMVvELmM=
+google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
@@ -1567,14 +1666,17 @@ gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRN
 gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
 grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o=
 honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
@@ -1593,6 +1695,7 @@ mvdan.cc/unparam v0.0.0-20190720180237-d51796306d8f/go.mod h1:4G1h5nDURzA3bwVMZI
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
+sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
 sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU=
 sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck=
 sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4 h1:JPJh2pk3+X4lXAkZIk2RuE/7/FoK9maXw+TNPJhVS/c=

hmy/blockchain.go

@@ -5,6 +5,8 @@ import (
 	"fmt"
 	"math/big"
 
+	v3 "github.com/harmony-one/harmony/block/v3"
+
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/bloombits"
 	"github.com/ethereum/go-ethereum/event"
@@ -174,10 +176,20 @@ func (hmy *Harmony) GetPreStakingBlockRewards(
 
 // GetLatestChainHeaders ..
 func (hmy *Harmony) GetLatestChainHeaders() *block.HeaderPair {
-	return &block.HeaderPair{
+	pair := &block.HeaderPair{
-		BeaconHeader: hmy.BeaconChain.CurrentHeader(),
+		BeaconHeader: &block.Header{Header: v3.NewHeader()},
-		ShardHeader:  hmy.BlockChain.CurrentHeader(),
+		ShardHeader:  &block.Header{Header: v3.NewHeader()},
+	}
+
+	if hmy.BeaconChain != nil {
+		pair.BeaconHeader = hmy.BeaconChain.CurrentHeader()
 	}
+
+	if hmy.BlockChain != nil {
+		pair.ShardHeader = hmy.BlockChain.CurrentHeader()
+	}
+
+	return pair
 }
 
 // GetLastCrossLinks ..

internal/configs/harmony/harmony.go

@@ -27,6 +27,7 @@ type HarmonyConfig struct {
 	Revert     *RevertConfig     `toml:",omitempty"`
 	Legacy     *LegacyConfig     `toml:",omitempty"`
 	Prometheus *PrometheusConfig `toml:",omitempty"`
+	TiKV       *TiKVConfig       `toml:",omitempty"`
 	DNSSync    DnsSync
 	ShardData  ShardDataConfig
 }
@@ -66,6 +67,18 @@ type GeneralConfig struct {
 	DataDir                string
 	TraceEnable            bool
 	EnablePruneBeaconChain bool
+	RunElasticMode         bool
+}
+
+type TiKVConfig struct {
+	Debug bool
+
+	PDAddr                      []string
+	Role                        string
+	StateDBCacheSizeInMB        uint32
+	StateDBCachePersistencePath string
+	StateDBRedisServerAddr      []string
+	StateDBRedisLRUTimeInDay    uint32
 }
 
 type ShardDataConfig struct {

internal/shardchain/dbfactory_tikv.go

@@ -0,0 +1,126 @@
+package shardchain
+
+import (
+	"fmt"
+	"io"
+	"sync"
+
+	"github.com/ethereum/go-ethereum/core/rawdb"
+	"github.com/harmony-one/harmony/internal/tikv"
+	tikvCommon "github.com/harmony-one/harmony/internal/tikv/common"
+	"github.com/harmony-one/harmony/internal/tikv/prefix"
+	"github.com/harmony-one/harmony/internal/tikv/remote"
+	"github.com/harmony-one/harmony/internal/tikv/statedb_cache"
+
+	"github.com/ethereum/go-ethereum/ethdb"
+)
+
+const (
+	LDBTiKVPrefix = "harmony_tikv"
+)
+
+type TiKvCacheConfig struct {
+	StateDBCacheSizeInMB        uint32
+	StateDBCachePersistencePath string
+	StateDBRedisServerAddr      string
+	StateDBRedisLRUTimeInDay    uint32
+}
+
+// TiKvFactory is a memory-backed blockchain database factory.
+type TiKvFactory struct {
+	cacheDBMap sync.Map
+
+	PDAddr      []string
+	Role        string
+	CacheConfig statedb_cache.StateDBCacheConfig
+}
+
+// getStateDB create statedb storage use tikv
+func (f *TiKvFactory) getRemoteDB(shardID uint32) (*prefix.PrefixDatabase, error) {
+	key := fmt.Sprintf("remote_%d_%s", shardID, f.Role)
+	if db, ok := f.cacheDBMap.Load(key); ok {
+		return db.(*prefix.PrefixDatabase), nil
+	} else {
+		prefixStr := []byte(fmt.Sprintf("%s_%d/", LDBTiKVPrefix, shardID))
+		remoteDatabase, err := remote.NewRemoteDatabase(f.PDAddr, f.Role == tikv.RoleReader)
+		if err != nil {
+			return nil, err
+		}
+
+		tmpDB := prefix.NewPrefixDatabase(prefixStr, remoteDatabase)
+		if loadedDB, loaded := f.cacheDBMap.LoadOrStore(key, tmpDB); loaded {
+			_ = tmpDB.Close()
+			return loadedDB.(*prefix.PrefixDatabase), nil
+		}
+
+		return tmpDB, nil
+	}
+}
+
+// getStateDB create statedb storage with local memory cache.
+func (f *TiKvFactory) getStateDB(shardID uint32) (*statedb_cache.StateDBCacheDatabase, error) {
+	key := fmt.Sprintf("state_db_%d_%s", shardID, f.Role)
+	if db, ok := f.cacheDBMap.Load(key); ok {
+		return db.(*statedb_cache.StateDBCacheDatabase), nil
+	} else {
+		db, err := f.getRemoteDB(shardID)
+		if err != nil {
+			return nil, err
+		}
+
+		tmpDB, err := statedb_cache.NewStateDBCacheDatabase(db, f.CacheConfig, f.Role == tikv.RoleReader)
+		if err != nil {
+			return nil, err
+		}
+
+		if loadedDB, loaded := f.cacheDBMap.LoadOrStore(key, tmpDB); loaded {
+			_ = tmpDB.Close()
+			return loadedDB.(*statedb_cache.StateDBCacheDatabase), nil
+		}
+
+		return tmpDB, nil
+	}
+}
+
+// NewChainDB returns a new memDB for the blockchain for given shard.
+func (f *TiKvFactory) NewChainDB(shardID uint32) (ethdb.Database, error) {
+	var database ethdb.KeyValueStore
+
+	db, err := f.getRemoteDB(shardID)
+	if err != nil {
+		return nil, err
+	}
+
+	database = tikvCommon.ToEthKeyValueStore(db)
+
+	return rawdb.NewDatabase(database), nil
+}
+
+// NewStateDB create shard tikv database
+func (f *TiKvFactory) NewStateDB(shardID uint32) (ethdb.Database, error) {
+	var database ethdb.KeyValueStore
+
+	cacheDatabase, err := f.getStateDB(shardID)
+	if err != nil {
+		return nil, err
+	}
+
+	database = tikvCommon.ToEthKeyValueStore(cacheDatabase)
+
+	return rawdb.NewDatabase(database), nil
+}
+
+// NewCacheStateDB create shard statedb storage with memory cache
+func (f *TiKvFactory) NewCacheStateDB(shardID uint32) (*statedb_cache.StateDBCacheDatabase, error) {
+	return f.getStateDB(shardID)
+}
+
+// CloseAllDB close all tikv database
+func (f *TiKvFactory) CloseAllDB() {
+	f.cacheDBMap.Range(func(_, value interface{}) bool {
+		if closer, ok := value.(io.Closer); ok {
+			_ = closer.Close()
+		}
+		return true
+	})
+}

internal/shardchain/shardchains.go

@@ -4,6 +4,10 @@ import (
 	"math/big"
 	"sync"
 
+	"github.com/harmony-one/harmony/core/state"
+	harmonyconfig "github.com/harmony-one/harmony/internal/configs/harmony"
+	"github.com/harmony-one/harmony/internal/shardchain/tikv_manage"
+
 	"github.com/harmony-one/harmony/shard"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -33,13 +37,14 @@ type Collection interface {
 // CollectionImpl is the main implementation of the shard chain collection.
 // See the Collection interface for details.
 type CollectionImpl struct {
-	dbFactory    DBFactory
+	dbFactory     DBFactory
-	dbInit       DBInitializer
+	dbInit        DBInitializer
-	engine       engine.Engine
+	engine        engine.Engine
-	mtx          sync.Mutex
+	mtx           sync.Mutex
-	pool         map[uint32]core.BlockChain
+	pool          map[uint32]core.BlockChain
-	disableCache map[uint32]bool
+	disableCache  map[uint32]bool
-	chainConfig  *params.ChainConfig
+	chainConfig   *params.ChainConfig
+	harmonyconfig *harmonyconfig.HarmonyConfig
 }
 
 // NewCollection creates and returns a new shard chain collection.
@@ -49,16 +54,18 @@ type CollectionImpl struct {
 // dbInit is the shard chain initializer to use when the database returned by
 // the factory is brand new (empty).
 func NewCollection(
+	harmonyconfig *harmonyconfig.HarmonyConfig,
 	dbFactory DBFactory, dbInit DBInitializer, engine engine.Engine,
 	chainConfig *params.ChainConfig,
 ) *CollectionImpl {
 	return &CollectionImpl{
-		dbFactory:    dbFactory,
+		harmonyconfig: harmonyconfig,
-		dbInit:       dbInit,
+		dbFactory:     dbFactory,
-		engine:       engine,
+		dbInit:        dbInit,
-		pool:         make(map[uint32]core.BlockChain),
+		engine:        engine,
-		disableCache: make(map[uint32]bool),
+		pool:          make(map[uint32]core.BlockChain),
-		chainConfig:  chainConfig,
+		disableCache:  make(map[uint32]bool),
+		chainConfig:   chainConfig,
 	}
 }
 
@@ -105,7 +112,6 @@ func (sc *CollectionImpl) ShardChain(shardID uint32, options ...core.Options) (c
 		// For beacon chain inside a shard chain, need to reset the eth chainID to shard 0's eth chainID in the config
 		chainConfig.EthCompatibleChainID = big.NewInt(chainConfig.EthCompatibleShard0ChainID.Int64())
 	}
-
 	opts := core.Options{}
 	if len(options) == 1 {
 		opts = options[0]
@@ -114,8 +120,13 @@ func (sc *CollectionImpl) ShardChain(shardID uint32, options ...core.Options) (c
 	if opts.EpochChain {
 		bc, err = core.NewEpochChain(db, &chainConfig, sc.engine, vm.Config{})
 	} else {
+		stateCache, err := initStateCache(db, sc, shardID)
+		if err != nil {
+			return nil, err
+		}
+
 		bc, err = core.NewBlockChainWithOptions(
-			db, cacheConfig, &chainConfig, sc.engine, vm.Config{}, nil, opts,
+			db, stateCache, cacheConfig, &chainConfig, sc.engine, vm.Config{}, nil, opts,
 		)
 	}
 
@@ -124,9 +135,28 @@ func (sc *CollectionImpl) ShardChain(shardID uint32, options ...core.Options) (c
 	}
 	db = nil // don't close
 	sc.pool[shardID] = bc
+
+	if sc.harmonyconfig != nil && sc.harmonyconfig.General.RunElasticMode {
+		// init the tikv mode
+		bc.InitTiKV(sc.harmonyconfig.TiKV)
+	}
+
 	return bc, nil
 }
 
+func initStateCache(db ethdb.Database, sc *CollectionImpl, shardID uint32) (state.Database, error) {
+	if sc.harmonyconfig != nil && sc.harmonyconfig.General.RunElasticMode {
+		// used for tikv mode, init state db using tikv storage
+		stateDB, err := tikv_manage.GetDefaultTiKVFactory().NewStateDB(shardID)
+		if err != nil {
+			return nil, err
+		}
+		return state.NewDatabaseWithCache(stateDB, 64), nil
+	} else {
+		return state.NewDatabase(db), nil
+	}
+}
+
 // DisableCache disables caching mode for newly opened chains.
 // It does not affect already open chains.  For best effect,
 // use this immediately after creating collection.

internal/shardchain/tikv_manage/factory_manage.go

@@ -0,0 +1,28 @@
+package tikv_manage
+
+import (
+	"sync"
+
+	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/harmony-one/harmony/internal/tikv/statedb_cache"
+)
+
+type TiKvFactory interface {
+	NewChainDB(shardID uint32) (ethdb.Database, error)
+	NewStateDB(shardID uint32) (ethdb.Database, error)
+	NewCacheStateDB(shardID uint32) (*statedb_cache.StateDBCacheDatabase, error)
+	CloseAllDB()
+}
+
+var tikvInit = sync.Once{}
+var tikvFactory TiKvFactory
+
+func SetDefaultTiKVFactory(factory TiKvFactory) {
+	tikvInit.Do(func() {
+		tikvFactory = factory
+	})
+}
+
+func GetDefaultTiKVFactory() (factory TiKvFactory) {
+	return tikvFactory
+}

internal/tikv/byte_alloc/alloc.go

@@ -0,0 +1,73 @@
+// from https://github.com/xtaci/smux/blob/master/alloc.go
+package byte_alloc
+
+import (
+	"sync"
+)
+
+// magic number
+var debruijinPos = [...]byte{0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31}
+
+// Allocator for incoming frames, optimized to prevent overwriting after zeroing
+type Allocator struct {
+	buffers []sync.Pool
+}
+
+// NewAllocator initiates a []byte allocator for frames less than 1048576 bytes,
+// the waste(memory fragmentation) of space allocation is guaranteed to be
+// no more than 50%.
+func NewAllocator() *Allocator {
+	alloc := new(Allocator)
+	alloc.buffers = make([]sync.Pool, 21) // 1B -> 1M
+	for k := range alloc.buffers {
+		size := 1 << uint32(k)
+		alloc.buffers[k].New = func() interface{} {
+			return make([]byte, size)
+		}
+	}
+	return alloc
+}
+
+// Get a []byte from pool with most appropriate cap
+func (alloc *Allocator) Get(size int) []byte {
+	if size <= 0 {
+		return nil
+	}
+
+	if size > 1048576 {
+		return make([]byte, size)
+	}
+
+	bits := msb(size)
+	if size == 1<<bits {
+		return alloc.buffers[bits].Get().([]byte)[:size]
+	} else {
+		return alloc.buffers[bits+1].Get().([]byte)[:size]
+	}
+}
+
+// Put returns a []byte to pool for future use,
+// which the cap must be exactly 2^n
+func (alloc *Allocator) Put(buf []byte) {
+	bufCap := cap(buf)
+	bits := msb(bufCap)
+	if bufCap == 0 || bufCap > 65536 || bufCap != 1<<bits {
+		return
+	}
+
+	alloc.buffers[bits].Put(buf)
+	return
+}
+
+// msb return the pos of most significiant bit
+// Equivalent to: uint(math.Floor(math.Log2(float64(n))))
+// http://supertech.csail.mit.edu/papers/debruijn.pdf
+func msb(size int) byte {
+	v := uint32(size)
+	v |= v >> 1
+	v |= v >> 2
+	v |= v >> 4
+	v |= v >> 8
+	v |= v >> 16
+	return debruijinPos[(v*0x07C4ACDD)>>27]
+}

internal/tikv/byte_alloc/defailt.go

@@ -0,0 +1,15 @@
+package byte_alloc
+
+var defaultAllocator *Allocator
+
+func init() {
+	defaultAllocator = NewAllocator()
+}
+
+func Get(size int) []byte {
+	return defaultAllocator.Get(size)
+}
+
+func Put(buf []byte) {
+	defaultAllocator.Put(buf)
+}

internal/tikv/common/errors.go

@@ -0,0 +1,10 @@
+package common
+
+import (
+	"errors"
+
+	"github.com/syndtr/goleveldb/leveldb"
+)
+
+var ErrEmptyKey = errors.New("empty key is not supported")
+var ErrNotFound = leveldb.ErrNotFound

internal/tikv/common/hack.go

@@ -0,0 +1,20 @@
+package common
+
+import (
+	"unsafe"
+)
+
+// String converts byte slice to string.
+func String(b []byte) string {
+	return *(*string)(unsafe.Pointer(&b))
+}
+
+// StringBytes converts string to byte slice.
+func StringBytes(s string) []byte {
+	return *(*[]byte)(unsafe.Pointer(
+		&struct {
+			string
+			Cap int
+		}{s, len(s)},
+	))
+}

internal/tikv/common/tikv_store.go

@@ -0,0 +1,41 @@
+package common
+
+import (
+	"io"
+
+	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/syndtr/goleveldb/leveldb/util"
+)
+
+type TiKVStore interface {
+	ethdb.KeyValueReader
+	ethdb.KeyValueWriter
+	ethdb.Batcher
+	ethdb.Stater
+	ethdb.Compacter
+	io.Closer
+
+	NewIterator(start, end []byte) ethdb.Iterator
+}
+
+// TiKVStoreWrapper simple wrapper to covert to ethdb.KeyValueStore
+type TiKVStoreWrapper struct {
+	TiKVStore
+}
+
+func (t *TiKVStoreWrapper) NewIterator() ethdb.Iterator {
+	return t.TiKVStore.NewIterator(nil, nil)
+}
+
+func (t *TiKVStoreWrapper) NewIteratorWithStart(start []byte) ethdb.Iterator {
+	return t.TiKVStore.NewIterator(start, nil)
+}
+
+func (t *TiKVStoreWrapper) NewIteratorWithPrefix(prefix []byte) ethdb.Iterator {
+	bytesPrefix := util.BytesPrefix(prefix)
+	return t.TiKVStore.NewIterator(bytesPrefix.Start, bytesPrefix.Limit)
+}
+
+func ToEthKeyValueStore(store TiKVStore) ethdb.KeyValueStore {
+	return &TiKVStoreWrapper{TiKVStore: store}
+}

internal/tikv/consts.go

@@ -0,0 +1,6 @@
+package tikv
+
+const (
+	RoleReader = "Reader"
+	RoleWriter = "Writer"
+)

internal/tikv/prefix/prefix_batch.go

@@ -0,0 +1,42 @@
+package prefix
+
+import "github.com/ethereum/go-ethereum/ethdb"
+
+type PrefixBatch struct {
+	prefix []byte
+	batch  ethdb.Batch
+}
+
+func newPrefixBatch(prefix []byte, batch ethdb.Batch) *PrefixBatch {
+	return &PrefixBatch{prefix: prefix, batch: batch}
+}
+
+// Put inserts the given value into the key-value data store.
+func (p *PrefixBatch) Put(key []byte, value []byte) error {
+	return p.batch.Put(append(append([]byte{}, p.prefix...), key...), value)
+}
+
+// Delete removes the key from the key-value data store.
+func (p *PrefixBatch) Delete(key []byte) error {
+	return p.batch.Delete(append(append([]byte{}, p.prefix...), key...))
+}
+
+// ValueSize retrieves the amount of data queued up for writing.
+func (p *PrefixBatch) ValueSize() int {
+	return p.batch.ValueSize()
+}
+
+// Write flushes any accumulated data to disk.
+func (p *PrefixBatch) Write() error {
+	return p.batch.Write()
+}
+
+// Reset resets the batch for reuse.
+func (p *PrefixBatch) Reset() {
+	p.batch.Reset()
+}
+
+// Replay replays the batch contents.
+func (p *PrefixBatch) Replay(w ethdb.KeyValueWriter) error {
+	return p.batch.Replay(newPrefixBatchReplay(p.prefix, w))
+}

internal/tikv/prefix/prefix_batch_replay.go

@@ -0,0 +1,35 @@
+package prefix
+
+import (
+	"bytes"
+
+	"github.com/ethereum/go-ethereum/ethdb"
+)
+
+type PrefixBatchReplay struct {
+	prefix    []byte
+	prefixLen int
+	w         ethdb.KeyValueWriter
+}
+
+func newPrefixBatchReplay(prefix []byte, w ethdb.KeyValueWriter) *PrefixBatchReplay {
+	return &PrefixBatchReplay{prefix: prefix, prefixLen: len(prefix), w: w}
+}
+
+// Put inserts the given value into the key-value data store.
+func (p *PrefixBatchReplay) Put(key []byte, value []byte) error {
+	if bytes.HasPrefix(key, p.prefix) {
+		return p.w.Put(key[p.prefixLen:], value)
+	} else {
+		return p.w.Put(key, value)
+	}
+}
+
+// Delete removes the key from the key-value data store.
+func (p *PrefixBatchReplay) Delete(key []byte) error {
+	if bytes.HasPrefix(key, p.prefix) {
+		return p.w.Delete(key[p.prefixLen:])
+	} else {
+		return p.w.Delete(key)
+	}
+}

internal/tikv/prefix/prefix_database.go

@@ -0,0 +1,109 @@
+package prefix
+
+import (
+	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/harmony-one/harmony/internal/tikv/byte_alloc"
+	"github.com/harmony-one/harmony/internal/tikv/common"
+)
+
+// PrefixDatabase is a wrapper to split the storage with prefix
+type PrefixDatabase struct {
+	prefix   []byte
+	db       common.TiKVStore
+	keysPool *byte_alloc.Allocator
+}
+
+func NewPrefixDatabase(prefix []byte, db common.TiKVStore) *PrefixDatabase {
+	return &PrefixDatabase{
+		prefix:   prefix,
+		db:       db,
+		keysPool: byte_alloc.NewAllocator(),
+	}
+}
+
+// makeKey use to create a key with prefix, keysPool can reduce gc pressure
+func (p *PrefixDatabase) makeKey(keys []byte) []byte {
+	prefixLen := len(p.prefix)
+	byt := p.keysPool.Get(len(keys) + prefixLen)
+	copy(byt, p.prefix)
+	copy(byt[prefixLen:], keys)
+
+	return byt
+}
+
+// Has retrieves if a key is present in the key-value data store.
+func (p *PrefixDatabase) Has(key []byte) (bool, error) {
+	return p.db.Has(p.makeKey(key))
+}
+
+// Get retrieves the given key if it's present in the key-value data store.
+func (p *PrefixDatabase) Get(key []byte) ([]byte, error) {
+	return p.db.Get(p.makeKey(key))
+}
+
+// Put inserts the given value into the key-value data store.
+func (p *PrefixDatabase) Put(key []byte, value []byte) error {
+	return p.db.Put(p.makeKey(key), value)
+}
+
+// Delete removes the key from the key-value data store.
+func (p *PrefixDatabase) Delete(key []byte) error {
+	return p.db.Delete(p.makeKey(key))
+}
+
+// NewBatch creates a write-only database that buffers changes to its host db
+// until a final write is called.
+func (p *PrefixDatabase) NewBatch() ethdb.Batch {
+	return newPrefixBatch(p.prefix, p.db.NewBatch())
+}
+
+// buildLimitUsePrefix build the limit byte from start byte, useful generating from prefix works
+func (p *PrefixDatabase) buildLimitUsePrefix() []byte {
+	var limit []byte
+	for i := len(p.prefix) - 1; i >= 0; i-- {
+		c := p.prefix[i]
+		if c < 0xff {
+			limit = make([]byte, i+1)
+			copy(limit, p.prefix)
+			limit[i] = c + 1
+			break
+		}
+	}
+
+	return limit
+}
+
+// NewIterator creates a binary-alphabetical iterator over the start to end keyspace
+// contained within the key-value database.
+func (p *PrefixDatabase) NewIterator(start, end []byte) ethdb.Iterator {
+	start = append(p.prefix, start...)
+
+	if len(end) == 0 {
+		end = p.buildLimitUsePrefix()
+	} else {
+		end = append(p.prefix, end...)
+	}
+
+	return newPrefixIterator(p.prefix, p.db.NewIterator(start, end))
+}
+
+// Stat returns a particular internal stat of the database.
+func (p *PrefixDatabase) Stat(property string) (string, error) {
+	return p.db.Stat(property)
+}
+
+// Compact flattens the underlying data store for the given key range. In essence,
+// deleted and overwritten versions are discarded, and the data is rearranged to
+// reduce the cost of operations needed to access them.
+//
+// A nil start is treated as a key before all keys in the data store; a nil limit
+// is treated as a key after all keys in the data store. If both is nil then it
+// will compact entire data store.
+func (p *PrefixDatabase) Compact(start []byte, limit []byte) error {
+	return p.db.Compact(start, limit)
+}
+
+// Close the storage
+func (p *PrefixDatabase) Close() error {
+	return p.db.Close()
+}

internal/tikv/prefix/prefix_iterator.go

@@ -0,0 +1,53 @@
+package prefix
+
+import (
+	"github.com/ethereum/go-ethereum/ethdb"
+)
+
+type PrefixIterator struct {
+	prefix    []byte
+	prefixLen int
+
+	it ethdb.Iterator
+}
+
+func newPrefixIterator(prefix []byte, it ethdb.Iterator) *PrefixIterator {
+	return &PrefixIterator{prefix: prefix, prefixLen: len(prefix), it: it}
+}
+
+// Next moves the iterator to the next key/value pair. It returns whether the
+// iterator is exhausted.
+func (i *PrefixIterator) Next() bool {
+	return i.it.Next()
+}
+
+// Error returns any accumulated error. Exhausting all the key/value pairs
+// is not considered to be an error.
+func (i *PrefixIterator) Error() error {
+	return i.it.Error()
+}
+
+// Key returns the key of the current key/value pair, or nil if done. The caller
+// should not modify the contents of the returned slice, and its contents may
+// change on the next call to Next.
+func (i *PrefixIterator) Key() []byte {
+	key := i.it.Key()
+	if len(key) < len(i.prefix) {
+		return nil
+	}
+
+	return key[i.prefixLen:]
+}
+
+// Value returns the value of the current key/value pair, or nil if done. The
+// caller should not modify the contents of the returned slice, and its contents
+// may change on the next call to Next.
+func (i *PrefixIterator) Value() []byte {
+	return i.it.Value()
+}
+
+// Release releases associated resources. Release should always succeed and can
+// be called multiple times without causing error.
+func (i *PrefixIterator) Release() {
+	i.it.Release()
+}

internal/tikv/redis_helper/common.go

@@ -0,0 +1,34 @@
+package redis_helper
+
+import (
+	"context"
+
+	"github.com/go-redis/redis/v8"
+)
+
+var redisInstance *redis.ClusterClient
+
+// Init used to init redis instance in tikv mode
+func Init(serverAddr []string) error {
+	option := &redis.ClusterOptions{
+		Addrs:    serverAddr,
+		PoolSize: 2,
+	}
+
+	rdb := redis.NewClusterClient(option)
+	err := rdb.Ping(context.Background()).Err()
+	if err != nil {
+		return err
+	}
+
+	redisInstance = rdb
+	return nil
+}
+
+// Close disconnect redis instance in tikv mode
+func Close() error {
+	if redisInstance != nil {
+		return redisInstance.Close()
+	}
+	return nil
+}

internal/tikv/redis_helper/lock.go

@@ -0,0 +1,72 @@
+package redis_helper
+
+import (
+	"context"
+	"crypto/rand"
+	"encoding/base64"
+
+	"github.com/go-redis/redis/v8"
+)
+
+type RedisPreempt struct {
+	key                      string
+	password                 string
+	lockScript, unlockScript *redis.Script
+	lastLockStatus           bool
+}
+
+// CreatePreempt used to create a redis preempt instance
+func CreatePreempt(key string) *RedisPreempt {
+	p := &RedisPreempt{
+		key: key,
+	}
+	p.init()
+
+	return p
+}
+
+// init redis preempt instance and some script
+func (p *RedisPreempt) init() {
+	byt := make([]byte, 18)
+	_, _ = rand.Read(byt)
+	p.password = base64.StdEncoding.EncodeToString(byt)
+	p.lockScript = redis.NewScript(`
+		if redis.call('get',KEYS[1]) == ARGV[1] then
+			return redis.call('expire', KEYS[1], ARGV[2])
+		else
+			return redis.call('set', KEYS[1], ARGV[1], 'ex', ARGV[2], 'nx')
+		end
+	`)
+	p.unlockScript = redis.NewScript(`
+		if redis.call('get',KEYS[1]) == ARGV[1] then
+			return redis.call('del', KEYS[1])
+		else
+			return 0
+		end
+	`)
+}
+
+// TryLock attempt to lock the master for ttlSecond
+func (p *RedisPreempt) TryLock(ttlSecond int) (ok bool, err error) {
+	ok, err = p.lockScript.Run(context.Background(), redisInstance, []string{p.key}, p.password, ttlSecond).Bool()
+	p.lastLockStatus = ok
+	return
+}
+
+// Unlock try to release the master permission
+func (p *RedisPreempt) Unlock() (bool, error) {
+	if p == nil {
+		return false, nil
+	}
+
+	return p.unlockScript.Run(context.Background(), redisInstance, []string{p.key}, p.password).Bool()
+}
+
+// LastLockStatus get the last preempt status
+func (p *RedisPreempt) LastLockStatus() bool {
+	if p == nil {
+		return false
+	}
+
+	return p.lastLockStatus
+}

internal/tikv/redis_helper/pubsub.go

@@ -0,0 +1,130 @@
+package redis_helper
+
+import (
+	"context"
+	"fmt"
+	"io"
+
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/harmony-one/harmony/core/types"
+	"github.com/harmony-one/harmony/internal/utils"
+	stakingTypes "github.com/harmony-one/harmony/staking/types"
+	"github.com/pkg/errors"
+)
+
+// BlockUpdate block update event
+type BlockUpdate struct {
+	BlkNum uint64
+	Logs   []*types.Log
+}
+
+// SubscribeShardUpdate subscribe block update event
+func SubscribeShardUpdate(shardID uint32, cb func(blkNum uint64, logs []*types.Log)) {
+	pubsub := redisInstance.Subscribe(context.Background(), fmt.Sprintf("shard_update_%d", shardID))
+	for message := range pubsub.Channel() {
+		block := &BlockUpdate{}
+		err := rlp.DecodeBytes([]byte(message.Payload), block)
+		if err != nil {
+			utils.Logger().Info().Err(err).Msg("redis subscribe shard update error")
+			continue
+		}
+		cb(block.BlkNum, block.Logs)
+	}
+}
+
+// PublishShardUpdate publish block update event
+func PublishShardUpdate(shardID uint32, blkNum uint64, logs []*types.Log) error {
+	msg, err := rlp.EncodeToBytes(&BlockUpdate{
+		BlkNum: blkNum,
+		Logs:   logs,
+	})
+	if err != nil {
+		return err
+	}
+	return redisInstance.Publish(context.Background(), fmt.Sprintf("shard_update_%d", shardID), msg).Err()
+}
+
+//TxPoolUpdate tx pool update event
+type TxPoolUpdate struct {
+	typ   string
+	Local bool
+	Tx    types.PoolTransaction
+}
+
+// DecodeRLP decode struct from binary stream
+func (t *TxPoolUpdate) DecodeRLP(stream *rlp.Stream) error {
+	if err := stream.Decode(&t.typ); err != nil {
+		return err
+	}
+	if err := stream.Decode(&t.Local); err != nil {
+		return err
+	}
+
+	switch t.typ {
+	case "types.EthTransaction":
+		var tmp = &types.EthTransaction{}
+		if err := stream.Decode(tmp); err != nil {
+			return err
+		}
+		t.Tx = tmp
+	case "types.Transaction":
+		var tmp = &types.Transaction{}
+		if err := stream.Decode(tmp); err != nil {
+			return err
+		}
+		t.Tx = tmp
+	case "stakingTypes.StakingTransaction":
+		var tmp = &stakingTypes.StakingTransaction{}
+		if err := stream.Decode(tmp); err != nil {
+			return err
+		}
+		t.Tx = tmp
+	default:
+		return errors.New("unknown txpool type")
+	}
+	return nil
+}
+
+// EncodeRLP encode struct to binary stream
+func (t *TxPoolUpdate) EncodeRLP(w io.Writer) error {
+	switch t.Tx.(type) {
+	case *types.EthTransaction:
+		t.typ = "types.EthTransaction"
+	case *types.Transaction:
+		t.typ = "types.Transaction"
+	case *stakingTypes.StakingTransaction:
+		t.typ = "stakingTypes.StakingTransaction"
+	}
+
+	if err := rlp.Encode(w, t.typ); err != nil {
+		return err
+	}
+	if err := rlp.Encode(w, t.Local); err != nil {
+		return err
+	}
+	return rlp.Encode(w, t.Tx)
+}
+
+// SubscribeTxPoolUpdate subscribe tx pool update event
+func SubscribeTxPoolUpdate(shardID uint32, cb func(tx types.PoolTransaction, local bool)) {
+	pubsub := redisInstance.Subscribe(context.Background(), fmt.Sprintf("txpool_update_%d", shardID))
+	for message := range pubsub.Channel() {
+		txu := &TxPoolUpdate{}
+		err := rlp.DecodeBytes([]byte(message.Payload), &txu)
+		if err != nil {
+			utils.Logger().Info().Err(err).Msg("redis subscribe shard update error")
+			continue
+		}
+		cb(txu.Tx, txu.Local)
+	}
+}
+
+// PublishTxPoolUpdate publish tx pool update event
+func PublishTxPoolUpdate(shardID uint32, tx types.PoolTransaction, local bool) error {
+	txu := &TxPoolUpdate{Local: local, Tx: tx}
+	msg, err := rlp.EncodeToBytes(txu)
+	if err != nil {
+		return err
+	}
+	return redisInstance.Publish(context.Background(), fmt.Sprintf("txpool_update_%d", shardID), msg).Err()
+}

internal/tikv/remote/remote_batch.go

@@ -0,0 +1,155 @@
+package remote
+
+import (
+	"bytes"
+	"context"
+	"sync"
+
+	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/harmony-one/harmony/internal/tikv/common"
+)
+
+type operate int
+
+const (
+	_ operate = iota
+	operatePut
+	operateDelete
+)
+
+type valueInfo struct {
+	operate  operate
+	key, val []byte
+}
+
+type RemoteBatch struct {
+	lock sync.Mutex
+
+	db *RemoteDatabase
+
+	size   int
+	valMap map[string]*valueInfo
+}
+
+func newRemoteBatch(db *RemoteDatabase) *RemoteBatch {
+	return &RemoteBatch{
+		db:     db,
+		valMap: make(map[string]*valueInfo),
+	}
+}
+
+func (b *RemoteBatch) copy(val []byte) []byte {
+	tmp := make([]byte, len(val))
+	copy(tmp, val)
+	return tmp
+}
+
+// Put inserts the given value into the key-value data store.
+func (b *RemoteBatch) Put(key []byte, value []byte) error {
+	if len(key) == 0 {
+		return common.ErrEmptyKey
+	}
+
+	if len(value) == 0 {
+		value = EmptyValueStub
+	}
+
+	b.lock.Lock()
+	defer b.lock.Unlock()
+
+	b.valMap[string(key)] = &valueInfo{
+		operate: operatePut,
+		key:     b.copy(key),
+		val:     b.copy(value),
+	}
+
+	b.size += len(key) + len(value)
+	return nil
+}
+
+// Delete removes the key from the key-value data store.
+func (b *RemoteBatch) Delete(key []byte) error {
+	b.lock.Lock()
+	defer b.lock.Unlock()
+
+	b.valMap[string(key)] = &valueInfo{
+		operate: operateDelete,
+		key:     b.copy(key),
+	}
+
+	b.size += len(key)
+	return nil
+}
+
+// ValueSize retrieves the amount of data queued up for writing.
+func (b *RemoteBatch) ValueSize() int {
+	return b.size
+}
+
+// Write flushes any accumulated data to disk.
+func (b *RemoteBatch) Write() error {
+	b.lock.Lock()
+	defer b.lock.Unlock()
+
+	batchWriteKey := make([][]byte, 0)
+	batchWriteValue := make([][]byte, 0)
+	batchDeleteKey := make([][]byte, 0)
+
+	for _, info := range b.valMap {
+		switch info.operate {
+		case operatePut:
+			batchWriteKey = append(batchWriteKey, info.key)
+			batchWriteValue = append(batchWriteValue, info.val)
+		case operateDelete:
+			batchDeleteKey = append(batchDeleteKey, info.key)
+		}
+	}
+
+	if len(batchWriteKey) > 0 {
+		err := b.db.client.BatchPut(context.Background(), batchWriteKey, batchWriteValue)
+		if err != nil {
+			return err
+		}
+	}
+
+	if len(batchDeleteKey) > 0 {
+		err := b.db.client.BatchDelete(context.Background(), batchDeleteKey)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// Reset resets the batch for reuse.
+func (b *RemoteBatch) Reset() {
+	b.lock.Lock()
+	defer b.lock.Unlock()
+
+	b.valMap = make(map[string]*valueInfo)
+	b.size = 0
+}
+
+// Replay replays the batch contents.
+func (b *RemoteBatch) Replay(w ethdb.KeyValueWriter) error {
+	var err error
+	for _, info := range b.valMap {
+		switch info.operate {
+		case operatePut:
+			if bytes.Compare(info.val, EmptyValueStub) == 0 {
+				err = w.Put(info.key, []byte{})
+			} else {
+				err = w.Put(info.key, info.val)
+			}
+		case operateDelete:
+			err = w.Delete(info.key)
+		}
+
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

internal/tikv/remote/remote_database.go

@@ -0,0 +1,139 @@
+package remote
+
+import (
+	"bytes"
+	"context"
+	"runtime/trace"
+	"sync/atomic"
+
+	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/harmony-one/harmony/internal/tikv/common"
+	"github.com/tikv/client-go/v2/config"
+	"github.com/tikv/client-go/v2/rawkv"
+)
+
+var EmptyValueStub = []byte("HarmonyTiKVEmptyValueStub")
+
+type RemoteDatabase struct {
+	client   *rawkv.Client
+	readOnly bool
+	isClose  uint64
+}
+
+func NewRemoteDatabase(pdAddr []string, readOnly bool) (*RemoteDatabase, error) {
+	client, err := rawkv.NewClient(context.Background(), pdAddr, config.DefaultConfig().Security)
+	if err != nil {
+		return nil, err
+	}
+
+	db := &RemoteDatabase{
+		client:   client,
+		readOnly: readOnly,
+	}
+
+	return db, nil
+}
+
+// ReadOnly set storage to readonly mode
+func (d *RemoteDatabase) ReadOnly() {
+	d.readOnly = true
+}
+
+// Has retrieves if a key is present in the key-value data store.
+func (d *RemoteDatabase) Has(key []byte) (bool, error) {
+	data, err := d.Get(key)
+	if err != nil {
+		if err == common.ErrNotFound {
+			return false, nil
+		}
+		return false, err
+	} else {
+		return len(data) != 0, nil
+	}
+}
+
+// Get retrieves the given key if it's present in the key-value data store.
+func (d *RemoteDatabase) Get(key []byte) ([]byte, error) {
+	if len(key) == 0 {
+		return nil, common.ErrEmptyKey
+	}
+
+	region := trace.StartRegion(context.Background(), "tikv Get")
+	defer region.End()
+
+	get, err := d.client.Get(context.Background(), key)
+	if err != nil {
+		return nil, err
+	}
+
+	if len(get) == 0 {
+		return nil, common.ErrNotFound
+	}
+
+	if len(get) == len(EmptyValueStub) && bytes.Compare(get, EmptyValueStub) == 0 {
+		get = get[:0]
+	}
+
+	return get, nil
+}
+
+// Put inserts the given value into the key-value data store.
+func (d *RemoteDatabase) Put(key []byte, value []byte) error {
+	if len(key) == 0 {
+		return common.ErrEmptyKey
+	}
+	if d.readOnly {
+		return nil
+	}
+
+	if len(value) == 0 {
+		value = EmptyValueStub
+	}
+
+	return d.client.Put(context.Background(), key, value)
+}
+
+// Delete removes the key from the key-value data store.
+func (d *RemoteDatabase) Delete(key []byte) error {
+	if len(key) == 0 {
+		return common.ErrEmptyKey
+	}
+	if d.readOnly {
+		return nil
+	}
+
+	return d.client.Delete(context.Background(), key)
+}
+
+// NewBatch creates a write-only database that buffers changes to its host db
+// until a final write is called.
+func (d *RemoteDatabase) NewBatch() ethdb.Batch {
+	if d.readOnly {
+		return newNopRemoteBatch(d)
+	}
+
+	return newRemoteBatch(d)
+}
+
+func (d *RemoteDatabase) NewIterator(start, end []byte) ethdb.Iterator {
+	return newRemoteIterator(d, start, end)
+}
+
+// Stat returns a particular internal stat of the database.
+func (d *RemoteDatabase) Stat(property string) (string, error) {
+	return "", common.ErrNotFound
+}
+
+// Compact tikv current not supprot manual compact
+func (d *RemoteDatabase) Compact(start []byte, limit []byte) error {
+	return nil
+}
+
+// Close disconnect the tikv
+func (d *RemoteDatabase) Close() error {
+	if atomic.CompareAndSwapUint64(&d.isClose, 0, 1) {
+		return d.client.Close()
+	}
+
+	return nil
+}

internal/tikv/remote/remote_iterator.go

@@ -0,0 +1,110 @@
+package remote
+
+import (
+	"context"
+	"sync"
+)
+
+const (
+	iteratorOnce = 300
+)
+
+type RemoteIterator struct {
+	db   *RemoteDatabase
+	lock sync.Mutex
+
+	limit []byte
+	start []byte
+
+	err error
+	end bool
+	pos int
+
+	keys, values             [][]byte
+	currentKey, currentValue []byte
+}
+
+func newRemoteIterator(db *RemoteDatabase, start, limit []byte) *RemoteIterator {
+	return &RemoteIterator{
+		db:    db,
+		start: start,
+		limit: limit,
+	}
+}
+
+// Next moves the iterator to the next key/value pair. It returns whether the
+// iterator is exhausted.
+func (i *RemoteIterator) Next() bool {
+	if i.end {
+		return false
+	}
+
+	if i.keys == nil {
+		if next := i.scanNext(); !next {
+			return false
+		}
+	}
+
+	i.currentKey = i.keys[i.pos]
+	i.currentValue = i.values[i.pos]
+	i.pos++
+
+	if i.pos >= len(i.keys) {
+		i.scanNext()
+	}
+
+	return true
+}
+
+// scanNext real scan from tikv, and cache it
+func (i *RemoteIterator) scanNext() bool {
+	keys, values, err := i.db.client.Scan(context.Background(), i.start, i.limit, iteratorOnce)
+	if err != nil {
+		i.err = err
+		i.end = true
+		return false
+	}
+
+	if len(keys) == 0 {
+		i.end = true
+		return false
+	} else {
+		i.start = append(keys[len(keys)-1], 0)
+	}
+
+	i.pos = 0
+	i.keys = keys
+	i.values = values
+	return true
+}
+
+// Error returns any accumulated error. Exhausting all the key/value pairs
+// is not considered to be an error.
+func (i *RemoteIterator) Error() error {
+	return i.err
+}
+
+// Key returns the key of the current key/value pair, or nil if done. The caller
+// should not modify the contents of the returned slice, and its contents may
+// change on the next call to Next.
+func (i *RemoteIterator) Key() []byte {
+	return i.currentKey
+}
+
+// Value returns the value of the current key/value pair, or nil if done. The
+// caller should not modify the contents of the returned slice, and its contents
+// may change on the next call to Next.
+func (i *RemoteIterator) Value() []byte {
+	return i.currentValue
+}
+
+// Release releases associated resources. Release should always succeed and can
+// be called multiple times without causing error.
+func (i *RemoteIterator) Release() {
+	i.db = nil
+	i.end = true
+	i.keys = nil
+	i.values = nil
+	i.currentKey = nil
+	i.currentValue = nil
+}

internal/tikv/remote/remote_nop_batch.go

@@ -0,0 +1,36 @@
+package remote
+
+import (
+	"github.com/ethereum/go-ethereum/ethdb"
+)
+
+// NopRemoteBatch on readonly mode, write operator will be reject
+type NopRemoteBatch struct {
+}
+
+func newNopRemoteBatch(db *RemoteDatabase) *NopRemoteBatch {
+	return &NopRemoteBatch{}
+}
+
+func (b *NopRemoteBatch) Put(key []byte, value []byte) error {
+	return nil
+}
+
+func (b *NopRemoteBatch) Delete(key []byte) error {
+	return nil
+}
+
+func (b *NopRemoteBatch) ValueSize() int {
+	return 0
+}
+
+func (b *NopRemoteBatch) Write() error {
+	return nil
+}
+
+func (b *NopRemoteBatch) Reset() {
+}
+
+func (b *NopRemoteBatch) Replay(w ethdb.KeyValueWriter) error {
+	return nil
+}

internal/tikv/statedb_cache/statedb_cache_batch.go

@@ -0,0 +1,45 @@
+package statedb_cache
+
+import (
+	"github.com/ethereum/go-ethereum/ethdb"
+)
+
+type StateDBCacheBatch struct {
+	db          *StateDBCacheDatabase
+	remoteBatch ethdb.Batch
+}
+
+func newStateDBCacheBatch(db *StateDBCacheDatabase, batch ethdb.Batch) *StateDBCacheBatch {
+	return &StateDBCacheBatch{db: db, remoteBatch: batch}
+}
+
+func (b *StateDBCacheBatch) Put(key []byte, value []byte) error {
+	return b.remoteBatch.Put(key, value)
+}
+
+func (b *StateDBCacheBatch) Delete(key []byte) error {
+	return b.remoteBatch.Delete(key)
+}
+
+func (b *StateDBCacheBatch) ValueSize() int {
+	// In ethdb, the size of each commit is too small, this controls the submission frequency
+	return b.remoteBatch.ValueSize() / 40
+}
+
+func (b *StateDBCacheBatch) Write() (err error) {
+	defer func() {
+		if err == nil {
+			_ = b.db.cacheWrite(b)
+		}
+	}()
+
+	return b.remoteBatch.Write()
+}
+
+func (b *StateDBCacheBatch) Reset() {
+	b.remoteBatch.Reset()
+}
+
+func (b *StateDBCacheBatch) Replay(w ethdb.KeyValueWriter) error {
+	return b.remoteBatch.Replay(w)
+}

internal/tikv/statedb_cache/statedb_cache_database.go

@@ -0,0 +1,373 @@
+package statedb_cache
+
+import (
+	"context"
+	"log"
+	"runtime"
+	"sync/atomic"
+	"time"
+
+	"github.com/VictoriaMetrics/fastcache"
+	"github.com/ethereum/go-ethereum/ethdb"
+	redis "github.com/go-redis/redis/v8"
+	"github.com/harmony-one/harmony/internal/tikv/common"
+)
+
+const (
+	maxMemoryEntrySize      = 64 * 1024
+	maxPipelineEntriesCount = 10000
+	cacheMinGapInSecond     = 600
+)
+
+type cacheWrapper struct {
+	*fastcache.Cache
+}
+
+// Put inserts the given value into the key-value data store.
+func (c *cacheWrapper) Put(key []byte, value []byte) error {
+	c.Cache.Set(key, value)
+	return nil
+}
+
+// Delete removes the key from the key-value data store.
+func (c *cacheWrapper) Delete(key []byte) error {
+	c.Cache.Del(key)
+	return nil
+}
+
+type redisPipelineWrapper struct {
+	redis.Pipeliner
+
+	expiredTime time.Duration
+}
+
+// Put inserts the given value into the key-value data store.
+func (c *redisPipelineWrapper) Put(key []byte, value []byte) error {
+	c.SetEX(context.Background(), string(key), value, c.expiredTime)
+	return nil
+}
+
+// Delete removes the key from the key-value data store.
+func (c *redisPipelineWrapper) Delete(key []byte) error {
+	c.Del(context.Background(), string(key))
+	return nil
+}
+
+type StateDBCacheConfig struct {
+	CacheSizeInMB        uint32
+	CachePersistencePath string
+	RedisServerAddr      []string
+	RedisLRUTimeInDay    uint32
+	DebugHitRate         bool
+}
+
+type StateDBCacheDatabase struct {
+	config          StateDBCacheConfig
+	enableReadCache bool
+	isClose         uint64
+	remoteDB        common.TiKVStore
+
+	// l1cache (memory)
+	l1Cache *cacheWrapper
+
+	// l2cache (redis)
+	l2Cache           *redis.ClusterClient
+	l2ReadOnly        bool
+	l2ExpiredTime     time.Duration
+	l2NextExpiredTime time.Time
+	l2ExpiredRefresh  chan string
+
+	// stats
+	l1HitCount           uint64
+	l2HitCount           uint64
+	missCount            uint64
+	notFoundOrErrorCount uint64
+}
+
+func NewStateDBCacheDatabase(remoteDB common.TiKVStore, config StateDBCacheConfig, readOnly bool) (*StateDBCacheDatabase, error) {
+	// create or load memory cache from file
+	cache := fastcache.LoadFromFileOrNew(config.CachePersistencePath, int(config.CacheSizeInMB)*1024*1024)
+
+	// create options
+	option := &redis.ClusterOptions{
+		Addrs:       config.RedisServerAddr,
+		PoolSize:    32,
+		IdleTimeout: 60 * time.Second,
+	}
+
+	if readOnly {
+		option.PoolSize = 16
+		option.ReadOnly = true
+	}
+
+	// check redis connection
+	rdb := redis.NewClusterClient(option)
+	err := rdb.Ping(context.Background()).Err()
+	if err != nil {
+		return nil, err
+	}
+
+	// reload redis cluster slots status
+	rdb.ReloadState(context.Background())
+
+	db := &StateDBCacheDatabase{
+		config:          config,
+		enableReadCache: true,
+		remoteDB:        remoteDB,
+		l1Cache:         &cacheWrapper{cache},
+		l2Cache:         rdb,
+		l2ReadOnly:      readOnly,
+		l2ExpiredTime:   time.Duration(config.RedisLRUTimeInDay) * 24 * time.Hour,
+	}
+
+	if !readOnly {
+		// Read a copy of the memory hit data into redis to improve the hit rate
+		// refresh read time to prevent recycling by lru
+		db.l2ExpiredRefresh = make(chan string, 100000)
+		go db.startL2ExpiredRefresh()
+	}
+
+	// print debug info
+	if config.DebugHitRate {
+		go func() {
+			for range time.Tick(5 * time.Second) {
+				db.cacheStatusPrint()
+			}
+		}()
+	}
+
+	return db, nil
+}
+
+// Has retrieves if a key is present in the key-value data store.
+func (c *StateDBCacheDatabase) Has(key []byte) (bool, error) {
+	return c.remoteDB.Has(key)
+}
+
+// Get retrieves the given key if it's present in the key-value data store.
+func (c *StateDBCacheDatabase) Get(key []byte) (ret []byte, err error) {
+	if c.enableReadCache {
+		var ok bool
+
+		// first, get data from memory cache
+		keyStr := string(key)
+		if ret, ok = c.l1Cache.HasGet(nil, key); ok {
+			if !c.l2ReadOnly {
+				select {
+				// refresh read time to prevent recycling by lru
+				case c.l2ExpiredRefresh <- keyStr:
+				default:
+				}
+			}
+
+			atomic.AddUint64(&c.l1HitCount, 1)
+			return ret, nil
+		}
+
+		defer func() {
+			if err == nil {
+				if len(ret) < maxMemoryEntrySize {
+					// set data to memory db if loaded from redis cache or leveldb
+					c.l1Cache.Set(key, ret)
+				}
+			}
+		}()
+
+		timeoutCtx, cancelFunc := context.WithTimeout(context.Background(), 5*time.Second)
+		defer cancelFunc()
+
+		// load data from redis cache
+		data := c.l2Cache.Get(timeoutCtx, keyStr)
+		if data.Err() == nil {
+			atomic.AddUint64(&c.l2HitCount, 1)
+			return data.Bytes()
+		} else if data.Err() != redis.Nil {
+			log.Printf("[Get WARN]Redis Error: %v", data.Err())
+		}
+
+		if !c.l2ReadOnly {
+			defer func() {
+				// set data to redis db if loaded from leveldb
+				if err == nil {
+					c.l2Cache.SetEX(context.Background(), keyStr, ret, c.l2ExpiredTime)
+				}
+			}()
+		}
+	}
+
+	ret, err = c.remoteDB.Get(key)
+	if err != nil {
+		atomic.AddUint64(&c.notFoundOrErrorCount, 1)
+	} else {
+		atomic.AddUint64(&c.missCount, 1)
+	}
+
+	return
+}
+
+// Put inserts the given value into the key-value data store.
+func (c *StateDBCacheDatabase) Put(key []byte, value []byte) (err error) {
+	if c.enableReadCache {
+		defer func() {
+			if err == nil {
+				if len(value) < maxMemoryEntrySize {
+					// memory db only accept the small data
+					c.l1Cache.Set(key, value)
+				}
+				if !c.l2ReadOnly {
+					timeoutCtx, cancelFunc := context.WithTimeout(context.Background(), 5*time.Second)
+					defer cancelFunc()
+
+					// send the data to redis
+					res := c.l2Cache.SetEX(timeoutCtx, string(key), value, c.l2ExpiredTime)
+					if res.Err() == nil {
+						log.Printf("[Put WARN]Redis Error: %v", res.Err())
+					}
+				}
+			}
+		}()
+	}
+
+	return c.remoteDB.Put(key, value)
+}
+
+// Delete removes the key from the key-value data store.
+func (c *StateDBCacheDatabase) Delete(key []byte) (err error) {
+	if c.enableReadCache {
+		defer func() {
+			if err == nil {
+				c.l1Cache.Del(key)
+				if !c.l2ReadOnly {
+					c.l2Cache.Del(context.Background(), string(key))
+				}
+			}
+		}()
+	}
+
+	return c.remoteDB.Delete(key)
+}
+
+// NewBatch creates a write-only database that buffers changes to its host db
+// until a final write is called.
+func (c *StateDBCacheDatabase) NewBatch() ethdb.Batch {
+	return newStateDBCacheBatch(c, c.remoteDB.NewBatch())
+}
+
+// Stat returns a particular internal stat of the database.
+func (c *StateDBCacheDatabase) Stat(property string) (string, error) {
+	switch property {
+	case "tikv.save.cache":
+		_ = c.l1Cache.SaveToFileConcurrent(c.config.CachePersistencePath, runtime.NumCPU())
+	}
+	return c.remoteDB.Stat(property)
+}
+
+func (c *StateDBCacheDatabase) Compact(start []byte, limit []byte) error {
+	return c.remoteDB.Compact(start, limit)
+}
+
+func (c *StateDBCacheDatabase) NewIterator(start, end []byte) ethdb.Iterator {
+	return c.remoteDB.NewIterator(start, end)
+}
+
+// Close disconnect the redis and save memory cache to file
+func (c *StateDBCacheDatabase) Close() error {
+	if atomic.CompareAndSwapUint64(&c.isClose, 0, 1) {
+		err := c.l1Cache.SaveToFileConcurrent(c.config.CachePersistencePath, runtime.NumCPU())
+		if err != nil {
+			log.Printf("save file to '%s' error: %v", c.config.CachePersistencePath, err)
+		}
+
+		if !c.l2ReadOnly {
+			close(c.l2ExpiredRefresh)
+			time.Sleep(time.Second)
+
+			for range c.l2ExpiredRefresh {
+				// nop, clear chan
+			}
+		}
+
+		_ = c.l2Cache.Close()
+
+		return c.remoteDB.Close()
+	}
+
+	return nil
+}
+
+// cacheWrite write batch to cache
+func (c *StateDBCacheDatabase) cacheWrite(b ethdb.Batch) error {
+	if !c.l2ReadOnly {
+		pipeline := c.l2Cache.Pipeline()
+
+		err := b.Replay(&redisPipelineWrapper{Pipeliner: pipeline, expiredTime: c.l2ExpiredTime})
+		if err != nil {
+			return err
+		}
+
+		_, err = pipeline.Exec(context.Background())
+		if err != nil {
+			log.Printf("[BatchWrite WARN]Redis Error: %v", err)
+		}
+	}
+
+	return b.Replay(c.l1Cache)
+}
+
+func (c *StateDBCacheDatabase) refreshL2ExpiredTime() {
+	unix := time.Now().Add(c.l2ExpiredTime).Unix()
+	unix = unix - (unix % cacheMinGapInSecond) + cacheMinGapInSecond
+	c.l2NextExpiredTime = time.Unix(unix, 0)
+}
+
+// startL2ExpiredRefresh batch refresh redis cache
+func (c *StateDBCacheDatabase) startL2ExpiredRefresh() {
+	ticker := time.NewTicker(time.Second)
+	defer ticker.Stop()
+
+	pipeline := c.l2Cache.Pipeline()
+	lastWrite := time.Now()
+
+	for {
+		select {
+		case key, ok := <-c.l2ExpiredRefresh:
+			if !ok {
+				return
+			}
+
+			pipeline.ExpireAt(context.Background(), key, c.l2NextExpiredTime)
+			if pipeline.Len() > maxPipelineEntriesCount {
+				_, _ = pipeline.Exec(context.Background())
+				lastWrite = time.Now()
+			}
+		case now := <-ticker.C:
+			c.refreshL2ExpiredTime()
+
+			if pipeline.Len() > 0 && now.Sub(lastWrite) > time.Second {
+				_, _ = pipeline.Exec(context.Background())
+				lastWrite = time.Now()
+			}
+		}
+	}
+}
+
+// print stats to stdout
+func (c *StateDBCacheDatabase) cacheStatusPrint() {
+	var state = &fastcache.Stats{}
+	state.Reset()
+	c.l1Cache.UpdateStats(state)
+
+	stats := c.l2Cache.PoolStats()
+	log.Printf("redis: TotalConns: %d, IdleConns: %d, StaleConns: %d", stats.TotalConns, stats.IdleConns, stats.StaleConns)
+	total := float64(c.l1HitCount + c.l2HitCount + c.missCount + c.notFoundOrErrorCount)
+
+	log.Printf(
+		"total: l1Hit: %d(%.2f%%), l2Hit: %d(%.2f%%), miss: %d(%.2f%%), notFoundOrErrorCount: %d, fastcache: GetCalls: %d, SetCalls: %d, Misses: %d, EntriesCount: %d, BytesSize: %d",
+		c.l1HitCount, float64(c.l1HitCount)/total*100, c.l2HitCount, float64(c.l2HitCount)/total*100, c.missCount, float64(c.missCount)/total*100, c.notFoundOrErrorCount,
+		state.GetCalls, state.SetCalls, state.Misses, state.EntriesCount, state.BytesSize,
+	)
+}
+
+func (c *StateDBCacheDatabase) ReplayCache(batch ethdb.Batch) error {
+	return c.cacheWrite(batch)
+}

internal/utils/bytes.go

@@ -1,6 +1,9 @@
 package utils
 
-import "encoding/hex"
+import (
+	"encoding/hex"
+	"math/big"
+)
 
 // use to look up number of 1 bit in 4 bits
 var halfByteLookup = [16]int{0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4}
@@ -44,3 +47,36 @@ func CountOneBits(arr []byte) int64 {
 	}
 	return int64(count)
 }
+
+func BytesMiddle(a, b []byte) []byte {
+	if a == nil && b == nil {
+		return []byte{128}
+	}
+
+	if len(a) > len(b) {
+		tmp := make([]byte, len(a))
+		if b == nil {
+			for i, _ := range tmp {
+				tmp[i] = 255
+			}
+		}
+		copy(tmp, b)
+		b = tmp
+	} else if len(a) < len(b) {
+		tmp := make([]byte, len(b))
+		if a == nil {
+			for i, _ := range tmp {
+				tmp[i] = 0
+			}
+		}
+		copy(tmp, a)
+		a = tmp
+	}
+
+	aI := big.NewInt(0).SetBytes(a)
+	bI := big.NewInt(0).SetBytes(b)
+
+	aI.Add(aI, bI)
+	aI.Div(aI, big.NewInt(2))
+	return aI.Bytes()
+}

node/node.go

@@ -1,14 +1,22 @@
 package node
 
 import (
+	"bytes"
 	"context"
 	"fmt"
+	"io/ioutil"
+	"log"
 	"math/big"
 	"os"
+	"runtime/pprof"
 	"strings"
 	"sync"
 	"time"
 
+	"github.com/harmony-one/harmony/internal/shardchain/tikv_manage"
+	"github.com/harmony-one/harmony/internal/tikv"
+	"github.com/harmony-one/harmony/internal/tikv/redis_helper"
+
 	"github.com/ethereum/go-ethereum/rlp"
 	harmonyconfig "github.com/harmony-one/harmony/internal/configs/harmony"
 	"github.com/harmony-one/harmony/internal/utils/crosslinks"
@@ -152,6 +160,11 @@ func (node *Node) Blockchain() core.BlockChain {
 
 // Beaconchain returns the beaconchain from node.
 func (node *Node) Beaconchain() core.BlockChain {
+	// tikv mode not have the BeaconChain storage
+	if node.HarmonyConfig != nil && node.HarmonyConfig.General.RunElasticMode && node.HarmonyConfig.General.ShardID != shard.BeaconChainShardID {
+		return nil
+	}
+
 	return node.chain(shard.BeaconChainShardID, core.Options{})
 }
 
@@ -222,6 +235,13 @@ func (node *Node) addPendingTransactions(newTxs types.Transactions) []error {
 			Msg("[addPendingTransactions] Node out of sync, ignoring transactions")
 		return nil
 	}
+
+	// in tikv mode, reader only accept the pending transaction from writer node, ignore the p2p message
+	if node.HarmonyConfig.General.RunElasticMode && node.HarmonyConfig.TiKV.Role == tikv.RoleReader {
+		log.Printf("skip reader addPendingTransactions: %#v", newTxs)
+		return nil
+	}
+
 	poolTxs := types.PoolTransactions{}
 	errs := []error{}
 	acceptCx := node.Blockchain().Config().AcceptsCrossTx(node.Blockchain().CurrentHeader().Epoch())
@@ -434,6 +454,11 @@ func (node *Node) validateNodeMessage(ctx context.Context, payload []byte) (
 		case proto_node.Sync:
 			nodeNodeMessageCounterVec.With(prometheus.Labels{"type": "block_sync"}).Inc()
 
+			// in tikv mode, not need BeaconChain message
+			if node.HarmonyConfig.General.RunElasticMode && node.HarmonyConfig.General.ShardID != shard.BeaconChainShardID {
+				return nil, 0, errIgnoreBeaconMsg
+			}
+
 			// checks whether the beacon block is larger than current block number
 			blocksPayload := payload[p2pNodeMsgPrefixSize+1:]
 			var blocks []*types.Block
@@ -997,7 +1022,7 @@ func New(
 	engine := chain.NewEngine()
 
 	collection := shardchain.NewCollection(
-		chainDBFactory, &core.GenesisInitializer{NetworkType: node.NodeConfig.GetNetworkType()}, engine, &chainConfig,
+		harmonyconfig, chainDBFactory, &core.GenesisInitializer{NetworkType: node.NodeConfig.GetNetworkType()}, engine, &chainConfig,
 	)
 
 	for shardID, archival := range isArchival {
@@ -1022,16 +1047,19 @@ func New(
 		}
 
 		if b1, b2 := beaconChain == nil, blockchain == nil; b1 || b2 {
-			var err error
+			// in tikv mode, not need BeaconChain
-			if b2 {
+			if !(node.HarmonyConfig != nil && node.HarmonyConfig.General.RunElasticMode) || node.HarmonyConfig.General.ShardID == shard.BeaconChainShardID {
-				shardID := node.NodeConfig.ShardID
+				var err error
-				// HACK get the real error reason
+				if b2 {
-				_, err = node.shardChains.ShardChain(shardID)
+					shardID := node.NodeConfig.ShardID
-			} else {
+					// HACK get the real error reason
-				_, err = node.shardChains.ShardChain(shard.BeaconChainShardID)
+					_, err = node.shardChains.ShardChain(shardID)
+				} else {
+					_, err = node.shardChains.ShardChain(shard.BeaconChainShardID)
+				}
+				fmt.Fprintf(os.Stderr, "Cannot initialize node: %v\n", err)
+				os.Exit(-1)
 			}
-			fmt.Fprintf(os.Stderr, "Cannot initialize node: %v\n", err)
-			os.Exit(-1)
 		}
 
 		node.BlockChannel = make(chan *types.Block)
@@ -1053,6 +1081,16 @@ func New(
 		txPoolConfig.Blacklist = blacklist
 		txPoolConfig.AllowedTxs = allowedTxs
 		txPoolConfig.Journal = fmt.Sprintf("%v/%v", node.NodeConfig.DBDir, txPoolConfig.Journal)
+		txPoolConfig.AddEvent = func(tx types.PoolTransaction, local bool) {
+			// in tikv mode, writer will publish tx pool update to all reader
+			if node.Blockchain().IsTikvWriterMaster() {
+				err := redis_helper.PublishTxPoolUpdate(uint32(harmonyconfig.General.ShardID), tx, local)
+				if err != nil {
+					utils.Logger().Info().Err(err).Msg("redis publish txpool update error")
+				}
+			}
+		}
+
 		node.TxPool = core.NewTxPool(txPoolConfig, node.Blockchain().Config(), blockchain, node.TransactionErrorSink)
 		node.CxPool = core.NewCxPool(core.CxPoolSize)
 		node.Worker = worker.New(node.Blockchain().Config(), blockchain, engine)
@@ -1108,8 +1146,11 @@ func New(
 		}()
 	}
 
-	// update reward values now that node is ready
+	// in tikv mode, not need BeaconChain
-	node.updateInitialRewardValues()
+	if !(node.HarmonyConfig != nil && node.HarmonyConfig.General.RunElasticMode) || node.HarmonyConfig.General.ShardID == shard.BeaconChainShardID {
+		// update reward values now that node is ready
+		node.updateInitialRewardValues()
+	}
 
 	// init metrics
 	initMetrics()
@@ -1273,6 +1314,18 @@ func (node *Node) ShutDown() {
 	node.Blockchain().Stop()
 	node.Beaconchain().Stop()
 
+	if node.HarmonyConfig.General.RunElasticMode {
+		_, _ = node.Blockchain().RedisPreempt().Unlock()
+		_, _ = node.Beaconchain().RedisPreempt().Unlock()
+
+		_ = redis_helper.Close()
+		time.Sleep(time.Second)
+
+		if storage := tikv_manage.GetDefaultTiKVFactory(); storage != nil {
+			storage.CloseAllDB()
+		}
+	}
+
 	const msg = "Successfully shut down!\n"
 	utils.Logger().Print(msg)
 	fmt.Print(msg)
@@ -1365,3 +1418,61 @@ func (node *Node) GetAddresses(epoch *big.Int) map[string]common.Address {
 func (node *Node) IsRunningBeaconChain() bool {
 	return node.NodeConfig.ShardID == shard.BeaconChainShardID
 }
+
+// syncFromTiKVWriter used for tikv mode, subscribe data from tikv writer
+func (node *Node) syncFromTiKVWriter() {
+	bc := node.Blockchain()
+
+	// subscribe block update
+	go redis_helper.SubscribeShardUpdate(bc.ShardID(), func(blkNum uint64, logs []*types.Log) {
+		// todo temp code to find and fix problems
+		// redis: 2022/07/09 03:51:23 pubsub.go:605: redis: &{%!s(*redis.PubSub=&{0xc002198d20 0xe0b160 0xe0b380 {0 0} 0xc0454265a0 map[shard_update_0:{}] map[] false 0xc0047c7800 0xc004946240 {1 {0 0}} 0xc0074269c0 <nil>}) %!s(chan *redis.Message=0xc004946180) %!s(chan interface {}=<nil>) %!s(chan struct {}=0xc0047c7b60) %!s(int=100) %!s(time.Duration=60000000000) %!s(time.Duration=3000000000)} channel is full for 1m0s (message is dropped)
+		doneChan := make(chan struct{}, 1)
+		go func() {
+			select {
+			case <-doneChan:
+				return
+			case <-time.After(5 * time.Minute):
+				buf := bytes.NewBuffer(nil)
+				err := pprof.Lookup("goroutine").WriteTo(buf, 1)
+				if err != nil {
+					panic(err)
+				}
+				err = ioutil.WriteFile(fmt.Sprintf("/tmp/%s", time.Now().Format("hmy_0102150405.error.log")), buf.Bytes(), 0644)
+				if err != nil {
+					panic(err)
+				}
+				// todo temp code to fix problems, restart self
+				os.Exit(1)
+			}
+		}()
+		defer close(doneChan)
+
+		err := bc.SyncFromTiKVWriter(blkNum, logs)
+		if err != nil {
+			utils.Logger().Warn().
+				Err(err).
+				Msg("cannot sync block from tikv writer")
+			return
+		}
+	})
+
+	// subscribe txpool update
+	if node.HarmonyConfig.TiKV.Role == tikv.RoleReader {
+		go redis_helper.SubscribeTxPoolUpdate(bc.ShardID(), func(tx types.PoolTransaction, local bool) {
+			var err error
+			if local {
+				err = node.TxPool.AddLocal(tx)
+			} else {
+				err = node.TxPool.AddRemote(tx)
+			}
+			if err != nil {
+				utils.Logger().Debug().
+					Err(err).
+					Interface("tx", tx).
+					Msg("cannot sync txpool from tikv writer")
+				return
+			}
+		})
+	}
+}

node/node_explorer.go

@@ -5,6 +5,8 @@ import (
 	"encoding/json"
 	"sync"
 
+	"github.com/harmony-one/harmony/internal/tikv"
+
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/rlp"
 	msg_pb "github.com/harmony-one/harmony/api/proto/message"
@@ -145,6 +147,11 @@ func (node *Node) TraceLoopForExplorer() {
 
 // AddNewBlockForExplorer add new block for explorer.
 func (node *Node) AddNewBlockForExplorer(block *types.Block) {
+	if node.HarmonyConfig.General.RunElasticMode && node.HarmonyConfig.TiKV.Role == tikv.RoleReader {
+		node.Consensus.FBFTLog.DeleteBlockByNumber(block.NumberU64())
+		return
+	}
+
 	utils.Logger().Info().Uint64("blockHeight", block.NumberU64()).Msg("[Explorer] Adding new block for explorer node")
 
 	if _, err := node.Blockchain().InsertChain([]*types.Block{block}, false); err == nil {
@@ -153,23 +160,38 @@ func (node *Node) AddNewBlockForExplorer(block *types.Block) {
 		}
 		// Clean up the blocks to avoid OOM.
 		node.Consensus.FBFTLog.DeleteBlockByNumber(block.NumberU64())
-		// Do dump all blocks from state syncing for explorer one time
+
-		// TODO: some blocks can be dumped before state syncing finished.
+		// if in tikv mode, only master writer node need dump all explorer block
-		// And they would be dumped again here. Please fix it.
+		if !node.HarmonyConfig.General.RunElasticMode || node.Blockchain().IsTikvWriterMaster() {
-		once.Do(func() {
+			// Do dump all blocks from state syncing for explorer one time
-			utils.Logger().Info().Int64("starting height", int64(block.NumberU64())-1).
+			// TODO: some blocks can be dumped before state syncing finished.
-				Msg("[Explorer] Populating explorer data from state synced blocks")
+			// And they would be dumped again here. Please fix it.
-			go func() {
+			once.Do(func() {
-				exp, err := node.getExplorerService()
+				utils.Logger().Info().Int64("starting height", int64(block.NumberU64())-1).
-				if err != nil {
+					Msg("[Explorer] Populating explorer data from state synced blocks")
-					// shall be unreachable
+				go func() {
-					utils.Logger().Fatal().Err(err).Msg("critical error in explorer node")
+					exp, err := node.getExplorerService()
-				}
+					if err != nil {
-				for blockHeight := int64(block.NumberU64()) - 1; blockHeight >= 0; blockHeight-- {
+						// shall be unreachable
-					exp.DumpCatchupBlock(node.Blockchain().GetBlockByNumber(uint64(blockHeight)))
+						utils.Logger().Fatal().Err(err).Msg("critical error in explorer node")
-				}
+					}
-			}()
+
-		})
+					if block.NumberU64() == 0 {
+						return
+					}
+
+					// get checkpoint bitmap and flip all bit
+					bitmap := exp.GetCheckpointBitmap()
+					bitmap.Flip(0, block.NumberU64())
+
+					// find all not processed block and dump it
+					iterator := bitmap.ReverseIterator()
+					for iterator.HasNext() {
+						exp.DumpCatchupBlock(node.Blockchain().GetBlockByNumber(iterator.Next()))
+					}
+				}()
+			})
+		}
 	} else {
 		utils.Logger().Error().Err(err).Msg("[Explorer] Error when adding new block for explorer node")
 	}
@@ -177,17 +199,20 @@ func (node *Node) AddNewBlockForExplorer(block *types.Block) {
 
 // ExplorerMessageHandler passes received message in node_handler to explorer service.
 func (node *Node) commitBlockForExplorer(block *types.Block) {
-	if block.ShardID() != node.NodeConfig.ShardID {
+	// if in tikv mode, only master writer node need dump explorer block
-		return
+	if !node.HarmonyConfig.General.RunElasticMode || (node.HarmonyConfig.TiKV.Role == tikv.RoleWriter && node.Blockchain().IsTikvWriterMaster()) {
-	}
+		if block.ShardID() != node.NodeConfig.ShardID {
-	// Dump new block into level db.
+			return
-	utils.Logger().Info().Uint64("blockNum", block.NumberU64()).Msg("[Explorer] Committing block into explorer DB")
+		}
-	exp, err := node.getExplorerService()
+		// Dump new block into level db.
-	if err != nil {
+		utils.Logger().Info().Uint64("blockNum", block.NumberU64()).Msg("[Explorer] Committing block into explorer DB")
-		// shall be unreachable
+		exp, err := node.getExplorerService()
-		utils.Logger().Fatal().Err(err).Msg("critical error in explorer node")
+		if err != nil {
+			// shall be unreachable
+			utils.Logger().Fatal().Err(err).Msg("critical error in explorer node")
+		}
+		exp.DumpNewBlock(block)
 	}
-	exp.DumpNewBlock(block)
 
 	curNum := block.NumberU64()
 	if curNum-100 > 0 {

node/node_syncing.go

@@ -8,6 +8,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/harmony-one/harmony/internal/tikv"
+
 	prom "github.com/harmony-one/harmony/api/service/prometheus"
 	"github.com/prometheus/client_golang/prometheus"
 
@@ -216,6 +218,10 @@ func (node *Node) doBeaconSyncing() {
 		return
 	}
 
+	if node.HarmonyConfig.General.RunElasticMode {
+		return
+	}
+
 	if !node.NodeConfig.Downloader {
 		// If Downloader is not working, we need also deal with blocks from beaconBlockChannel
 		go func(node *Node) {
@@ -319,7 +325,22 @@ func (node *Node) StartGRPCSyncClient() {
 			Msg("SupportBeaconSyncing")
 		go node.doBeaconSyncing()
 	}
-	node.supportSyncing()
+}
+
+// NodeSyncing makes sure to start all the processes needed to sync the node based on different configuration factors.
+func (node *Node) NodeSyncing() {
+	if node.HarmonyConfig.General.RunElasticMode {
+		node.syncFromTiKVWriter() // this is for both reader and backup writers
+
+		if node.HarmonyConfig.TiKV.Role == tikv.RoleReader {
+			node.Consensus.UpdateConsensusInformation()
+		}
+		if node.HarmonyConfig.TiKV.Role == tikv.RoleWriter {
+			node.supportSyncing() // the writer needs to be in sync with it's other peers
+		}
+	} else if !node.HarmonyConfig.General.IsOffline && node.HarmonyConfig.DNSSync.Client {
+		node.supportSyncing() // for non-writer-reader mode a.k.a tikv nodes
+	}
 }
 
 // supportSyncing keeps sleeping until it's doing consensus or it's a leader.

node/service_setup.go

@@ -27,7 +27,7 @@ func (node *Node) RegisterValidatorServices() {
 func (node *Node) RegisterExplorerServices() {
 	// Register explorer service.
 	node.serviceManager.Register(
-		service.SupportExplorer, explorer.New(&node.SelfPeer, node.Blockchain(), node),
+		service.SupportExplorer, explorer.New(node.HarmonyConfig, &node.SelfPeer, node.Blockchain(), node),
 	)
 }
 

node/worker/worker_test.go

@@ -5,6 +5,8 @@ import (
 	"math/rand"
 	"testing"
 
+	"github.com/harmony-one/harmony/core/state"
+
 	"github.com/ethereum/go-ethereum/core/rawdb"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -43,7 +45,7 @@ func TestNewWorker(t *testing.T) {
 
 	genesis := gspec.MustCommit(database)
 	_ = genesis
-	chain, err := core.NewBlockChain(database, nil, gspec.Config, engine, vm.Config{}, nil)
+	chain, err := core.NewBlockChain(database, state.NewDatabase(database), nil, gspec.Config, engine, vm.Config{}, nil)
 
 	if err != nil {
 		t.Error(err)
@@ -70,7 +72,7 @@ func TestCommitTransactions(t *testing.T) {
 	)
 
 	gspec.MustCommit(database)
-	chain, _ := core.NewBlockChain(database, nil, gspec.Config, engine, vm.Config{}, nil)
+	chain, _ := core.NewBlockChain(database, state.NewDatabase(database), nil, gspec.Config, engine, vm.Config{}, nil)
 
 	// Create a new worker
 	worker := New(params.TestChainConfig, chain, engine)

rosetta/services/call_service.go

@@ -85,7 +85,7 @@ func (c *CallAPIService) Call(
 func NewCallAPIService(hmy *hmy.Harmony, limiterEnable bool, rateLimit int) server.CallAPIServicer {
 	return &CallAPIService{
 		hmy:                 hmy,
-		publicContractAPI:   rpc2.NewPublicContractAPI(hmy, rpc2.V2),
+		publicContractAPI:   rpc2.NewPublicContractAPI(hmy, rpc2.V2, limiterEnable, rateLimit),
 		publicStakingAPI:    rpc2.NewPublicStakingAPI(hmy, rpc2.V2),
 		publicBlockChainAPI: rpc2.NewPublicBlockchainAPI(hmy, rpc2.V2, limiterEnable, rateLimit),
 	}

rpc/contract.go

@@ -35,11 +35,16 @@ type PublicContractService struct {
 }
 
 // NewPublicContractAPI creates a new API for the RPC interface
-func NewPublicContractAPI(hmy *hmy.Harmony, version Version) rpc.API {
+func NewPublicContractAPI(hmy *hmy.Harmony, version Version, limiterEnable bool, limit int) rpc.API {
+	var limiter *rate.Limiter
+	if limiterEnable {
+		limiter = rate.NewLimiter(rate.Limit(limit), limit)
+	}
+
 	return rpc.API{
 		Namespace: version.Namespace(),
 		Version:   APIVersion,
-		Service:   &PublicContractService{hmy, version, rate.NewLimiter(200, 1500)},
+		Service:   &PublicContractService{hmy, version, limiter},
 		Public:    true,
 	}
 }

rpc/pool.go

@@ -37,11 +37,15 @@ type PublicPoolService struct {
 }
 
 // NewPublicPoolAPI creates a new API for the RPC interface
-func NewPublicPoolAPI(hmy *hmy.Harmony, version Version) rpc.API {
+func NewPublicPoolAPI(hmy *hmy.Harmony, version Version, limiterEnable bool, limit int) rpc.API {
+	var limiter *rate.Limiter
+	if limiterEnable {
+		limiter = rate.NewLimiter(rate.Limit(limit), limit)
+	}
 	return rpc.API{
 		Namespace: version.Namespace(),
 		Version:   APIVersion,
-		Service:   &PublicPoolService{hmy, version, rate.NewLimiter(2, 5)},
+		Service:   &PublicPoolService{hmy, version, limiter},
 		Public:    true,
 	}
 }

rpc/rpc.go

@@ -158,12 +158,12 @@ func getAPIs(hmy *hmy.Harmony, config nodeconfig.RPCServerConfig) []rpc.API {
 		NewPublicHarmonyAPI(hmy, V2),
 		NewPublicBlockchainAPI(hmy, V1, config.RateLimiterEnabled, config.RequestsPerSecond),
 		NewPublicBlockchainAPI(hmy, V2, config.RateLimiterEnabled, config.RequestsPerSecond),
-		NewPublicContractAPI(hmy, V1),
+		NewPublicContractAPI(hmy, V1, config.RateLimiterEnabled, config.RequestsPerSecond),
-		NewPublicContractAPI(hmy, V2),
+		NewPublicContractAPI(hmy, V2, config.RateLimiterEnabled, config.RequestsPerSecond),
 		NewPublicTransactionAPI(hmy, V1),
 		NewPublicTransactionAPI(hmy, V2),
-		NewPublicPoolAPI(hmy, V1),
+		NewPublicPoolAPI(hmy, V1, config.RateLimiterEnabled, config.RequestsPerSecond),
-		NewPublicPoolAPI(hmy, V2),
+		NewPublicPoolAPI(hmy, V2, config.RateLimiterEnabled, config.RequestsPerSecond),
 	}
 
 	// Legacy methods (subject to removal)
@@ -185,9 +185,9 @@ func getAPIs(hmy *hmy.Harmony, config nodeconfig.RPCServerConfig) []rpc.API {
 		publicAPIs = append(publicAPIs,
 			NewPublicHarmonyAPI(hmy, Eth),
 			NewPublicBlockchainAPI(hmy, Eth, config.RateLimiterEnabled, config.RequestsPerSecond),
-			NewPublicContractAPI(hmy, Eth),
+			NewPublicContractAPI(hmy, Eth, config.RateLimiterEnabled, config.RequestsPerSecond),
 			NewPublicTransactionAPI(hmy, Eth),
-			NewPublicPoolAPI(hmy, Eth),
+			NewPublicPoolAPI(hmy, Eth, config.RateLimiterEnabled, config.RequestsPerSecond),
 			eth.NewPublicEthService(hmy, "eth"),
 		)
 	}

test/chain/main.go

@@ -15,6 +15,7 @@ import (
 	blockfactory "github.com/harmony-one/harmony/block/factory"
 	"github.com/harmony-one/harmony/core"
 	core_state "github.com/harmony-one/harmony/core/state"
+	harmonyState "github.com/harmony-one/harmony/core/state"
 	"github.com/harmony-one/harmony/core/types"
 	"github.com/harmony-one/harmony/core/vm"
 	"github.com/harmony-one/harmony/crypto/hash"
@@ -205,7 +206,7 @@ func playFaucetContract(chain core.BlockChain) {
 
 func main() {
 	genesis := gspec.MustCommit(database)
-	chain, _ := core.NewBlockChain(database, nil, gspec.Config, chain.Engine(), vm.Config{}, nil)
+	chain, _ := core.NewBlockChain(database, harmonyState.NewDatabase(database), nil, gspec.Config, chain.Engine(), vm.Config{}, nil)
 	txpool := core.NewTxPool(core.DefaultTxPoolConfig, chainConfig, chain, types.NewTransactionErrorSink())
 
 	backend := &testWorkerBackend{

test/chain/reward/main.go

@@ -109,7 +109,7 @@ func main() {
 	genesis := gspec.MustCommit(database)
 	_ = genesis
 	engine := chain.NewEngine()
-	bc, _ := core.NewBlockChain(database, nil, gspec.Config, engine, vm.Config{}, nil)
+	bc, _ := core.NewBlockChain(database, state.NewDatabase(database), nil, gspec.Config, engine, vm.Config{}, nil)
 	statedb, _ := state.New(common2.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()))
 	msg := createValidator()
 	statedb.AddBalance(msg.ValidatorAddress, new(big.Int).Mul(big.NewInt(5e18), big.NewInt(2000)))