lutty
3 years ago
committed by
Lutty
parent
9df446a23d
commit
720f9a6a02
@ -0,0 +1,167 @@ |
||||
package state |
||||
|
||||
import ( |
||||
"bytes" |
||||
"log" |
||||
"sync" |
||||
"sync/atomic" |
||||
"time" |
||||
|
||||
"github.com/ethereum/go-ethereum/common" |
||||
"github.com/ethereum/go-ethereum/rlp" |
||||
"github.com/ethereum/go-ethereum/trie" |
||||
"github.com/harmony-one/harmony/internal/utils" |
||||
) |
||||
|
||||
type prefetchJob struct { |
||||
accountAddr []byte |
||||
account *Account |
||||
start, end []byte |
||||
} |
||||
|
||||
// Prefetch If redis is empty, the hit rate will be too low and the synchronization block speed will be slow
|
||||
// this function will parallel load the latest block statedb to redis
|
||||
// this function used by debug or first time to init tikv cluster
|
||||
func (s *DB) Prefetch(parallel int) { |
||||
wg := sync.WaitGroup{} |
||||
|
||||
jobChan := make(chan *prefetchJob, 10000) |
||||
waitWorker := int64(parallel) |
||||
|
||||
// start parallel workers
|
||||
for i := 0; i < parallel; i++ { |
||||
go func() { |
||||
defer wg.Done() |
||||
for job := range jobChan { |
||||
atomic.AddInt64(&waitWorker, -1) |
||||
s.prefetchWorker(job, jobChan) |
||||
atomic.AddInt64(&waitWorker, 1) |
||||
} |
||||
}() |
||||
|
||||
wg.Add(1) |
||||
} |
||||
|
||||
// add first jobs
|
||||
for i := 0; i < 255; i++ { |
||||
start := []byte{byte(i)} |
||||
end := []byte{byte(i + 1)} |
||||
if i == parallel-1 { |
||||
end = nil |
||||
} else if i == 0 { |
||||
start = nil |
||||
} |
||||
|
||||
jobChan <- &prefetchJob{ |
||||
start: start, |
||||
end: end, |
||||
} |
||||
} |
||||
|
||||
// wait all worker done
|
||||
start := time.Now() |
||||
log.Println("Prefetch start") |
||||
var sleepCount int |
||||
for sleepCount < 3 { |
||||
time.Sleep(time.Second) |
||||
waitWorkerCount := atomic.LoadInt64(&waitWorker) |
||||
if waitWorkerCount >= int64(parallel) { |
||||
sleepCount++ |
||||
} else { |
||||
sleepCount = 0 |
||||
} |
||||
} |
||||
|
||||
close(jobChan) |
||||
wg.Wait() |
||||
end := time.Now() |
||||
log.Println("Prefetch end, use", end.Sub(start)) |
||||
} |
||||
|
||||
// prefetchWorker used to process one job
|
||||
func (s *DB) prefetchWorker(job *prefetchJob, jobs chan *prefetchJob) { |
||||
if job.account == nil { |
||||
// scan one account
|
||||
nodeIterator := s.trie.NodeIterator(job.start) |
||||
it := trie.NewIterator(nodeIterator) |
||||
|
||||
for it.Next() { |
||||
if job.end != nil && bytes.Compare(it.Key, job.end) >= 0 { |
||||
return |
||||
} |
||||
|
||||
// build account data from main trie tree
|
||||
var data Account |
||||
if err := rlp.DecodeBytes(it.Value, &data); err != nil { |
||||
panic(err) |
||||
} |
||||
addrBytes := s.trie.GetKey(it.Key) |
||||
addr := common.BytesToAddress(addrBytes) |
||||
obj := newObject(s, addr, data) |
||||
if data.CodeHash != nil { |
||||
obj.Code(s.db) |
||||
} |
||||
|
||||
// build account trie tree
|
||||
storageIt := trie.NewIterator(obj.getTrie(s.db).NodeIterator(nil)) |
||||
storageJob := &prefetchJob{ |
||||
accountAddr: addrBytes, |
||||
account: &data, |
||||
} |
||||
|
||||
// fetch data
|
||||
s.prefetchAccountStorage(jobs, storageJob, storageIt) |
||||
} |
||||
} else { |
||||
// scan main trie tree
|
||||
obj := newObject(s, common.BytesToAddress(job.accountAddr), *job.account) |
||||
storageIt := trie.NewIterator(obj.getTrie(s.db).NodeIterator(job.start)) |
||||
|
||||
// fetch data
|
||||
s.prefetchAccountStorage(jobs, job, storageIt) |
||||
} |
||||
} |
||||
|
||||
// prefetchAccountStorage used for fetch account storage
|
||||
func (s *DB) prefetchAccountStorage(jobs chan *prefetchJob, job *prefetchJob, it *trie.Iterator) { |
||||
start := time.Now() |
||||
count := 0 |
||||
|
||||
for it.Next() { |
||||
if job.end != nil && bytes.Compare(it.Key, job.end) >= 0 { |
||||
return |
||||
} |
||||
|
||||
count++ |
||||
|
||||
// if fetch one account job used more than 15s, then split the job
|
||||
if count%10000 == 0 && time.Now().Sub(start) > 15*time.Second { |
||||
middle := utils.BytesMiddle(it.Key, job.end) |
||||
startJob := &prefetchJob{ |
||||
accountAddr: job.accountAddr, |
||||
account: job.account, |
||||
start: it.Key, |
||||
end: middle, |
||||
} |
||||
endJob := &prefetchJob{ |
||||
accountAddr: job.accountAddr, |
||||
account: job.account, |
||||
start: middle, |
||||
end: job.end, |
||||
} |
||||
|
||||
select { |
||||
case jobs <- endJob: |
||||
select { |
||||
case jobs <- startJob: |
||||
return |
||||
default: |
||||
job.end = startJob.end |
||||
start = time.Now() |
||||
} |
||||
default: |
||||
log.Println("job full, continue") |
||||
} |
||||
} |
||||
} |
||||
} |
||||
@ -0,0 +1,80 @@ |
||||
package state |
||||
|
||||
import ( |
||||
"bytes" |
||||
"sync" |
||||
"sync/atomic" |
||||
|
||||
"github.com/ethereum/go-ethereum/common" |
||||
"github.com/ethereum/go-ethereum/rlp" |
||||
"github.com/ethereum/go-ethereum/trie" |
||||
"github.com/harmony-one/harmony/internal/shardchain/tikv_manage" |
||||
) |
||||
|
||||
var secureKeyPrefix = []byte("secure-key-") |
||||
|
||||
// DiffAndCleanCache clean block tire data from redis, Used to reduce redis storage and increase hit rate
|
||||
func (s *DB) DiffAndCleanCache(shardId uint32, to *DB) (int, error) { |
||||
// create difference iterator
|
||||
it, _ := trie.NewDifferenceIterator(to.trie.NodeIterator(nil), s.trie.NodeIterator(nil)) |
||||
db, err := tikv_manage.GetDefaultTiKVFactory().NewCacheStateDB(shardId) |
||||
if err != nil { |
||||
return 0, err |
||||
} |
||||
|
||||
batch := db.NewBatch() |
||||
wg := &sync.WaitGroup{} |
||||
count := uint64(0) |
||||
for it.Next(true) { |
||||
if !it.Leaf() { |
||||
// delete it if trie leaf node
|
||||
atomic.AddUint64(&count, 1) |
||||
_ = batch.Delete(it.Hash().Bytes()) |
||||
} else { |
||||
|
||||
// build account data
|
||||
addrBytes := s.trie.GetKey(it.LeafKey()) |
||||
addr := common.BytesToAddress(addrBytes) |
||||
|
||||
var fromAccount, toAccount Account |
||||
if err := rlp.DecodeBytes(it.LeafBlob(), &fromAccount); err != nil { |
||||
continue |
||||
} |
||||
|
||||
if toByte, err := to.trie.TryGet(addrBytes); err != nil { |
||||
continue |
||||
} else if err := rlp.DecodeBytes(toByte, &toAccount); err != nil { |
||||
continue |
||||
} |
||||
|
||||
// if account not changed, skip
|
||||
if bytes.Compare(fromAccount.Root.Bytes(), toAccount.Root.Bytes()) == 0 { |
||||
continue |
||||
} |
||||
|
||||
// create account difference iterator
|
||||
fromAccountTrie := newObject(s, addr, fromAccount).getTrie(s.db) |
||||
toAccountTrie := newObject(to, addr, toAccount).getTrie(to.db) |
||||
accountIt, _ := trie.NewDifferenceIterator(toAccountTrie.NodeIterator(nil), fromAccountTrie.NodeIterator(nil)) |
||||
|
||||
// parallel to delete data
|
||||
wg.Add(1) |
||||
go func() { |
||||
defer wg.Done() |
||||
for accountIt.Next(true) { |
||||
atomic.AddUint64(&count, 1) |
||||
|
||||
if !accountIt.Leaf() { |
||||
_ = batch.Delete(accountIt.Hash().Bytes()) |
||||
} else { |
||||
_ = batch.Delete(append(append([]byte{}, secureKeyPrefix...), accountIt.LeafKey()...)) |
||||
} |
||||
} |
||||
}() |
||||
} |
||||
} |
||||
|
||||
wg.Wait() |
||||
|
||||
return int(count), db.ReplayCache(batch) |
||||
} |
||||
@ -0,0 +1,126 @@ |
||||
package shardchain |
||||
|
||||
import ( |
||||
"fmt" |
||||
"io" |
||||
"sync" |
||||
|
||||
"github.com/ethereum/go-ethereum/core/rawdb" |
||||
"github.com/harmony-one/harmony/internal/tikv" |
||||
tikvCommon "github.com/harmony-one/harmony/internal/tikv/common" |
||||
"github.com/harmony-one/harmony/internal/tikv/prefix" |
||||
"github.com/harmony-one/harmony/internal/tikv/remote" |
||||
"github.com/harmony-one/harmony/internal/tikv/statedb_cache" |
||||
|
||||
"github.com/ethereum/go-ethereum/ethdb" |
||||
) |
||||
|
||||
const ( |
||||
LDBTiKVPrefix = "harmony_tikv" |
||||
) |
||||
|
||||
type TiKvCacheConfig struct { |
||||
StateDBCacheSizeInMB uint32 |
||||
StateDBCachePersistencePath string |
||||
StateDBRedisServerAddr string |
||||
StateDBRedisLRUTimeInDay uint32 |
||||
} |
||||
|
||||
// TiKvFactory is a memory-backed blockchain database factory.
|
||||
type TiKvFactory struct { |
||||
cacheDBMap sync.Map |
||||
|
||||
PDAddr []string |
||||
Role string |
||||
CacheConfig statedb_cache.StateDBCacheConfig |
||||
} |
||||
|
||||
// getStateDB create statedb storage use tikv
|
||||
func (f *TiKvFactory) getRemoteDB(shardID uint32) (*prefix.PrefixDatabase, error) { |
||||
key := fmt.Sprintf("remote_%d_%s", shardID, f.Role) |
||||
if db, ok := f.cacheDBMap.Load(key); ok { |
||||
return db.(*prefix.PrefixDatabase), nil |
||||
} else { |
||||
prefixStr := []byte(fmt.Sprintf("%s_%d/", LDBTiKVPrefix, shardID)) |
||||
remoteDatabase, err := remote.NewRemoteDatabase(f.PDAddr, f.Role == tikv.RoleReader) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
tmpDB := prefix.NewPrefixDatabase(prefixStr, remoteDatabase) |
||||
if loadedDB, loaded := f.cacheDBMap.LoadOrStore(key, tmpDB); loaded { |
||||
_ = tmpDB.Close() |
||||
return loadedDB.(*prefix.PrefixDatabase), nil |
||||
} |
||||
|
||||
return tmpDB, nil |
||||
} |
||||
} |
||||
|
||||
// getStateDB create statedb storage with local memory cahce
|
||||
func (f *TiKvFactory) getStateDB(shardID uint32) (*statedb_cache.StateDBCacheDatabase, error) { |
||||
key := fmt.Sprintf("state_db_%d_%s", shardID, f.Role) |
||||
if db, ok := f.cacheDBMap.Load(key); ok { |
||||
return db.(*statedb_cache.StateDBCacheDatabase), nil |
||||
} else { |
||||
db, err := f.getRemoteDB(shardID) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
tmpDB, err := statedb_cache.NewStateDBCacheDatabase(db, f.CacheConfig, f.Role == tikv.RoleReader) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
if loadedDB, loaded := f.cacheDBMap.LoadOrStore(key, tmpDB); loaded { |
||||
_ = tmpDB.Close() |
||||
return loadedDB.(*statedb_cache.StateDBCacheDatabase), nil |
||||
} |
||||
|
||||
return tmpDB, nil |
||||
} |
||||
} |
||||
|
||||
// NewChainDB returns a new memDB for the blockchain for given shard.
|
||||
func (f *TiKvFactory) NewChainDB(shardID uint32) (ethdb.Database, error) { |
||||
var database ethdb.KeyValueStore |
||||
|
||||
db, err := f.getRemoteDB(shardID) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
database = tikvCommon.ToEthKeyValueStore(db) |
||||
|
||||
return rawdb.NewDatabase(database), nil |
||||
} |
||||
|
||||
// NewStateDB create shard tikv database
|
||||
func (f *TiKvFactory) NewStateDB(shardID uint32) (ethdb.Database, error) { |
||||
var database ethdb.KeyValueStore |
||||
|
||||
cacheDatabase, err := f.getStateDB(shardID) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
database = tikvCommon.ToEthKeyValueStore(cacheDatabase) |
||||
|
||||
return rawdb.NewDatabase(database), nil |
||||
} |
||||
|
||||
// NewCacheStateDB create shard statedb storage with memory cache
|
||||
func (f *TiKvFactory) NewCacheStateDB(shardID uint32) (*statedb_cache.StateDBCacheDatabase, error) { |
||||
return f.getStateDB(shardID) |
||||
} |
||||
|
||||
// CloseAllDB close all tikv database
|
||||
func (f *TiKvFactory) CloseAllDB() { |
||||
f.cacheDBMap.Range(func(_, value interface{}) bool { |
||||
if closer, ok := value.(io.Closer); ok { |
||||
_ = closer.Close() |
||||
} |
||||
return true |
||||
}) |
||||
} |
||||
@ -0,0 +1,28 @@ |
||||
package tikv_manage |
||||
|
||||
import ( |
||||
"sync" |
||||
|
||||
"github.com/ethereum/go-ethereum/ethdb" |
||||
"github.com/harmony-one/harmony/internal/tikv/statedb_cache" |
||||
) |
||||
|
||||
type TiKvFactory interface { |
||||
NewChainDB(shardID uint32) (ethdb.Database, error) |
||||
NewStateDB(shardID uint32) (ethdb.Database, error) |
||||
NewCacheStateDB(shardID uint32) (*statedb_cache.StateDBCacheDatabase, error) |
||||
CloseAllDB() |
||||
} |
||||
|
||||
var tikvInit = sync.Once{} |
||||
var tikvFactory TiKvFactory |
||||
|
||||
func SetDefaultTiKVFactory(factory TiKvFactory) { |
||||
tikvInit.Do(func() { |
||||
tikvFactory = factory |
||||
}) |
||||
} |
||||
|
||||
func GetDefaultTiKVFactory() (factory TiKvFactory) { |
||||
return tikvFactory |
||||
} |
||||
@ -0,0 +1,73 @@ |
||||
// from https://github.com/xtaci/smux/blob/master/alloc.go
|
||||
package byte_alloc |
||||
|
||||
import ( |
||||
"sync" |
||||
) |
||||
|
||||
// magic number
|
||||
var debruijinPos = [...]byte{0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31} |
||||
|
||||
// Allocator for incoming frames, optimized to prevent overwriting after zeroing
|
||||
type Allocator struct { |
||||
buffers []sync.Pool |
||||
} |
||||
|
||||
// NewAllocator initiates a []byte allocator for frames less than 1048576 bytes,
|
||||
// the waste(memory fragmentation) of space allocation is guaranteed to be
|
||||
// no more than 50%.
|
||||
func NewAllocator() *Allocator { |
||||
alloc := new(Allocator) |
||||
alloc.buffers = make([]sync.Pool, 21) // 1B -> 1M
|
||||
for k := range alloc.buffers { |
||||
size := 1 << uint32(k) |
||||
alloc.buffers[k].New = func() interface{} { |
||||
return make([]byte, size) |
||||
} |
||||
} |
||||
return alloc |
||||
} |
||||
|
||||
// Get a []byte from pool with most appropriate cap
|
||||
func (alloc *Allocator) Get(size int) []byte { |
||||
if size <= 0 { |
||||
return nil |
||||
} |
||||
|
||||
if size > 1048576 { |
||||
return make([]byte, size) |
||||
} |
||||
|
||||
bits := msb(size) |
||||
if size == 1<<bits { |
||||
return alloc.buffers[bits].Get().([]byte)[:size] |
||||
} else { |
||||
return alloc.buffers[bits+1].Get().([]byte)[:size] |
||||
} |
||||
} |
||||
|
||||
// Put returns a []byte to pool for future use,
|
||||
// which the cap must be exactly 2^n
|
||||
func (alloc *Allocator) Put(buf []byte) { |
||||
bufCap := cap(buf) |
||||
bits := msb(bufCap) |
||||
if bufCap == 0 || bufCap > 65536 || bufCap != 1<<bits { |
||||
return |
||||
} |
||||
|
||||
alloc.buffers[bits].Put(buf) |
||||
return |
||||
} |
||||
|
||||
// msb return the pos of most significiant bit
|
||||
// Equivalent to: uint(math.Floor(math.Log2(float64(n))))
|
||||
// http://supertech.csail.mit.edu/papers/debruijn.pdf
|
||||
func msb(size int) byte { |
||||
v := uint32(size) |
||||
v |= v >> 1 |
||||
v |= v >> 2 |
||||
v |= v >> 4 |
||||
v |= v >> 8 |
||||
v |= v >> 16 |
||||
return debruijinPos[(v*0x07C4ACDD)>>27] |
||||
} |
||||
@ -0,0 +1,15 @@ |
||||
package byte_alloc |
||||
|
||||
var defaultAllocator *Allocator |
||||
|
||||
func init() { |
||||
defaultAllocator = NewAllocator() |
||||
} |
||||
|
||||
func Get(size int) []byte { |
||||
return defaultAllocator.Get(size) |
||||
} |
||||
|
||||
func Put(buf []byte) { |
||||
defaultAllocator.Put(buf) |
||||
} |
||||
@ -0,0 +1,10 @@ |
||||
package common |
||||
|
||||
import ( |
||||
"errors" |
||||
|
||||
"github.com/syndtr/goleveldb/leveldb" |
||||
) |
||||
|
||||
var ErrEmptyKey = errors.New("empty key is not supported") |
||||
var ErrNotFound = leveldb.ErrNotFound |
||||
@ -0,0 +1,20 @@ |
||||
package common |
||||
|
||||
import ( |
||||
"unsafe" |
||||
) |
||||
|
||||
// String converts byte slice to string.
|
||||
func String(b []byte) string { |
||||
return *(*string)(unsafe.Pointer(&b)) |
||||
} |
||||
|
||||
// StringBytes converts string to byte slice.
|
||||
func StringBytes(s string) []byte { |
||||
return *(*[]byte)(unsafe.Pointer( |
||||
&struct { |
||||
string |
||||
Cap int |
||||
}{s, len(s)}, |
||||
)) |
||||
} |
||||
@ -0,0 +1,41 @@ |
||||
package common |
||||
|
||||
import ( |
||||
"io" |
||||
|
||||
"github.com/ethereum/go-ethereum/ethdb" |
||||
"github.com/syndtr/goleveldb/leveldb/util" |
||||
) |
||||
|
||||
type TiKVStore interface { |
||||
ethdb.KeyValueReader |
||||
ethdb.KeyValueWriter |
||||
ethdb.Batcher |
||||
ethdb.Stater |
||||
ethdb.Compacter |
||||
io.Closer |
||||
|
||||
NewIterator(start, end []byte) ethdb.Iterator |
||||
} |
||||
|
||||
// TiKVStoreWrapper simple wrapper to covert to ethdb.KeyValueStore
|
||||
type TiKVStoreWrapper struct { |
||||
TiKVStore |
||||
} |
||||
|
||||
func (t *TiKVStoreWrapper) NewIterator() ethdb.Iterator { |
||||
return t.TiKVStore.NewIterator(nil, nil) |
||||
} |
||||
|
||||
func (t *TiKVStoreWrapper) NewIteratorWithStart(start []byte) ethdb.Iterator { |
||||
return t.TiKVStore.NewIterator(start, nil) |
||||
} |
||||
|
||||
func (t *TiKVStoreWrapper) NewIteratorWithPrefix(prefix []byte) ethdb.Iterator { |
||||
bytesPrefix := util.BytesPrefix(prefix) |
||||
return t.TiKVStore.NewIterator(bytesPrefix.Start, bytesPrefix.Limit) |
||||
} |
||||
|
||||
func ToEthKeyValueStore(store TiKVStore) ethdb.KeyValueStore { |
||||
return &TiKVStoreWrapper{TiKVStore: store} |
||||
} |
||||
@ -0,0 +1,6 @@ |
||||
package tikv |
||||
|
||||
const ( |
||||
RoleReader = "Reader" |
||||
RoleWriter = "Writer" |
||||
) |
||||
@ -0,0 +1,42 @@ |
||||
package prefix |
||||
|
||||
import "github.com/ethereum/go-ethereum/ethdb" |
||||
|
||||
type PrefixBatch struct { |
||||
prefix []byte |
||||
batch ethdb.Batch |
||||
} |
||||
|
||||
func newPrefixBatch(prefix []byte, batch ethdb.Batch) *PrefixBatch { |
||||
return &PrefixBatch{prefix: prefix, batch: batch} |
||||
} |
||||
|
||||
// Put inserts the given value into the key-value data store.
|
||||
func (p *PrefixBatch) Put(key []byte, value []byte) error { |
||||
return p.batch.Put(append(append([]byte{}, p.prefix...), key...), value) |
||||
} |
||||
|
||||
// Delete removes the key from the key-value data store.
|
||||
func (p *PrefixBatch) Delete(key []byte) error { |
||||
return p.batch.Delete(append(append([]byte{}, p.prefix...), key...)) |
||||
} |
||||
|
||||
// ValueSize retrieves the amount of data queued up for writing.
|
||||
func (p *PrefixBatch) ValueSize() int { |
||||
return p.batch.ValueSize() |
||||
} |
||||
|
||||
// Write flushes any accumulated data to disk.
|
||||
func (p *PrefixBatch) Write() error { |
||||
return p.batch.Write() |
||||
} |
||||
|
||||
// Reset resets the batch for reuse.
|
||||
func (p *PrefixBatch) Reset() { |
||||
p.batch.Reset() |
||||
} |
||||
|
||||
// Replay replays the batch contents.
|
||||
func (p *PrefixBatch) Replay(w ethdb.KeyValueWriter) error { |
||||
return p.batch.Replay(newPrefixBatchReplay(p.prefix, w)) |
||||
} |
||||
@ -0,0 +1,35 @@ |
||||
package prefix |
||||
|
||||
import ( |
||||
"bytes" |
||||
|
||||
"github.com/ethereum/go-ethereum/ethdb" |
||||
) |
||||
|
||||
type PrefixBatchReplay struct { |
||||
prefix []byte |
||||
prefixLen int |
||||
w ethdb.KeyValueWriter |
||||
} |
||||
|
||||
func newPrefixBatchReplay(prefix []byte, w ethdb.KeyValueWriter) *PrefixBatchReplay { |
||||
return &PrefixBatchReplay{prefix: prefix, prefixLen: len(prefix), w: w} |
||||
} |
||||
|
||||
// Put inserts the given value into the key-value data store.
|
||||
func (p *PrefixBatchReplay) Put(key []byte, value []byte) error { |
||||
if bytes.HasPrefix(key, p.prefix) { |
||||
return p.w.Put(key[p.prefixLen:], value) |
||||
} else { |
||||
return p.w.Put(key, value) |
||||
} |
||||
} |
||||
|
||||
// Delete removes the key from the key-value data store.
|
||||
func (p *PrefixBatchReplay) Delete(key []byte) error { |
||||
if bytes.HasPrefix(key, p.prefix) { |
||||
return p.w.Delete(key[p.prefixLen:]) |
||||
} else { |
||||
return p.w.Delete(key) |
||||
} |
||||
} |
||||
@ -0,0 +1,109 @@ |
||||
package prefix |
||||
|
||||
import ( |
||||
"github.com/ethereum/go-ethereum/ethdb" |
||||
"github.com/harmony-one/harmony/internal/tikv/byte_alloc" |
||||
"github.com/harmony-one/harmony/internal/tikv/common" |
||||
) |
||||
|
||||
// PrefixDatabase is a wrapper to split the storage with prefix
|
||||
type PrefixDatabase struct { |
||||
prefix []byte |
||||
db common.TiKVStore |
||||
keysPool *byte_alloc.Allocator |
||||
} |
||||
|
||||
func NewPrefixDatabase(prefix []byte, db common.TiKVStore) *PrefixDatabase { |
||||
return &PrefixDatabase{ |
||||
prefix: prefix, |
||||
db: db, |
||||
keysPool: byte_alloc.NewAllocator(), |
||||
} |
||||
} |
||||
|
||||
// makeKey use to create a key with prefix, keysPool can reduce gc pressure
|
||||
func (p *PrefixDatabase) makeKey(keys []byte) []byte { |
||||
prefixLen := len(p.prefix) |
||||
byt := p.keysPool.Get(len(keys) + prefixLen) |
||||
copy(byt, p.prefix) |
||||
copy(byt[prefixLen:], keys) |
||||
|
||||
return byt |
||||
} |
||||
|
||||
// Has retrieves if a key is present in the key-value data store.
|
||||
func (p *PrefixDatabase) Has(key []byte) (bool, error) { |
||||
return p.db.Has(p.makeKey(key)) |
||||
} |
||||
|
||||
// Get retrieves the given key if it's present in the key-value data store.
|
||||
func (p *PrefixDatabase) Get(key []byte) ([]byte, error) { |
||||
return p.db.Get(p.makeKey(key)) |
||||
} |
||||
|
||||
// Put inserts the given value into the key-value data store.
|
||||
func (p *PrefixDatabase) Put(key []byte, value []byte) error { |
||||
return p.db.Put(p.makeKey(key), value) |
||||
} |
||||
|
||||
// Delete removes the key from the key-value data store.
|
||||
func (p *PrefixDatabase) Delete(key []byte) error { |
||||
return p.db.Delete(p.makeKey(key)) |
||||
} |
||||
|
||||
// NewBatch creates a write-only database that buffers changes to its host db
|
||||
// until a final write is called.
|
||||
func (p *PrefixDatabase) NewBatch() ethdb.Batch { |
||||
return newPrefixBatch(p.prefix, p.db.NewBatch()) |
||||
} |
||||
|
||||
// buildLimitUsePrefix build the limit byte from start byte, useful generating from prefix works
|
||||
func (p *PrefixDatabase) buildLimitUsePrefix() []byte { |
||||
var limit []byte |
||||
for i := len(p.prefix) - 1; i >= 0; i-- { |
||||
c := p.prefix[i] |
||||
if c < 0xff { |
||||
limit = make([]byte, i+1) |
||||
copy(limit, p.prefix) |
||||
limit[i] = c + 1 |
||||
break |
||||
} |
||||
} |
||||
|
||||
return limit |
||||
} |
||||
|
||||
// NewIterator creates a binary-alphabetical iterator over the start to end keyspace
|
||||
// contained within the key-value database.
|
||||
func (p *PrefixDatabase) NewIterator(start, end []byte) ethdb.Iterator { |
||||
start = append(p.prefix, start...) |
||||
|
||||
if len(end) == 0 { |
||||
end = p.buildLimitUsePrefix() |
||||
} else { |
||||
end = append(p.prefix, end...) |
||||
} |
||||
|
||||
return newPrefixIterator(p.prefix, p.db.NewIterator(start, end)) |
||||
} |
||||
|
||||
// Stat returns a particular internal stat of the database.
|
||||
func (p *PrefixDatabase) Stat(property string) (string, error) { |
||||
return p.db.Stat(property) |
||||
} |
||||
|
||||
// Compact flattens the underlying data store for the given key range. In essence,
|
||||
// deleted and overwritten versions are discarded, and the data is rearranged to
|
||||
// reduce the cost of operations needed to access them.
|
||||
//
|
||||
// A nil start is treated as a key before all keys in the data store; a nil limit
|
||||
// is treated as a key after all keys in the data store. If both is nil then it
|
||||
// will compact entire data store.
|
||||
func (p *PrefixDatabase) Compact(start []byte, limit []byte) error { |
||||
return p.db.Compact(start, limit) |
||||
} |
||||
|
||||
// Close the storage
|
||||
func (p *PrefixDatabase) Close() error { |
||||
return p.db.Close() |
||||
} |
||||
@ -0,0 +1,53 @@ |
||||
package prefix |
||||
|
||||
import ( |
||||
"github.com/ethereum/go-ethereum/ethdb" |
||||
) |
||||
|
||||
type PrefixIterator struct { |
||||
prefix []byte |
||||
prefixLen int |
||||
|
||||
it ethdb.Iterator |
||||
} |
||||
|
||||
func newPrefixIterator(prefix []byte, it ethdb.Iterator) *PrefixIterator { |
||||
return &PrefixIterator{prefix: prefix, prefixLen: len(prefix), it: it} |
||||
} |
||||
|
||||
// Next moves the iterator to the next key/value pair. It returns whether the
|
||||
// iterator is exhausted.
|
||||
func (i *PrefixIterator) Next() bool { |
||||
return i.it.Next() |
||||
} |
||||
|
||||
// Error returns any accumulated error. Exhausting all the key/value pairs
|
||||
// is not considered to be an error.
|
||||
func (i *PrefixIterator) Error() error { |
||||
return i.it.Error() |
||||
} |
||||
|
||||
// Key returns the key of the current key/value pair, or nil if done. The caller
|
||||
// should not modify the contents of the returned slice, and its contents may
|
||||
// change on the next call to Next.
|
||||
func (i *PrefixIterator) Key() []byte { |
||||
key := i.it.Key() |
||||
if len(key) < len(i.prefix) { |
||||
return nil |
||||
} |
||||
|
||||
return key[i.prefixLen:] |
||||
} |
||||
|
||||
// Value returns the value of the current key/value pair, or nil if done. The
|
||||
// caller should not modify the contents of the returned slice, and its contents
|
||||
// may change on the next call to Next.
|
||||
func (i *PrefixIterator) Value() []byte { |
||||
return i.it.Value() |
||||
} |
||||
|
||||
// Release releases associated resources. Release should always succeed and can
|
||||
// be called multiple times without causing error.
|
||||
func (i *PrefixIterator) Release() { |
||||
i.it.Release() |
||||
} |
||||
@ -0,0 +1,34 @@ |
||||
package redis_helper |
||||
|
||||
import ( |
||||
"context" |
||||
|
||||
"github.com/go-redis/redis/v8" |
||||
) |
||||
|
||||
var redisInstance *redis.ClusterClient |
||||
|
||||
// Init used to init redis instance in tikv mode
|
||||
func Init(serverAddr []string) error { |
||||
option := &redis.ClusterOptions{ |
||||
Addrs: serverAddr, |
||||
PoolSize: 2, |
||||
} |
||||
|
||||
rdb := redis.NewClusterClient(option) |
||||
err := rdb.Ping(context.Background()).Err() |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
redisInstance = rdb |
||||
return nil |
||||
} |
||||
|
||||
// Close disconnect redis instance in tikv mode
|
||||
func Close() error { |
||||
if redisInstance != nil { |
||||
return redisInstance.Close() |
||||
} |
||||
return nil |
||||
} |
||||
@ -0,0 +1,72 @@ |
||||
package redis_helper |
||||
|
||||
import ( |
||||
"context" |
||||
"crypto/rand" |
||||
"encoding/base64" |
||||
|
||||
"github.com/go-redis/redis/v8" |
||||
) |
||||
|
||||
type RedisPreempt struct { |
||||
key string |
||||
password string |
||||
lockScript, unlockScript *redis.Script |
||||
lastLockStatus bool |
||||
} |
||||
|
||||
// CreatePreempt used to create a redis preempt instance
|
||||
func CreatePreempt(key string) *RedisPreempt { |
||||
p := &RedisPreempt{ |
||||
key: key, |
||||
} |
||||
p.init() |
||||
|
||||
return p |
||||
} |
||||
|
||||
// init redis preempt instance and some script
|
||||
func (p *RedisPreempt) init() { |
||||
byt := make([]byte, 18) |
||||
_, _ = rand.Read(byt) |
||||
p.password = base64.StdEncoding.EncodeToString(byt) |
||||
p.lockScript = redis.NewScript(` |
||||
if redis.call('get',KEYS[1]) == ARGV[1] then |
||||
return redis.call('expire', KEYS[1], ARGV[2]) |
||||
else |
||||
return redis.call('set', KEYS[1], ARGV[1], 'ex', ARGV[2], 'nx') |
||||
end |
||||
`) |
||||
p.unlockScript = redis.NewScript(` |
||||
if redis.call('get',KEYS[1]) == ARGV[1] then |
||||
return redis.call('del', KEYS[1]) |
||||
else |
||||
return 0 |
||||
end |
||||
`) |
||||
} |
||||
|
||||
// TryLock attempt to lock the master for ttlSecond
|
||||
func (p *RedisPreempt) TryLock(ttlSecond int) (ok bool, err error) { |
||||
ok, err = p.lockScript.Run(context.Background(), redisInstance, []string{p.key}, p.password, ttlSecond).Bool() |
||||
p.lastLockStatus = ok |
||||
return |
||||
} |
||||
|
||||
// Unlock try to release the master permission
|
||||
func (p *RedisPreempt) Unlock() (bool, error) { |
||||
if p == nil { |
||||
return false, nil |
||||
} |
||||
|
||||
return p.unlockScript.Run(context.Background(), redisInstance, []string{p.key}, p.password).Bool() |
||||
} |
||||
|
||||
// LastLockStatus get the last preempt status
|
||||
func (p *RedisPreempt) LastLockStatus() bool { |
||||
if p == nil { |
||||
return false |
||||
} |
||||
|
||||
return p.lastLockStatus |
||||
} |
||||
@ -0,0 +1,130 @@ |
||||
package redis_helper |
||||
|
||||
import ( |
||||
"context" |
||||
"fmt" |
||||
"io" |
||||
|
||||
"github.com/ethereum/go-ethereum/rlp" |
||||
"github.com/harmony-one/harmony/core/types" |
||||
"github.com/harmony-one/harmony/internal/utils" |
||||
stakingTypes "github.com/harmony-one/harmony/staking/types" |
||||
"github.com/pkg/errors" |
||||
) |
||||
|
||||
// BlockUpdate block update event
|
||||
type BlockUpdate struct { |
||||
BlkNum uint64 |
||||
Logs []*types.Log |
||||
} |
||||
|
||||
// SubscribeShardUpdate subscribe block update event
|
||||
func SubscribeShardUpdate(shardID uint32, cb func(blkNum uint64, logs []*types.Log)) { |
||||
pubsub := redisInstance.Subscribe(context.Background(), fmt.Sprintf("shard_update_%d", shardID)) |
||||
for message := range pubsub.Channel() { |
||||
block := &BlockUpdate{} |
||||
err := rlp.DecodeBytes([]byte(message.Payload), block) |
||||
if err != nil { |
||||
utils.Logger().Info().Err(err).Msg("redis subscribe shard update error") |
||||
continue |
||||
} |
||||
cb(block.BlkNum, block.Logs) |
||||
} |
||||
} |
||||
|
||||
// PublishShardUpdate publish block update event
|
||||
func PublishShardUpdate(shardID uint32, blkNum uint64, logs []*types.Log) error { |
||||
msg, err := rlp.EncodeToBytes(&BlockUpdate{ |
||||
BlkNum: blkNum, |
||||
Logs: logs, |
||||
}) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
return redisInstance.Publish(context.Background(), fmt.Sprintf("shard_update_%d", shardID), msg).Err() |
||||
} |
||||
|
||||
//TxPoolUpdate tx pool update event
|
||||
type TxPoolUpdate struct { |
||||
typ string |
||||
Local bool |
||||
Tx types.PoolTransaction |
||||
} |
||||
|
||||
// DecodeRLP decode struct from binary stream
|
||||
func (t *TxPoolUpdate) DecodeRLP(stream *rlp.Stream) error { |
||||
if err := stream.Decode(&t.typ); err != nil { |
||||
return err |
||||
} |
||||
if err := stream.Decode(&t.Local); err != nil { |
||||
return err |
||||
} |
||||
|
||||
switch t.typ { |
||||
case "types.EthTransaction": |
||||
var tmp = &types.EthTransaction{} |
||||
if err := stream.Decode(tmp); err != nil { |
||||
return err |
||||
} |
||||
t.Tx = tmp |
||||
case "types.Transaction": |
||||
var tmp = &types.Transaction{} |
||||
if err := stream.Decode(tmp); err != nil { |
||||
return err |
||||
} |
||||
t.Tx = tmp |
||||
case "stakingTypes.StakingTransaction": |
||||
var tmp = &stakingTypes.StakingTransaction{} |
||||
if err := stream.Decode(tmp); err != nil { |
||||
return err |
||||
} |
||||
t.Tx = tmp |
||||
default: |
||||
return errors.New("unknown txpool type") |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
// EncodeRLP encode struct to binary stream
|
||||
func (t *TxPoolUpdate) EncodeRLP(w io.Writer) error { |
||||
switch t.Tx.(type) { |
||||
case *types.EthTransaction: |
||||
t.typ = "types.EthTransaction" |
||||
case *types.Transaction: |
||||
t.typ = "types.Transaction" |
||||
case *stakingTypes.StakingTransaction: |
||||
t.typ = "stakingTypes.StakingTransaction" |
||||
} |
||||
|
||||
if err := rlp.Encode(w, t.typ); err != nil { |
||||
return err |
||||
} |
||||
if err := rlp.Encode(w, t.Local); err != nil { |
||||
return err |
||||
} |
||||
return rlp.Encode(w, t.Tx) |
||||
} |
||||
|
||||
// SubscribeTxPoolUpdate subscribe tx pool update event
|
||||
func SubscribeTxPoolUpdate(shardID uint32, cb func(tx types.PoolTransaction, local bool)) { |
||||
pubsub := redisInstance.Subscribe(context.Background(), fmt.Sprintf("txpool_update_%d", shardID)) |
||||
for message := range pubsub.Channel() { |
||||
txu := &TxPoolUpdate{} |
||||
err := rlp.DecodeBytes([]byte(message.Payload), &txu) |
||||
if err != nil { |
||||
utils.Logger().Info().Err(err).Msg("redis subscribe shard update error") |
||||
continue |
||||
} |
||||
cb(txu.Tx, txu.Local) |
||||
} |
||||
} |
||||
|
||||
// PublishTxPoolUpdate publish tx pool update event
|
||||
func PublishTxPoolUpdate(shardID uint32, tx types.PoolTransaction, local bool) error { |
||||
txu := &TxPoolUpdate{Local: local, Tx: tx} |
||||
msg, err := rlp.EncodeToBytes(txu) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
return redisInstance.Publish(context.Background(), fmt.Sprintf("txpool_update_%d", shardID), msg).Err() |
||||
} |
||||
@ -0,0 +1,119 @@ |
||||
package remote |
||||
|
||||
import ( |
||||
"bytes" |
||||
"context" |
||||
"sync" |
||||
|
||||
"github.com/ethereum/go-ethereum/ethdb" |
||||
"github.com/harmony-one/harmony/internal/tikv/common" |
||||
) |
||||
|
||||
type RemoteBatch struct { |
||||
db *RemoteDatabase |
||||
lock sync.Mutex |
||||
|
||||
size int |
||||
batchWriteKey [][]byte |
||||
batchWriteValue [][]byte |
||||
batchDeleteKey [][]byte |
||||
} |
||||
|
||||
func newRemoteBatch(db *RemoteDatabase) *RemoteBatch { |
||||
return &RemoteBatch{db: db} |
||||
} |
||||
|
||||
// Put inserts the given value into the key-value data store.
|
||||
func (b *RemoteBatch) Put(key []byte, value []byte) error { |
||||
if len(key) == 0 { |
||||
return common.ErrEmptyKey |
||||
} |
||||
|
||||
if len(value) == 0 { |
||||
value = EmptyValueStub |
||||
} |
||||
|
||||
b.lock.Lock() |
||||
defer b.lock.Unlock() |
||||
|
||||
b.batchWriteKey = append(b.batchWriteKey, key) |
||||
b.batchWriteValue = append(b.batchWriteValue, value) |
||||
b.size += len(key) + len(value) |
||||
return nil |
||||
} |
||||
|
||||
// Delete removes the key from the key-value data store.
|
||||
func (b *RemoteBatch) Delete(key []byte) error { |
||||
b.lock.Lock() |
||||
defer b.lock.Unlock() |
||||
|
||||
b.batchDeleteKey = append(b.batchDeleteKey, key) |
||||
b.size += len(key) |
||||
return nil |
||||
} |
||||
|
||||
// ValueSize retrieves the amount of data queued up for writing.
|
||||
func (b *RemoteBatch) ValueSize() int { |
||||
return b.size |
||||
} |
||||
|
||||
// Write flushes any accumulated data to disk.
|
||||
func (b *RemoteBatch) Write() error { |
||||
b.lock.Lock() |
||||
defer b.lock.Unlock() |
||||
|
||||
if len(b.batchWriteKey) > 0 { |
||||
err := b.db.client.BatchPut(context.Background(), b.batchWriteKey, b.batchWriteValue) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
} |
||||
|
||||
if len(b.batchDeleteKey) > 0 { |
||||
err := b.db.client.BatchDelete(context.Background(), b.batchDeleteKey) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
} |
||||
|
||||
return nil |
||||
} |
||||
|
||||
// Reset resets the batch for reuse.
|
||||
func (b *RemoteBatch) Reset() { |
||||
b.lock.Lock() |
||||
defer b.lock.Unlock() |
||||
|
||||
b.batchWriteKey = b.batchWriteKey[:0] |
||||
b.batchWriteValue = b.batchWriteValue[:0] |
||||
b.batchDeleteKey = b.batchDeleteKey[:0] |
||||
b.size = 0 |
||||
} |
||||
|
||||
// Replay replays the batch contents.
|
||||
func (b *RemoteBatch) Replay(w ethdb.KeyValueWriter) error { |
||||
for i, key := range b.batchWriteKey { |
||||
if bytes.Compare(b.batchWriteValue[i], EmptyValueStub) == 0 { |
||||
err := w.Put(key, []byte{}) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
} else { |
||||
err := w.Put(key, b.batchWriteValue[i]) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
} |
||||
} |
||||
|
||||
if len(b.batchDeleteKey) > 0 { |
||||
for _, key := range b.batchDeleteKey { |
||||
err := w.Delete(key) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
} |
||||
} |
||||
|
||||
return nil |
||||
} |
||||
@ -0,0 +1,139 @@ |
||||
package remote |
||||
|
||||
import ( |
||||
"bytes" |
||||
"context" |
||||
"runtime/trace" |
||||
"sync/atomic" |
||||
|
||||
"github.com/ethereum/go-ethereum/ethdb" |
||||
"github.com/harmony-one/harmony/internal/tikv/common" |
||||
"github.com/tikv/client-go/v2/config" |
||||
"github.com/tikv/client-go/v2/rawkv" |
||||
) |
||||
|
||||
var EmptyValueStub = []byte("HarmonyTiKVEmptyValueStub") |
||||
|
||||
type RemoteDatabase struct { |
||||
client *rawkv.Client |
||||
readOnly bool |
||||
isClose uint64 |
||||
} |
||||
|
||||
func NewRemoteDatabase(pdAddr []string, readOnly bool) (*RemoteDatabase, error) { |
||||
client, err := rawkv.NewClient(context.Background(), pdAddr, config.DefaultConfig().Security) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
db := &RemoteDatabase{ |
||||
client: client, |
||||
readOnly: readOnly, |
||||
} |
||||
|
||||
return db, nil |
||||
} |
||||
|
||||
// ReadOnly set storage to readonly mode
|
||||
func (d *RemoteDatabase) ReadOnly() { |
||||
d.readOnly = true |
||||
} |
||||
|
||||
// Has retrieves if a key is present in the key-value data store.
|
||||
func (d *RemoteDatabase) Has(key []byte) (bool, error) { |
||||
data, err := d.Get(key) |
||||
if err != nil { |
||||
if err == common.ErrNotFound { |
||||
return false, nil |
||||
} |
||||
return false, err |
||||
} else { |
||||
return len(data) != 0, nil |
||||
} |
||||
} |
||||
|
||||
// Get retrieves the given key if it's present in the key-value data store.
|
||||
func (d *RemoteDatabase) Get(key []byte) ([]byte, error) { |
||||
if len(key) == 0 { |
||||
return nil, common.ErrEmptyKey |
||||
} |
||||
|
||||
region := trace.StartRegion(context.Background(), "tikv Get") |
||||
defer region.End() |
||||
|
||||
get, err := d.client.Get(context.Background(), key) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
if len(get) == 0 { |
||||
return nil, common.ErrNotFound |
||||
} |
||||
|
||||
if len(get) == len(EmptyValueStub) && bytes.Compare(get, EmptyValueStub) == 0 { |
||||
get = get[:0] |
||||
} |
||||
|
||||
return get, nil |
||||
} |
||||
|
||||
// Put inserts the given value into the key-value data store.
|
||||
func (d *RemoteDatabase) Put(key []byte, value []byte) error { |
||||
if len(key) == 0 { |
||||
return common.ErrEmptyKey |
||||
} |
||||
if d.readOnly { |
||||
return nil |
||||
} |
||||
|
||||
if len(value) == 0 { |
||||
value = EmptyValueStub |
||||
} |
||||
|
||||
return d.client.Put(context.Background(), key, value) |
||||
} |
||||
|
||||
// Delete removes the key from the key-value data store.
|
||||
func (d *RemoteDatabase) Delete(key []byte) error { |
||||
if len(key) == 0 { |
||||
return common.ErrEmptyKey |
||||
} |
||||
if d.readOnly { |
||||
return nil |
||||
} |
||||
|
||||
return d.client.Delete(context.Background(), key) |
||||
} |
||||
|
||||
// NewBatch creates a write-only database that buffers changes to its host db
|
||||
// until a final write is called.
|
||||
func (d *RemoteDatabase) NewBatch() ethdb.Batch { |
||||
if d.readOnly { |
||||
return newNopRemoteBatch(d) |
||||
} |
||||
|
||||
return newRemoteBatch(d) |
||||
} |
||||
|
||||
func (d *RemoteDatabase) NewIterator(start, end []byte) ethdb.Iterator { |
||||
return newRemoteIterator(d, start, end) |
||||
} |
||||
|
||||
// Stat returns a particular internal stat of the database.
|
||||
func (d *RemoteDatabase) Stat(property string) (string, error) { |
||||
return "", common.ErrNotFound |
||||
} |
||||
|
||||
// Compact tikv current not supprot manual compact
|
||||
func (d *RemoteDatabase) Compact(start []byte, limit []byte) error { |
||||
return nil |
||||
} |
||||
|
||||
// Close disconnect the tikv
|
||||
func (d *RemoteDatabase) Close() error { |
||||
if atomic.CompareAndSwapUint64(&d.isClose, 0, 1) { |
||||
return d.client.Close() |
||||
} |
||||
|
||||
return nil |
||||
} |
||||
@ -0,0 +1,110 @@ |
||||
package remote |
||||
|
||||
import ( |
||||
"context" |
||||
"sync" |
||||
) |
||||
|
||||
const ( |
||||
iteratorOnce = 300 |
||||
) |
||||
|
||||
type RemoteIterator struct { |
||||
db *RemoteDatabase |
||||
lock sync.Mutex |
||||
|
||||
limit []byte |
||||
start []byte |
||||
|
||||
err error |
||||
end bool |
||||
pos int |
||||
|
||||
keys, values [][]byte |
||||
currentKey, currentValue []byte |
||||
} |
||||
|
||||
func newRemoteIterator(db *RemoteDatabase, start, limit []byte) *RemoteIterator { |
||||
return &RemoteIterator{ |
||||
db: db, |
||||
start: start, |
||||
limit: limit, |
||||
} |
||||
} |
||||
|
||||
// Next moves the iterator to the next key/value pair. It returns whether the
|
||||
// iterator is exhausted.
|
||||
func (i *RemoteIterator) Next() bool { |
||||
if i.end { |
||||
return false |
||||
} |
||||
|
||||
if i.keys == nil { |
||||
if next := i.scanNext(); !next { |
||||
return false |
||||
} |
||||
} |
||||
|
||||
i.currentKey = i.keys[i.pos] |
||||
i.currentValue = i.values[i.pos] |
||||
i.pos++ |
||||
|
||||
if i.pos >= len(i.keys) { |
||||
i.scanNext() |
||||
} |
||||
|
||||
return true |
||||
} |
||||
|
||||
// scanNext real scan from tikv, and cache it
|
||||
func (i *RemoteIterator) scanNext() bool { |
||||
keys, values, err := i.db.client.Scan(context.Background(), i.start, i.limit, iteratorOnce) |
||||
if err != nil { |
||||
i.err = err |
||||
i.end = true |
||||
return false |
||||
} |
||||
|
||||
if len(keys) == 0 { |
||||
i.end = true |
||||
return false |
||||
} else { |
||||
i.start = append(keys[len(keys)-1], 0) |
||||
} |
||||
|
||||
i.pos = 0 |
||||
i.keys = keys |
||||
i.values = values |
||||
return true |
||||
} |
||||
|
||||
// Error returns any accumulated error. Exhausting all the key/value pairs
|
||||
// is not considered to be an error.
|
||||
func (i *RemoteIterator) Error() error { |
||||
return i.err |
||||
} |
||||
|
||||
// Key returns the key of the current key/value pair, or nil if done. The caller
|
||||
// should not modify the contents of the returned slice, and its contents may
|
||||
// change on the next call to Next.
|
||||
func (i *RemoteIterator) Key() []byte { |
||||
return i.currentKey |
||||
} |
||||
|
||||
// Value returns the value of the current key/value pair, or nil if done. The
|
||||
// caller should not modify the contents of the returned slice, and its contents
|
||||
// may change on the next call to Next.
|
||||
func (i *RemoteIterator) Value() []byte { |
||||
return i.currentValue |
||||
} |
||||
|
||||
// Release releases associated resources. Release should always succeed and can
|
||||
// be called multiple times without causing error.
|
||||
func (i *RemoteIterator) Release() { |
||||
i.db = nil |
||||
i.end = true |
||||
i.keys = nil |
||||
i.values = nil |
||||
i.currentKey = nil |
||||
i.currentValue = nil |
||||
} |
||||
@ -0,0 +1,36 @@ |
||||
package remote |
||||
|
||||
import ( |
||||
"github.com/ethereum/go-ethereum/ethdb" |
||||
) |
||||
|
||||
// NopRemoteBatch on readonly mode, write operator will be reject
|
||||
type NopRemoteBatch struct { |
||||
} |
||||
|
||||
func newNopRemoteBatch(db *RemoteDatabase) *NopRemoteBatch { |
||||
return &NopRemoteBatch{} |
||||
} |
||||
|
||||
func (b *NopRemoteBatch) Put(key []byte, value []byte) error { |
||||
return nil |
||||
} |
||||
|
||||
func (b *NopRemoteBatch) Delete(key []byte) error { |
||||
return nil |
||||
} |
||||
|
||||
func (b *NopRemoteBatch) ValueSize() int { |
||||
return 0 |
||||
} |
||||
|
||||
func (b *NopRemoteBatch) Write() error { |
||||
return nil |
||||
} |
||||
|
||||
func (b *NopRemoteBatch) Reset() { |
||||
} |
||||
|
||||
func (b *NopRemoteBatch) Replay(w ethdb.KeyValueWriter) error { |
||||
return nil |
||||
} |
||||
@ -0,0 +1,45 @@ |
||||
package statedb_cache |
||||
|
||||
import ( |
||||
"github.com/ethereum/go-ethereum/ethdb" |
||||
) |
||||
|
||||
type StateDBCacheBatch struct { |
||||
db *StateDBCacheDatabase |
||||
remoteBatch ethdb.Batch |
||||
} |
||||
|
||||
func newStateDBCacheBatch(db *StateDBCacheDatabase, batch ethdb.Batch) *StateDBCacheBatch { |
||||
return &StateDBCacheBatch{db: db, remoteBatch: batch} |
||||
} |
||||
|
||||
func (b *StateDBCacheBatch) Put(key []byte, value []byte) error { |
||||
return b.remoteBatch.Put(key, value) |
||||
} |
||||
|
||||
func (b *StateDBCacheBatch) Delete(key []byte) error { |
||||
return b.remoteBatch.Delete(key) |
||||
} |
||||
|
||||
func (b *StateDBCacheBatch) ValueSize() int { |
||||
// In ethdb, the size of each commit is too small, this controls the submission frequency
|
||||
return b.remoteBatch.ValueSize() / 40 |
||||
} |
||||
|
||||
func (b *StateDBCacheBatch) Write() (err error) { |
||||
defer func() { |
||||
if err == nil { |
||||
_ = b.db.cacheWrite(b) |
||||
} |
||||
}() |
||||
|
||||
return b.remoteBatch.Write() |
||||
} |
||||
|
||||
func (b *StateDBCacheBatch) Reset() { |
||||
b.remoteBatch.Reset() |
||||
} |
||||
|
||||
func (b *StateDBCacheBatch) Replay(w ethdb.KeyValueWriter) error { |
||||
return b.remoteBatch.Replay(w) |
||||
} |
||||
@ -0,0 +1,373 @@ |
||||
package statedb_cache |
||||
|
||||
import ( |
||||
"context" |
||||
"log" |
||||
"runtime" |
||||
"sync/atomic" |
||||
"time" |
||||
|
||||
"github.com/VictoriaMetrics/fastcache" |
||||
"github.com/ethereum/go-ethereum/ethdb" |
||||
redis "github.com/go-redis/redis/v8" |
||||
"github.com/harmony-one/harmony/internal/tikv/common" |
||||
) |
||||
|
||||
const ( |
||||
maxMemoryEntrySize = 64 * 1024 |
||||
maxPipelineEntriesCount = 10000 |
||||
cacheMinGapInSecond = 600 |
||||
) |
||||
|
||||
type cacheWrapper struct { |
||||
*fastcache.Cache |
||||
} |
||||
|
||||
// Put inserts the given value into the key-value data store.
|
||||
func (c *cacheWrapper) Put(key []byte, value []byte) error { |
||||
c.Cache.Set(key, value) |
||||
return nil |
||||
} |
||||
|
||||
// Delete removes the key from the key-value data store.
|
||||
func (c *cacheWrapper) Delete(key []byte) error { |
||||
c.Cache.Del(key) |
||||
return nil |
||||
} |
||||
|
||||
type redisPipelineWrapper struct { |
||||
redis.Pipeliner |
||||
|
||||
expiredTime time.Duration |
||||
} |
||||
|
||||
// Put inserts the given value into the key-value data store.
|
||||
func (c *redisPipelineWrapper) Put(key []byte, value []byte) error { |
||||
c.SetEX(context.Background(), string(key), value, c.expiredTime) |
||||
return nil |
||||
} |
||||
|
||||
// Delete removes the key from the key-value data store.
|
||||
func (c *redisPipelineWrapper) Delete(key []byte) error { |
||||
c.Del(context.Background(), string(key)) |
||||
return nil |
||||
} |
||||
|
||||
type StateDBCacheConfig struct { |
||||
CacheSizeInMB uint32 |
||||
CachePersistencePath string |
||||
RedisServerAddr []string |
||||
RedisLRUTimeInDay uint32 |
||||
DebugHitRate bool |
||||
} |
||||
|
||||
type StateDBCacheDatabase struct { |
||||
config StateDBCacheConfig |
||||
enableReadCache bool |
||||
isClose uint64 |
||||
remoteDB common.TiKVStore |
||||
|
||||
// l1cache (memory)
|
||||
l1Cache *cacheWrapper |
||||
|
||||
// l2cache (redis)
|
||||
l2Cache *redis.ClusterClient |
||||
l2ReadOnly bool |
||||
l2ExpiredTime time.Duration |
||||
l2NextExpiredTime time.Time |
||||
l2ExpiredRefresh chan string |
||||
|
||||
// stats
|
||||
l1HitCount uint64 |
||||
l2HitCount uint64 |
||||
missCount uint64 |
||||
notFoundOrErrorCount uint64 |
||||
} |
||||
|
||||
func NewStateDBCacheDatabase(remoteDB common.TiKVStore, config StateDBCacheConfig, readOnly bool) (*StateDBCacheDatabase, error) { |
||||
// create or load memory cache from file
|
||||
cache := fastcache.LoadFromFileOrNew(config.CachePersistencePath, int(config.CacheSizeInMB)*1024*1024) |
||||
|
||||
// create options
|
||||
option := &redis.ClusterOptions{ |
||||
Addrs: config.RedisServerAddr, |
||||
PoolSize: 32, |
||||
IdleTimeout: 60 * time.Second, |
||||
} |
||||
|
||||
if readOnly { |
||||
option.PoolSize = 16 |
||||
option.ReadOnly = true |
||||
} |
||||
|
||||
// check redis connection
|
||||
rdb := redis.NewClusterClient(option) |
||||
err := rdb.Ping(context.Background()).Err() |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
// reload redis cluster slots status
|
||||
rdb.ReloadState(context.Background()) |
||||
|
||||
db := &StateDBCacheDatabase{ |
||||
config: config, |
||||
enableReadCache: true, |
||||
remoteDB: remoteDB, |
||||
l1Cache: &cacheWrapper{cache}, |
||||
l2Cache: rdb, |
||||
l2ReadOnly: readOnly, |
||||
l2ExpiredTime: time.Duration(config.RedisLRUTimeInDay) * 24 * time.Hour, |
||||
} |
||||
|
||||
if !readOnly { |
||||
// Read a copy of the memory hit data into redis to improve the hit rate
|
||||
// refresh read time to prevent recycling by lru
|
||||
db.l2ExpiredRefresh = make(chan string, 100000) |
||||
go db.startL2ExpiredRefresh() |
||||
} |
||||
|
||||
// print debug info
|
||||
if config.DebugHitRate { |
||||
go func() { |
||||
for range time.Tick(5 * time.Second) { |
||||
db.cacheStatusPrint() |
||||
} |
||||
}() |
||||
} |
||||
|
||||
return db, nil |
||||
} |
||||
|
||||
// Has retrieves if a key is present in the key-value data store.
|
||||
func (c *StateDBCacheDatabase) Has(key []byte) (bool, error) { |
||||
return c.remoteDB.Has(key) |
||||
} |
||||
|
||||
// Get retrieves the given key if it's present in the key-value data store.
|
||||
func (c *StateDBCacheDatabase) Get(key []byte) (ret []byte, err error) { |
||||
if c.enableReadCache { |
||||
var ok bool |
||||
|
||||
// first, get data from memory cache
|
||||
keyStr := string(key) |
||||
if ret, ok = c.l1Cache.HasGet(nil, key); ok { |
||||
if !c.l2ReadOnly { |
||||
select { |
||||
// refresh read time to prevent recycling by lru
|
||||
case c.l2ExpiredRefresh <- keyStr: |
||||
default: |
||||
} |
||||
} |
||||
|
||||
atomic.AddUint64(&c.l1HitCount, 1) |
||||
return ret, nil |
||||
} |
||||
|
||||
defer func() { |
||||
if err == nil { |
||||
if len(ret) < maxMemoryEntrySize { |
||||
// set data to memory db if loaded from redis cache or leveldb
|
||||
c.l1Cache.Set(key, ret) |
||||
} |
||||
} |
||||
}() |
||||
|
||||
timeoutCtx, cancelFunc := context.WithTimeout(context.Background(), 5*time.Second) |
||||
defer cancelFunc() |
||||
|
||||
// load data from redis cache
|
||||
data := c.l2Cache.Get(timeoutCtx, keyStr) |
||||
if data.Err() == nil { |
||||
atomic.AddUint64(&c.l2HitCount, 1) |
||||
return data.Bytes() |
||||
} else if data.Err() != redis.Nil { |
||||
log.Printf("[Get WARN]Redis Error: %v", data.Err()) |
||||
} |
||||
|
||||
if !c.l2ReadOnly { |
||||
defer func() { |
||||
// set data to redis db if loaded from leveldb
|
||||
if err == nil { |
||||
c.l2Cache.SetEX(context.Background(), keyStr, ret, c.l2ExpiredTime) |
||||
} |
||||
}() |
||||
} |
||||
} |
||||
|
||||
ret, err = c.remoteDB.Get(key) |
||||
if err != nil { |
||||
atomic.AddUint64(&c.notFoundOrErrorCount, 1) |
||||
} else { |
||||
atomic.AddUint64(&c.missCount, 1) |
||||
} |
||||
|
||||
return |
||||
} |
||||
|
||||
// Put inserts the given value into the key-value data store.
|
||||
func (c *StateDBCacheDatabase) Put(key []byte, value []byte) (err error) { |
||||
if c.enableReadCache { |
||||
defer func() { |
||||
if err == nil { |
||||
if len(value) < maxMemoryEntrySize { |
||||
// memory db only accept the small data
|
||||
c.l1Cache.Set(key, value) |
||||
} |
||||
if !c.l2ReadOnly { |
||||
timeoutCtx, cancelFunc := context.WithTimeout(context.Background(), 5*time.Second) |
||||
defer cancelFunc() |
||||
|
||||
// send the data to redis
|
||||
res := c.l2Cache.SetEX(timeoutCtx, string(key), value, c.l2ExpiredTime) |
||||
if res.Err() == nil { |
||||
log.Printf("[Put WARN]Redis Error: %v", res.Err()) |
||||
} |
||||
} |
||||
} |
||||
}() |
||||
} |
||||
|
||||
return c.remoteDB.Put(key, value) |
||||
} |
||||
|
||||
// Delete removes the key from the key-value data store.
|
||||
func (c *StateDBCacheDatabase) Delete(key []byte) (err error) { |
||||
if c.enableReadCache { |
||||
defer func() { |
||||
if err == nil { |
||||
c.l1Cache.Del(key) |
||||
if !c.l2ReadOnly { |
||||
c.l2Cache.Del(context.Background(), string(key)) |
||||
} |
||||
} |
||||
}() |
||||
} |
||||
|
||||
return c.remoteDB.Delete(key) |
||||
} |
||||
|
||||
// NewBatch creates a write-only database that buffers changes to its host db
|
||||
// until a final write is called.
|
||||
func (c *StateDBCacheDatabase) NewBatch() ethdb.Batch { |
||||
return newStateDBCacheBatch(c, c.remoteDB.NewBatch()) |
||||
} |
||||
|
||||
// Stat returns a particular internal stat of the database.
|
||||
func (c *StateDBCacheDatabase) Stat(property string) (string, error) { |
||||
switch property { |
||||
case "tikv.save.cache": |
||||
_ = c.l1Cache.SaveToFileConcurrent(c.config.CachePersistencePath, runtime.NumCPU()) |
||||
} |
||||
return c.remoteDB.Stat(property) |
||||
} |
||||
|
||||
func (c *StateDBCacheDatabase) Compact(start []byte, limit []byte) error { |
||||
return c.remoteDB.Compact(start, limit) |
||||
} |
||||
|
||||
func (c *StateDBCacheDatabase) NewIterator(start, end []byte) ethdb.Iterator { |
||||
return c.remoteDB.NewIterator(start, end) |
||||
} |
||||
|
||||
// Close disconnect the redis and save memory cache to file
|
||||
func (c *StateDBCacheDatabase) Close() error { |
||||
if atomic.CompareAndSwapUint64(&c.isClose, 0, 1) { |
||||
err := c.l1Cache.SaveToFileConcurrent(c.config.CachePersistencePath, runtime.NumCPU()) |
||||
if err != nil { |
||||
log.Printf("save file to '%s' error: %v", c.config.CachePersistencePath, err) |
||||
} |
||||
|
||||
if !c.l2ReadOnly { |
||||
close(c.l2ExpiredRefresh) |
||||
time.Sleep(time.Second) |
||||
|
||||
for range c.l2ExpiredRefresh { |
||||
// nop, clear chan
|
||||
} |
||||
} |
||||
|
||||
_ = c.l2Cache.Close() |
||||
|
||||
return c.remoteDB.Close() |
||||
} |
||||
|
||||
return nil |
||||
} |
||||
|
||||
// cacheWrite write batch to cache
|
||||
func (c *StateDBCacheDatabase) cacheWrite(b ethdb.Batch) error { |
||||
if !c.l2ReadOnly { |
||||
pipeline := c.l2Cache.Pipeline() |
||||
|
||||
err := b.Replay(&redisPipelineWrapper{Pipeliner: pipeline, expiredTime: c.l2ExpiredTime}) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
_, err = pipeline.Exec(context.Background()) |
||||
if err != nil { |
||||
log.Printf("[BatchWrite WARN]Redis Error: %v", err) |
||||
} |
||||
} |
||||
|
||||
return b.Replay(c.l1Cache) |
||||
} |
||||
|
||||
func (c *StateDBCacheDatabase) refreshL2ExpiredTime() { |
||||
unix := time.Now().Add(c.l2ExpiredTime).Unix() |
||||
unix = unix - (unix % cacheMinGapInSecond) + cacheMinGapInSecond |
||||
c.l2NextExpiredTime = time.Unix(unix, 0) |
||||
} |
||||
|
||||
// startL2ExpiredRefresh batch refresh redis cache
|
||||
func (c *StateDBCacheDatabase) startL2ExpiredRefresh() { |
||||
ticker := time.NewTicker(time.Second) |
||||
defer ticker.Stop() |
||||
|
||||
pipeline := c.l2Cache.Pipeline() |
||||
lastWrite := time.Now() |
||||
|
||||
for { |
||||
select { |
||||
case key, ok := <-c.l2ExpiredRefresh: |
||||
if !ok { |
||||
return |
||||
} |
||||
|
||||
pipeline.ExpireAt(context.Background(), key, c.l2NextExpiredTime) |
||||
if pipeline.Len() > maxPipelineEntriesCount { |
||||
_, _ = pipeline.Exec(context.Background()) |
||||
lastWrite = time.Now() |
||||
} |
||||
case now := <-ticker.C: |
||||
c.refreshL2ExpiredTime() |
||||
|
||||
if pipeline.Len() > 0 && now.Sub(lastWrite) > time.Second { |
||||
_, _ = pipeline.Exec(context.Background()) |
||||
lastWrite = time.Now() |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
// print stats to stdout
|
||||
func (c *StateDBCacheDatabase) cacheStatusPrint() { |
||||
var state = &fastcache.Stats{} |
||||
state.Reset() |
||||
c.l1Cache.UpdateStats(state) |
||||
|
||||
stats := c.l2Cache.PoolStats() |
||||
log.Printf("redis: TotalConns: %d, IdleConns: %d, StaleConns: %d", stats.TotalConns, stats.IdleConns, stats.StaleConns) |
||||
total := float64(c.l1HitCount + c.l2HitCount + c.missCount + c.notFoundOrErrorCount) |
||||
|
||||
log.Printf( |
||||
"total: l1Hit: %d(%.2f%%), l2Hit: %d(%.2f%%), miss: %d(%.2f%%), notFoundOrErrorCount: %d, fastcache: GetCalls: %d, SetCalls: %d, Misses: %d, EntriesCount: %d, BytesSize: %d", |
||||
c.l1HitCount, float64(c.l1HitCount)/total*100, c.l2HitCount, float64(c.l2HitCount)/total*100, c.missCount, float64(c.missCount)/total*100, c.notFoundOrErrorCount, |
||||
state.GetCalls, state.SetCalls, state.Misses, state.EntriesCount, state.BytesSize, |
||||
) |
||||
} |
||||
|
||||
func (c *StateDBCacheDatabase) ReplayCache(batch ethdb.Batch) error { |
||||
return c.cacheWrite(batch) |
||||
} |
||||
Loading…
Reference in new issue