woop/api/service/explorer/schema.go

package explorer

import (
	"encoding/binary"
	"fmt"

	"github.com/RoaringBitmap/roaring/roaring64"
	"github.com/ethereum/go-ethereum/common"
	"github.com/ethereum/go-ethereum/rlp"
	goversion "github.com/hashicorp/go-version"
	"github.com/pkg/errors"
	"github.com/syndtr/goleveldb/leveldb"
	"go.uber.org/zap/buffer"
)

const (
	LegAddressPrefix = "ad_"
	CheckpointBitmap = "checkpoint_bitmap"
	TracePrefix      = "tr_"

	oneAddrByteLen = 42 // byte size of string "one1..."
)

// readCheckpointBitmap read explorer checkpoint bitmap from storage
func readCheckpointBitmap(db databaseReader) (*roaring64.Bitmap, error) {
	bitmapByte, err := db.Get([]byte(CheckpointBitmap))
	if err != nil {
		if err == leveldb.ErrNotFound {
			return roaring64.NewBitmap(), nil
		}
		return nil, err
	}

	rb := roaring64.NewBitmap()
	err = rb.UnmarshalBinary(bitmapByte)
	if err != nil {
		return nil, err
	}

	return rb, nil
}

// writeCheckpointBitmap write explorer checkpoint bitmap to storage
func writeCheckpointBitmap(db databaseWriter, rb *roaring64.Bitmap) error {
	bitmapByte, err := rb.MarshalBinary()
	if err != nil {
		return err
	}

	return db.Put([]byte(CheckpointBitmap), bitmapByte)
}

func getTraceResultKey(key []byte) []byte {
	return append([]byte(TracePrefix), key...)
}
func isTraceResultInDB(db databaseReader, key []byte) (bool, error) {
	key = getTraceResultKey(key)
	return db.Has(key)
}

func writeTraceResult(db databaseWriter, key []byte, data []byte) error {
	key = getTraceResultKey(key)
	return db.Put(key, data)
}

func getTraceResult(db databaseReader, key []byte) ([]byte, error) {
	key = getTraceResultKey(key)
	return db.Get(key)
}

// New schema

var (
	versionKey     = []byte("version")
	versionV100, _ = goversion.NewVersion("1.0.0")
)

// isVersionV100 return whether the version is larger than or equal to 1.0.0
func isVersionV100(db databaseReader) (bool, error) {
	curVer, err := readVersion(db)
	if err != nil {
		if errors.Is(err, leveldb.ErrNotFound) {
			return false, nil
		}
		return false, err
	}
	return curVer.GreaterThanOrEqual(versionV100), nil
}

func readVersion(db databaseReader) (*goversion.Version, error) {
	key := versionKey
	val, err := db.Get(key)
	if err != nil {
		return nil, err
	}
	return goversion.NewVersion(string(val))
}

func writeVersion(db databaseWriter, ver *goversion.Version) error {
	key := versionKey
	val := []byte(ver.String())
	return db.Put(key, val)
}

var (
	addrPrefix                = []byte("addr")
	txnPrefix                 = []byte("tx")
	addrNormalTxnIndexPrefix  = []byte("at")
	addrStakingTxnIndexPrefix = []byte("stk")
)

// bPool is the sync pool for reusing the memory for allocating db keys
var bPool = buffer.NewPool()

func getAddressKey(addr oneAddress) []byte {
	b := bPool.Get()
	defer b.Free()

	_, _ = b.Write(addrPrefix)
	_, _ = b.Write([]byte(addr))
	return b.Bytes()
}

func getAddressFromAddressKey(key []byte) (oneAddress, error) {
	if len(key) < len(addrPrefix)+oneAddrByteLen {
		return "", errors.New("address key size unexpected")
	}
	addrBytes := key[len(addrPrefix) : len(addrPrefix)+oneAddrByteLen]
	return oneAddress(addrBytes), nil
}

func writeAddressEntry(db databaseWriter, addr oneAddress) error {
	key := getAddressKey(addr)
	return db.Put(key, []byte{})
}

func isAddressWritten(db databaseReader, addr oneAddress) (bool, error) {
	key := getAddressKey(addr)
	return db.Has(key)
}

func getAllAddresses(db databaseReader) ([]oneAddress, error) {
	var addrs []oneAddress
	it := db.NewPrefixIterator(addrPrefix)
	defer it.Release()
	for it.Next() {
		addr, err := getAddressFromAddressKey(it.Key())
		if err != nil {
			return nil, err
		}
		addrs = append(addrs, addr)
	}
	return addrs, nil
}

func getAddressesInRange(db databaseReader, start oneAddress, size int) ([]oneAddress, error) {
	var addrs []oneAddress
	startKey := getAddressKey(start)
	it := db.NewSizedIterator(startKey, size)
	defer it.Release()

	for it.Next() {
		addr, err := getAddressFromAddressKey(it.Key())
		if err != nil {
			return nil, err
		}
		addrs = append(addrs, addr)
	}
	return addrs, it.Error()
}

// getTxnKey return the transaction key. It's simply a prefix with the transaction hash
func getTxnKey(txHash common.Hash) []byte {
	b := bPool.Get()
	defer b.Free()

	_, _ = b.Write(txnPrefix)
	_, _ = b.Write(txHash[:])
	return b.Bytes()
}

func readTxnByHash(db databaseReader, txHash common.Hash) (*TxRecord, error) {
	key := getTxnKey(txHash)
	b, err := db.Get(key)
	if err != nil {
		return nil, err
	}
	var tx *TxRecord
	if err := rlp.DecodeBytes(b, &tx); err != nil {
		return nil, err
	}
	return tx, nil
}

func writeTxn(db databaseWriter, txHash common.Hash, tx *TxRecord) error {
	key := getTxnKey(txHash)
	bs, err := rlp.EncodeToBytes(tx)
	if err != nil {
		return err
	}
	return db.Put(key, bs)
}

// normalTxnIndex is a single entry of address-transaction index for normal transaction.
// The key of the entry in db is a combination of txnPrefix, account address, block
// number of the transaction, transaction index in the block, and transaction hash
type normalTxnIndex struct {
	addr        oneAddress
	blockNumber uint64
	txnIndex    uint64
	txnHash     common.Hash
}

func (index normalTxnIndex) key() []byte {
	b := bPool.Get()
	defer b.Free()

	_, _ = b.Write(addrNormalTxnIndexPrefix)
	_, _ = b.Write([]byte(index.addr))
	_ = binary.Write(b, binary.BigEndian, index.blockNumber)
	_ = binary.Write(b, binary.BigEndian, index.txnIndex)
	_, _ = b.Write(index.txnHash[:])
	return b.Bytes()
}

func normalTxnIndexPrefixByAddr(addr oneAddress) []byte {
	b := bPool.Get()
	defer b.Free()

	_, _ = b.Write(addrNormalTxnIndexPrefix)
	_, _ = b.Write([]byte(addr))
	return b.Bytes()
}

func txnHashFromNormalTxnIndexKey(key []byte) (common.Hash, error) {
	txStart := len(addrNormalTxnIndexPrefix) + oneAddrByteLen + 8 + 8
	expSize := txStart + common.HashLength
	if len(key) < expSize {
		return common.Hash{}, errors.New("unexpected key size")
	}
	var txHash common.Hash
	copy(txHash[:], key[txStart:expSize])
	return txHash, nil
}

func getNormalTxnHashesByAccount(db databaseReader, addr oneAddress) ([]common.Hash, []TxType, error) {
	var (
		txHashes []common.Hash
		tts      []TxType
	)
	prefix := normalTxnIndexPrefixByAddr(addr)
	err := forEachAtPrefix(db, prefix, func(key, val []byte) error {
		txHash, err := txnHashFromNormalTxnIndexKey(key)
		if err != nil {
			return err
		}
		if len(val) < 0 {
			return errors.New("val size not expected")
		}
		tts = append(tts, TxType(val[0]))
		txHashes = append(txHashes, txHash)
		return nil
	})
	if err != nil {
		return nil, nil, err
	}
	return txHashes, tts, nil
}

func writeNormalTxnIndex(db databaseWriter, entry normalTxnIndex, tt TxType) error {
	key := entry.key()
	return db.Put(key, []byte{byte(tt)})
}

type stakingTxnIndex struct {
	addr        oneAddress
	blockNumber uint64
	txnIndex    uint64
	txnHash     common.Hash
}

func (index stakingTxnIndex) key() []byte {
	b := bPool.Get()
	defer b.Free()

	_, _ = b.Write(addrStakingTxnIndexPrefix)
	_, _ = b.Write([]byte(index.addr))
	_ = binary.Write(b, binary.BigEndian, index.blockNumber)
	_ = binary.Write(b, binary.BigEndian, index.txnIndex)
	_, _ = b.Write(index.txnHash[:])
	return b.Bytes()
}

func stakingTxnIndexPrefixByAddr(addr oneAddress) []byte {
	b := bPool.Get()
	defer b.Free()

	_, _ = b.Write(addrStakingTxnIndexPrefix)
	_, _ = b.Write([]byte(addr))
	return b.Bytes()
}

func txnHashFromStakingTxnIndexKey(key []byte) (common.Hash, error) {
	txStart := len(addrStakingTxnIndexPrefix) + oneAddrByteLen + 8 + 8
	expSize := txStart + common.HashLength
	if len(key) < expSize {
		return common.Hash{}, errors.New("unexpected key size")
	}
	var txHash common.Hash
	copy(txHash[:], key[txStart:expSize])
	return txHash, nil
}

func getStakingTxnHashesByAccount(db databaseReader, addr oneAddress) ([]common.Hash, []TxType, error) {
	var (
		txHashes []common.Hash
		tts      []TxType
	)
	prefix := stakingTxnIndexPrefixByAddr(addr)
	err := forEachAtPrefix(db, prefix, func(key, val []byte) error {
		txHash, err := txnHashFromStakingTxnIndexKey(key)
		if err != nil {
			return err
		}
		if len(val) < 0 {
			return errors.New("val size not expected")
		}
		tts = append(tts, TxType(val[0]))
		txHashes = append(txHashes, txHash)
		return nil
	})
	if err != nil {
		return nil, nil, err
	}
	return txHashes, tts, nil
}

func writeStakingTxnIndex(db databaseWriter, entry stakingTxnIndex, tt TxType) error {
	key := entry.key()
	return db.Put(key, []byte{byte(tt)})
}

func forEachAtPrefix(db databaseReader, prefix []byte, f func(key, val []byte) error) error {
	it := db.NewPrefixIterator(prefix)
	defer it.Release()

	for it.Next() {
		err := f(it.Key(), it.Value())
		if err != nil {
			return err
		}
	}
	return it.Error()
}

// Legacy Schema

// LegGetAddressKey ...
func LegGetAddressKey(address oneAddress) []byte {
	return []byte(fmt.Sprintf("%s%s", LegAddressPrefix, address))
}