diff --git a/internal/db/db.go b/internal/db/db.go
deleted file mode 100644
index 4bd5f3db9..000000000
--- a/internal/db/db.go
+++ /dev/null
@@ -1,242 +0,0 @@
-package db
-
-import (
- "sync"
- "time"
-
- "github.com/harmony-one/harmony/log"
- "github.com/syndtr/goleveldb/leveldb"
- "github.com/syndtr/goleveldb/leveldb/errors"
- "github.com/syndtr/goleveldb/leveldb/filter"
- "github.com/syndtr/goleveldb/leveldb/iterator"
- "github.com/syndtr/goleveldb/leveldb/opt"
- "github.com/syndtr/goleveldb/leveldb/util"
-)
-
-// Constants for db which can be used to customize later.
-const (
- writePauseWarningThrottler = 1 * time.Minute
-)
-
-// LDBDatabase is database based on leveldb.
-type LDBDatabase struct {
- fn string // filename for reporting
- db *leveldb.DB // LevelDB instance
-
- quitLock sync.Mutex // Mutex protecting the quit channel access
- quitChan chan chan error // Quit channel to stop the metrics collection before closing the database
-
- log log.Logger // Contextual logger tracking the database path
-}
-
-// NewLDBDatabase returns a LevelDB wrapped object.
-func NewLDBDatabase(file string, cache int, handles int) (*LDBDatabase, error) {
- logger := log.New("database", file)
-
- // Ensure we have some minimal caching and file guarantees
- if cache < 16 {
- cache = 16
- }
- if handles < 16 {
- handles = 16
- }
- logger.Info("Allocated cache and file handles", "cache", cache, "handles", handles)
-
- // Open the db and recover any potential corruptions
- db, err := leveldb.OpenFile(file, &opt.Options{
- OpenFilesCacheCapacity: handles,
- BlockCacheCapacity: cache / 2 * opt.MiB,
- WriteBuffer: cache / 4 * opt.MiB, // Two of these are used internally
- Filter: filter.NewBloomFilter(10),
- })
- if _, corrupted := err.(*errors.ErrCorrupted); corrupted {
- db, err = leveldb.RecoverFile(file, nil)
- }
- // (Re)check for errors and abort if opening of the db failed
- if err != nil {
- return nil, err
- }
- return &LDBDatabase{
- fn: file,
- db: db,
- log: logger,
- }, nil
-}
-
-// Path returns the path to the database directory.
-func (db *LDBDatabase) Path() string {
- return db.fn
-}
-
-// Put puts the given key / value to the queue
-func (db *LDBDatabase) Put(key []byte, value []byte) error {
- return db.db.Put(key, value, nil)
-}
-
-// Has is used to check if the given key is included into the database.
-func (db *LDBDatabase) Has(key []byte) (bool, error) {
- return db.db.Has(key, nil)
-}
-
-// Get returns the given key if it's present.
-func (db *LDBDatabase) Get(key []byte) ([]byte, error) {
- dat, err := db.db.Get(key, nil)
- if err != nil {
- return nil, err
- }
- return dat, nil
-}
-
-// Delete deletes the key from the queue and database
-func (db *LDBDatabase) Delete(key []byte) error {
- return db.db.Delete(key, nil)
-}
-
-// NewIterator returns the current iterator of the db.
-func (db *LDBDatabase) NewIterator() iterator.Iterator {
- return db.db.NewIterator(nil, nil)
-}
-
-// NewIteratorWithPrefix returns a iterator to iterate over subset of database content with a particular prefix.
-func (db *LDBDatabase) NewIteratorWithPrefix(prefix []byte) iterator.Iterator {
- return db.db.NewIterator(util.BytesPrefix(prefix), nil)
-}
-
-// Close closes the database.
-func (db *LDBDatabase) Close() {
- // Stop the metrics collection to avoid internal database races
- db.quitLock.Lock()
- defer db.quitLock.Unlock()
-
- if db.quitChan != nil {
- errc := make(chan error)
- db.quitChan <- errc
- if err := <-errc; err != nil {
- db.log.Error("Metrics collection failed", "err", err)
- }
- db.quitChan = nil
- }
- err := db.db.Close()
- if err == nil {
- db.log.Info("Database closed")
- } else {
- db.log.Error("Failed to close database", "err", err)
- }
-}
-
-// LDB returns the pointer to leveldb on which the LDBDatabase is built.
-func (db *LDBDatabase) LDB() *leveldb.DB {
- return db.db
-}
-
-/* TODO(minhdoan): Might add meter func from ethereum-go repo
- */
-
-// NewBatch returns Batch interface for a series of leveldb transactions.
-func (db *LDBDatabase) NewBatch() Batch {
- return &ldbBatch{db: db.db, b: new(leveldb.Batch)}
-}
-
-type ldbBatch struct {
- db *leveldb.DB
- b *leveldb.Batch
- size int
-}
-
-// Put is used to put key, value into the batch of transactions.
-func (b *ldbBatch) Put(key, value []byte) error {
- b.b.Put(key, value)
- b.size += len(value)
- return nil
-}
-
-// Delete is used to delete the item associated with the given key as a part of the batch.
-func (b *ldbBatch) Delete(key []byte) error {
- b.b.Delete(key)
- b.size++
- return nil
-}
-
-// Write writes the patch of transactions.
-func (b *ldbBatch) Write() error {
- return b.db.Write(b.b, nil)
-}
-
-// ValueSize returns the size of the patch.
-func (b *ldbBatch) ValueSize() int {
- return b.size
-}
-
-// Reset resets the batch.
-func (b *ldbBatch) Reset() {
- b.b.Reset()
- b.size = 0
-}
-
-type table struct {
- db Database
- prefix string
-}
-
-// NewTable returns a Database object that prefixes all keys with a given
-// string.
-func NewTable(db Database, prefix string) Database {
- return &table{
- db: db,
- prefix: prefix,
- }
-}
-
-func (dt *table) Put(key []byte, value []byte) error {
- return dt.db.Put(append([]byte(dt.prefix), key...), value)
-}
-
-func (dt *table) Has(key []byte) (bool, error) {
- return dt.db.Has(append([]byte(dt.prefix), key...))
-}
-
-func (dt *table) Get(key []byte) ([]byte, error) {
- return dt.db.Get(append([]byte(dt.prefix), key...))
-}
-
-func (dt *table) Delete(key []byte) error {
- return dt.db.Delete(append([]byte(dt.prefix), key...))
-}
-
-func (dt *table) Close() {
- // Do nothing; don't close the underlying DB.
-}
-
-type tableBatch struct {
- batch Batch
- prefix string
-}
-
-// NewTableBatch returns a Batch object which prefixes all keys with a given string.
-func NewTableBatch(db Database, prefix string) Batch {
- return &tableBatch{db.NewBatch(), prefix}
-}
-
-func (dt *table) NewBatch() Batch {
- return &tableBatch{dt.db.NewBatch(), dt.prefix}
-}
-
-func (tb *tableBatch) Put(key, value []byte) error {
- return tb.batch.Put(append([]byte(tb.prefix), key...), value)
-}
-
-func (tb *tableBatch) Delete(key []byte) error {
- return tb.batch.Delete(append([]byte(tb.prefix), key...))
-}
-
-func (tb *tableBatch) Write() error {
- return tb.batch.Write()
-}
-
-func (tb *tableBatch) ValueSize() int {
- return tb.batch.ValueSize()
-}
-
-func (tb *tableBatch) Reset() {
- tb.batch.Reset()
-}
diff --git a/internal/db/db_test.go b/internal/db/db_test.go
deleted file mode 100644
index 13a5bb122..000000000
--- a/internal/db/db_test.go
+++ /dev/null
@@ -1,194 +0,0 @@
-package db
-
-import (
- "bytes"
- "fmt"
- "io/ioutil"
- "os"
- "strconv"
- "sync"
- "testing"
-)
-
-func newTestLDB() (*LDBDatabase, func()) {
- dirname, err := ioutil.TempDir(os.TempDir(), "db_test_")
- if err != nil {
- panic("failed to create test file: " + err.Error())
- }
- db, err := NewLDBDatabase(dirname, 0, 0)
- if err != nil {
- panic("failed to create test database: " + err.Error())
- }
-
- return db, func() {
- db.Close()
- os.RemoveAll(dirname)
- }
-}
-
-var testValues = []string{"", "a", "1251", "\x00123\x00"}
-
-func TestLDB_PutGet(t *testing.T) {
- db, remove := newTestLDB()
- defer remove()
- testPutGet(db, t)
-}
-
-func TestMemoryDB_PutGet(t *testing.T) {
- testPutGet(NewMemDatabase(), t)
-}
-
-func testPutGet(db Database, t *testing.T) {
- t.Parallel()
-
- for _, k := range testValues {
- err := db.Put([]byte(k), nil)
- if err != nil {
- t.Fatalf("put failed: %v", err)
- }
- }
-
- for _, k := range testValues {
- data, err := db.Get([]byte(k))
- if err != nil {
- t.Fatalf("get failed: %v", err)
- }
- if len(data) != 0 {
- t.Fatalf("get returned wrong result, got %q expected nil", string(data))
- }
- }
-
- _, err := db.Get([]byte("non-exist-key"))
- if err == nil {
- t.Fatalf("expect to return a not found error")
- }
-
- for _, v := range testValues {
- err := db.Put([]byte(v), []byte(v))
- if err != nil {
- t.Fatalf("put failed: %v", err)
- }
- }
-
- for _, v := range testValues {
- data, err := db.Get([]byte(v))
- if err != nil {
- t.Fatalf("get failed: %v", err)
- }
- if !bytes.Equal(data, []byte(v)) {
- t.Fatalf("get returned wrong result, got %q expected %q", string(data), v)
- }
- }
-
- for _, v := range testValues {
- err := db.Put([]byte(v), []byte("?"))
- if err != nil {
- t.Fatalf("put override failed: %v", err)
- }
- }
-
- for _, v := range testValues {
- data, err := db.Get([]byte(v))
- if err != nil {
- t.Fatalf("get failed: %v", err)
- }
- if !bytes.Equal(data, []byte("?")) {
- t.Fatalf("get returned wrong result, got %q expected ?", string(data))
- }
- }
-
- for _, v := range testValues {
- orig, err := db.Get([]byte(v))
- if err != nil {
- t.Fatalf("get failed: %v", err)
- }
- orig[0] = byte(0xff)
- data, err := db.Get([]byte(v))
- if err != nil {
- t.Fatalf("get failed: %v", err)
- }
- if !bytes.Equal(data, []byte("?")) {
- t.Fatalf("get returned wrong result, got %q expected ?", string(data))
- }
- }
-
- for _, v := range testValues {
- err := db.Delete([]byte(v))
- if err != nil {
- t.Fatalf("delete %q failed: %v", v, err)
- }
- }
-
- for _, v := range testValues {
- _, err := db.Get([]byte(v))
- if err == nil {
- t.Fatalf("got deleted value %q", v)
- }
- }
-}
-
-func TestLDB_ParallelPutGet(t *testing.T) {
- db, remove := newTestLDB()
- defer remove()
- testParallelPutGet(db, t)
-}
-
-func TestMemoryDB_ParallelPutGet(t *testing.T) {
- testParallelPutGet(NewMemDatabase(), t)
-}
-
-func testParallelPutGet(db Database, t *testing.T) {
- const n = 8
- var pending sync.WaitGroup
-
- pending.Add(n)
- for i := 0; i < n; i++ {
- go func(key string) {
- defer pending.Done()
- err := db.Put([]byte(key), []byte("v"+key))
- if err != nil {
- panic("put failed: " + err.Error())
- }
- }(strconv.Itoa(i))
- }
- pending.Wait()
-
- pending.Add(n)
- for i := 0; i < n; i++ {
- go func(key string) {
- defer pending.Done()
- data, err := db.Get([]byte(key))
- if err != nil {
- panic("get failed: " + err.Error())
- }
- if !bytes.Equal(data, []byte("v"+key)) {
- panic(fmt.Sprintf("get failed, got %q expected %q", []byte(data), []byte("v"+key)))
- }
- }(strconv.Itoa(i))
- }
- pending.Wait()
-
- pending.Add(n)
- for i := 0; i < n; i++ {
- go func(key string) {
- defer pending.Done()
- err := db.Delete([]byte(key))
- if err != nil {
- panic("delete failed: " + err.Error())
- }
- }(strconv.Itoa(i))
- }
- pending.Wait()
-
- pending.Add(n)
- for i := 0; i < n; i++ {
- go func(key string) {
- defer pending.Done()
- _, err := db.Get([]byte(key))
- if err == nil {
- panic("get succeeded")
- }
- }(strconv.Itoa(i))
- }
- pending.Wait()
-}
diff --git a/internal/db/interface.go b/internal/db/interface.go
deleted file mode 100644
index f3d8cb830..000000000
--- a/internal/db/interface.go
+++ /dev/null
@@ -1,36 +0,0 @@
-package db
-
-// IdealBatchSize is the max size of batch transactions.
-// The value was determined empirically.
-const IdealBatchSize = 100 * 1024
-
-// Putter wraps the database write operation supported by both batches and regular databases.
-type Putter interface {
- Put(key []byte, value []byte) error
-}
-
-// Deleter wraps the database delete operation supported by both batches and regular databases.
-type Deleter interface {
- Delete(key []byte) error
-}
-
-// Database wraps all database operations. All methods are safe for concurrent use.
-type Database interface {
- Putter
- Deleter
- Get(key []byte) ([]byte, error)
- Has(key []byte) (bool, error)
- Close()
- NewBatch() Batch
-}
-
-// Batch is a write-only database that commits changes to its host database
-// when Write is called. Batch cannot be used concurrently.
-type Batch interface {
- Putter
- Deleter
- ValueSize() int // amount of data in the batch
- Write() error
- // Reset resets the batch for reuse
- Reset()
-}
diff --git a/internal/db/memory_db.go b/internal/db/memory_db.go
deleted file mode 100644
index f81e0e797..000000000
--- a/internal/db/memory_db.go
+++ /dev/null
@@ -1,135 +0,0 @@
-package db
-
-import (
- "errors"
- "sync"
-
- "github.com/harmony-one/harmony/internal/utils"
-)
-
-// MemDatabase is the test memory database. It won't be used for any production.
-type MemDatabase struct {
- db map[string][]byte
- lock sync.RWMutex
-}
-
-// NewMemDatabase returns a pointer of the new creation of MemDatabase.
-func NewMemDatabase() *MemDatabase {
- return &MemDatabase{
- db: make(map[string][]byte),
- }
-}
-
-// NewMemDatabaseWithCap returns a pointer of the new creation of MemDatabase with the given size.
-func NewMemDatabaseWithCap(size int) *MemDatabase {
- return &MemDatabase{
- db: make(map[string][]byte, size),
- }
-}
-
-// Put puts (key, value) item into MemDatabase.
-func (db *MemDatabase) Put(key []byte, value []byte) error {
- db.lock.Lock()
- defer db.lock.Unlock()
-
- db.db[string(key)] = utils.CopyBytes(value)
- return nil
-}
-
-// Has checks if the key is included into MemDatabase.
-func (db *MemDatabase) Has(key []byte) (bool, error) {
- db.lock.RLock()
- defer db.lock.RUnlock()
-
- _, ok := db.db[string(key)]
- return ok, nil
-}
-
-// Get gets value of the given key.
-func (db *MemDatabase) Get(key []byte) ([]byte, error) {
- db.lock.RLock()
- defer db.lock.RUnlock()
-
- if entry, ok := db.db[string(key)]; ok {
- return utils.CopyBytes(entry), nil
- }
- return nil, errors.New("not found")
-}
-
-// Keys returns all keys of the given MemDatabase.
-func (db *MemDatabase) Keys() [][]byte {
- db.lock.RLock()
- defer db.lock.RUnlock()
-
- keys := [][]byte{}
- for key := range db.db {
- keys = append(keys, []byte(key))
- }
- return keys
-}
-
-// Delete deletes the given key.
-func (db *MemDatabase) Delete(key []byte) error {
- db.lock.Lock()
- defer db.lock.Unlock()
-
- delete(db.db, string(key))
- return nil
-}
-
-// Close closes the given db.
-func (db *MemDatabase) Close() {}
-
-// NewBatch returns a batch of MemDatabase transactions.
-func (db *MemDatabase) NewBatch() Batch {
- return &memBatch{db: db}
-}
-
-// Len returns the length of the given db.
-func (db *MemDatabase) Len() int { return len(db.db) }
-
-type kv struct {
- k, v []byte
- del bool
-}
-
-type memBatch struct {
- db *MemDatabase
- writes []kv
- size int
-}
-
-func (b *memBatch) Put(key, value []byte) error {
- b.writes = append(b.writes, kv{utils.CopyBytes(key), utils.CopyBytes(value), false})
- b.size += len(value)
- return nil
-}
-
-func (b *memBatch) Delete(key []byte) error {
- b.writes = append(b.writes, kv{utils.CopyBytes(key), nil, true})
- b.size++
- return nil
-}
-
-func (b *memBatch) Write() error {
- b.db.lock.Lock()
- defer b.db.lock.Unlock()
-
- for _, kv := range b.writes {
- if kv.del {
- delete(b.db.db, string(kv.k))
- continue
- }
- b.db.db[string(kv.k)] = kv.v
- }
- return nil
-}
-
-func (b *memBatch) ValueSize() int {
- return b.size
-}
-
-func (b *memBatch) Reset() {
- b.writes = b.writes[:0]
- b.size = 0
-}
diff --git a/internal/trie/database.go b/internal/trie/database.go
deleted file mode 100644
index 499282421..000000000
--- a/internal/trie/database.go
+++ /dev/null
@@ -1,782 +0,0 @@
-// Copyright 2018 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see .
-
-package trie
-
-import (
- "fmt"
- "io"
- "sync"
- "time"
-
- "github.com/ethereum/go-ethereum/common"
- "github.com/ethereum/go-ethereum/log"
- "github.com/ethereum/go-ethereum/metrics"
- "github.com/ethereum/go-ethereum/rlp"
- hdb "github.com/harmony-one/harmony/internal/db"
-)
-
-var (
- memcacheFlushTimeTimer = metrics.NewRegisteredResettingTimer("trie/memcache/flush/time", nil)
- memcacheFlushNodesMeter = metrics.NewRegisteredMeter("trie/memcache/flush/nodes", nil)
- memcacheFlushSizeMeter = metrics.NewRegisteredMeter("trie/memcache/flush/size", nil)
-
- memcacheGCTimeTimer = metrics.NewRegisteredResettingTimer("trie/memcache/gc/time", nil)
- memcacheGCNodesMeter = metrics.NewRegisteredMeter("trie/memcache/gc/nodes", nil)
- memcacheGCSizeMeter = metrics.NewRegisteredMeter("trie/memcache/gc/size", nil)
-
- memcacheCommitTimeTimer = metrics.NewRegisteredResettingTimer("trie/memcache/commit/time", nil)
- memcacheCommitNodesMeter = metrics.NewRegisteredMeter("trie/memcache/commit/nodes", nil)
- memcacheCommitSizeMeter = metrics.NewRegisteredMeter("trie/memcache/commit/size", nil)
-)
-
-// secureKeyPrefix is the database key prefix used to store trie node preimages.
-var secureKeyPrefix = []byte("secure-key-")
-
-// secureKeyLength is the length of the above prefix + 32byte hash.
-const secureKeyLength = 11 + 32
-
-// DatabaseReader wraps the Get and Has method of a backing store for the trie.
-type DatabaseReader interface {
- // Get retrieves the value associated with key from the database.
- Get(key []byte) (value []byte, err error)
-
- // Has retrieves whether a key is present in the database.
- Has(key []byte) (bool, error)
-}
-
-// Database is an intermediate write layer between the trie data structures and
-// the disk database. The aim is to accumulate trie writes in-memory and only
-// periodically flush a couple tries to disk, garbage collecting the remainder.
-type Database struct {
- diskdb hdb.Database // Persistent storage for matured trie nodes
-
- nodes map[common.Hash]*cachedNode // Data and references relationships of a node
- oldest common.Hash // Oldest tracked node, flush-list head
- newest common.Hash // Newest tracked node, flush-list tail
-
- preimages map[common.Hash][]byte // Preimages of nodes from the secure trie
- seckeybuf [secureKeyLength]byte // Ephemeral buffer for calculating preimage keys
-
- gctime time.Duration // Time spent on garbage collection since last commit
- gcnodes uint64 // Nodes garbage collected since last commit
- gcsize common.StorageSize // Data storage garbage collected since last commit
-
- flushtime time.Duration // Time spent on data flushing since last commit
- flushnodes uint64 // Nodes flushed since last commit
- flushsize common.StorageSize // Data storage flushed since last commit
-
- nodesSize common.StorageSize // Storage size of the nodes cache (exc. flushlist)
- preimagesSize common.StorageSize // Storage size of the preimages cache
-
- lock sync.RWMutex
-}
-
-// rawNode is a simple binary blob used to differentiate between collapsed trie
-// nodes and already encoded RLP binary blobs (while at the same time store them
-// in the same cache fields).
-type rawNode []byte
-
-func (n rawNode) canUnload(uint16, uint16) bool { panic("this should never end up in a live trie") }
-func (n rawNode) cache() (hashNode, bool) { panic("this should never end up in a live trie") }
-func (n rawNode) fstring(ind string) string { panic("this should never end up in a live trie") }
-
-// rawFullNode represents only the useful data content of a full node, with the
-// caches and flags stripped out to minimize its data storage. This type honors
-// the same RLP encoding as the original parent.
-type rawFullNode [17]node
-
-func (n rawFullNode) canUnload(uint16, uint16) bool { panic("this should never end up in a live trie") }
-func (n rawFullNode) cache() (hashNode, bool) { panic("this should never end up in a live trie") }
-func (n rawFullNode) fstring(ind string) string { panic("this should never end up in a live trie") }
-
-func (n rawFullNode) EncodeRLP(w io.Writer) error {
- var nodes [17]node
-
- for i, child := range n {
- if child != nil {
- nodes[i] = child
- } else {
- nodes[i] = nilValueNode
- }
- }
- return rlp.Encode(w, nodes)
-}
-
-// rawShortNode represents only the useful data content of a short node, with the
-// caches and flags stripped out to minimize its data storage. This type honors
-// the same RLP encoding as the original parent.
-type rawShortNode struct {
- Key []byte
- Val node
-}
-
-func (n rawShortNode) canUnload(uint16, uint16) bool { panic("this should never end up in a live trie") }
-func (n rawShortNode) cache() (hashNode, bool) { panic("this should never end up in a live trie") }
-func (n rawShortNode) fstring(ind string) string { panic("this should never end up in a live trie") }
-
-// cachedNode is all the information we know about a single cached node in the
-// memory database write layer.
-type cachedNode struct {
- node node // Cached collapsed trie node, or raw rlp data
- size uint16 // Byte size of the useful cached data
-
- parents uint16 // Number of live nodes referencing this one
- children map[common.Hash]uint16 // External children referenced by this node
-
- flushPrev common.Hash // Previous node in the flush-list
- flushNext common.Hash // Next node in the flush-list
-}
-
-// rlp returns the raw rlp encoded blob of the cached node, either directly from
-// the cache, or by regenerating it from the collapsed node.
-func (n *cachedNode) rlp() []byte {
- if node, ok := n.node.(rawNode); ok {
- return node
- }
- blob, err := rlp.EncodeToBytes(n.node)
- if err != nil {
- panic(err)
- }
- return blob
-}
-
-// obj returns the decoded and expanded trie node, either directly from the cache,
-// or by regenerating it from the rlp encoded blob.
-func (n *cachedNode) obj(hash common.Hash, cachegen uint16) node {
- if node, ok := n.node.(rawNode); ok {
- return mustDecodeNode(hash[:], node, cachegen)
- }
- return expandNode(hash[:], n.node, cachegen)
-}
-
-// childs returns all the tracked children of this node, both the implicit ones
-// from inside the node as well as the explicit ones from outside the node.
-func (n *cachedNode) childs() []common.Hash {
- children := make([]common.Hash, 0, 16)
- for child := range n.children {
- children = append(children, child)
- }
- if _, ok := n.node.(rawNode); !ok {
- gatherChildren(n.node, &children)
- }
- return children
-}
-
-// gatherChildren traverses the node hierarchy of a collapsed storage node and
-// retrieves all the hashnode children.
-func gatherChildren(n node, children *[]common.Hash) {
- switch n := n.(type) {
- case *rawShortNode:
- gatherChildren(n.Val, children)
-
- case rawFullNode:
- for i := 0; i < 16; i++ {
- gatherChildren(n[i], children)
- }
- case hashNode:
- *children = append(*children, common.BytesToHash(n))
-
- case valueNode, nil:
-
- default:
- panic(fmt.Sprintf("unknown node type: %T", n))
- }
-}
-
-// simplifyNode traverses the hierarchy of an expanded memory node and discards
-// all the internal caches, returning a node that only contains the raw data.
-func simplifyNode(n node) node {
- switch n := n.(type) {
- case *shortNode:
- // Short nodes discard the flags and cascade
- return &rawShortNode{Key: n.Key, Val: simplifyNode(n.Val)}
-
- case *fullNode:
- // Full nodes discard the flags and cascade
- node := rawFullNode(n.Children)
- for i := 0; i < len(node); i++ {
- if node[i] != nil {
- node[i] = simplifyNode(node[i])
- }
- }
- return node
-
- case valueNode, hashNode, rawNode:
- return n
-
- default:
- panic(fmt.Sprintf("unknown node type: %T", n))
- }
-}
-
-// expandNode traverses the node hierarchy of a collapsed storage node and converts
-// all fields and keys into expanded memory form.
-func expandNode(hash hashNode, n node, cachegen uint16) node {
- switch n := n.(type) {
- case *rawShortNode:
- // Short nodes need key and child expansion
- return &shortNode{
- Key: compactToHex(n.Key),
- Val: expandNode(nil, n.Val, cachegen),
- flags: nodeFlag{
- hash: hash,
- gen: cachegen,
- },
- }
-
- case rawFullNode:
- // Full nodes need child expansion
- node := &fullNode{
- flags: nodeFlag{
- hash: hash,
- gen: cachegen,
- },
- }
- for i := 0; i < len(node.Children); i++ {
- if n[i] != nil {
- node.Children[i] = expandNode(nil, n[i], cachegen)
- }
- }
- return node
-
- case valueNode, hashNode:
- return n
-
- default:
- panic(fmt.Sprintf("unknown node type: %T", n))
- }
-}
-
-// NewDatabase creates a new trie database to store ephemeral trie content before
-// its written out to disk or garbage collected.
-func NewDatabase(diskdb hdb.Database) *Database {
- return &Database{
- diskdb: diskdb,
- nodes: map[common.Hash]*cachedNode{{}: {}},
- preimages: make(map[common.Hash][]byte),
- }
-}
-
-// DiskDB retrieves the persistent storage backing the trie database.
-func (db *Database) DiskDB() DatabaseReader {
- return db.diskdb
-}
-
-// InsertBlob writes a new reference tracked blob to the memory database if it's
-// yet unknown. This method should only be used for non-trie nodes that require
-// reference counting, since trie nodes are garbage collected directly through
-// their embedded children.
-func (db *Database) InsertBlob(hash common.Hash, blob []byte) {
- db.lock.Lock()
- defer db.lock.Unlock()
-
- db.insert(hash, blob, rawNode(blob))
-}
-
-// insert inserts a collapsed trie node into the memory database. This method is
-// a more generic version of InsertBlob, supporting both raw blob insertions as
-// well ex trie node insertions. The blob must always be specified to allow proper
-// size tracking.
-func (db *Database) insert(hash common.Hash, blob []byte, node node) {
- // If the node's already cached, skip
- if _, ok := db.nodes[hash]; ok {
- return
- }
- // Create the cached entry for this node
- entry := &cachedNode{
- node: simplifyNode(node),
- size: uint16(len(blob)),
- flushPrev: db.newest,
- }
- for _, child := range entry.childs() {
- if c := db.nodes[child]; c != nil {
- c.parents++
- }
- }
- db.nodes[hash] = entry
-
- // Update the flush-list endpoints
- if db.oldest == (common.Hash{}) {
- db.oldest, db.newest = hash, hash
- } else {
- db.nodes[db.newest].flushNext, db.newest = hash, hash
- }
- db.nodesSize += common.StorageSize(common.HashLength + entry.size)
-}
-
-// insertPreimage writes a new trie node pre-image to the memory database if it's
-// yet unknown. The method will make a copy of the slice.
-//
-// Note, this method assumes that the database's lock is held!
-func (db *Database) insertPreimage(hash common.Hash, preimage []byte) {
- if _, ok := db.preimages[hash]; ok {
- return
- }
- db.preimages[hash] = common.CopyBytes(preimage)
- db.preimagesSize += common.StorageSize(common.HashLength + len(preimage))
-}
-
-// node retrieves a cached trie node from memory, or returns nil if none can be
-// found in the memory cache.
-func (db *Database) node(hash common.Hash, cachegen uint16) node {
- // Retrieve the node from cache if available
- db.lock.RLock()
- node := db.nodes[hash]
- db.lock.RUnlock()
-
- if node != nil {
- return node.obj(hash, cachegen)
- }
- // Content unavailable in memory, attempt to retrieve from disk
- enc, err := db.diskdb.Get(hash[:])
- if err != nil || enc == nil {
- return nil
- }
- return mustDecodeNode(hash[:], enc, cachegen)
-}
-
-// Node retrieves an encoded cached trie node from memory. If it cannot be found
-// cached, the method queries the persistent database for the content.
-func (db *Database) Node(hash common.Hash) ([]byte, error) {
- // Retrieve the node from cache if available
- db.lock.RLock()
- node := db.nodes[hash]
- db.lock.RUnlock()
-
- if node != nil {
- return node.rlp(), nil
- }
- // Content unavailable in memory, attempt to retrieve from disk
- return db.diskdb.Get(hash[:])
-}
-
-// preimage retrieves a cached trie node pre-image from memory. If it cannot be
-// found cached, the method queries the persistent database for the content.
-func (db *Database) preimage(hash common.Hash) ([]byte, error) {
- // Retrieve the node from cache if available
- db.lock.RLock()
- preimage := db.preimages[hash]
- db.lock.RUnlock()
-
- if preimage != nil {
- return preimage, nil
- }
- // Content unavailable in memory, attempt to retrieve from disk
- return db.diskdb.Get(db.secureKey(hash[:]))
-}
-
-// secureKey returns the database key for the preimage of key, as an ephemeral
-// buffer. The caller must not hold onto the return value because it will become
-// invalid on the next call.
-func (db *Database) secureKey(key []byte) []byte {
- buf := append(db.seckeybuf[:0], secureKeyPrefix...)
- buf = append(buf, key...)
- return buf
-}
-
-// Nodes retrieves the hashes of all the nodes cached within the memory database.
-// This method is extremely expensive and should only be used to validate internal
-// states in test code.
-func (db *Database) Nodes() []common.Hash {
- db.lock.RLock()
- defer db.lock.RUnlock()
-
- var hashes = make([]common.Hash, 0, len(db.nodes))
- for hash := range db.nodes {
- if hash != (common.Hash{}) { // Special case for "root" references/nodes
- hashes = append(hashes, hash)
- }
- }
- return hashes
-}
-
-// Reference adds a new reference from a parent node to a child node.
-func (db *Database) Reference(child common.Hash, parent common.Hash) {
- db.lock.RLock()
- defer db.lock.RUnlock()
-
- db.reference(child, parent)
-}
-
-// reference is the private locked version of Reference.
-func (db *Database) reference(child common.Hash, parent common.Hash) {
- // If the node does not exist, it's a node pulled from disk, skip
- node, ok := db.nodes[child]
- if !ok {
- return
- }
- // If the reference already exists, only duplicate for roots
- if db.nodes[parent].children == nil {
- db.nodes[parent].children = make(map[common.Hash]uint16)
- } else if _, ok = db.nodes[parent].children[child]; ok && parent != (common.Hash{}) {
- return
- }
- node.parents++
- db.nodes[parent].children[child]++
-}
-
-// Dereference removes an existing reference from a root node.
-func (db *Database) Dereference(root common.Hash) {
- // Sanity check to ensure that the meta-root is not removed
- if root == (common.Hash{}) {
- log.Error("Attempted to dereference the trie cache meta root")
- return
- }
- db.lock.Lock()
- defer db.lock.Unlock()
-
- nodes, storage, start := len(db.nodes), db.nodesSize, time.Now()
- db.dereference(root, common.Hash{})
-
- db.gcnodes += uint64(nodes - len(db.nodes))
- db.gcsize += storage - db.nodesSize
- db.gctime += time.Since(start)
-
- memcacheGCTimeTimer.Update(time.Since(start))
- memcacheGCSizeMeter.Mark(int64(storage - db.nodesSize))
- memcacheGCNodesMeter.Mark(int64(nodes - len(db.nodes)))
-
- log.Debug("Dereferenced trie from memory database", "nodes", nodes-len(db.nodes), "size", storage-db.nodesSize, "time", time.Since(start),
- "gcnodes", db.gcnodes, "gcsize", db.gcsize, "gctime", db.gctime, "livenodes", len(db.nodes), "livesize", db.nodesSize)
-}
-
-// dereference is the private locked version of Dereference.
-func (db *Database) dereference(child common.Hash, parent common.Hash) {
- // Dereference the parent-child
- node := db.nodes[parent]
-
- if node.children != nil && node.children[child] > 0 {
- node.children[child]--
- if node.children[child] == 0 {
- delete(node.children, child)
- }
- }
- // If the child does not exist, it's a previously committed node.
- node, ok := db.nodes[child]
- if !ok {
- return
- }
- // If there are no more references to the child, delete it and cascade
- if node.parents > 0 {
- // This is a special cornercase where a node loaded from disk (i.e. not in the
- // memcache any more) gets reinjected as a new node (short node split into full,
- // then reverted into short), causing a cached node to have no parents. That is
- // no problem in itself, but don't make maxint parents out of it.
- node.parents--
- }
- if node.parents == 0 {
- // Remove the node from the flush-list
- switch child {
- case db.oldest:
- db.oldest = node.flushNext
- db.nodes[node.flushNext].flushPrev = common.Hash{}
- case db.newest:
- db.newest = node.flushPrev
- db.nodes[node.flushPrev].flushNext = common.Hash{}
- default:
- db.nodes[node.flushPrev].flushNext = node.flushNext
- db.nodes[node.flushNext].flushPrev = node.flushPrev
- }
- // Dereference all children and delete the node
- for _, hash := range node.childs() {
- db.dereference(hash, child)
- }
- delete(db.nodes, child)
- db.nodesSize -= common.StorageSize(common.HashLength + int(node.size))
- }
-}
-
-// Cap iteratively flushes old but still referenced trie nodes until the total
-// memory usage goes below the given threshold.
-func (db *Database) Cap(limit common.StorageSize) error {
- // Create a database batch to flush persistent data out. It is important that
- // outside code doesn't see an inconsistent state (referenced data removed from
- // memory cache during commit but not yet in persistent storage). This is ensured
- // by only uncaching existing data when the database write finalizes.
- db.lock.RLock()
-
- nodes, storage, start := len(db.nodes), db.nodesSize, time.Now()
- batch := db.diskdb.NewBatch()
-
- // db.nodesSize only contains the useful data in the cache, but when reporting
- // the total memory consumption, the maintenance metadata is also needed to be
- // counted. For every useful node, we track 2 extra hashes as the flushlist.
- size := db.nodesSize + common.StorageSize((len(db.nodes)-1)*2*common.HashLength)
-
- // If the preimage cache got large enough, push to disk. If it's still small
- // leave for later to deduplicate writes.
- flushPreimages := db.preimagesSize > 4*1024*1024
- if flushPreimages {
- for hash, preimage := range db.preimages {
- if err := batch.Put(db.secureKey(hash[:]), preimage); err != nil {
- log.Error("Failed to commit preimage from trie database", "err", err)
- db.lock.RUnlock()
- return err
- }
- if batch.ValueSize() > hdb.IdealBatchSize {
- if err := batch.Write(); err != nil {
- db.lock.RUnlock()
- return err
- }
- batch.Reset()
- }
- }
- }
- // Keep committing nodes from the flush-list until we're below allowance
- oldest := db.oldest
- for size > limit && oldest != (common.Hash{}) {
- // Fetch the oldest referenced node and push into the batch
- node := db.nodes[oldest]
- if err := batch.Put(oldest[:], node.rlp()); err != nil {
- db.lock.RUnlock()
- return err
- }
- // If we exceeded the ideal batch size, commit and reset
- if batch.ValueSize() >= hdb.IdealBatchSize {
- if err := batch.Write(); err != nil {
- log.Error("Failed to write flush list to disk", "err", err)
- db.lock.RUnlock()
- return err
- }
- batch.Reset()
- }
- // Iterate to the next flush item, or abort if the size cap was achieved. Size
- // is the total size, including both the useful cached data (hash -> blob), as
- // well as the flushlist metadata (2*hash). When flushing items from the cache,
- // we need to reduce both.
- size -= common.StorageSize(3*common.HashLength + int(node.size))
- oldest = node.flushNext
- }
- // Flush out any remainder data from the last batch
- if err := batch.Write(); err != nil {
- log.Error("Failed to write flush list to disk", "err", err)
- db.lock.RUnlock()
- return err
- }
- db.lock.RUnlock()
-
- // Write successful, clear out the flushed data
- db.lock.Lock()
- defer db.lock.Unlock()
-
- if flushPreimages {
- db.preimages = make(map[common.Hash][]byte)
- db.preimagesSize = 0
- }
- for db.oldest != oldest {
- node := db.nodes[db.oldest]
- delete(db.nodes, db.oldest)
- db.oldest = node.flushNext
-
- db.nodesSize -= common.StorageSize(common.HashLength + int(node.size))
- }
- if db.oldest != (common.Hash{}) {
- db.nodes[db.oldest].flushPrev = common.Hash{}
- }
- db.flushnodes += uint64(nodes - len(db.nodes))
- db.flushsize += storage - db.nodesSize
- db.flushtime += time.Since(start)
-
- memcacheFlushTimeTimer.Update(time.Since(start))
- memcacheFlushSizeMeter.Mark(int64(storage - db.nodesSize))
- memcacheFlushNodesMeter.Mark(int64(nodes - len(db.nodes)))
-
- log.Debug("Persisted nodes from memory database", "nodes", nodes-len(db.nodes), "size", storage-db.nodesSize, "time", time.Since(start),
- "flushnodes", db.flushnodes, "flushsize", db.flushsize, "flushtime", db.flushtime, "livenodes", len(db.nodes), "livesize", db.nodesSize)
-
- return nil
-}
-
-// Commit iterates over all the children of a particular node, writes them out
-// to disk, forcefully tearing down all references in both directions.
-//
-// As a side effect, all pre-images accumulated up to this point are also written.
-func (db *Database) Commit(node common.Hash, report bool) error {
- // Create a database batch to flush persistent data out. It is important that
- // outside code doesn't see an inconsistent state (referenced data removed from
- // memory cache during commit but not yet in persistent storage). This is ensured
- // by only uncaching existing data when the database write finalizes.
- db.lock.RLock()
-
- start := time.Now()
- batch := db.diskdb.NewBatch()
-
- // Move all of the accumulated preimages into a write batch
- for hash, preimage := range db.preimages {
- if err := batch.Put(db.secureKey(hash[:]), preimage); err != nil {
- log.Error("Failed to commit preimage from trie database", "err", err)
- db.lock.RUnlock()
- return err
- }
- if batch.ValueSize() > hdb.IdealBatchSize {
- if err := batch.Write(); err != nil {
- return err
- }
- batch.Reset()
- }
- }
- // Move the trie itself into the batch, flushing if enough data is accumulated
- nodes, storage := len(db.nodes), db.nodesSize
- if err := db.commit(node, batch); err != nil {
- log.Error("Failed to commit trie from trie database", "err", err)
- db.lock.RUnlock()
- return err
- }
- // Write batch ready, unlock for readers during persistence
- if err := batch.Write(); err != nil {
- log.Error("Failed to write trie to disk", "err", err)
- db.lock.RUnlock()
- return err
- }
- db.lock.RUnlock()
-
- // Write successful, clear out the flushed data
- db.lock.Lock()
- defer db.lock.Unlock()
-
- db.preimages = make(map[common.Hash][]byte)
- db.preimagesSize = 0
-
- db.uncache(node)
-
- memcacheCommitTimeTimer.Update(time.Since(start))
- memcacheCommitSizeMeter.Mark(int64(storage - db.nodesSize))
- memcacheCommitNodesMeter.Mark(int64(nodes - len(db.nodes)))
-
- logger := log.Info
- if !report {
- logger = log.Debug
- }
- logger("Persisted trie from memory database", "nodes", nodes-len(db.nodes)+int(db.flushnodes), "size", storage-db.nodesSize+db.flushsize, "time", time.Since(start)+db.flushtime,
- "gcnodes", db.gcnodes, "gcsize", db.gcsize, "gctime", db.gctime, "livenodes", len(db.nodes), "livesize", db.nodesSize)
-
- // Reset the garbage collection statistics
- db.gcnodes, db.gcsize, db.gctime = 0, 0, 0
- db.flushnodes, db.flushsize, db.flushtime = 0, 0, 0
-
- return nil
-}
-
-// commit is the private locked version of Commit.
-func (db *Database) commit(hash common.Hash, batch hdb.Batch) error {
- // If the node does not exist, it's a previously committed node
- node, ok := db.nodes[hash]
- if !ok {
- return nil
- }
- for _, child := range node.childs() {
- if err := db.commit(child, batch); err != nil {
- return err
- }
- }
- if err := batch.Put(hash[:], node.rlp()); err != nil {
- return err
- }
- // If we've reached an optimal batch size, commit and start over
- if batch.ValueSize() >= hdb.IdealBatchSize {
- if err := batch.Write(); err != nil {
- return err
- }
- batch.Reset()
- }
- return nil
-}
-
-// uncache is the post-processing step of a commit operation where the already
-// persisted trie is removed from the cache. The reason behind the two-phase
-// commit is to ensure consistent data availability while moving from memory
-// to disk.
-func (db *Database) uncache(hash common.Hash) {
- // If the node does not exist, we're done on this path
- node, ok := db.nodes[hash]
- if !ok {
- return
- }
- // Node still exists, remove it from the flush-list
- switch hash {
- case db.oldest:
- db.oldest = node.flushNext
- db.nodes[node.flushNext].flushPrev = common.Hash{}
- case db.newest:
- db.newest = node.flushPrev
- db.nodes[node.flushPrev].flushNext = common.Hash{}
- default:
- db.nodes[node.flushPrev].flushNext = node.flushNext
- db.nodes[node.flushNext].flushPrev = node.flushPrev
- }
- // Uncache the node's subtries and remove the node itself too
- for _, child := range node.childs() {
- db.uncache(child)
- }
- delete(db.nodes, hash)
- db.nodesSize -= common.StorageSize(common.HashLength + int(node.size))
-}
-
-// Size returns the current storage size of the memory cache in front of the
-// persistent database layer.
-func (db *Database) Size() (common.StorageSize, common.StorageSize) {
- db.lock.RLock()
- defer db.lock.RUnlock()
-
- // db.nodesSize only contains the useful data in the cache, but when reporting
- // the total memory consumption, the maintenance metadata is also needed to be
- // counted. For every useful node, we track 2 extra hashes as the flushlist.
- var flushlistSize = common.StorageSize((len(db.nodes) - 1) * 2 * common.HashLength)
- return db.nodesSize + flushlistSize, db.preimagesSize
-}
-
-// verifyIntegrity is a debug method to iterate over the entire trie stored in
-// memory and check whether every node is reachable from the meta root. The goal
-// is to find any errors that might cause memory leaks and or trie nodes to go
-// missing.
-//
-// This method is extremely CPU and memory intensive, only use when must.
-func (db *Database) verifyIntegrity() {
- // Iterate over all the cached nodes and accumulate them into a set
- reachable := map[common.Hash]struct{}{{}: {}}
-
- for child := range db.nodes[common.Hash{}].children {
- db.accumulate(child, reachable)
- }
- // Find any unreachable but cached nodes
- unreachable := []string{}
- for hash, node := range db.nodes {
- if _, ok := reachable[hash]; !ok {
- unreachable = append(unreachable, fmt.Sprintf("%x: {Node: %v, Parents: %d, Prev: %x, Next: %x}",
- hash, node.node, node.parents, node.flushPrev, node.flushNext))
- }
- }
- if len(unreachable) != 0 {
- panic(fmt.Sprintf("trie cache memory leak: %v", unreachable))
- }
-}
-
-// accumulate iterates over the trie defined by hash and accumulates all the
-// cached children found in memory.
-func (db *Database) accumulate(hash common.Hash, reachable map[common.Hash]struct{}) {
- // Mark the node reachable if present in the memory cache
- node, ok := db.nodes[hash]
- if !ok {
- return
- }
- reachable[hash] = struct{}{}
-
- // Iterate over all the children and accumulate them too
- for _, child := range node.childs() {
- db.accumulate(child, reachable)
- }
-}
diff --git a/internal/trie/encoding.go b/internal/trie/encoding.go
deleted file mode 100644
index 5f120de63..000000000
--- a/internal/trie/encoding.go
+++ /dev/null
@@ -1,113 +0,0 @@
-// Copyright 2014 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see .
-
-package trie
-
-// Trie keys are dealt with in three distinct encodings:
-//
-// KEYBYTES encoding contains the actual key and nothing else. This encoding is the
-// input to most API functions.
-//
-// HEX encoding contains one byte for each nibble of the key and an optional trailing
-// 'terminator' byte of value 0x10 which indicates whether or not the node at the key
-// contains a value. Hex key encoding is used for nodes loaded in memory because it's
-// convenient to access.
-//
-// COMPACT encoding is defined by the Ethereum Yellow Paper (it's called "hex prefix
-// encoding" there) and contains the bytes of the key and a flag. The high nibble of the
-// first byte contains the flag; the lowest bit encoding the oddness of the length and
-// the second-lowest encoding whether the node at the key is a value node. The low nibble
-// of the first byte is zero in the case of an even number of nibbles and the first nibble
-// in the case of an odd number. All remaining nibbles (now an even number) fit properly
-// into the remaining bytes. Compact encoding is used for nodes stored on disk.
-
-func hexToCompact(hex []byte) []byte {
- terminator := byte(0)
- if hasTerm(hex) {
- terminator = 1
- hex = hex[:len(hex)-1]
- }
- buf := make([]byte, len(hex)/2+1)
- buf[0] = terminator << 5 // the flag byte
- if len(hex)&1 == 1 {
- buf[0] |= 1 << 4 // odd flag
- buf[0] |= hex[0] // first nibble is contained in the first byte
- hex = hex[1:]
- }
- decodeNibbles(hex, buf[1:])
- return buf
-}
-
-func compactToHex(compact []byte) []byte {
- base := keybytesToHex(compact)
- // delete terminator flag
- if base[0] < 2 {
- base = base[:len(base)-1]
- }
- // apply odd flag
- chop := 2 - base[0]&1
- return base[chop:]
-}
-
-func keybytesToHex(str []byte) []byte {
- l := len(str)*2 + 1
- var nibbles = make([]byte, l)
- for i, b := range str {
- nibbles[i*2] = b / 16
- nibbles[i*2+1] = b % 16
- }
- nibbles[l-1] = 16
- return nibbles
-}
-
-// hexToKeybytes turns hex nibbles into key bytes.
-// This can only be used for keys of even length.
-func hexToKeybytes(hex []byte) []byte {
- if hasTerm(hex) {
- hex = hex[:len(hex)-1]
- }
- if len(hex)&1 != 0 {
- panic("can't convert hex key of odd length")
- }
- key := make([]byte, len(hex)/2)
- decodeNibbles(hex, key)
- return key
-}
-
-func decodeNibbles(nibbles []byte, bytes []byte) {
- for bi, ni := 0, 0; ni < len(nibbles); bi, ni = bi+1, ni+2 {
- bytes[bi] = nibbles[ni]<<4 | nibbles[ni+1]
- }
-}
-
-// prefixLen returns the length of the common prefix of a and b.
-func prefixLen(a, b []byte) int {
- var i, length = 0, len(a)
- if len(b) < length {
- length = len(b)
- }
- for ; i < length; i++ {
- if a[i] != b[i] {
- break
- }
- }
- return i
-}
-
-// hasTerm returns whether a hex key has the terminator flag.
-func hasTerm(s []byte) bool {
- return len(s) > 0 && s[len(s)-1] == 16
-}
diff --git a/internal/trie/encoding_test.go b/internal/trie/encoding_test.go
deleted file mode 100644
index 97d8da136..000000000
--- a/internal/trie/encoding_test.go
+++ /dev/null
@@ -1,104 +0,0 @@
-// Copyright 2014 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see .
-
-package trie
-
-import (
- "bytes"
- "testing"
-)
-
-func TestHexCompact(t *testing.T) {
- tests := []struct{ hex, compact []byte }{
- // empty keys, with and without terminator.
- {hex: []byte{}, compact: []byte{0x00}},
- {hex: []byte{16}, compact: []byte{0x20}},
- // odd length, no terminator
- {hex: []byte{1, 2, 3, 4, 5}, compact: []byte{0x11, 0x23, 0x45}},
- // even length, no terminator
- {hex: []byte{0, 1, 2, 3, 4, 5}, compact: []byte{0x00, 0x01, 0x23, 0x45}},
- // odd length, terminator
- {hex: []byte{15, 1, 12, 11, 8, 16 /*term*/}, compact: []byte{0x3f, 0x1c, 0xb8}},
- // even length, terminator
- {hex: []byte{0, 15, 1, 12, 11, 8, 16 /*term*/}, compact: []byte{0x20, 0x0f, 0x1c, 0xb8}},
- }
- for _, test := range tests {
- if c := hexToCompact(test.hex); !bytes.Equal(c, test.compact) {
- t.Errorf("hexToCompact(%x) -> %x, want %x", test.hex, c, test.compact)
- }
- if h := compactToHex(test.compact); !bytes.Equal(h, test.hex) {
- t.Errorf("compactToHex(%x) -> %x, want %x", test.compact, h, test.hex)
- }
- }
-}
-
-func TestHexKeybytes(t *testing.T) {
- tests := []struct{ key, hexIn, hexOut []byte }{
- {key: []byte{}, hexIn: []byte{16}, hexOut: []byte{16}},
- {key: []byte{}, hexIn: []byte{}, hexOut: []byte{16}},
- {
- key: []byte{0x12, 0x34, 0x56},
- hexIn: []byte{1, 2, 3, 4, 5, 6, 16},
- hexOut: []byte{1, 2, 3, 4, 5, 6, 16},
- },
- {
- key: []byte{0x12, 0x34, 0x5},
- hexIn: []byte{1, 2, 3, 4, 0, 5, 16},
- hexOut: []byte{1, 2, 3, 4, 0, 5, 16},
- },
- {
- key: []byte{0x12, 0x34, 0x56},
- hexIn: []byte{1, 2, 3, 4, 5, 6},
- hexOut: []byte{1, 2, 3, 4, 5, 6, 16},
- },
- }
- for _, test := range tests {
- if h := keybytesToHex(test.key); !bytes.Equal(h, test.hexOut) {
- t.Errorf("keybytesToHex(%x) -> %x, want %x", test.key, h, test.hexOut)
- }
- if k := hexToKeybytes(test.hexIn); !bytes.Equal(k, test.key) {
- t.Errorf("hexToKeybytes(%x) -> %x, want %x", test.hexIn, k, test.key)
- }
- }
-}
-
-func BenchmarkHexToCompact(b *testing.B) {
- testBytes := []byte{0, 15, 1, 12, 11, 8, 16 /*term*/}
- for i := 0; i < b.N; i++ {
- hexToCompact(testBytes)
- }
-}
-
-func BenchmarkCompactToHex(b *testing.B) {
- testBytes := []byte{0, 15, 1, 12, 11, 8, 16 /*term*/}
- for i := 0; i < b.N; i++ {
- compactToHex(testBytes)
- }
-}
-
-func BenchmarkKeybytesToHex(b *testing.B) {
- testBytes := []byte{7, 6, 6, 5, 7, 2, 6, 2, 16}
- for i := 0; i < b.N; i++ {
- keybytesToHex(testBytes)
- }
-}
-
-func BenchmarkHexToKeybytes(b *testing.B) {
- testBytes := []byte{7, 6, 6, 5, 7, 2, 6, 2, 16}
- for i := 0; i < b.N; i++ {
- hexToKeybytes(testBytes)
- }
-}
diff --git a/internal/trie/errors.go b/internal/trie/errors.go
deleted file mode 100644
index 567b80078..000000000
--- a/internal/trie/errors.go
+++ /dev/null
@@ -1,35 +0,0 @@
-// Copyright 2015 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see .
-
-package trie
-
-import (
- "fmt"
-
- "github.com/ethereum/go-ethereum/common"
-)
-
-// MissingNodeError is returned by the trie functions (TryGet, TryUpdate, TryDelete)
-// in the case where a trie node is not present in the local database. It contains
-// information necessary for retrieving the missing node.
-type MissingNodeError struct {
- NodeHash common.Hash // hash of the missing node
- Path []byte // hex-encoded path to the missing node
-}
-
-func (err *MissingNodeError) Error() string {
- return fmt.Sprintf("missing trie node %x (path %x)", err.NodeHash, err.Path)
-}
diff --git a/internal/trie/hasher.go b/internal/trie/hasher.go
deleted file mode 100644
index 9d6756b6f..000000000
--- a/internal/trie/hasher.go
+++ /dev/null
@@ -1,218 +0,0 @@
-// Copyright 2016 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see .
-
-package trie
-
-import (
- "hash"
- "sync"
-
- "github.com/ethereum/go-ethereum/common"
- "github.com/ethereum/go-ethereum/rlp"
- "golang.org/x/crypto/sha3"
-)
-
-type hasher struct {
- tmp sliceBuffer
- sha keccakState
- cachegen uint16
- cachelimit uint16
- onleaf LeafCallback
-}
-
-// keccakState wraps sha3.state. In addition to the usual hash methods, it also supports
-// Read to get a variable amount of data from the hash state. Read is faster than Sum
-// because it doesn't copy the internal state, but also modifies the internal state.
-type keccakState interface {
- hash.Hash
- Read([]byte) (int, error)
-}
-
-type sliceBuffer []byte
-
-func (b *sliceBuffer) Write(data []byte) (n int, err error) {
- *b = append(*b, data...)
- return len(data), nil
-}
-
-func (b *sliceBuffer) Reset() {
- *b = (*b)[:0]
-}
-
-// hashers live in a global db.
-var hasherPool = sync.Pool{
- New: func() interface{} {
- return &hasher{
- tmp: make(sliceBuffer, 0, 550), // cap is as large as a full fullNode.
- sha: sha3.NewLegacyKeccak256().(keccakState),
- }
- },
-}
-
-func newHasher(cachegen, cachelimit uint16, onleaf LeafCallback) *hasher {
- h := hasherPool.Get().(*hasher)
- h.cachegen, h.cachelimit, h.onleaf = cachegen, cachelimit, onleaf
- return h
-}
-
-func returnHasherToPool(h *hasher) {
- hasherPool.Put(h)
-}
-
-// hash collapses a node down into a hash node, also returning a copy of the
-// original node initialized with the computed hash to replace the original one.
-func (h *hasher) hash(n node, db *Database, force bool) (node, node, error) {
- // If we're not storing the node, just hashing, use available cached data
- if hash, dirty := n.cache(); hash != nil {
- if db == nil {
- return hash, n, nil
- }
- if n.canUnload(h.cachegen, h.cachelimit) {
- // Unload the node from cache. All of its subnodes will have a lower or equal
- // cache generation number.
- cacheUnloadCounter.Inc(1)
- return hash, hash, nil
- }
- if !dirty {
- return hash, n, nil
- }
- }
- // Trie not processed yet or needs storage, walk the children
- collapsed, cached, err := h.hashChildren(n, db)
- if err != nil {
- return hashNode{}, n, err
- }
- hashed, err := h.store(collapsed, db, force)
- if err != nil {
- return hashNode{}, n, err
- }
- // Cache the hash of the node for later reuse and remove
- // the dirty flag in commit mode. It's fine to assign these values directly
- // without copying the node first because hashChildren copies it.
- cachedHash, _ := hashed.(hashNode)
- switch cn := cached.(type) {
- case *shortNode:
- cn.flags.hash = cachedHash
- if db != nil {
- cn.flags.dirty = false
- }
- case *fullNode:
- cn.flags.hash = cachedHash
- if db != nil {
- cn.flags.dirty = false
- }
- }
- return hashed, cached, nil
-}
-
-// hashChildren replaces the children of a node with their hashes if the encoded
-// size of the child is larger than a hash, returning the collapsed node as well
-// as a replacement for the original node with the child hashes cached in.
-func (h *hasher) hashChildren(original node, db *Database) (node, node, error) {
- var err error
-
- switch n := original.(type) {
- case *shortNode:
- // Hash the short node's child, caching the newly hashed subtree
- collapsed, cached := n.copy(), n.copy()
- collapsed.Key = hexToCompact(n.Key)
- cached.Key = common.CopyBytes(n.Key)
-
- if _, ok := n.Val.(valueNode); !ok {
- collapsed.Val, cached.Val, err = h.hash(n.Val, db, false)
- if err != nil {
- return original, original, err
- }
- }
- return collapsed, cached, nil
-
- case *fullNode:
- // Hash the full node's children, caching the newly hashed subtrees
- collapsed, cached := n.copy(), n.copy()
-
- for i := 0; i < 16; i++ {
- if n.Children[i] != nil {
- collapsed.Children[i], cached.Children[i], err = h.hash(n.Children[i], db, false)
- if err != nil {
- return original, original, err
- }
- }
- }
- cached.Children[16] = n.Children[16]
- return collapsed, cached, nil
-
- default:
- // Value and hash nodes don't have children so they're left as were
- return n, original, nil
- }
-}
-
-// store hashes the node n and if we have a storage layer specified, it writes
-// the key/value pair to it and tracks any node->child references as well as any
-// node->external trie references.
-func (h *hasher) store(n node, db *Database, force bool) (node, error) {
- // Don't store hashes or empty nodes.
- if _, isHash := n.(hashNode); n == nil || isHash {
- return n, nil
- }
- // Generate the RLP encoding of the node
- h.tmp.Reset()
- if err := rlp.Encode(&h.tmp, n); err != nil {
- panic("encode error: " + err.Error())
- }
- if len(h.tmp) < 32 && !force {
- return n, nil // Nodes smaller than 32 bytes are stored inside their parent
- }
- // Larger nodes are replaced by their hash and stored in the database.
- hash, _ := n.cache()
- if hash == nil {
- hash = h.makeHashNode(h.tmp)
- }
-
- if db != nil {
- // We are pooling the trie nodes into an intermediate memory cache
- hash := common.BytesToHash(hash)
-
- db.lock.Lock()
- db.insert(hash, h.tmp, n)
- db.lock.Unlock()
-
- // Track external references from account->storage trie
- if h.onleaf != nil {
- switch n := n.(type) {
- case *shortNode:
- if child, ok := n.Val.(valueNode); ok {
- h.onleaf(child, hash)
- }
- case *fullNode:
- for i := 0; i < 16; i++ {
- if child, ok := n.Children[i].(valueNode); ok {
- h.onleaf(child, hash)
- }
- }
- }
- }
- }
- return hash, nil
-}
-
-func (h *hasher) makeHashNode(data []byte) hashNode {
- n := make(hashNode, h.sha.Size())
- h.sha.Reset()
- h.sha.Write(data)
- h.sha.Read(n)
- return n
-}
diff --git a/internal/trie/iterator.go b/internal/trie/iterator.go
deleted file mode 100644
index 00b890eb8..000000000
--- a/internal/trie/iterator.go
+++ /dev/null
@@ -1,575 +0,0 @@
-// Copyright 2014 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see .
-
-package trie
-
-import (
- "bytes"
- "container/heap"
- "errors"
-
- "github.com/ethereum/go-ethereum/common"
- "github.com/ethereum/go-ethereum/rlp"
-)
-
-// Iterator is a key-value trie iterator that traverses a Trie.
-type Iterator struct {
- nodeIt NodeIterator
-
- Key []byte // Current data key on which the iterator is positioned on
- Value []byte // Current data value on which the iterator is positioned on
- Err error
-}
-
-// NewIterator creates a new key-value iterator from a node iterator
-func NewIterator(it NodeIterator) *Iterator {
- return &Iterator{
- nodeIt: it,
- }
-}
-
-// Next moves the iterator forward one key-value entry.
-func (it *Iterator) Next() bool {
- for it.nodeIt.Next(true) {
- if it.nodeIt.Leaf() {
- it.Key = it.nodeIt.LeafKey()
- it.Value = it.nodeIt.LeafBlob()
- return true
- }
- }
- it.Key = nil
- it.Value = nil
- it.Err = it.nodeIt.Error()
- return false
-}
-
-// Prove generates the Merkle proof for the leaf node the iterator is currently
-// positioned on.
-func (it *Iterator) Prove() [][]byte {
- return it.nodeIt.LeafProof()
-}
-
-// NodeIterator is an iterator to traverse the trie pre-order.
-type NodeIterator interface {
- // Next moves the iterator to the next node. If the parameter is false, any child
- // nodes will be skipped.
- Next(bool) bool
-
- // Error returns the error status of the iterator.
- Error() error
-
- // Hash returns the hash of the current node.
- Hash() common.Hash
-
- // Parent returns the hash of the parent of the current node. The hash may be the one
- // grandparent if the immediate parent is an internal node with no hash.
- Parent() common.Hash
-
- // Path returns the hex-encoded path to the current node.
- // Callers must not retain references to the return value after calling Next.
- // For leaf nodes, the last element of the path is the 'terminator symbol' 0x10.
- Path() []byte
-
- // Leaf returns true iff the current node is a leaf node.
- Leaf() bool
-
- // LeafKey returns the key of the leaf. The method panics if the iterator is not
- // positioned at a leaf. Callers must not retain references to the value after
- // calling Next.
- LeafKey() []byte
-
- // LeafBlob returns the content of the leaf. The method panics if the iterator
- // is not positioned at a leaf. Callers must not retain references to the value
- // after calling Next.
- LeafBlob() []byte
-
- // LeafProof returns the Merkle proof of the leaf. The method panics if the
- // iterator is not positioned at a leaf. Callers must not retain references
- // to the value after calling Next.
- LeafProof() [][]byte
-}
-
-// nodeIteratorState represents the iteration state at one particular node of the
-// trie, which can be resumed at a later invocation.
-type nodeIteratorState struct {
- hash common.Hash // Hash of the node being iterated (nil if not standalone)
- node node // Trie node being iterated
- parent common.Hash // Hash of the first full ancestor node (nil if current is the root)
- index int // Child to be processed next
- pathlen int // Length of the path to this node
-}
-
-type nodeIterator struct {
- trie *Trie // Trie being iterated
- stack []*nodeIteratorState // Hierarchy of trie nodes persisting the iteration state
- path []byte // Path to the current node
- err error // Failure set in case of an internal error in the iterator
-}
-
-// errIteratorEnd is stored in nodeIterator.err when iteration is done.
-var errIteratorEnd = errors.New("end of iteration")
-
-// seekError is stored in nodeIterator.err if the initial seek has failed.
-type seekError struct {
- key []byte
- err error
-}
-
-func (e seekError) Error() string {
- return "seek error: " + e.err.Error()
-}
-
-func newNodeIterator(trie *Trie, start []byte) NodeIterator {
- if trie.Hash() == emptyState {
- return new(nodeIterator)
- }
- it := &nodeIterator{trie: trie}
- it.err = it.seek(start)
- return it
-}
-
-func (it *nodeIterator) Hash() common.Hash {
- if len(it.stack) == 0 {
- return common.Hash{}
- }
- return it.stack[len(it.stack)-1].hash
-}
-
-func (it *nodeIterator) Parent() common.Hash {
- if len(it.stack) == 0 {
- return common.Hash{}
- }
- return it.stack[len(it.stack)-1].parent
-}
-
-func (it *nodeIterator) Leaf() bool {
- return hasTerm(it.path)
-}
-
-func (it *nodeIterator) LeafKey() []byte {
- if len(it.stack) > 0 {
- if _, ok := it.stack[len(it.stack)-1].node.(valueNode); ok {
- return hexToKeybytes(it.path)
- }
- }
- panic("not at leaf")
-}
-
-func (it *nodeIterator) LeafBlob() []byte {
- if len(it.stack) > 0 {
- if node, ok := it.stack[len(it.stack)-1].node.(valueNode); ok {
- return []byte(node)
- }
- }
- panic("not at leaf")
-}
-
-func (it *nodeIterator) LeafProof() [][]byte {
- if len(it.stack) > 0 {
- if _, ok := it.stack[len(it.stack)-1].node.(valueNode); ok {
- hasher := newHasher(0, 0, nil)
- proofs := make([][]byte, 0, len(it.stack))
-
- for i, item := range it.stack[:len(it.stack)-1] {
- // Gather nodes that end up as hash nodes (or the root)
- node, _, _ := hasher.hashChildren(item.node, nil)
- hashed, _ := hasher.store(node, nil, false)
- if _, ok := hashed.(hashNode); ok || i == 0 {
- enc, _ := rlp.EncodeToBytes(node)
- proofs = append(proofs, enc)
- }
- }
- return proofs
- }
- }
- panic("not at leaf")
-}
-
-func (it *nodeIterator) Path() []byte {
- return it.path
-}
-
-func (it *nodeIterator) Error() error {
- if it.err == errIteratorEnd {
- return nil
- }
- if seek, ok := it.err.(seekError); ok {
- return seek.err
- }
- return it.err
-}
-
-// Next moves the iterator to the next node, returning whether there are any
-// further nodes. In case of an internal error this method returns false and
-// sets the Error field to the encountered failure. If `descend` is false,
-// skips iterating over any subnodes of the current node.
-func (it *nodeIterator) Next(descend bool) bool {
- if it.err == errIteratorEnd {
- return false
- }
- if seek, ok := it.err.(seekError); ok {
- if it.err = it.seek(seek.key); it.err != nil {
- return false
- }
- }
- // Otherwise step forward with the iterator and report any errors.
- state, parentIndex, path, err := it.peek(descend)
- it.err = err
- if it.err != nil {
- return false
- }
- it.push(state, parentIndex, path)
- return true
-}
-
-func (it *nodeIterator) seek(prefix []byte) error {
- // The path we're looking for is the hex encoded key without terminator.
- key := keybytesToHex(prefix)
- key = key[:len(key)-1]
- // Move forward until we're just before the closest match to key.
- for {
- state, parentIndex, path, err := it.peek(bytes.HasPrefix(key, it.path))
- if err == errIteratorEnd {
- return errIteratorEnd
- } else if err != nil {
- return seekError{prefix, err}
- } else if bytes.Compare(path, key) >= 0 {
- return nil
- }
- it.push(state, parentIndex, path)
- }
-}
-
-// peek creates the next state of the iterator.
-func (it *nodeIterator) peek(descend bool) (*nodeIteratorState, *int, []byte, error) {
- if len(it.stack) == 0 {
- // Initialize the iterator if we've just started.
- root := it.trie.Hash()
- state := &nodeIteratorState{node: it.trie.root, index: -1}
- if root != emptyRoot {
- state.hash = root
- }
- err := state.resolve(it.trie, nil)
- return state, nil, nil, err
- }
- if !descend {
- // If we're skipping children, pop the current node first
- it.pop()
- }
-
- // Continue iteration to the next child
- for len(it.stack) > 0 {
- parent := it.stack[len(it.stack)-1]
- ancestor := parent.hash
- if (ancestor == common.Hash{}) {
- ancestor = parent.parent
- }
- state, path, ok := it.nextChild(parent, ancestor)
- if ok {
- if err := state.resolve(it.trie, path); err != nil {
- return parent, &parent.index, path, err
- }
- return state, &parent.index, path, nil
- }
- // No more child nodes, move back up.
- it.pop()
- }
- return nil, nil, nil, errIteratorEnd
-}
-
-func (st *nodeIteratorState) resolve(tr *Trie, path []byte) error {
- if hash, ok := st.node.(hashNode); ok {
- resolved, err := tr.resolveHash(hash, path)
- if err != nil {
- return err
- }
- st.node = resolved
- st.hash = common.BytesToHash(hash)
- }
- return nil
-}
-
-func (it *nodeIterator) nextChild(parent *nodeIteratorState, ancestor common.Hash) (*nodeIteratorState, []byte, bool) {
- switch node := parent.node.(type) {
- case *fullNode:
- // Full node, move to the first non-nil child.
- for i := parent.index + 1; i < len(node.Children); i++ {
- child := node.Children[i]
- if child != nil {
- hash, _ := child.cache()
- state := &nodeIteratorState{
- hash: common.BytesToHash(hash),
- node: child,
- parent: ancestor,
- index: -1,
- pathlen: len(it.path),
- }
- path := append(it.path, byte(i))
- parent.index = i - 1
- return state, path, true
- }
- }
- case *shortNode:
- // Short node, return the pointer singleton child
- if parent.index < 0 {
- hash, _ := node.Val.cache()
- state := &nodeIteratorState{
- hash: common.BytesToHash(hash),
- node: node.Val,
- parent: ancestor,
- index: -1,
- pathlen: len(it.path),
- }
- path := append(it.path, node.Key...)
- return state, path, true
- }
- }
- return parent, it.path, false
-}
-
-func (it *nodeIterator) push(state *nodeIteratorState, parentIndex *int, path []byte) {
- it.path = path
- it.stack = append(it.stack, state)
- if parentIndex != nil {
- *parentIndex++
- }
-}
-
-func (it *nodeIterator) pop() {
- parent := it.stack[len(it.stack)-1]
- it.path = it.path[:parent.pathlen]
- it.stack = it.stack[:len(it.stack)-1]
-}
-
-func compareNodes(a, b NodeIterator) int {
- if cmp := bytes.Compare(a.Path(), b.Path()); cmp != 0 {
- return cmp
- }
- if a.Leaf() && !b.Leaf() {
- return -1
- } else if b.Leaf() && !a.Leaf() {
- return 1
- }
- if cmp := bytes.Compare(a.Hash().Bytes(), b.Hash().Bytes()); cmp != 0 {
- return cmp
- }
- if a.Leaf() && b.Leaf() {
- return bytes.Compare(a.LeafBlob(), b.LeafBlob())
- }
- return 0
-}
-
-type differenceIterator struct {
- a, b NodeIterator // Nodes returned are those in b - a.
- eof bool // Indicates a has run out of elements
- count int // Number of nodes scanned on either trie
-}
-
-// NewDifferenceIterator constructs a NodeIterator that iterates over elements in b that
-// are not in a. Returns the iterator, and a pointer to an integer recording the number
-// of nodes seen.
-func NewDifferenceIterator(a, b NodeIterator) (NodeIterator, *int) {
- a.Next(true)
- it := &differenceIterator{
- a: a,
- b: b,
- }
- return it, &it.count
-}
-
-func (it *differenceIterator) Hash() common.Hash {
- return it.b.Hash()
-}
-
-func (it *differenceIterator) Parent() common.Hash {
- return it.b.Parent()
-}
-
-func (it *differenceIterator) Leaf() bool {
- return it.b.Leaf()
-}
-
-func (it *differenceIterator) LeafKey() []byte {
- return it.b.LeafKey()
-}
-
-func (it *differenceIterator) LeafBlob() []byte {
- return it.b.LeafBlob()
-}
-
-func (it *differenceIterator) LeafProof() [][]byte {
- return it.b.LeafProof()
-}
-
-func (it *differenceIterator) Path() []byte {
- return it.b.Path()
-}
-
-func (it *differenceIterator) Next(bool) bool {
- // Invariants:
- // - We always advance at least one element in b.
- // - At the start of this function, a's path is lexically greater than b's.
- if !it.b.Next(true) {
- return false
- }
- it.count++
-
- if it.eof {
- // a has reached eof, so we just return all elements from b
- return true
- }
-
- for {
- switch compareNodes(it.a, it.b) {
- case -1:
- // b jumped past a; advance a
- if !it.a.Next(true) {
- it.eof = true
- return true
- }
- it.count++
- case 1:
- // b is before a
- return true
- case 0:
- // a and b are identical; skip this whole subtree if the nodes have hashes
- hasHash := it.a.Hash() == common.Hash{}
- if !it.b.Next(hasHash) {
- return false
- }
- it.count++
- if !it.a.Next(hasHash) {
- it.eof = true
- return true
- }
- it.count++
- }
- }
-}
-
-func (it *differenceIterator) Error() error {
- if err := it.a.Error(); err != nil {
- return err
- }
- return it.b.Error()
-}
-
-type nodeIteratorHeap []NodeIterator
-
-func (h nodeIteratorHeap) Len() int { return len(h) }
-func (h nodeIteratorHeap) Less(i, j int) bool { return compareNodes(h[i], h[j]) < 0 }
-func (h nodeIteratorHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] }
-func (h *nodeIteratorHeap) Push(x interface{}) { *h = append(*h, x.(NodeIterator)) }
-func (h *nodeIteratorHeap) Pop() interface{} {
- n := len(*h)
- x := (*h)[n-1]
- *h = (*h)[0 : n-1]
- return x
-}
-
-type unionIterator struct {
- items *nodeIteratorHeap // Nodes returned are the union of the ones in these iterators
- count int // Number of nodes scanned across all tries
-}
-
-// NewUnionIterator constructs a NodeIterator that iterates over elements in the union
-// of the provided NodeIterators. Returns the iterator, and a pointer to an integer
-// recording the number of nodes visited.
-func NewUnionIterator(iters []NodeIterator) (NodeIterator, *int) {
- h := make(nodeIteratorHeap, len(iters))
- copy(h, iters)
- heap.Init(&h)
-
- ui := &unionIterator{items: &h}
- return ui, &ui.count
-}
-
-func (it *unionIterator) Hash() common.Hash {
- return (*it.items)[0].Hash()
-}
-
-func (it *unionIterator) Parent() common.Hash {
- return (*it.items)[0].Parent()
-}
-
-func (it *unionIterator) Leaf() bool {
- return (*it.items)[0].Leaf()
-}
-
-func (it *unionIterator) LeafKey() []byte {
- return (*it.items)[0].LeafKey()
-}
-
-func (it *unionIterator) LeafBlob() []byte {
- return (*it.items)[0].LeafBlob()
-}
-
-func (it *unionIterator) LeafProof() [][]byte {
- return (*it.items)[0].LeafProof()
-}
-
-func (it *unionIterator) Path() []byte {
- return (*it.items)[0].Path()
-}
-
-// Next returns the next node in the union of tries being iterated over.
-//
-// It does this by maintaining a heap of iterators, sorted by the iteration
-// order of their next elements, with one entry for each source trie. Each
-// time Next() is called, it takes the least element from the heap to return,
-// advancing any other iterators that also point to that same element. These
-// iterators are called with descend=false, since we know that any nodes under
-// these nodes will also be duplicates, found in the currently selected iterator.
-// Whenever an iterator is advanced, it is pushed back into the heap if it still
-// has elements remaining.
-//
-// In the case that descend=false - eg, we're asked to ignore all subnodes of the
-// current node - we also advance any iterators in the heap that have the current
-// path as a prefix.
-func (it *unionIterator) Next(descend bool) bool {
- if len(*it.items) == 0 {
- return false
- }
-
- // Get the next key from the union
- least := heap.Pop(it.items).(NodeIterator)
-
- // Skip over other nodes as long as they're identical, or, if we're not descending, as
- // long as they have the same prefix as the current node.
- for len(*it.items) > 0 && ((!descend && bytes.HasPrefix((*it.items)[0].Path(), least.Path())) || compareNodes(least, (*it.items)[0]) == 0) {
- skipped := heap.Pop(it.items).(NodeIterator)
- // Skip the whole subtree if the nodes have hashes; otherwise just skip this node
- if skipped.Next(skipped.Hash() == common.Hash{}) {
- it.count++
- // If there are more elements, push the iterator back on the heap
- heap.Push(it.items, skipped)
- }
- }
- if least.Next(descend) {
- it.count++
- heap.Push(it.items, least)
- }
- return len(*it.items) > 0
-}
-
-func (it *unionIterator) Error() error {
- for i := 0; i < len(*it.items); i++ {
- if err := (*it.items)[i].Error(); err != nil {
- return err
- }
- }
- return nil
-}
diff --git a/internal/trie/iterator_test.go b/internal/trie/iterator_test.go
deleted file mode 100644
index f54127efd..000000000
--- a/internal/trie/iterator_test.go
+++ /dev/null
@@ -1,435 +0,0 @@
-// Copyright 2014 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see .
-
-package trie
-
-import (
- "bytes"
- "fmt"
- "math/rand"
- "testing"
-
- "github.com/ethereum/go-ethereum/common"
- hdb "github.com/harmony-one/harmony/internal/db"
-)
-
-func TestIterator(t *testing.T) {
- trie := newEmpty()
- vals := []struct{ k, v string }{
- {"do", "verb"},
- {"ether", "wookiedoo"},
- {"horse", "stallion"},
- {"shaman", "horse"},
- {"doge", "coin"},
- {"dog", "puppy"},
- {"somethingveryoddindeedthis is", "myothernodedata"},
- }
- all := make(map[string]string)
- for _, val := range vals {
- all[val.k] = val.v
- trie.Update([]byte(val.k), []byte(val.v))
- }
- trie.Commit(nil)
-
- found := make(map[string]string)
- it := NewIterator(trie.NodeIterator(nil))
- for it.Next() {
- found[string(it.Key)] = string(it.Value)
- }
-
- for k, v := range all {
- if found[k] != v {
- t.Errorf("iterator value mismatch for %s: got %q want %q", k, found[k], v)
- }
- }
-}
-
-type kv struct {
- k, v []byte
- t bool
-}
-
-func TestIteratorLargeData(t *testing.T) {
- trie := newEmpty()
- vals := make(map[string]*kv)
-
- for i := byte(0); i < 255; i++ {
- value := &kv{common.LeftPadBytes([]byte{i}, 32), []byte{i}, false}
- value2 := &kv{common.LeftPadBytes([]byte{10, i}, 32), []byte{i}, false}
- trie.Update(value.k, value.v)
- trie.Update(value2.k, value2.v)
- vals[string(value.k)] = value
- vals[string(value2.k)] = value2
- }
-
- it := NewIterator(trie.NodeIterator(nil))
- for it.Next() {
- vals[string(it.Key)].t = true
- }
-
- var untouched []*kv
- for _, value := range vals {
- if !value.t {
- untouched = append(untouched, value)
- }
- }
-
- if len(untouched) > 0 {
- t.Errorf("Missed %d nodes", len(untouched))
- for _, value := range untouched {
- t.Error(value)
- }
- }
-}
-
-// Tests that the node iterator indeed walks over the entire database contents.
-func TestNodeIteratorCoverage(t *testing.T) {
- // Create some arbitrary test trie to iterate
- db, trie, _ := makeTestTrie()
-
- // Gather all the node hashes found by the iterator
- hashes := make(map[common.Hash]struct{})
- for it := trie.NodeIterator(nil); it.Next(true); {
- if it.Hash() != (common.Hash{}) {
- hashes[it.Hash()] = struct{}{}
- }
- }
- // Cross check the hashes and the database itself
- for hash := range hashes {
- if _, err := db.Node(hash); err != nil {
- t.Errorf("failed to retrieve reported node %x: %v", hash, err)
- }
- }
- for hash, obj := range db.nodes {
- if obj != nil && hash != (common.Hash{}) {
- if _, ok := hashes[hash]; !ok {
- t.Errorf("state entry not reported %x", hash)
- }
- }
- }
- for _, key := range db.diskdb.(*hdb.MemDatabase).Keys() {
- if _, ok := hashes[common.BytesToHash(key)]; !ok {
- t.Errorf("state entry not reported %x", key)
- }
- }
-}
-
-type kvs struct{ k, v string }
-
-var testdata1 = []kvs{
- {"barb", "ba"},
- {"bard", "bc"},
- {"bars", "bb"},
- {"bar", "b"},
- {"fab", "z"},
- {"food", "ab"},
- {"foos", "aa"},
- {"foo", "a"},
-}
-
-var testdata2 = []kvs{
- {"aardvark", "c"},
- {"bar", "b"},
- {"barb", "bd"},
- {"bars", "be"},
- {"fab", "z"},
- {"foo", "a"},
- {"foos", "aa"},
- {"food", "ab"},
- {"jars", "d"},
-}
-
-func TestIteratorSeek(t *testing.T) {
- trie := newEmpty()
- for _, val := range testdata1 {
- trie.Update([]byte(val.k), []byte(val.v))
- }
-
- // Seek to the middle.
- it := NewIterator(trie.NodeIterator([]byte("fab")))
- if err := checkIteratorOrder(testdata1[4:], it); err != nil {
- t.Fatal(err)
- }
-
- // Seek to a non-existent key.
- it = NewIterator(trie.NodeIterator([]byte("barc")))
- if err := checkIteratorOrder(testdata1[1:], it); err != nil {
- t.Fatal(err)
- }
-
- // Seek beyond the end.
- it = NewIterator(trie.NodeIterator([]byte("z")))
- if err := checkIteratorOrder(nil, it); err != nil {
- t.Fatal(err)
- }
-}
-
-func checkIteratorOrder(want []kvs, it *Iterator) error {
- for it.Next() {
- if len(want) == 0 {
- return fmt.Errorf("didn't expect any more values, got key %q", it.Key)
- }
- if !bytes.Equal(it.Key, []byte(want[0].k)) {
- return fmt.Errorf("wrong key: got %q, want %q", it.Key, want[0].k)
- }
- want = want[1:]
- }
- if len(want) > 0 {
- return fmt.Errorf("iterator ended early, want key %q", want[0])
- }
- return nil
-}
-
-func TestDifferenceIterator(t *testing.T) {
- triea := newEmpty()
- for _, val := range testdata1 {
- triea.Update([]byte(val.k), []byte(val.v))
- }
- triea.Commit(nil)
-
- trieb := newEmpty()
- for _, val := range testdata2 {
- trieb.Update([]byte(val.k), []byte(val.v))
- }
- trieb.Commit(nil)
-
- found := make(map[string]string)
- di, _ := NewDifferenceIterator(triea.NodeIterator(nil), trieb.NodeIterator(nil))
- it := NewIterator(di)
- for it.Next() {
- found[string(it.Key)] = string(it.Value)
- }
-
- all := []struct{ k, v string }{
- {"aardvark", "c"},
- {"barb", "bd"},
- {"bars", "be"},
- {"jars", "d"},
- }
- for _, item := range all {
- if found[item.k] != item.v {
- t.Errorf("iterator value mismatch for %s: got %v want %v", item.k, found[item.k], item.v)
- }
- }
- if len(found) != len(all) {
- t.Errorf("iterator count mismatch: got %d values, want %d", len(found), len(all))
- }
-}
-
-func TestUnionIterator(t *testing.T) {
- triea := newEmpty()
- for _, val := range testdata1 {
- triea.Update([]byte(val.k), []byte(val.v))
- }
- triea.Commit(nil)
-
- trieb := newEmpty()
- for _, val := range testdata2 {
- trieb.Update([]byte(val.k), []byte(val.v))
- }
- trieb.Commit(nil)
-
- di, _ := NewUnionIterator([]NodeIterator{triea.NodeIterator(nil), trieb.NodeIterator(nil)})
- it := NewIterator(di)
-
- all := []struct{ k, v string }{
- {"aardvark", "c"},
- {"barb", "ba"},
- {"barb", "bd"},
- {"bard", "bc"},
- {"bars", "bb"},
- {"bars", "be"},
- {"bar", "b"},
- {"fab", "z"},
- {"food", "ab"},
- {"foos", "aa"},
- {"foo", "a"},
- {"jars", "d"},
- }
-
- for i, kv := range all {
- if !it.Next() {
- t.Errorf("Iterator ends prematurely at element %d", i)
- }
- if kv.k != string(it.Key) {
- t.Errorf("iterator value mismatch for element %d: got key %s want %s", i, it.Key, kv.k)
- }
- if kv.v != string(it.Value) {
- t.Errorf("iterator value mismatch for element %d: got value %s want %s", i, it.Value, kv.v)
- }
- }
- if it.Next() {
- t.Errorf("Iterator returned extra values.")
- }
-}
-
-func TestIteratorNoDups(t *testing.T) {
- var tr Trie
- for _, val := range testdata1 {
- tr.Update([]byte(val.k), []byte(val.v))
- }
- checkIteratorNoDups(t, tr.NodeIterator(nil), nil)
-}
-
-// This test checks that nodeIterator.Next can be retried after inserting missing trie nodes.
-func TestIteratorContinueAfterErrorDisk(t *testing.T) { testIteratorContinueAfterError(t, false) }
-func TestIteratorContinueAfterErrorMemonly(t *testing.T) { testIteratorContinueAfterError(t, true) }
-
-func testIteratorContinueAfterError(t *testing.T, memonly bool) {
- diskdb := hdb.NewMemDatabase()
- triedb := NewDatabase(diskdb)
-
- tr, _ := New(common.Hash{}, triedb)
- for _, val := range testdata1 {
- tr.Update([]byte(val.k), []byte(val.v))
- }
- tr.Commit(nil)
- if !memonly {
- triedb.Commit(tr.Hash(), true)
- }
- wantNodeCount := checkIteratorNoDups(t, tr.NodeIterator(nil), nil)
-
- var (
- diskKeys [][]byte
- memKeys []common.Hash
- )
- if memonly {
- memKeys = triedb.Nodes()
- } else {
- diskKeys = diskdb.Keys()
- }
- for i := 0; i < 20; i++ {
- // Create trie that will load all nodes from DB.
- tr, _ := New(tr.Hash(), triedb)
-
- // Remove a random node from the database. It can't be the root node
- // because that one is already loaded.
- var (
- rkey common.Hash
- rval []byte
- robj *cachedNode
- )
- for {
- if memonly {
- rkey = memKeys[rand.Intn(len(memKeys))]
- } else {
- copy(rkey[:], diskKeys[rand.Intn(len(diskKeys))])
- }
- if rkey != tr.Hash() {
- break
- }
- }
- if memonly {
- robj = triedb.nodes[rkey]
- delete(triedb.nodes, rkey)
- } else {
- rval, _ = diskdb.Get(rkey[:])
- diskdb.Delete(rkey[:])
- }
- // Iterate until the error is hit.
- seen := make(map[string]bool)
- it := tr.NodeIterator(nil)
- checkIteratorNoDups(t, it, seen)
- missing, ok := it.Error().(*MissingNodeError)
- if !ok || missing.NodeHash != rkey {
- t.Fatal("didn't hit missing node, got", it.Error())
- }
-
- // Add the node back and continue iteration.
- if memonly {
- triedb.nodes[rkey] = robj
- } else {
- diskdb.Put(rkey[:], rval)
- }
- checkIteratorNoDups(t, it, seen)
- if it.Error() != nil {
- t.Fatal("unexpected error", it.Error())
- }
- if len(seen) != wantNodeCount {
- t.Fatal("wrong node iteration count, got", len(seen), "want", wantNodeCount)
- }
- }
-}
-
-// Similar to the test above, this one checks that failure to create nodeIterator at a
-// certain key prefix behaves correctly when Next is called. The expectation is that Next
-// should retry seeking before returning true for the first time.
-func TestIteratorContinueAfterSeekErrorDisk(t *testing.T) {
- testIteratorContinueAfterSeekError(t, false)
-}
-func TestIteratorContinueAfterSeekErrorMemonly(t *testing.T) {
- testIteratorContinueAfterSeekError(t, true)
-}
-
-func testIteratorContinueAfterSeekError(t *testing.T, memonly bool) {
- // Commit test trie to db, then remove the node containing "bars".
- diskdb := hdb.NewMemDatabase()
- triedb := NewDatabase(diskdb)
-
- ctr, _ := New(common.Hash{}, triedb)
- for _, val := range testdata1 {
- ctr.Update([]byte(val.k), []byte(val.v))
- }
- root, _ := ctr.Commit(nil)
- if !memonly {
- triedb.Commit(root, true)
- }
- barNodeHash := common.HexToHash("05041990364eb72fcb1127652ce40d8bab765f2bfe53225b1170d276cc101c2e")
- var (
- barNodeBlob []byte
- barNodeObj *cachedNode
- )
- if memonly {
- barNodeObj = triedb.nodes[barNodeHash]
- delete(triedb.nodes, barNodeHash)
- } else {
- barNodeBlob, _ = diskdb.Get(barNodeHash[:])
- diskdb.Delete(barNodeHash[:])
- }
- // Create a new iterator that seeks to "bars". Seeking can't proceed because
- // the node is missing.
- tr, _ := New(root, triedb)
- it := tr.NodeIterator([]byte("bars"))
- missing, ok := it.Error().(*MissingNodeError)
- if !ok {
- t.Fatal("want MissingNodeError, got", it.Error())
- } else if missing.NodeHash != barNodeHash {
- t.Fatal("wrong node missing")
- }
- // Reinsert the missing node.
- if memonly {
- triedb.nodes[barNodeHash] = barNodeObj
- } else {
- diskdb.Put(barNodeHash[:], barNodeBlob)
- }
- // Check that iteration produces the right set of values.
- if err := checkIteratorOrder(testdata1[2:], NewIterator(it)); err != nil {
- t.Fatal(err)
- }
-}
-
-func checkIteratorNoDups(t *testing.T, it NodeIterator, seen map[string]bool) int {
- if seen == nil {
- seen = make(map[string]bool)
- }
- for it.Next(true) {
- if seen[string(it.Path())] {
- t.Fatalf("iterator visited node path %x twice", it.Path())
- }
- seen[string(it.Path())] = true
- }
- return len(seen)
-}
diff --git a/internal/trie/node.go b/internal/trie/node.go
deleted file mode 100644
index 1fafb7a53..000000000
--- a/internal/trie/node.go
+++ /dev/null
@@ -1,237 +0,0 @@
-// Copyright 2014 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see .
-
-package trie
-
-import (
- "fmt"
- "io"
- "strings"
-
- "github.com/ethereum/go-ethereum/common"
- "github.com/ethereum/go-ethereum/rlp"
-)
-
-var indices = []string{"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "a", "b", "c", "d", "e", "f", "[17]"}
-
-type node interface {
- fstring(string) string
- cache() (hashNode, bool)
- canUnload(cachegen, cachelimit uint16) bool
-}
-
-type (
- fullNode struct {
- Children [17]node // Actual trie node data to encode/decode (needs custom encoder)
- flags nodeFlag
- }
- shortNode struct {
- Key []byte
- Val node
- flags nodeFlag
- }
- hashNode []byte
- valueNode []byte
-)
-
-// nilValueNode is used when collapsing internal trie nodes for hashing, since
-// unset children need to serialize correctly.
-var nilValueNode = valueNode(nil)
-
-// EncodeRLP encodes a full node into the consensus RLP format.
-func (n *fullNode) EncodeRLP(w io.Writer) error {
- var nodes [17]node
-
- for i, child := range &n.Children {
- if child != nil {
- nodes[i] = child
- } else {
- nodes[i] = nilValueNode
- }
- }
- return rlp.Encode(w, nodes)
-}
-
-func (n *fullNode) copy() *fullNode { copy := *n; return © }
-func (n *shortNode) copy() *shortNode { copy := *n; return © }
-
-// nodeFlag contains caching-related metadata about a node.
-type nodeFlag struct {
- hash hashNode // cached hash of the node (may be nil)
- gen uint16 // cache generation counter
- dirty bool // whether the node has changes that must be written to the database
-}
-
-// canUnload tells whether a node can be unloaded.
-func (n *nodeFlag) canUnload(cachegen, cachelimit uint16) bool {
- return !n.dirty && cachegen-n.gen >= cachelimit
-}
-
-func (n *fullNode) canUnload(gen, limit uint16) bool { return n.flags.canUnload(gen, limit) }
-func (n *shortNode) canUnload(gen, limit uint16) bool { return n.flags.canUnload(gen, limit) }
-func (n hashNode) canUnload(uint16, uint16) bool { return false }
-func (n valueNode) canUnload(uint16, uint16) bool { return false }
-
-func (n *fullNode) cache() (hashNode, bool) { return n.flags.hash, n.flags.dirty }
-func (n *shortNode) cache() (hashNode, bool) { return n.flags.hash, n.flags.dirty }
-func (n hashNode) cache() (hashNode, bool) { return nil, true }
-func (n valueNode) cache() (hashNode, bool) { return nil, true }
-
-// Pretty printing.
-func (n *fullNode) String() string { return n.fstring("") }
-func (n *shortNode) String() string { return n.fstring("") }
-func (n hashNode) String() string { return n.fstring("") }
-func (n valueNode) String() string { return n.fstring("") }
-
-func (n *fullNode) fstring(ind string) string {
- resp := fmt.Sprintf("[\n%s ", ind)
- for i, node := range &n.Children {
- if node == nil {
- resp += fmt.Sprintf("%s: ", indices[i])
- } else {
- resp += fmt.Sprintf("%s: %v", indices[i], node.fstring(ind+" "))
- }
- }
- return resp + fmt.Sprintf("\n%s] ", ind)
-}
-func (n *shortNode) fstring(ind string) string {
- return fmt.Sprintf("{%x: %v} ", n.Key, n.Val.fstring(ind+" "))
-}
-func (n hashNode) fstring(ind string) string {
- return fmt.Sprintf("<%x> ", []byte(n))
-}
-func (n valueNode) fstring(ind string) string {
- return fmt.Sprintf("%x ", []byte(n))
-}
-
-func mustDecodeNode(hash, buf []byte, cachegen uint16) node {
- n, err := decodeNode(hash, buf, cachegen)
- if err != nil {
- panic(fmt.Sprintf("node %x: %v", hash, err))
- }
- return n
-}
-
-// decodeNode parses the RLP encoding of a trie node.
-func decodeNode(hash, buf []byte, cachegen uint16) (node, error) {
- if len(buf) == 0 {
- return nil, io.ErrUnexpectedEOF
- }
- elems, _, err := rlp.SplitList(buf)
- if err != nil {
- return nil, fmt.Errorf("decode error: %v", err)
- }
- switch c, _ := rlp.CountValues(elems); c {
- case 2:
- n, err := decodeShort(hash, elems, cachegen)
- return n, wrapError(err, "short")
- case 17:
- n, err := decodeFull(hash, elems, cachegen)
- return n, wrapError(err, "full")
- default:
- return nil, fmt.Errorf("invalid number of list elements: %v", c)
- }
-}
-
-func decodeShort(hash, elems []byte, cachegen uint16) (node, error) {
- kbuf, rest, err := rlp.SplitString(elems)
- if err != nil {
- return nil, err
- }
- flag := nodeFlag{hash: hash, gen: cachegen}
- key := compactToHex(kbuf)
- if hasTerm(key) {
- // value node
- val, _, err := rlp.SplitString(rest)
- if err != nil {
- return nil, fmt.Errorf("invalid value node: %v", err)
- }
- return &shortNode{key, append(valueNode{}, val...), flag}, nil
- }
- r, _, err := decodeRef(rest, cachegen)
- if err != nil {
- return nil, wrapError(err, "val")
- }
- return &shortNode{key, r, flag}, nil
-}
-
-func decodeFull(hash, elems []byte, cachegen uint16) (*fullNode, error) {
- n := &fullNode{flags: nodeFlag{hash: hash, gen: cachegen}}
- for i := 0; i < 16; i++ {
- cld, rest, err := decodeRef(elems, cachegen)
- if err != nil {
- return n, wrapError(err, fmt.Sprintf("[%d]", i))
- }
- n.Children[i], elems = cld, rest
- }
- val, _, err := rlp.SplitString(elems)
- if err != nil {
- return n, err
- }
- if len(val) > 0 {
- n.Children[16] = append(valueNode{}, val...)
- }
- return n, nil
-}
-
-const hashLen = len(common.Hash{})
-
-func decodeRef(buf []byte, cachegen uint16) (node, []byte, error) {
- kind, val, rest, err := rlp.Split(buf)
- if err != nil {
- return nil, buf, err
- }
- switch {
- case kind == rlp.List:
- // 'embedded' node reference. The encoding must be smaller
- // than a hash in order to be valid.
- if size := len(buf) - len(rest); size > hashLen {
- err := fmt.Errorf("oversized embedded node (size is %d bytes, want size < %d)", size, hashLen)
- return nil, buf, err
- }
- n, err := decodeNode(nil, buf, cachegen)
- return n, rest, err
- case kind == rlp.String && len(val) == 0:
- // empty node
- return nil, rest, nil
- case kind == rlp.String && len(val) == 32:
- return append(hashNode{}, val...), rest, nil
- default:
- return nil, nil, fmt.Errorf("invalid RLP string size %d (want 0 or 32)", len(val))
- }
-}
-
-// wraps a decoding error with information about the path to the
-// invalid child node (for debugging encoding issues).
-type decodeError struct {
- what error
- stack []string
-}
-
-func wrapError(err error, ctx string) error {
- if err == nil {
- return nil
- }
- if decErr, ok := err.(*decodeError); ok {
- decErr.stack = append(decErr.stack, ctx)
- return decErr
- }
- return &decodeError{err, []string{ctx}}
-}
-
-func (err *decodeError) Error() string {
- return fmt.Sprintf("%v (decode path: %s)", err.what, strings.Join(err.stack, "<-"))
-}
diff --git a/internal/trie/node_test.go b/internal/trie/node_test.go
deleted file mode 100644
index 7ad1ff9e7..000000000
--- a/internal/trie/node_test.go
+++ /dev/null
@@ -1,58 +0,0 @@
-// Copyright 2016 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see .
-
-package trie
-
-import "testing"
-
-func TestCanUnload(t *testing.T) {
- tests := []struct {
- flag nodeFlag
- cachegen, cachelimit uint16
- want bool
- }{
- {
- flag: nodeFlag{dirty: true, gen: 0},
- want: false,
- },
- {
- flag: nodeFlag{dirty: false, gen: 0},
- cachegen: 0, cachelimit: 0,
- want: true,
- },
- {
- flag: nodeFlag{dirty: false, gen: 65534},
- cachegen: 65535, cachelimit: 1,
- want: true,
- },
- {
- flag: nodeFlag{dirty: false, gen: 65534},
- cachegen: 0, cachelimit: 1,
- want: true,
- },
- {
- flag: nodeFlag{dirty: false, gen: 1},
- cachegen: 65535, cachelimit: 1,
- want: true,
- },
- }
-
- for _, test := range tests {
- if got := test.flag.canUnload(test.cachegen, test.cachelimit); got != test.want {
- t.Errorf("%+v\n got %t, want %t", test, got, test.want)
- }
- }
-}
diff --git a/internal/trie/proof.go b/internal/trie/proof.go
deleted file mode 100644
index af405eb8b..000000000
--- a/internal/trie/proof.go
+++ /dev/null
@@ -1,153 +0,0 @@
-// Copyright 2015 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see .
-
-package trie
-
-import (
- "bytes"
- "fmt"
-
- "github.com/ethereum/go-ethereum/common"
- "github.com/ethereum/go-ethereum/crypto"
- "github.com/ethereum/go-ethereum/log"
- "github.com/ethereum/go-ethereum/rlp"
- "github.com/harmony-one/harmony/internal/db"
-)
-
-// Prove constructs a merkle proof for key. The result contains all encoded nodes
-// on the path to the value at key. The value itself is also included in the last
-// node and can be retrieved by verifying the proof.
-//
-// If the trie does not contain a value for key, the returned proof contains all
-// nodes of the longest existing prefix of the key (at least the root node), ending
-// with the node that proves the absence of the key.
-func (t *Trie) Prove(key []byte, fromLevel uint, proofDb db.Putter) error {
- // Collect all nodes on the path to key.
- key = keybytesToHex(key)
- nodes := []node{}
- tn := t.root
- for len(key) > 0 && tn != nil {
- switch n := tn.(type) {
- case *shortNode:
- if len(key) < len(n.Key) || !bytes.Equal(n.Key, key[:len(n.Key)]) {
- // The trie doesn't contain the key.
- tn = nil
- } else {
- tn = n.Val
- key = key[len(n.Key):]
- }
- nodes = append(nodes, n)
- case *fullNode:
- tn = n.Children[key[0]]
- key = key[1:]
- nodes = append(nodes, n)
- case hashNode:
- var err error
- tn, err = t.resolveHash(n, nil)
- if err != nil {
- log.Error(fmt.Sprintf("Unhandled trie error: %v", err))
- return err
- }
- default:
- panic(fmt.Sprintf("%T: invalid node: %v", tn, tn))
- }
- }
- hasher := newHasher(0, 0, nil)
- for i, n := range nodes {
- // Don't bother checking for errors here since hasher panics
- // if encoding doesn't work and we're not writing to any database.
- n, _, _ = hasher.hashChildren(n, nil)
- hn, _ := hasher.store(n, nil, false)
- if hash, ok := hn.(hashNode); ok || i == 0 {
- // If the node's database encoding is a hash (or is the
- // root node), it becomes a proof element.
- if fromLevel > 0 {
- fromLevel--
- } else {
- enc, _ := rlp.EncodeToBytes(n)
- if !ok {
- hash = crypto.Keccak256(enc)
- }
- proofDb.Put(hash, enc)
- }
- }
- }
- return nil
-}
-
-// Prove constructs a merkle proof for key. The result contains all encoded nodes
-// on the path to the value at key. The value itself is also included in the last
-// node and can be retrieved by verifying the proof.
-//
-// If the trie does not contain a value for key, the returned proof contains all
-// nodes of the longest existing prefix of the key (at least the root node), ending
-// with the node that proves the absence of the key.
-func (t *SecureTrie) Prove(key []byte, fromLevel uint, proofDb db.Putter) error {
- return t.trie.Prove(key, fromLevel, proofDb)
-}
-
-// VerifyProof checks merkle proofs. The given proof must contain the value for
-// key in a trie with the given root hash. VerifyProof returns an error if the
-// proof contains invalid trie nodes or the wrong value.
-func VerifyProof(rootHash common.Hash, key []byte, proofDb DatabaseReader) (value []byte, nodes int, err error) {
- key = keybytesToHex(key)
- wantHash := rootHash
- for i := 0; ; i++ {
- buf, _ := proofDb.Get(wantHash[:])
- if buf == nil {
- return nil, i, fmt.Errorf("proof node %d (hash %064x) missing", i, wantHash)
- }
- n, err := decodeNode(wantHash[:], buf, 0)
- if err != nil {
- return nil, i, fmt.Errorf("bad proof node %d: %v", i, err)
- }
- keyrest, cld := get(n, key)
- switch cld := cld.(type) {
- case nil:
- // The trie doesn't contain the key.
- return nil, i, nil
- case hashNode:
- key = keyrest
- copy(wantHash[:], cld)
- case valueNode:
- return cld, i + 1, nil
- }
- }
-}
-
-func get(tn node, key []byte) ([]byte, node) {
- for {
- switch n := tn.(type) {
- case *shortNode:
- if len(key) < len(n.Key) || !bytes.Equal(n.Key, key[:len(n.Key)]) {
- return nil, nil
- }
- tn = n.Val
- key = key[len(n.Key):]
- case *fullNode:
- tn = n.Children[key[0]]
- key = key[1:]
- case hashNode:
- return key, n
- case nil:
- return key, nil
- case valueNode:
- return nil, n
- default:
- panic(fmt.Sprintf("%T: invalid node: %v", tn, tn))
- }
- }
-}
diff --git a/internal/trie/proof_test.go b/internal/trie/proof_test.go
deleted file mode 100644
index 996f87478..000000000
--- a/internal/trie/proof_test.go
+++ /dev/null
@@ -1,218 +0,0 @@
-// Copyright 2015 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see .
-
-package trie
-
-import (
- "bytes"
- crand "crypto/rand"
- mrand "math/rand"
- "testing"
- "time"
-
- "github.com/ethereum/go-ethereum/common"
- "github.com/ethereum/go-ethereum/crypto"
- "github.com/ethereum/go-ethereum/ethdb"
-)
-
-func init() {
- mrand.Seed(time.Now().Unix())
-}
-
-// makeProvers creates Merkle trie provers based on different implementations to
-// test all variations.
-func makeProvers(trie *Trie) []func(key []byte) *ethdb.MemDatabase {
- var provers []func(key []byte) *ethdb.MemDatabase
-
- // Create a direct trie based Merkle prover
- provers = append(provers, func(key []byte) *ethdb.MemDatabase {
- proof := ethdb.NewMemDatabase()
- trie.Prove(key, 0, proof)
- return proof
- })
- // Create a leaf iterator based Merkle prover
- provers = append(provers, func(key []byte) *ethdb.MemDatabase {
- proof := ethdb.NewMemDatabase()
- if it := NewIterator(trie.NodeIterator(key)); it.Next() && bytes.Equal(key, it.Key) {
- for _, p := range it.Prove() {
- proof.Put(crypto.Keccak256(p), p)
- }
- }
- return proof
- })
- return provers
-}
-
-func TestProof(t *testing.T) {
- trie, vals := randomTrie(500)
- root := trie.Hash()
- for i, prover := range makeProvers(trie) {
- for _, kv := range vals {
- proof := prover(kv.k)
- if proof == nil {
- t.Fatalf("prover %d: missing key %x while constructing proof", i, kv.k)
- }
- val, _, err := VerifyProof(root, kv.k, proof)
- if err != nil {
- t.Fatalf("prover %d: failed to verify proof for key %x: %v\nraw proof: %x", i, kv.k, err, proof)
- }
- if !bytes.Equal(val, kv.v) {
- t.Fatalf("prover %d: verified value mismatch for key %x: have %x, want %x", i, kv.k, val, kv.v)
- }
- }
- }
-}
-
-func TestOneElementProof(t *testing.T) {
- trie := new(Trie)
- updateString(trie, "k", "v")
- for i, prover := range makeProvers(trie) {
- proof := prover([]byte("k"))
- if proof == nil {
- t.Fatalf("prover %d: nil proof", i)
- }
- if proof.Len() != 1 {
- t.Errorf("prover %d: proof should have one element", i)
- }
- val, _, err := VerifyProof(trie.Hash(), []byte("k"), proof)
- if err != nil {
- t.Fatalf("prover %d: failed to verify proof: %v\nraw proof: %x", i, err, proof)
- }
- if !bytes.Equal(val, []byte("v")) {
- t.Fatalf("prover %d: verified value mismatch: have %x, want 'k'", i, val)
- }
- }
-}
-
-func TestBadProof(t *testing.T) {
- trie, vals := randomTrie(800)
- root := trie.Hash()
- for i, prover := range makeProvers(trie) {
- for _, kv := range vals {
- proof := prover(kv.k)
- if proof == nil {
- t.Fatalf("prover %d: nil proof", i)
- }
- key := proof.Keys()[mrand.Intn(proof.Len())]
- val, _ := proof.Get(key)
- proof.Delete(key)
-
- mutateByte(val)
- proof.Put(crypto.Keccak256(val), val)
-
- if _, _, err := VerifyProof(root, kv.k, proof); err == nil {
- t.Fatalf("prover %d: expected proof to fail for key %x", i, kv.k)
- }
- }
- }
-}
-
-// Tests that missing keys can also be proven. The test explicitly uses a single
-// entry trie and checks for missing keys both before and after the single entry.
-func TestMissingKeyProof(t *testing.T) {
- trie := new(Trie)
- updateString(trie, "k", "v")
-
- for i, key := range []string{"a", "j", "l", "z"} {
- proof := ethdb.NewMemDatabase()
- trie.Prove([]byte(key), 0, proof)
-
- if proof.Len() != 1 {
- t.Errorf("test %d: proof should have one element", i)
- }
- val, _, err := VerifyProof(trie.Hash(), []byte(key), proof)
- if err != nil {
- t.Fatalf("test %d: failed to verify proof: %v\nraw proof: %x", i, err, proof)
- }
- if val != nil {
- t.Fatalf("test %d: verified value mismatch: have %x, want nil", i, val)
- }
- }
-}
-
-// mutateByte changes one byte in b.
-func mutateByte(b []byte) {
- for r := mrand.Intn(len(b)); ; {
- new := byte(mrand.Intn(255))
- if new != b[r] {
- b[r] = new
- break
- }
- }
-}
-
-func BenchmarkProve(b *testing.B) {
- trie, vals := randomTrie(100)
- var keys []string
- for k := range vals {
- keys = append(keys, k)
- }
-
- b.ResetTimer()
- for i := 0; i < b.N; i++ {
- kv := vals[keys[i%len(keys)]]
- proofs := ethdb.NewMemDatabase()
- if trie.Prove(kv.k, 0, proofs); len(proofs.Keys()) == 0 {
- b.Fatalf("zero length proof for %x", kv.k)
- }
- }
-}
-
-func BenchmarkVerifyProof(b *testing.B) {
- trie, vals := randomTrie(100)
- root := trie.Hash()
- var keys []string
- var proofs []*ethdb.MemDatabase
- for k := range vals {
- keys = append(keys, k)
- proof := ethdb.NewMemDatabase()
- trie.Prove([]byte(k), 0, proof)
- proofs = append(proofs, proof)
- }
-
- b.ResetTimer()
- for i := 0; i < b.N; i++ {
- im := i % len(keys)
- if _, _, err := VerifyProof(root, []byte(keys[im]), proofs[im]); err != nil {
- b.Fatalf("key %x: %v", keys[im], err)
- }
- }
-}
-
-func randomTrie(n int) (*Trie, map[string]*kv) {
- trie := new(Trie)
- vals := make(map[string]*kv)
- for i := byte(0); i < 100; i++ {
- value := &kv{common.LeftPadBytes([]byte{i}, 32), []byte{i}, false}
- value2 := &kv{common.LeftPadBytes([]byte{i + 10}, 32), []byte{i}, false}
- trie.Update(value.k, value.v)
- trie.Update(value2.k, value2.v)
- vals[string(value.k)] = value
- vals[string(value2.k)] = value2
- }
- for i := 0; i < n; i++ {
- value := &kv{randBytes(32), randBytes(20), false}
- trie.Update(value.k, value.v)
- vals[string(value.k)] = value
- }
- return trie, vals
-}
-
-func randBytes(n int) []byte {
- r := make([]byte, n)
- crand.Read(r)
- return r
-}
diff --git a/internal/trie/secure_trie.go b/internal/trie/secure_trie.go
deleted file mode 100644
index 6a50cfd5a..000000000
--- a/internal/trie/secure_trie.go
+++ /dev/null
@@ -1,203 +0,0 @@
-// Copyright 2015 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see .
-
-package trie
-
-import (
- "fmt"
-
- "github.com/ethereum/go-ethereum/common"
- "github.com/ethereum/go-ethereum/log"
-)
-
-// SecureTrie wraps a trie with key hashing. In a secure trie, all
-// access operations hash the key using keccak256. This prevents
-// calling code from creating long chains of nodes that
-// increase the access time.
-//
-// Contrary to a regular trie, a SecureTrie can only be created with
-// New and must have an attached database. The database also stores
-// the preimage of each key.
-//
-// SecureTrie is not safe for concurrent use.
-type SecureTrie struct {
- trie Trie
- hashKeyBuf [common.HashLength]byte
- secKeyCache map[string][]byte
- secKeyCacheOwner *SecureTrie // Pointer to self, replace the key cache on mismatch
-}
-
-// NewSecure creates a trie with an existing root node from a backing database
-// and optional intermediate in-memory node pool.
-//
-// If root is the zero hash or the sha3 hash of an empty string, the
-// trie is initially empty. Otherwise, New will panic if db is nil
-// and returns MissingNodeError if the root node cannot be found.
-//
-// Accessing the trie loads nodes from the database or node pool on demand.
-// Loaded nodes are kept around until their 'cache generation' expires.
-// A new cache generation is created by each call to Commit.
-// cachelimit sets the number of past cache generations to keep.
-func NewSecure(root common.Hash, db *Database, cachelimit uint16) (*SecureTrie, error) {
- if db == nil {
- panic("trie.NewSecure called without a database")
- }
- trie, err := New(root, db)
- if err != nil {
- return nil, err
- }
- trie.SetCacheLimit(cachelimit)
- return &SecureTrie{trie: *trie}, nil
-}
-
-// Get returns the value for key stored in the trie.
-// The value bytes must not be modified by the caller.
-func (t *SecureTrie) Get(key []byte) []byte {
- res, err := t.TryGet(key)
- if err != nil {
- log.Error(fmt.Sprintf("Unhandled trie error: %v", err))
- }
- return res
-}
-
-// TryGet returns the value for key stored in the trie.
-// The value bytes must not be modified by the caller.
-// If a node was not found in the database, a MissingNodeError is returned.
-func (t *SecureTrie) TryGet(key []byte) ([]byte, error) {
- return t.trie.TryGet(t.hashKey(key))
-}
-
-// Update associates key with value in the trie. Subsequent calls to
-// Get will return value. If value has length zero, any existing value
-// is deleted from the trie and calls to Get will return nil.
-//
-// The value bytes must not be modified by the caller while they are
-// stored in the trie.
-func (t *SecureTrie) Update(key, value []byte) {
- if err := t.TryUpdate(key, value); err != nil {
- log.Error(fmt.Sprintf("Unhandled trie error: %v", err))
- }
-}
-
-// TryUpdate associates key with value in the trie. Subsequent calls to
-// Get will return value. If value has length zero, any existing value
-// is deleted from the trie and calls to Get will return nil.
-//
-// The value bytes must not be modified by the caller while they are
-// stored in the trie.
-//
-// If a node was not found in the database, a MissingNodeError is returned.
-func (t *SecureTrie) TryUpdate(key, value []byte) error {
- hk := t.hashKey(key)
- err := t.trie.TryUpdate(hk, value)
- if err != nil {
- return err
- }
- t.getSecKeyCache()[string(hk)] = common.CopyBytes(key)
- return nil
-}
-
-// Delete removes any existing value for key from the trie.
-func (t *SecureTrie) Delete(key []byte) {
- if err := t.TryDelete(key); err != nil {
- log.Error(fmt.Sprintf("Unhandled trie error: %v", err))
- }
-}
-
-// TryDelete removes any existing value for key from the trie.
-// If a node was not found in the database, a MissingNodeError is returned.
-func (t *SecureTrie) TryDelete(key []byte) error {
- hk := t.hashKey(key)
- delete(t.getSecKeyCache(), string(hk))
- return t.trie.TryDelete(hk)
-}
-
-// GetKey returns the sha3 preimage of a hashed key that was
-// previously used to store a value.
-func (t *SecureTrie) GetKey(shaKey []byte) []byte {
- if key, ok := t.getSecKeyCache()[string(shaKey)]; ok {
- return key
- }
- key, _ := t.trie.db.preimage(common.BytesToHash(shaKey))
- return key
-}
-
-// Commit writes all nodes and the secure hash pre-images to the trie's database.
-// Nodes are stored with their sha3 hash as the key.
-//
-// Committing flushes nodes from memory. Subsequent Get calls will load nodes
-// from the database.
-func (t *SecureTrie) Commit(onleaf LeafCallback) (root common.Hash, err error) {
- // Write all the pre-images to the actual disk database
- if len(t.getSecKeyCache()) > 0 {
- t.trie.db.lock.Lock()
- for hk, key := range t.secKeyCache {
- t.trie.db.insertPreimage(common.BytesToHash([]byte(hk)), key)
- }
- t.trie.db.lock.Unlock()
-
- t.secKeyCache = make(map[string][]byte)
- }
- // Commit the trie to its intermediate node database
- return t.trie.Commit(onleaf)
-}
-
-// Hash returns the root hash of SecureTrie. It does not write to the
-// database and can be used even if the trie doesn't have one.
-func (t *SecureTrie) Hash() common.Hash {
- return t.trie.Hash()
-}
-
-// Root returns the root hash of SecureTrie.
-// Deprecated: use Hash instead.
-func (t *SecureTrie) Root() []byte {
- return t.trie.Root()
-}
-
-// Copy returns a copy of SecureTrie.
-func (t *SecureTrie) Copy() *SecureTrie {
- cpy := *t
- return &cpy
-}
-
-// NodeIterator returns an iterator that returns nodes of the underlying trie. Iteration
-// starts at the key after the given start key.
-func (t *SecureTrie) NodeIterator(start []byte) NodeIterator {
- return t.trie.NodeIterator(start)
-}
-
-// hashKey returns the hash of key as an ephemeral buffer.
-// The caller must not hold onto the return value because it will become
-// invalid on the next call to hashKey or secKey.
-func (t *SecureTrie) hashKey(key []byte) []byte {
- h := newHasher(0, 0, nil)
- h.sha.Reset()
- h.sha.Write(key)
- buf := h.sha.Sum(t.hashKeyBuf[:0])
- returnHasherToPool(h)
- return buf
-}
-
-// getSecKeyCache returns the current secure key cache, creating a new one if
-// ownership changed (i.e. the current secure trie is a copy of another owning
-// the actual cache).
-func (t *SecureTrie) getSecKeyCache() map[string][]byte {
- if t != t.secKeyCacheOwner {
- t.secKeyCacheOwner = t
- t.secKeyCache = make(map[string][]byte)
- }
- return t.secKeyCache
-}
diff --git a/internal/trie/secure_trie_test.go b/internal/trie/secure_trie_test.go
deleted file mode 100644
index 5df369f15..000000000
--- a/internal/trie/secure_trie_test.go
+++ /dev/null
@@ -1,145 +0,0 @@
-// Copyright 2015 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see .
-
-package trie
-
-import (
- "bytes"
- "runtime"
- "sync"
- "testing"
-
- "github.com/ethereum/go-ethereum/common"
- "github.com/ethereum/go-ethereum/crypto"
- "github.com/harmony-one/harmony/internal/db"
-)
-
-func newEmptySecure() *SecureTrie {
- trie, _ := NewSecure(common.Hash{}, NewDatabase(db.NewMemDatabase()), 0)
- return trie
-}
-
-// makeTestSecureTrie creates a large enough secure trie for testing.
-func makeTestSecureTrie() (*Database, *SecureTrie, map[string][]byte) {
- // Create an empty trie
- triedb := NewDatabase(db.NewMemDatabase())
-
- trie, _ := NewSecure(common.Hash{}, triedb, 0)
-
- // Fill it with some arbitrary data
- content := make(map[string][]byte)
- for i := byte(0); i < 255; i++ {
- // Map the same data under multiple keys
- key, val := common.LeftPadBytes([]byte{1, i}, 32), []byte{i}
- content[string(key)] = val
- trie.Update(key, val)
-
- key, val = common.LeftPadBytes([]byte{2, i}, 32), []byte{i}
- content[string(key)] = val
- trie.Update(key, val)
-
- // Add some other data to inflate the trie
- for j := byte(3); j < 13; j++ {
- key, val = common.LeftPadBytes([]byte{j, i}, 32), []byte{j, i}
- content[string(key)] = val
- trie.Update(key, val)
- }
- }
- trie.Commit(nil)
-
- // Return the generated trie
- return triedb, trie, content
-}
-
-func TestSecureDelete(t *testing.T) {
- trie := newEmptySecure()
- vals := []struct{ k, v string }{
- {"do", "verb"},
- {"ether", "wookiedoo"},
- {"horse", "stallion"},
- {"shaman", "horse"},
- {"doge", "coin"},
- {"ether", ""},
- {"dog", "puppy"},
- {"shaman", ""},
- }
- for _, val := range vals {
- if val.v != "" {
- trie.Update([]byte(val.k), []byte(val.v))
- } else {
- trie.Delete([]byte(val.k))
- }
- }
- hash := trie.Hash()
- exp := common.HexToHash("29b235a58c3c25ab83010c327d5932bcf05324b7d6b1185e650798034783ca9d")
- if hash != exp {
- t.Errorf("expected %x got %x", exp, hash)
- }
-}
-
-func TestSecureGetKey(t *testing.T) {
- trie := newEmptySecure()
- trie.Update([]byte("foo"), []byte("bar"))
-
- key := []byte("foo")
- value := []byte("bar")
- seckey := crypto.Keccak256(key)
-
- if !bytes.Equal(trie.Get(key), value) {
- t.Errorf("Get did not return bar")
- }
- if k := trie.GetKey(seckey); !bytes.Equal(k, key) {
- t.Errorf("GetKey returned %q, want %q", k, key)
- }
-}
-
-func TestSecureTrieConcurrency(t *testing.T) {
- // Create an initial trie and copy if for concurrent access
- _, trie, _ := makeTestSecureTrie()
-
- threads := runtime.NumCPU()
- tries := make([]*SecureTrie, threads)
- for i := 0; i < threads; i++ {
- cpy := *trie
- tries[i] = &cpy
- }
- // Start a batch of goroutines interactng with the trie
- pend := new(sync.WaitGroup)
- pend.Add(threads)
- for i := 0; i < threads; i++ {
- go func(index int) {
- defer pend.Done()
-
- for j := byte(0); j < 255; j++ {
- // Map the same data under multiple keys
- key, val := common.LeftPadBytes([]byte{byte(index), 1, j}, 32), []byte{j}
- tries[index].Update(key, val)
-
- key, val = common.LeftPadBytes([]byte{byte(index), 2, j}, 32), []byte{j}
- tries[index].Update(key, val)
-
- // Add some other data to inflate the trie
- for k := byte(3); k < 13; k++ {
- key, val = common.LeftPadBytes([]byte{byte(index), k, j}, 32), []byte{k, j}
- tries[index].Update(key, val)
- }
- }
- tries[index].Commit(nil)
- }(i)
- }
- // Wait for all threads to finish
- pend.Wait()
-}
diff --git a/internal/trie/sync.go b/internal/trie/sync.go
deleted file mode 100644
index 67dff5a8b..000000000
--- a/internal/trie/sync.go
+++ /dev/null
@@ -1,330 +0,0 @@
-// Copyright 2015 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see .
-
-package trie
-
-import (
- "errors"
- "fmt"
-
- "github.com/ethereum/go-ethereum/common"
- "github.com/ethereum/go-ethereum/common/prque"
- "github.com/ethereum/go-ethereum/ethdb"
-)
-
-// ErrNotRequested is returned by the trie sync when it's requested to process a
-// node it did not request.
-var ErrNotRequested = errors.New("not requested")
-
-// ErrAlreadyProcessed is returned by the trie sync when it's requested to process a
-// node it already processed previously.
-var ErrAlreadyProcessed = errors.New("already processed")
-
-// request represents a scheduled or already in-flight state retrieval request.
-type request struct {
- hash common.Hash // Hash of the node data content to retrieve
- data []byte // Data content of the node, cached until all subtrees complete
- raw bool // Whether this is a raw entry (code) or a trie node
-
- parents []*request // Parent state nodes referencing this entry (notify all upon completion)
- depth int // Depth level within the trie the node is located to prioritise DFS
- deps int // Number of dependencies before allowed to commit this node
-
- callback LeafCallback // Callback to invoke if a leaf node it reached on this branch
-}
-
-// SyncResult is a simple list to return missing nodes along with their request
-// hashes.
-type SyncResult struct {
- Hash common.Hash // Hash of the originally unknown trie node
- Data []byte // Data content of the retrieved node
-}
-
-// syncMemBatch is an in-memory buffer of successfully downloaded but not yet
-// persisted data items.
-type syncMemBatch struct {
- batch map[common.Hash][]byte // In-memory membatch of recently completed items
- order []common.Hash // Order of completion to prevent out-of-order data loss
-}
-
-// newSyncMemBatch allocates a new memory-buffer for not-yet persisted trie nodes.
-func newSyncMemBatch() *syncMemBatch {
- return &syncMemBatch{
- batch: make(map[common.Hash][]byte),
- order: make([]common.Hash, 0, 256),
- }
-}
-
-// Sync is the main state trie synchronisation scheduler, which provides yet
-// unknown trie hashes to retrieve, accepts node data associated with said hashes
-// and reconstructs the trie step by step until all is done.
-type Sync struct {
- database DatabaseReader // Persistent database to check for existing entries
- membatch *syncMemBatch // Memory buffer to avoid frequent database writes
- requests map[common.Hash]*request // Pending requests pertaining to a key hash
- queue *prque.Prque // Priority queue with the pending requests
-}
-
-// NewSync creates a new trie data download scheduler.
-func NewSync(root common.Hash, database DatabaseReader, callback LeafCallback) *Sync {
- ts := &Sync{
- database: database,
- membatch: newSyncMemBatch(),
- requests: make(map[common.Hash]*request),
- queue: prque.New(nil),
- }
- ts.AddSubTrie(root, 0, common.Hash{}, callback)
- return ts
-}
-
-// AddSubTrie registers a new trie to the sync code, rooted at the designated parent.
-func (s *Sync) AddSubTrie(root common.Hash, depth int, parent common.Hash, callback LeafCallback) {
- // Short circuit if the trie is empty or already known
- if root == emptyRoot {
- return
- }
- if _, ok := s.membatch.batch[root]; ok {
- return
- }
- key := root.Bytes()
- blob, _ := s.database.Get(key)
- if local, err := decodeNode(key, blob, 0); local != nil && err == nil {
- return
- }
- // Assemble the new sub-trie sync request
- req := &request{
- hash: root,
- depth: depth,
- callback: callback,
- }
- // If this sub-trie has a designated parent, link them together
- if parent != (common.Hash{}) {
- ancestor := s.requests[parent]
- if ancestor == nil {
- panic(fmt.Sprintf("sub-trie ancestor not found: %x", parent))
- }
- ancestor.deps++
- req.parents = append(req.parents, ancestor)
- }
- s.schedule(req)
-}
-
-// AddRawEntry schedules the direct retrieval of a state entry that should not be
-// interpreted as a trie node, but rather accepted and stored into the database
-// as is. This method's goal is to support misc state metadata retrievals (e.g.
-// contract code).
-func (s *Sync) AddRawEntry(hash common.Hash, depth int, parent common.Hash) {
- // Short circuit if the entry is empty or already known
- if hash == emptyState {
- return
- }
- if _, ok := s.membatch.batch[hash]; ok {
- return
- }
- if ok, _ := s.database.Has(hash.Bytes()); ok {
- return
- }
- // Assemble the new sub-trie sync request
- req := &request{
- hash: hash,
- raw: true,
- depth: depth,
- }
- // If this sub-trie has a designated parent, link them together
- if parent != (common.Hash{}) {
- ancestor := s.requests[parent]
- if ancestor == nil {
- panic(fmt.Sprintf("raw-entry ancestor not found: %x", parent))
- }
- ancestor.deps++
- req.parents = append(req.parents, ancestor)
- }
- s.schedule(req)
-}
-
-// Missing retrieves the known missing nodes from the trie for retrieval.
-func (s *Sync) Missing(max int) []common.Hash {
- requests := []common.Hash{}
- for !s.queue.Empty() && (max == 0 || len(requests) < max) {
- requests = append(requests, s.queue.PopItem().(common.Hash))
- }
- return requests
-}
-
-// Process injects a batch of retrieved trie nodes data, returning if something
-// was committed to the database and also the index of an entry if processing of
-// it failed.
-func (s *Sync) Process(results []SyncResult) (bool, int, error) {
- committed := false
-
- for i, item := range results {
- // If the item was not requested, bail out
- request := s.requests[item.Hash]
- if request == nil {
- return committed, i, ErrNotRequested
- }
- if request.data != nil {
- return committed, i, ErrAlreadyProcessed
- }
- // If the item is a raw entry request, commit directly
- if request.raw {
- request.data = item.Data
- s.commit(request)
- committed = true
- continue
- }
- // Decode the node data content and update the request
- node, err := decodeNode(item.Hash[:], item.Data, 0)
- if err != nil {
- return committed, i, err
- }
- request.data = item.Data
-
- // Create and schedule a request for all the children nodes
- requests, err := s.children(request, node)
- if err != nil {
- return committed, i, err
- }
- if len(requests) == 0 && request.deps == 0 {
- s.commit(request)
- committed = true
- continue
- }
- request.deps += len(requests)
- for _, child := range requests {
- s.schedule(child)
- }
- }
- return committed, 0, nil
-}
-
-// Commit flushes the data stored in the internal membatch out to persistent
-// storage, returning the number of items written and any occurred error.
-func (s *Sync) Commit(dbw ethdb.Putter) (int, error) {
- // Dump the membatch into a database dbw
- for i, key := range s.membatch.order {
- if err := dbw.Put(key[:], s.membatch.batch[key]); err != nil {
- return i, err
- }
- }
- written := len(s.membatch.order)
-
- // Drop the membatch data and return
- s.membatch = newSyncMemBatch()
- return written, nil
-}
-
-// Pending returns the number of state entries currently pending for download.
-func (s *Sync) Pending() int {
- return len(s.requests)
-}
-
-// schedule inserts a new state retrieval request into the fetch queue. If there
-// is already a pending request for this node, the new request will be discarded
-// and only a parent reference added to the old one.
-func (s *Sync) schedule(req *request) {
- // If we're already requesting this node, add a new reference and stop
- if old, ok := s.requests[req.hash]; ok {
- old.parents = append(old.parents, req.parents...)
- return
- }
- // Schedule the request for future retrieval
- s.queue.Push(req.hash, int64(req.depth))
- s.requests[req.hash] = req
-}
-
-// children retrieves all the missing children of a state trie entry for future
-// retrieval scheduling.
-func (s *Sync) children(req *request, object node) ([]*request, error) {
- // Gather all the children of the node, irrelevant whether known or not
- type child struct {
- node node
- depth int
- }
- children := []child{}
-
- switch node := (object).(type) {
- case *shortNode:
- children = []child{{
- node: node.Val,
- depth: req.depth + len(node.Key),
- }}
- case *fullNode:
- for i := 0; i < 17; i++ {
- if node.Children[i] != nil {
- children = append(children, child{
- node: node.Children[i],
- depth: req.depth + 1,
- })
- }
- }
- default:
- panic(fmt.Sprintf("unknown node: %+v", node))
- }
- // Iterate over the children, and request all unknown ones
- requests := make([]*request, 0, len(children))
- for _, child := range children {
- // Notify any external watcher of a new key/value node
- if req.callback != nil {
- if node, ok := (child.node).(valueNode); ok {
- if err := req.callback(node, req.hash); err != nil {
- return nil, err
- }
- }
- }
- // If the child references another node, resolve or schedule
- if node, ok := (child.node).(hashNode); ok {
- // Try to resolve the node from the local database
- hash := common.BytesToHash(node)
- if _, ok := s.membatch.batch[hash]; ok {
- continue
- }
- if ok, _ := s.database.Has(node); ok {
- continue
- }
- // Locally unknown node, schedule for retrieval
- requests = append(requests, &request{
- hash: hash,
- parents: []*request{req},
- depth: child.depth,
- callback: req.callback,
- })
- }
- }
- return requests, nil
-}
-
-// commit finalizes a retrieval request and stores it into the membatch. If any
-// of the referencing parent requests complete due to this commit, they are also
-// committed themselves.
-func (s *Sync) commit(req *request) (err error) {
- // Write the node content to the membatch
- s.membatch.batch[req.hash] = req.data
- s.membatch.order = append(s.membatch.order, req.hash)
-
- delete(s.requests, req.hash)
-
- // Check all parents for completion
- for _, parent := range req.parents {
- parent.deps--
- if parent.deps == 0 {
- if err := s.commit(parent); err != nil {
- return err
- }
- }
- }
- return nil
-}
diff --git a/internal/trie/sync_test.go b/internal/trie/sync_test.go
deleted file mode 100644
index f2d9c5504..000000000
--- a/internal/trie/sync_test.go
+++ /dev/null
@@ -1,358 +0,0 @@
-// Copyright 2015 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see .
-
-package trie
-
-import (
- "bytes"
- "testing"
-
- "github.com/ethereum/go-ethereum/common"
- "github.com/harmony-one/harmony/internal/db"
-)
-
-// makeTestTrie create a sample test trie to test node-wise reconstruction.
-func makeTestTrie() (*Database, *Trie, map[string][]byte) {
- // Create an empty trie
- triedb := NewDatabase(db.NewMemDatabase())
- trie, _ := New(common.Hash{}, triedb)
-
- // Fill it with some arbitrary data
- content := make(map[string][]byte)
- for i := byte(0); i < 255; i++ {
- // Map the same data under multiple keys
- key, val := common.LeftPadBytes([]byte{1, i}, 32), []byte{i}
- content[string(key)] = val
- trie.Update(key, val)
-
- key, val = common.LeftPadBytes([]byte{2, i}, 32), []byte{i}
- content[string(key)] = val
- trie.Update(key, val)
-
- // Add some other data to inflate the trie
- for j := byte(3); j < 13; j++ {
- key, val = common.LeftPadBytes([]byte{j, i}, 32), []byte{j, i}
- content[string(key)] = val
- trie.Update(key, val)
- }
- }
- trie.Commit(nil)
-
- // Return the generated trie
- return triedb, trie, content
-}
-
-// checkTrieContents cross references a reconstructed trie with an expected data
-// content map.
-func checkTrieContents(t *testing.T, db *Database, root []byte, content map[string][]byte) {
- // Check root availability and trie contents
- trie, err := New(common.BytesToHash(root), db)
- if err != nil {
- t.Fatalf("failed to create trie at %x: %v", root, err)
- }
- if err := checkTrieConsistency(db, common.BytesToHash(root)); err != nil {
- t.Fatalf("inconsistent trie at %x: %v", root, err)
- }
- for key, val := range content {
- if have := trie.Get([]byte(key)); !bytes.Equal(have, val) {
- t.Errorf("entry %x: content mismatch: have %x, want %x", key, have, val)
- }
- }
-}
-
-// checkTrieConsistency checks that all nodes in a trie are indeed present.
-func checkTrieConsistency(db *Database, root common.Hash) error {
- // Create and iterate a trie rooted in a subnode
- trie, err := New(root, db)
- if err != nil {
- return nil // Consider a non existent state consistent
- }
- it := trie.NodeIterator(nil)
- for it.Next(true) {
- }
- return it.Error()
-}
-
-// Tests that an empty trie is not scheduled for syncing.
-func TestEmptySync(t *testing.T) {
- dbA := NewDatabase(db.NewMemDatabase())
- dbB := NewDatabase(db.NewMemDatabase())
- emptyA, _ := New(common.Hash{}, dbA)
- emptyB, _ := New(emptyRoot, dbB)
-
- for i, trie := range []*Trie{emptyA, emptyB} {
- if req := NewSync(trie.Hash(), db.NewMemDatabase(), nil).Missing(1); len(req) != 0 {
- t.Errorf("test %d: content requested for empty trie: %v", i, req)
- }
- }
-}
-
-// Tests that given a root hash, a trie can sync iteratively on a single thread,
-// requesting retrieval tasks and returning all of them in one go.
-func TestIterativeSyncIndividual(t *testing.T) { testIterativeSync(t, 1) }
-func TestIterativeSyncBatched(t *testing.T) { testIterativeSync(t, 100) }
-
-func testIterativeSync(t *testing.T, batch int) {
- // Create a random trie to copy
- srcDb, srcTrie, srcData := makeTestTrie()
-
- // Create a destination trie and sync with the scheduler
- diskdb := db.NewMemDatabase()
- triedb := NewDatabase(diskdb)
- sched := NewSync(srcTrie.Hash(), diskdb, nil)
-
- queue := append([]common.Hash{}, sched.Missing(batch)...)
- for len(queue) > 0 {
- results := make([]SyncResult, len(queue))
- for i, hash := range queue {
- data, err := srcDb.Node(hash)
- if err != nil {
- t.Fatalf("failed to retrieve node data for %x: %v", hash, err)
- }
- results[i] = SyncResult{hash, data}
- }
- if _, index, err := sched.Process(results); err != nil {
- t.Fatalf("failed to process result #%d: %v", index, err)
- }
- if index, err := sched.Commit(diskdb); err != nil {
- t.Fatalf("failed to commit data #%d: %v", index, err)
- }
- queue = append(queue[:0], sched.Missing(batch)...)
- }
- // Cross check that the two tries are in sync
- checkTrieContents(t, triedb, srcTrie.Root(), srcData)
-}
-
-// Tests that the trie scheduler can correctly reconstruct the state even if only
-// partial results are returned, and the others sent only later.
-func TestIterativeDelayedSync(t *testing.T) {
- // Create a random trie to copy
- srcDb, srcTrie, srcData := makeTestTrie()
-
- // Create a destination trie and sync with the scheduler
- diskdb := db.NewMemDatabase()
- triedb := NewDatabase(diskdb)
- sched := NewSync(srcTrie.Hash(), diskdb, nil)
-
- queue := append([]common.Hash{}, sched.Missing(10000)...)
- for len(queue) > 0 {
- // Sync only half of the scheduled nodes
- results := make([]SyncResult, len(queue)/2+1)
- for i, hash := range queue[:len(results)] {
- data, err := srcDb.Node(hash)
- if err != nil {
- t.Fatalf("failed to retrieve node data for %x: %v", hash, err)
- }
- results[i] = SyncResult{hash, data}
- }
- if _, index, err := sched.Process(results); err != nil {
- t.Fatalf("failed to process result #%d: %v", index, err)
- }
- if index, err := sched.Commit(diskdb); err != nil {
- t.Fatalf("failed to commit data #%d: %v", index, err)
- }
- queue = append(queue[len(results):], sched.Missing(10000)...)
- }
- // Cross check that the two tries are in sync
- checkTrieContents(t, triedb, srcTrie.Root(), srcData)
-}
-
-// Tests that given a root hash, a trie can sync iteratively on a single thread,
-// requesting retrieval tasks and returning all of them in one go, however in a
-// random order.
-func TestIterativeRandomSyncIndividual(t *testing.T) { testIterativeRandomSync(t, 1) }
-func TestIterativeRandomSyncBatched(t *testing.T) { testIterativeRandomSync(t, 100) }
-
-func testIterativeRandomSync(t *testing.T, batch int) {
- // Create a random trie to copy
- srcDb, srcTrie, srcData := makeTestTrie()
-
- // Create a destination trie and sync with the scheduler
- diskdb := db.NewMemDatabase()
- triedb := NewDatabase(diskdb)
- sched := NewSync(srcTrie.Hash(), diskdb, nil)
-
- queue := make(map[common.Hash]struct{})
- for _, hash := range sched.Missing(batch) {
- queue[hash] = struct{}{}
- }
- for len(queue) > 0 {
- // Fetch all the queued nodes in a random order
- results := make([]SyncResult, 0, len(queue))
- for hash := range queue {
- data, err := srcDb.Node(hash)
- if err != nil {
- t.Fatalf("failed to retrieve node data for %x: %v", hash, err)
- }
- results = append(results, SyncResult{hash, data})
- }
- // Feed the retrieved results back and queue new tasks
- if _, index, err := sched.Process(results); err != nil {
- t.Fatalf("failed to process result #%d: %v", index, err)
- }
- if index, err := sched.Commit(diskdb); err != nil {
- t.Fatalf("failed to commit data #%d: %v", index, err)
- }
- queue = make(map[common.Hash]struct{})
- for _, hash := range sched.Missing(batch) {
- queue[hash] = struct{}{}
- }
- }
- // Cross check that the two tries are in sync
- checkTrieContents(t, triedb, srcTrie.Root(), srcData)
-}
-
-// Tests that the trie scheduler can correctly reconstruct the state even if only
-// partial results are returned (Even those randomly), others sent only later.
-func TestIterativeRandomDelayedSync(t *testing.T) {
- // Create a random trie to copy
- srcDb, srcTrie, srcData := makeTestTrie()
-
- // Create a destination trie and sync with the scheduler
- diskdb := db.NewMemDatabase()
- triedb := NewDatabase(diskdb)
- sched := NewSync(srcTrie.Hash(), diskdb, nil)
-
- queue := make(map[common.Hash]struct{})
- for _, hash := range sched.Missing(10000) {
- queue[hash] = struct{}{}
- }
- for len(queue) > 0 {
- // Sync only half of the scheduled nodes, even those in random order
- results := make([]SyncResult, 0, len(queue)/2+1)
- for hash := range queue {
- data, err := srcDb.Node(hash)
- if err != nil {
- t.Fatalf("failed to retrieve node data for %x: %v", hash, err)
- }
- results = append(results, SyncResult{hash, data})
-
- if len(results) >= cap(results) {
- break
- }
- }
- // Feed the retrieved results back and queue new tasks
- if _, index, err := sched.Process(results); err != nil {
- t.Fatalf("failed to process result #%d: %v", index, err)
- }
- if index, err := sched.Commit(diskdb); err != nil {
- t.Fatalf("failed to commit data #%d: %v", index, err)
- }
- for _, result := range results {
- delete(queue, result.Hash)
- }
- for _, hash := range sched.Missing(10000) {
- queue[hash] = struct{}{}
- }
- }
- // Cross check that the two tries are in sync
- checkTrieContents(t, triedb, srcTrie.Root(), srcData)
-}
-
-// Tests that a trie sync will not request nodes multiple times, even if they
-// have such references.
-func TestDuplicateAvoidanceSync(t *testing.T) {
- // Create a random trie to copy
- srcDb, srcTrie, srcData := makeTestTrie()
-
- // Create a destination trie and sync with the scheduler
- diskdb := db.NewMemDatabase()
- triedb := NewDatabase(diskdb)
- sched := NewSync(srcTrie.Hash(), diskdb, nil)
-
- queue := append([]common.Hash{}, sched.Missing(0)...)
- requested := make(map[common.Hash]struct{})
-
- for len(queue) > 0 {
- results := make([]SyncResult, len(queue))
- for i, hash := range queue {
- data, err := srcDb.Node(hash)
- if err != nil {
- t.Fatalf("failed to retrieve node data for %x: %v", hash, err)
- }
- if _, ok := requested[hash]; ok {
- t.Errorf("hash %x already requested once", hash)
- }
- requested[hash] = struct{}{}
-
- results[i] = SyncResult{hash, data}
- }
- if _, index, err := sched.Process(results); err != nil {
- t.Fatalf("failed to process result #%d: %v", index, err)
- }
- if index, err := sched.Commit(diskdb); err != nil {
- t.Fatalf("failed to commit data #%d: %v", index, err)
- }
- queue = append(queue[:0], sched.Missing(0)...)
- }
- // Cross check that the two tries are in sync
- checkTrieContents(t, triedb, srcTrie.Root(), srcData)
-}
-
-// Tests that at any point in time during a sync, only complete sub-tries are in
-// the database.
-func TestIncompleteSync(t *testing.T) {
- // Create a random trie to copy
- srcDb, srcTrie, _ := makeTestTrie()
-
- // Create a destination trie and sync with the scheduler
- diskdb := db.NewMemDatabase()
- triedb := NewDatabase(diskdb)
- sched := NewSync(srcTrie.Hash(), diskdb, nil)
-
- added := []common.Hash{}
- queue := append([]common.Hash{}, sched.Missing(1)...)
- for len(queue) > 0 {
- // Fetch a batch of trie nodes
- results := make([]SyncResult, len(queue))
- for i, hash := range queue {
- data, err := srcDb.Node(hash)
- if err != nil {
- t.Fatalf("failed to retrieve node data for %x: %v", hash, err)
- }
- results[i] = SyncResult{hash, data}
- }
- // Process each of the trie nodes
- if _, index, err := sched.Process(results); err != nil {
- t.Fatalf("failed to process result #%d: %v", index, err)
- }
- if index, err := sched.Commit(diskdb); err != nil {
- t.Fatalf("failed to commit data #%d: %v", index, err)
- }
- for _, result := range results {
- added = append(added, result.Hash)
- }
- // Check that all known sub-tries in the synced trie are complete
- for _, root := range added {
- if err := checkTrieConsistency(triedb, root); err != nil {
- t.Fatalf("trie inconsistent: %v", err)
- }
- }
- // Fetch the next batch to retrieve
- queue = append(queue[:0], sched.Missing(1)...)
- }
- // Sanity check that removing any node from the database is detected
- for _, node := range added[1:] {
- key := node.Bytes()
- value, _ := diskdb.Get(key)
-
- diskdb.Delete(key)
- if err := checkTrieConsistency(triedb, added[0]); err == nil {
- t.Fatalf("trie inconsistency not caught, missing: %x", key)
- }
- diskdb.Put(key, value)
- }
-}
diff --git a/internal/trie/trie.go b/internal/trie/trie.go
deleted file mode 100644
index af424d4ac..000000000
--- a/internal/trie/trie.go
+++ /dev/null
@@ -1,474 +0,0 @@
-// Copyright 2014 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see .
-
-// Package trie implements Merkle Patricia Tries.
-package trie
-
-import (
- "bytes"
- "fmt"
-
- "github.com/ethereum/go-ethereum/common"
- "github.com/ethereum/go-ethereum/crypto"
- "github.com/ethereum/go-ethereum/log"
- "github.com/ethereum/go-ethereum/metrics"
-)
-
-var (
- // emptyRoot is the known root hash of an empty trie.
- emptyRoot = common.HexToHash("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421")
-
- // emptyState is the known hash of an empty state trie entry.
- emptyState = crypto.Keccak256Hash(nil)
-)
-
-var (
- cacheMissCounter = metrics.NewRegisteredCounter("trie/cachemiss", nil)
- cacheUnloadCounter = metrics.NewRegisteredCounter("trie/cacheunload", nil)
-)
-
-// CacheMisses retrieves a global counter measuring the number of cache misses
-// the trie had since process startup. This isn't useful for anything apart from
-// trie debugging purposes.
-func CacheMisses() int64 {
- return cacheMissCounter.Count()
-}
-
-// CacheUnloads retrieves a global counter measuring the number of cache unloads
-// the trie did since process startup. This isn't useful for anything apart from
-// trie debugging purposes.
-func CacheUnloads() int64 {
- return cacheUnloadCounter.Count()
-}
-
-// LeafCallback is a callback type invoked when a trie operation reaches a leaf
-// node. It's used by state sync and commit to allow handling external references
-// between account and storage tries.
-type LeafCallback func(leaf []byte, parent common.Hash) error
-
-// Trie is a Merkle Patricia Trie.
-// The zero value is an empty trie with no database.
-// Use New to create a trie that sits on top of a database.
-//
-// Trie is not safe for concurrent use.
-type Trie struct {
- db *Database
- root node
-
- // Cache generation values.
- // cachegen increases by one with each commit operation.
- // new nodes are tagged with the current generation and unloaded
- // when their generation is older than than cachegen-cachelimit.
- cachegen, cachelimit uint16
-}
-
-// SetCacheLimit sets the number of 'cache generations' to keep.
-// A cache generation is created by a call to Commit.
-func (t *Trie) SetCacheLimit(l uint16) {
- t.cachelimit = l
-}
-
-// newFlag returns the cache flag value for a newly created node.
-func (t *Trie) newFlag() nodeFlag {
- return nodeFlag{dirty: true, gen: t.cachegen}
-}
-
-// New creates a trie with an existing root node from db.
-//
-// If root is the zero hash or the sha3 hash of an empty string, the
-// trie is initially empty and does not require a database. Otherwise,
-// New will panic if db is nil and returns a MissingNodeError if root does
-// not exist in the database. Accessing the trie loads nodes from db on demand.
-func New(root common.Hash, db *Database) (*Trie, error) {
- if db == nil {
- panic("trie.New called without a database")
- }
- trie := &Trie{
- db: db,
- }
- if root != (common.Hash{}) && root != emptyRoot {
- rootnode, err := trie.resolveHash(root[:], nil)
- if err != nil {
- return nil, err
- }
- trie.root = rootnode
- }
- return trie, nil
-}
-
-// NodeIterator returns an iterator that returns nodes of the trie. Iteration starts at
-// the key after the given start key.
-func (t *Trie) NodeIterator(start []byte) NodeIterator {
- return newNodeIterator(t, start)
-}
-
-// Get returns the value for key stored in the trie.
-// The value bytes must not be modified by the caller.
-func (t *Trie) Get(key []byte) []byte {
- res, err := t.TryGet(key)
- if err != nil {
- log.Error(fmt.Sprintf("Unhandled trie error: %v", err))
- }
- return res
-}
-
-// TryGet returns the value for key stored in the trie.
-// The value bytes must not be modified by the caller.
-// If a node was not found in the database, a MissingNodeError is returned.
-func (t *Trie) TryGet(key []byte) ([]byte, error) {
- key = keybytesToHex(key)
- value, newroot, didResolve, err := t.tryGet(t.root, key, 0)
- if err == nil && didResolve {
- t.root = newroot
- }
- return value, err
-}
-
-func (t *Trie) tryGet(origNode node, key []byte, pos int) (value []byte, newnode node, didResolve bool, err error) {
- switch n := (origNode).(type) {
- case nil:
- return nil, nil, false, nil
- case valueNode:
- return n, n, false, nil
- case *shortNode:
- if len(key)-pos < len(n.Key) || !bytes.Equal(n.Key, key[pos:pos+len(n.Key)]) {
- // key not found in trie
- return nil, n, false, nil
- }
- value, newnode, didResolve, err = t.tryGet(n.Val, key, pos+len(n.Key))
- if err == nil && didResolve {
- n = n.copy()
- n.Val = newnode
- n.flags.gen = t.cachegen
- }
- return value, n, didResolve, err
- case *fullNode:
- value, newnode, didResolve, err = t.tryGet(n.Children[key[pos]], key, pos+1)
- if err == nil && didResolve {
- n = n.copy()
- n.flags.gen = t.cachegen
- n.Children[key[pos]] = newnode
- }
- return value, n, didResolve, err
- case hashNode:
- child, err := t.resolveHash(n, key[:pos])
- if err != nil {
- return nil, n, true, err
- }
- value, newnode, _, err := t.tryGet(child, key, pos)
- return value, newnode, true, err
- default:
- panic(fmt.Sprintf("%T: invalid node: %v", origNode, origNode))
- }
-}
-
-// Update associates key with value in the trie. Subsequent calls to
-// Get will return value. If value has length zero, any existing value
-// is deleted from the trie and calls to Get will return nil.
-//
-// The value bytes must not be modified by the caller while they are
-// stored in the trie.
-func (t *Trie) Update(key, value []byte) {
- if err := t.TryUpdate(key, value); err != nil {
- log.Error(fmt.Sprintf("Unhandled trie error: %v", err))
- }
-}
-
-// TryUpdate associates key with value in the trie. Subsequent calls to
-// Get will return value. If value has length zero, any existing value
-// is deleted from the trie and calls to Get will return nil.
-//
-// The value bytes must not be modified by the caller while they are
-// stored in the trie.
-//
-// If a node was not found in the database, a MissingNodeError is returned.
-func (t *Trie) TryUpdate(key, value []byte) error {
- k := keybytesToHex(key)
- if len(value) != 0 {
- _, n, err := t.insert(t.root, nil, k, valueNode(value))
- if err != nil {
- return err
- }
- t.root = n
- } else {
- _, n, err := t.delete(t.root, nil, k)
- if err != nil {
- return err
- }
- t.root = n
- }
- return nil
-}
-
-func (t *Trie) insert(n node, prefix, key []byte, value node) (bool, node, error) {
- if len(key) == 0 {
- if v, ok := n.(valueNode); ok {
- return !bytes.Equal(v, value.(valueNode)), value, nil
- }
- return true, value, nil
- }
- switch n := n.(type) {
- case *shortNode:
- matchlen := prefixLen(key, n.Key)
- // If the whole key matches, keep this short node as is
- // and only update the value.
- if matchlen == len(n.Key) {
- dirty, nn, err := t.insert(n.Val, append(prefix, key[:matchlen]...), key[matchlen:], value)
- if !dirty || err != nil {
- return false, n, err
- }
- return true, &shortNode{n.Key, nn, t.newFlag()}, nil
- }
- // Otherwise branch out at the index where they differ.
- branch := &fullNode{flags: t.newFlag()}
- var err error
- _, branch.Children[n.Key[matchlen]], err = t.insert(nil, append(prefix, n.Key[:matchlen+1]...), n.Key[matchlen+1:], n.Val)
- if err != nil {
- return false, nil, err
- }
- _, branch.Children[key[matchlen]], err = t.insert(nil, append(prefix, key[:matchlen+1]...), key[matchlen+1:], value)
- if err != nil {
- return false, nil, err
- }
- // Replace this shortNode with the branch if it occurs at index 0.
- if matchlen == 0 {
- return true, branch, nil
- }
- // Otherwise, replace it with a short node leading up to the branch.
- return true, &shortNode{key[:matchlen], branch, t.newFlag()}, nil
-
- case *fullNode:
- dirty, nn, err := t.insert(n.Children[key[0]], append(prefix, key[0]), key[1:], value)
- if !dirty || err != nil {
- return false, n, err
- }
- n = n.copy()
- n.flags = t.newFlag()
- n.Children[key[0]] = nn
- return true, n, nil
-
- case nil:
- return true, &shortNode{key, value, t.newFlag()}, nil
-
- case hashNode:
- // We've hit a part of the trie that isn't loaded yet. Load
- // the node and insert into it. This leaves all child nodes on
- // the path to the value in the trie.
- rn, err := t.resolveHash(n, prefix)
- if err != nil {
- return false, nil, err
- }
- dirty, nn, err := t.insert(rn, prefix, key, value)
- if !dirty || err != nil {
- return false, rn, err
- }
- return true, nn, nil
-
- default:
- panic(fmt.Sprintf("%T: invalid node: %v", n, n))
- }
-}
-
-// Delete removes any existing value for key from the trie.
-func (t *Trie) Delete(key []byte) {
- if err := t.TryDelete(key); err != nil {
- log.Error(fmt.Sprintf("Unhandled trie error: %v", err))
- }
-}
-
-// TryDelete removes any existing value for key from the trie.
-// If a node was not found in the database, a MissingNodeError is returned.
-func (t *Trie) TryDelete(key []byte) error {
- k := keybytesToHex(key)
- _, n, err := t.delete(t.root, nil, k)
- if err != nil {
- return err
- }
- t.root = n
- return nil
-}
-
-// delete returns the new root of the trie with key deleted.
-// It reduces the trie to minimal form by simplifying
-// nodes on the way up after deleting recursively.
-func (t *Trie) delete(n node, prefix, key []byte) (bool, node, error) {
- switch n := n.(type) {
- case *shortNode:
- matchlen := prefixLen(key, n.Key)
- if matchlen < len(n.Key) {
- return false, n, nil // don't replace n on mismatch
- }
- if matchlen == len(key) {
- return true, nil, nil // remove n entirely for whole matches
- }
- // The key is longer than n.Key. Remove the remaining suffix
- // from the subtrie. Child can never be nil here since the
- // subtrie must contain at least two other values with keys
- // longer than n.Key.
- dirty, child, err := t.delete(n.Val, append(prefix, key[:len(n.Key)]...), key[len(n.Key):])
- if !dirty || err != nil {
- return false, n, err
- }
- switch child := child.(type) {
- case *shortNode:
- // Deleting from the subtrie reduced it to another
- // short node. Merge the nodes to avoid creating a
- // shortNode{..., shortNode{...}}. Use concat (which
- // always creates a new slice) instead of append to
- // avoid modifying n.Key since it might be shared with
- // other nodes.
- return true, &shortNode{concat(n.Key, child.Key...), child.Val, t.newFlag()}, nil
- default:
- return true, &shortNode{n.Key, child, t.newFlag()}, nil
- }
-
- case *fullNode:
- dirty, nn, err := t.delete(n.Children[key[0]], append(prefix, key[0]), key[1:])
- if !dirty || err != nil {
- return false, n, err
- }
- n = n.copy()
- n.flags = t.newFlag()
- n.Children[key[0]] = nn
-
- // Check how many non-nil entries are left after deleting and
- // reduce the full node to a short node if only one entry is
- // left. Since n must've contained at least two children
- // before deletion (otherwise it would not be a full node) n
- // can never be reduced to nil.
- //
- // When the loop is done, pos contains the index of the single
- // value that is left in n or -2 if n contains at least two
- // values.
- pos := -1
- for i, cld := range &n.Children {
- if cld != nil {
- if pos == -1 {
- pos = i
- } else {
- pos = -2
- break
- }
- }
- }
- if pos >= 0 {
- if pos != 16 {
- // If the remaining entry is a short node, it replaces
- // n and its key gets the missing nibble tacked to the
- // front. This avoids creating an invalid
- // shortNode{..., shortNode{...}}. Since the entry
- // might not be loaded yet, resolve it just for this
- // check.
- cnode, err := t.resolve(n.Children[pos], prefix)
- if err != nil {
- return false, nil, err
- }
- if cnode, ok := cnode.(*shortNode); ok {
- k := append([]byte{byte(pos)}, cnode.Key...)
- return true, &shortNode{k, cnode.Val, t.newFlag()}, nil
- }
- }
- // Otherwise, n is replaced by a one-nibble short node
- // containing the child.
- return true, &shortNode{[]byte{byte(pos)}, n.Children[pos], t.newFlag()}, nil
- }
- // n still contains at least two values and cannot be reduced.
- return true, n, nil
-
- case valueNode:
- return true, nil, nil
-
- case nil:
- return false, nil, nil
-
- case hashNode:
- // We've hit a part of the trie that isn't loaded yet. Load
- // the node and delete from it. This leaves all child nodes on
- // the path to the value in the trie.
- rn, err := t.resolveHash(n, prefix)
- if err != nil {
- return false, nil, err
- }
- dirty, nn, err := t.delete(rn, prefix, key)
- if !dirty || err != nil {
- return false, rn, err
- }
- return true, nn, nil
-
- default:
- panic(fmt.Sprintf("%T: invalid node: %v (%v)", n, n, key))
- }
-}
-
-func concat(s1 []byte, s2 ...byte) []byte {
- r := make([]byte, len(s1)+len(s2))
- copy(r, s1)
- copy(r[len(s1):], s2)
- return r
-}
-
-func (t *Trie) resolve(n node, prefix []byte) (node, error) {
- if n, ok := n.(hashNode); ok {
- return t.resolveHash(n, prefix)
- }
- return n, nil
-}
-
-func (t *Trie) resolveHash(n hashNode, prefix []byte) (node, error) {
- cacheMissCounter.Inc(1)
-
- hash := common.BytesToHash(n)
- if node := t.db.node(hash, t.cachegen); node != nil {
- return node, nil
- }
- return nil, &MissingNodeError{NodeHash: hash, Path: prefix}
-}
-
-// Root returns the root hash of the trie.
-// Deprecated: use Hash instead.
-func (t *Trie) Root() []byte { return t.Hash().Bytes() }
-
-// Hash returns the root hash of the trie. It does not write to the
-// database and can be used even if the trie doesn't have one.
-func (t *Trie) Hash() common.Hash {
- hash, cached, _ := t.hashRoot(nil, nil)
- t.root = cached
- return common.BytesToHash(hash.(hashNode))
-}
-
-// Commit writes all nodes to the trie's memory database, tracking the internal
-// and external (for account tries) references.
-func (t *Trie) Commit(onleaf LeafCallback) (root common.Hash, err error) {
- if t.db == nil {
- panic("commit called on trie with nil database")
- }
- hash, cached, err := t.hashRoot(t.db, onleaf)
- if err != nil {
- return common.Hash{}, err
- }
- t.root = cached
- t.cachegen++
- return common.BytesToHash(hash.(hashNode)), nil
-}
-
-func (t *Trie) hashRoot(db *Database, onleaf LeafCallback) (node, node, error) {
- if t.root == nil {
- return hashNode(emptyRoot.Bytes()), nil, nil
- }
- h := newHasher(t.cachegen, t.cachelimit, onleaf)
- defer returnHasherToPool(h)
- return h.hash(t.root, db, true)
-}
diff --git a/internal/trie/trie_test.go b/internal/trie/trie_test.go
deleted file mode 100644
index cff9a4c86..000000000
--- a/internal/trie/trie_test.go
+++ /dev/null
@@ -1,615 +0,0 @@
-// Copyright 2014 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see .
-
-package trie
-
-import (
- "bytes"
- "encoding/binary"
- "errors"
- "fmt"
- "io/ioutil"
- "math/big"
- "math/rand"
- "os"
- "reflect"
- "testing"
- "testing/quick"
-
- "github.com/davecgh/go-spew/spew"
- "github.com/ethereum/go-ethereum/common"
- "github.com/ethereum/go-ethereum/crypto"
- "github.com/ethereum/go-ethereum/rlp"
- "github.com/harmony-one/harmony/internal/db"
-)
-
-func init() {
- spew.Config.Indent = " "
- spew.Config.DisableMethods = false
-}
-
-// Used for testing
-func newEmpty() *Trie {
- trie, _ := New(common.Hash{}, NewDatabase(db.NewMemDatabase()))
- return trie
-}
-
-func TestEmptyTrie(t *testing.T) {
- var trie Trie
- res := trie.Hash()
- exp := emptyRoot
- if res != common.Hash(exp) {
- t.Errorf("expected %x got %x", exp, res)
- }
-}
-
-func TestNull(t *testing.T) {
- var trie Trie
- key := make([]byte, 32)
- value := []byte("test")
- trie.Update(key, value)
- if !bytes.Equal(trie.Get(key), value) {
- t.Fatal("wrong value")
- }
-}
-
-func TestMissingRoot(t *testing.T) {
- trie, err := New(common.HexToHash("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33"), NewDatabase(db.NewMemDatabase()))
- if trie != nil {
- t.Error("New returned non-nil trie for invalid root")
- }
- if _, ok := err.(*MissingNodeError); !ok {
- t.Errorf("New returned wrong error: %v", err)
- }
-}
-
-func TestMissingNodeDisk(t *testing.T) { testMissingNode(t, false) }
-func TestMissingNodeMemonly(t *testing.T) { testMissingNode(t, true) }
-
-func testMissingNode(t *testing.T, memonly bool) {
- diskdb := db.NewMemDatabase()
- triedb := NewDatabase(diskdb)
-
- trie, _ := New(common.Hash{}, triedb)
- updateString(trie, "120000", "qwerqwerqwerqwerqwerqwerqwerqwer")
- updateString(trie, "123456", "asdfasdfasdfasdfasdfasdfasdfasdf")
- root, _ := trie.Commit(nil)
- if !memonly {
- triedb.Commit(root, true)
- }
-
- trie, _ = New(root, triedb)
- _, err := trie.TryGet([]byte("120000"))
- if err != nil {
- t.Errorf("Unexpected error: %v", err)
- }
- trie, _ = New(root, triedb)
- _, err = trie.TryGet([]byte("120099"))
- if err != nil {
- t.Errorf("Unexpected error: %v", err)
- }
- trie, _ = New(root, triedb)
- _, err = trie.TryGet([]byte("123456"))
- if err != nil {
- t.Errorf("Unexpected error: %v", err)
- }
- trie, _ = New(root, triedb)
- err = trie.TryUpdate([]byte("120099"), []byte("zxcvzxcvzxcvzxcvzxcvzxcvzxcvzxcv"))
- if err != nil {
- t.Errorf("Unexpected error: %v", err)
- }
- trie, _ = New(root, triedb)
- err = trie.TryDelete([]byte("123456"))
- if err != nil {
- t.Errorf("Unexpected error: %v", err)
- }
-
- hash := common.HexToHash("0xe1d943cc8f061a0c0b98162830b970395ac9315654824bf21b73b891365262f9")
- if memonly {
- delete(triedb.nodes, hash)
- } else {
- diskdb.Delete(hash[:])
- }
-
- trie, _ = New(root, triedb)
- _, err = trie.TryGet([]byte("120000"))
- if _, ok := err.(*MissingNodeError); !ok {
- t.Errorf("Wrong error: %v", err)
- }
- trie, _ = New(root, triedb)
- _, err = trie.TryGet([]byte("120099"))
- if _, ok := err.(*MissingNodeError); !ok {
- t.Errorf("Wrong error: %v", err)
- }
- trie, _ = New(root, triedb)
- _, err = trie.TryGet([]byte("123456"))
- if err != nil {
- t.Errorf("Unexpected error: %v", err)
- }
- trie, _ = New(root, triedb)
- err = trie.TryUpdate([]byte("120099"), []byte("zxcv"))
- if _, ok := err.(*MissingNodeError); !ok {
- t.Errorf("Wrong error: %v", err)
- }
- trie, _ = New(root, triedb)
- err = trie.TryDelete([]byte("123456"))
- if _, ok := err.(*MissingNodeError); !ok {
- t.Errorf("Wrong error: %v", err)
- }
-}
-
-func TestInsert(t *testing.T) {
- trie := newEmpty()
-
- updateString(trie, "doe", "reindeer")
- updateString(trie, "dog", "puppy")
- updateString(trie, "dogglesworth", "cat")
-
- exp := common.HexToHash("8aad789dff2f538bca5d8ea56e8abe10f4c7ba3a5dea95fea4cd6e7c3a1168d3")
- root := trie.Hash()
- if root != exp {
- t.Errorf("exp %x got %x", exp, root)
- }
-
- trie = newEmpty()
- updateString(trie, "A", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
-
- exp = common.HexToHash("d23786fb4a010da3ce639d66d5e904a11dbc02746d1ce25029e53290cabf28ab")
- root, err := trie.Commit(nil)
- if err != nil {
- t.Fatalf("commit error: %v", err)
- }
- if root != exp {
- t.Errorf("exp %x got %x", exp, root)
- }
-}
-
-func TestGet(t *testing.T) {
- trie := newEmpty()
- updateString(trie, "doe", "reindeer")
- updateString(trie, "dog", "puppy")
- updateString(trie, "dogglesworth", "cat")
-
- for i := 0; i < 2; i++ {
- res := getString(trie, "dog")
- if !bytes.Equal(res, []byte("puppy")) {
- t.Errorf("expected puppy got %x", res)
- }
-
- unknown := getString(trie, "unknown")
- if unknown != nil {
- t.Errorf("expected nil got %x", unknown)
- }
-
- if i == 1 {
- return
- }
- trie.Commit(nil)
- }
-}
-
-func TestDelete(t *testing.T) {
- trie := newEmpty()
- vals := []struct{ k, v string }{
- {"do", "verb"},
- {"ether", "wookiedoo"},
- {"horse", "stallion"},
- {"shaman", "horse"},
- {"doge", "coin"},
- {"ether", ""},
- {"dog", "puppy"},
- {"shaman", ""},
- }
- for _, val := range vals {
- if val.v != "" {
- updateString(trie, val.k, val.v)
- } else {
- deleteString(trie, val.k)
- }
- }
-
- hash := trie.Hash()
- exp := common.HexToHash("5991bb8c6514148a29db676a14ac506cd2cd5775ace63c30a4fe457715e9ac84")
- if hash != exp {
- t.Errorf("expected %x got %x", exp, hash)
- }
-}
-
-func TestEmptyValues(t *testing.T) {
- trie := newEmpty()
-
- vals := []struct{ k, v string }{
- {"do", "verb"},
- {"ether", "wookiedoo"},
- {"horse", "stallion"},
- {"shaman", "horse"},
- {"doge", "coin"},
- {"ether", ""},
- {"dog", "puppy"},
- {"shaman", ""},
- }
- for _, val := range vals {
- updateString(trie, val.k, val.v)
- }
-
- hash := trie.Hash()
- exp := common.HexToHash("5991bb8c6514148a29db676a14ac506cd2cd5775ace63c30a4fe457715e9ac84")
- if hash != exp {
- t.Errorf("expected %x got %x", exp, hash)
- }
-}
-
-func TestReplication(t *testing.T) {
- trie := newEmpty()
- vals := []struct{ k, v string }{
- {"do", "verb"},
- {"ether", "wookiedoo"},
- {"horse", "stallion"},
- {"shaman", "horse"},
- {"doge", "coin"},
- {"dog", "puppy"},
- {"somethingveryoddindeedthis is", "myothernodedata"},
- }
- for _, val := range vals {
- updateString(trie, val.k, val.v)
- }
- exp, err := trie.Commit(nil)
- if err != nil {
- t.Fatalf("commit error: %v", err)
- }
-
- // create a new trie on top of the database and check that lookups work.
- trie2, err := New(exp, trie.db)
- if err != nil {
- t.Fatalf("can't recreate trie at %x: %v", exp, err)
- }
- for _, kv := range vals {
- if string(getString(trie2, kv.k)) != kv.v {
- t.Errorf("trie2 doesn't have %q => %q", kv.k, kv.v)
- }
- }
- hash, err := trie2.Commit(nil)
- if err != nil {
- t.Fatalf("commit error: %v", err)
- }
- if hash != exp {
- t.Errorf("root failure. expected %x got %x", exp, hash)
- }
-
- // perform some insertions on the new trie.
- vals2 := []struct{ k, v string }{
- {"do", "verb"},
- {"ether", "wookiedoo"},
- {"horse", "stallion"},
- // {"shaman", "horse"},
- // {"doge", "coin"},
- // {"ether", ""},
- // {"dog", "puppy"},
- // {"somethingveryoddindeedthis is", "myothernodedata"},
- // {"shaman", ""},
- }
- for _, val := range vals2 {
- updateString(trie2, val.k, val.v)
- }
- if hash := trie2.Hash(); hash != exp {
- t.Errorf("root failure. expected %x got %x", exp, hash)
- }
-}
-
-func TestLargeValue(t *testing.T) {
- trie := newEmpty()
- trie.Update([]byte("key1"), []byte{99, 99, 99, 99})
- trie.Update([]byte("key2"), bytes.Repeat([]byte{1}, 32))
- trie.Hash()
-}
-
-type countingDB struct {
- db.Database
- gets map[string]int
-}
-
-func (db *countingDB) Get(key []byte) ([]byte, error) {
- db.gets[string(key)]++
- return db.Database.Get(key)
-}
-
-// TestCacheUnload checks that decoded nodes are unloaded after a
-// certain number of commit operations.
-func TestCacheUnload(t *testing.T) {
- // Create test trie with two branches.
- trie := newEmpty()
- key1 := "---------------------------------"
- key2 := "---some other branch"
- updateString(trie, key1, "this is the branch of key1.")
- updateString(trie, key2, "this is the branch of key2.")
-
- root, _ := trie.Commit(nil)
- trie.db.Commit(root, true)
-
- // Commit the trie repeatedly and access key1.
- // The branch containing it is loaded from DB exactly two times:
- // in the 0th and 6th iteration.
- db := &countingDB{Database: trie.db.diskdb, gets: make(map[string]int)}
- trie, _ = New(root, NewDatabase(db))
- trie.SetCacheLimit(5)
- for i := 0; i < 12; i++ {
- getString(trie, key1)
- trie.Commit(nil)
- }
- // Check that it got loaded two times.
- for dbkey, count := range db.gets {
- if count != 2 {
- t.Errorf("db key %x loaded %d times, want %d times", []byte(dbkey), count, 2)
- }
- }
-}
-
-// randTest performs random trie operations.
-// Instances of this test are created by Generate.
-type randTest []randTestStep
-
-type randTestStep struct {
- op int
- key []byte // for opUpdate, opDelete, opGet
- value []byte // for opUpdate
- err error // for debugging
-}
-
-const (
- opUpdate = iota
- opDelete
- opGet
- opCommit
- opHash
- opReset
- opItercheckhash
- opCheckCacheInvariant
- opMax // boundary value, not an actual op
-)
-
-func (randTest) Generate(r *rand.Rand, size int) reflect.Value {
- var allKeys [][]byte
- genKey := func() []byte {
- if len(allKeys) < 2 || r.Intn(100) < 10 {
- // new key
- key := make([]byte, r.Intn(50))
- r.Read(key)
- allKeys = append(allKeys, key)
- return key
- }
- // use existing key
- return allKeys[r.Intn(len(allKeys))]
- }
-
- var steps randTest
- for i := 0; i < size; i++ {
- step := randTestStep{op: r.Intn(opMax)}
- switch step.op {
- case opUpdate:
- step.key = genKey()
- step.value = make([]byte, 8)
- binary.BigEndian.PutUint64(step.value, uint64(i))
- case opGet, opDelete:
- step.key = genKey()
- }
- steps = append(steps, step)
- }
- return reflect.ValueOf(steps)
-}
-
-func runRandTest(rt randTest) bool {
- triedb := NewDatabase(db.NewMemDatabase())
-
- tr, _ := New(common.Hash{}, triedb)
- values := make(map[string]string) // tracks content of the trie
-
- for i, step := range rt {
- switch step.op {
- case opUpdate:
- tr.Update(step.key, step.value)
- values[string(step.key)] = string(step.value)
- case opDelete:
- tr.Delete(step.key)
- delete(values, string(step.key))
- case opGet:
- v := tr.Get(step.key)
- want := values[string(step.key)]
- if string(v) != want {
- rt[i].err = fmt.Errorf("mismatch for key 0x%x, got 0x%x want 0x%x", step.key, v, want)
- }
- case opCommit:
- _, rt[i].err = tr.Commit(nil)
- case opHash:
- tr.Hash()
- case opReset:
- hash, err := tr.Commit(nil)
- if err != nil {
- rt[i].err = err
- return false
- }
- newtr, err := New(hash, triedb)
- if err != nil {
- rt[i].err = err
- return false
- }
- tr = newtr
- case opItercheckhash:
- checktr, _ := New(common.Hash{}, triedb)
- it := NewIterator(tr.NodeIterator(nil))
- for it.Next() {
- checktr.Update(it.Key, it.Value)
- }
- if tr.Hash() != checktr.Hash() {
- rt[i].err = fmt.Errorf("hash mismatch in opItercheckhash")
- }
- case opCheckCacheInvariant:
- rt[i].err = checkCacheInvariant(tr.root, nil, tr.cachegen, false, 0)
- }
- // Abort the test on error.
- if rt[i].err != nil {
- return false
- }
- }
- return true
-}
-
-func checkCacheInvariant(n, parent node, parentCachegen uint16, parentDirty bool, depth int) error {
- var children []node
- var flag nodeFlag
- switch n := n.(type) {
- case *shortNode:
- flag = n.flags
- children = []node{n.Val}
- case *fullNode:
- flag = n.flags
- children = n.Children[:]
- default:
- return nil
- }
-
- errorf := func(format string, args ...interface{}) error {
- msg := fmt.Sprintf(format, args...)
- msg += fmt.Sprintf("\nat depth %d node %s", depth, spew.Sdump(n))
- msg += fmt.Sprintf("parent: %s", spew.Sdump(parent))
- return errors.New(msg)
- }
- if flag.gen > parentCachegen {
- return errorf("cache invariant violation: %d > %d\n", flag.gen, parentCachegen)
- }
- if depth > 0 && !parentDirty && flag.dirty {
- return errorf("cache invariant violation: %d > %d\n", flag.gen, parentCachegen)
- }
- for _, child := range children {
- if err := checkCacheInvariant(child, n, flag.gen, flag.dirty, depth+1); err != nil {
- return err
- }
- }
- return nil
-}
-
-func TestRandom(t *testing.T) {
- if err := quick.Check(runRandTest, nil); err != nil {
- if cerr, ok := err.(*quick.CheckError); ok {
- t.Fatalf("random test iteration %d failed: %s", cerr.Count, spew.Sdump(cerr.In))
- }
- t.Fatal(err)
- }
-}
-
-func BenchmarkGet(b *testing.B) { benchGet(b, false) }
-func BenchmarkGetDB(b *testing.B) { benchGet(b, true) }
-func BenchmarkUpdateBE(b *testing.B) { benchUpdate(b, binary.BigEndian) }
-func BenchmarkUpdateLE(b *testing.B) { benchUpdate(b, binary.LittleEndian) }
-
-const benchElemCount = 20000
-
-func benchGet(b *testing.B, commit bool) {
- trie := new(Trie)
- if commit {
- _, tmpdb := tempDB()
- trie, _ = New(common.Hash{}, tmpdb)
- }
- k := make([]byte, 32)
- for i := 0; i < benchElemCount; i++ {
- binary.LittleEndian.PutUint64(k, uint64(i))
- trie.Update(k, k)
- }
- binary.LittleEndian.PutUint64(k, benchElemCount/2)
- if commit {
- trie.Commit(nil)
- }
-
- b.ResetTimer()
- for i := 0; i < b.N; i++ {
- trie.Get(k)
- }
- b.StopTimer()
-
- if commit {
- ldb := trie.db.diskdb.(*db.LDBDatabase)
- ldb.Close()
- os.RemoveAll(ldb.Path())
- }
-}
-
-func benchUpdate(b *testing.B, e binary.ByteOrder) *Trie {
- trie := newEmpty()
- k := make([]byte, 32)
- for i := 0; i < b.N; i++ {
- e.PutUint64(k, uint64(i))
- trie.Update(k, k)
- }
- return trie
-}
-
-// Benchmarks the trie hashing. Since the trie caches the result of any operation,
-// we cannot use b.N as the number of hashing rouns, since all rounds apart from
-// the first one will be NOOP. As such, we'll use b.N as the number of account to
-// insert into the trie before measuring the hashing.
-func BenchmarkHash(b *testing.B) {
- // Make the random benchmark deterministic
- random := rand.New(rand.NewSource(0))
-
- // Create a realistic account trie to hash
- addresses := make([][20]byte, b.N)
- for i := 0; i < len(addresses); i++ {
- for j := 0; j < len(addresses[i]); j++ {
- addresses[i][j] = byte(random.Intn(256))
- }
- }
- accounts := make([][]byte, len(addresses))
- for i := 0; i < len(accounts); i++ {
- var (
- nonce = uint64(random.Int63())
- balance = new(big.Int).Rand(random, new(big.Int).Exp(common.Big2, common.Big256, nil))
- root = emptyRoot
- code = crypto.Keccak256(nil)
- )
- accounts[i], _ = rlp.EncodeToBytes([]interface{}{nonce, balance, root, code})
- }
- // Insert the accounts into the trie and hash it
- trie := newEmpty()
- for i := 0; i < len(addresses); i++ {
- trie.Update(crypto.Keccak256(addresses[i][:]), accounts[i])
- }
- b.ResetTimer()
- b.ReportAllocs()
- trie.Hash()
-}
-
-func tempDB() (string, *Database) {
- dir, err := ioutil.TempDir("", "trie-bench")
- if err != nil {
- panic(fmt.Sprintf("can't create temporary directory: %v", err))
- }
- diskdb, err := db.NewLDBDatabase(dir, 256, 0)
- if err != nil {
- panic(fmt.Sprintf("can't create temporary database: %v", err))
- }
- return dir, NewDatabase(diskdb)
-}
-
-func getString(trie *Trie, k string) []byte {
- return trie.Get([]byte(k))
-}
-
-func updateString(trie *Trie, k, v string) {
- trie.Update([]byte(k), []byte(v))
-}
-
-func deleteString(trie *Trie, k string) {
- trie.Delete([]byte(k))
-}
diff --git a/log/CONTRIBUTORS b/log/CONTRIBUTORS
deleted file mode 100644
index a0866713b..000000000
--- a/log/CONTRIBUTORS
+++ /dev/null
@@ -1,11 +0,0 @@
-Contributors to log15:
-
-- Aaron L
-- Alan Shreve
-- Chris Hines
-- Ciaran Downey
-- Dmitry Chestnykh
-- Evan Shaw
-- Péter Szilágyi
-- Trevor Gattis
-- Vincent Vanackere
diff --git a/log/LICENSE b/log/LICENSE
deleted file mode 100644
index 5f0d1fb6a..000000000
--- a/log/LICENSE
+++ /dev/null
@@ -1,13 +0,0 @@
-Copyright 2014 Alan Shreve
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
diff --git a/log/README.md b/log/README.md
deleted file mode 100644
index b4476577b..000000000
--- a/log/README.md
+++ /dev/null
@@ -1,77 +0,0 @@
-
-
-# log15 [](https://godoc.org/github.com/inconshreveable/log15) [](https://travis-ci.org/inconshreveable/log15)
-
-Package log15 provides an opinionated, simple toolkit for best-practice logging in Go (golang) that is both human and machine readable. It is modeled after the Go standard library's [`io`](http://golang.org/pkg/io/) and [`net/http`](http://golang.org/pkg/net/http/) packages and is an alternative to the standard library's [`log`](http://golang.org/pkg/log/) package.
-
-## Features
-- A simple, easy-to-understand API
-- Promotes structured logging by encouraging use of key/value pairs
-- Child loggers which inherit and add their own private context
-- Lazy evaluation of expensive operations
-- Simple Handler interface allowing for construction of flexible, custom logging configurations with a tiny API.
-- Color terminal support
-- Built-in support for logging to files, streams, syslog, and the network
-- Support for forking records to multiple handlers, buffering records for output, failing over from failed handler writes, + more
-
-## Versioning
-The API of the master branch of log15 should always be considered unstable. If you want to rely on a stable API,
-you must vendor the library.
-
-## Importing
-
-```go
-import log "github.com/inconshreveable/log15"
-```
-
-## Examples
-
-```go
-// all loggers can have key/value context
-srvlog := log.New("module", "app/server")
-
-// all log messages can have key/value context
-srvlog.Warn("abnormal conn rate", "rate", curRate, "low", lowRate, "high", highRate)
-
-// child loggers with inherited context
-connlog := srvlog.New("raddr", c.RemoteAddr())
-connlog.Info("connection open")
-
-// lazy evaluation
-connlog.Debug("ping remote", "latency", log.Lazy{pingRemote})
-
-// flexible configuration
-srvlog.SetHandler(log.MultiHandler(
- log.StreamHandler(os.Stderr, log.LogfmtFormat()),
- log.LvlFilterHandler(
- log.LvlError,
- log.Must.FileHandler("errors.json", log.JSONFormat()))))
-```
-
-Will result in output that looks like this:
-
-```
-WARN[06-17|21:58:10] abnormal conn rate module=app/server rate=0.500 low=0.100 high=0.800
-INFO[06-17|21:58:10] connection open module=app/server raddr=10.0.0.1
-```
-
-## Breaking API Changes
-The following commits broke API stability. This reference is intended to help you understand the consequences of updating to a newer version
-of log15.
-
-- 57a084d014d4150152b19e4e531399a7145d1540 - Added a `Get()` method to the `Logger` interface to retrieve the current handler
-- 93404652ee366648fa622b64d1e2b67d75a3094a - `Record` field `Call` changed to `stack.Call` with switch to `github.com/go-stack/stack`
-- a5e7613673c73281f58e15a87d2cf0cf111e8152 - Restored `syslog.Priority` argument to the `SyslogXxx` handler constructors
-
-## FAQ
-
-### The varargs style is brittle and error prone! Can I have type safety please?
-Yes. Use `log.Ctx`:
-
-```go
-srvlog := log.New(log.Ctx{"module": "app/server"})
-srvlog.Warn("abnormal conn rate", log.Ctx{"rate": curRate, "low": lowRate, "high": highRate})
-```
-
-## License
-Apache
diff --git a/log/README_ETHEREUM.md b/log/README_ETHEREUM.md
deleted file mode 100644
index f6c42ccc0..000000000
--- a/log/README_ETHEREUM.md
+++ /dev/null
@@ -1,5 +0,0 @@
-This package is a fork of https://github.com/inconshreveable/log15, with some
-minor modifications required by the go-ethereum codebase:
-
- * Support for log level `trace`
- * Modified behavior to exit on `critical` failure
diff --git a/log/doc.go b/log/doc.go
deleted file mode 100644
index 9aa72bf42..000000000
--- a/log/doc.go
+++ /dev/null
@@ -1,334 +0,0 @@
-package log
-
-/*
-Package log15 provides an opinionated, simple toolkit for best-practice logging that is
-both human and machine readable. It is modeled after the standard library's io and net/http
-packages.
-
-This package enforces you to only log key/value pairs. Keys must be strings. Values may be
-any type that you like. The default output format is logfmt, but you may also choose to use
-JSON instead if that suits you. Here's how you log:
-
- log.Info("page accessed", "path", r.URL.Path, "user_id", user.id)
-
-This will output a line that looks like:
-
- lvl=info t=2014-05-02T16:07:23-0700 msg="page accessed" path=/org/71/profile user_id=9
-
-Getting Started
-
-To get started, you'll want to import the library:
-
- import log "github.com/inconshreveable/log15"
-
-
-Now you're ready to start logging:
-
- func main() {
- log.Info("Program starting", "args", os.Args())
- }
-
-
-Convention
-
-Because recording a human-meaningful message is common and good practice, the first argument to every
-logging method is the value to the *implicit* key 'msg'.
-
-Additionally, the level you choose for a message will be automatically added with the key 'lvl', and so
-will the current timestamp with key 't'.
-
-You may supply any additional context as a set of key/value pairs to the logging function. log15 allows
-you to favor terseness, ordering, and speed over safety. This is a reasonable tradeoff for
-logging functions. You don't need to explicitly state keys/values, log15 understands that they alternate
-in the variadic argument list:
-
- log.Warn("size out of bounds", "low", lowBound, "high", highBound, "val", val)
-
-If you really do favor your type-safety, you may choose to pass a log.Ctx instead:
-
- log.Warn("size out of bounds", log.Ctx{"low": lowBound, "high": highBound, "val": val})
-
-
-Context loggers
-
-Frequently, you want to add context to a logger so that you can track actions associated with it. An http
-request is a good example. You can easily create new loggers that have context that is automatically included
-with each log line:
-
- requestlogger := log.New("path", r.URL.Path)
-
- // later
- requestlogger.Debug("db txn commit", "duration", txnTimer.Finish())
-
-This will output a log line that includes the path context that is attached to the logger:
-
- lvl=dbug t=2014-05-02T16:07:23-0700 path=/repo/12/add_hook msg="db txn commit" duration=0.12
-
-
-Handlers
-
-The Handler interface defines where log lines are printed to and how they are formated. Handler is a
-single interface that is inspired by net/http's handler interface:
-
- type Handler interface {
- Log(r *Record) error
- }
-
-
-Handlers can filter records, format them, or dispatch to multiple other Handlers.
-This package implements a number of Handlers for common logging patterns that are
-easily composed to create flexible, custom logging structures.
-
-Here's an example handler that prints logfmt output to Stdout:
-
- handler := log.StreamHandler(os.Stdout, log.LogfmtFormat())
-
-Here's an example handler that defers to two other handlers. One handler only prints records
-from the rpc package in logfmt to standard out. The other prints records at Error level
-or above in JSON formatted output to the file /var/log/service.json
-
- handler := log.MultiHandler(
- log.LvlFilterHandler(log.LvlError, log.Must.FileHandler("/var/log/service.json", log.JSONFormat())),
- log.MatchFilterHandler("pkg", "app/rpc" log.StdoutHandler())
- )
-
-Logging File Names and Line Numbers
-
-This package implements three Handlers that add debugging information to the
-context, CallerFileHandler, CallerFuncHandler and CallerStackHandler. Here's
-an example that adds the source file and line number of each logging call to
-the context.
-
- h := log.CallerFileHandler(log.StdoutHandler)
- log.Root().SetHandler(h)
- ...
- log.Error("open file", "err", err)
-
-This will output a line that looks like:
-
- lvl=eror t=2014-05-02T16:07:23-0700 msg="open file" err="file not found" caller=data.go:42
-
-Here's an example that logs the call stack rather than just the call site.
-
- h := log.CallerStackHandler("%+v", log.StdoutHandler)
- log.Root().SetHandler(h)
- ...
- log.Error("open file", "err", err)
-
-This will output a line that looks like:
-
- lvl=eror t=2014-05-02T16:07:23-0700 msg="open file" err="file not found" stack="[pkg/data.go:42 pkg/cmd/main.go]"
-
-The "%+v" format instructs the handler to include the path of the source file
-relative to the compile time GOPATH. The github.com/go-stack/stack package
-documents the full list of formatting verbs and modifiers available.
-
-Custom Handlers
-
-The Handler interface is so simple that it's also trivial to write your own. Let's create an
-example handler which tries to write to one handler, but if that fails it falls back to
-writing to another handler and includes the error that it encountered when trying to write
-to the primary. This might be useful when trying to log over a network socket, but if that
-fails you want to log those records to a file on disk.
-
- type BackupHandler struct {
- Primary Handler
- Secondary Handler
- }
-
- func (h *BackupHandler) Log (r *Record) error {
- err := h.Primary.Log(r)
- if err != nil {
- r.Ctx = append(ctx, "primary_err", err)
- return h.Secondary.Log(r)
- }
- return nil
- }
-
-This pattern is so useful that a generic version that handles an arbitrary number of Handlers
-is included as part of this library called FailoverHandler.
-
-Logging Expensive Operations
-
-Sometimes, you want to log values that are extremely expensive to compute, but you don't want to pay
-the price of computing them if you haven't turned up your logging level to a high level of detail.
-
-This package provides a simple type to annotate a logging operation that you want to be evaluated
-lazily, just when it is about to be logged, so that it would not be evaluated if an upstream Handler
-filters it out. Just wrap any function which takes no arguments with the log.Lazy type. For example:
-
- func factorRSAKey() (factors []int) {
- // return the factors of a very large number
- }
-
- log.Debug("factors", log.Lazy{factorRSAKey})
-
-If this message is not logged for any reason (like logging at the Error level), then
-factorRSAKey is never evaluated.
-
-Dynamic context values
-
-The same log.Lazy mechanism can be used to attach context to a logger which you want to be
-evaluated when the message is logged, but not when the logger is created. For example, let's imagine
-a game where you have Player objects:
-
- type Player struct {
- name string
- alive bool
- log.Logger
- }
-
-You always want to log a player's name and whether they're alive or dead, so when you create the player
-object, you might do:
-
- p := &Player{name: name, alive: true}
- p.Logger = log.New("name", p.name, "alive", p.alive)
-
-Only now, even after a player has died, the logger will still report they are alive because the logging
-context is evaluated when the logger was created. By using the Lazy wrapper, we can defer the evaluation
-of whether the player is alive or not to each log message, so that the log records will reflect the player's
-current state no matter when the log message is written:
-
- p := &Player{name: name, alive: true}
- isAlive := func() bool { return p.alive }
- player.Logger = log.New("name", p.name, "alive", log.Lazy{isAlive})
-
-Terminal Format
-
-If log15 detects that stdout is a terminal, it will configure the default
-handler for it (which is log.StdoutHandler) to use TerminalFormat. This format
-logs records nicely for your terminal, including color-coded output based
-on log level.
-
-Error Handling
-
-Becasuse log15 allows you to step around the type system, there are a few ways you can specify
-invalid arguments to the logging functions. You could, for example, wrap something that is not
-a zero-argument function with log.Lazy or pass a context key that is not a string. Since logging libraries
-are typically the mechanism by which errors are reported, it would be onerous for the logging functions
-to return errors. Instead, log15 handles errors by making these guarantees to you:
-
-- Any log record containing an error will still be printed with the error explained to you as part of the log record.
-
-- Any log record containing an error will include the context key LOG15_ERROR, enabling you to easily
-(and if you like, automatically) detect if any of your logging calls are passing bad values.
-
-Understanding this, you might wonder why the Handler interface can return an error value in its Log method. Handlers
-are encouraged to return errors only if they fail to write their log records out to an external source like if the
-syslog daemon is not responding. This allows the construction of useful handlers which cope with those failures
-like the FailoverHandler.
-
-Library Use
-
-log15 is intended to be useful for library authors as a way to provide configurable logging to
-users of their library. Best practice for use in a library is to always disable all output for your logger
-by default and to provide a public Logger instance that consumers of your library can configure. Like so:
-
- package yourlib
-
- import "github.com/inconshreveable/log15"
-
- var Log = log.New()
-
- func init() {
- Log.SetHandler(log.DiscardHandler())
- }
-
-Users of your library may then enable it if they like:
-
- import "github.com/inconshreveable/log15"
- import "example.com/yourlib"
-
- func main() {
- handler := // custom handler setup
- yourlib.Log.SetHandler(handler)
- }
-
-Best practices attaching logger context
-
-The ability to attach context to a logger is a powerful one. Where should you do it and why?
-I favor embedding a Logger directly into any persistent object in my application and adding
-unique, tracing context keys to it. For instance, imagine I am writing a web browser:
-
- type Tab struct {
- url string
- render *RenderingContext
- // ...
-
- Logger
- }
-
- func NewTab(url string) *Tab {
- return &Tab {
- // ...
- url: url,
-
- Logger: log.New("url", url),
- }
- }
-
-When a new tab is created, I assign a logger to it with the url of
-the tab as context so it can easily be traced through the logs.
-Now, whenever we perform any operation with the tab, we'll log with its
-embedded logger and it will include the tab title automatically:
-
- tab.Debug("moved position", "idx", tab.idx)
-
-There's only one problem. What if the tab url changes? We could
-use log.Lazy to make sure the current url is always written, but that
-would mean that we couldn't trace a tab's full lifetime through our
-logs after the user navigate to a new URL.
-
-Instead, think about what values to attach to your loggers the
-same way you think about what to use as a key in a SQL database schema.
-If it's possible to use a natural key that is unique for the lifetime of the
-object, do so. But otherwise, log15's ext package has a handy RandId
-function to let you generate what you might call "surrogate keys"
-They're just random hex identifiers to use for tracing. Back to our
-Tab example, we would prefer to set up our Logger like so:
-
- import logext "github.com/inconshreveable/log15/ext"
-
- t := &Tab {
- // ...
- url: url,
- }
-
- t.Logger = log.New("id", logext.RandId(8), "url", log.Lazy{t.getUrl})
- return t
-
-Now we'll have a unique traceable identifier even across loading new urls, but
-we'll still be able to see the tab's current url in the log messages.
-
-Must
-
-For all Handler functions which can return an error, there is a version of that
-function which will return no error but panics on failure. They are all available
-on the Must object. For example:
-
- log.Must.FileHandler("/path", log.JSONFormat)
- log.Must.NetHandler("tcp", ":1234", log.JSONFormat)
-
-Inspiration and Credit
-
-All of the following excellent projects inspired the design of this library:
-
-code.google.com/p/log4go
-
-github.com/op/go-logging
-
-github.com/technoweenie/grohl
-
-github.com/Sirupsen/logrus
-
-github.com/kr/logfmt
-
-github.com/spacemonkeygo/spacelog
-
-golang's stdlib, notably io and net/http
-
-The Name
-
-https://xkcd.com/927/
-
-*/
diff --git a/log/format.go b/log/format.go
deleted file mode 100644
index 2db2e6bf6..000000000
--- a/log/format.go
+++ /dev/null
@@ -1,364 +0,0 @@
-package log
-
-import (
- "bytes"
- "encoding/json"
- "fmt"
- "reflect"
- "strconv"
- "strings"
- "sync"
- "sync/atomic"
- "time"
- "unicode/utf8"
-)
-
-const (
- timeFormat = "2006-01-02T15:04:05-0700"
- termTimeFormat = "01-02|15:04:05.999999"
- floatFormat = 'f'
- termMsgJust = 40
-)
-
-// locationTrims are trimmed for display to avoid unwieldy log lines.
-var locationTrims = []string{
- "github.com/ethereum/go-ethereum/",
-}
-
-// PrintOrigins sets or unsets log location (file:line) printing for terminal
-// format output.
-func PrintOrigins(print bool) {
- if print {
- atomic.StoreUint32(&locationEnabled, 1)
- } else {
- atomic.StoreUint32(&locationEnabled, 0)
- }
-}
-
-// locationEnabled is an atomic flag controlling whether the terminal formatter
-// should append the log locations too when printing entries.
-var locationEnabled uint32
-
-// locationLength is the maxmimum path length encountered, which all logs are
-// padded to to aid in alignment.
-var locationLength uint32
-
-// fieldPadding is a global map with maximum field value lengths seen until now
-// to allow padding log contexts in a bit smarter way.
-var fieldPadding = make(map[string]int)
-
-// fieldPaddingLock is a global mutex protecting the field padding map.
-var fieldPaddingLock sync.RWMutex
-
-// Format interface.
-type Format interface {
- Format(r *Record) []byte
-}
-
-// FormatFunc returns a new Format object which uses
-// the given function to perform record formatting.
-func FormatFunc(f func(*Record) []byte) Format {
- return formatFunc(f)
-}
-
-type formatFunc func(*Record) []byte
-
-func (f formatFunc) Format(r *Record) []byte {
- return f(r)
-}
-
-// TerminalStringer is an analogous interface to the stdlib stringer, allowing
-// own types to have custom shortened serialization formats when printed to the
-// screen.
-type TerminalStringer interface {
- TerminalString() string
-}
-
-// TerminalFormat formats log records optimized for human readability on
-// a terminal with color-coded level output and terser human friendly timestamp.
-// This format should only be used for interactive programs or while developing.
-//
-// [TIME] [LEVEL] MESAGE key=value key=value ...
-//
-// Example:
-//
-// [May 16 20:58:45] [DBUG] remove route ns=haproxy addr=127.0.0.1:50002
-//
-func TerminalFormat(usecolor bool) Format {
- return FormatFunc(func(r *Record) []byte {
- var color = 0
- if usecolor {
- switch r.Lvl {
- case LvlCrit:
- color = 35
- case LvlError:
- color = 31
- case LvlWarn:
- color = 33
- case LvlInfo:
- color = 32
- case LvlDebug:
- color = 36
- case LvlTrace:
- color = 34
- }
- }
-
- b := &bytes.Buffer{}
- lvl := r.Lvl.AlignedString()
- if atomic.LoadUint32(&locationEnabled) != 0 {
- // Log origin printing was requested, format the location path and line number
- location := fmt.Sprintf("%+v", r.Call)
- for _, prefix := range locationTrims {
- location = strings.TrimPrefix(location, prefix)
- }
- // Maintain the maximum location length for fancyer alignment
- align := int(atomic.LoadUint32(&locationLength))
- if align < len(location) {
- align = len(location)
- atomic.StoreUint32(&locationLength, uint32(align))
- }
- padding := strings.Repeat(" ", align-len(location))
-
- // Assemble and print the log heading
- if color > 0 {
- fmt.Fprintf(b, "\x1b[%dm%s\x1b[0m[%s|%s]%s %s ", color, lvl, r.Time.Format(termTimeFormat), location, padding, r.Msg)
- } else {
- fmt.Fprintf(b, "%s[%s|%s]%s %s ", lvl, r.Time.Format(termTimeFormat), location, padding, r.Msg)
- }
- } else {
- if color > 0 {
- fmt.Fprintf(b, "\x1b[%dm%s\x1b[0m[%s] %s ", color, lvl, r.Time.Format(termTimeFormat), r.Msg)
- } else {
- fmt.Fprintf(b, "%s[%s] %s ", lvl, r.Time.Format(termTimeFormat), r.Msg)
- }
- }
- // try to justify the log output for short messages
- length := utf8.RuneCountInString(r.Msg)
- if len(r.Ctx) > 0 && length < termMsgJust {
- b.Write(bytes.Repeat([]byte{' '}, termMsgJust-length))
- }
- // print the keys logfmt style
- logfmt(b, r.Ctx, color, true)
- return b.Bytes()
- })
-}
-
-// LogfmtFormat prints records in logfmt format, an easy machine-parseable but human-readable
-// format for key/value pairs.
-//
-// For more details see: http://godoc.org/github.com/kr/logfmt
-//
-func LogfmtFormat() Format {
- return FormatFunc(func(r *Record) []byte {
- common := []interface{}{r.KeyNames.Time, r.Time, r.KeyNames.Lvl, r.Lvl, r.KeyNames.Msg, r.Msg}
- buf := &bytes.Buffer{}
- logfmt(buf, append(common, r.Ctx...), 0, false)
- return buf.Bytes()
- })
-}
-
-func logfmt(buf *bytes.Buffer, ctx []interface{}, color int, term bool) {
- for i := 0; i < len(ctx); i += 2 {
- if i != 0 {
- buf.WriteByte(' ')
- }
-
- k, ok := ctx[i].(string)
- v := formatLogfmtValue(ctx[i+1], term)
- if !ok {
- k, v = errorKey, formatLogfmtValue(k, term)
- }
-
- // XXX: we should probably check that all of your key bytes aren't invalid
- fieldPaddingLock.RLock()
- padding := fieldPadding[k]
- fieldPaddingLock.RUnlock()
-
- length := utf8.RuneCountInString(v)
- if padding < length {
- padding = length
-
- fieldPaddingLock.Lock()
- fieldPadding[k] = padding
- fieldPaddingLock.Unlock()
- }
- if color > 0 {
- fmt.Fprintf(buf, "\x1b[%dm%s\x1b[0m=", color, k)
- } else {
- buf.WriteString(k)
- buf.WriteByte('=')
- }
- buf.WriteString(v)
- if i < len(ctx)-2 {
- buf.Write(bytes.Repeat([]byte{' '}, padding-length))
- }
- }
- buf.WriteByte('\n')
-}
-
-// JSONFormat formats log records as JSON objects separated by newlines.
-// It is the equivalent of JSONFormatEx(false, true).
-func JSONFormat() Format {
- return JSONFormatEx(false, true)
-}
-
-// JSONFormatEx formats log records as JSON objects. If pretty is true,
-// records will be pretty-printed. If lineSeparated is true, records
-// will be logged with a new line between each record.
-func JSONFormatEx(pretty, lineSeparated bool) Format {
- jsonMarshal := json.Marshal
- if pretty {
- jsonMarshal = func(v interface{}) ([]byte, error) {
- return json.MarshalIndent(v, "", " ")
- }
- }
-
- return FormatFunc(func(r *Record) []byte {
- props := make(map[string]interface{})
-
- props[r.KeyNames.Time] = r.Time
- props[r.KeyNames.Lvl] = r.Lvl.String()
- props[r.KeyNames.Msg] = r.Msg
-
- for i := 0; i < len(r.Ctx); i += 2 {
- k, ok := r.Ctx[i].(string)
- if !ok {
- props[errorKey] = fmt.Sprintf("%+v is not a string key", r.Ctx[i])
- }
- props[k] = formatJSONValue(r.Ctx[i+1])
- }
-
- b, err := jsonMarshal(props)
- if err != nil {
- b, _ = jsonMarshal(map[string]string{
- errorKey: err.Error(),
- })
- return b
- }
-
- if lineSeparated {
- b = append(b, '\n')
- }
-
- return b
- })
-}
-
-func formatShared(value interface{}) (result interface{}) {
- defer func() {
- if err := recover(); err != nil {
- if v := reflect.ValueOf(value); v.Kind() == reflect.Ptr && v.IsNil() {
- result = "nil"
- } else {
- panic(err)
- }
- }
- }()
-
- switch v := value.(type) {
- case time.Time:
- return v.Format(timeFormat)
-
- case error:
- return v.Error()
-
- case fmt.Stringer:
- return v.String()
-
- default:
- return v
- }
-}
-
-func formatJSONValue(value interface{}) interface{} {
- value = formatShared(value)
- switch value.(type) {
- case int, int8, int16, int32, int64, float32, float64, uint, uint8, uint16, uint32, uint64, string:
- return value
- default:
- return fmt.Sprintf("%+v", value)
- }
-}
-
-// formatValue formats a value for serialization
-func formatLogfmtValue(value interface{}, term bool) string {
- if value == nil {
- return "nil"
- }
-
- if t, ok := value.(time.Time); ok {
- // Performance optimization: No need for escaping since the provided
- // timeFormat doesn't have any escape characters, and escaping is
- // expensive.
- return t.Format(timeFormat)
- }
- if term {
- if s, ok := value.(TerminalStringer); ok {
- // Custom terminal stringer provided, use that
- return escapeString(s.TerminalString())
- }
- }
- value = formatShared(value)
- switch v := value.(type) {
- case bool:
- return strconv.FormatBool(v)
- case float32:
- return strconv.FormatFloat(float64(v), floatFormat, 3, 64)
- case float64:
- return strconv.FormatFloat(v, floatFormat, 3, 64)
- case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64:
- return fmt.Sprintf("%d", value)
- case string:
- return escapeString(v)
- default:
- return escapeString(fmt.Sprintf("%+v", value))
- }
-}
-
-var stringBufPool = sync.Pool{
- New: func() interface{} { return new(bytes.Buffer) },
-}
-
-func escapeString(s string) string {
- needsQuotes := false
- needsEscape := false
- for _, r := range s {
- if r <= ' ' || r == '=' || r == '"' {
- needsQuotes = true
- }
- if r == '\\' || r == '"' || r == '\n' || r == '\r' || r == '\t' {
- needsEscape = true
- }
- }
- if !needsEscape && !needsQuotes {
- return s
- }
- e := stringBufPool.Get().(*bytes.Buffer)
- e.WriteByte('"')
- for _, r := range s {
- switch r {
- case '\\', '"':
- e.WriteByte('\\')
- e.WriteByte(byte(r))
- case '\n':
- e.WriteString("\\n")
- case '\r':
- e.WriteString("\\r")
- case '\t':
- e.WriteString("\\t")
- default:
- e.WriteRune(r)
- }
- }
- e.WriteByte('"')
- var ret string
- if needsQuotes {
- ret = e.String()
- } else {
- ret = string(e.Bytes()[1 : e.Len()-1])
- }
- e.Reset()
- stringBufPool.Put(e)
- return ret
-}
diff --git a/log/handler.go b/log/handler.go
deleted file mode 100644
index 3c99114dc..000000000
--- a/log/handler.go
+++ /dev/null
@@ -1,359 +0,0 @@
-package log
-
-import (
- "fmt"
- "io"
- "net"
- "os"
- "reflect"
- "sync"
-
- "github.com/go-stack/stack"
-)
-
-// Handler defines where and how log records are written.
-// A Logger prints its log records by writing to a Handler.
-// Handlers are composable, providing you great flexibility in combining
-// them to achieve the logging structure that suits your applications.
-type Handler interface {
- Log(r *Record) error
-}
-
-// FuncHandler returns a Handler that logs records with the given
-// function.
-func FuncHandler(fn func(r *Record) error) Handler {
- return funcHandler(fn)
-}
-
-type funcHandler func(r *Record) error
-
-func (h funcHandler) Log(r *Record) error {
- return h(r)
-}
-
-// StreamHandler writes log records to an io.Writer
-// with the given format. StreamHandler can be used
-// to easily begin writing log records to other
-// outputs.
-//
-// StreamHandler wraps itself with LazyHandler and SyncHandler
-// to evaluate Lazy objects and perform safe concurrent writes.
-func StreamHandler(wr io.Writer, fmtr Format) Handler {
- h := FuncHandler(func(r *Record) error {
- _, err := wr.Write(fmtr.Format(r))
- return err
- })
- return LazyHandler(SyncHandler(h))
-}
-
-// SyncHandler can be wrapped around a handler to guarantee that
-// only a single Log operation can proceed at a time. It's necessary
-// for thread-safe concurrent writes.
-func SyncHandler(h Handler) Handler {
- var mu sync.Mutex
- return FuncHandler(func(r *Record) error {
- defer mu.Unlock()
- mu.Lock()
- return h.Log(r)
- })
-}
-
-// FileHandler returns a handler which writes log records to the give file
-// using the given format. If the path
-// already exists, FileHandler will append to the given file. If it does not,
-// FileHandler will create the file with mode 0644.
-func FileHandler(path string, fmtr Format) (Handler, error) {
- f, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644)
- if err != nil {
- return nil, err
- }
- return closingHandler{f, StreamHandler(f, fmtr)}, nil
-}
-
-// NetHandler opens a socket to the given address and writes records
-// over the connection.
-func NetHandler(network, addr string, fmtr Format) (Handler, error) {
- conn, err := net.Dial(network, addr)
- if err != nil {
- return nil, err
- }
-
- return closingHandler{conn, StreamHandler(conn, fmtr)}, nil
-}
-
-// XXX: closingHandler is essentially unused at the moment
-// it's meant for a future time when the Handler interface supports
-// a possible Close() operation
-type closingHandler struct {
- io.WriteCloser
- Handler
-}
-
-func (h *closingHandler) Close() error {
- return h.WriteCloser.Close()
-}
-
-// CallerFileHandler returns a Handler that adds the line number and file of
-// the calling function to the context with key "caller".
-func CallerFileHandler(h Handler) Handler {
- return FuncHandler(func(r *Record) error {
- r.Ctx = append(r.Ctx, "caller", fmt.Sprint(r.Call))
- return h.Log(r)
- })
-}
-
-// CallerFuncHandler returns a Handler that adds the calling function name to
-// the context with key "fn".
-func CallerFuncHandler(h Handler) Handler {
- return FuncHandler(func(r *Record) error {
- r.Ctx = append(r.Ctx, "fn", formatCall("%+n", r.Call))
- return h.Log(r)
- })
-}
-
-// This function is here to please go vet on Go < 1.8.
-func formatCall(format string, c stack.Call) string {
- return fmt.Sprintf(format, c)
-}
-
-// CallerStackHandler returns a Handler that adds a stack trace to the context
-// with key "stack". The stack trace is formated as a space separated list of
-// call sites inside matching []'s. The most recent call site is listed first.
-// Each call site is formatted according to format. See the documentation of
-// package github.com/go-stack/stack for the list of supported formats.
-func CallerStackHandler(format string, h Handler) Handler {
- return FuncHandler(func(r *Record) error {
- s := stack.Trace().TrimBelow(r.Call).TrimRuntime()
- if len(s) > 0 {
- r.Ctx = append(r.Ctx, "stack", fmt.Sprintf(format, s))
- }
- return h.Log(r)
- })
-}
-
-// FilterHandler returns a Handler that only writes records to the
-// wrapped Handler if the given function evaluates true. For example,
-// to only log records where the 'err' key is not nil:
-//
-// logger.SetHandler(FilterHandler(func(r *Record) bool {
-// for i := 0; i < len(r.Ctx); i += 2 {
-// if r.Ctx[i] == "err" {
-// return r.Ctx[i+1] != nil
-// }
-// }
-// return false
-// }, h))
-//
-func FilterHandler(fn func(r *Record) bool, h Handler) Handler {
- return FuncHandler(func(r *Record) error {
- if fn(r) {
- return h.Log(r)
- }
- return nil
- })
-}
-
-// MatchFilterHandler returns a Handler that only writes records
-// to the wrapped Handler if the given key in the logged
-// context matches the value. For example, to only log records
-// from your ui package:
-//
-// log.MatchFilterHandler("pkg", "app/ui", log.StdoutHandler)
-//
-func MatchFilterHandler(key string, value interface{}, h Handler) Handler {
- return FilterHandler(func(r *Record) (pass bool) {
- switch key {
- case r.KeyNames.Lvl:
- return r.Lvl == value
- case r.KeyNames.Time:
- return r.Time == value
- case r.KeyNames.Msg:
- return r.Msg == value
- }
-
- for i := 0; i < len(r.Ctx); i += 2 {
- if r.Ctx[i] == key {
- return r.Ctx[i+1] == value
- }
- }
- return false
- }, h)
-}
-
-// LvlFilterHandler returns a Handler that only writes
-// records which are less than the given verbosity
-// level to the wrapped Handler. For example, to only
-// log Error/Crit records:
-//
-// log.LvlFilterHandler(log.LvlError, log.StdoutHandler)
-//
-func LvlFilterHandler(maxLvl Lvl, h Handler) Handler {
- return FilterHandler(func(r *Record) (pass bool) {
- return r.Lvl <= maxLvl
- }, h)
-}
-
-// MultiHandler dispatches any write to each of its handlers.
-// This is useful for writing different types of log information
-// to different locations. For example, to log to a file and
-// standard error:
-//
-// log.MultiHandler(
-// log.Must.FileHandler("/var/log/app.log", log.LogfmtFormat()),
-// log.StderrHandler)
-//
-func MultiHandler(hs ...Handler) Handler {
- return FuncHandler(func(r *Record) error {
- for _, h := range hs {
- // what to do about failures?
- h.Log(r)
- }
- return nil
- })
-}
-
-// FailoverHandler writes all log records to the first handler
-// specified, but will failover and write to the second handler if
-// the first handler has failed, and so on for all handlers specified.
-// For example you might want to log to a network socket, but failover
-// to writing to a file if the network fails, and then to
-// standard out if the file write fails:
-//
-// log.FailoverHandler(
-// log.Must.NetHandler("tcp", ":9090", log.JSONFormat()),
-// log.Must.FileHandler("/var/log/app.log", log.LogfmtFormat()),
-// log.StdoutHandler)
-//
-// All writes that do not go to the first handler will add context with keys of
-// the form "failover_err_{idx}" which explain the error encountered while
-// trying to write to the handlers before them in the list.
-func FailoverHandler(hs ...Handler) Handler {
- return FuncHandler(func(r *Record) error {
- var err error
- for i, h := range hs {
- err = h.Log(r)
- if err == nil {
- return nil
- }
- r.Ctx = append(r.Ctx, fmt.Sprintf("failover_err_%d", i), err)
- }
-
- return err
- })
-}
-
-// ChannelHandler writes all records to the given channel.
-// It blocks if the channel is full. Useful for async processing
-// of log messages, it's used by BufferedHandler.
-func ChannelHandler(recs chan<- *Record) Handler {
- return FuncHandler(func(r *Record) error {
- recs <- r
- return nil
- })
-}
-
-// BufferedHandler writes all records to a buffered
-// channel of the given size which flushes into the wrapped
-// handler whenever it is available for writing. Since these
-// writes happen asynchronously, all writes to a BufferedHandler
-// never return an error and any errors from the wrapped handler are ignored.
-func BufferedHandler(bufSize int, h Handler) Handler {
- recs := make(chan *Record, bufSize)
- go func() {
- for m := range recs {
- _ = h.Log(m)
- }
- }()
- return ChannelHandler(recs)
-}
-
-// LazyHandler writes all values to the wrapped handler after evaluating
-// any lazy functions in the record's context. It is already wrapped
-// around StreamHandler and SyslogHandler in this library, you'll only need
-// it if you write your own Handler.
-func LazyHandler(h Handler) Handler {
- return FuncHandler(func(r *Record) error {
- // go through the values (odd indices) and reassign
- // the values of any lazy fn to the result of its execution
- hadErr := false
- for i := 1; i < len(r.Ctx); i += 2 {
- lz, ok := r.Ctx[i].(Lazy)
- if ok {
- v, err := evaluateLazy(lz)
- if err != nil {
- hadErr = true
- r.Ctx[i] = err
- } else {
- if cs, ok := v.(stack.CallStack); ok {
- v = cs.TrimBelow(r.Call).TrimRuntime()
- }
- r.Ctx[i] = v
- }
- }
- }
-
- if hadErr {
- r.Ctx = append(r.Ctx, errorKey, "bad lazy")
- }
-
- return h.Log(r)
- })
-}
-
-func evaluateLazy(lz Lazy) (interface{}, error) {
- t := reflect.TypeOf(lz.Fn)
-
- if t.Kind() != reflect.Func {
- return nil, fmt.Errorf("INVALID_LAZY, not func: %+v", lz.Fn)
- }
-
- if t.NumIn() > 0 {
- return nil, fmt.Errorf("INVALID_LAZY, func takes args: %+v", lz.Fn)
- }
-
- if t.NumOut() == 0 {
- return nil, fmt.Errorf("INVALID_LAZY, no func return val: %+v", lz.Fn)
- }
-
- value := reflect.ValueOf(lz.Fn)
- results := value.Call([]reflect.Value{})
- if len(results) == 1 {
- return results[0].Interface(), nil
- }
- values := make([]interface{}, len(results))
- for i, v := range results {
- values[i] = v.Interface()
- }
- return values, nil
-}
-
-// DiscardHandler reports success for all writes but does nothing.
-// It is useful for dynamically disabling logging at runtime via
-// a Logger's SetHandler method.
-func DiscardHandler() Handler {
- return FuncHandler(func(r *Record) error {
- return nil
- })
-}
-
-// Must provides the following Handler creation functions
-// which instead of returning an error parameter only return a Handler
-// and panic on failure: FileHandler, NetHandler, SyslogHandler, SyslogNetHandler
-var Must muster
-
-func must(h Handler, err error) Handler {
- if err != nil {
- panic(err)
- }
- return h
-}
-
-type muster struct{}
-
-func (m muster) FileHandler(path string, fmtr Format) Handler {
- return must(FileHandler(path, fmtr))
-}
-
-func (m muster) NetHandler(network, addr string, fmtr Format) Handler {
- return must(NetHandler(network, addr, fmtr))
-}
diff --git a/log/handler_glog.go b/log/handler_glog.go
deleted file mode 100644
index f8b932fd1..000000000
--- a/log/handler_glog.go
+++ /dev/null
@@ -1,227 +0,0 @@
-// Copyright 2017 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see .
-
-package log
-
-import (
- "errors"
- "fmt"
- "regexp"
- "runtime"
- "strconv"
- "strings"
- "sync"
- "sync/atomic"
-)
-
-// errVmoduleSyntax is returned when a user vmodule pattern is invalid.
-var errVmoduleSyntax = errors.New("expect comma-separated list of filename=N")
-
-// errTraceSyntax is returned when a user backtrace pattern is invalid.
-var errTraceSyntax = errors.New("expect file.go:234")
-
-// GlogHandler is a log handler that mimics the filtering features of Google's
-// glog logger: setting global log levels; overriding with callsite pattern
-// matches; and requesting backtraces at certain positions.
-type GlogHandler struct {
- origin Handler // The origin handler this wraps
-
- level uint32 // Current log level, atomically accessible
- override uint32 // Flag whether overrides are used, atomically accessible
- backtrace uint32 // Flag whether backtrace location is set
-
- patterns []pattern // Current list of patterns to override with
- siteCache map[uintptr]Lvl // Cache of callsite pattern evaluations
- location string // file:line location where to do a stackdump at
- lock sync.RWMutex // Lock protecting the override pattern list
-}
-
-// NewGlogHandler creates a new log handler with filtering functionality similar
-// to Google's glog logger. The returned handler implements Handler.
-func NewGlogHandler(h Handler) *GlogHandler {
- return &GlogHandler{
- origin: h,
- }
-}
-
-// pattern contains a filter for the Vmodule option, holding a verbosity level
-// and a file pattern to match.
-type pattern struct {
- pattern *regexp.Regexp
- level Lvl
-}
-
-// Verbosity sets the glog verbosity ceiling. The verbosity of individual packages
-// and source files can be raised using Vmodule.
-func (h *GlogHandler) Verbosity(level Lvl) {
- atomic.StoreUint32(&h.level, uint32(level))
-}
-
-// Vmodule sets the glog verbosity pattern.
-//
-// The syntax of the argument is a comma-separated list of pattern=N, where the
-// pattern is a literal file name or "glob" pattern matching and N is a V level.
-//
-// For instance:
-//
-// pattern="gopher.go=3"
-// sets the V level to 3 in all Go files named "gopher.go"
-//
-// pattern="foo=3"
-// sets V to 3 in all files of any packages whose import path ends in "foo"
-//
-// pattern="foo/*=3"
-// sets V to 3 in all files of any packages whose import path contains "foo"
-func (h *GlogHandler) Vmodule(ruleset string) error {
- var filter []pattern
- for _, rule := range strings.Split(ruleset, ",") {
- // Empty strings such as from a trailing comma can be ignored
- if len(rule) == 0 {
- continue
- }
- // Ensure we have a pattern = level filter rule
- parts := strings.Split(rule, "=")
- if len(parts) != 2 {
- return errVmoduleSyntax
- }
- parts[0] = strings.TrimSpace(parts[0])
- parts[1] = strings.TrimSpace(parts[1])
- if len(parts[0]) == 0 || len(parts[1]) == 0 {
- return errVmoduleSyntax
- }
- // Parse the level and if correct, assemble the filter rule
- level, err := strconv.Atoi(parts[1])
- if err != nil {
- return errVmoduleSyntax
- }
- if level <= 0 {
- continue // Ignore. It's harmless but no point in paying the overhead.
- }
- // Compile the rule pattern into a regular expression
- matcher := ".*"
- for _, comp := range strings.Split(parts[0], "/") {
- if comp == "*" {
- matcher += "(/.*)?"
- } else if comp != "" {
- matcher += "/" + regexp.QuoteMeta(comp)
- }
- }
- if !strings.HasSuffix(parts[0], ".go") {
- matcher += "/[^/]+\\.go"
- }
- matcher = matcher + "$"
-
- re, _ := regexp.Compile(matcher)
- filter = append(filter, pattern{re, Lvl(level)})
- }
- // Swap out the vmodule pattern for the new filter system
- h.lock.Lock()
- defer h.lock.Unlock()
-
- h.patterns = filter
- h.siteCache = make(map[uintptr]Lvl)
- atomic.StoreUint32(&h.override, uint32(len(filter)))
-
- return nil
-}
-
-// BacktraceAt sets the glog backtrace location. When set to a file and line
-// number holding a logging statement, a stack trace will be written to the Info
-// log whenever execution hits that statement.
-//
-// Unlike with Vmodule, the ".go" must be present.
-func (h *GlogHandler) BacktraceAt(location string) error {
- // Ensure the backtrace location contains two non-empty elements
- parts := strings.Split(location, ":")
- if len(parts) != 2 {
- return errTraceSyntax
- }
- parts[0] = strings.TrimSpace(parts[0])
- parts[1] = strings.TrimSpace(parts[1])
- if len(parts[0]) == 0 || len(parts[1]) == 0 {
- return errTraceSyntax
- }
- // Ensure the .go prefix is present and the line is valid
- if !strings.HasSuffix(parts[0], ".go") {
- return errTraceSyntax
- }
- if _, err := strconv.Atoi(parts[1]); err != nil {
- return errTraceSyntax
- }
- // All seems valid
- h.lock.Lock()
- defer h.lock.Unlock()
-
- h.location = location
- atomic.StoreUint32(&h.backtrace, uint32(len(location)))
-
- return nil
-}
-
-// Log implements Handler.Log, filtering a log record through the global, local
-// and backtrace filters, finally emitting it if either allow it through.
-func (h *GlogHandler) Log(r *Record) error {
- // If backtracing is requested, check whether this is the callsite
- if atomic.LoadUint32(&h.backtrace) > 0 {
- // Everything below here is slow. Although we could cache the call sites the
- // same way as for vmodule, backtracing is so rare it's not worth the extra
- // complexity.
- h.lock.RLock()
- match := h.location == r.Call.String()
- h.lock.RUnlock()
-
- if match {
- // Callsite matched, raise the log level to info and gather the stacks
- r.Lvl = LvlInfo
-
- buf := make([]byte, 1024*1024)
- buf = buf[:runtime.Stack(buf, true)]
- r.Msg += "\n\n" + string(buf)
- }
- }
- // If the global log level allows, fast track logging
- if atomic.LoadUint32(&h.level) >= uint32(r.Lvl) {
- return h.origin.Log(r)
- }
- // If no local overrides are present, fast track skipping
- if atomic.LoadUint32(&h.override) == 0 {
- return nil
- }
- // Check callsite cache for previously calculated log levels
- h.lock.RLock()
- lvl, ok := h.siteCache[r.Call.PC()]
- h.lock.RUnlock()
-
- // If we didn't cache the callsite yet, calculate it
- if !ok {
- h.lock.Lock()
- for _, rule := range h.patterns {
- if rule.pattern.MatchString(fmt.Sprintf("%+s", r.Call)) {
- h.siteCache[r.Call.PC()], lvl, ok = rule.level, rule.level, true
- break
- }
- }
- // If no rule matched, remember to drop log the next time
- if !ok {
- h.siteCache[r.Call.PC()] = 0
- }
- h.lock.Unlock()
- }
- if lvl >= r.Lvl {
- return h.origin.Log(r)
- }
- return nil
-}
diff --git a/log/handler_go13.go b/log/handler_go13.go
deleted file mode 100644
index 0843ed0e5..000000000
--- a/log/handler_go13.go
+++ /dev/null
@@ -1,26 +0,0 @@
-// +build !go1.4
-
-package log
-
-import (
- "sync/atomic"
- "unsafe"
-)
-
-// swapHandler wraps another handler that may be swapped out
-// dynamically at runtime in a thread-safe fashion.
-type swapHandler struct {
- handler unsafe.Pointer
-}
-
-func (h *swapHandler) Log(r *Record) error {
- return h.Get().Log(r)
-}
-
-func (h *swapHandler) Get() Handler {
- return *(*Handler)(atomic.LoadPointer(&h.handler))
-}
-
-func (h *swapHandler) Swap(newHandler Handler) {
- atomic.StorePointer(&h.handler, unsafe.Pointer(&newHandler))
-}
diff --git a/log/handler_go14.go b/log/handler_go14.go
deleted file mode 100644
index 05dedbf2a..000000000
--- a/log/handler_go14.go
+++ /dev/null
@@ -1,23 +0,0 @@
-// +build go1.4
-
-package log
-
-import "sync/atomic"
-
-// swapHandler wraps another handler that may be swapped out
-// dynamically at runtime in a thread-safe fashion.
-type swapHandler struct {
- handler atomic.Value
-}
-
-func (h *swapHandler) Log(r *Record) error {
- return (*h.handler.Load().(*Handler)).Log(r)
-}
-
-func (h *swapHandler) Swap(newHandler Handler) {
- h.handler.Store(&newHandler)
-}
-
-func (h *swapHandler) Get() Handler {
- return *h.handler.Load().(*Handler)
-}
diff --git a/log/logger.go b/log/logger.go
deleted file mode 100644
index 5e5924a3d..000000000
--- a/log/logger.go
+++ /dev/null
@@ -1,244 +0,0 @@
-package log
-
-import (
- "fmt"
- "os"
- "time"
-
- "github.com/go-stack/stack"
-)
-
-const timeKey = "t"
-const lvlKey = "lvl"
-const msgKey = "msg"
-const errorKey = "LOG15_ERROR"
-const skipLevel = 2
-
-// Lvl type.
-type Lvl int
-
-// Constants.
-const (
- LvlCrit Lvl = iota
- LvlError
- LvlWarn
- LvlInfo
- LvlDebug
- LvlTrace
-)
-
-// AlignedString returns a 5-character string containing the name of a Lvl.
-func (l Lvl) AlignedString() string {
- switch l {
- case LvlTrace:
- return "TRACE"
- case LvlDebug:
- return "DEBUG"
- case LvlInfo:
- return "INFO "
- case LvlWarn:
- return "WARN "
- case LvlError:
- return "ERROR"
- case LvlCrit:
- return "CRIT "
- default:
- panic("bad level")
- }
-}
-
-// Strings returns the name of a Lvl.
-func (l Lvl) String() string {
- switch l {
- case LvlTrace:
- return "trce"
- case LvlDebug:
- return "dbug"
- case LvlInfo:
- return "info"
- case LvlWarn:
- return "warn"
- case LvlError:
- return "eror"
- case LvlCrit:
- return "crit"
- default:
- panic("bad level")
- }
-}
-
-// LvlFromString returns the appropriate Lvl from a string name.
-// Useful for parsing command line args and configuration files.
-func LvlFromString(lvlString string) (Lvl, error) {
- switch lvlString {
- case "trace", "trce":
- return LvlTrace, nil
- case "debug", "dbug":
- return LvlDebug, nil
- case "info":
- return LvlInfo, nil
- case "warn":
- return LvlWarn, nil
- case "error", "eror":
- return LvlError, nil
- case "crit":
- return LvlCrit, nil
- default:
- return LvlDebug, fmt.Errorf("Unknown level: %v", lvlString)
- }
-}
-
-// A Record is what a Logger asks its handler to write
-type Record struct {
- Time time.Time
- Lvl Lvl
- Msg string
- Ctx []interface{}
- Call stack.Call
- KeyNames RecordKeyNames
-}
-
-// RecordKeyNames gets stored in a Record when the write function is executed.
-type RecordKeyNames struct {
- Time string
- Msg string
- Lvl string
-}
-
-// A Logger writes key/value pairs to a Handler
-type Logger interface {
- // New returns a new Logger that has this logger's context plus the given context
- New(ctx ...interface{}) Logger
-
- // GetHandler gets the handler associated with the logger.
- GetHandler() Handler
-
- // SetHandler updates the logger to write records to the specified handler.
- SetHandler(h Handler)
-
- // Log a message at the given level with context key/value pairs
- Trace(msg string, ctx ...interface{})
- Debug(msg string, ctx ...interface{})
- Info(msg string, ctx ...interface{})
- Warn(msg string, ctx ...interface{})
- Error(msg string, ctx ...interface{})
- Crit(msg string, ctx ...interface{})
-}
-
-type logger struct {
- ctx []interface{}
- h *swapHandler
-}
-
-func (l *logger) write(msg string, lvl Lvl, ctx []interface{}, skip int) {
- l.h.Log(&Record{
- Time: time.Now(),
- Lvl: lvl,
- Msg: msg,
- Ctx: newContext(l.ctx, ctx),
- Call: stack.Caller(skip),
- KeyNames: RecordKeyNames{
- Time: timeKey,
- Msg: msgKey,
- Lvl: lvlKey,
- },
- })
-}
-
-func (l *logger) New(ctx ...interface{}) Logger {
- child := &logger{newContext(l.ctx, ctx), new(swapHandler)}
- child.SetHandler(l.h)
- return child
-}
-
-func newContext(prefix []interface{}, suffix []interface{}) []interface{} {
- normalizedSuffix := normalize(suffix)
- newCtx := make([]interface{}, len(prefix)+len(normalizedSuffix))
- n := copy(newCtx, prefix)
- copy(newCtx[n:], normalizedSuffix)
- return newCtx
-}
-
-func (l *logger) Trace(msg string, ctx ...interface{}) {
- l.write(msg, LvlTrace, ctx, skipLevel)
-}
-
-func (l *logger) Debug(msg string, ctx ...interface{}) {
- l.write(msg, LvlDebug, ctx, skipLevel)
-}
-
-func (l *logger) Info(msg string, ctx ...interface{}) {
- l.write(msg, LvlInfo, ctx, skipLevel)
-}
-
-func (l *logger) Warn(msg string, ctx ...interface{}) {
- l.write(msg, LvlWarn, ctx, skipLevel)
-}
-
-func (l *logger) Error(msg string, ctx ...interface{}) {
- l.write(msg, LvlError, ctx, skipLevel)
-}
-
-func (l *logger) Crit(msg string, ctx ...interface{}) {
- l.write(msg, LvlCrit, ctx, skipLevel)
- os.Exit(1)
-}
-
-func (l *logger) GetHandler() Handler {
- return l.h.Get()
-}
-
-func (l *logger) SetHandler(h Handler) {
- l.h.Swap(h)
-}
-
-func normalize(ctx []interface{}) []interface{} {
- // if the caller passed a Ctx object, then expand it
- if len(ctx) == 1 {
- if ctxMap, ok := ctx[0].(Ctx); ok {
- ctx = ctxMap.toArray()
- }
- }
-
- // ctx needs to be even because it's a series of key/value pairs
- // no one wants to check for errors on logging functions,
- // so instead of erroring on bad input, we'll just make sure
- // that things are the right length and users can fix bugs
- // when they see the output looks wrong
- if len(ctx)%2 != 0 {
- ctx = append(ctx, nil, errorKey, "Normalized odd number of arguments by adding nil")
- }
-
- return ctx
-}
-
-// Lazy allows you to defer calculation of a logged value that is expensive
-// to compute until it is certain that it must be evaluated with the given filters.
-//
-// Lazy may also be used in conjunction with a Logger's New() function
-// to generate a child logger which always reports the current value of changing
-// state.
-//
-// You may wrap any function which takes no arguments to Lazy. It may return any
-// number of values of any type.
-type Lazy struct {
- Fn interface{}
-}
-
-// Ctx is a map of key/value pairs to pass as context to a log function
-// Use this only if you really need greater safety around the arguments you pass
-// to the logging functions.
-type Ctx map[string]interface{}
-
-func (c Ctx) toArray() []interface{} {
- arr := make([]interface{}, len(c)*2)
-
- i := 0
- for k, v := range c {
- arr[i] = k
- arr[i+1] = v
- i += 2
- }
-
- return arr
-}
diff --git a/log/root.go b/log/root.go
deleted file mode 100644
index 84d73b6f6..000000000
--- a/log/root.go
+++ /dev/null
@@ -1,72 +0,0 @@
-package log
-
-import (
- "os"
-)
-
-var (
- root = &logger{[]interface{}{}, new(swapHandler)}
- // StdoutHandler handles stdout
- StdoutHandler = StreamHandler(os.Stdout, TerminalFormat(false))
- // StderrHandler handles stderr
- StderrHandler = StreamHandler(os.Stderr, LogfmtFormat())
-)
-
-func init() {
- root.SetHandler(StdoutHandler)
-}
-
-// New returns a new logger with the given context.
-// New is a convenient alias for Root().New
-func New(ctx ...interface{}) Logger {
- return root.New(ctx...)
-}
-
-// Root returns the root logger
-func Root() Logger {
- return root
-}
-
-// The following functions bypass the exported logger methods (logger.Debug,
-// etc.) to keep the call depth the same for all paths to logger.write so
-// runtime.Caller(2) always refers to the call site in client code.
-
-// Trace is a convenient alias for Root().Trace
-func Trace(msg string, ctx ...interface{}) {
- root.write(msg, LvlTrace, ctx, skipLevel)
-}
-
-// Debug is a convenient alias for Root().Debug
-func Debug(msg string, ctx ...interface{}) {
- root.write(msg, LvlDebug, ctx, skipLevel)
-}
-
-// Info is a convenient alias for Root().Info
-func Info(msg string, ctx ...interface{}) {
- root.write(msg, LvlInfo, ctx, skipLevel)
-}
-
-// Warn is a convenient alias for Root().Warn
-func Warn(msg string, ctx ...interface{}) {
- root.write(msg, LvlWarn, ctx, skipLevel)
-}
-
-// Error is a convenient alias for Root().Error
-func Error(msg string, ctx ...interface{}) {
- root.write(msg, LvlError, ctx, skipLevel)
-}
-
-// Crit is a convenient alias for Root().Crit
-func Crit(msg string, ctx ...interface{}) {
- root.write(msg, LvlCrit, ctx, skipLevel)
- os.Exit(1)
-}
-
-// Output is a convenient alias for write, allowing for the modification of
-// the calldepth (number of stack frames to skip).
-// calldepth influences the reported line number of the log message.
-// A calldepth of zero reports the immediate caller of Output.
-// Non-zero calldepth skips as many stack frames.
-func Output(msg string, lvl Lvl, calldepth int, ctx ...interface{}) {
- root.write(msg, lvl, ctx, calldepth+skipLevel)
-}
diff --git a/log/syslog.go b/log/syslog.go
deleted file mode 100644
index 71a17b30b..000000000
--- a/log/syslog.go
+++ /dev/null
@@ -1,57 +0,0 @@
-// +build !windows,!plan9
-
-package log
-
-import (
- "log/syslog"
- "strings"
-)
-
-// SyslogHandler opens a connection to the system syslog daemon by calling
-// syslog.New and writes all records to it.
-func SyslogHandler(priority syslog.Priority, tag string, fmtr Format) (Handler, error) {
- wr, err := syslog.New(priority, tag)
- return sharedSyslog(fmtr, wr, err)
-}
-
-// SyslogNetHandler opens a connection to a log daemon over the network and writes
-// all log records to it.
-func SyslogNetHandler(net, addr string, priority syslog.Priority, tag string, fmtr Format) (Handler, error) {
- wr, err := syslog.Dial(net, addr, priority, tag)
- return sharedSyslog(fmtr, wr, err)
-}
-
-func sharedSyslog(fmtr Format, sysWr *syslog.Writer, err error) (Handler, error) {
- if err != nil {
- return nil, err
- }
- h := FuncHandler(func(r *Record) error {
- var syslogFn = sysWr.Info
- switch r.Lvl {
- case LvlCrit:
- syslogFn = sysWr.Crit
- case LvlError:
- syslogFn = sysWr.Err
- case LvlWarn:
- syslogFn = sysWr.Warning
- case LvlInfo:
- syslogFn = sysWr.Info
- case LvlDebug:
- syslogFn = sysWr.Debug
- case LvlTrace:
- syslogFn = func(m string) error { return nil } // There's no syslog level for trace
- }
-
- s := strings.TrimSpace(string(fmtr.Format(r)))
- return syslogFn(s)
- })
- return LazyHandler(&closingHandler{sysWr, h}), nil
-}
-
-func (m muster) SyslogHandler(priority syslog.Priority, tag string, fmtr Format) Handler {
- return must(SyslogHandler(priority, tag, fmtr))
-}
-
-func (m muster) SyslogNetHandler(net, addr string, priority syslog.Priority, tag string, fmtr Format) Handler {
- return must(SyslogNetHandler(net, addr, priority, tag, fmtr))
-}
diff --git a/log/term/LICENSE b/log/term/LICENSE
deleted file mode 100644
index f090cb42f..000000000
--- a/log/term/LICENSE
+++ /dev/null
@@ -1,21 +0,0 @@
-The MIT License (MIT)
-
-Copyright (c) 2014 Simon Eskildsen
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
diff --git a/log/term/terminal_appengine.go b/log/term/terminal_appengine.go
deleted file mode 100644
index c1b5d2a3b..000000000
--- a/log/term/terminal_appengine.go
+++ /dev/null
@@ -1,13 +0,0 @@
-// Based on ssh/terminal:
-// Copyright 2013 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build appengine
-
-package term
-
-// IsTty always returns false on AppEngine.
-func IsTty(fd uintptr) bool {
- return false
-}
diff --git a/log/term/terminal_darwin.go b/log/term/terminal_darwin.go
deleted file mode 100644
index 9e5b18668..000000000
--- a/log/term/terminal_darwin.go
+++ /dev/null
@@ -1,14 +0,0 @@
-// Based on ssh/terminal:
-// Copyright 2013 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-// +build !appengine
-
-package term
-
-import "syscall"
-
-const ioctlReadTermios = syscall.TIOCGETA
-
-// Termios is syscall.Termios.
-type Termios syscall.Termios
diff --git a/log/term/terminal_freebsd.go b/log/term/terminal_freebsd.go
deleted file mode 100644
index cfaceab33..000000000
--- a/log/term/terminal_freebsd.go
+++ /dev/null
@@ -1,18 +0,0 @@
-package term
-
-import (
- "syscall"
-)
-
-const ioctlReadTermios = syscall.TIOCGETA
-
-// Go 1.2 doesn't include Termios for FreeBSD. This should be added in 1.3 and this could be merged with terminal_darwin.
-type Termios struct {
- Iflag uint32
- Oflag uint32
- Cflag uint32
- Lflag uint32
- Cc [20]uint8
- Ispeed uint32
- Ospeed uint32
-}
diff --git a/log/term/terminal_linux.go b/log/term/terminal_linux.go
deleted file mode 100644
index 6cb862c05..000000000
--- a/log/term/terminal_linux.go
+++ /dev/null
@@ -1,15 +0,0 @@
-// Based on ssh/terminal:
-// Copyright 2013 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build !appengine
-
-package term
-
-import "syscall"
-
-const ioctlReadTermios = syscall.TCGETS
-
-// Termios ...
-type Termios syscall.Termios
diff --git a/log/term/terminal_netbsd.go b/log/term/terminal_netbsd.go
deleted file mode 100644
index f9bb9e1c2..000000000
--- a/log/term/terminal_netbsd.go
+++ /dev/null
@@ -1,7 +0,0 @@
-package term
-
-import "syscall"
-
-const ioctlReadTermios = syscall.TIOCGETA
-
-type Termios syscall.Termios
diff --git a/log/term/terminal_notwindows.go b/log/term/terminal_notwindows.go
deleted file mode 100644
index c9af534f6..000000000
--- a/log/term/terminal_notwindows.go
+++ /dev/null
@@ -1,20 +0,0 @@
-// Based on ssh/terminal:
-// Copyright 2011 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build linux,!appengine darwin freebsd openbsd netbsd
-
-package term
-
-import (
- "syscall"
- "unsafe"
-)
-
-// IsTty returns true if the given file descriptor is a terminal.
-func IsTty(fd uintptr) bool {
- var termios Termios
- _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, fd, ioctlReadTermios, uintptr(unsafe.Pointer(&termios)), 0, 0, 0)
- return err == 0
-}
diff --git a/log/term/terminal_openbsd.go b/log/term/terminal_openbsd.go
deleted file mode 100644
index f9bb9e1c2..000000000
--- a/log/term/terminal_openbsd.go
+++ /dev/null
@@ -1,7 +0,0 @@
-package term
-
-import "syscall"
-
-const ioctlReadTermios = syscall.TIOCGETA
-
-type Termios syscall.Termios
diff --git a/log/term/terminal_solaris.go b/log/term/terminal_solaris.go
deleted file mode 100644
index 033c16324..000000000
--- a/log/term/terminal_solaris.go
+++ /dev/null
@@ -1,9 +0,0 @@
-package term
-
-import "golang.org/x/sys/unix"
-
-// IsTty returns true if the given file descriptor is a terminal.
-func IsTty(fd uintptr) bool {
- _, err := unix.IoctlGetTermios(int(fd), unix.TCGETA)
- return err == nil
-}
diff --git a/log/term/terminal_windows.go b/log/term/terminal_windows.go
deleted file mode 100644
index df3c30c15..000000000
--- a/log/term/terminal_windows.go
+++ /dev/null
@@ -1,26 +0,0 @@
-// Based on ssh/terminal:
-// Copyright 2011 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build windows
-
-package term
-
-import (
- "syscall"
- "unsafe"
-)
-
-var kernel32 = syscall.NewLazyDLL("kernel32.dll")
-
-var (
- procGetConsoleMode = kernel32.NewProc("GetConsoleMode")
-)
-
-// IsTty returns true if the given file descriptor is a terminal.
-func IsTty(fd uintptr) bool {
- var st uint32
- r, _, e := syscall.Syscall(procGetConsoleMode.Addr(), 2, fd, uintptr(unsafe.Pointer(&st)), 0)
- return r != 0 && e == 0
-}