Merge pull request #1256 from LeoHChen/merge_r3_to_master

Merge r3 to master

consensus/consensus.go

@@ -7,6 +7,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/harmony-one/harmony/core"
+
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/harmony-one/bls/ffi/go/bls"
@@ -69,7 +71,7 @@ type Consensus struct {
 	vcLock       sync.Mutex // mutex for view change
 
 	// The chain reader for the blockchain this consensus is working on
-	ChainReader consensus_engine.ChainReader
+	ChainReader *core.BlockChain
 
 	// map of nodeID to validator Peer object
 	validators sync.Map // key is the hex string of the blsKey, value is p2p.Peer

consensus/consensus_v2.go

@@ -711,6 +711,7 @@ func (consensus *Consensus) finalizeCommits() {
 		return
 	}
 	consensus.PbftLog.AddMessage(pbftMsg)
+	consensus.ChainReader.WriteLastCommits(pbftMsg.Payload)
 
 	// find correct block content
 	block := consensus.PbftLog.GetBlockByHash(consensus.blockHash)
@@ -819,6 +820,7 @@ func (consensus *Consensus) onCommitted(msg *msg_pb.Message) {
 		consensus.getLogger().Warn().Msg("[OnCommitted] unable to parse msg")
 		return
 	}
+
 	if recvMsg.BlockNum < consensus.blockNum {
 		consensus.getLogger().Info().
 			Uint64("MsgBlockNum", recvMsg.BlockNum).
@@ -852,6 +854,7 @@ func (consensus *Consensus) onCommitted(msg *msg_pb.Message) {
 	}
 
 	consensus.PbftLog.AddMessage(recvMsg)
+	consensus.ChainReader.WriteLastCommits(recvMsg.Payload)
 	consensus.getLogger().Debug().
 		Uint64("MsgViewID", recvMsg.ViewID).
 		Uint64("MsgBlockNum", recvMsg.BlockNum).
@@ -904,17 +907,21 @@ func (consensus *Consensus) LastCommitSig() ([]byte, []byte, error) {
 	if consensus.blockNum <= 1 {
 		return nil, nil, nil
 	}
-	msgs := consensus.PbftLog.GetMessagesByTypeSeq(msg_pb.MessageType_COMMITTED, consensus.blockNum-1)
+	lastCommits, err := consensus.ChainReader.ReadLastCommits()
-	if len(msgs) != 1 {
+	if err != nil || len(lastCommits) < 96 {
-		return nil, nil, ctxerror.New("SetLastCommitSig failed with wrong number of committed message", "numCommittedMsg", len(msgs))
+		msgs := consensus.PbftLog.GetMessagesByTypeSeq(msg_pb.MessageType_COMMITTED, consensus.blockNum-1)
+		if len(msgs) != 1 {
+			return nil, nil, ctxerror.New("GetLastCommitSig failed with wrong number of committed message", "numCommittedMsg", len(msgs))
+		}
+		lastCommits = msgs[0].Payload
 	}
 	//#### Read payload data from committed msg
 	aggSig := make([]byte, 96)
-	bitmap := make([]byte, len(msgs[0].Payload)-96)
+	bitmap := make([]byte, len(lastCommits)-96)
 	offset := 0
-	copy(aggSig[:], msgs[0].Payload[offset:offset+96])
+	copy(aggSig[:], lastCommits[offset:offset+96])
 	offset += 96
-	copy(bitmap[:], msgs[0].Payload[offset:])
+	copy(bitmap[:], lastCommits[offset:])
 	//#### END Read payload data from committed msg
 	return aggSig, bitmap, nil
 }

core/blockchain.go

@@ -65,6 +65,7 @@ const (
 	badBlockLimit       = 10
 	triesInMemory       = 128
 	shardCacheLimit     = 2
+	commitsCacheLimit   = 10
 	epochCacheLimit     = 10
 
 	// BlockChainVersion ensures that an incompatible database forces a resync from scratch.
@@ -118,14 +119,15 @@ type BlockChain struct {
 	currentBlock     atomic.Value // Current head of the block chain
 	currentFastBlock atomic.Value // Current head of the fast-sync chain (may be above the block chain!)
 
-	stateCache      state.Database // State database to reuse between imports (contains state cache)
+	stateCache       state.Database // State database to reuse between imports (contains state cache)
-	bodyCache       *lru.Cache     // Cache for the most recent block bodies
+	bodyCache        *lru.Cache     // Cache for the most recent block bodies
-	bodyRLPCache    *lru.Cache     // Cache for the most recent block bodies in RLP encoded format
+	bodyRLPCache     *lru.Cache     // Cache for the most recent block bodies in RLP encoded format
-	receiptsCache   *lru.Cache     // Cache for the most recent receipts per block
+	receiptsCache    *lru.Cache     // Cache for the most recent receipts per block
-	blockCache      *lru.Cache     // Cache for the most recent entire blocks
+	blockCache       *lru.Cache     // Cache for the most recent entire blocks
-	futureBlocks    *lru.Cache     // future blocks are blocks added for later processing
+	futureBlocks     *lru.Cache     // future blocks are blocks added for later processing
-	shardStateCache *lru.Cache
+	shardStateCache  *lru.Cache
-	epochCache      *lru.Cache // Cache epoch number → first block number
+	lastCommitsCache *lru.Cache
+	epochCache       *lru.Cache // Cache epoch number → first block number
 
 	quit    chan struct{} // blockchain quit channel
 	running int32         // running must be called atomically
@@ -159,26 +161,28 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par
 	futureBlocks, _ := lru.New(maxFutureBlocks)
 	badBlocks, _ := lru.New(badBlockLimit)
 	shardCache, _ := lru.New(shardCacheLimit)
+	commitsCache, _ := lru.New(commitsCacheLimit)
 	epochCache, _ := lru.New(epochCacheLimit)
 
 	bc := &BlockChain{
-		chainConfig:     chainConfig,
+		chainConfig:      chainConfig,
-		cacheConfig:     cacheConfig,
+		cacheConfig:      cacheConfig,
-		db:              db,
+		db:               db,
-		triegc:          prque.New(nil),
+		triegc:           prque.New(nil),
-		stateCache:      state.NewDatabase(db),
+		stateCache:       state.NewDatabase(db),
-		quit:            make(chan struct{}),
+		quit:             make(chan struct{}),
-		shouldPreserve:  shouldPreserve,
+		shouldPreserve:   shouldPreserve,
-		bodyCache:       bodyCache,
+		bodyCache:        bodyCache,
-		bodyRLPCache:    bodyRLPCache,
+		bodyRLPCache:     bodyRLPCache,
-		receiptsCache:   receiptsCache,
+		receiptsCache:    receiptsCache,
-		blockCache:      blockCache,
+		blockCache:       blockCache,
-		futureBlocks:    futureBlocks,
+		futureBlocks:     futureBlocks,
-		shardStateCache: shardCache,
+		shardStateCache:  shardCache,
-		epochCache:      epochCache,
+		lastCommitsCache: commitsCache,
-		engine:          engine,
+		epochCache:       epochCache,
-		vmConfig:        vmConfig,
+		engine:           engine,
-		badBlocks:       badBlocks,
+		vmConfig:         vmConfig,
+		badBlocks:        badBlocks,
 	}
 	bc.SetValidator(NewBlockValidator(chainConfig, bc, engine))
 	bc.SetProcessor(NewStateProcessor(chainConfig, bc, engine))
@@ -1753,6 +1757,29 @@ func (bc *BlockChain) WriteShardStateBytes(
 	return nil
 }
 
+// ReadLastCommits retrieves last commits.
+func (bc *BlockChain) ReadLastCommits() ([]byte, error) {
+	if cached, ok := bc.lastCommitsCache.Get("lastCommits"); ok {
+		lastCommits := cached.([]byte)
+		return lastCommits, nil
+	}
+	lastCommits, err := rawdb.ReadLastCommits(bc.db)
+	if err != nil {
+		return nil, err
+	}
+	return lastCommits, nil
+}
+
+// WriteLastCommits saves the commits of last block.
+func (bc *BlockChain) WriteLastCommits(lastCommits []byte) error {
+	err := rawdb.WriteLastCommits(bc.db, lastCommits)
+	if err != nil {
+		return err
+	}
+	bc.lastCommitsCache.Add("lastCommits", lastCommits)
+	return nil
+}
+
 // GetVdfByNumber retrieves the rand seed given the block number, return 0 if not exist
 func (bc *BlockChain) GetVdfByNumber(number uint64) [32]byte {
 	header := bc.GetHeaderByNumber(number)

core/rawdb/accessors_chain.go

@@ -447,6 +447,27 @@ func WriteShardStateBytes(
 	return nil
 }
 
+// ReadLastCommits retrieves LastCommits.
+func ReadLastCommits(db DatabaseReader) ([]byte, error) {
+	var data []byte
+	data, err := db.Get(lastCommitsKey)
+	if err != nil {
+		return nil, ctxerror.New("cannot read last commits from rawdb").WithCause(err)
+	}
+	return data, nil
+}
+
+// WriteLastCommits stores last commits into database.
+func WriteLastCommits(
+	db DatabaseWriter, data []byte,
+) (err error) {
+	if err = db.Put(lastCommitsKey, data); err != nil {
+		return ctxerror.New("cannot write last commits").WithCause(err)
+	}
+	utils.GetLogger().Info("wrote last commits", "numShards", len(data))
+	return nil
+}
+
 // ReadEpochBlockNumber retrieves the epoch block number for the given epoch,
 // or nil if the given epoch is not found in the database.
 func ReadEpochBlockNumber(db DatabaseReader, epoch *big.Int) (*big.Int, error) {

core/rawdb/schema.go

@@ -55,6 +55,7 @@ var (
 	bloomBitsPrefix = []byte("B") // bloomBitsPrefix + bit (uint16 big endian) + section (uint64 big endian) + hash -> bloom bits
 
 	shardStatePrefix = []byte("ss") // shardStatePrefix + num (uint64 big endian) + hash -> shardState
+	lastCommitsKey   = []byte("LastCommits")
 
 	preimagePrefix = []byte("secure-key-")      // preimagePrefix + hash -> preimage
 	configPrefix   = []byte("ethereum-config-") // config prefix for the db

internal/configs/sharding/mainnet.go

@@ -7,8 +7,9 @@ import (
 )
 
 const (
-	mainnetEpochBlock1 = 294912 // 18 * 2^14
+	mainnetEpochBlock1 = 327680 // 20 * 2^14
 	blocksPerShard     = 16384  // 2^14
+	mainnetV1Epoch     = 1
 )
 
 // MainnetSchedule is the mainnet sharding configuration schedule.
@@ -18,10 +19,9 @@ type mainnetSchedule struct{}
 
 func (mainnetSchedule) InstanceForEpoch(epoch *big.Int) Instance {
 	switch {
-	//case epoch.Cmp(big.NewInt(1000)) >= 0:
+	case epoch.Cmp(big.NewInt(mainnetV1Epoch)) >= 0:
-	//	return mainnet6400
+		// first resharding epoch around 07/29/2019 7:30am PDT
-	//case epoch.Cmp(big.NewInt(100)) >= 0:
+		return mainnetV1
-	//	return mainnetV2
 	default: // genesis
 		return mainnetV0
 	}
@@ -53,8 +53,9 @@ func (ms mainnetSchedule) IsLastBlock(blockNum uint64) bool {
 	}
 }
 
-var mainnetReshardingEpoch = make([]*big.Int, 0)
+var mainnetReshardingEpoch = []*big.Int{big.NewInt(0), big.NewInt(mainnetV1Epoch)}
 var mainnetV0 = MustNewInstance(4, 150, 112, genesis.HarmonyAccounts, genesis.FoundationalNodeAccounts, mainnetReshardingEpoch)
+var mainnetV1 = MustNewInstance(4, 151, 112, genesis.HarmonyAccounts, genesis.FoundationalNodeAccountsV1, mainnetReshardingEpoch)
 
 //var mainnetV2 = MustNewInstance(8, 200, 100)
 //var mainnet6400 = MustNewInstance(16, 400, 50)

node/node_genesis.go

@@ -8,6 +8,9 @@ import (
 	"strings"
 
 	"github.com/ethereum/go-ethereum/common"
+
+	nodeconfig "github.com/harmony-one/harmony/internal/configs/node"
+
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/params"
@@ -15,10 +18,8 @@ import (
 	"github.com/harmony-one/harmony/core"
 	"github.com/harmony-one/harmony/core/types"
 	common2 "github.com/harmony-one/harmony/internal/common"
-	nodeconfig "github.com/harmony-one/harmony/internal/configs/node"
 	"github.com/harmony-one/harmony/internal/genesis"
 	"github.com/harmony-one/harmony/internal/utils"
-	demo "github.com/harmony-one/harmony/test/demo"
 )
 
 const (
@@ -77,7 +78,6 @@ func (node *Node) SetupGenesisBlock(db ethdb.Database, shardID uint32, myShardSt
 	case nodeconfig.Testnet:
 		fallthrough
 	case nodeconfig.Localnet:
-		AddDemoAddressesToGenesisAlloc(genesisAlloc)
 		fallthrough
 	case nodeconfig.Devnet:
 		chainConfig = *params.TestnetChainConfig
@@ -151,13 +151,3 @@ func AddNodeAddressesToGenesisAlloc(genesisAlloc core.GenesisAlloc) {
 		genesisAlloc[address] = core.GenesisAccount{Balance: testBankFunds}
 	}
 }
-
-// AddDemoAddressesToGenesisAlloc funds demo addresses with free fund.
-func AddDemoAddressesToGenesisAlloc(genesisAlloc core.GenesisAlloc) {
-	for _, account := range demo.DemoAccounts {
-		testBankFunds := big.NewInt(InitFreeFund)
-		testBankFunds = testBankFunds.Mul(testBankFunds, big.NewInt(params.Ether))
-		address := common.HexToAddress(account.Address)
-		genesisAlloc[address] = core.GenesisAccount{Balance: testBankFunds}
-	}
-}

test/demo/demo_constants.go

@@ -1,27 +0,0 @@
-package demo
-
-// DeployAccount is the accounts used for development.
-type DeployAccount struct {
-	Address string
-	Private string
-	Public  string
-}
-
-// DemoAccounts is the accounts used for demo.
-var DemoAccounts = [...]DeployAccount{
-	{Address: "0x1a3e7a44ee21101d7D64FBf29B0F6F1fc295F723", Private: "27978f895b11d9c737e1ab1623fde722c04b4f9ccb4ab776bf15932cc72d7c66", Public: "0x1a3e7a44ee21101d7D64FBf29B0F6F1fc295F723"},
-	{Address: "0x10A02A0a6e95a676AE23e2db04BEa3D1B8b7ca2E", Private: "371cb68abe6a6101ac88603fc847e0c013a834253acee5315884d2c4e387ebca", Public: "0x10A02A0a6e95a676AE23e2db04BEa3D1B8b7ca2E"},
-	{Address: "0x3e881F6C36A3A14a2D1816b0A5471d1caBB16F33", Private: "3f8af52063c6648be37d4b33559f784feb16d8e5ffaccf082b3657ea35b05977", Public: "0x3e881F6C36A3A14a2D1816b0A5471d1caBB16F33"},
-	{Address: "0x9d72989b68777a1f3FfD6F1DB079f1928373eE52", Private: "df77927961152e6a080ac299e7af2135fc0fb02eb044d0d7bbb1e8c5ad523809", Public: "0x9d72989b68777a1f3FfD6F1DB079f1928373eE52"},
-	{Address: "0x67957240b6eB045E17B47dcE98102f09aaC03435", Private: "fcff43741ad2dd0b232efb159dc47736bbb16f11a79aaeec39b388d06f91116d", Public: "0x67957240b6eB045E17B47dcE98102f09aaC03435"},
-	{Address: "0xf70fBDB1AD002baDF19024785b1a4bf6F841F558", Private: "916d3d78b7f413452434e89f9c1f1d136995ef02d7dc8038e84cc9cef4a02b96", Public: "0xf70fBDB1AD002baDF19024785b1a4bf6F841F558"},
-	{Address: "0x3f1A559be93C9456Ca75712535Fd522f5EC22c6B", Private: "f5967bd87fd2b9dbf51855a2a75ef0a811c84953b3b300ffe90c430a5c856303", Public: "0x3f1A559be93C9456Ca75712535Fd522f5EC22c6B"},
-	{Address: "0xedD257B4e0F5e7d632c737f4277e93b64DC268FC", Private: "f02f7b3bb5aa03aa97f9e030020dd9ca306b209742fafe018104a3207a70a3c9", Public: "0xedD257B4e0F5e7d632c737f4277e93b64DC268FC"},
-	{Address: "0x66A74477FC1dd0F4924ed943C1d2F1Dece3Ab138", Private: "0436864cc15772448f88dd40554592ff6c91a6c1a389d965ad26ee143db1234d", Public: "0x66A74477FC1dd0F4924ed943C1d2F1Dece3Ab138"},
-	{Address: "0x04178CdbCe3a9Ff9Ea385777aFc4b78B3E745281", Private: "dea956e530073ab23d9cae704f5d068482b1977c3173c9efd697c48a7fd3ce83", Public: "0x04178CdbCe3a9Ff9Ea385777aFc4b78B3E745281"},
-	{Address: "0x46C61d50874A7A06D29FF89a710AbBD0856265be", Private: "af539d4ace07a9f601a8d3a6ca6f914d5a9fabe09cfe7d62ebc2348fc95f03a4", Public: "0x46C61d50874A7A06D29FF89a710AbBD0856265be"},
-	{Address: "0xfE9BABE6904C28E31971337738FBCBAF8c72873e", Private: "7d24797eeba0cdac9bf943f0d82c4b18eb206108d6e1b7f610471594c0c94306", Public: "0xfE9BABE6904C28E31971337738FBCBAF8c72873e"},
-	{Address: "0x3f78622de8D8f87EAa0E8b28C2851e2450E91250", Private: "4fa2fecce1becfaf7e5fba5394caacb318333b04071462b5ca850ee5a406dcfe", Public: "0x3f78622de8D8f87EAa0E8b28C2851e2450E91250"},
-	{Address: "0xd2Cb501B40D3a9a013A38267a4d2A4Cf6bD2CAa8", Private: "3c8642f7188e05acc4467d9e2aa7fd539e82aa90a5497257cf0ecbb98ed3b88f", Public: "0xd2Cb501B40D3a9a013A38267a4d2A4Cf6bD2CAa8"},
-	{Address: "0x2676e6dd2d7618be14cb4c18a355c81bf7aac647", Private: "bf29f6a33b2c24a8b5182ef44cc35ce87534ef827c8dfbc1e6bb536aa52f8563", Public: "0x2676e6dd2d7618be14cb4c18a355c81bf7aac647"},
-}