From f4748d7bcc1b12549bb74470affae4b14d3f90e8 Mon Sep 17 00:00:00 2001 From: Rongjian Lan Date: Tue, 12 Mar 2019 13:35:19 -0700 Subject: [PATCH 1/3] Add shard state hash into genesis --- core/gen_genesis.go | 58 ++++++++++++++++++++++++------------------- core/genesis.go | 46 ++++++++++++++++++---------------- core/resharding.go | 21 +++++++++------- node/node_genesis.go | 7 +++--- node/node_newblock.go | 4 +-- 5 files changed, 74 insertions(+), 62 deletions(-) diff --git a/core/gen_genesis.go b/core/gen_genesis.go index f10c683d9..d9eb029fd 100644 --- a/core/gen_genesis.go +++ b/core/gen_genesis.go @@ -18,19 +18,20 @@ var _ = (*genesisSpecMarshaling)(nil) // MarshalJSON marshals as JSON. func (g Genesis) MarshalJSON() ([]byte, error) { type Genesis struct { - Config *params.ChainConfig `json:"config"` - Nonce math.HexOrDecimal64 `json:"nonce"` - ShardID uint32 `json:"shardID"` - Timestamp math.HexOrDecimal64 `json:"timestamp"` - ExtraData hexutil.Bytes `json:"extraData"` - GasLimit math.HexOrDecimal64 `json:"gasLimit" gencodec:"required"` - Difficulty *math.HexOrDecimal256 `json:"difficulty" gencodec:"required"` - Mixhash common.Hash `json:"mixHash"` - Coinbase common.Address `json:"coinbase"` - Alloc map[common.UnprefixedAddress]GenesisAccount `json:"alloc" gencodec:"required"` - Number math.HexOrDecimal64 `json:"number"` - GasUsed math.HexOrDecimal64 `json:"gasUsed"` - ParentHash common.Hash `json:"parentHash"` + Config *params.ChainConfig `json:"config"` + Nonce math.HexOrDecimal64 `json:"nonce"` + ShardID uint32 `json:"shardID"` + Timestamp math.HexOrDecimal64 `json:"timestamp"` + ExtraData hexutil.Bytes `json:"extraData"` + GasLimit math.HexOrDecimal64 `json:"gasLimit" gencodec:"required"` + Difficulty *math.HexOrDecimal256 `json:"difficulty" gencodec:"required"` + Mixhash common.Hash `json:"mixHash"` + Coinbase common.Address `json:"coinbase"` + Alloc map[common.UnprefixedAddress]GenesisAccount `json:"alloc" gencodec:"required"` + ShardStateHash common.Hash `json:"shardStateHash"` + Number math.HexOrDecimal64 `json:"number"` + GasUsed math.HexOrDecimal64 `json:"gasUsed"` + ParentHash common.Hash `json:"parentHash"` } var enc Genesis enc.Config = g.Config @@ -48,6 +49,7 @@ func (g Genesis) MarshalJSON() ([]byte, error) { enc.Alloc[common.UnprefixedAddress(k)] = v } } + enc.ShardStateHash = g.ShardStateHash enc.Number = math.HexOrDecimal64(g.Number) enc.GasUsed = math.HexOrDecimal64(g.GasUsed) enc.ParentHash = g.ParentHash @@ -57,19 +59,20 @@ func (g Genesis) MarshalJSON() ([]byte, error) { // UnmarshalJSON unmarshals from JSON. func (g *Genesis) UnmarshalJSON(input []byte) error { type Genesis struct { - Config *params.ChainConfig `json:"config"` - Nonce *math.HexOrDecimal64 `json:"nonce"` - ShardID *uint32 `json:"shardID"` - Timestamp *math.HexOrDecimal64 `json:"timestamp"` - ExtraData *hexutil.Bytes `json:"extraData"` - GasLimit *math.HexOrDecimal64 `json:"gasLimit" gencodec:"required"` - Difficulty *math.HexOrDecimal256 `json:"difficulty" gencodec:"required"` - Mixhash *common.Hash `json:"mixHash"` - Coinbase *common.Address `json:"coinbase"` - Alloc map[common.UnprefixedAddress]GenesisAccount `json:"alloc" gencodec:"required"` - Number *math.HexOrDecimal64 `json:"number"` - GasUsed *math.HexOrDecimal64 `json:"gasUsed"` - ParentHash *common.Hash `json:"parentHash"` + Config *params.ChainConfig `json:"config"` + Nonce *math.HexOrDecimal64 `json:"nonce"` + ShardID *uint32 `json:"shardID"` + Timestamp *math.HexOrDecimal64 `json:"timestamp"` + ExtraData *hexutil.Bytes `json:"extraData"` + GasLimit *math.HexOrDecimal64 `json:"gasLimit" gencodec:"required"` + Difficulty *math.HexOrDecimal256 `json:"difficulty" gencodec:"required"` + Mixhash *common.Hash `json:"mixHash"` + Coinbase *common.Address `json:"coinbase"` + Alloc map[common.UnprefixedAddress]GenesisAccount `json:"alloc" gencodec:"required"` + ShardStateHash *common.Hash `json:"shardStateHash"` + Number *math.HexOrDecimal64 `json:"number"` + GasUsed *math.HexOrDecimal64 `json:"gasUsed"` + ParentHash *common.Hash `json:"parentHash"` } var dec Genesis if err := json.Unmarshal(input, &dec); err != nil { @@ -111,6 +114,9 @@ func (g *Genesis) UnmarshalJSON(input []byte) error { for k, v := range dec.Alloc { g.Alloc[common.Address(k)] = v } + if dec.ShardStateHash != nil { + g.ShardStateHash = *dec.ShardStateHash + } if dec.Number != nil { g.Number = uint64(*dec.Number) } diff --git a/core/genesis.go b/core/genesis.go index 828276052..65aade753 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -45,16 +45,17 @@ var errGenesisNoConfig = errors.New("genesis has no chain configuration") // Genesis specifies the header fields, state of a genesis block. It also defines hard // fork switch-over blocks through the chain configuration. type Genesis struct { - Config *params.ChainConfig `json:"config"` - Nonce uint64 `json:"nonce"` - ShardID uint32 `json:"shardID"` - Timestamp uint64 `json:"timestamp"` - ExtraData []byte `json:"extraData"` - GasLimit uint64 `json:"gasLimit" gencodec:"required"` - Difficulty *big.Int `json:"difficulty" gencodec:"required"` - Mixhash common.Hash `json:"mixHash"` - Coinbase common.Address `json:"coinbase"` - Alloc GenesisAlloc `json:"alloc" gencodec:"required"` + Config *params.ChainConfig `json:"config"` + Nonce uint64 `json:"nonce"` + ShardID uint32 `json:"shardID"` + Timestamp uint64 `json:"timestamp"` + ExtraData []byte `json:"extraData"` + GasLimit uint64 `json:"gasLimit" gencodec:"required"` + Difficulty *big.Int `json:"difficulty" gencodec:"required"` + Mixhash common.Hash `json:"mixHash"` + Coinbase common.Address `json:"coinbase"` + Alloc GenesisAlloc `json:"alloc" gencodec:"required"` + ShardStateHash common.Hash `json:"shardStateHash"` // These fields are used for consensus tests. Please don't use them // in actual genesis blocks. @@ -237,18 +238,19 @@ func (g *Genesis) ToBlock(db ethdb.Database) *types.Block { } root := statedb.IntermediateRoot(false) head := &types.Header{ - Number: new(big.Int).SetUint64(g.Number), - Nonce: types.EncodeNonce(g.Nonce), - ShardID: types.EncodeShardID(g.ShardID), - Time: new(big.Int).SetUint64(g.Timestamp), - ParentHash: g.ParentHash, - Extra: g.ExtraData, - GasLimit: g.GasLimit, - GasUsed: g.GasUsed, - Difficulty: g.Difficulty, - MixDigest: g.Mixhash, - Coinbase: g.Coinbase, - Root: root, + Number: new(big.Int).SetUint64(g.Number), + Nonce: types.EncodeNonce(g.Nonce), + ShardID: types.EncodeShardID(g.ShardID), + Time: new(big.Int).SetUint64(g.Timestamp), + ParentHash: g.ParentHash, + Extra: g.ExtraData, + GasLimit: g.GasLimit, + GasUsed: g.GasUsed, + Difficulty: g.Difficulty, + MixDigest: g.Mixhash, + Coinbase: g.Coinbase, + Root: root, + ShardStateHash: g.ShardStateHash, } if g.GasLimit == 0 { head.GasLimit = 10000000000 // TODO(RJ): figure out better solution. // params.GenesisGasLimit diff --git a/core/resharding.go b/core/resharding.go index 471ad43f5..18de1795a 100644 --- a/core/resharding.go +++ b/core/resharding.go @@ -19,9 +19,12 @@ import ( const ( // InitialSeed is the initial random seed, a magic number to answer everything, remove later InitialSeed uint32 = 42 - // FirstEpoch is the number of the first epoch. - // TODO(minhdoan): we should design the first epoch as 0. Please figure out how to change other logic to make it 0 - FirstEpoch = 1 + // GenesisEpoch is the number of the first genesis epoch. + GenesisEpoch = 0 + // GenesisShardNum is the number of shard at genesis + GenesisShardNum = 3 + // GenesisShardSize is the size of each shard at genesis + GenesisShardSize = 10 ) // ShardingState is data structure hold the sharding state @@ -134,8 +137,8 @@ func GetShardingStateFromBlockChain(bc *BlockChain, epoch uint64) *ShardingState // CalculateNewShardState get sharding state from previous epoch and calculate sharding state for new epoch // TODO: currently, we just mock everything func CalculateNewShardState(bc *BlockChain, epoch uint64, stakeInfo *map[common.Address]*structs.StakeInfo) types.ShardState { - if epoch == FirstEpoch { - return getInitShardState(3, 10) + if epoch == GenesisEpoch { + return GetInitShardState() } ss := GetShardingStateFromBlockChain(bc, epoch-1) newNodeList := ss.UpdateShardingState(stakeInfo) @@ -190,13 +193,13 @@ func (ss *ShardingState) calculateKickoutRate(newNodeList []types.NodeID) float6 return math.Max(0.1, math.Min(rate, 1.0)) } -// getInitShardState returns the initial shard state at genesis. -func getInitShardState(numberOfShards, numNodesPerShard int) types.ShardState { +// GetInitShardState returns the initial shard state at genesis. +func GetInitShardState() types.ShardState { shardState := types.ShardState{} - for i := 0; i < numberOfShards; i++ { + for i := 0; i < GenesisShardNum; i++ { com := types.Committee{ShardID: uint32(i)} if i == 0 { - for j := 0; j < numNodesPerShard; j++ { + for j := 0; j < GenesisShardSize; j++ { priKey := bls.SecretKey{} priKey.SetHexString(contract.InitialBeaconChainAccounts[i].Private) addrBytes := priKey.GetPublicKey().GetAddress() diff --git a/node/node_genesis.go b/node/node_genesis.go index 4b373a651..9a7cf364c 100644 --- a/node/node_genesis.go +++ b/node/node_genesis.go @@ -43,9 +43,10 @@ func (node *Node) GenesisBlockSetup(db ethdb.Database) (*core.BlockChain, error) chainConfig := params.TestChainConfig chainConfig.ChainID = big.NewInt(int64(node.Consensus.ShardID)) // Use ChainID as piggybacked ShardID gspec := core.Genesis{ - Config: chainConfig, - Alloc: genesisAlloc, - ShardID: uint32(node.Consensus.ShardID), + Config: chainConfig, + Alloc: genesisAlloc, + ShardID: uint32(node.Consensus.ShardID), + ShardStateHash: core.GetInitShardState().Hash(), } // Store genesis block into db. diff --git a/node/node_newblock.go b/node/node_newblock.go index 79aa8ba52..a18f70297 100644 --- a/node/node_newblock.go +++ b/node/node_newblock.go @@ -60,7 +60,7 @@ func (node *Node) WaitForConsensusReady(readySignal chan struct{}, stopChan chan } else { // add new shard state if it's epoch block // TODO(minhdoan): only happens for beaconchain - node.addNewShardState(block) + node.addNewShardStateHash(block) newBlock = block break } @@ -78,7 +78,7 @@ func (node *Node) WaitForConsensusReady(readySignal chan struct{}, stopChan chan }() } -func (node *Node) addNewShardState(block *types.Block) { +func (node *Node) addNewShardStateHash(block *types.Block) { shardState := node.blockchain.GetNewShardState(block, &node.CurrentStakes) if shardState != nil { shardHash := shardState.Hash() From 8fa84d223002dfb1835608feedc2a93a2c222eb3 Mon Sep 17 00:00:00 2001 From: Rongjian Lan Date: Tue, 12 Mar 2019 16:31:23 -0700 Subject: [PATCH 2/3] Setup genesis shard state, and fix other issues --- core/blockchain.go | 8 ++++---- core/resharding.go | 10 +++++++--- node/node.go | 3 +++ node/node_handler.go | 4 ++-- test/configs/beaconchain.txt | 1 + 5 files changed, 17 insertions(+), 9 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index 1ad96c429..fddf75b52 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1732,8 +1732,8 @@ func (bc *BlockChain) ValidateNewShardState(block *types.Block, stakeInfo *map[c return nil } -// InsertNewShardState insert new shard state into epoch block -func (bc *BlockChain) InsertNewShardState(block *types.Block, stakeInfo *map[common.Address]*structs.StakeInfo) { +// StoreNewShardState insert new shard state into epoch block +func (bc *BlockChain) StoreNewShardState(block *types.Block, stakeInfo *map[common.Address]*structs.StakeInfo) { // write state into db. shardState := bc.GetNewShardState(block, stakeInfo) if shardState == nil { @@ -1742,8 +1742,8 @@ func (bc *BlockChain) InsertNewShardState(block *types.Block, stakeInfo *map[com hash := block.Hash() number := block.NumberU64() rawdb.WriteShardState(bc.db, hash, number, shardState) - utils.GetLogInstance().Debug("[resharding] save new shard state success", "shardStateHash", shardState.Hash()) + utils.GetLogInstance().Debug("[Resharding] Saved new shard state success", "shardStateHash", shardState.Hash()) for _, c := range shardState { - utils.GetLogInstance().Debug("[resharding] new shard information", "shardID", c.ShardID, "NodeList", c.NodeList) + utils.GetLogInstance().Debug("[Resharding] Shard Info", "shardID", c.ShardID, "NodeList", c.NodeList) } } diff --git a/core/resharding.go b/core/resharding.go index 18de1795a..c07aa44a1 100644 --- a/core/resharding.go +++ b/core/resharding.go @@ -86,8 +86,11 @@ func (ss *ShardingState) cuckooResharding(percent float64) { // assignLeaders will first add new nodes into shards, then use cuckoo rule to reshard to get new shard state func (ss *ShardingState) assignLeaders() { for i := 0; i < ss.numShards; i++ { - Shuffle(ss.shardState[i].NodeList) - ss.shardState[i].Leader = ss.shardState[i].NodeList[0] + // At genesis epoch, the shards are empty. + if len(ss.shardState[i].NodeList) > 0 { + Shuffle(ss.shardState[i].NodeList) + ss.shardState[i].Leader = ss.shardState[i].NodeList[0] + } } } @@ -143,6 +146,7 @@ func CalculateNewShardState(bc *BlockChain, epoch uint64, stakeInfo *map[common. ss := GetShardingStateFromBlockChain(bc, epoch-1) newNodeList := ss.UpdateShardingState(stakeInfo) percent := ss.calculateKickoutRate(newNodeList) + utils.GetLogInstance().Info("Kickout Percentage", "percentage", percent) ss.Reshard(newNodeList, percent) return ss.shardState } @@ -201,7 +205,7 @@ func GetInitShardState() types.ShardState { if i == 0 { for j := 0; j < GenesisShardSize; j++ { priKey := bls.SecretKey{} - priKey.SetHexString(contract.InitialBeaconChainAccounts[i].Private) + priKey.SetHexString(contract.InitialBeaconChainAccounts[j].Private) addrBytes := priKey.GetPublicKey().GetAddress() address := hex.EncodeToString(addrBytes[:]) com.NodeList = append(com.NodeList, types.NodeID(address)) diff --git a/node/node.go b/node/node.go index 73adfb43a..a536e2bff 100644 --- a/node/node.go +++ b/node/node.go @@ -240,6 +240,9 @@ func New(host p2p.Host, consensusObj *consensus.Consensus, db ethdb.Database) *N os.Exit(1) } node.blockchain = chain + // Store the genesis shard state into db. + node.blockchain.StoreNewShardState(node.blockchain.CurrentBlock(), nil) + node.BlockChannel = make(chan *types.Block) node.ConfirmedBlockChannel = make(chan *types.Block) node.BeaconBlockChannel = make(chan *types.Block) diff --git a/node/node_handler.go b/node/node_handler.go index 6ea3e0a06..9dbf0cefa 100644 --- a/node/node_handler.go +++ b/node/node_handler.go @@ -154,7 +154,7 @@ func (node *Node) messageHandler(content []byte, sender string) { utils.GetLogInstance().Info("NET: received message: Node/Control") controlType := msgPayload[0] if proto_node.ControlMessageType(controlType) == proto_node.STOP { - utils.GetLogInstance().Debug("Stopping Node", "node", node, "numBlocks", node.blockchain.CurrentBlock().NumberU64(), "numTxsProcessed", node.countNumTransactionsInBlockchain()) + utils.GetLogInstance().Debug("Stopping Node", "numBlocks", node.blockchain.CurrentBlock().NumberU64(), "numTxsProcessed", node.countNumTransactionsInBlockchain()) var avgBlockSizeInBytes common.StorageSize txCount := 0 @@ -298,7 +298,7 @@ func (node *Node) PostConsensusProcessing(newBlock *types.Block) { func (node *Node) AddNewBlock(newBlock *types.Block) { blockNum, err := node.blockchain.InsertChain([]*types.Block{newBlock}) - node.blockchain.InsertNewShardState(newBlock, &node.CurrentStakes) + node.blockchain.StoreNewShardState(newBlock, &node.CurrentStakes) if err != nil { utils.GetLogInstance().Debug("Error adding new block to blockchain", "blockNum", blockNum, "Error", err) } else { diff --git a/test/configs/beaconchain.txt b/test/configs/beaconchain.txt index a6cfdea2e..cf674290b 100644 --- a/test/configs/beaconchain.txt +++ b/test/configs/beaconchain.txt @@ -8,3 +8,4 @@ 127.0.0.1 9007 validator 0 127.0.0.1 9008 validator 0 127.0.0.1 9009 validator 0 +127.0.0.1 19999 client 0 From c4a70adc9a2e38a10fbe184c7684915581dbb167 Mon Sep 17 00:00:00 2001 From: Rongjian Lan Date: Tue, 12 Mar 2019 17:15:37 -0700 Subject: [PATCH 3/3] Add TODO --- core/resharding.go | 1 + 1 file changed, 1 insertion(+) diff --git a/core/resharding.go b/core/resharding.go index c07aa44a1..fc0c2a6cb 100644 --- a/core/resharding.go +++ b/core/resharding.go @@ -198,6 +198,7 @@ func (ss *ShardingState) calculateKickoutRate(newNodeList []types.NodeID) float6 } // GetInitShardState returns the initial shard state at genesis. +// TODO: make the deploy.sh config file in sync with genesis constants. func GetInitShardState() types.ShardState { shardState := types.ShardState{} for i := 0; i < GenesisShardNum; i++ {