From f0747bdd001dc99e1a3a05e0d0557baea5dfc9e0 Mon Sep 17 00:00:00 2001 From: Andy Wu Date: Wed, 16 Oct 2019 21:01:45 -0700 Subject: [PATCH 01/45] fn key swaps for epoch 51 find a duplicated key which was introduced in epoch 46 re-targeting epoch 54; verified with Li --- internal/configs/sharding/mainnet.go | 4 ++-- internal/configs/sharding/shardingconfig_test.go | 2 +- internal/genesis/foundational.go | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/configs/sharding/mainnet.go b/internal/configs/sharding/mainnet.go index 6470c0ec8..9b032e5c4 100644 --- a/internal/configs/sharding/mainnet.go +++ b/internal/configs/sharding/mainnet.go @@ -27,7 +27,7 @@ const ( mainnetV1_2Epoch = 25 mainnetV1_3Epoch = 36 mainnetV1_4Epoch = 46 - mainnetV1_5Epoch = 50 + mainnetV1_5Epoch = 54 mainnetMaxTxAmountLimit = 1e3 // unit is interface{} One mainnetMaxNumRecentTxsPerAccountLimit = 1e2 @@ -50,7 +50,7 @@ type mainnetSchedule struct{} func (mainnetSchedule) InstanceForEpoch(epoch *big.Int) Instance { switch { case epoch.Cmp(big.NewInt(mainnetV1_5Epoch)) >= 0: - // forty-nine resharding epoch (for shard 0) around 17/10/2019 4:05:16 PDT + // 54 resharding epoch (for shard 0) around 23/10/2019 ~10:05 PDT return mainnetV1_5 case epoch.Cmp(big.NewInt(mainnetV1_4Epoch)) >= 0: // forty-sixth resharding epoch around 10/10/2019 8:06pm PDT diff --git a/internal/configs/sharding/shardingconfig_test.go b/internal/configs/sharding/shardingconfig_test.go index 210fe36dd..07534a890 100644 --- a/internal/configs/sharding/shardingconfig_test.go +++ b/internal/configs/sharding/shardingconfig_test.go @@ -36,7 +36,7 @@ func TestMainnetInstanceForEpoch(t *testing.T) { mainnetV1_4, }, { - big.NewInt(50), + big.NewInt(54), mainnetV1_5, }, } diff --git a/internal/genesis/foundational.go b/internal/genesis/foundational.go index 5cd326269..13f846713 100644 --- a/internal/genesis/foundational.go +++ b/internal/genesis/foundational.go @@ -2680,7 +2680,7 @@ var FoundationalNodeAccountsV1_4 = []DeployAccount{ {Index: "319", Address: "one19c4uqfzezuws7e4ka4kvc5r09suks2ghpyg6xw", BlsPublicKey: "51b2019b222df63fc99d202b03834dee09f1ef11e25a03592a96c1d01bca2bedfc25e0f26d88dcbb8a7176e30e1ec116"}, } -// FoundationalNodeAccountsV1_5 are the accounts for the foundational nodes from Epoch 50. +// FoundationalNodeAccountsV1_5 are the accounts for the foundational nodes from Epoch 54. var FoundationalNodeAccountsV1_5 = []DeployAccount{ {Index: "0", Address: "one1y0xcf40fg65n2ehm8fx5vda4thrkymhpg45ecj", BlsPublicKey: "9e70e8d76851f6e8dc648255acdd57bb5c49cdae7571aed43f86e9f140a6343caed2ffa860919d03e0912411fee4850a"}, {Index: "1", Address: "one18lp2w7ghhuajdpzl8zqeddza97u92wtkfcwpjk", BlsPublicKey: "fce3097d9fc234d34d6eaef3eecd0365d435d1118f69f2da1ed2a69ba725270771572e40347c222aca784cb973307b11"}, @@ -2800,7 +2800,7 @@ var FoundationalNodeAccountsV1_5 = []DeployAccount{ {Index: "115", Address: "one14ajehwyxpzpzxhke77mhtt0z6k5z6cevgf6rfa", BlsPublicKey: "52ba9ca9d046ac237214e81438b054d42b17c16654b041562723d8e6e928f92a83e6373da28a821d285ebfe118e81884"}, {Index: "116", Address: "one1hxqhp9tls9r4v5hz208g93exhvz5ak258ut7d2", BlsPublicKey: "95bad32a857901a2eecf20aa516a6fc0c21d85015ba0dc70a966f0bd70b0f3bc0f5af356fac630ef53e5e1a329d7fe0a"}, {Index: "117", Address: "one1wt5darzj8wd385xl8stccj4sv6553hgckaypfr", BlsPublicKey: "9622f8a5590d6ef8ca94e6c866d663aa0398caf00a88b2dd059dc7a63daa8600828a85737eca4e595caa382b5d407205"}, - {Index: "118", Address: "one19saqljg2w5n402p589y6xenjc6lan46a9l9tah", BlsPublicKey: "bcd24c722dc5dd3727bc3f027e3f681e4d1f5a552513d158645833eb8d8d39ec1076370b55e063aeed5a7825eb6aa20a"}, + {Index: "118", Address: "one1k80wv3uvfw5r0qhzp9yxn94u4jxu8my2xwuk87", BlsPublicKey: "bcd24c722dc5dd3727bc3f027e3f681e4d1f5a552513d158645833eb8d8d39ec1076370b55e063aeed5a7825eb6aa20a"}, {Index: "119", Address: "one1kwqkyzq2pmhvufe9528g9nd966ur54v6auzruf", BlsPublicKey: "aaac4eb8260e6cee7f19fbcae721ce2d68f125461953a583adca44407194452e7ac41de0757e2921c8fed83469172f92"}, {Index: "120", Address: "one1gjas4xurmc0rguafq63ql65rwuxayukm74w2mn", BlsPublicKey: "d6c8cf5553fa77257d26ba6b201294a2a497d070d420ab76c044efc0f4325f40b5664e7a7f973940ef1ea57530215886"}, {Index: "121", Address: "one1pkw7wnplp077fn6phv2kfejw3u7wvx0m9vppzc", BlsPublicKey: "92d5e3fb5d3f1e64af4be7c0acbd457b68a2ec59cf34aaaa0bac04d0e0346b283a65e0227378a60e1fe7af2407d9c50a"}, From 3c394abac3affc7f5d6e436fd150085577468953 Mon Sep 17 00:00:00 2001 From: Eugene Kim Date: Tue, 22 Oct 2019 19:51:17 +0000 Subject: [PATCH 02/45] Test newer FN tables --- internal/genesis/genesis_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/genesis/genesis_test.go b/internal/genesis/genesis_test.go index 8c5e7672a..620ef0734 100644 --- a/internal/genesis/genesis_test.go +++ b/internal/genesis/genesis_test.go @@ -57,6 +57,9 @@ func TestCommitteeAccounts(test *testing.T) { testAccounts(test, FoundationalNodeAccountsV1) testAccounts(test, FoundationalNodeAccountsV1_1) testAccounts(test, FoundationalNodeAccountsV1_2) + testAccounts(test, FoundationalNodeAccountsV1_3) + testAccounts(test, FoundationalNodeAccountsV1_4) + testAccounts(test, FoundationalNodeAccountsV1_5) testAccounts(test, HarmonyAccounts) testAccounts(test, TNHarmonyAccounts) testAccounts(test, TNFoundationalAccounts) From 2c1819a4e2c97dd100046f0a3aaa86ff1c01f490 Mon Sep 17 00:00:00 2001 From: chao Date: Thu, 24 Oct 2019 13:41:22 -0700 Subject: [PATCH 03/45] remove unused const --- staking/params.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/staking/params.go b/staking/params.go index 7c3ce9e57..a5a55e7e0 100644 --- a/staking/params.go +++ b/staking/params.go @@ -7,12 +7,10 @@ import ( const ( isValidatorKeyStr = "Harmony/IsValidator/v0" isValidatorStr = "Harmony/IsAValidator/v0" - isNotValidatorStr = "Harmony/IsNotAValidator/v0" ) // keys used to retrieve staking related informatio var ( IsValidatorKey = crypto.Keccak256Hash([]byte(isValidatorKeyStr)) IsValidator = crypto.Keccak256Hash([]byte(isValidatorStr)) - IsNotValidator = crypto.Keccak256Hash([]byte(isNotValidatorStr)) ) From 89ef7317ff6d9bc9640d981c573ca30e78a9a267 Mon Sep 17 00:00:00 2001 From: chao Date: Thu, 24 Oct 2019 14:34:43 -0700 Subject: [PATCH 04/45] add commitTransaction code for staking --- node/node_newblock.go | 4 ++++ node/worker/worker.go | 13 +++++++++---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/node/node_newblock.go b/node/node_newblock.go index 711cb7225..6288970bd 100644 --- a/node/node_newblock.go +++ b/node/node_newblock.go @@ -90,6 +90,10 @@ func (node *Node) proposeNewBlock() (*types.Block, error) { return nil, err } + if len(selectedStakingTxs) > 0 { + utils.Logger().Debug().Int("selectedStakingtxs", len(selectedStakingTxs)).Msg("hehe, staking txs proposed") + } + // Prepare cross shard transaction receipts receiptsList := node.proposeReceiptsProof() if len(receiptsList) != 0 { diff --git a/node/worker/worker.go b/node/worker/worker.go index a72f1c29d..221c62503 100644 --- a/node/worker/worker.go +++ b/node/worker/worker.go @@ -161,6 +161,7 @@ func (w *Worker) SelectStakingTransactionsForNewBlock( } selected := staking.StakingTransactions{} + // TODO: chao add total gas fee checking when needed unselected := staking.StakingTransactions{} invalid := staking.StakingTransactions{} for _, tx := range txs { @@ -181,7 +182,6 @@ func (w *Worker) SelectStakingTransactionsForNewBlock( w.current.header.GasUsed()).Msg("[SelectStakingTransaction] Block gas limit and usage info") return selected, unselected, invalid - } func (w *Worker) commitStakingTransaction(tx *staking.StakingTransaction, coinbase common.Address) ([]*types.Log, error) { @@ -256,9 +256,14 @@ func (w *Worker) CommitTransactions(txs types.Transactions, stakingTxns staking. } } - for _, stakingTx := range stakingTxns { - _ = stakingTx - // TODO: add logic to commit staking txns + for _, tx := range stakingTxns { + snap := w.current.state.Snapshot() + _, err := w.commitStakingTransaction(tx, coinbase) + if err != nil { + w.current.state.RevertToSnapshot(snap) + return err + + } } return nil } From af0a6608adcf391c7542458558bb189673690714 Mon Sep 17 00:00:00 2001 From: chao Date: Thu, 24 Oct 2019 14:59:17 -0700 Subject: [PATCH 05/45] update staking cli to latest data structure --- cmd/staking/root.go | 46 +++++++++++++++++++++++---------------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/cmd/staking/root.go b/cmd/staking/root.go index cfdbb2896..7a2cac6f9 100644 --- a/cmd/staking/root.go +++ b/cmd/staking/root.go @@ -18,6 +18,7 @@ import ( "github.com/harmony-one/harmony/accounts" "github.com/harmony-one/harmony/accounts/keystore" common2 "github.com/harmony-one/harmony/internal/common" + numeric "github.com/harmony-one/harmony/numeric" "github.com/harmony-one/harmony/shard" staking "github.com/harmony-one/harmony/staking/types" "github.com/spf13/cobra" @@ -58,29 +59,30 @@ func (s *staker) run(cmd *cobra.Command, args []string) error { p.DeserializeHexStr(testBLSPubKey) pub := shard.BlsPublicKey{} pub.FromLibBLSPublicKey(p) - // return staking.DirectiveCreateValidator, staking.CreateValidator{ - // Description: staking.Description{ - // Name: "something", - // Identity: "something else", - // Website: "some site, harmony.one", - // SecurityContact: "mr.smith", - // Details: "blah blah details", - // }, - // CommissionRates: staking.CommissionRates{ - // Rate: staking.NewDec(100), - // MaxRate: staking.NewDec(150), - // MaxChangeRate: staking.NewDec(5), - // }, - // MinSelfDelegation: big.NewInt(10), - // StakingAddress: common.Address(dAddr), - // PubKey: pub, - // Amount: big.NewInt(100), - // } - return staking.DirectiveDelegate, staking.Delegate{ - common.Address(dAddr), - common.Address(dAddr), - big.NewInt(10), + return staking.DirectiveCreateValidator, staking.CreateValidator{ + Description: &staking.Description{ + Name: "SuperHero", + Identity: "YouWouldNotKnow", + Website: "Secret Website", + SecurityContact: "Mr.DoubleZeroSeven", + Details: "blah blah blah", + }, + CommissionRates: staking.CommissionRates{ + Rate: numeric.NewDec(100), + MaxRate: numeric.NewDec(150), + MaxChangeRate: numeric.NewDec(5), + }, + MinSelfDelegation: big.NewInt(10), + MaxTotalDelegation: big.NewInt(3000), + ValidatorAddress: common.Address(dAddr), + SlotPubKeys: []shard.BlsPublicKey{pub}, + Amount: big.NewInt(100), } + // return staking.DirectiveDelegate, staking.Delegate{ + // common.Address(dAddr), + // common.Address(dAddr), + // big.NewInt(10), + // } } stakingTx, err := staking.NewStakingTransaction(2, 100, gasPrice, stakePayloadMaker) From 6ceede98f53621058214705567ade4fcd973979b Mon Sep 17 00:00:00 2001 From: chao Date: Thu, 24 Oct 2019 14:59:45 -0700 Subject: [PATCH 06/45] add crash protection for staking msg type cast --- staking/types/transaction.go | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/staking/types/transaction.go b/staking/types/transaction.go index a383b56da..3ef937131 100644 --- a/staking/types/transaction.go +++ b/staking/types/transaction.go @@ -12,6 +12,10 @@ import ( "github.com/harmony-one/harmony/crypto/hash" ) +var ( + errStakingTransactionTypeCastErr = errors.New("Cannot type cast to matching staking type") +) + type txdata struct { Directive StakeMsg interface{} @@ -159,19 +163,34 @@ func (tx *StakingTransaction) StakingMsgToBytes() (by []byte, err error) { switch stakeType { case DirectiveCreateValidator: - createValidator := tx.StakingMessage().(CreateValidator) + createValidator, ok := tx.StakingMessage().(CreateValidator) + if !ok { + return nil, errStakingTransactionTypeCastErr + } by, err = rlp.EncodeToBytes(createValidator) case DirectiveEditValidator: - editValidator := tx.StakingMessage().(EditValidator) + editValidator, ok := tx.StakingMessage().(EditValidator) + if !ok { + return nil, errStakingTransactionTypeCastErr + } by, err = rlp.EncodeToBytes(editValidator) case DirectiveDelegate: - delegate := tx.StakingMessage().(Delegate) + delegate, ok := tx.StakingMessage().(Delegate) + if !ok { + return nil, errStakingTransactionTypeCastErr + } by, err = rlp.EncodeToBytes(delegate) case DirectiveUndelegate: - undelegate := tx.StakingMessage().(Undelegate) + undelegate, ok := tx.StakingMessage().(Undelegate) + if !ok { + return nil, errStakingTransactionTypeCastErr + } by, err = rlp.EncodeToBytes(undelegate) case DirectiveCollectRewards: - collectRewards := tx.StakingMessage().(CollectRewards) + collectRewards, ok := tx.StakingMessage().(CollectRewards) + if !ok { + return nil, errStakingTransactionTypeCastErr + } by, err = rlp.EncodeToBytes(collectRewards) default: by = []byte{} From e75b4a49e290ee851ee617b3d5e886f6ebee575c Mon Sep 17 00:00:00 2001 From: Dennis Won Date: Wed, 30 Oct 2019 21:54:51 +0000 Subject: [PATCH 07/45] make process state sync func to return error --- api/service/syncing/errors.go | 12 +++- api/service/syncing/syncing.go | 104 ++++++++++++++++++++------------- node/node_syncing.go | 2 +- 3 files changed, 72 insertions(+), 46 deletions(-) diff --git a/api/service/syncing/errors.go b/api/service/syncing/errors.go index a70193583..ae8c7dae9 100644 --- a/api/service/syncing/errors.go +++ b/api/service/syncing/errors.go @@ -4,7 +4,13 @@ import "errors" // Errors ... var ( - ErrRegistrationFail = errors.New("[SYNC]: registration failed") - ErrGetBlock = errors.New("[SYNC]: get block failed") - ErrGetBlockHash = errors.New("[SYNC]: get blockhash failed") + ErrRegistrationFail = errors.New("[SYNC]: registration failed") + ErrGetBlock = errors.New("[SYNC]: get block failed") + ErrGetBlockHash = errors.New("[SYNC]: get blockhash failed") + ErrProcessStateSync = errors.New("[SYNC]: get blockhash failed") + ErrGetConsensusHashes = errors.New("[SYNC]: get consensus hashes failed") + ErrGenStateSyncTaskQueue = errors.New("[SYNC]: generate state sync task queue failed") + ErrDownloadBlocks = errors.New("[SYNC]: get download blocks failed") + ErrUpdateBlockAndStatus = errors.New("[SYNC]: update block and status failed") + ErrGenerateNewState = errors.New("[SYNC]: get generate new state failed") ) diff --git a/api/service/syncing/syncing.go b/api/service/syncing/syncing.go index 45f73dca7..6c1a849a7 100644 --- a/api/service/syncing/syncing.go +++ b/api/service/syncing/syncing.go @@ -26,13 +26,15 @@ import ( // Constants for syncing. const ( - TimesToFail = 5 // Downloadblocks service retry limit - RegistrationNumber = 3 - SyncingPortDifference = 3000 - inSyncThreshold = 0 // when peerBlockHeight - myBlockHeight <= inSyncThreshold, it's ready to join consensus - BatchSize uint32 = 1000 //maximum size for one query of block hashes - SyncLoopFrequency = 1 // unit in second - LastMileBlocksSize = 10 + downloadBlocksRetryLimit = 5 // downloadBlocks service retry limit + TimesToFail = 5 // downloadBlocks service retry limit + RegistrationNumber = 3 + SyncingPortDifference = 3000 + inSyncThreshold = 0 // when peerBlockHeight - myBlockHeight <= inSyncThreshold, it's ready to join consensus + SyncLoopBatchSize uint32 = 1000 // maximum size for one query of block hashes + verifyHeaderBatchSize uint64 = 100 // block chain header verification batch size + SyncLoopFrequency = 1 // unit in second + LastMileBlocksSize = 10 ) // SyncPeerConfig is peer config to sync. @@ -333,26 +335,27 @@ func (sc *SyncConfig) GetBlockHashesConsensusAndCleanUp() { sc.cleanUpPeers(maxFirstID) } -// GetConsensusHashes gets all hashes needed to download. -func (ss *StateSync) GetConsensusHashes(startHash []byte, size uint32) { +// getConsensusHashes gets all hashes needed to download. +func (ss *StateSync) getConsensusHashes(startHash []byte, size uint32) { var wg sync.WaitGroup ss.syncConfig.ForEachPeer(func(peerConfig *SyncPeerConfig) (brk bool) { wg.Add(1) go func() { defer wg.Done() + response := peerConfig.client.GetBlockHashes(startHash, size, ss.selfip, ss.selfport) if response == nil { utils.Logger().Warn(). Str("peerIP", peerConfig.ip). Str("peerPort", peerConfig.port). - Msg("[SYNC] GetConsensusHashes Nil Response") + Msg("[SYNC] getConsensusHashes Nil Response") return } if len(response.Payload) > int(size+1) { utils.Logger().Warn(). Uint32("requestSize", size). Int("respondSize", len(response.Payload)). - Msg("[SYNC] GetConsensusHashes: receive more blockHahses than request!") + Msg("[SYNC] getConsensusHashes: receive more blockHahses than request!") peerConfig.blockHashes = response.Payload[:size+1] } else { peerConfig.blockHashes = response.Payload @@ -404,7 +407,7 @@ func (ss *StateSync) downloadBlocks(bc *core.BlockChain) { if err != nil || len(payload) == 0 { count++ utils.Logger().Error().Err(err).Int("failNumber", count).Msg("[SYNC] downloadBlocks: GetBlocks failed") - if count > TimesToFail { + if count > downloadBlocksRetryLimit { break } if err := ss.stateSyncTaskQueue.Put(syncTask); err != nil { @@ -424,7 +427,7 @@ func (ss *StateSync) downloadBlocks(bc *core.BlockChain) { if err != nil { count++ utils.Logger().Error().Err(err).Msg("[SYNC] downloadBlocks: failed to DecodeBytes from received new block") - if count > TimesToFail { + if count > downloadBlocksRetryLimit { break } if err := ss.stateSyncTaskQueue.Put(syncTask); err != nil { @@ -527,50 +530,55 @@ func (ss *StateSync) getBlockFromLastMileBlocksByParentHash(parentHash common.Ha return nil } -func (ss *StateSync) updateBlockAndStatus(block *types.Block, bc *core.BlockChain, worker *worker.Worker) bool { - utils.Logger().Info().Str("blockHex", bc.CurrentBlock().Hash().Hex()).Msg("[SYNC] Current Block") +func (ss *StateSync) updateBlockAndStatus(block *types.Block, bc *core.BlockChain, worker *worker.Worker) error { + utils.Logger().Info().Str("blockHex", bc.CurrentBlock().Hash().Hex()).Msg("[SYNC] updateBlockAndStatus: Current Block") // Verify block signatures if block.NumberU64() > 1 { // Verify signature every 100 blocks - verifySig := block.NumberU64()%100 == 0 + verifySig := block.NumberU64()%verifyHeaderBatchSize == 0 err := bc.Engine().VerifyHeader(bc, block.Header(), verifySig) if err != nil { - utils.Logger().Error().Err(err).Msgf("[SYNC] failed verifying signatures for new block %d", block.NumberU64()) - utils.Logger().Debug().Interface("block", bc.CurrentBlock()).Msg("[SYNC] Rolling back last 99 blocks!") - for i := 0; i < 99; i++ { - bc.Rollback([]common.Hash{bc.CurrentBlock().Hash()}) + utils.Logger().Error().Err(err).Msgf("[SYNC] updateBlockAndStatus: failed verifying signatures for new block %d", block.NumberU64()) + + utils.Logger().Debug().Interface("block", bc.CurrentBlock()).Msg("[SYNC] updateBlockAndStatus: Rolling back last 99 blocks!") + var hashes []common.Hash + for i := uint64(0); i < verifyHeaderBatchSize-1; i++ { + hashes = append(hashes, bc.CurrentBlock().Hash()) } - return false + bc.Rollback(hashes) + return err } } _, err := bc.InsertChain([]*types.Block{block}, false /* verifyHeaders */) if err != nil { - utils.Logger().Error().Err(err).Msgf("[SYNC] Error adding new block to blockchain %d %d", block.NumberU64(), block.ShardID()) + utils.Logger().Error().Err(err).Msgf("[SYNC] updateBlockAndStatus: Error adding new block to blockchain %d %d", block.NumberU64(), block.ShardID()) - utils.Logger().Debug().Interface("block", bc.CurrentBlock()).Msg("[SYNC] Rolling back current block!") + utils.Logger().Debug().Interface("block", bc.CurrentBlock()).Msg("[SYNC] updateBlockAndStatus: Rolling back current block!") bc.Rollback([]common.Hash{bc.CurrentBlock().Hash()}) - return false + return err } utils.Logger().Info(). Uint64("blockHeight", bc.CurrentBlock().NumberU64()). Str("blockHex", bc.CurrentBlock().Hash().Hex()). - Msg("[SYNC] new block added to blockchain") - return true + Msg("[SYNC] updateBlockAndStatus: new block added to blockchain") + return nil } // generateNewState will construct most recent state from downloaded blocks -func (ss *StateSync) generateNewState(bc *core.BlockChain, worker *worker.Worker) { +func (ss *StateSync) generateNewState(bc *core.BlockChain, worker *worker.Worker) error { // update blocks created before node start sync parentHash := bc.CurrentBlock().Hash() + + var err error for { block := ss.getBlockFromOldBlocksByParentHash(parentHash) if block == nil { break } - ok := ss.updateBlockAndStatus(block, bc, worker) - if !ok { + err = ss.updateBlockAndStatus(block, bc, worker) + if err != nil { break } parentHash = block.Hash() @@ -586,8 +594,8 @@ func (ss *StateSync) generateNewState(bc *core.BlockChain, worker *worker.Worker if block == nil { break } - ok := ss.updateBlockAndStatus(block, bc, worker) - if !ok { + err = ss.updateBlockAndStatus(block, bc, worker) + if err != nil { break } parentHash = block.Hash() @@ -607,25 +615,26 @@ func (ss *StateSync) generateNewState(bc *core.BlockChain, worker *worker.Worker if block == nil { break } - ok := ss.updateBlockAndStatus(block, bc, worker) - if !ok { + err = ss.updateBlockAndStatus(block, bc, worker) + if err != nil { break } parentHash = block.Hash() } + + return err } // ProcessStateSync processes state sync from the blocks received but not yet processed so far -// TODO: return error -func (ss *StateSync) ProcessStateSync(startHash []byte, size uint32, bc *core.BlockChain, worker *worker.Worker) { +func (ss *StateSync) ProcessStateSync(startHash []byte, size uint32, bc *core.BlockChain, worker *worker.Worker) error { // Gets consensus hashes. - ss.GetConsensusHashes(startHash, size) + ss.getConsensusHashes(startHash, size) ss.generateStateSyncTaskQueue(bc) // Download blocks. if ss.stateSyncTaskQueue.Len() > 0 { ss.downloadBlocks(bc) } - ss.generateNewState(bc, worker) + return ss.generateNewState(bc, worker) } func (peerConfig *SyncPeerConfig) registerToBroadcast(peerHash []byte, ip, port string) error { @@ -738,17 +747,28 @@ Loop: currentHeight := bc.CurrentBlock().NumberU64() if currentHeight >= otherHeight { - utils.Logger().Info().Msgf("[SYNC] Node is now IN SYNC! (isBeacon: %t, ShardID: %d, otherHeight: %d, currentHeight: %d)", isBeacon, bc.ShardID(), otherHeight, currentHeight) + utils.Logger().Info(). + Msgf("[SYNC] Node is now IN SYNC! (isBeacon: %t, ShardID: %d, otherHeight: %d, currentHeight: %d)", + isBeacon, bc.ShardID(), otherHeight, currentHeight) break Loop } else { - utils.Logger().Debug().Msgf("[SYNC] Node is Not in Sync (isBeacon: %t, ShardID: %d, otherHeight: %d, currentHeight: %d)", isBeacon, bc.ShardID(), otherHeight, currentHeight) + utils.Logger().Debug(). + Msgf("[SYNC] Node is Not in Sync (isBeacon: %t, ShardID: %d, otherHeight: %d, currentHeight: %d)", + isBeacon, bc.ShardID(), otherHeight, currentHeight) } startHash := bc.CurrentBlock().Hash() size := uint32(otherHeight - currentHeight) - if size > BatchSize { - size = BatchSize + if size > SyncLoopBatchSize { + size = SyncLoopBatchSize + } + err := ss.ProcessStateSync(startHash[:], size, bc, worker) + if err != nil { + utils.Logger().Error().Err(err). + Msgf("[SYNC] ProcessStateSync failed (isBeacon: %t, ShardID: %d, otherHeight: %d, currentHeight: %d)", + isBeacon, bc.ShardID(), otherHeight, currentHeight) + // should we still call UpdateConsensusInformation() upon state sync failure? + // how to handle error here? } - ss.ProcessStateSync(startHash[:], size, bc, worker) ss.purgeOldBlocksFromCache() if consensus != nil { consensus.UpdateConsensusInformation() diff --git a/node/node_syncing.go b/node/node_syncing.go index a1fdf67c4..359496716 100644 --- a/node/node_syncing.go +++ b/node/node_syncing.go @@ -332,7 +332,7 @@ func (node *Node) CalculateResponse(request *downloader_pb.DownloaderRequest, in if request.BlockHash == nil { return response, fmt.Errorf("[SYNC] GetBlockHashes Request BlockHash is NIL") } - if request.Size == 0 || request.Size > syncing.BatchSize { + if request.Size == 0 || request.Size > syncing.SyncLoopBatchSize { return response, fmt.Errorf("[SYNC] GetBlockHashes Request contains invalid Size %v", request.Size) } size := uint64(request.Size) From fb77926d0fdc9b692a6a0973c4a7f4539e4ba6c8 Mon Sep 17 00:00:00 2001 From: Nye Liu Date: Thu, 31 Oct 2019 23:57:06 +0000 Subject: [PATCH 08/45] Comment out "Dumping block" message When running explorer, every time we restart it, this dumps megabytes of messages into zerolog, overwhelming logstash. --- api/service/explorer/storage.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/service/explorer/storage.go b/api/service/explorer/storage.go index 85b669719..f034fb966 100644 --- a/api/service/explorer/storage.go +++ b/api/service/explorer/storage.go @@ -89,7 +89,7 @@ func (storage *Storage) GetDB() *ethdb.LDBDatabase { // Dump extracts information from block and index them into lvdb for explorer. func (storage *Storage) Dump(block *types.Block, height uint64) { - utils.Logger().Info().Uint64("block height", height).Msg("Dumping block") + //utils.Logger().Debug().Uint64("block height", height).Msg("Dumping block") if block == nil { return } From 8a534ee88cb813f16c1fa9ec126ceeedfc97c3cc Mon Sep 17 00:00:00 2001 From: coolcottontail Date: Thu, 31 Oct 2019 22:20:03 -0700 Subject: [PATCH 09/45] switch the order of validatoraddress in staking messages --- staking/types/messages.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/staking/types/messages.go b/staking/types/messages.go index 3a25d6ce4..9fb487601 100644 --- a/staking/types/messages.go +++ b/staking/types/messages.go @@ -48,11 +48,11 @@ func (d Directive) String() string { // CreateValidator - type for creating a new validator type CreateValidator struct { + ValidatorAddress common.Address `json:"validator_address" yaml:"validator_address"` Description *Description `json:"description" yaml:"description"` CommissionRates `json:"commission" yaml:"commission"` MinSelfDelegation *big.Int `json:"min_self_delegation" yaml:"min_self_delegation"` MaxTotalDelegation *big.Int `json:"max_total_delegation" yaml:"max_total_delegation"` - ValidatorAddress common.Address `json:"validator_address" yaml:"validator_address"` SlotPubKeys []shard.BlsPublicKey `json:"slot_pub_keys" yaml:"slot_pub_keys"` Amount *big.Int `json:"amount" yaml:"amount"` } From 0bf5c0925dc9b957d9f896a2c30a7a9668318f75 Mon Sep 17 00:00:00 2001 From: coolcottontail Date: Fri, 1 Nov 2019 09:14:07 -0700 Subject: [PATCH 10/45] fixed goimports --- staking/types/messages.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/staking/types/messages.go b/staking/types/messages.go index 9fb487601..0a1678ac0 100644 --- a/staking/types/messages.go +++ b/staking/types/messages.go @@ -48,8 +48,8 @@ func (d Directive) String() string { // CreateValidator - type for creating a new validator type CreateValidator struct { - ValidatorAddress common.Address `json:"validator_address" yaml:"validator_address"` - Description *Description `json:"description" yaml:"description"` + ValidatorAddress common.Address `json:"validator_address" yaml:"validator_address"` + Description *Description `json:"description" yaml:"description"` CommissionRates `json:"commission" yaml:"commission"` MinSelfDelegation *big.Int `json:"min_self_delegation" yaml:"min_self_delegation"` MaxTotalDelegation *big.Int `json:"max_total_delegation" yaml:"max_total_delegation"` From 7b0ce75a8996af787c5b178b35f665a04d2b12af Mon Sep 17 00:00:00 2001 From: Leo Chen Date: Wed, 30 Oct 2019 03:25:07 +0000 Subject: [PATCH 11/45] [build] do not upload so files if build statically Signed-off-by: Leo Chen --- scripts/go_executable_build.sh | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/scripts/go_executable_build.sh b/scripts/go_executable_build.sh index 9d4528417..8d3d34f74 100755 --- a/scripts/go_executable_build.sh +++ b/scripts/go_executable_build.sh @@ -138,13 +138,15 @@ function upload [ -e $BINDIR/$bin ] && $AWSCLI s3 cp $BINDIR/$bin s3://${BUCKET}$FOLDER/$bin --acl public-read done - for lib in "${!LIB[@]}"; do - if [ -e ${LIB[$lib]} ]; then - $AWSCLI s3 cp ${LIB[$lib]} s3://${BUCKET}$FOLDER/$lib --acl public-read - else - echo "!! MISSING ${LIB[$lib]} !!" - fi - done + if [ "$STATIC" != "true" ]; then + for lib in "${!LIB[@]}"; do + if [ -e ${LIB[$lib]} ]; then + $AWSCLI s3 cp ${LIB[$lib]} s3://${BUCKET}$FOLDER/$lib --acl public-read + else + echo "!! MISSING ${LIB[$lib]} !!" + fi + done + fi [ -e $BINDIR/md5sum.txt ] && $AWSCLI s3 cp $BINDIR/md5sum.txt s3://${BUCKET}$FOLDER/md5sum.txt --acl public-read } @@ -177,13 +179,15 @@ function release fi done - for lib in "${!LIB[@]}"; do - if [ -e ${LIB[$lib]} ]; then - $AWSCLI s3 cp ${LIB[$lib]} s3://${PUBBUCKET}/$FOLDER/$lib --acl public-read - else - echo "!! MISSING ${LIB[$lib]} !!" - fi - done + if [ "$STATIC" != "true" ]; then + for lib in "${!LIB[@]}"; do + if [ -e ${LIB[$lib]} ]; then + $AWSCLI s3 cp ${LIB[$lib]} s3://${PUBBUCKET}/$FOLDER/$lib --acl public-read + else + echo "!! MISSING ${LIB[$lib]} !!" + fi + done + fi [ -e $BINDIR/md5sum.txt ] && $AWSCLI s3 cp $BINDIR/md5sum.txt s3://${PUBBUCKET}/$FOLDER/md5sum.txt --acl public-read } From 005321f9e9da6eb2d3d8b03b2476a9c96216457a Mon Sep 17 00:00:00 2001 From: Leo Chen Date: Fri, 1 Nov 2019 22:58:30 +0000 Subject: [PATCH 12/45] [static] exit if not running on Linux Signed-off-by: Leo Chen --- scripts/go_executable_build.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/go_executable_build.sh b/scripts/go_executable_build.sh index 8d3d34f74..2a0300847 100755 --- a/scripts/go_executable_build.sh +++ b/scripts/go_executable_build.sh @@ -92,6 +92,11 @@ EOF function build_only { + if [[ "$STATIC" == "true" && "$GOOS" == "darwin" ]]; then + echo "static build only supported on Linux platform" + exit 2 + fi + VERSION=$(git rev-list --count HEAD) COMMIT=$(git describe --always --long --dirty) BUILTAT=$(date +%FT%T%z) From 1ee2ab5866c1295ac55ff5cb2ba5f26ef5506594 Mon Sep 17 00:00:00 2001 From: chao Date: Fri, 1 Nov 2019 18:21:37 -0700 Subject: [PATCH 13/45] debugging --- cmd/staking/root.go | 8 +- core/blockchain.go | 142 +++++++++++++++++++--------------- core/rawdb/accessors_chain.go | 23 +++--- core/state_processor.go | 18 +++++ core/state_transition.go | 1 + core/types/transaction.go | 2 +- internal/utils/utils.go | 10 +++ node/node.go | 2 +- node/node_newblock.go | 8 +- node/worker/worker.go | 14 ++-- 10 files changed, 135 insertions(+), 93 deletions(-) diff --git a/cmd/staking/root.go b/cmd/staking/root.go index 8442d9715..a7e2d3242 100644 --- a/cmd/staking/root.go +++ b/cmd/staking/root.go @@ -42,8 +42,8 @@ const ( // Harmony protocol assume beacon chain shard is only place to send // staking, later need to consider logic when beacon chain shard rotates stakingShard = 0 - testAccount = "one1a0x3d6xpmr6f8wsyaxd9v36pytvp48zckswvv9" - testBLSPubKey = "b9486167ab9087ab818dc4ce026edb5bf216863364c32e42df2af03c5ced1ad181e7d12f0e6dd5307a73b62247608611" + testAccount = "one1pdv9lrdwl0rg5vglh4xtyrv3wjk3wsqket7zxy" + testBLSPubKey = "65f55eb3052f9e9f632b2923be594ba77c55543f5c58ee1454b9cfd658d25e06373b0f7d42a19c84768139ea294f6204" testAccountPassword = "" ) @@ -53,7 +53,7 @@ func (s *staker) run(cmd *cobra.Command, args []string) error { ks := keystore.NewKeyStore(keystoreDir, scryptN, scryptP) account := accounts.Account{Address: dAddr} ks.Unlock(account, testAccountPassword) - gasPrice := big.NewInt(0) + gasPrice := big.NewInt(1) stakePayloadMaker := func() (staking.Directive, interface{}) { p := &bls.PublicKey{} p.DeserializeHexStr(testBLSPubKey) @@ -85,7 +85,7 @@ func (s *staker) run(cmd *cobra.Command, args []string) error { // } } - stakingTx, err := staking.NewStakingTransaction(2, 100, gasPrice, stakePayloadMaker) + stakingTx, err := staking.NewStakingTransaction(0, 600000, gasPrice, stakePayloadMaker) if err != nil { return err } diff --git a/core/blockchain.go b/core/blockchain.go index 8445c2906..d9e500481 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -59,19 +59,19 @@ var ( ) const ( - bodyCacheLimit = 256 - blockCacheLimit = 256 - receiptsCacheLimit = 32 - maxFutureBlocks = 256 - maxTimeFutureBlocks = 30 - badBlockLimit = 10 - triesInMemory = 128 - shardCacheLimit = 2 - commitsCacheLimit = 10 - epochCacheLimit = 10 - randomnessCacheLimit = 10 - stakingCacheLimit = 256 - validatorMapCacheLimit = 2 + bodyCacheLimit = 256 + blockCacheLimit = 256 + receiptsCacheLimit = 32 + maxFutureBlocks = 256 + maxTimeFutureBlocks = 30 + badBlockLimit = 10 + triesInMemory = 128 + shardCacheLimit = 2 + commitsCacheLimit = 10 + epochCacheLimit = 10 + randomnessCacheLimit = 10 + stakingCacheLimit = 256 + validatorListCacheLimit = 2 // BlockChainVersion ensures that an incompatible database forces a resync from scratch. BlockChainVersion = 3 @@ -124,18 +124,18 @@ type BlockChain struct { currentBlock atomic.Value // Current head of the block chain currentFastBlock atomic.Value // Current head of the fast-sync chain (may be above the block chain!) - stateCache state.Database // State database to reuse between imports (contains state cache) - bodyCache *lru.Cache // Cache for the most recent block bodies - bodyRLPCache *lru.Cache // Cache for the most recent block bodies in RLP encoded format - receiptsCache *lru.Cache // Cache for the most recent receipts per block - blockCache *lru.Cache // Cache for the most recent entire blocks - futureBlocks *lru.Cache // future blocks are blocks added for later processing - shardStateCache *lru.Cache - lastCommitsCache *lru.Cache - epochCache *lru.Cache // Cache epoch number → first block number - randomnessCache *lru.Cache // Cache for vrf/vdf - stakingCache *lru.Cache // Cache for staking validator - validatorMapCache *lru.Cache // Cache of validator list + stateCache state.Database // State database to reuse between imports (contains state cache) + bodyCache *lru.Cache // Cache for the most recent block bodies + bodyRLPCache *lru.Cache // Cache for the most recent block bodies in RLP encoded format + receiptsCache *lru.Cache // Cache for the most recent receipts per block + blockCache *lru.Cache // Cache for the most recent entire blocks + futureBlocks *lru.Cache // future blocks are blocks added for later processing + shardStateCache *lru.Cache + lastCommitsCache *lru.Cache + epochCache *lru.Cache // Cache epoch number → first block number + randomnessCache *lru.Cache // Cache for vrf/vdf + stakingCache *lru.Cache // Cache for staking validator + validatorListCache *lru.Cache // Cache of validator list quit chan struct{} // blockchain quit channel running int32 // running must be called atomically @@ -173,30 +173,30 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par epochCache, _ := lru.New(epochCacheLimit) randomnessCache, _ := lru.New(randomnessCacheLimit) stakingCache, _ := lru.New(stakingCacheLimit) - validatorMapCache, _ := lru.New(validatorMapCacheLimit) + validatorListCache, _ := lru.New(validatorListCacheLimit) bc := &BlockChain{ - chainConfig: chainConfig, - cacheConfig: cacheConfig, - db: db, - triegc: prque.New(nil), - stateCache: state.NewDatabase(db), - quit: make(chan struct{}), - shouldPreserve: shouldPreserve, - bodyCache: bodyCache, - bodyRLPCache: bodyRLPCache, - receiptsCache: receiptsCache, - blockCache: blockCache, - futureBlocks: futureBlocks, - shardStateCache: shardCache, - lastCommitsCache: commitsCache, - epochCache: epochCache, - randomnessCache: randomnessCache, - stakingCache: stakingCache, - validatorMapCache: validatorMapCache, - engine: engine, - vmConfig: vmConfig, - badBlocks: badBlocks, + chainConfig: chainConfig, + cacheConfig: cacheConfig, + db: db, + triegc: prque.New(nil), + stateCache: state.NewDatabase(db), + quit: make(chan struct{}), + shouldPreserve: shouldPreserve, + bodyCache: bodyCache, + bodyRLPCache: bodyRLPCache, + receiptsCache: receiptsCache, + blockCache: blockCache, + futureBlocks: futureBlocks, + shardStateCache: shardCache, + lastCommitsCache: commitsCache, + epochCache: epochCache, + randomnessCache: randomnessCache, + stakingCache: stakingCache, + validatorListCache: validatorListCache, + engine: engine, + vmConfig: vmConfig, + badBlocks: badBlocks, } bc.SetValidator(NewBlockValidator(chainConfig, bc, engine)) bc.SetProcessor(NewStateProcessor(chainConfig, bc, engine)) @@ -229,12 +229,14 @@ func (bc *BlockChain) ValidateNewBlock(block *types.Block) error { // Process block using the parent state as reference point. receipts, cxReceipts, _, usedGas, err := bc.processor.Process(block, state, bc.vmConfig) if err != nil { + utils.Logger().Info().Msgf("hehe1, usedGas: %v, err: %v", usedGas, err) bc.reportBlock(block, receipts, err) return err } err = bc.Validator().ValidateState(block, bc.CurrentBlock(), state, receipts, cxReceipts, usedGas) if err != nil { + utils.Logger().Info().Msgf("hehe2, err: %v", err) bc.reportBlock(block, receipts, err) return err } @@ -2292,22 +2294,22 @@ func (bc *BlockChain) WriteStakingValidator(v *staking.ValidatorWrapper) error { return nil } -// ReadValidatorMap reads the addresses of current all validators -func (bc *BlockChain) ReadValidatorMap() (map[common.Address]struct{}, error) { - if cached, ok := bc.validatorMapCache.Get("validatorMap"); ok { +// ReadValidatorList reads the addresses of current all validators +func (bc *BlockChain) ReadValidatorList() ([]common.Address, error) { + if cached, ok := bc.validatorListCache.Get("validatorList"); ok { by := cached.([]byte) - m := make(map[common.Address]struct{}) + m := []common.Address{} if err := rlp.DecodeBytes(by, &m); err != nil { return nil, err } return m, nil } - return rawdb.ReadValidatorMap(bc.db) + return rawdb.ReadValidatorList(bc.db) } -// WriteValidatorMap writes the list of validator addresses to database -func (bc *BlockChain) WriteValidatorMap(addrs map[common.Address]struct{}) error { - err := rawdb.WriteValidatorMap(bc.db, addrs) +// WriteValidatorList writes the list of validator addresses to database +func (bc *BlockChain) WriteValidatorList(addrs []common.Address) error { + err := rawdb.WriteValidatorList(bc.db, addrs) if err != nil { return err } @@ -2315,24 +2317,36 @@ func (bc *BlockChain) WriteValidatorMap(addrs map[common.Address]struct{}) error if err != nil { return err } - bc.validatorMapCache.Add("validatorMap", by) + bc.validatorListCache.Add("validatorList", by) return nil } -// UpdateValidatorMap updates the validator map according to staking transaction -func (bc *BlockChain) UpdateValidatorMap(tx *staking.StakingTransaction) error { +// UpdateValidatorList updates the validator map according to staking transaction +func (bc *BlockChain) UpdateValidatorList(tx *staking.StakingTransaction) error { + // TODO: simply the logic here in staking/types/transaction.go + payload, err := tx.RLPEncodeStakeMsg() + if err != nil { + return err + } + decodePayload, err := staking.RLPDecodeStakeMsg(payload, tx.StakingType()) + if err != nil { + return err + } switch tx.StakingType() { case staking.DirectiveCreateValidator: - createValidator := tx.StakingMessage().(staking.CreateValidator) - m, err := bc.ReadValidatorMap() + createValidator := decodePayload.(*staking.CreateValidator) + list, err := bc.ReadValidatorList() if err != nil { return err } - if m == nil { - m = make(map[common.Address]struct{}) + if list == nil { + list = []common.Address{} + } + beforeLen := len(list) + list = utils.AppendIfMissing(list, createValidator.ValidatorAddress) + if len(list) > beforeLen { + err = bc.WriteValidatorList(list) } - m[createValidator.ValidatorAddress] = struct{}{} - err = bc.WriteValidatorMap(m) return err // following cases are placeholder for now diff --git a/core/rawdb/accessors_chain.go b/core/rawdb/accessors_chain.go index fb0ed780f..4c5a714c4 100644 --- a/core/rawdb/accessors_chain.go +++ b/core/rawdb/accessors_chain.go @@ -641,29 +641,28 @@ func WriteStakingValidator(db DatabaseWriter, v *staking.ValidatorWrapper) error return err } -// ReadValidatorMap retrieves staking validator by its address -func ReadValidatorMap(db DatabaseReader) (map[common.Address]struct{}, error) { - data, err := db.Get([]byte("validatorMap")) +// ReadValidatorList retrieves staking validator by its address +func ReadValidatorList(db DatabaseReader) ([]common.Address, error) { + data, err := db.Get([]byte("validatorList")) if len(data) == 0 || err != nil { - utils.Logger().Info().Err(err).Msg("ReadValidatorMap") - return nil, err + return []common.Address{}, nil } - addrs := make(map[common.Address]struct{}) + addrs := []common.Address{} if err := rlp.DecodeBytes(data, &addrs); err != nil { - utils.Logger().Error().Err(err).Msg("Unable to Decode validator Map from database") + utils.Logger().Error().Err(err).Msg("Unable to Decode validator List from database") return nil, err } return addrs, nil } -// WriteValidatorMap stores staking validator's information by its address -func WriteValidatorMap(db DatabaseWriter, addrs map[common.Address]struct{}) error { +// WriteValidatorList stores staking validator's information by its address +func WriteValidatorList(db DatabaseWriter, addrs []common.Address) error { bytes, err := rlp.EncodeToBytes(addrs) if err != nil { - utils.Logger().Error().Msg("[WriteValidatorMap] Failed to encode") + utils.Logger().Error().Msg("[WriteValidatorList] Failed to encode") } - if err := db.Put([]byte("validatorMap"), bytes); err != nil { - utils.Logger().Error().Msg("[WriteValidatorMap] Failed to store to database") + if err := db.Put([]byte("validatorList"), bytes); err != nil { + utils.Logger().Error().Msg("[WriteValidatorList] Failed to store to database") } return err } diff --git a/core/state_processor.go b/core/state_processor.go index 3fd9b1d69..71acc6259 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -86,6 +86,22 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.DB, cfg vm.C allLogs = append(allLogs, receipt.Logs...) } + // Iterate over staking transactions + L := len(block.Transactions()) + utils.Logger().Info().Msgf("oops: len stakingTx : %v", len(block.StakingTransactions())) + for i, tx := range block.StakingTransactions() { + statedb.Prepare(tx.Hash(), block.Hash(), i+L) + receipt, _, err := + ApplyStakingTransaction(p.config, p.bc, &coinbase, gp, statedb, header, tx, usedGas, cfg) + utils.Logger().Info().Msgf("hehe, i: %v, usedGas: %v, err: %v", i, *usedGas, err) + + if err != nil { + return nil, nil, nil, 0, err + } + receipts = append(receipts, receipt) + allLogs = append(allLogs, receipt.Logs...) + } + // incomingReceipts should always be processed after transactions (to be consistent with the block proposal) for _, cx := range block.IncomingReceipts() { err := ApplyIncomingReceipt(p.config, statedb, header, cx) @@ -201,8 +217,10 @@ func ApplyStakingTransaction( // Create a new environment which holds all relevant information // about the transaction and calling mechanisms. vmenv := vm.NewEVM(context, statedb, config, cfg) + // Apply the transaction to the current state (included in the env) gas, err = ApplyStakingMessage(vmenv, msg, gp) + utils.Logger().Info().Msgf("ApplyStakingMessage: usedGas: %v, err: %v", gas, err) // even there is error, we charge it if err != nil { diff --git a/core/state_transition.go b/core/state_transition.go index 81aa55a65..934c3f222 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -271,6 +271,7 @@ func (st *StateTransition) StakingTransitionDb() (usedGas uint64, err error) { return 0, err } msg := st.msg + sender := vm.AccountRef(msg.From()) homestead := st.evm.ChainConfig().IsS3(st.evm.EpochNumber) // s3 includes homestead diff --git a/core/types/transaction.go b/core/types/transaction.go index 22df976a0..36cbf4a0a 100644 --- a/core/types/transaction.go +++ b/core/types/transaction.go @@ -640,7 +640,7 @@ func (m Message) Type() TransactionType { } // SetType set the type of message -func (m Message) SetType(typ TransactionType) { +func (m *Message) SetType(typ TransactionType) { m.txType = typ } diff --git a/internal/utils/utils.go b/internal/utils/utils.go index 8b3e0653b..5222e62e5 100644 --- a/internal/utils/utils.go +++ b/internal/utils/utils.go @@ -245,3 +245,13 @@ func GetPendingCXKey(shardID uint32, blockNum uint64) string { key := strconv.FormatUint(uint64(shardID), 10) + "-" + strconv.FormatUint(blockNum, 10) return key } + +// AppendIfMissing returns a list of unique addresses +func AppendIfMissing(slice []common.Address, addr common.Address) []common.Address { + for _, ele := range slice { + if ele == addr { + return slice + } + } + return append(slice, addr) +} diff --git a/node/node.go b/node/node.go index 782089b8e..e068aa700 100644 --- a/node/node.go +++ b/node/node.go @@ -406,7 +406,7 @@ func (node *Node) getTransactionsForNewBlock(coinbase common.Address) (types.Tra Int("remainPending", len(node.pendingStakingTransactions)). Int("selected", len(unselectedStaking)). Int("invalidDiscarded", len(invalidStaking)). - Msg("Selecting Transactions") + Msg("Selecting Staking Transactions") return selected, selectedStaking } diff --git a/node/node_newblock.go b/node/node_newblock.go index 6288970bd..458449131 100644 --- a/node/node_newblock.go +++ b/node/node_newblock.go @@ -82,6 +82,9 @@ func (node *Node) proposeNewBlock() (*types.Block, error) { // Prepare transactions including staking transactions selectedTxs, selectedStakingTxs := node.getTransactionsForNewBlock(coinbase) + if len(selectedStakingTxs) > 0 { + utils.Logger().Debug().Int("selectedStakingtxs", len(selectedStakingTxs)).Msg("hehe, staking txs proposed") + } if err := node.Worker.CommitTransactions(selectedTxs, selectedStakingTxs, coinbase); err != nil { ctxerror.Log15(utils.GetLogger().Error, @@ -90,10 +93,6 @@ func (node *Node) proposeNewBlock() (*types.Block, error) { return nil, err } - if len(selectedStakingTxs) > 0 { - utils.Logger().Debug().Int("selectedStakingtxs", len(selectedStakingTxs)).Msg("hehe, staking txs proposed") - } - // Prepare cross shard transaction receipts receiptsList := node.proposeReceiptsProof() if len(receiptsList) != 0 { @@ -124,7 +123,6 @@ func (node *Node) proposeNewBlock() (*types.Block, error) { WithCause(err)) return nil, err } - return node.Worker.FinalizeNewBlock(sig, mask, node.Consensus.GetViewID(), coinbase, crossLinks, shardState) } diff --git a/node/worker/worker.go b/node/worker/worker.go index bad91b911..b797665e4 100644 --- a/node/worker/worker.go +++ b/node/worker/worker.go @@ -163,14 +163,13 @@ func (w *Worker) SelectStakingTransactionsForNewBlock( // only beaconchain process staking transaction if w.chain.ShardID() != values.BeaconChainShardID { + utils.Logger().Warn().Msgf("Invalid shardID: %v", w.chain.ShardID()) return nil, nil, nil } - // TODO: gas pool should be initialized once for both normal and staking transactions - // staking transaction share the same gasPool with normal transactions - //if w.current.gasPool == nil { - // w.current.gasPool = new(core.GasPool).AddGas(w.current.header.GasLimit()) - //} + if w.current.gasPool == nil { + w.current.gasPool = new(core.GasPool).AddGas(w.current.header.GasLimit()) + } selected := staking.StakingTransactions{} // TODO: chao add total gas fee checking when needed @@ -210,10 +209,11 @@ func (w *Worker) commitStakingTransaction(tx *staking.StakingTransaction, coinba return nil, fmt.Errorf("nil staking receipt") } - err = w.chain.UpdateValidatorMap(tx) + err = w.chain.UpdateValidatorList(tx) // keep offchain database consistency with onchain we need revert // but it should not happend unless local database corrupted if err != nil { + utils.Logger().Debug().Msgf("oops, UpdateValidatorList failed, err: %+v", err) w.current.state.RevertToSnapshot(snap) return nil, err } @@ -425,6 +425,8 @@ func (w *Worker) FinalizeNewBlock(sig []byte, signers []byte, viewID uint64, coi if err != nil { return nil, ctxerror.New("cannot finalize block").WithCause(err) } + + utils.Logger().Debug().Msgf("hehehe, len stxs: %v", len(block.StakingTransactions())) return block, nil } From d49dafa85ec5c1219df6821d301e03bc091afc76 Mon Sep 17 00:00:00 2001 From: chao Date: Fri, 1 Nov 2019 20:12:38 -0700 Subject: [PATCH 14/45] debug block encode/decode --- consensus/consensus_v2.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 58c314b3b..6aa8e5a09 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -95,12 +95,17 @@ func (consensus *Consensus) announce(block *types.Block) { utils.Logger().Debug().Msg("[Announce] Failed encoding block") return } + utils.Logger().Debug().Msgf("haha1, before:%v ", len(block.StakingTransactions())) encodedBlockHeader, err := rlp.EncodeToBytes(block.Header()) if err != nil { utils.Logger().Debug().Msg("[Announce] Failed encoding block header") return } + var blockObj1 types.Block + err = rlp.DecodeBytes(encodedBlock, &blockObj1) + utils.Logger().Debug().Msgf("haha2, after stks:= %v", len(blockObj1.StakingTransactions())) + consensus.block = encodedBlock consensus.blockHeader = encodedBlockHeader msgToSend := consensus.constructAnnounceMessage() @@ -407,10 +412,12 @@ func (consensus *Consensus) onPrepare(msg *msg_pb.Message) { msg := &msg_pb.Message{} _ = protobuf.Unmarshal(msgPayload, msg) FBFTMsg, err := ParseFBFTMessage(msg) + if err != nil { utils.Logger().Warn().Err(err).Msg("[OnPrepare] Unable to parse pbft message") return } + consensus.FBFTLog.AddMessage(FBFTMsg) // Leader add commit phase signature blockNumHash := make([]byte, 8) From 99ab46c6efdff9875c84b8e54c3c23f2f6b551e6 Mon Sep 17 00:00:00 2001 From: chao Date: Sat, 2 Nov 2019 14:17:36 -0700 Subject: [PATCH 15/45] fix block encoding/decoding issue for blockV2 and headerV3 --- block/v3/header.go | 415 +++++++++++++++++++++++++++++++++++++- consensus/consensus_v2.go | 5 - core/blockchain.go | 2 - core/state_processor.go | 2 - core/types/block.go | 3 + core/types/bodyv0.go | 8 + core/types/bodyv1.go | 8 + internal/params/config.go | 4 +- node/node_newblock.go | 3 - node/worker/worker.go | 1 - 10 files changed, 431 insertions(+), 20 deletions(-) diff --git a/block/v3/header.go b/block/v3/header.go index fe054a359..db54626ff 100644 --- a/block/v3/header.go +++ b/block/v3/header.go @@ -1,16 +1,421 @@ package v3 import ( - v2 "github.com/harmony-one/harmony/block/v2" + "io" + "math/big" + "unsafe" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + ethtypes "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" + "github.com/rs/zerolog" + + blockif "github.com/harmony-one/harmony/block/interface" + "github.com/harmony-one/harmony/crypto/hash" + "github.com/harmony-one/harmony/shard" ) -// Header v3 has the same structure as v2 header -// It is used to identify the body v3 which including staking txs +// Header is the V3 block header. +// V3 block header is exactly the same +// we copy the code instead of embedded v2 header into v3 +// when we do type checking in NewBodyForMatchingHeader +// the embedded structure will return v2 header type instead of v3 type type Header struct { - v2.Header + fields headerFields +} + +// EncodeRLP encodes the header fields into RLP format. +func (h *Header) EncodeRLP(w io.Writer) error { + return rlp.Encode(w, &h.fields) +} + +// DecodeRLP decodes the given RLP decode stream into the header fields. +func (h *Header) DecodeRLP(s *rlp.Stream) error { + return s.Decode(&h.fields) } // NewHeader creates a new header object. func NewHeader() *Header { - return &Header{*v2.NewHeader()} + return &Header{headerFields{ + Number: new(big.Int), + Time: new(big.Int), + ViewID: new(big.Int), + Epoch: new(big.Int), + }} +} + +type headerFields struct { + ParentHash common.Hash `json:"parentHash" gencodec:"required"` + Coinbase common.Address `json:"miner" gencodec:"required"` + Root common.Hash `json:"stateRoot" gencodec:"required"` + TxHash common.Hash `json:"transactionsRoot" gencodec:"required"` + ReceiptHash common.Hash `json:"receiptsRoot" gencodec:"required"` + OutgoingReceiptHash common.Hash `json:"outgoingReceiptsRoot" gencodec:"required"` + IncomingReceiptHash common.Hash `json:"incomingReceiptsRoot" gencodec:"required"` + Bloom ethtypes.Bloom `json:"logsBloom" gencodec:"required"` + Number *big.Int `json:"number" gencodec:"required"` + GasLimit uint64 `json:"gasLimit" gencodec:"required"` + GasUsed uint64 `json:"gasUsed" gencodec:"required"` + Time *big.Int `json:"timestamp" gencodec:"required"` + Extra []byte `json:"extraData" gencodec:"required"` + MixDigest common.Hash `json:"mixHash" gencodec:"required"` + // Additional Fields + ViewID *big.Int `json:"viewID" gencodec:"required"` + Epoch *big.Int `json:"epoch" gencodec:"required"` + ShardID uint32 `json:"shardID" gencodec:"required"` + LastCommitSignature [96]byte `json:"lastCommitSignature" gencodec:"required"` + LastCommitBitmap []byte `json:"lastCommitBitmap" gencodec:"required"` // Contains which validator signed + ShardStateHash common.Hash `json:"shardStateRoot"` + Vrf []byte `json:"vrf"` + Vdf []byte `json:"vdf"` + ShardState []byte `json:"shardState"` + CrossLinks []byte `json:"crossLink"` +} + +// ParentHash is the header hash of the parent block. For the genesis block +// which has no parent by definition, this field is zeroed out. +func (h *Header) ParentHash() common.Hash { + return h.fields.ParentHash +} + +// SetParentHash sets the parent hash field. +func (h *Header) SetParentHash(newParentHash common.Hash) { + h.fields.ParentHash = newParentHash +} + +// Coinbase is the address of the node that proposed this block and all +// transactions in it. +func (h *Header) Coinbase() common.Address { + return h.fields.Coinbase +} + +// SetCoinbase sets the coinbase address field. +func (h *Header) SetCoinbase(newCoinbase common.Address) { + h.fields.Coinbase = newCoinbase +} + +// Root is the state (account) trie root hash. +func (h *Header) Root() common.Hash { + return h.fields.Root +} + +// SetRoot sets the state trie root hash field. +func (h *Header) SetRoot(newRoot common.Hash) { + h.fields.Root = newRoot +} + +// TxHash is the transaction trie root hash. +func (h *Header) TxHash() common.Hash { + return h.fields.TxHash +} + +// SetTxHash sets the transaction trie root hash field. +func (h *Header) SetTxHash(newTxHash common.Hash) { + h.fields.TxHash = newTxHash +} + +// ReceiptHash is the same-shard transaction receipt trie hash. +func (h *Header) ReceiptHash() common.Hash { + return h.fields.ReceiptHash +} + +// SetReceiptHash sets the same-shard transaction receipt trie hash. +func (h *Header) SetReceiptHash(newReceiptHash common.Hash) { + h.fields.ReceiptHash = newReceiptHash +} + +// OutgoingReceiptHash is the egress transaction receipt trie hash. +func (h *Header) OutgoingReceiptHash() common.Hash { + return h.fields.OutgoingReceiptHash +} + +// SetOutgoingReceiptHash sets the egress transaction receipt trie hash. +func (h *Header) SetOutgoingReceiptHash(newOutgoingReceiptHash common.Hash) { + h.fields.OutgoingReceiptHash = newOutgoingReceiptHash +} + +// IncomingReceiptHash is the ingress transaction receipt trie hash. +func (h *Header) IncomingReceiptHash() common.Hash { + return h.fields.IncomingReceiptHash +} + +// SetIncomingReceiptHash sets the ingress transaction receipt trie hash. +func (h *Header) SetIncomingReceiptHash(newIncomingReceiptHash common.Hash) { + h.fields.IncomingReceiptHash = newIncomingReceiptHash +} + +// Bloom is the Bloom filter that indexes accounts and topics logged by smart +// contract transactions (executions) in this block. +func (h *Header) Bloom() ethtypes.Bloom { + return h.fields.Bloom +} + +// SetBloom sets the smart contract log Bloom filter for this block. +func (h *Header) SetBloom(newBloom ethtypes.Bloom) { + h.fields.Bloom = newBloom +} + +// Number is the block number. +// +// The returned instance is a copy; the caller may do anything with it. +func (h *Header) Number() *big.Int { + return new(big.Int).Set(h.fields.Number) +} + +// SetNumber sets the block number. +// +// It stores a copy; the caller may freely modify the original. +func (h *Header) SetNumber(newNumber *big.Int) { + h.fields.Number = new(big.Int).Set(newNumber) +} + +// GasLimit is the gas limit for transactions in this block. +func (h *Header) GasLimit() uint64 { + return h.fields.GasLimit +} + +// SetGasLimit sets the gas limit for transactions in this block. +func (h *Header) SetGasLimit(newGasLimit uint64) { + h.fields.GasLimit = newGasLimit +} + +// GasUsed is the amount of gas used by transactions in this block. +func (h *Header) GasUsed() uint64 { + return h.fields.GasUsed +} + +// SetGasUsed sets the amount of gas used by transactions in this block. +func (h *Header) SetGasUsed(newGasUsed uint64) { + h.fields.GasUsed = newGasUsed +} + +// Time is the UNIX timestamp of this block. +// +// The returned instance is a copy; the caller may do anything with it. +func (h *Header) Time() *big.Int { + return new(big.Int).Set(h.fields.Time) +} + +// SetTime sets the UNIX timestamp of this block. +// +// It stores a copy; the caller may freely modify the original. +func (h *Header) SetTime(newTime *big.Int) { + h.fields.Time = new(big.Int).Set(newTime) +} + +// Extra is the extra data field of this block. +// +// The returned slice is a copy; the caller may do anything with it. +func (h *Header) Extra() []byte { + return append(h.fields.Extra[:0:0], h.fields.Extra...) +} + +// SetExtra sets the extra data field of this block. +// +// It stores a copy; the caller may freely modify the original. +func (h *Header) SetExtra(newExtra []byte) { + h.fields.Extra = append(newExtra[:0:0], newExtra...) +} + +// MixDigest is the mixhash. +// +// This field is a remnant from Ethereum, and Harmony does not use it and always +// zeroes it out. +func (h *Header) MixDigest() common.Hash { + return h.fields.MixDigest +} + +// SetMixDigest sets the mixhash of this block. +func (h *Header) SetMixDigest(newMixDigest common.Hash) { + h.fields.MixDigest = newMixDigest +} + +// ViewID is the ID of the view in which this block was originally proposed. +// +// It normally increases by one for each subsequent block, or by more than one +// if one or more PBFT/FBFT view changes have occurred. +// +// The returned instance is a copy; the caller may do anything with it. +func (h *Header) ViewID() *big.Int { + return new(big.Int).Set(h.fields.ViewID) +} + +// SetViewID sets the view ID in which the block was originally proposed. +// +// It stores a copy; the caller may freely modify the original. +func (h *Header) SetViewID(newViewID *big.Int) { + h.fields.ViewID = new(big.Int).Set(newViewID) +} + +// Epoch is the epoch number of this block. +// +// The returned instance is a copy; the caller may do anything with it. +func (h *Header) Epoch() *big.Int { + return new(big.Int).Set(h.fields.Epoch) +} + +// SetEpoch sets the epoch number of this block. +// +// It stores a copy; the caller may freely modify the original. +func (h *Header) SetEpoch(newEpoch *big.Int) { + h.fields.Epoch = new(big.Int).Set(newEpoch) +} + +// ShardID is the shard ID to which this block belongs. +func (h *Header) ShardID() uint32 { + return h.fields.ShardID +} + +// SetShardID sets the shard ID to which this block belongs. +func (h *Header) SetShardID(newShardID uint32) { + h.fields.ShardID = newShardID +} + +// LastCommitSignature is the FBFT commit group signature for the last block. +func (h *Header) LastCommitSignature() [96]byte { + return h.fields.LastCommitSignature +} + +// SetLastCommitSignature sets the FBFT commit group signature for the last +// block. +func (h *Header) SetLastCommitSignature(newLastCommitSignature [96]byte) { + h.fields.LastCommitSignature = newLastCommitSignature +} + +// LastCommitBitmap is the signatory bitmap of the previous block. Bit +// positions index into committee member array. +// +// The returned slice is a copy; the caller may do anything with it. +func (h *Header) LastCommitBitmap() []byte { + return append(h.fields.LastCommitBitmap[:0:0], h.fields.LastCommitBitmap...) +} + +// SetLastCommitBitmap sets the signatory bitmap of the previous block. +// +// It stores a copy; the caller may freely modify the original. +func (h *Header) SetLastCommitBitmap(newLastCommitBitmap []byte) { + h.fields.LastCommitBitmap = append(newLastCommitBitmap[:0:0], newLastCommitBitmap...) +} + +// ShardStateHash is the shard state hash. +func (h *Header) ShardStateHash() common.Hash { + return h.fields.ShardStateHash +} + +// SetShardStateHash sets the shard state hash. +func (h *Header) SetShardStateHash(newShardStateHash common.Hash) { + h.fields.ShardStateHash = newShardStateHash +} + +// Vrf is the output of the VRF for the epoch. +// +// The returned slice is a copy; the caller may do anything with it. +func (h *Header) Vrf() []byte { + return append(h.fields.Vrf[:0:0], h.fields.Vrf...) +} + +// SetVrf sets the output of the VRF for the epoch. +// +// It stores a copy; the caller may freely modify the original. +func (h *Header) SetVrf(newVrf []byte) { + h.fields.Vrf = append(newVrf[:0:0], newVrf...) +} + +// Vdf is the output of the VDF for the epoch. +// +// The returned slice is a copy; the caller may do anything with it. +func (h *Header) Vdf() []byte { + return append(h.fields.Vdf[:0:0], h.fields.Vdf...) +} + +// SetVdf sets the output of the VDF for the epoch. +// +// It stores a copy; the caller may freely modify the original. +func (h *Header) SetVdf(newVdf []byte) { + h.fields.Vdf = append(newVdf[:0:0], newVdf...) +} + +// ShardState is the RLP-encoded form of shard state (list of committees) for +// the next epoch. +// +// The returned slice is a copy; the caller may do anything with it. +func (h *Header) ShardState() []byte { + return append(h.fields.ShardState[:0:0], h.fields.ShardState...) +} + +// SetShardState sets the RLP-encoded form of shard state +// +// It stores a copy; the caller may freely modify the original. +func (h *Header) SetShardState(newShardState []byte) { + h.fields.ShardState = append(newShardState[:0:0], newShardState...) +} + +// CrossLinks is the RLP-encoded form of non-beacon block headers chosen to be +// canonical by the beacon committee. This field is present only on beacon +// chain block headers. +// +// The returned slice is a copy; the caller may do anything with it. +func (h *Header) CrossLinks() []byte { + return append(h.fields.CrossLinks[:0:0], h.fields.CrossLinks...) +} + +// SetCrossLinks sets the RLP-encoded form of non-beacon block headers chosen to +// be canonical by the beacon committee. +// +// It stores a copy; the caller may freely modify the original. +func (h *Header) SetCrossLinks(newCrossLinks []byte) { + h.fields.CrossLinks = append(newCrossLinks[:0:0], newCrossLinks...) +} + +// field type overrides for gencodec +type headerMarshaling struct { + Difficulty *hexutil.Big + Number *hexutil.Big + GasLimit hexutil.Uint64 + GasUsed hexutil.Uint64 + Time *hexutil.Big + Extra hexutil.Bytes + Hash common.Hash `json:"hash"` // adds call to Hash() in MarshalJSON +} + +// Hash returns the block hash of the header, which is simply the keccak256 hash of its +// RLP encoding. +func (h *Header) Hash() common.Hash { + return hash.FromRLP(h) +} + +// Size returns the approximate memory used by all internal contents. It is used +// to approximate and limit the memory consumption of various caches. +func (h *Header) Size() common.StorageSize { + // TODO: update with new fields + return common.StorageSize(unsafe.Sizeof(*h)) + common.StorageSize(len(h.Extra())+(h.Number().BitLen()+h.Time().BitLen())/8) +} + +// Logger returns a sub-logger with block contexts added. +func (h *Header) Logger(logger *zerolog.Logger) *zerolog.Logger { + nlogger := logger. + With(). + Str("blockHash", h.Hash().Hex()). + Uint32("blockShard", h.ShardID()). + Uint64("blockEpoch", h.Epoch().Uint64()). + Uint64("blockNumber", h.Number().Uint64()). + Logger() + return &nlogger +} + +// GetShardState returns the deserialized shard state object. +func (h *Header) GetShardState() (shard.State, error) { + shardState := shard.State{} + err := rlp.DecodeBytes(h.ShardState(), &shardState) + if err != nil { + return nil, err + } + return shardState, nil +} + +// Copy returns a copy of the given header. +func (h *Header) Copy() blockif.Header { + cpy := *h + return &cpy } diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 6aa8e5a09..598226c56 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -95,17 +95,12 @@ func (consensus *Consensus) announce(block *types.Block) { utils.Logger().Debug().Msg("[Announce] Failed encoding block") return } - utils.Logger().Debug().Msgf("haha1, before:%v ", len(block.StakingTransactions())) encodedBlockHeader, err := rlp.EncodeToBytes(block.Header()) if err != nil { utils.Logger().Debug().Msg("[Announce] Failed encoding block header") return } - var blockObj1 types.Block - err = rlp.DecodeBytes(encodedBlock, &blockObj1) - utils.Logger().Debug().Msgf("haha2, after stks:= %v", len(blockObj1.StakingTransactions())) - consensus.block = encodedBlock consensus.blockHeader = encodedBlockHeader msgToSend := consensus.constructAnnounceMessage() diff --git a/core/blockchain.go b/core/blockchain.go index d9e500481..bfd60a3b3 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -229,14 +229,12 @@ func (bc *BlockChain) ValidateNewBlock(block *types.Block) error { // Process block using the parent state as reference point. receipts, cxReceipts, _, usedGas, err := bc.processor.Process(block, state, bc.vmConfig) if err != nil { - utils.Logger().Info().Msgf("hehe1, usedGas: %v, err: %v", usedGas, err) bc.reportBlock(block, receipts, err) return err } err = bc.Validator().ValidateState(block, bc.CurrentBlock(), state, receipts, cxReceipts, usedGas) if err != nil { - utils.Logger().Info().Msgf("hehe2, err: %v", err) bc.reportBlock(block, receipts, err) return err } diff --git a/core/state_processor.go b/core/state_processor.go index 71acc6259..f72c2e809 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -88,12 +88,10 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.DB, cfg vm.C // Iterate over staking transactions L := len(block.Transactions()) - utils.Logger().Info().Msgf("oops: len stakingTx : %v", len(block.StakingTransactions())) for i, tx := range block.StakingTransactions() { statedb.Prepare(tx.Hash(), block.Hash(), i+L) receipt, _, err := ApplyStakingTransaction(p.config, p.bc, &coinbase, gp, statedb, header, tx, usedGas, cfg) - utils.Logger().Info().Msgf("hehe, i: %v, usedGas: %v, err: %v", i, *usedGas, err) if err != nil { return nil, nil, nil, 0, err diff --git a/core/types/block.go b/core/types/block.go index 2e67ee0bc..f450b3824 100644 --- a/core/types/block.go +++ b/core/types/block.go @@ -83,6 +83,9 @@ type BodyInterface interface { // Transactions returns a deep copy the list of transactions in this block. Transactions() []*Transaction + // StakingTransactions returns a deep copy of staking transactions + StakingTransactions() []*staking.StakingTransaction + // TransactionAt returns the transaction at the given index in this block. // It returns nil if index is out of bounds. TransactionAt(index int) *Transaction diff --git a/core/types/bodyv0.go b/core/types/bodyv0.go index a6c4a1da0..78042af19 100644 --- a/core/types/bodyv0.go +++ b/core/types/bodyv0.go @@ -7,6 +7,7 @@ import ( "github.com/harmony-one/harmony/block" "github.com/harmony-one/harmony/internal/utils" + staking "github.com/harmony-one/harmony/staking/types" ) // BodyV0 is the V0 block body @@ -79,6 +80,13 @@ func (b *BodyV0) IncomingReceipts() (incomingReceipts CXReceiptsProofs) { return nil } +// StakingTransactions returns the list of staking transactions. +// The returned list is a deep copy; the caller may do anything with it without +// affecting the original. +func (b *BodyV0) StakingTransactions() (txs []*staking.StakingTransaction) { + return nil +} + // SetIncomingReceipts sets the list of incoming cross-shard transaction // receipts of this block with a dep copy of the given list. func (b *BodyV0) SetIncomingReceipts(newIncomingReceipts CXReceiptsProofs) { diff --git a/core/types/bodyv1.go b/core/types/bodyv1.go index e14520c88..fcc08b73c 100644 --- a/core/types/bodyv1.go +++ b/core/types/bodyv1.go @@ -6,6 +6,7 @@ import ( "github.com/ethereum/go-ethereum/rlp" "github.com/harmony-one/harmony/block" + staking "github.com/harmony-one/harmony/staking/types" ) // BodyV1 is the V1 block body @@ -30,6 +31,13 @@ func (b *BodyV1) Transactions() (txs []*Transaction) { return txs } +// StakingTransactions returns the list of staking transactions. +// The returned list is a deep copy; the caller may do anything with it without +// affecting the original. +func (b *BodyV1) StakingTransactions() (txs []*staking.StakingTransaction) { + return nil +} + // TransactionAt returns the transaction at the given index in this block. // It returns nil if index is out of bounds. func (b *BodyV1) TransactionAt(index int) *Transaction { diff --git a/internal/params/config.go b/internal/params/config.go index 9cc33d551..6d5f19133 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -35,8 +35,8 @@ var ( TestnetChainConfig = &ChainConfig{ ChainID: TestnetChainID, CrossTxEpoch: big.NewInt(0), - CrossLinkEpoch: EpochTBD, - StakingEpoch: EpochTBD, + CrossLinkEpoch: big.NewInt(0), + StakingEpoch: big.NewInt(0), EIP155Epoch: big.NewInt(0), S3Epoch: big.NewInt(0), } diff --git a/node/node_newblock.go b/node/node_newblock.go index 458449131..faeff3426 100644 --- a/node/node_newblock.go +++ b/node/node_newblock.go @@ -82,9 +82,6 @@ func (node *Node) proposeNewBlock() (*types.Block, error) { // Prepare transactions including staking transactions selectedTxs, selectedStakingTxs := node.getTransactionsForNewBlock(coinbase) - if len(selectedStakingTxs) > 0 { - utils.Logger().Debug().Int("selectedStakingtxs", len(selectedStakingTxs)).Msg("hehe, staking txs proposed") - } if err := node.Worker.CommitTransactions(selectedTxs, selectedStakingTxs, coinbase); err != nil { ctxerror.Log15(utils.GetLogger().Error, diff --git a/node/worker/worker.go b/node/worker/worker.go index b797665e4..d646363f5 100644 --- a/node/worker/worker.go +++ b/node/worker/worker.go @@ -426,7 +426,6 @@ func (w *Worker) FinalizeNewBlock(sig []byte, signers []byte, viewID uint64, coi return nil, ctxerror.New("cannot finalize block").WithCause(err) } - utils.Logger().Debug().Msgf("hehehe, len stxs: %v", len(block.StakingTransactions())) return block, nil } From 0f71a3e9842cc780c2474e7db21584e11ccb961b Mon Sep 17 00:00:00 2001 From: chao Date: Sat, 2 Nov 2019 15:10:59 -0700 Subject: [PATCH 16/45] move update validator list from proposal stage to block commit stage --- core/blockchain.go | 13 +++++++++++++ internal/params/config.go | 3 +-- node/node_handler.go | 4 ++++ node/worker/worker.go | 8 -------- 4 files changed, 18 insertions(+), 10 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index bfd60a3b3..fb092dad5 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1101,6 +1101,19 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types. bc.WriteCXReceiptsProofSpent(block.IncomingReceipts()) } + if bc.chainConfig.IsStaking(block.Epoch()) { + for _, tx := range block.StakingTransactions() { + err = bc.UpdateValidatorList(tx) + // keep offchain database consistency with onchain we need revert + // but it should not happend unless local database corrupted + if err != nil { + utils.Logger().Debug().Msgf("oops, UpdateValidatorList failed, err: %+v", err) + return NonStatTy, err + } + + } + } + // If the total difficulty is higher than our known, add it to the canonical chain // Second clause in the if statement reduces the vulnerability to selfish mining. // Please refer to http://www.cs.cornell.edu/~ie53/publications/btcProcFC.pdf diff --git a/internal/params/config.go b/internal/params/config.go index 6d5f19133..9bb5e49e0 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -148,8 +148,7 @@ func (c *ChainConfig) IsCrossTx(epoch *big.Int) bool { // IsStaking determines whether it is staking epoch func (c *ChainConfig) IsStaking(epoch *big.Int) bool { - stkEpoch := new(big.Int).Add(c.StakingEpoch, common.Big1) - return isForked(stkEpoch, epoch) + return isForked(c.StakingEpoch, epoch) } // IsCrossLink returns whether epoch is either equal to the CrossLink fork epoch or greater. diff --git a/node/node_handler.go b/node/node_handler.go index c67f533f0..e57edb089 100644 --- a/node/node_handler.go +++ b/node/node_handler.go @@ -410,6 +410,10 @@ func (node *Node) PostConsensusProcessing(newBlock *types.Block, commitSigAndBit // AddNewBlock is usedd to add new block into the blockchain. func (node *Node) AddNewBlock(newBlock *types.Block) error { _, err := node.Blockchain().InsertChain([]*types.Block{newBlock}, true /* verifyHeaders */) + if len(newBlock.StakingTransactions()) >= 1 { + addrs, err := node.Blockchain().ReadValidatorList() + utils.Logger().Debug().Msgf("validator list updated, err=%v, len(addrs)=%v", err, len(addrs)) + } if err != nil { utils.Logger().Error(). Err(err). diff --git a/node/worker/worker.go b/node/worker/worker.go index d646363f5..d998d0e27 100644 --- a/node/worker/worker.go +++ b/node/worker/worker.go @@ -209,14 +209,6 @@ func (w *Worker) commitStakingTransaction(tx *staking.StakingTransaction, coinba return nil, fmt.Errorf("nil staking receipt") } - err = w.chain.UpdateValidatorList(tx) - // keep offchain database consistency with onchain we need revert - // but it should not happend unless local database corrupted - if err != nil { - utils.Logger().Debug().Msgf("oops, UpdateValidatorList failed, err: %+v", err) - w.current.state.RevertToSnapshot(snap) - return nil, err - } w.current.stkingTxs = append(w.current.stkingTxs, tx) w.current.receipts = append(w.current.receipts, receipt) return receipt.Logs, nil From aed4b17e8a58b8fbc090bd154e150b90c228437f Mon Sep 17 00:00:00 2001 From: chao Date: Sat, 2 Nov 2019 18:26:58 -0700 Subject: [PATCH 17/45] fix rlp decoding issue for validatorWrapper --- core/blockchain.go | 38 ++++++++++---------------------------- core/state/statedb.go | 2 ++ hmy/api_backend.go | 3 ++- node/node_handler.go | 14 +++++++++++--- staking/types/validator.go | 23 +++++++++++++++++++---- 5 files changed, 44 insertions(+), 36 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index fb092dad5..169928b72 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -41,11 +41,9 @@ import ( "github.com/harmony-one/harmony/core/state" "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/core/vm" - internal_common "github.com/harmony-one/harmony/internal/common" "github.com/harmony-one/harmony/internal/ctxerror" "github.com/harmony-one/harmony/internal/params" "github.com/harmony-one/harmony/internal/utils" - "github.com/harmony-one/harmony/numeric" "github.com/harmony-one/harmony/shard" staking "github.com/harmony-one/harmony/staking/types" lru "github.com/hashicorp/golang-lru" @@ -2381,32 +2379,16 @@ func (bc *BlockChain) ValidatorCandidates() []common.Address { } // ValidatorInformation returns the information of validator -func (bc *BlockChain) ValidatorInformation(addr common.Address) *staking.Validator { - commission := staking.Commission{ - UpdateHeight: big.NewInt(0), - } - commission.CommissionRates = staking.CommissionRates{ - Rate: numeric.Dec{Int: big.NewInt(0)}, - MaxRate: numeric.Dec{Int: big.NewInt(0)}, - MaxChangeRate: numeric.Dec{Int: big.NewInt(0)}, - } - validator := &staking.Validator{ - Address: internal_common.ParseAddr("0x0000000000000000000000000000000000000000000000000000000000000000"), - SlotPubKeys: make([]shard.BlsPublicKey, 0), - Stake: big.NewInt(0), - UnbondingHeight: big.NewInt(0), - MinSelfDelegation: big.NewInt(0), - Active: false, - } - validator.Commission = commission - validator.Description = staking.Description{ - Name: "lol", - Identity: "lol", - Website: "lol", - SecurityContact: "lol", - Details: "lol", - } - return validator +func (bc *BlockChain) ValidatorInformation(addr common.Address) (*staking.Validator, error) { + state, err := bc.StateAt(bc.CurrentBlock().Root()) + if err != nil || state == nil { + return nil, err + } + wrapper := state.GetStakingInfo(addr) + if wrapper == nil { + return nil, fmt.Errorf("ValidatorInformation not found: %v", addr) + } + return &wrapper.Validator, nil } // DelegatorsInformation returns up to date information of delegators of a given validator address diff --git a/core/state/statedb.go b/core/state/statedb.go index a8a539b50..af7d99353 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -691,6 +691,7 @@ func (db *DB) GetStakingInfo(addr common.Address) *stk.ValidatorWrapper { val := stk.ValidatorWrapper{} err := rlp.DecodeBytes(by, &val) if err != nil { + fmt.Printf("GetStakingInfo unable to decode: %v\n", err) return nil } return &val @@ -703,6 +704,7 @@ func (db *DB) UpdateStakingInfo(addr common.Address, val *stk.ValidatorWrapper) return err } db.SetCode(addr, by) + return nil } diff --git a/hmy/api_backend.go b/hmy/api_backend.go index 346b68061..4e583bc73 100644 --- a/hmy/api_backend.go +++ b/hmy/api_backend.go @@ -302,7 +302,8 @@ func (b *APIBackend) GetValidatorCandidates() []common.Address { // GetValidatorInformation returns the information of validator func (b *APIBackend) GetValidatorInformation(addr common.Address) *staking.Validator { - return b.hmy.BlockChain().ValidatorInformation(addr) + val, _ := b.hmy.BlockChain().ValidatorInformation(addr) + return val } // GetDelegatorsInformation returns up to date information of delegators of a given validator address diff --git a/node/node_handler.go b/node/node_handler.go index e57edb089..184e45575 100644 --- a/node/node_handler.go +++ b/node/node_handler.go @@ -410,10 +410,18 @@ func (node *Node) PostConsensusProcessing(newBlock *types.Block, commitSigAndBit // AddNewBlock is usedd to add new block into the blockchain. func (node *Node) AddNewBlock(newBlock *types.Block) error { _, err := node.Blockchain().InsertChain([]*types.Block{newBlock}, true /* verifyHeaders */) - if len(newBlock.StakingTransactions()) >= 1 { - addrs, err := node.Blockchain().ReadValidatorList() - utils.Logger().Debug().Msgf("validator list updated, err=%v, len(addrs)=%v", err, len(addrs)) + + // Debug only + addrs, err := node.Blockchain().ReadValidatorList() + utils.Logger().Debug().Msgf("validator list updated, err=%v, len(addrs)=%v", err, len(addrs)) + for i, addr := range addrs { + val, err := node.Blockchain().ValidatorInformation(addr) + if err != nil { + utils.Logger().Debug().Msgf("ValidatorInformation Error %v: err %v", i, err) + } + utils.Logger().Debug().Msgf("ValidatorInformation %v: %v", i, val) } + if err != nil { utils.Logger().Error(). Err(err). diff --git a/staking/types/validator.go b/staking/types/validator.go index aee4743c4..542987998 100644 --- a/staking/types/validator.go +++ b/staking/types/validator.go @@ -2,6 +2,7 @@ package types import ( "errors" + "fmt" "math/big" "github.com/ethereum/go-ethereum/common" @@ -27,10 +28,10 @@ var ( // ValidatorWrapper contains validator and its delegation information type ValidatorWrapper struct { - Validator `json:"validator" yaml:"validator"` - Delegations []Delegation `json:"delegations" yaml:"delegations"` - SnapshotValidator *Validator `json:"snapshot_validator" yaml:"snaphost_validator"` - SnapshotDelegations []Delegation `json:"snapshot_delegations" yaml:"snapshot_delegations"` + Validator `json:"validator" yaml:"validator" rlp:"nil"` + Delegations []Delegation `json:"delegations" yaml:"delegations" rlp:"nil"` + SnapshotValidator *Validator `json:"snapshot_validator" yaml:"snaphost_validator" rlp:"nil"` + SnapshotDelegations []Delegation `json:"snapshot_delegations" yaml:"snapshot_delegations" rlp:"nil"` } // Validator - data fields for a validator @@ -169,3 +170,17 @@ func UpdateValidatorFromEditMsg(validator *Validator, edit *EditValidator) error } return nil } + +// String returns a human readable string representation of a validator. +func (v *Validator) String() string { + return fmt.Sprintf(`Validator + Address: %s + SlotPubKeys: %s + Stake: %s + Unbonding Height: %v + Minimum SelfDelegation: %v + Description: %v + Commission: %v`, v.Address, v.SlotPubKeys, + v.Stake, v.UnbondingHeight, + v.MinSelfDelegation, v.Description, v.Commission) +} From c58e47512b245643aec87ebddc080c64f7b7f9f7 Mon Sep 17 00:00:00 2001 From: Chao Ma Date: Sun, 3 Nov 2019 17:55:24 -0800 Subject: [PATCH 18/45] beautify validator String method --- staking/types/validator.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/staking/types/validator.go b/staking/types/validator.go index 542987998..99960d54d 100644 --- a/staking/types/validator.go +++ b/staking/types/validator.go @@ -54,6 +54,15 @@ type Validator struct { Description `json:"description" yaml:"description"` } +func printSlotPubKeys(pubKeys []shard.BlsPublicKey) string { + str := "[" + for i, key := range pubKeys { + str += fmt.Sprintf("%d: %s,", i, key.Hex()) + } + str += "]" + return str +} + // Description - some possible IRL connections type Description struct { Name string `json:"name" yaml:"name"` // name @@ -180,7 +189,7 @@ func (v *Validator) String() string { Unbonding Height: %v Minimum SelfDelegation: %v Description: %v - Commission: %v`, v.Address, v.SlotPubKeys, + Commission: %v`, v.Address.Hex(), printSlotPubKeys(v.SlotPubKeys), v.Stake, v.UnbondingHeight, v.MinSelfDelegation, v.Description, v.Commission) } From 71e4914bbfec8d04610e7089c085418588459cd0 Mon Sep 17 00:00:00 2001 From: Chao Ma Date: Sun, 3 Nov 2019 21:52:53 -0800 Subject: [PATCH 19/45] modify cmd for better staking tx sending; fix editvalidator call --- cmd/staking/root.go | 89 +++++++++++++++++++++++++++------------ staking/types/messages.go | 34 +++++++-------- 2 files changed, 80 insertions(+), 43 deletions(-) diff --git a/cmd/staking/root.go b/cmd/staking/root.go index a7e2d3242..2f07513bb 100644 --- a/cmd/staking/root.go +++ b/cmd/staking/root.go @@ -22,6 +22,7 @@ import ( "github.com/harmony-one/harmony/shard" staking "github.com/harmony-one/harmony/staking/types" "github.com/spf13/cobra" + "github.com/spf13/viper" ) const ( @@ -35,57 +36,86 @@ var ( queryID = 0 s = &staker{} localNetChain = big.NewInt(2) - dAddr, _ = common2.Bech32ToAddress(testAccount) ) const ( // Harmony protocol assume beacon chain shard is only place to send // staking, later need to consider logic when beacon chain shard rotates stakingShard = 0 - testAccount = "one1pdv9lrdwl0rg5vglh4xtyrv3wjk3wsqket7zxy" - testBLSPubKey = "65f55eb3052f9e9f632b2923be594ba77c55543f5c58ee1454b9cfd658d25e06373b0f7d42a19c84768139ea294f6204" testAccountPassword = "" ) +// command line options var +var ( + nonce = 0 + cmdType = "create" + name = "NewName" + index = 0 + minDele = 777 + + testAccounts = []string{ + "one1pdv9lrdwl0rg5vglh4xtyrv3wjk3wsqket7zxy", + "one12fuf7x9rgtdgqg7vgq0962c556m3p7afsxgvll"} + testBLSPubKeys = []string{ + "65f55eb3052f9e9f632b2923be594ba77c55543f5c58ee1454b9cfd658d25e06373b0f7d42a19c84768139ea294f6204", + "02c8ff0b88f313717bc3a627d2f8bb172ba3ad3bb9ba3ecb8eed4b7c878653d3d4faf769876c528b73f343967f74a917"} +) + func (s *staker) run(cmd *cobra.Command, args []string) error { scryptN := keystore.StandardScryptN scryptP := keystore.StandardScryptP ks := keystore.NewKeyStore(keystoreDir, scryptN, scryptP) + dAddr, _ := common2.Bech32ToAddress(testAccounts[index]) account := accounts.Account{Address: dAddr} ks.Unlock(account, testAccountPassword) gasPrice := big.NewInt(1) stakePayloadMaker := func() (staking.Directive, interface{}) { p := &bls.PublicKey{} - p.DeserializeHexStr(testBLSPubKey) + p.DeserializeHexStr(testBLSPubKeys[index]) pub := shard.BlsPublicKey{} pub.FromLibBLSPublicKey(p) - return staking.DirectiveCreateValidator, staking.CreateValidator{ + if cmdType == "create" { + return staking.DirectiveCreateValidator, staking.CreateValidator{ + Description: &staking.Description{ + Name: "SuperHero", + Identity: "YouWouldNotKnow", + Website: "Secret Website", + SecurityContact: "Mr.DoubleZeroSeven", + Details: "blah blah blah", + }, + CommissionRates: staking.CommissionRates{ + Rate: numeric.NewDec(100), + MaxRate: numeric.NewDec(150), + MaxChangeRate: numeric.NewDec(5), + }, + MinSelfDelegation: big.NewInt(10), + MaxTotalDelegation: big.NewInt(3000), + ValidatorAddress: common.Address(dAddr), + SlotPubKeys: []shard.BlsPublicKey{pub}, + Amount: big.NewInt(100), + } + } + /* + ValidatorAddress common.Address `json:"validator_address" yaml:"validator_address"` + Description *Description `json:"description" yaml:"description"` + CommissionRate *numeric.Dec `json:"commission_rate" yaml:"commission_rate"` + MinSelfDelegation *big.Int `json:"min_self_delegation" yaml:"min_self_delegation"` + MaxTotalDelegation *big.Int `json:"max_total_delegation" yaml:"max_total_delegation"` + SlotKeyToRemove *shard.BlsPublicKey `json:"slot_key_to_remove" yaml:"slot_key_to_remove"` + SlotKeyToAdd *shard.BlsPublicKey `json:"slot_key_to_add" yaml:"slot_key_to_add"` + } + */ + + return staking.DirectiveEditValidator, staking.EditValidator{ Description: &staking.Description{ - Name: "SuperHero", - Identity: "YouWouldNotKnow", - Website: "Secret Website", - SecurityContact: "Mr.DoubleZeroSeven", - Details: "blah blah blah", - }, - CommissionRates: staking.CommissionRates{ - Rate: numeric.NewDec(100), - MaxRate: numeric.NewDec(150), - MaxChangeRate: numeric.NewDec(5), + Name: name, }, - MinSelfDelegation: big.NewInt(10), - MaxTotalDelegation: big.NewInt(3000), - ValidatorAddress: common.Address(dAddr), - SlotPubKeys: []shard.BlsPublicKey{pub}, - Amount: big.NewInt(100), + MinSelfDelegation: big.NewInt(int64(minDele)), + ValidatorAddress: common.Address(dAddr), } - // return staking.DirectiveDelegate, staking.Delegate{ - // common.Address(dAddr), - // common.Address(dAddr), - // big.NewInt(10), - // } } - stakingTx, err := staking.NewStakingTransaction(0, 600000, gasPrice, stakePayloadMaker) + stakingTx, err := staking.NewStakingTransaction(uint64(nonce), 600000, gasPrice, stakePayloadMaker) if err != nil { return err } @@ -154,7 +184,13 @@ func baseRequest(method, node string, params interface{}) ([]byte, error) { } func init() { + rootCmd.PersistentFlags().IntVarP(&index, "index", "i", 0, "account index:0|1") + rootCmd.PersistentFlags().IntVarP(&nonce, "nonce", "n", 0, "nonce of address") + rootCmd.PersistentFlags().StringVarP(&cmdType, "type", "t", "create", "type of commands: create|edit") + rootCmd.PersistentFlags().StringVarP(&name, "name", "m", "ANewName", "Name of Validator") + rootCmd.PersistentFlags().IntVarP(&minDele, "minDele", "d", 666, "MinSelfDelegation Fee") + viper.BindPFlag("nonce", rootCmd.PersistentFlags().Lookup("nonce")) rootCmd.AddCommand(&cobra.Command{ Use: "staking-iterate", Short: "run through staking process", @@ -170,6 +206,7 @@ func init() { os.Exit(0) }, }) + } var ( diff --git a/staking/types/messages.go b/staking/types/messages.go index 3a25d6ce4..035007a06 100644 --- a/staking/types/messages.go +++ b/staking/types/messages.go @@ -48,38 +48,38 @@ func (d Directive) String() string { // CreateValidator - type for creating a new validator type CreateValidator struct { - Description *Description `json:"description" yaml:"description"` - CommissionRates `json:"commission" yaml:"commission"` - MinSelfDelegation *big.Int `json:"min_self_delegation" yaml:"min_self_delegation"` - MaxTotalDelegation *big.Int `json:"max_total_delegation" yaml:"max_total_delegation"` - ValidatorAddress common.Address `json:"validator_address" yaml:"validator_address"` - SlotPubKeys []shard.BlsPublicKey `json:"slot_pub_keys" yaml:"slot_pub_keys"` - Amount *big.Int `json:"amount" yaml:"amount"` + Description *Description `json:"description" yaml:"description" rlp:"nil"` + CommissionRates `json:"commission" yaml:"commission" rlp:"nil"` + MinSelfDelegation *big.Int `json:"min_self_delegation" yaml:"min_self_delegation" rlp:"nil"` + MaxTotalDelegation *big.Int `json:"max_total_delegation" yaml:"max_total_delegation" rlp:"nil"` + ValidatorAddress common.Address `json:"validator_address" yaml:"validator_address" rlp:"nil"` + SlotPubKeys []shard.BlsPublicKey `json:"slot_pub_keys" yaml:"slot_pub_keys" rlp:"nil"` + Amount *big.Int `json:"amount" yaml:"amount" rlp:"nil"` } // EditValidator - type for edit existing validator type EditValidator struct { - ValidatorAddress common.Address `json:"validator_address" yaml:"validator_address"` + ValidatorAddress common.Address `json:"validator_address" yaml:"validator_address" rlp:"nil"` Description *Description `json:"description" yaml:"description"` - CommissionRate *numeric.Dec `json:"commission_rate" yaml:"commission_rate"` - MinSelfDelegation *big.Int `json:"min_self_delegation" yaml:"min_self_delegation"` - MaxTotalDelegation *big.Int `json:"max_total_delegation" yaml:"max_total_delegation"` - SlotKeyToRemove *shard.BlsPublicKey `json:"slot_key_to_remove" yaml:"slot_key_to_remove"` - SlotKeyToAdd *shard.BlsPublicKey `json:"slot_key_to_add" yaml:"slot_key_to_add"` + CommissionRate *numeric.Dec `json:"commission_rate" yaml:"commission_rate" rlp:"nil" rlp:"nil"` + MinSelfDelegation *big.Int `json:"min_self_delegation" yaml:"min_self_delegation" rlp:"nil"` + MaxTotalDelegation *big.Int `json:"max_total_delegation" yaml:"max_total_delegation" rlp:"nil"` + SlotKeyToRemove *shard.BlsPublicKey `json:"slot_key_to_remove" yaml:"slot_key_to_remove" rlp:"nil"` + SlotKeyToAdd *shard.BlsPublicKey `json:"slot_key_to_add" yaml:"slot_key_to_add" rlp:"nil"` } // Delegate - type for delegating to a validator type Delegate struct { DelegatorAddress common.Address `json:"delegator_address" yaml:"delegator_address"` ValidatorAddress common.Address `json:"validator_address" yaml:"validator_address"` - Amount *big.Int `json:"amount" yaml:"amount"` + Amount *big.Int `json:"amount" yaml:"amount" rlp:"nil"` } // Undelegate - type for removing delegation responsibility type Undelegate struct { - DelegatorAddress common.Address `json:"delegator_address" yaml:"delegator_address"` - ValidatorAddress common.Address `json:"validator_address" yaml:"validator_address"` - Amount *big.Int `json:"amount" yaml:"amount"` + DelegatorAddress common.Address `json:"delegator_address" yaml:"delegator_address" rlp:"nil"` + ValidatorAddress common.Address `json:"validator_address" yaml:"validator_address" rlp:"nil"` + Amount *big.Int `json:"amount" yaml:"amount" rlp:"nil"` } // CollectRewards - type for collecting token rewards From 74d774de0532a5359bd9fe6e8b531f8e4064024c Mon Sep 17 00:00:00 2001 From: Chao Ma Date: Sun, 3 Nov 2019 23:36:50 -0800 Subject: [PATCH 20/45] add more fields editing in updateEditValidator; add commission ratels in staking tx sending; add current/future staking validator api call; --- cmd/staking/root.go | 16 ++++++++++----- core/blockchain.go | 28 ++++++++++++++++++++++++-- core/state_transition.go | 9 ++++++++- node/node_handler.go | 5 +++++ staking/types/commission.go | 13 +++++++++++++ staking/types/validator.go | 39 +++++++++++++++++++++++++++++++++++-- 6 files changed, 100 insertions(+), 10 deletions(-) diff --git a/cmd/staking/root.go b/cmd/staking/root.go index 2f07513bb..88b605dab 100644 --- a/cmd/staking/root.go +++ b/cmd/staking/root.go @@ -22,7 +22,6 @@ import ( "github.com/harmony-one/harmony/shard" staking "github.com/harmony-one/harmony/staking/types" "github.com/spf13/cobra" - "github.com/spf13/viper" ) const ( @@ -52,6 +51,7 @@ var ( name = "NewName" index = 0 minDele = 777 + rate = "0.0" testAccounts = []string{ "one1pdv9lrdwl0rg5vglh4xtyrv3wjk3wsqket7zxy", @@ -74,6 +74,10 @@ func (s *staker) run(cmd *cobra.Command, args []string) error { p.DeserializeHexStr(testBLSPubKeys[index]) pub := shard.BlsPublicKey{} pub.FromLibBLSPublicKey(p) + + ra, _ := numeric.NewDecFromStr("27.27") + maxRate, _ := numeric.NewDecFromStr("150.99") + maxChangeRate, _ := numeric.NewDecFromStr("0.5") if cmdType == "create" { return staking.DirectiveCreateValidator, staking.CreateValidator{ Description: &staking.Description{ @@ -84,9 +88,9 @@ func (s *staker) run(cmd *cobra.Command, args []string) error { Details: "blah blah blah", }, CommissionRates: staking.CommissionRates{ - Rate: numeric.NewDec(100), - MaxRate: numeric.NewDec(150), - MaxChangeRate: numeric.NewDec(5), + Rate: ra, + MaxRate: maxRate, + MaxChangeRate: maxChangeRate, }, MinSelfDelegation: big.NewInt(10), MaxTotalDelegation: big.NewInt(3000), @@ -106,11 +110,13 @@ func (s *staker) run(cmd *cobra.Command, args []string) error { } */ + newRate, _ := numeric.NewDecFromStr(rate) return staking.DirectiveEditValidator, staking.EditValidator{ Description: &staking.Description{ Name: name, }, MinSelfDelegation: big.NewInt(int64(minDele)), + CommissionRate: &newRate, ValidatorAddress: common.Address(dAddr), } } @@ -189,8 +195,8 @@ func init() { rootCmd.PersistentFlags().StringVarP(&cmdType, "type", "t", "create", "type of commands: create|edit") rootCmd.PersistentFlags().StringVarP(&name, "name", "m", "ANewName", "Name of Validator") rootCmd.PersistentFlags().IntVarP(&minDele, "minDele", "d", 666, "MinSelfDelegation Fee") + rootCmd.PersistentFlags().StringVarP(&rate, "rate", "r", "22.22", "Commision Rate") - viper.BindPFlag("nonce", rootCmd.PersistentFlags().Lookup("nonce")) rootCmd.AddCommand(&cobra.Command{ Use: "staking-iterate", Short: "run through staking process", diff --git a/core/blockchain.go b/core/blockchain.go index 169928b72..f6fefdc1d 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -2370,12 +2370,36 @@ func (bc *BlockChain) UpdateValidatorList(tx *staking.StakingTransaction) error // CurrentValidatorAddresses returns the address of active validators for current epoch func (bc *BlockChain) CurrentValidatorAddresses() []common.Address { - return make([]common.Address, 0) + list, err := bc.ReadValidatorList() + if err != nil { + return make([]common.Address, 0) + } + + currentEpoch := bc.CurrentBlock().Epoch() + + filtered := []common.Address{} + for _, addr := range list { + val, err := bc.ValidatorInformation(addr) + if err != nil { + continue + } + epoch := ShardingSchedule.CalcEpochNumber(val.CreationHeight.Uint64()) + if epoch.Cmp(currentEpoch) >= 0 { + // wait for next epoch + continue + } + filtered = append(filtered, addr) + } + return filtered } // ValidatorCandidates returns the up to date validator candidates for next epoch func (bc *BlockChain) ValidatorCandidates() []common.Address { - return make([]common.Address, 0) + list, err := bc.ReadValidatorList() + if err != nil { + return make([]common.Address, 0) + } + return list } // ValidatorInformation returns the information of validator diff --git a/core/state_transition.go b/core/state_transition.go index 934c3f222..dc68c62f6 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -335,6 +335,7 @@ func (st *StateTransition) applyCreateValidatorTx(nv *staking.CreateValidator, b return err } v.UpdateHeight = blockNum + v.CreationHeight = blockNum wrapper := staking.ValidatorWrapper{*v, nil, nil, nil} if err := st.state.UpdateStakingInfo(v.Address, &wrapper); err != nil { return err @@ -348,10 +349,16 @@ func (st *StateTransition) applyEditValidatorTx(ev *staking.EditValidator, block return errValidatorNotExist } wrapper := st.state.GetStakingInfo(ev.ValidatorAddress) + + oldRate := wrapper.Validator.Rate if err := staking.UpdateValidatorFromEditMsg(&wrapper.Validator, ev); err != nil { return err } - wrapper.Validator.UpdateHeight = blockNum + newRate := wrapper.Validator.Rate + // update the commision rate change height + if oldRate.IsNil() || (!newRate.IsNil() && !oldRate.Equal(newRate)) { + wrapper.Validator.UpdateHeight = blockNum + } if err := st.state.UpdateStakingInfo(ev.ValidatorAddress, wrapper); err != nil { return err } diff --git a/node/node_handler.go b/node/node_handler.go index 184e45575..57f2d89ae 100644 --- a/node/node_handler.go +++ b/node/node_handler.go @@ -421,6 +421,11 @@ func (node *Node) AddNewBlock(newBlock *types.Block) error { } utils.Logger().Debug().Msgf("ValidatorInformation %v: %v", i, val) } + currAddrs := node.Blockchain().CurrentValidatorAddresses() + utils.Logger().Debug().Msgf("CurrentValidators : %v", currAddrs) + candidates := node.Blockchain().ValidatorCandidates() + utils.Logger().Debug().Msgf("CandidateValidators : %v", candidates) + // Finish debug if err != nil { utils.Logger().Error(). diff --git a/staking/types/commission.go b/staking/types/commission.go index 3176ba807..43de6565c 100644 --- a/staking/types/commission.go +++ b/staking/types/commission.go @@ -1,6 +1,7 @@ package types import ( + "fmt" "math/big" "github.com/harmony-one/harmony/numeric" @@ -22,3 +23,15 @@ type ( MaxChangeRate numeric.Dec `json:"max_change_rate" yaml:"max_change_rate"` // maximum increase of the validator commission every epoch, as a fraction } ) + +// String returns a human readable string representation of a validator. +func (c Commission) String() string { + return fmt.Sprintf(` + Commission: + Rate: %s + MaxRate: %s + MaxChangeRate: %s + UpdateHeight: %v`, + c.Rate, c.MaxRate, c.MaxChangeRate, + c.UpdateHeight) +} diff --git a/staking/types/validator.go b/staking/types/validator.go index 99960d54d..3edbf831b 100644 --- a/staking/types/validator.go +++ b/staking/types/validator.go @@ -46,12 +46,16 @@ type Validator struct { UnbondingHeight *big.Int `json:"unbonding_height" yaml:"unbonding_height"` // validator's self declared minimum self delegation MinSelfDelegation *big.Int `json:"min_self_delegation" yaml:"min_self_delegation"` + // maximum total delgation allowed + MaxTotalDelegation *big.Int `json:"min_self_delegation" yaml:"min_self_delegation"` // Is the validator active in the validating process or not Active bool `json:"active" yaml:"active"` // commission parameters Commission `json:"commission" yaml:"commission"` // description for the validator Description `json:"description" yaml:"description"` + // CreationHeight is the height of creation + CreationHeight *big.Int `json:"creation_height" yaml:"creation_height"` } func printSlotPubKeys(pubKeys []shard.BlsPublicKey) string { @@ -153,12 +157,13 @@ func CreateValidatorFromNewMsg(val *CreateValidator) (*Validator, error) { pubKeys := []shard.BlsPublicKey{} pubKeys = append(pubKeys, val.SlotPubKeys...) v := Validator{val.ValidatorAddress, pubKeys, - val.Amount, new(big.Int), val.MinSelfDelegation, false, - commission, desc} + val.Amount, new(big.Int), val.MinSelfDelegation, val.MaxTotalDelegation, false, + commission, desc, big.NewInt(0)} return &v, nil } // UpdateValidatorFromEditMsg updates validator from EditValidator message +// TODO check the validity of the fields of edit message func UpdateValidatorFromEditMsg(validator *Validator, edit *EditValidator) error { if validator.Address != edit.ValidatorAddress { return errAddressNotMatch @@ -172,11 +177,41 @@ func UpdateValidatorFromEditMsg(validator *Validator, edit *EditValidator) error if edit.CommissionRate != nil { validator.Rate = *edit.CommissionRate + if err != nil { + return err + } + //TODO update other rates } if edit.MinSelfDelegation != nil { validator.MinSelfDelegation = edit.MinSelfDelegation } + + if edit.MaxTotalDelegation != nil { + validator.MaxTotalDelegation = edit.MaxTotalDelegation + } + + if edit.SlotKeyToAdd != nil { + for _, key := range validator.SlotPubKeys { + if key == *edit.SlotKeyToAdd { + break + } + validator.SlotPubKeys = append(validator.SlotPubKeys, *edit.SlotKeyToAdd) + } + } + + if edit.SlotKeyToRemove != nil { + index := -1 + for i, key := range validator.SlotPubKeys { + if key == *edit.SlotKeyToRemove { + index = i + } + } + // we found key to be removed + if index >= 0 { + validator.SlotPubKeys = append(validator.SlotPubKeys[:index], validator.SlotPubKeys[index+1:]...) + } + } return nil } From 0edfedee1d8649848d03601072877e05caf12c32 Mon Sep 17 00:00:00 2001 From: Sebastian Johnsson Date: Mon, 4 Nov 2019 23:56:53 +0700 Subject: [PATCH 21/45] Update Pangaea to use the new bootnodes (54.218.73.167 & 18.232.171.117) --- scripts/node.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/node.sh b/scripts/node.sh index d3d56ff38..f427c602f 100755 --- a/scripts/node.sh +++ b/scripts/node.sh @@ -229,8 +229,8 @@ beta) ;; pangaea) bootnodes=( - /ip4/54.86.126.90/tcp/9889/p2p/Qmdfjtk6hPoyrH1zVD9PEH4zfWLo38dP2mDvvKXfh3tnEv - /ip4/52.40.84.2/tcp/9889/p2p/QmZJJx6AdaoEkGLrYG4JeLCKeCKDjnFz2wfHNHxAqFSGA9 + /ip4/54.218.73.167/tcp/9876/p2p/QmWBVCPXQmc2ULigm3b9ayCZa15gj25kywiQQwPhHCZeXj + /ip4/18.232.171.117/tcp/9876/p2p/QmfJ71Eb7XTDs8hX2vPJ8un4L7b7RiDk6zCzWVxLXGA6MA ) REL=pangaea network_type=pangaea From 02057507dff76544ed71496bcb8dab4f7ea1bb97 Mon Sep 17 00:00:00 2001 From: Sebastian Johnsson Date: Mon, 4 Nov 2019 23:57:20 +0700 Subject: [PATCH 22/45] Remove previous beta/testnet section from node.sh --- scripts/node.sh | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/scripts/node.sh b/scripts/node.sh index f427c602f..77842429e 100755 --- a/scripts/node.sh +++ b/scripts/node.sh @@ -217,16 +217,6 @@ main) network_type=mainnet dns_zone=t.hmny.io ;; -beta) - bootnodes=( - /ip4/54.213.43.194/tcp/9868/p2p/QmZJJx6AdaoEkGLrYG4JeLCKeCKDjnFz2wfHNHxAqFSGA9 - /ip4/100.26.90.187/tcp/9868/p2p/Qmdfjtk6hPoyrH1zVD9PEH4zfWLo38dP2mDvvKXfh3tnEv - /ip4/13.113.101.219/tcp/12018/p2p/QmQayinFSgMMw5cSpDUiD9pQ2WeP6WNmGxpZ6ou3mdVFJX - ) - REL=testnet - network_type=testnet - dns_zone=b.hmny.io - ;; pangaea) bootnodes=( /ip4/54.218.73.167/tcp/9876/p2p/QmWBVCPXQmc2ULigm3b9ayCZa15gj25kywiQQwPhHCZeXj From 5dd83cd4be960aeb1ac668b24e124d240f5a4a7f Mon Sep 17 00:00:00 2001 From: chao Date: Mon, 4 Nov 2019 14:05:16 -0800 Subject: [PATCH 23/45] comment out staking debug code which can be uncommented later for debugging; we are not done yet --- core/blockchain.go | 1 + node/node_handler.go | 30 ++++++++++++++++-------------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index f6fefdc1d..7d4af6818 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -2344,6 +2344,7 @@ func (bc *BlockChain) UpdateValidatorList(tx *staking.StakingTransaction) error switch tx.StakingType() { case staking.DirectiveCreateValidator: createValidator := decodePayload.(*staking.CreateValidator) + // TODO: batch add validator list instead of one by one list, err := bc.ReadValidatorList() if err != nil { return err diff --git a/node/node_handler.go b/node/node_handler.go index 57f2d89ae..2041ccdd5 100644 --- a/node/node_handler.go +++ b/node/node_handler.go @@ -411,21 +411,23 @@ func (node *Node) PostConsensusProcessing(newBlock *types.Block, commitSigAndBit func (node *Node) AddNewBlock(newBlock *types.Block) error { _, err := node.Blockchain().InsertChain([]*types.Block{newBlock}, true /* verifyHeaders */) - // Debug only - addrs, err := node.Blockchain().ReadValidatorList() - utils.Logger().Debug().Msgf("validator list updated, err=%v, len(addrs)=%v", err, len(addrs)) - for i, addr := range addrs { - val, err := node.Blockchain().ValidatorInformation(addr) - if err != nil { - utils.Logger().Debug().Msgf("ValidatorInformation Error %v: err %v", i, err) + /* + // Debug only + addrs, err := node.Blockchain().ReadValidatorList() + utils.Logger().Debug().Msgf("validator list updated, err=%v, len(addrs)=%v", err, len(addrs)) + for i, addr := range addrs { + val, err := node.Blockchain().ValidatorInformation(addr) + if err != nil { + utils.Logger().Debug().Msgf("ValidatorInformation Error %v: err %v", i, err) + } + utils.Logger().Debug().Msgf("ValidatorInformation %v: %v", i, val) } - utils.Logger().Debug().Msgf("ValidatorInformation %v: %v", i, val) - } - currAddrs := node.Blockchain().CurrentValidatorAddresses() - utils.Logger().Debug().Msgf("CurrentValidators : %v", currAddrs) - candidates := node.Blockchain().ValidatorCandidates() - utils.Logger().Debug().Msgf("CandidateValidators : %v", candidates) - // Finish debug + currAddrs := node.Blockchain().CurrentValidatorAddresses() + utils.Logger().Debug().Msgf("CurrentValidators : %v", currAddrs) + candidates := node.Blockchain().ValidatorCandidates() + utils.Logger().Debug().Msgf("CandidateValidators : %v", candidates) + // Finish debug + */ if err != nil { utils.Logger().Error(). From 5827fc231ef78042f73d8583cd8e643aa61a6be4 Mon Sep 17 00:00:00 2001 From: Eugene Kim Date: Tue, 5 Nov 2019 00:40:25 +0000 Subject: [PATCH 24/45] Use -network_type=testnet for Pangaea/testnet It should be lockstep-updated after we deprecate/remove -network_type=pangaea later. --- scripts/node.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/node.sh b/scripts/node.sh index 77842429e..1d40f5c14 100755 --- a/scripts/node.sh +++ b/scripts/node.sh @@ -223,7 +223,7 @@ pangaea) /ip4/18.232.171.117/tcp/9876/p2p/QmfJ71Eb7XTDs8hX2vPJ8un4L7b7RiDk6zCzWVxLXGA6MA ) REL=pangaea - network_type=pangaea + network_type=testnet dns_zone=p.hmny.io ;; *) From 3aa28fa9e4f8e9202b9b37f6071e3f2523b7834d Mon Sep 17 00:00:00 2001 From: Eugene Kim Date: Tue, 5 Nov 2019 00:42:55 +0000 Subject: [PATCH 25/45] Deprecate -N beta with a warning message --- scripts/node.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scripts/node.sh b/scripts/node.sh index 1d40f5c14..d2c5379db 100755 --- a/scripts/node.sh +++ b/scripts/node.sh @@ -217,7 +217,12 @@ main) network_type=mainnet dns_zone=t.hmny.io ;; -pangaea) +beta|pangaea) + case "${network}" in + beta) + msg "WARNING: -N beta has been deprecated and will be removed in a future release; please use -N pangaea instead." + ;; + esac bootnodes=( /ip4/54.218.73.167/tcp/9876/p2p/QmWBVCPXQmc2ULigm3b9ayCZa15gj25kywiQQwPhHCZeXj /ip4/18.232.171.117/tcp/9876/p2p/QmfJ71Eb7XTDs8hX2vPJ8un4L7b7RiDk6zCzWVxLXGA6MA From e9013af8053bfa9981fe7349f848621d82d03dba Mon Sep 17 00:00:00 2001 From: Eugene Kim Date: Tue, 5 Nov 2019 01:13:36 +0000 Subject: [PATCH 26/45] Print filename upon curl failure --- scripts/node.sh | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/scripts/node.sh b/scripts/node.sh index d2c5379db..44182b13d 100755 --- a/scripts/node.sh +++ b/scripts/node.sh @@ -292,12 +292,20 @@ verify_checksum() { } download_binaries() { - local outdir + local outdir status ${do_not_download} && return 0 outdir="${1:-.}" mkdir -p "${outdir}" for bin in "${BIN[@]}"; do - curl -sSf http://${BUCKET}.s3.amazonaws.com/${FOLDER}${bin} -o "${outdir}/${bin}" || return $? + status=0 + curl -sSf http://${BUCKET}.s3.amazonaws.com/${FOLDER}${bin} -o "${outdir}/${bin}" || status=$? + case "${status}" in + 0) ;; + *) + msg "cannot download ${bin} (status ${status})" + return ${status} + ;; + esac verify_checksum "${outdir}" "${bin}" md5sum.txt || return $? msg "downloaded ${bin}" done From 943d1b38bbfcbba66c22a8b11d6108731b7623b3 Mon Sep 17 00:00:00 2001 From: Eugene Kim Date: Tue, 5 Nov 2019 01:14:46 +0000 Subject: [PATCH 27/45] Mandate arguments to any_new_binaries and download_binaries --- scripts/node.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/node.sh b/scripts/node.sh index 44182b13d..1298f27a7 100755 --- a/scripts/node.sh +++ b/scripts/node.sh @@ -294,7 +294,7 @@ verify_checksum() { download_binaries() { local outdir status ${do_not_download} && return 0 - outdir="${1:-.}" + outdir="${1}" mkdir -p "${outdir}" for bin in "${BIN[@]}"; do status=0 @@ -452,7 +452,7 @@ esac any_new_binaries() { local outdir ${do_not_download} && return 0 - outdir="${1:-.}" + outdir="${1}" mkdir -p "${outdir}" curl -sSf http://${BUCKET}.s3.amazonaws.com/${FOLDER}md5sum.txt -o "${outdir}/md5sum.txt.new" || return $? if diff $outdir/md5sum.txt.new md5sum.txt @@ -464,11 +464,11 @@ any_new_binaries() { fi } -if any_new_binaries +if any_new_binaries . then msg "binaries did not change" else - download_binaries || err 69 "initial node software update failed" + download_binaries . || err 69 "initial node software update failed" fi NODE_PORT=9000 From f75abe17a76ac4840d40ea6401c6893296570cc4 Mon Sep 17 00:00:00 2001 From: Eugene Kim Date: Tue, 5 Nov 2019 01:26:29 +0000 Subject: [PATCH 28/45] Download/verify files listed in md5sum.txt That is, do not hardcode list of files. --- scripts/node.sh | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/scripts/node.sh b/scripts/node.sh index 1298f27a7..7973dd9b3 100755 --- a/scripts/node.sh +++ b/scripts/node.sh @@ -249,11 +249,9 @@ fi if [ "$OS" == "Darwin" ]; then FOLDER=release/darwin-x86_64/$REL/ - BIN=( harmony libbls384_256.dylib libcrypto.1.0.0.dylib libgmp.10.dylib libgmpxx.4.dylib libmcl.dylib md5sum.txt ) fi if [ "$OS" == "Linux" ]; then FOLDER=release/linux-x86_64/$REL/ - BIN=( harmony libbls384_256.so libcrypto.so.10 libgmp.so.10 libgmpxx.so.4 libmcl.so md5sum.txt ) fi extract_checksum() { @@ -296,7 +294,7 @@ download_binaries() { ${do_not_download} && return 0 outdir="${1}" mkdir -p "${outdir}" - for bin in "${BIN[@]}"; do + for bin in $(cut -c35- "${outdir}/md5sum.txt"); do status=0 curl -sSf http://${BUCKET}.s3.amazonaws.com/${FOLDER}${bin} -o "${outdir}/${bin}" || status=$? case "${status}" in @@ -310,7 +308,7 @@ download_binaries() { msg "downloaded ${bin}" done chmod +x "${outdir}/harmony" - (cd "${outdir}" && exec openssl sha256 "${BIN[@]}") > "${outdir}/harmony-checksums.txt" + (cd "${outdir}" && exec openssl sha256 $(cut -c35- md5sum.txt)) > "${outdir}/harmony-checksums.txt" } check_free_disk() { @@ -607,7 +605,7 @@ kill_node() { continue fi msg "binaries changed; moving from staging into main" - (cd staging; exec mv harmony-checksums.txt "${BIN[@]}" ..) || continue + (cd staging; exec mv harmony-checksums.txt $(cut -c35- md5sum.txt) ..) || continue msg "binaries updated, killing node to restart" kill_node done From bbcb9f02748abbca4a535040a72a9e956eea8c86 Mon Sep 17 00:00:00 2001 From: Eugene Kim Date: Tue, 5 Nov 2019 01:27:25 +0000 Subject: [PATCH 29/45] Avoid unconditional download in download-only mode --- scripts/node.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/scripts/node.sh b/scripts/node.sh index 7973dd9b3..b162d24ed 100755 --- a/scripts/node.sh +++ b/scripts/node.sh @@ -403,8 +403,13 @@ download_harmony_db_file() { } if ${download_only}; then - download_binaries staging || err 69 "download node software failed" - msg "downloaded files are in staging direectory" + if any_new_binaries staging + then + msg "binaries did not change in staging" + else + download_binaries staging || err 69 "download node software failed" + msg "downloaded files are in staging direectory" + fi exit 0 fi From 70b0a6258d33df19042a6edbeb2129321a0b1cb7 Mon Sep 17 00:00:00 2001 From: Leo Chen Date: Tue, 5 Nov 2019 10:52:42 -0800 Subject: [PATCH 30/45] [node.sh] updated version Signed-off-by: Leo Chen --- scripts/node.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/node.sh b/scripts/node.sh index b162d24ed..cb6b4e668 100755 --- a/scripts/node.sh +++ b/scripts/node.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -version="v1 20190924.0" +version="v1 20191105.1" unset -v progname progname="${0##*/}" From 56d8be156260e1d10f1f310ac50bd8f56b5a2642 Mon Sep 17 00:00:00 2001 From: flicker-harmony Date: Wed, 6 Nov 2019 01:22:08 +0300 Subject: [PATCH 31/45] Fix security overflow issue --- internal/hmyapi/util.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/hmyapi/util.go b/internal/hmyapi/util.go index 81464f112..b8fcd65d2 100644 --- a/internal/hmyapi/util.go +++ b/internal/hmyapi/util.go @@ -23,7 +23,7 @@ func ReturnWithPagination(hashes []common.Hash, args TxHistoryArgs) []common.Has if args.PageSize > 0 { pageSize = args.PageSize } - if pageSize*pageIndex >= len(hashes) { + if pageIndex < 0 || pageSize*pageIndex >= len(hashes) { return make([]common.Hash, 0) } if pageSize*pageIndex+pageSize > len(hashes) { From feceec6e38a6c2ad4993194db3c12f43bb439ec5 Mon Sep 17 00:00:00 2001 From: Leo Chen Date: Wed, 6 Nov 2019 01:01:25 +0000 Subject: [PATCH 32/45] [net] only listen to tcp4 address Signed-off-by: Leo Chen --- api/client/service/server.go | 2 +- api/proto/message/server.go | 2 +- api/service/syncing/downloader/server.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/api/client/service/server.go b/api/client/service/server.go index 1e554eb31..c6cea57b2 100644 --- a/api/client/service/server.go +++ b/api/client/service/server.go @@ -43,7 +43,7 @@ func (s *Server) GetFreeToken(ctx context.Context, request *proto.GetFreeTokenRe func (s *Server) Start(ip, port string) (*grpc.Server, error) { // TODO(minhdoan): Currently not using ip. Fix it later. addr := net.JoinHostPort("", port) - lis, err := net.Listen("tcp", addr) + lis, err := net.Listen("tcp4", addr) if err != nil { log.Fatalf("failed to listen: %v", err) } diff --git a/api/proto/message/server.go b/api/proto/message/server.go index 20b90d5db..da0f51174 100644 --- a/api/proto/message/server.go +++ b/api/proto/message/server.go @@ -81,7 +81,7 @@ func (s *Server) Process(ctx context.Context, message *Message) (*Response, erro // Start starts the Server on given ip and port. func (s *Server) Start() (*grpc.Server, error) { addr := net.JoinHostPort(IP, Port) - lis, err := net.Listen("tcp", addr) + lis, err := net.Listen("tcp4", addr) if err != nil { log.Fatalf("failed to listen: %v", err) } diff --git a/api/service/syncing/downloader/server.go b/api/service/syncing/downloader/server.go index d1953aa8f..ea82c19b9 100644 --- a/api/service/syncing/downloader/server.go +++ b/api/service/syncing/downloader/server.go @@ -43,7 +43,7 @@ func (s *Server) Query(ctx context.Context, request *pb.DownloaderRequest) (*pb. // Start starts the Server on given ip and port. func (s *Server) Start(ip, port string) (*grpc.Server, error) { addr := net.JoinHostPort("", port) - lis, err := net.Listen("tcp", addr) + lis, err := net.Listen("tcp4", addr) if err != nil { log.Fatalf("[SYNC] failed to listen: %v", err) } From 64ed37a1d6263630b2db1eb4798693cb795032ea Mon Sep 17 00:00:00 2001 From: Leo Chen Date: Wed, 6 Nov 2019 01:37:52 +0000 Subject: [PATCH 33/45] [node] disable client support service on mainnet Signed-off-by: Leo Chen --- node/service_setup.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/node/service_setup.go b/node/service_setup.go index 8d4212d93..1818481df 100644 --- a/node/service_setup.go +++ b/node/service_setup.go @@ -27,8 +27,11 @@ func (node *Node) setupForValidator() { node.serviceManager.RegisterService(service.Consensus, consensus.New(node.BlockChannel, node.Consensus, node.startConsensus)) // Register new block service. node.serviceManager.RegisterService(service.BlockProposal, blockproposal.New(node.Consensus.ReadySignal, node.WaitForConsensusReadyV2)) - // Register client support service. - node.serviceManager.RegisterService(service.ClientSupport, clientsupport.New(node.Blockchain().State, node.CallFaucetContract, node.SelfPeer.IP, node.SelfPeer.Port)) + + if node.NodeConfig.GetNetworkType() != nodeconfig.Mainnet { + // Register client support service. + node.serviceManager.RegisterService(service.ClientSupport, clientsupport.New(node.Blockchain().State, node.CallFaucetContract, node.SelfPeer.IP, node.SelfPeer.Port)) + } // Register new metrics service if node.NodeConfig.GetMetricsFlag() { node.serviceManager.RegisterService(service.Metrics, metrics.New(&node.SelfPeer, node.NodeConfig.ConsensusPubKey.SerializeToHexStr(), node.NodeConfig.GetPushgatewayIP(), node.NodeConfig.GetPushgatewayPort())) From e410b13c70338e9cc3acb0f8e9e554bcd332fd87 Mon Sep 17 00:00:00 2001 From: flicker-harmony Date: Wed, 6 Nov 2019 19:18:39 +0300 Subject: [PATCH 34/45] Fix int overflow by int -> uint32 and uint64 conversion --- internal/hmyapi/transactionpool.go | 4 ++-- internal/hmyapi/util.go | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/hmyapi/transactionpool.go b/internal/hmyapi/transactionpool.go index f90122756..8f52f81f5 100644 --- a/internal/hmyapi/transactionpool.go +++ b/internal/hmyapi/transactionpool.go @@ -24,8 +24,8 @@ var ( // TxHistoryArgs is struct to make GetTransactionsHistory request type TxHistoryArgs struct { Address string `json:"address"` - PageIndex int `json:"pageIndex"` - PageSize int `json:"pageSize"` + PageIndex uint32 `json:"pageIndex"` + PageSize uint32 `json:"pageSize"` FullTx bool `json:"fullTx"` TxType string `json:"txType"` Order string `json:"order"` diff --git a/internal/hmyapi/util.go b/internal/hmyapi/util.go index b8fcd65d2..2fe5e0c86 100644 --- a/internal/hmyapi/util.go +++ b/internal/hmyapi/util.go @@ -13,7 +13,7 @@ import ( // defaultPageSize is to have default pagination. const ( - defaultPageSize = 100 + defaultPageSize = uint32(100) ) // ReturnWithPagination returns result with pagination (offset, page in TxHistoryArgs). @@ -23,10 +23,10 @@ func ReturnWithPagination(hashes []common.Hash, args TxHistoryArgs) []common.Has if args.PageSize > 0 { pageSize = args.PageSize } - if pageIndex < 0 || pageSize*pageIndex >= len(hashes) { + if uint64(pageSize)*uint64(pageIndex) >= uint64(len(hashes)) { return make([]common.Hash, 0) } - if pageSize*pageIndex+pageSize > len(hashes) { + if uint64(pageSize)*uint64(pageIndex)+uint64(pageSize) > uint64(len(hashes)) { return hashes[pageSize*pageIndex:] } return hashes[pageSize*pageIndex : pageSize*pageIndex+pageSize] From 085e820a1192c25b3d9a6d10a5b0cd9505ff4489 Mon Sep 17 00:00:00 2001 From: Rongjian Lan Date: Thu, 7 Nov 2019 14:10:40 -0800 Subject: [PATCH 35/45] Fix epoch change leader block proposal issue --- consensus/consensus.go | 3 --- consensus/consensus_service.go | 27 ++++++++++++++------------- consensus/consensus_v2.go | 7 +++++++ consensus/view_change.go | 1 + 4 files changed, 22 insertions(+), 16 deletions(-) diff --git a/consensus/consensus.go b/consensus/consensus.go index 458338a08..54e43cf32 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -79,9 +79,6 @@ type Consensus struct { // If the number of validators is less than minPeers, the consensus won't start MinPeers int - // Leader's address - leader p2p.Peer - CommitteePublicKeys map[string]bool pubKeyLock sync.Mutex diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index fd2b430c9..b8d9ea667 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -85,17 +85,6 @@ func (consensus *Consensus) signAndMarshalConsensusMessage(message *msg_pb.Messa return marshaledMessage, nil } -// SetLeaderPubKey deserialize the public key of consensus leader -func (consensus *Consensus) SetLeaderPubKey(k []byte) error { - consensus.leader.ConsensusPubKey = &bls.PublicKey{} - return consensus.leader.ConsensusPubKey.Deserialize(k) -} - -// GetLeaderPubKey returns the public key of consensus leader -func (consensus *Consensus) GetLeaderPubKey() *bls.PublicKey { - return consensus.leader.ConsensusPubKey -} - // GetNodeIDs returns Node IDs of all nodes in the same shard func (consensus *Consensus) GetNodeIDs() []libp2p_peer.ID { nodes := make([]libp2p_peer.ID, 0) @@ -131,8 +120,7 @@ func (consensus *Consensus) UpdatePublicKeys(pubKeys []*bls.PublicKey) int64 { utils.Logger().Info().Int("index", i).Str("BlsPubKey", pubKey).Msg("Member") consensus.CommitteePublicKeys[pubKey] = true } - // TODO: use pubkey to identify leader rather than p2p.Peer. - consensus.leader = p2p.Peer{ConsensusPubKey: pubKeys[0]} + consensus.LeaderPubKey = pubKeys[0] utils.Logger().Info(). Str("info", consensus.LeaderPubKey.SerializeToHexStr()).Msg("My Leader") @@ -498,6 +486,7 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode { } // update public keys committee + oldLeader := consensus.LeaderPubKey consensus.getLogger().Info(). Int("numPubKeys", len(pubKeys)). Msg("[UpdateConsensusInformation] Successfully updated public keys") @@ -525,6 +514,18 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode { if hasError { return Syncing } + + // If the leader changed and I myself become the leader + if !consensus.LeaderPubKey.IsEqual(oldLeader) && consensus.LeaderPubKey.IsEqual(consensus.PubKey) { + go func() { + utils.Logger().Debug(). + Str("myKey", consensus.PubKey.SerializeToHexStr()). + Uint64("viewID", consensus.viewID). + Uint64("block", consensus.blockNum). + Msg("[onEpochChange] I am the New Leader") + consensus.ReadySignal <- struct{}{} + }() + } return Normal } } diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 598226c56..8da4c5459 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -1026,6 +1026,7 @@ func (consensus *Consensus) tryCatchup() { } utils.Logger().Info().Msg("[TryCatchup] prepared message found to commit") + // TODO(Chao): Explain the reasoning for these code consensus.blockHash = [32]byte{} consensus.blockNum = consensus.blockNum + 1 consensus.viewID = msgs[0].ViewID + 1 @@ -1129,6 +1130,11 @@ func (consensus *Consensus) Start(blockChannel chan *types.Block, stopChan chan utils.Logger().Info().Msg("Node is out of sync") case newBlock := <-blockChannel: + // Debug code to trigger leader change. + //if consensus.ShardID == 0 && newBlock.NumberU64() == 2 && strings.Contains(consensus.PubKey.SerializeToHexStr(), "65f55eb") { + // continue + //} + utils.Logger().Info(). Uint64("MsgBlockNum", newBlock.NumberU64()). Msg("[ConsensusMainLoop] Received Proposed New Block!") @@ -1223,6 +1229,7 @@ func (consensus *Consensus) Start(blockChannel chan *types.Block, stopChan chan consensus.handleMessageUpdate(msg) case viewID := <-consensus.commitFinishChan: + // Only Leader execute this condition func() { consensus.mutex.Lock() defer consensus.mutex.Unlock() diff --git a/consensus/view_change.go b/consensus/view_change.go index dc43b44b6..db2aa8e91 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -355,6 +355,7 @@ func (consensus *Consensus) onViewChange(msg *msg_pb.Message) { consensus.LeaderPubKey = consensus.PubKey consensus.ResetState() if len(consensus.m1Payload) == 0 { + // TODO(Chao): explain why ReadySignal is sent only in this case but not the other case. go func() { consensus.ReadySignal <- struct{}{} }() From 3570e318708dcabbcefac0b969ff4787fa8417f4 Mon Sep 17 00:00:00 2001 From: Rongjian Lan Date: Thu, 7 Nov 2019 14:51:07 -0800 Subject: [PATCH 36/45] Fix validator at index 0 not showing BINGO at the epoch block --- node/node_handler.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/node/node_handler.go b/node/node_handler.go index 2041ccdd5..afd745607 100644 --- a/node/node_handler.go +++ b/node/node_handler.go @@ -315,8 +315,6 @@ func (node *Node) PostConsensusProcessing(newBlock *types.Block, commitSigAndBit Err(err). Msg("Error when adding new block") return - } else if core.IsEpochLastBlock(newBlock) { - node.Consensus.UpdateConsensusInformation() } // Update last consensus time for metrics @@ -349,6 +347,11 @@ func (node *Node) PostConsensusProcessing(newBlock *types.Block, commitSigAndBit // Broadcast client requested missing cross shard receipts if there is any node.BroadcastMissingCXReceipts() + // Update consensus keys at last so the change of leader don't mess up Z + if core.IsEpochLastBlock(newBlock) { + node.Consensus.UpdateConsensusInformation() + } + // TODO chao: uncomment this after beacon syncing is stable // node.Blockchain().UpdateCXReceiptsCheckpointsByBlock(newBlock) From ee593e321564798c1f7722245096abe361277327 Mon Sep 17 00:00:00 2001 From: Rongjian Lan Date: Thu, 7 Nov 2019 14:55:43 -0800 Subject: [PATCH 37/45] Fix comment --- node/node_handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/node_handler.go b/node/node_handler.go index afd745607..5551333bf 100644 --- a/node/node_handler.go +++ b/node/node_handler.go @@ -347,7 +347,7 @@ func (node *Node) PostConsensusProcessing(newBlock *types.Block, commitSigAndBit // Broadcast client requested missing cross shard receipts if there is any node.BroadcastMissingCXReceipts() - // Update consensus keys at last so the change of leader don't mess up Z + // Update consensus keys at last so the change of leader status doesn't mess up normal flow if core.IsEpochLastBlock(newBlock) { node.Consensus.UpdateConsensusInformation() } From 98532249b4d55bf7dc5655efc0d87507201603f7 Mon Sep 17 00:00:00 2001 From: Rongjian Lan Date: Thu, 7 Nov 2019 15:12:27 -0800 Subject: [PATCH 38/45] Reduce wait time for bootnode connection to 1 min --- api/service/networkinfo/service.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/service/networkinfo/service.go b/api/service/networkinfo/service.go index 0d87c298f..7501bac79 100644 --- a/api/service/networkinfo/service.go +++ b/api/service/networkinfo/service.go @@ -41,8 +41,8 @@ type Service struct { // ConnectionRetry set the number of retry of connection to bootnode in case the initial connection is failed var ( - // retry for 10 minutes and give up then - ConnectionRetry = 300 + // retry for 1 minutes and give up then + ConnectionRetry = 30 // context ctx context.Context From 812bbcda5c034157d955776413b88def12c830fd Mon Sep 17 00:00:00 2001 From: Rongjian Lan Date: Thu, 7 Nov 2019 15:15:19 -0800 Subject: [PATCH 39/45] Reduce wait time for bootnode connection to 30s --- api/service/networkinfo/service.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/service/networkinfo/service.go b/api/service/networkinfo/service.go index 7501bac79..eb437abd8 100644 --- a/api/service/networkinfo/service.go +++ b/api/service/networkinfo/service.go @@ -41,8 +41,8 @@ type Service struct { // ConnectionRetry set the number of retry of connection to bootnode in case the initial connection is failed var ( - // retry for 1 minutes and give up then - ConnectionRetry = 30 + // retry for 30s and give up then + ConnectionRetry = 15 // context ctx context.Context From e86f25a5b922ccabb91d44c72fb4ada709eb45a5 Mon Sep 17 00:00:00 2001 From: Rongjian Lan Date: Thu, 7 Nov 2019 16:04:21 -0800 Subject: [PATCH 40/45] Do not propose new block if I am not the leader --- node/node_newblock.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/node_newblock.go b/node/node_newblock.go index faeff3426..b47445201 100644 --- a/node/node_newblock.go +++ b/node/node_newblock.go @@ -42,7 +42,7 @@ func (node *Node) WaitForConsensusReadyV2(readySignal chan struct{}, stopChan ch Msg("Consensus new block proposal: STOPPED!") return case <-readySignal: - for { + for node.Consensus != nil && node.Consensus.IsLeader() { time.Sleep(PeriodicBlock) if time.Now().Before(deadline) { continue From 4e628224e66724d68f36639527b1de02c52e9fe2 Mon Sep 17 00:00:00 2001 From: Edgar Aroutiounian Date: Sat, 9 Nov 2019 15:05:19 -0800 Subject: [PATCH 41/45] [rpc] Fix mistake how nodemetadata gets ShardID, fix older networkType mistake (#1817) --- hmy/api_backend.go | 2 +- internal/configs/node/config.go | 1 + internal/hmyapi/harmony.go | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/hmy/api_backend.go b/hmy/api_backend.go index 4e583bc73..f22857630 100644 --- a/hmy/api_backend.go +++ b/hmy/api_backend.go @@ -236,7 +236,7 @@ func (b *APIBackend) RPCGasCap() *big.Int { return b.hmy.RPCGasCap // TODO(ricl): should be hmy.config.RPCGasCap } -// GetShardID returns the gas cap of rpc +// GetShardID returns shardID of this node func (b *APIBackend) GetShardID() uint32 { return b.hmy.shardID } diff --git a/internal/configs/node/config.go b/internal/configs/node/config.go index 1744011fc..d633a1790 100644 --- a/internal/configs/node/config.go +++ b/internal/configs/node/config.go @@ -224,6 +224,7 @@ func (conf *ConfigType) Role() Role { // SetNetworkType set the networkType func SetNetworkType(networkType NetworkType) { + defaultConfig.networkType = networkType for i := range shardConfigs { shardConfigs[i].networkType = networkType } diff --git a/internal/hmyapi/harmony.go b/internal/hmyapi/harmony.go index 5dc544a23..51f26cbb3 100644 --- a/internal/hmyapi/harmony.go +++ b/internal/hmyapi/harmony.go @@ -53,7 +53,7 @@ type NodeMetadata struct { ShardID uint32 `json:"shard-id"` } -// GetNodeMetadata produces a NodeMetadata record. Note the data is from the answering RPC +// GetNodeMetadata produces a NodeMetadata record, data is from the answering RPC node func (s *PublicHarmonyAPI) GetNodeMetadata() NodeMetadata { cfg := nodeconfig.GetDefaultConfig() return NodeMetadata{ @@ -62,6 +62,6 @@ func (s *PublicHarmonyAPI) GetNodeMetadata() NodeMetadata { string(cfg.GetNetworkType()), s.b.ChainConfig().ChainID.String(), s.b.IsLeader(), - cfg.GetShardID(), + s.b.GetShardID(), } } From e6a4fbea4f60201df528597e3f04b534a1d98b45 Mon Sep 17 00:00:00 2001 From: Edgar Aroutiounian Date: Sun, 10 Nov 2019 20:57:29 -0800 Subject: [PATCH 42/45] [committee] Factor out committee membership, provide entry for alternative committee membership (#1818) * [committee] Move core.ShardingSchedule to shard.Schedule * [consensus] Remove redundant PublicKeys field of Consensus as Decider maintains that * [committee] Use committee package to pick PublicKeys * [committee] Use committee inplace of CalculateShardState * [committee] Remove core/resharding.go, complete usage of committee as implementation replacement * [committee] Address PR comments --- api/service/resharding/service.go | 97 -------- cmd/client/txgen/main.go | 17 +- cmd/harmony/main.go | 47 ++-- consensus/consensus.go | 3 - consensus/consensus_leader_msg_test.go | 6 +- consensus/consensus_service.go | 39 ++-- consensus/consensus_service_test.go | 8 +- consensus/consensus_test.go | 4 +- consensus/consensus_v2.go | 8 +- consensus/consensus_validator_msg_test.go | 6 +- consensus/fbft_log_test.go | 4 +- core/blockchain.go | 39 ++-- core/core_test.go | 3 +- core/genesis.go | 5 + core/resharding.go | 259 --------------------- core/resharding.md | 12 - core/resharding_test.go | 149 ------------ core/state_processor.go | 3 +- core/values/blockchain.go | 10 - drand/drand_leader.go | 6 +- internal/chain/engine.go | 7 +- internal/configs/sharding/localnet.go | 13 +- internal/hmyapi/blockchain.go | 5 +- internal/params/config.go | 2 +- node/node.go | 41 ++-- node/node_cross_shard.go | 10 +- node/node_explorer.go | 3 +- node/node_genesis.go | 7 +- node/node_handler.go | 70 +----- node/node_handler_test.go | 6 +- node/node_newblock.go | 21 +- node/node_resharding.go | 15 +- node/node_test.go | 8 +- node/worker/worker.go | 12 +- shard/committee/assignment.go | 261 ++++++++++++++++++++++ shard/shard_state.go | 76 +++++-- shard/shard_state_test.go | 36 +-- shard/values.go | 19 ++ 38 files changed, 534 insertions(+), 803 deletions(-) delete mode 100644 api/service/resharding/service.go delete mode 100644 core/resharding.go delete mode 100644 core/resharding.md delete mode 100644 core/resharding_test.go delete mode 100644 core/values/blockchain.go create mode 100644 shard/committee/assignment.go create mode 100644 shard/values.go diff --git a/api/service/resharding/service.go b/api/service/resharding/service.go deleted file mode 100644 index 18558ab39..000000000 --- a/api/service/resharding/service.go +++ /dev/null @@ -1,97 +0,0 @@ -package resharding - -import ( - "time" - - "github.com/ethereum/go-ethereum/rpc" - msg_pb "github.com/harmony-one/harmony/api/proto/message" - "github.com/harmony-one/harmony/core" - "github.com/harmony-one/harmony/internal/utils" -) - -// Constants for resharding service. -const ( - ReshardingCheckTime = time.Second -) - -// Service is the role conversion service. -type Service struct { - stopChan chan struct{} - stoppedChan chan struct{} - messageChan chan *msg_pb.Message - beaconChain *core.BlockChain -} - -// New returns role conversion service. -func New(beaconChain *core.BlockChain) *Service { - return &Service{beaconChain: beaconChain} -} - -// StartService starts role conversion service. -func (s *Service) StartService() { - s.stopChan = make(chan struct{}) - s.stoppedChan = make(chan struct{}) - - s.Init() - s.Run(s.stopChan, s.stoppedChan) -} - -// Init initializes role conversion service. -func (s *Service) Init() { -} - -// Run runs role conversion. -func (s *Service) Run(stopChan chan struct{}, stoppedChan chan struct{}) { - go func() { - defer close(stoppedChan) - for { - select { - default: - utils.Logger().Info().Msg("Running role conversion") - // TODO: Write some logic here. - s.DoService() - case <-stopChan: - return - } - } - }() -} - -// DoService does role conversion. -func (s *Service) DoService() { - tick := time.NewTicker(ReshardingCheckTime) - // Get current shard state hash. - currentShardStateHash := s.beaconChain.CurrentBlock().Header().ShardStateHash() - for { - select { - case <-tick.C: - LatestShardStateHash := s.beaconChain.CurrentBlock().Header().ShardStateHash() - if currentShardStateHash != LatestShardStateHash { - // TODO(minhdoan): Add resharding logic later after modifying the resharding func as it current doesn't calculate the role (leader/validator) - } - } - } -} - -// StopService stops role conversion service. -func (s *Service) StopService() { - utils.Logger().Info().Msg("Stopping role conversion service") - s.stopChan <- struct{}{} - <-s.stoppedChan - utils.Logger().Info().Msg("Role conversion stopped") -} - -// NotifyService notify service -func (s *Service) NotifyService(params map[string]interface{}) { - return -} - -// SetMessageChan sets up message channel to service. -func (s *Service) SetMessageChan(messageChan chan *msg_pb.Message) { - s.messageChan = messageChan -} - -// APIs for the services. -func (s *Service) APIs() []rpc.API { - return nil -} diff --git a/cmd/client/txgen/main.go b/cmd/client/txgen/main.go index 5363d193d..fe8b9b4c6 100644 --- a/cmd/client/txgen/main.go +++ b/cmd/client/txgen/main.go @@ -10,29 +10,28 @@ import ( "sync" "time" - "github.com/harmony-one/harmony/consensus" - "github.com/harmony-one/harmony/consensus/quorum" - "github.com/harmony-one/harmony/core" - "github.com/harmony-one/harmony/internal/ctxerror" - "github.com/harmony-one/harmony/internal/shardchain" - "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" bls2 "github.com/harmony-one/bls/ffi/go/bls" - "github.com/harmony-one/harmony/internal/params" - "github.com/harmony-one/harmony/api/client" proto_node "github.com/harmony-one/harmony/api/proto/node" "github.com/harmony-one/harmony/common/denominations" + "github.com/harmony-one/harmony/consensus" + "github.com/harmony-one/harmony/consensus/quorum" + "github.com/harmony-one/harmony/core" "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/crypto/bls" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" + "github.com/harmony-one/harmony/internal/ctxerror" "github.com/harmony-one/harmony/internal/genesis" + "github.com/harmony-one/harmony/internal/params" + "github.com/harmony-one/harmony/internal/shardchain" "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/node" "github.com/harmony-one/harmony/p2p" p2p_host "github.com/harmony-one/harmony/p2p/host" "github.com/harmony-one/harmony/p2p/p2pimpl" + "github.com/harmony-one/harmony/shard" ) var ( @@ -105,7 +104,7 @@ func setUpTXGen() *node.Node { txGen := node.New(myhost, consensusObj, chainDBFactory, false) //Changed it : no longer archival node. txGen.Client = client.NewClient(txGen.GetHost(), uint32(shardID)) consensusObj.ChainReader = txGen.Blockchain() - genesisShardingConfig := core.ShardingSchedule.InstanceForEpoch(big.NewInt(core.GenesisEpoch)) + genesisShardingConfig := shard.Schedule.InstanceForEpoch(big.NewInt(core.GenesisEpoch)) startIdx := 0 endIdx := startIdx + genesisShardingConfig.NumNodesPerShard() pubs := []*bls2.PublicKey{} diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index 5f890c3f6..efe440d8e 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -16,7 +16,6 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/harmony-one/bls/ffi/go/bls" - "github.com/harmony-one/harmony/api/service/syncing" "github.com/harmony-one/harmony/consensus" "github.com/harmony-one/harmony/consensus/quorum" @@ -34,6 +33,7 @@ import ( "github.com/harmony-one/harmony/node" "github.com/harmony-one/harmony/p2p" "github.com/harmony-one/harmony/p2p/p2pimpl" + "github.com/harmony-one/harmony/shard" ) // Version string variables @@ -71,7 +71,6 @@ func printVersion() { os.Exit(0) } -// Flags var ( ip = flag.String("ip", "127.0.0.1", "ip of the node") port = flag.String("port", "9000", "port of the node.") @@ -102,7 +101,6 @@ var ( syncFreq = flag.Int("sync_freq", 60, "unit in seconds") // beaconSyncFreq indicates beaconchain sync frequency beaconSyncFreq = flag.Int("beacon_sync_freq", 60, "unit in seconds") - // blockPeriod indicates the how long the leader waits to propose a new block. blockPeriod = flag.Int("block_period", 8, "how long in second the leader waits to propose a new block.") leaderOverride = flag.Bool("leader_override", false, "true means override the default leader role and acts as validator") @@ -113,34 +111,25 @@ var ( blsKeyFile = flag.String("blskey_file", "", "The encrypted file of bls serialized private key by passphrase.") blsPass = flag.String("blspass", "", "The file containing passphrase to decrypt the encrypted bls file.") blsPassphrase string - // Sharding configuration parameters for devnet devnetNumShards = flag.Uint("dn_num_shards", 2, "number of shards for -network_type=devnet (default: 2)") devnetShardSize = flag.Int("dn_shard_size", 10, "number of nodes per shard for -network_type=devnet (default 10)") devnetHarmonySize = flag.Int("dn_hmy_size", -1, "number of Harmony-operated nodes per shard for -network_type=devnet; negative (default) means equal to -dn_shard_size") - // logConn logs incoming/outgoing connections - logConn = flag.Bool("log_conn", false, "log incoming/outgoing connections") - - keystoreDir = flag.String("keystore", hmykey.DefaultKeyStoreDir, "The default keystore directory") - + logConn = flag.Bool("log_conn", false, "log incoming/outgoing connections") + keystoreDir = flag.String("keystore", hmykey.DefaultKeyStoreDir, "The default keystore directory") initialAccount = &genesis.DeployAccount{} - // logging verbosity verbosity = flag.Int("verbosity", 5, "Logging verbosity: 0=silent, 1=error, 2=warn, 3=info, 4=debug, 5=detail (default: 5)") - // dbDir is the database directory. dbDir = flag.String("db_dir", "", "blockchain database directory") - // Disable view change. disableViewChange = flag.Bool("disable_view_change", false, "Do not propose view change (testing only)") - // metrics flag to collct meetrics or not, pushgateway ip and port for metrics metricsFlag = flag.Bool("metrics", false, "Collect and upload node metrics") pushgatewayIP = flag.String("pushgateway_ip", "grafana.harmony.one", "Metrics view ip") pushgatewayPort = flag.String("pushgateway_port", "9091", "Metrics view port") - - publicRPC = flag.Bool("public_rpc", false, "Enable Public RPC Access (default: false)") + publicRPC = flag.Bool("public_rpc", false, "Enable Public RPC Access (default: false)") ) func initSetup() { @@ -203,13 +192,13 @@ func passphraseForBls() { } func setupInitialAccount() (isLeader bool) { - genesisShardingConfig := core.ShardingSchedule.InstanceForEpoch(big.NewInt(core.GenesisEpoch)) + genesisShardingConfig := shard.Schedule.InstanceForEpoch(big.NewInt(core.GenesisEpoch)) pubKey := setupConsensusKey(nodeconfig.GetDefaultConfig()) reshardingEpoch := genesisShardingConfig.ReshardingEpoch() if reshardingEpoch != nil && len(reshardingEpoch) > 0 { for _, epoch := range reshardingEpoch { - config := core.ShardingSchedule.InstanceForEpoch(epoch) + config := shard.Schedule.InstanceForEpoch(epoch) isLeader, initialAccount = config.FindAccount(pubKey.SerializeToHexStr()) if initialAccount != nil { break @@ -323,7 +312,7 @@ func setupConsensusAndNode(nodeConfig *nodeconfig.ConfigType) *node.Node { switch { case *networkType == nodeconfig.Localnet: - epochConfig := core.ShardingSchedule.InstanceForEpoch(ethCommon.Big0) + epochConfig := shard.Schedule.InstanceForEpoch(ethCommon.Big0) selfPort, err := strconv.ParseUint(*port, 10, 16) if err != nil { utils.Logger().Fatal(). @@ -359,9 +348,9 @@ func setupConsensusAndNode(nodeConfig *nodeconfig.ConfigType) *node.Node { currentNode.NodeConfig.SetClientGroupID(nodeconfig.NewClientGroupIDByShardID(nodeconfig.ShardID(*shardID))) case "validator": currentNode.NodeConfig.SetRole(nodeconfig.Validator) - if nodeConfig.ShardID == 0 { - currentNode.NodeConfig.SetShardGroupID(nodeconfig.NewGroupIDByShardID(0)) - currentNode.NodeConfig.SetClientGroupID(nodeconfig.NewClientGroupIDByShardID(0)) + if nodeConfig.ShardID == shard.BeaconChainShardID { + currentNode.NodeConfig.SetShardGroupID(nodeconfig.NewGroupIDByShardID(shard.BeaconChainShardID)) + currentNode.NodeConfig.SetClientGroupID(nodeconfig.NewClientGroupIDByShardID(shard.BeaconChainShardID)) } else { currentNode.NodeConfig.SetShardGroupID(nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(nodeConfig.ShardID))) currentNode.NodeConfig.SetClientGroupID(nodeconfig.NewClientGroupIDByShardID(nodeconfig.ShardID(nodeConfig.ShardID))) @@ -381,8 +370,8 @@ func setupConsensusAndNode(nodeConfig *nodeconfig.ConfigType) *node.Node { // currentNode.DRand = dRand // This needs to be executed after consensus and drand are setup - if err := currentNode.CalculateInitShardState(); err != nil { - ctxerror.Crit(utils.GetLogger(), err, "CalculateInitShardState failed", + if err := currentNode.InitConsensusWithValidators(); err != nil { + ctxerror.Crit(utils.GetLogger(), err, "InitConsensusWithMembers failed", "shardID", *shardID) } @@ -429,13 +418,13 @@ func main() { switch *networkType { case nodeconfig.Mainnet: - core.ShardingSchedule = shardingconfig.MainnetSchedule + shard.Schedule = shardingconfig.MainnetSchedule case nodeconfig.Testnet: - core.ShardingSchedule = shardingconfig.TestnetSchedule + shard.Schedule = shardingconfig.TestnetSchedule case nodeconfig.Pangaea: - core.ShardingSchedule = shardingconfig.PangaeaSchedule + shard.Schedule = shardingconfig.PangaeaSchedule case nodeconfig.Localnet: - core.ShardingSchedule = shardingconfig.LocalnetSchedule + shard.Schedule = shardingconfig.LocalnetSchedule case nodeconfig.Devnet: if *devnetHarmonySize < 0 { *devnetHarmonySize = *devnetShardSize @@ -448,7 +437,7 @@ func main() { err) os.Exit(1) } - core.ShardingSchedule = shardingconfig.NewFixedSchedule(devnetConfig) + shard.Schedule = shardingconfig.NewFixedSchedule(devnetConfig) } initSetup() @@ -476,7 +465,7 @@ func main() { currentNode.SetSyncFreq(*syncFreq) currentNode.SetBeaconSyncFreq(*beaconSyncFreq) - if nodeConfig.ShardID != 0 && currentNode.NodeConfig.Role() != nodeconfig.ExplorerNode { + if nodeConfig.ShardID != shard.BeaconChainShardID && currentNode.NodeConfig.Role() != nodeconfig.ExplorerNode { utils.GetLogInstance().Info("SupportBeaconSyncing", "shardID", currentNode.Blockchain().ShardID(), "shardID", nodeConfig.ShardID) go currentNode.SupportBeaconSyncing() } diff --git a/consensus/consensus.go b/consensus/consensus.go index 54e43cf32..fc59d5a2c 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -79,8 +79,6 @@ type Consensus struct { // If the number of validators is less than minPeers, the consensus won't start MinPeers int - CommitteePublicKeys map[string]bool - pubKeyLock sync.Mutex // private/public keys of current node @@ -216,7 +214,6 @@ func New( consensus.current = State{mode: Normal} // FBFT timeout consensus.consensusTimeout = createTimeout() - consensus.CommitteePublicKeys = make(map[string]bool) consensus.validators.Store(leader.ConsensusPubKey.SerializeToHexStr(), leader) if blsPriKey != nil { diff --git a/consensus/consensus_leader_msg_test.go b/consensus/consensus_leader_msg_test.go index 22f5e81ae..9abff5be1 100644 --- a/consensus/consensus_leader_msg_test.go +++ b/consensus/consensus_leader_msg_test.go @@ -7,12 +7,12 @@ import ( "github.com/harmony-one/harmony/api/proto" msg_pb "github.com/harmony-one/harmony/api/proto/message" "github.com/harmony-one/harmony/consensus/quorum" - "github.com/harmony-one/harmony/core/values" "github.com/harmony-one/harmony/crypto/bls" "github.com/harmony-one/harmony/internal/ctxerror" "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/p2p" "github.com/harmony-one/harmony/p2p/p2pimpl" + "github.com/harmony-one/harmony/shard" ) func TestConstructAnnounceMessage(test *testing.T) { @@ -24,7 +24,7 @@ func TestConstructAnnounceMessage(test *testing.T) { } decider := quorum.NewDecider(quorum.SuperMajorityVote) consensus, err := New( - host, values.BeaconChainShardID, leader, bls.RandPrivateKey(), decider, + host, shard.BeaconChainShardID, leader, bls.RandPrivateKey(), decider, ) if err != nil { test.Fatalf("Cannot create consensus: %v", err) @@ -57,7 +57,7 @@ func TestConstructPreparedMessage(test *testing.T) { } decider := quorum.NewDecider(quorum.SuperMajorityVote) consensus, err := New( - host, values.BeaconChainShardID, leader, bls.RandPrivateKey(), decider, + host, shard.BeaconChainShardID, leader, bls.RandPrivateKey(), decider, ) if err != nil { test.Fatalf("Cannot craeate consensus: %v", err) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index b8d9ea667..2f5bba52a 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -14,7 +14,6 @@ import ( "github.com/harmony-one/harmony/block" consensus_engine "github.com/harmony-one/harmony/consensus/engine" "github.com/harmony-one/harmony/consensus/quorum" - "github.com/harmony-one/harmony/core" "github.com/harmony-one/harmony/core/types" bls_cosi "github.com/harmony-one/harmony/crypto/bls" "github.com/harmony-one/harmony/crypto/hash" @@ -23,6 +22,8 @@ import ( "github.com/harmony-one/harmony/internal/profiler" "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/p2p" + "github.com/harmony-one/harmony/shard" + "github.com/harmony-one/harmony/shard/committee" libp2p_peer "github.com/libp2p/go-libp2p-peer" "github.com/rs/zerolog" ) @@ -110,17 +111,14 @@ func (consensus *Consensus) DebugPrintPublicKeys() { utils.Logger().Debug().Strs("PublicKeys", keys).Int("count", len(keys)).Msgf("Debug Public Keys") } -// UpdatePublicKeys updates the PublicKeys variable, protected by a mutex +// UpdatePublicKeys updates the PublicKeys for quorum on current subcommittee, protected by a mutex func (consensus *Consensus) UpdatePublicKeys(pubKeys []*bls.PublicKey) int64 { consensus.pubKeyLock.Lock() consensus.Decider.UpdateParticipants(pubKeys) - consensus.CommitteePublicKeys = map[string]bool{} utils.Logger().Info().Msg("My Committee updated") - for i, pubKey := range consensus.Decider.DumpParticipants() { - utils.Logger().Info().Int("index", i).Str("BlsPubKey", pubKey).Msg("Member") - consensus.CommitteePublicKeys[pubKey] = true + for i := range pubKeys { + utils.Logger().Info().Int("index", i).Str("BLSPubKey", pubKeys[i].SerializeToHexStr()).Msg("Member") } - consensus.LeaderPubKey = pubKeys[0] utils.Logger().Info(). Str("info", consensus.LeaderPubKey.SerializeToHexStr()).Msg("My Leader") @@ -230,8 +228,7 @@ func (consensus *Consensus) ToggleConsensusCheck() { // IsValidatorInCommittee returns whether the given validator BLS address is part of my committee func (consensus *Consensus) IsValidatorInCommittee(pubKey *bls.PublicKey) bool { - _, ok := consensus.CommitteePublicKeys[pubKey.SerializeToHexStr()] - return ok + return consensus.Decider.IndexOf(pubKey) != -1 } // Verify the signature of the message are valid from the signer's public key. @@ -458,22 +455,21 @@ func (consensus *Consensus) getLeaderPubKeyFromCoinbase(header *block.Header) (* func (consensus *Consensus) UpdateConsensusInformation() Mode { pubKeys := []*bls.PublicKey{} hasError := false - header := consensus.ChainReader.CurrentHeader() - epoch := header.Epoch() - curPubKeys := core.CalculatePublicKeys(epoch, header.ShardID()) + _, curPubKeys := committee.WithStakingEnabled.ComputePublicKeys( + epoch, consensus.ChainReader, int(header.ShardID()), + ) consensus.numPrevPubKeys = len(curPubKeys) - consensus.getLogger().Info().Msg("[UpdateConsensusInformation] Updating.....") - - if core.IsEpochLastBlockByHeader(header) { + if shard.Schedule.IsLastBlock(header.Number().Uint64()) { // increase epoch by one if it's the last block consensus.SetEpochNum(epoch.Uint64() + 1) consensus.getLogger().Info().Uint64("headerNum", header.Number().Uint64()). Msg("[UpdateConsensusInformation] Epoch updated for next epoch") - nextEpoch := new(big.Int).Add(epoch, common.Big1) - pubKeys = core.CalculatePublicKeys(nextEpoch, header.ShardID()) + _, pubKeys = committee.WithStakingEnabled.ComputePublicKeys( + new(big.Int).Add(epoch, common.Big1), consensus.ChainReader, int(header.ShardID()), + ) } else { consensus.SetEpochNum(epoch.Uint64()) pubKeys = curPubKeys @@ -493,7 +489,8 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode { consensus.UpdatePublicKeys(pubKeys) // take care of possible leader change during the epoch - if !core.IsEpochLastBlockByHeader(header) && header.Number().Uint64() != 0 { + if !shard.Schedule.IsLastBlock(header.Number().Uint64()) && + header.Number().Uint64() != 0 { leaderPubKey, err := consensus.getLeaderPubKeyFromCoinbase(header) if err != nil || leaderPubKey == nil { consensus.getLogger().Debug().Err(err). @@ -508,9 +505,9 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode { } } - for _, key := range pubKeys { + for i := range pubKeys { // in committee - if key.IsEqual(consensus.PubKey) { + if pubKeys[i].IsEqual(consensus.PubKey) { if hasError { return Syncing } @@ -544,7 +541,7 @@ func (consensus *Consensus) IsLeader() bool { // NeedsRandomNumberGeneration returns true if the current epoch needs random number generation func (consensus *Consensus) NeedsRandomNumberGeneration(epoch *big.Int) bool { - if consensus.ShardID == 0 && epoch.Uint64() >= core.ShardingSchedule.RandomnessStartingEpoch() { + if consensus.ShardID == 0 && epoch.Uint64() >= shard.Schedule.RandomnessStartingEpoch() { return true } diff --git a/consensus/consensus_service_test.go b/consensus/consensus_service_test.go index 36673a1f6..ae5042f5d 100644 --- a/consensus/consensus_service_test.go +++ b/consensus/consensus_service_test.go @@ -6,11 +6,11 @@ import ( msg_pb "github.com/harmony-one/harmony/api/proto/message" "github.com/harmony-one/harmony/consensus/quorum" - "github.com/harmony-one/harmony/core/values" "github.com/harmony-one/harmony/crypto/bls" "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/p2p" "github.com/harmony-one/harmony/p2p/p2pimpl" + "github.com/harmony-one/harmony/shard" ) func TestPopulateMessageFields(t *testing.T) { @@ -23,7 +23,7 @@ func TestPopulateMessageFields(t *testing.T) { blsPriKey := bls.RandPrivateKey() decider := quorum.NewDecider(quorum.SuperMajorityVote) consensus, err := New( - host, values.BeaconChainShardID, leader, blsPriKey, decider, + host, shard.BeaconChainShardID, leader, blsPriKey, decider, ) if err != nil { t.Fatalf("Cannot craeate consensus: %v", err) @@ -60,7 +60,7 @@ func TestSignAndMarshalConsensusMessage(t *testing.T) { } decider := quorum.NewDecider(quorum.SuperMajorityVote) consensus, err := New( - host, values.BeaconChainShardID, leader, bls.RandPrivateKey(), decider, + host, shard.BeaconChainShardID, leader, bls.RandPrivateKey(), decider, ) if err != nil { t.Fatalf("Cannot craeate consensus: %v", err) @@ -88,7 +88,7 @@ func TestSetViewID(t *testing.T) { } decider := quorum.NewDecider(quorum.SuperMajorityVote) consensus, err := New( - host, values.BeaconChainShardID, leader, bls.RandPrivateKey(), decider, + host, shard.BeaconChainShardID, leader, bls.RandPrivateKey(), decider, ) if err != nil { t.Fatalf("Cannot craeate consensus: %v", err) diff --git a/consensus/consensus_test.go b/consensus/consensus_test.go index a67002d6c..87a1ee11f 100644 --- a/consensus/consensus_test.go +++ b/consensus/consensus_test.go @@ -4,11 +4,11 @@ import ( "testing" "github.com/harmony-one/harmony/consensus/quorum" - "github.com/harmony-one/harmony/core/values" "github.com/harmony-one/harmony/crypto/bls" "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/p2p" "github.com/harmony-one/harmony/p2p/p2pimpl" + "github.com/harmony-one/harmony/shard" ) func TestNew(test *testing.T) { @@ -20,7 +20,7 @@ func TestNew(test *testing.T) { } decider := quorum.NewDecider(quorum.SuperMajorityVote) consensus, err := New( - host, values.BeaconChainShardID, leader, bls.RandPrivateKey(), decider, + host, shard.BeaconChainShardID, leader, bls.RandPrivateKey(), decider, ) if err != nil { test.Fatalf("Cannot craeate consensus: %v", err) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 8da4c5459..cc917476c 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -14,7 +14,6 @@ import ( msg_pb "github.com/harmony-one/harmony/api/proto/message" "github.com/harmony-one/harmony/block" "github.com/harmony-one/harmony/consensus/quorum" - "github.com/harmony-one/harmony/core" "github.com/harmony-one/harmony/core/types" vrf_bls "github.com/harmony-one/harmony/crypto/vrf/bls" "github.com/harmony-one/harmony/internal/chain" @@ -22,6 +21,7 @@ import ( "github.com/harmony-one/harmony/internal/ctxerror" "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/p2p/host" + "github.com/harmony-one/harmony/shard" "github.com/harmony-one/vdf/src/vdf_go" ) @@ -1187,7 +1187,7 @@ func (consensus *Consensus) Start(blockChannel chan *types.Block, stopChan chan if err == nil { vdfInProgress = false // Verify the randomness - vdfObject := vdf_go.New(core.ShardingSchedule.VdfDifficulty(), seed) + vdfObject := vdf_go.New(shard.Schedule.VdfDifficulty(), seed) if !vdfObject.Verify(vdfOutput) { consensus.getLogger().Warn(). Uint64("MsgBlockNum", newBlock.NumberU64()). @@ -1323,7 +1323,7 @@ func (consensus *Consensus) GenerateVdfAndProof(newBlock *types.Block, vrfBlockN // TODO ek – limit concurrency go func() { - vdf := vdf_go.New(core.ShardingSchedule.VdfDifficulty(), seed) + vdf := vdf_go.New(shard.Schedule.VdfDifficulty(), seed) outputChannel := vdf.GetOutputChannel() start := time.Now() vdf.Execute() @@ -1364,7 +1364,7 @@ func (consensus *Consensus) ValidateVdfAndProof(headerObj *block.Header) bool { } } - vdfObject := vdf_go.New(core.ShardingSchedule.VdfDifficulty(), seed) + vdfObject := vdf_go.New(shard.Schedule.VdfDifficulty(), seed) vdfOutput := [516]byte{} copy(vdfOutput[:], headerObj.Vdf()) if vdfObject.Verify(vdfOutput) { diff --git a/consensus/consensus_validator_msg_test.go b/consensus/consensus_validator_msg_test.go index 0bb6658b1..4a8305777 100644 --- a/consensus/consensus_validator_msg_test.go +++ b/consensus/consensus_validator_msg_test.go @@ -7,11 +7,11 @@ import ( "github.com/harmony-one/harmony/api/proto" msg_pb "github.com/harmony-one/harmony/api/proto/message" "github.com/harmony-one/harmony/consensus/quorum" - "github.com/harmony-one/harmony/core/values" "github.com/harmony-one/harmony/crypto/bls" "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/p2p" "github.com/harmony-one/harmony/p2p/p2pimpl" + "github.com/harmony-one/harmony/shard" ) func TestConstructPrepareMessage(test *testing.T) { @@ -23,7 +23,7 @@ func TestConstructPrepareMessage(test *testing.T) { } decider := quorum.NewDecider(quorum.SuperMajorityVote) consensus, err := New( - host, values.BeaconChainShardID, leader, bls.RandPrivateKey(), decider, + host, shard.BeaconChainShardID, leader, bls.RandPrivateKey(), decider, ) if err != nil { test.Fatalf("Cannot craeate consensus: %v", err) @@ -54,7 +54,7 @@ func TestConstructCommitMessage(test *testing.T) { } decider := quorum.NewDecider(quorum.SuperMajorityVote) consensus, err := New( - host, values.BeaconChainShardID, leader, bls.RandPrivateKey(), decider, + host, shard.BeaconChainShardID, leader, bls.RandPrivateKey(), decider, ) if err != nil { test.Fatalf("Cannot craeate consensus: %v", err) diff --git a/consensus/fbft_log_test.go b/consensus/fbft_log_test.go index bcf390ea9..a3f545ab1 100644 --- a/consensus/fbft_log_test.go +++ b/consensus/fbft_log_test.go @@ -7,11 +7,11 @@ import ( "github.com/harmony-one/harmony/api/proto" msg_pb "github.com/harmony-one/harmony/api/proto/message" "github.com/harmony-one/harmony/consensus/quorum" - "github.com/harmony-one/harmony/core/values" "github.com/harmony-one/harmony/crypto/bls" "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/p2p" "github.com/harmony-one/harmony/p2p/p2pimpl" + "github.com/harmony-one/harmony/shard" ) func constructAnnounceMessage(t *testing.T) []byte { @@ -23,7 +23,7 @@ func constructAnnounceMessage(t *testing.T) []byte { } decider := quorum.NewDecider(quorum.SuperMajorityVote) consensus, err := New( - host, values.BeaconChainShardID, leader, bls.RandPrivateKey(), decider, + host, shard.BeaconChainShardID, leader, bls.RandPrivateKey(), decider, ) if err != nil { t.Fatalf("Cannot create consensus: %v", err) diff --git a/core/blockchain.go b/core/blockchain.go index 7d4af6818..33fb29f92 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -45,6 +45,7 @@ import ( "github.com/harmony-one/harmony/internal/params" "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/shard" + "github.com/harmony-one/harmony/shard/committee" staking "github.com/harmony-one/harmony/staking/types" lru "github.com/hashicorp/golang-lru" ) @@ -246,27 +247,16 @@ func IsEpochBlock(block *types.Block) bool { // genesis block is the first epoch block return true } - return ShardingSchedule.IsLastBlock(block.NumberU64() - 1) + return shard.Schedule.IsLastBlock(block.NumberU64() - 1) } // EpochFirstBlock returns the block number of the first block of an epoch. // TODO: instead of using fixed epoch schedules, determine the first block by epoch changes. func EpochFirstBlock(epoch *big.Int) *big.Int { - if epoch.Cmp(big.NewInt(0)) == 0 { - return big.NewInt(0) + if epoch.Cmp(big.NewInt(GenesisEpoch)) == 0 { + return big.NewInt(GenesisEpoch) } - return big.NewInt(int64(ShardingSchedule.EpochLastBlock(epoch.Uint64()-1) + 1)) -} - -// IsEpochLastBlock returns whether this block is the last block of an epoch. -func IsEpochLastBlock(block *types.Block) bool { - return ShardingSchedule.IsLastBlock(block.NumberU64()) -} - -// IsEpochLastBlockByHeader returns whether this block is the last block of an epoch -// given block header -func IsEpochLastBlockByHeader(header *block.Header) bool { - return ShardingSchedule.IsLastBlock(header.Number().Uint64()) + return big.NewInt(int64(shard.Schedule.EpochLastBlock(epoch.Uint64()-1) + 1)) } func (bc *BlockChain) getProcInterrupt() bool { @@ -1083,7 +1073,7 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types. epoch := block.Header().Epoch() if bc.chainConfig.IsCrossTx(block.Epoch()) { - shardingConfig := ShardingSchedule.InstanceForEpoch(epoch) + shardingConfig := shard.Schedule.InstanceForEpoch(epoch) shardNum := int(shardingConfig.NumShards()) for i := 0; i < shardNum; i++ { if i == int(block.ShardID()) { @@ -1943,7 +1933,18 @@ func (bc *BlockChain) GetShardState(epoch *big.Int) (shard.State, error) { if err == nil { // TODO ek – distinguish ErrNotFound return shardState, err } - shardState, err = CalculateNewShardState(bc, epoch) + + if epoch.Cmp(big.NewInt(GenesisEpoch)) == 0 { + shardState, err = committee.WithStakingEnabled.ReadFromComputation( + big.NewInt(GenesisEpoch), *bc.Config(), nil, + ) + } else { + prevEpoch := new(big.Int).Sub(epoch, common.Big1) + shardState, err = committee.WithStakingEnabled.ReadFromChain( + prevEpoch, bc, + ) + } + if err != nil { return nil, err } @@ -2164,7 +2165,7 @@ func (bc *BlockChain) CXMerkleProof(shardID uint32, block *types.Block) (*types. } epoch := block.Header().Epoch() - shardingConfig := ShardingSchedule.InstanceForEpoch(epoch) + shardingConfig := shard.Schedule.InstanceForEpoch(epoch) shardNum := int(shardingConfig.NumShards()) for i := 0; i < shardNum; i++ { @@ -2384,7 +2385,7 @@ func (bc *BlockChain) CurrentValidatorAddresses() []common.Address { if err != nil { continue } - epoch := ShardingSchedule.CalcEpochNumber(val.CreationHeight.Uint64()) + epoch := shard.Schedule.CalcEpochNumber(val.CreationHeight.Uint64()) if epoch.Cmp(currentEpoch) >= 0 { // wait for next epoch continue diff --git a/core/core_test.go b/core/core_test.go index 36ddde25b..4ea9ee2c8 100644 --- a/core/core_test.go +++ b/core/core_test.go @@ -7,6 +7,7 @@ import ( blockfactory "github.com/harmony-one/harmony/block/factory" "github.com/harmony-one/harmony/core/types" shardingconfig "github.com/harmony-one/harmony/internal/configs/sharding" + "github.com/harmony-one/harmony/shard" ) func TestIsEpochBlock(t *testing.T) { @@ -58,7 +59,7 @@ func TestIsEpochBlock(t *testing.T) { }, } for i, test := range tests { - ShardingSchedule = test.schedule + shard.Schedule = test.schedule r := IsEpochBlock(test.block) if r != test.expected { t.Errorf("index: %v, expected: %v, got: %v\n", i, test.expected, r) diff --git a/core/genesis.go b/core/genesis.go index 9e2d3ecc0..cd3d4aa21 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -46,6 +46,11 @@ import ( var errGenesisNoConfig = errors.New("genesis has no chain configuration") +const ( + // GenesisEpoch is the number of the genesis epoch. + GenesisEpoch = 0 +) + // Genesis specifies the header fields, state of a genesis block. It also defines hard // fork switch-over blocks through the chain configuration. type Genesis struct { diff --git a/core/resharding.go b/core/resharding.go deleted file mode 100644 index 2686549f7..000000000 --- a/core/resharding.go +++ /dev/null @@ -1,259 +0,0 @@ -package core - -import ( - "encoding/hex" - "errors" - "math/big" - "math/rand" - "sort" - - "github.com/ethereum/go-ethereum/common" - "github.com/harmony-one/bls/ffi/go/bls" - common2 "github.com/harmony-one/harmony/internal/common" - shardingconfig "github.com/harmony-one/harmony/internal/configs/sharding" - "github.com/harmony-one/harmony/internal/ctxerror" - "github.com/harmony-one/harmony/internal/utils" - "github.com/harmony-one/harmony/shard" -) - -const ( - // GenesisEpoch is the number of the genesis epoch. - GenesisEpoch = 0 - // CuckooRate is the percentage of nodes getting reshuffled in the second step of cuckoo resharding. - CuckooRate = 0.1 -) - -// ShardingState is data structure hold the sharding state -type ShardingState struct { - epoch uint64 // current epoch - rnd uint64 // random seed for resharding - numShards int // TODO ek – equal to len(shardState); remove this - shardState shard.State -} - -// sortedCommitteeBySize will sort shards by size -// Suppose there are N shards, the first N/2 larger shards are called active committees -// the rest N/2 smaller committees are called inactive committees -// actually they are all just normal shards -// TODO: sort the committee weighted by total staking instead of shard size -func (ss *ShardingState) sortCommitteeBySize() { - sort.Slice(ss.shardState, func(i, j int) bool { - return len(ss.shardState[i].NodeList) > len(ss.shardState[j].NodeList) - }) -} - -// assignNewNodes add new nodes into the N/2 active committees evenly -func (ss *ShardingState) assignNewNodes(newNodeList []shard.NodeID) { - ss.sortCommitteeBySize() - numActiveShards := ss.numShards / 2 - Shuffle(newNodeList) - for i, nid := range newNodeList { - id := 0 - if numActiveShards > 0 { - id = i % numActiveShards - } - if id < len(ss.shardState) { - ss.shardState[id].NodeList = append(ss.shardState[id].NodeList, nid) - } else { - utils.Logger().Error().Int("id", id).Int("shardState Count", len(ss.shardState)).Msg("assignNewNodes index out of range") - } - } -} - -// cuckooResharding uses cuckoo rule to reshard X% of active committee(shards) into inactive committee(shards) -func (ss *ShardingState) cuckooResharding(percent float64) { - numActiveShards := ss.numShards / 2 - kickedNodes := []shard.NodeID{} - for i := range ss.shardState { - if i >= numActiveShards { - break - } - numKicked := int(percent * float64(len(ss.shardState[i].NodeList))) - if numKicked == 0 { - numKicked++ // At least kick one node out - } - length := len(ss.shardState[i].NodeList) - if length-numKicked <= 0 { - continue // Never empty a shard - } - tmp := ss.shardState[i].NodeList[length-numKicked:] - kickedNodes = append(kickedNodes, tmp...) - ss.shardState[i].NodeList = ss.shardState[i].NodeList[:length-numKicked] - } - - Shuffle(kickedNodes) - numInactiveShards := ss.numShards - numActiveShards - for i, nid := range kickedNodes { - id := numActiveShards - if numInactiveShards > 0 { - id += i % numInactiveShards - } - ss.shardState[id].NodeList = append(ss.shardState[id].NodeList, nid) - } -} - -// Reshard will first add new nodes into shards, then use cuckoo rule to reshard to get new shard state -func (ss *ShardingState) Reshard(newNodeList []shard.NodeID, percent float64) { - rand.Seed(int64(ss.rnd)) - ss.sortCommitteeBySize() - - // Take out and preserve leaders - leaders := []shard.NodeID{} - for i := 0; i < ss.numShards; i++ { - if len(ss.shardState[i].NodeList) > 0 { - leaders = append(leaders, ss.shardState[i].NodeList[0]) - ss.shardState[i].NodeList = ss.shardState[i].NodeList[1:] - // Also shuffle the rest of the nodes - Shuffle(ss.shardState[i].NodeList) - } - } - - ss.assignNewNodes(newNodeList) - ss.cuckooResharding(percent) - - // Put leader back - if len(leaders) < ss.numShards { - utils.Logger().Error().Msg("Not enough leaders to assign to shards") - } - for i := 0; i < ss.numShards; i++ { - ss.shardState[i].NodeList = append([]shard.NodeID{leaders[i]}, ss.shardState[i].NodeList...) - } -} - -// Shuffle will shuffle the list with result uniquely determined by seed, assuming there is no repeat items in the list -func Shuffle(list []shard.NodeID) { - // Sort to make sure everyone will generate the same with the same rand seed. - sort.Slice(list, func(i, j int) bool { - return shard.CompareNodeIDByBLSKey(list[i], list[j]) == -1 - }) - rand.Shuffle(len(list), func(i, j int) { - list[i], list[j] = list[j], list[i] - }) -} - -// GetEpochFromBlockNumber calculates the epoch number the block belongs to -func GetEpochFromBlockNumber(blockNumber uint64) uint64 { - return ShardingSchedule.CalcEpochNumber(blockNumber).Uint64() -} - -// GetShardingStateFromBlockChain will retrieve random seed and shard map from beacon chain for given a epoch -func GetShardingStateFromBlockChain(bc *BlockChain, epoch *big.Int) (*ShardingState, error) { - if bc == nil { - return nil, errors.New("no blockchain is supplied to get shard state") - } - shardState, err := bc.ReadShardState(epoch) - if err != nil { - return nil, err - } - shardState = shardState.DeepCopy() - - // TODO(RJ,HB): use real randomness for resharding - //blockNumber := GetBlockNumberFromEpoch(epoch.Uint64()) - //rndSeedBytes := bc.GetVdfByNumber(blockNumber) - rndSeed := uint64(0) - - return &ShardingState{epoch: epoch.Uint64(), rnd: rndSeed, shardState: shardState, numShards: len(shardState)}, nil -} - -// CalculateNewShardState get sharding state from previous epoch and calculate sharding state for new epoch -func CalculateNewShardState(bc *BlockChain, epoch *big.Int) (shard.State, error) { - if epoch.Cmp(big.NewInt(GenesisEpoch)) == 0 { - return CalculateInitShardState(), nil - } - prevEpoch := new(big.Int).Sub(epoch, common.Big1) - ss, err := GetShardingStateFromBlockChain(bc, prevEpoch) - if err != nil { - return nil, ctxerror.New("cannot retrieve previous sharding state"). - WithCause(err) - } - utils.Logger().Info().Float64("percentage", CuckooRate).Msg("Cuckoo Rate") - return ss.shardState, nil -} - -// TODO ek – shardingSchedule should really be part of a general-purpose network -// configuration. We are OK for the time being, -// until the day we should let one node process join multiple networks. - -// ShardingSchedule is the sharding configuration schedule. -// Depends on the type of the network. Defaults to the mainnet schedule. -var ShardingSchedule shardingconfig.Schedule = shardingconfig.MainnetSchedule - -// CalculateInitShardState returns the initial shard state at genesis. -func CalculateInitShardState() shard.State { - return CalculateShardState(big.NewInt(GenesisEpoch)) -} - -// CalculateShardState returns the shard state based on epoch number -// This api for getting shard state is what should be used to get shard state regardless of -// current chain dependency (ex. getting shard state from block header received during cross-shard transaction) -func CalculateShardState(epoch *big.Int) shard.State { - utils.Logger().Info().Int64("epoch", epoch.Int64()).Msg("Get Shard State of Epoch.") - shardingConfig := ShardingSchedule.InstanceForEpoch(epoch) - shardNum := int(shardingConfig.NumShards()) - shardHarmonyNodes := shardingConfig.NumHarmonyOperatedNodesPerShard() - shardSize := shardingConfig.NumNodesPerShard() - hmyAccounts := shardingConfig.HmyAccounts() - fnAccounts := shardingConfig.FnAccounts() - - shardState := shard.State{} - for i := 0; i < shardNum; i++ { - com := shard.Committee{ShardID: uint32(i)} - for j := 0; j < shardHarmonyNodes; j++ { - index := i + j*shardNum // The initial account to use for genesis nodes - - pub := &bls.PublicKey{} - pub.DeserializeHexStr(hmyAccounts[index].BlsPublicKey) - pubKey := shard.BlsPublicKey{} - pubKey.FromLibBLSPublicKey(pub) - // TODO: directly read address for bls too - curNodeID := shard.NodeID{ - EcdsaAddress: common2.ParseAddr(hmyAccounts[index].Address), - BlsPublicKey: pubKey, - } - com.NodeList = append(com.NodeList, curNodeID) - } - - // add FN runner's key - for j := shardHarmonyNodes; j < shardSize; j++ { - index := i + (j-shardHarmonyNodes)*shardNum - - pub := &bls.PublicKey{} - pub.DeserializeHexStr(fnAccounts[index].BlsPublicKey) - - pubKey := shard.BlsPublicKey{} - pubKey.FromLibBLSPublicKey(pub) - // TODO: directly read address for bls too - curNodeID := shard.NodeID{ - EcdsaAddress: common2.ParseAddr(fnAccounts[index].Address), - BlsPublicKey: pubKey, - } - com.NodeList = append(com.NodeList, curNodeID) - } - shardState = append(shardState, com) - } - return shardState -} - -// CalculatePublicKeys returns the publickeys given epoch and shardID -func CalculatePublicKeys(epoch *big.Int, shardID uint32) []*bls.PublicKey { - shardState := CalculateShardState(epoch) - - // Update validator public keys - committee := shardState.FindCommitteeByID(shardID) - if committee == nil { - utils.Logger().Warn().Uint32("shardID", shardID).Uint64("epoch", epoch.Uint64()).Msg("Cannot find committee") - return nil - } - pubKeys := []*bls.PublicKey{} - for _, node := range committee.NodeList { - pubKey := &bls.PublicKey{} - pubKeyBytes := node.BlsPublicKey[:] - err := pubKey.Deserialize(pubKeyBytes) - if err != nil { - utils.Logger().Warn().Str("pubKeyBytes", hex.EncodeToString(pubKeyBytes)).Msg("Cannot Deserialize pubKey") - return nil - } - pubKeys = append(pubKeys, pubKey) - } - return pubKeys -} diff --git a/core/resharding.md b/core/resharding.md deleted file mode 100644 index 54dadb24d..000000000 --- a/core/resharding.md +++ /dev/null @@ -1,12 +0,0 @@ -## Resharding - -In current design, the epoch is defined to be fixed length, the epoch length is a constant parameter BlocksPerEpoch. In future, it will be dynamically adjustable according to security parameter. During the epoch transition, suppose there are N shards, we sort the shards according to the size of active nodes (that had staking for next epoch). The first N/2 larger shards will be called active committees, and the last N/2 smaller shards will be called inactive committees. Don't be confused by -the name, they are all normal shards with same function. - -All the information about sharding will be stored in BeaconChain. A sharding state is defined as a map which maps each NodeID to the ShardID the node belongs to. Every node will have a unique NodeID and be mapped to one ShardID. At the beginning of a new epoch, the BeaconChain leader will propose a new block containing the new sharding state, the new sharding state is uniquely determined by the randomness generated by distributed randomness protocol. During the consensus process, all the validators will perform the same calculation and verify the proposed sharding state is valid. After consensus is reached, each node will write the new sharding state into the block. This block is called epoch block. In current code, it's the first block of each epoch in BeaconChain. - -The main function of resharding is CalculcateNewShardState. It will take 3 inputs: newNodeList, oldShardState, randomSeed and output newShardState. -The newNodeList will be retrieved from BeaconChain staking transaction during the previous epoch. The randomSeed and oldShardState is stored in previous epoch block. It should be noticed that the randomSeed generation currently is mocked. After the distributed randomness protocol(drand) is ready, the drand service will generate the random seed for resharding. - -The resharding process is as follows: we first get newNodeList from staking transactions from previous epoch and assign the new nodes evenly into the N/2 active committees. Then, we kick out X% of nodes from each active committees and put these kicked out nodes into inactive committees evenly. The percentage X roughly equals to the percentage of new nodes into active committee in order to balance the committee size. - diff --git a/core/resharding_test.go b/core/resharding_test.go deleted file mode 100644 index dd59cc5ed..000000000 --- a/core/resharding_test.go +++ /dev/null @@ -1,149 +0,0 @@ -package core - -import ( - "fmt" - "math/rand" - "strconv" - "testing" - - "github.com/ethereum/go-ethereum/common" - - "github.com/harmony-one/harmony/shard" - - "github.com/stretchr/testify/assert" -) - -var ( - blsPubKey1 = [48]byte{} - blsPubKey2 = [48]byte{} - blsPubKey3 = [48]byte{} - blsPubKey4 = [48]byte{} - blsPubKey5 = [48]byte{} - blsPubKey6 = [48]byte{} - blsPubKey7 = [48]byte{} - blsPubKey8 = [48]byte{} - blsPubKey9 = [48]byte{} - blsPubKey10 = [48]byte{} -) - -func init() { - copy(blsPubKey1[:], []byte("random key 1")) - copy(blsPubKey2[:], []byte("random key 2")) - copy(blsPubKey3[:], []byte("random key 3")) - copy(blsPubKey4[:], []byte("random key 4")) - copy(blsPubKey5[:], []byte("random key 5")) - copy(blsPubKey6[:], []byte("random key 6")) - copy(blsPubKey7[:], []byte("random key 7")) - copy(blsPubKey8[:], []byte("random key 8")) - copy(blsPubKey9[:], []byte("random key 9")) - copy(blsPubKey10[:], []byte("random key 10")) -} - -func fakeGetInitShardState(numberOfShards, numOfNodes int) shard.State { - rand.Seed(int64(42)) - shardState := shard.State{} - for i := 0; i < numberOfShards; i++ { - sid := uint32(i) - com := shard.Committee{ShardID: sid} - for j := 0; j < numOfNodes; j++ { - nid := strconv.Itoa(int(rand.Int63())) - blsPubKey := [48]byte{} - copy(blsPubKey1[:], []byte(nid)) - com.NodeList = append(com.NodeList, shard.NodeID{ - EcdsaAddress: common.BytesToAddress([]byte(nid)), - BlsPublicKey: blsPubKey, - }) - } - shardState = append(shardState, com) - } - return shardState -} - -func fakeNewNodeList(seed int64) []shard.NodeID { - rand.Seed(seed) - numNewNodes := rand.Intn(10) - nodeList := []shard.NodeID{} - for i := 0; i < numNewNodes; i++ { - nid := strconv.Itoa(int(rand.Int63())) - blsPubKey := [48]byte{} - copy(blsPubKey1[:], []byte(nid)) - nodeList = append(nodeList, shard.NodeID{ - EcdsaAddress: common.BytesToAddress([]byte(nid)), - BlsPublicKey: blsPubKey, - }) - } - return nodeList -} - -func TestFakeNewNodeList(t *testing.T) { - nodeList := fakeNewNodeList(42) - fmt.Println("newNodeList: ", nodeList) -} - -func TestShuffle(t *testing.T) { - nodeList := []shard.NodeID{ - {EcdsaAddress: common.Address{0x12}, BlsPublicKey: blsPubKey1}, - {EcdsaAddress: common.Address{0x22}, BlsPublicKey: blsPubKey2}, - {EcdsaAddress: common.Address{0x32}, BlsPublicKey: blsPubKey3}, - {EcdsaAddress: common.Address{0x42}, BlsPublicKey: blsPubKey4}, - {EcdsaAddress: common.Address{0x52}, BlsPublicKey: blsPubKey5}, - {EcdsaAddress: common.Address{0x62}, BlsPublicKey: blsPubKey6}, - {EcdsaAddress: common.Address{0x72}, BlsPublicKey: blsPubKey7}, - {EcdsaAddress: common.Address{0x82}, BlsPublicKey: blsPubKey8}, - {EcdsaAddress: common.Address{0x92}, BlsPublicKey: blsPubKey9}, - {EcdsaAddress: common.Address{0x02}, BlsPublicKey: blsPubKey10}, - } - - cpList := []shard.NodeID{} - cpList = append(cpList, nodeList...) - Shuffle(nodeList) - cnt := 0 - for i := 0; i < 10; i++ { - if cpList[i] == nodeList[i] { - cnt++ - } - } - if cnt == 10 { - t.Error("Shuffle list is the same as original list") - } - return -} - -func TestSortCommitteeBySize(t *testing.T) { - shardState := fakeGetInitShardState(6, 10) - ss := &ShardingState{epoch: 1, rnd: 42, shardState: shardState, numShards: len(shardState)} - ss.sortCommitteeBySize() - for i := 0; i < ss.numShards-1; i++ { - assert.Equal(t, true, len(ss.shardState[i].NodeList) >= len(ss.shardState[i+1].NodeList)) - } -} - -func TestUpdateShardState(t *testing.T) { - shardState := fakeGetInitShardState(6, 10) - ss := &ShardingState{epoch: 1, rnd: 42, shardState: shardState, numShards: len(shardState)} - newNodeList := []shard.NodeID{ - {EcdsaAddress: common.Address{0x12}, BlsPublicKey: blsPubKey1}, - {EcdsaAddress: common.Address{0x22}, BlsPublicKey: blsPubKey2}, - {EcdsaAddress: common.Address{0x32}, BlsPublicKey: blsPubKey3}, - {EcdsaAddress: common.Address{0x42}, BlsPublicKey: blsPubKey4}, - {EcdsaAddress: common.Address{0x52}, BlsPublicKey: blsPubKey5}, - {EcdsaAddress: common.Address{0x62}, BlsPublicKey: blsPubKey6}, - } - - ss.Reshard(newNodeList, 0.2) - assert.Equal(t, 6, ss.numShards) -} - -func TestAssignNewNodes(t *testing.T) { - shardState := fakeGetInitShardState(2, 2) - ss := &ShardingState{epoch: 1, rnd: 42, shardState: shardState, numShards: len(shardState)} - newNodes := []shard.NodeID{ - {EcdsaAddress: common.Address{0x12}, BlsPublicKey: blsPubKey1}, - {EcdsaAddress: common.Address{0x22}, BlsPublicKey: blsPubKey2}, - {EcdsaAddress: common.Address{0x32}, BlsPublicKey: blsPubKey3}, - } - - ss.assignNewNodes(newNodes) - assert.Equal(t, 2, ss.numShards) - assert.Equal(t, 5, len(ss.shardState[0].NodeList)) -} diff --git a/core/state_processor.go b/core/state_processor.go index f72c2e809..fee37267e 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -30,6 +30,7 @@ import ( "github.com/harmony-one/harmony/internal/ctxerror" "github.com/harmony-one/harmony/internal/params" "github.com/harmony-one/harmony/internal/utils" + "github.com/harmony-one/harmony/shard" staking "github.com/harmony-one/harmony/staking/types" ) @@ -122,7 +123,7 @@ func getTransactionType(config *params.ChainConfig, header *block.Header, tx *ty if header.ShardID() == tx.ShardID() && (!config.IsCrossTx(header.Epoch()) || tx.ShardID() == tx.ToShardID()) { return types.SameShardTx } - numShards := ShardingSchedule.InstanceForEpoch(header.Epoch()).NumShards() + numShards := shard.Schedule.InstanceForEpoch(header.Epoch()).NumShards() // Assuming here all the shards are consecutive from 0 to n-1, n is total number of shards if tx.ShardID() != tx.ToShardID() && header.ShardID() == tx.ShardID() && tx.ToShardID() < numShards { return types.SubtractionOnly diff --git a/core/values/blockchain.go b/core/values/blockchain.go deleted file mode 100644 index 1225e2277..000000000 --- a/core/values/blockchain.go +++ /dev/null @@ -1,10 +0,0 @@ -package values - -const ( - // BeaconChainShardID is the ShardID of the BeaconChain - BeaconChainShardID = 0 - // VotingPowerReduceBlockThreshold roughly corresponds to 3 hours - VotingPowerReduceBlockThreshold = 1350 - // VotingPowerFullReduce roughly corresponds to 12 hours - VotingPowerFullReduce = 4 * VotingPowerReduceBlockThreshold -) diff --git a/drand/drand_leader.go b/drand/drand_leader.go index f9d7346be..52b844c7a 100644 --- a/drand/drand_leader.go +++ b/drand/drand_leader.go @@ -4,17 +4,17 @@ import ( "bytes" "time" - "github.com/harmony-one/harmony/crypto/bls" - protobuf "github.com/golang/protobuf/proto" msg_pb "github.com/harmony-one/harmony/api/proto/message" "github.com/harmony-one/harmony/core" "github.com/harmony-one/harmony/core/types" + "github.com/harmony-one/harmony/crypto/bls" "github.com/harmony-one/harmony/crypto/vdf" "github.com/harmony-one/harmony/crypto/vrf/p256" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/p2p/host" + "github.com/harmony-one/harmony/shard" ) const ( @@ -30,7 +30,7 @@ func (dRand *DRand) WaitForEpochBlock(blockChannel chan *types.Block, stopChan c default: // keep waiting for epoch block newBlock := <-blockChannel - if core.IsEpochLastBlock(newBlock) { + if shard.Schedule.IsLastBlock(newBlock.Number().Uint64()) { dRand.init(newBlock) } // TODO: use real vrf diff --git a/internal/chain/engine.go b/internal/chain/engine.go index c9f0d9306..e927e6a65 100644 --- a/internal/chain/engine.go +++ b/internal/chain/engine.go @@ -8,12 +8,12 @@ import ( "github.com/harmony-one/bls/ffi/go/bls" "github.com/harmony-one/harmony/block" "github.com/harmony-one/harmony/consensus/engine" - "github.com/harmony-one/harmony/core" "github.com/harmony-one/harmony/core/state" "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/internal/ctxerror" "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/shard" + "github.com/harmony-one/harmony/shard/committee" staking "github.com/harmony-one/harmony/staking/types" "github.com/pkg/errors" "golang.org/x/crypto/sha3" @@ -116,6 +116,7 @@ func (e *engineImpl) VerifySeal(chain engine.ChainReader, header *block.Header) return nil } publicKeys, err := ReadPublicKeysFromLastBlock(chain, header) + if err != nil { return ctxerror.New("[VerifySeal] Cannot retrieve publickeys from last block").WithCause(err) } @@ -166,7 +167,7 @@ func (e *engineImpl) Finalize( func QuorumForBlock(chain engine.ChainReader, h *block.Header, reCalculate bool) (quorum int, err error) { var ss shard.State if reCalculate { - ss = core.CalculateShardState(h.Epoch()) + ss, _ = committee.WithStakingEnabled.ReadFromComputation(h.Epoch(), *chain.Config(), nil) } else { ss, err = chain.ReadShardState(h.Epoch()) if err != nil { @@ -225,7 +226,7 @@ func GetPublicKeys(chain engine.ChainReader, header *block.Header, reCalculate b var shardState shard.State var err error if reCalculate { - shardState = core.CalculateShardState(header.Epoch()) + shardState, _ = committee.WithStakingEnabled.ReadFromComputation(header.Epoch(), *chain.Config(), nil) } else { shardState, err = chain.ReadShardState(header.Epoch()) if err != nil { diff --git a/internal/configs/sharding/localnet.go b/internal/configs/sharding/localnet.go index 2a0f0b302..a36a3b714 100644 --- a/internal/configs/sharding/localnet.go +++ b/internal/configs/sharding/localnet.go @@ -152,8 +152,11 @@ func (ls localnetSchedule) GetShardingStructure(numShard, shardID int) []map[str return res } -var localnetReshardingEpoch = []*big.Int{big.NewInt(0), big.NewInt(localnetV1Epoch), big.NewInt(localnetV2Epoch)} - -var localnetV0 = MustNewInstance(2, 7, 5, genesis.LocalHarmonyAccounts, genesis.LocalFnAccounts, localnetReshardingEpoch) -var localnetV1 = MustNewInstance(2, 8, 5, genesis.LocalHarmonyAccountsV1, genesis.LocalFnAccountsV1, localnetReshardingEpoch) -var localnetV2 = MustNewInstance(2, 9, 6, genesis.LocalHarmonyAccountsV2, genesis.LocalFnAccountsV2, localnetReshardingEpoch) +var ( + localnetReshardingEpoch = []*big.Int{ + big.NewInt(0), big.NewInt(localnetV1Epoch), big.NewInt(localnetV2Epoch), + } + localnetV0 = MustNewInstance(2, 7, 5, genesis.LocalHarmonyAccounts, genesis.LocalFnAccounts, localnetReshardingEpoch) + localnetV1 = MustNewInstance(2, 8, 5, genesis.LocalHarmonyAccountsV1, genesis.LocalFnAccountsV1, localnetReshardingEpoch) + localnetV2 = MustNewInstance(2, 9, 6, genesis.LocalHarmonyAccountsV2, genesis.LocalFnAccountsV2, localnetReshardingEpoch) +) diff --git a/internal/hmyapi/blockchain.go b/internal/hmyapi/blockchain.go index 525f239ee..ab7ade411 100644 --- a/internal/hmyapi/blockchain.go +++ b/internal/hmyapi/blockchain.go @@ -20,6 +20,7 @@ import ( internal_bls "github.com/harmony-one/harmony/crypto/bls" internal_common "github.com/harmony-one/harmony/internal/common" "github.com/harmony-one/harmony/internal/utils" + "github.com/harmony-one/harmony/shard" ) const ( @@ -359,10 +360,10 @@ func (s *PublicBlockChainAPI) GetDelegatorsInformation(ctx context.Context, addr func (s *PublicBlockChainAPI) GetShardingStructure(ctx context.Context) ([]map[string]interface{}, error) { // Get header and number of shards. epoch := s.GetEpoch(ctx) - numShard := core.ShardingSchedule.InstanceForEpoch(big.NewInt(int64(epoch))).NumShards() + numShard := shard.Schedule.InstanceForEpoch(big.NewInt(int64(epoch))).NumShards() // Return shareding structure for each case. - return core.ShardingSchedule.GetShardingStructure(int(numShard), int(s.b.GetShardID())), nil + return shard.Schedule.GetShardingStructure(int(numShard), int(s.b.GetShardID())), nil } // GetShardID returns shard ID of the requested node. diff --git a/internal/params/config.go b/internal/params/config.go index 9bb5e49e0..21af1fda3 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -36,7 +36,7 @@ var ( ChainID: TestnetChainID, CrossTxEpoch: big.NewInt(0), CrossLinkEpoch: big.NewInt(0), - StakingEpoch: big.NewInt(0), + StakingEpoch: EpochTBD, EIP155Epoch: big.NewInt(0), S3Epoch: big.NewInt(0), } diff --git a/node/node.go b/node/node.go index e068aa700..8e97b23e0 100644 --- a/node/node.go +++ b/node/node.go @@ -20,7 +20,6 @@ import ( "github.com/harmony-one/harmony/contracts" "github.com/harmony-one/harmony/core" "github.com/harmony-one/harmony/core/types" - "github.com/harmony-one/harmony/core/values" "github.com/harmony-one/harmony/drand" "github.com/harmony-one/harmony/internal/chain" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" @@ -33,6 +32,7 @@ import ( "github.com/harmony-one/harmony/p2p" p2p_host "github.com/harmony-one/harmony/p2p/host" "github.com/harmony-one/harmony/shard" + "github.com/harmony-one/harmony/shard/committee" staking "github.com/harmony-one/harmony/staking/types" ) @@ -277,7 +277,7 @@ func (node *Node) tryBroadcast(tx *types.Transaction) { // Add new transactions to the pending transaction list. func (node *Node) addPendingTransactions(newTxs types.Transactions) { - txPoolLimit := core.ShardingSchedule.MaxTxPoolSizeLimit() + txPoolLimit := shard.Schedule.MaxTxPoolSizeLimit() node.pendingTxMutex.Lock() for _, tx := range newTxs { if _, ok := node.pendingTransactions[tx.Hash()]; !ok { @@ -293,7 +293,7 @@ func (node *Node) addPendingTransactions(newTxs types.Transactions) { // Add new staking transactions to the pending staking transaction list. func (node *Node) addPendingStakingTransactions(newStakingTxs staking.StakingTransactions) { - txPoolLimit := core.ShardingSchedule.MaxTxPoolSizeLimit() + txPoolLimit := shard.Schedule.MaxTxPoolSizeLimit() node.pendingStakingTxMutex.Lock() for _, tx := range newStakingTxs { if _, ok := node.pendingStakingTransactions[tx.Hash()]; !ok { @@ -350,7 +350,7 @@ func (node *Node) AddPendingReceipts(receipts *types.CXReceiptsProof) { // Take out a subset of valid transactions from the pending transaction list // Note the pending transaction list will then contain the rest of the txs func (node *Node) getTransactionsForNewBlock(coinbase common.Address) (types.Transactions, staking.StakingTransactions) { - txsThrottleConfig := core.ShardingSchedule.TxsThrottleConfig() + txsThrottleConfig := shard.Schedule.TxsThrottleConfig() // the next block number to be added in consensus protocol, which is always one more than current chain header block newBlockNum := node.Blockchain().CurrentBlock().NumberU64() + 1 @@ -483,7 +483,8 @@ func New(host p2p.Host, consensusObj *consensus.Consensus, chainDBFactory shardc node.chainConfig = chainConfig collection := shardchain.NewCollection( - chainDBFactory, &genesisInitializer{&node}, chain.Engine, &chainConfig) + chainDBFactory, &genesisInitializer{&node}, chain.Engine, &chainConfig, + ) if isArchival { collection.DisableCache() } @@ -505,7 +506,7 @@ func New(host p2p.Host, consensusObj *consensus.Consensus, chainDBFactory shardc node.CxPool = core.NewCxPool(core.CxPoolSize) node.Worker = worker.New(node.Blockchain().Config(), blockchain, chain.Engine) - if node.Blockchain().ShardID() != values.BeaconChainShardID { + if node.Blockchain().ShardID() != shard.BeaconChainShardID { node.BeaconWorker = worker.New(node.Beaconchain().Config(), beaconChain, chain.Engine) } @@ -545,44 +546,42 @@ func New(host p2p.Host, consensusObj *consensus.Consensus, chainDBFactory shardc // Setup initial state of syncing. node.peerRegistrationRecord = make(map[string]*syncConfig) - node.startConsensus = make(chan struct{}) - go node.bootstrapConsensus() - return &node } -// CalculateInitShardState initialize shard state from latest epoch and update committee pub keys for consensus and drand -func (node *Node) CalculateInitShardState() (err error) { +// InitConsensusWithValidators initialize shard state from latest epoch and update committee pub +// keys for consensus and drand +func (node *Node) InitConsensusWithValidators() (err error) { if node.Consensus == nil { - return ctxerror.New("[CalculateInitShardState] consenus is nil; Cannot figure out shardID") + return ctxerror.New("[InitConsensusWithValidators] consenus is nil; Cannot figure out shardID") } shardID := node.Consensus.ShardID - - // Get genesis epoch shard state from chain blockNum := node.Blockchain().CurrentBlock().NumberU64() node.Consensus.SetMode(consensus.Listening) - epoch := core.ShardingSchedule.CalcEpochNumber(blockNum) + epoch := shard.Schedule.CalcEpochNumber(blockNum) utils.Logger().Info(). Uint64("blockNum", blockNum). Uint32("shardID", shardID). Uint64("epoch", epoch.Uint64()). - Msg("[CalculateInitShardState] Try To Get PublicKeys from database") - pubKeys := core.CalculatePublicKeys(epoch, shardID) + Msg("[InitConsensusWithValidators] Try To Get PublicKeys") + _, pubKeys := committee.WithStakingEnabled.ComputePublicKeys( + epoch, node.Consensus.ChainReader, int(shardID), + ) if len(pubKeys) == 0 { return ctxerror.New( - "[CalculateInitShardState] PublicKeys is Empty, Cannot update public keys", + "[InitConsensusWithValidators] PublicKeys is Empty, Cannot update public keys", "shardID", shardID, "blockNum", blockNum) } - for _, key := range pubKeys { - if key.IsEqual(node.Consensus.PubKey) { + for i := range pubKeys { + if pubKeys[i].IsEqual(node.Consensus.PubKey) { utils.Logger().Info(). Uint64("blockNum", blockNum). Int("numPubKeys", len(pubKeys)). - Msg("[CalculateInitShardState] Successfully updated public keys") + Msg("[InitConsensusWithValidators] Successfully updated public keys") node.Consensus.UpdatePublicKeys(pubKeys) node.Consensus.SetMode(consensus.Normal) return nil diff --git a/node/node_cross_shard.go b/node/node_cross_shard.go index e1040db43..ee306c389 100644 --- a/node/node_cross_shard.go +++ b/node/node_cross_shard.go @@ -4,13 +4,9 @@ import ( "encoding/binary" "errors" - "github.com/harmony-one/harmony/p2p/host" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/rlp" - "github.com/harmony-one/bls/ffi/go/bls" - proto_node "github.com/harmony-one/harmony/api/proto/node" "github.com/harmony-one/harmony/block" "github.com/harmony-one/harmony/core" @@ -19,6 +15,8 @@ import ( nodeconfig "github.com/harmony-one/harmony/internal/configs/node" "github.com/harmony-one/harmony/internal/ctxerror" "github.com/harmony-one/harmony/internal/utils" + "github.com/harmony-one/harmony/p2p/host" + "github.com/harmony-one/harmony/shard" ) // BroadcastCXReceipts broadcasts cross shard receipts to correspoding @@ -38,7 +36,7 @@ func (node *Node) BroadcastCXReceipts(newBlock *types.Block, lastCommits []byte) //#### END Read payload data from committed msg epoch := newBlock.Header().Epoch() - shardingConfig := core.ShardingSchedule.InstanceForEpoch(epoch) + shardingConfig := shard.Schedule.InstanceForEpoch(epoch) shardNum := int(shardingConfig.NumShards()) myShardID := node.Consensus.ShardID utils.Logger().Info().Int("shardNum", shardNum).Uint32("myShardID", myShardID).Uint64("blockNum", newBlock.NumberU64()).Msg("[BroadcastCXReceipts]") @@ -345,7 +343,7 @@ func (node *Node) ProposeCrossLinkDataForBeaconchain() (types.CrossLinks, error) Uint64("blockNum", node.Blockchain().CurrentBlock().NumberU64()+1). Msg("Proposing cross links ...") curBlock := node.Blockchain().CurrentBlock() - numShards := core.ShardingSchedule.InstanceForEpoch(curBlock.Header().Epoch()).NumShards() + numShards := shard.Schedule.InstanceForEpoch(curBlock.Header().Epoch()).NumShards() shardCrossLinks := make([]types.CrossLinks, numShards) diff --git a/node/node_explorer.go b/node/node_explorer.go index 64ba627d8..70fa2fe6e 100644 --- a/node/node_explorer.go +++ b/node/node_explorer.go @@ -14,6 +14,7 @@ import ( "github.com/harmony-one/harmony/core" "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/internal/utils" + "github.com/harmony-one/harmony/shard" ) var once sync.Once @@ -107,7 +108,7 @@ func (node *Node) ExplorerMessageHandler(payload []byte) { func (node *Node) AddNewBlockForExplorer(block *types.Block) { utils.Logger().Debug().Uint64("blockHeight", block.NumberU64()).Msg("[Explorer] Adding new block for explorer node") if err := node.AddNewBlock(block); err == nil { - if core.IsEpochLastBlock(block) { + if shard.Schedule.IsLastBlock(block.Number().Uint64()) { node.Consensus.UpdateConsensusInformation() } // Clean up the blocks to avoid OOM. diff --git a/node/node_genesis.go b/node/node_genesis.go index 47f36760e..bc9740743 100644 --- a/node/node_genesis.go +++ b/node/node_genesis.go @@ -21,6 +21,7 @@ import ( "github.com/harmony-one/harmony/internal/params" "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/shard" + "github.com/harmony-one/harmony/shard/committee" ) const ( @@ -41,8 +42,10 @@ type genesisInitializer struct { // InitChainDB sets up a new genesis block in the database for the given shard. func (gi *genesisInitializer) InitChainDB(db ethdb.Database, shardID uint32) error { - shardState := core.CalculateInitShardState() - if shardID != 0 { + shardState, _ := committee.WithStakingEnabled.ReadFromComputation( + big.NewInt(core.GenesisEpoch), gi.node.chainConfig, nil, + ) + if shardID != shard.BeaconChainShardID { // store only the local shard for shard chains c := shardState.FindCommitteeByID(shardID) if c == nil { diff --git a/node/node_handler.go b/node/node_handler.go index 5551333bf..947cbfd00 100644 --- a/node/node_handler.go +++ b/node/node_handler.go @@ -5,11 +5,9 @@ import ( "context" "math/big" "math/rand" - "sync" "sync/atomic" "time" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/rlp" "github.com/harmony-one/bls/ffi/go/bls" @@ -19,7 +17,6 @@ import ( proto_discovery "github.com/harmony-one/harmony/api/proto/discovery" proto_node "github.com/harmony-one/harmony/api/proto/node" "github.com/harmony-one/harmony/block" - "github.com/harmony-one/harmony/core" "github.com/harmony-one/harmony/core/types" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" "github.com/harmony-one/harmony/internal/ctxerror" @@ -325,7 +322,7 @@ func (node *Node) PostConsensusProcessing(newBlock *types.Block, commitSigAndBit if node.NodeConfig.ShardID == 0 { node.BroadcastNewBlock(newBlock) } - if node.NodeConfig.ShardID != 0 && newBlock.Epoch().Cmp(node.Blockchain().Config().CrossLinkEpoch) >= 0 { + if node.NodeConfig.ShardID != shard.BeaconChainShardID && newBlock.Epoch().Cmp(node.Blockchain().Config().CrossLinkEpoch) >= 0 { node.BroadcastCrossLinkHeader(newBlock) } node.BroadcastCXReceipts(newBlock, commitSigAndBitmap) @@ -348,7 +345,7 @@ func (node *Node) PostConsensusProcessing(newBlock *types.Block, commitSigAndBit node.BroadcastMissingCXReceipts() // Update consensus keys at last so the change of leader status doesn't mess up normal flow - if core.IsEpochLastBlock(newBlock) { + if shard.Schedule.IsLastBlock(newBlock.Number().Uint64()) { node.Consensus.UpdateConsensusInformation() } @@ -380,33 +377,7 @@ func (node *Node) PostConsensusProcessing(newBlock *types.Block, commitSigAndBit // node.ConfirmedBlockChannel <- newBlock // }() //} - - // TODO: enable staking - // TODO: update staking information once per epoch. - //node.UpdateStakingList(node.QueryStakeInfo()) - //node.printStakingList() } - - // TODO: enable shard state update - //newBlockHeader := newBlock.Header() - //if newBlockHeader.ShardStateHash != (common.Hash{}) { - // if node.Consensus.ShardID == 0 { - // // TODO ek – this is a temp hack until beacon chain sync is fixed - // // End-of-epoch block on beacon chain; block's EpochState is the - // // master resharding table. Broadcast it to the network. - // if err := node.broadcastEpochShardState(newBlock); err != nil { - // e := ctxerror.New("cannot broadcast shard state").WithCause(err) - // ctxerror.Log15(utils.Logger().Error, e) - // } - // } - // shardState, err := newBlockHeader.CalculateShardState() - // if err != nil { - // e := ctxerror.New("cannot get shard state from header").WithCause(err) - // ctxerror.Log15(utils.Logger().Error, e) - // } else { - // node.transitionIntoNextEpoch(shardState) - // } - //} } } @@ -448,43 +419,6 @@ func (node *Node) AddNewBlock(newBlock *types.Block) error { return err } -type genesisNode struct { - ShardID uint32 - MemberIndex int - NodeID shard.NodeID -} - -var ( - genesisCatalogOnce sync.Once - genesisNodeByStakingAddress = make(map[common.Address]*genesisNode) - genesisNodeByConsensusKey = make(map[shard.BlsPublicKey]*genesisNode) -) - -func initGenesisCatalog() { - genesisShardState := core.CalculateInitShardState() - for _, committee := range genesisShardState { - for i, nodeID := range committee.NodeList { - genesisNode := &genesisNode{ - ShardID: committee.ShardID, - MemberIndex: i, - NodeID: nodeID, - } - genesisNodeByStakingAddress[nodeID.EcdsaAddress] = genesisNode - genesisNodeByConsensusKey[nodeID.BlsPublicKey] = genesisNode - } - } -} - -func getGenesisNodeByStakingAddress(address common.Address) *genesisNode { - genesisCatalogOnce.Do(initGenesisCatalog) - return genesisNodeByStakingAddress[address] -} - -func getGenesisNodeByConsensusKey(key shard.BlsPublicKey) *genesisNode { - genesisCatalogOnce.Do(initGenesisCatalog) - return genesisNodeByConsensusKey[key] -} - func (node *Node) pingMessageHandler(msgPayload []byte, sender libp2p_peer.ID) int { ping, err := proto_discovery.GetPingMessage(msgPayload) if err != nil { diff --git a/node/node_handler_test.go b/node/node_handler_test.go index 49a012270..e1609b9e0 100644 --- a/node/node_handler_test.go +++ b/node/node_handler_test.go @@ -6,12 +6,12 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/harmony-one/harmony/consensus" "github.com/harmony-one/harmony/consensus/quorum" - "github.com/harmony-one/harmony/core/values" "github.com/harmony-one/harmony/crypto/bls" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/p2p" "github.com/harmony-one/harmony/p2p/p2pimpl" + "github.com/harmony-one/harmony/shard" ) func TestAddNewBlock(t *testing.T) { @@ -25,7 +25,7 @@ func TestAddNewBlock(t *testing.T) { } decider := quorum.NewDecider(quorum.SuperMajorityVote) consensus, err := consensus.New( - host, values.BeaconChainShardID, leader, blsKey, decider, + host, shard.BeaconChainShardID, leader, blsKey, decider, ) if err != nil { t.Fatalf("Cannot craeate consensus: %v", err) @@ -58,7 +58,7 @@ func TestVerifyNewBlock(t *testing.T) { } decider := quorum.NewDecider(quorum.SuperMajorityVote) consensus, err := consensus.New( - host, values.BeaconChainShardID, leader, blsKey, decider, + host, shard.BeaconChainShardID, leader, blsKey, decider, ) if err != nil { t.Fatalf("Cannot craeate consensus: %v", err) diff --git a/node/node_newblock.go b/node/node_newblock.go index b47445201..f6df9c1bd 100644 --- a/node/node_newblock.go +++ b/node/node_newblock.go @@ -6,11 +6,11 @@ import ( "time" "github.com/ethereum/go-ethereum/common" - "github.com/harmony-one/harmony/core" "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/internal/ctxerror" "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/shard" + "github.com/harmony-one/harmony/shard/committee" ) // Constants of proposing a new block @@ -124,12 +124,13 @@ func (node *Node) proposeNewBlock() (*types.Block, error) { } func (node *Node) proposeShardStateWithoutBeaconSync(block *types.Block) shard.State { - if block == nil || !core.IsEpochLastBlock(block) { + if block == nil || !shard.Schedule.IsLastBlock(block.Number().Uint64()) { return nil } - - nextEpoch := new(big.Int).Add(block.Header().Epoch(), common.Big1) - return core.CalculateShardState(nextEpoch) + shardState, _ := committee.WithStakingEnabled.ReadFromComputation( + new(big.Int).Add(block.Header().Epoch(), common.Big1), node.chainConfig, nil, + ) + return shardState } func (node *Node) proposeShardState(block *types.Block) error { @@ -144,13 +145,15 @@ func (node *Node) proposeShardState(block *types.Block) error { func (node *Node) proposeBeaconShardState(block *types.Block) error { // TODO ek - replace this with variable epoch logic. - if !core.IsEpochLastBlock(block) { + if !shard.Schedule.IsLastBlock(block.Number().Uint64()) { // We haven't reached the end of this epoch; don't propose yet. return nil } - nextEpoch := new(big.Int).Add(block.Header().Epoch(), common.Big1) - // TODO: add logic for EPoS - shardState, err := core.CalculateNewShardState(node.Blockchain(), nextEpoch) + // TODO Use ReadFromComputation + prevEpoch := new(big.Int).Sub(block.Header().Epoch(), common.Big1) + shardState, err := committee.WithStakingEnabled.ReadFromChain( + prevEpoch, node.Blockchain(), + ) if err != nil { return err } diff --git a/node/node_resharding.go b/node/node_resharding.go index fa9bc1e63..0c55369d9 100644 --- a/node/node_resharding.go +++ b/node/node_resharding.go @@ -15,13 +15,13 @@ import ( "github.com/ethereum/go-ethereum/rlp" "github.com/harmony-one/bls/ffi/go/bls" proto_node "github.com/harmony-one/harmony/api/proto/node" - "github.com/harmony-one/harmony/core" "github.com/harmony-one/harmony/core/types" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" "github.com/harmony-one/harmony/internal/ctxerror" "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/p2p/host" "github.com/harmony-one/harmony/shard" + "github.com/harmony-one/harmony/shard/committee" ) // validateNewShardState validate whether the new shard state root matches @@ -30,8 +30,8 @@ func (node *Node) validateNewShardState(block *types.Block) error { header := block.Header() if header.ShardStateHash() == (common.Hash{}) { // No new shard state was proposed - if block.ShardID() == 0 { - if core.IsEpochLastBlock(block) { + if block.ShardID() == shard.BeaconChainShardID { + if shard.Schedule.IsLastBlock(block.Number().Uint64()) { // TODO ek - invoke view change return errors.New("beacon leader did not propose resharding") } @@ -51,14 +51,17 @@ func (node *Node) validateNewShardState(block *types.Block) error { return err } proposed := *shardState - if block.ShardID() == 0 { + if block.ShardID() == shard.BeaconChainShardID { // Beacon validators independently recalculate the master state and // compare it against the proposed copy. - nextEpoch := new(big.Int).Add(block.Header().Epoch(), common.Big1) // TODO ek – this may be called from regular shards, // for vetting beacon chain blocks received during block syncing. // DRand may or or may not get in the way. Test this out. - expected, err := core.CalculateNewShardState(node.Blockchain(), nextEpoch) + expected, err := committee.WithStakingEnabled.ReadFromChain( + new(big.Int).Sub(block.Header().Epoch(), common.Big1), + node.Beaconchain(), + ) + if err != nil { return ctxerror.New("cannot calculate expected shard state"). WithCause(err) diff --git a/node/node_test.go b/node/node_test.go index a84cc1122..77cd65d08 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -11,7 +11,6 @@ import ( proto_discovery "github.com/harmony-one/harmony/api/proto/discovery" "github.com/harmony-one/harmony/consensus" "github.com/harmony-one/harmony/consensus/quorum" - "github.com/harmony-one/harmony/core/values" bls2 "github.com/harmony-one/harmony/crypto/bls" "github.com/harmony-one/harmony/crypto/pki" "github.com/harmony-one/harmony/drand" @@ -19,6 +18,7 @@ import ( "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/p2p" "github.com/harmony-one/harmony/p2p/p2pimpl" + "github.com/harmony-one/harmony/shard" "github.com/stretchr/testify/assert" ) @@ -35,7 +35,7 @@ func TestNewNode(t *testing.T) { } decider := quorum.NewDecider(quorum.SuperMajorityVote) consensus, err := consensus.New( - host, values.BeaconChainShardID, leader, blsKey, decider, + host, shard.BeaconChainShardID, leader, blsKey, decider, ) if err != nil { t.Fatalf("Cannot craeate consensus: %v", err) @@ -202,7 +202,7 @@ func TestAddPeers(t *testing.T) { } decider := quorum.NewDecider(quorum.SuperMajorityVote) consensus, err := consensus.New( - host, values.BeaconChainShardID, leader, blsKey, decider, + host, shard.BeaconChainShardID, leader, blsKey, decider, ) if err != nil { t.Fatalf("Cannot craeate consensus: %v", err) @@ -252,7 +252,7 @@ func TestAddBeaconPeer(t *testing.T) { } decider := quorum.NewDecider(quorum.SuperMajorityVote) consensus, err := consensus.New( - host, values.BeaconChainShardID, leader, blsKey, decider, + host, shard.BeaconChainShardID, leader, blsKey, decider, ) if err != nil { t.Fatalf("Cannot craeate consensus: %v", err) diff --git a/node/worker/worker.go b/node/worker/worker.go index d998d0e27..8fc3a2054 100644 --- a/node/worker/worker.go +++ b/node/worker/worker.go @@ -13,13 +13,13 @@ import ( "github.com/harmony-one/harmony/core" "github.com/harmony-one/harmony/core/state" "github.com/harmony-one/harmony/core/types" - "github.com/harmony-one/harmony/core/values" "github.com/harmony-one/harmony/core/vm" shardingconfig "github.com/harmony-one/harmony/internal/configs/sharding" "github.com/harmony-one/harmony/internal/ctxerror" "github.com/harmony-one/harmony/internal/params" "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/shard" + "github.com/harmony-one/harmony/shard/committee" staking "github.com/harmony-one/harmony/staking/types" ) @@ -162,7 +162,7 @@ func (w *Worker) SelectStakingTransactionsForNewBlock( coinbase common.Address) (staking.StakingTransactions, staking.StakingTransactions, staking.StakingTransactions) { // only beaconchain process staking transaction - if w.chain.ShardID() != values.BeaconChainShardID { + if w.chain.ShardID() != shard.BeaconChainShardID { utils.Logger().Warn().Msgf("Invalid shardID: %v", w.chain.ShardID()) return nil, nil, nil } @@ -365,11 +365,13 @@ func (w *Worker) IncomingReceipts() []*types.CXReceiptsProof { // ProposeShardStateWithoutBeaconSync proposes the next shard state for next epoch. func (w *Worker) ProposeShardStateWithoutBeaconSync() shard.State { - if !core.ShardingSchedule.IsLastBlock(w.current.header.Number().Uint64()) { + if !shard.Schedule.IsLastBlock(w.current.header.Number().Uint64()) { return nil } - nextEpoch := new(big.Int).Add(w.current.header.Epoch(), common.Big1) - return core.CalculateShardState(nextEpoch) + shardState, _ := committee.WithStakingEnabled.ReadFromComputation( + new(big.Int).Add(w.current.header.Epoch(), common.Big1), *w.config, nil, + ) + return shardState } // FinalizeNewBlock generate a new block for the next consensus round. diff --git a/shard/committee/assignment.go b/shard/committee/assignment.go new file mode 100644 index 000000000..69afb9259 --- /dev/null +++ b/shard/committee/assignment.go @@ -0,0 +1,261 @@ +package committee + +import ( + "math/big" + "sort" + + "github.com/ethereum/go-ethereum/common" + "github.com/harmony-one/bls/ffi/go/bls" + "github.com/harmony-one/harmony/block" + common2 "github.com/harmony-one/harmony/internal/common" + shardingconfig "github.com/harmony-one/harmony/internal/configs/sharding" + "github.com/harmony-one/harmony/internal/ctxerror" + "github.com/harmony-one/harmony/internal/params" + "github.com/harmony-one/harmony/internal/utils" + "github.com/harmony-one/harmony/numeric" + "github.com/harmony-one/harmony/shard" + staking "github.com/harmony-one/harmony/staking/types" +) + +// StateID means reading off whole network when using calls that accept +// a shardID parameter +const StateID = -1 + +// MembershipList .. +type MembershipList interface { + ReadFromComputation( + epoch *big.Int, config params.ChainConfig, reader StakingCandidatesReader, + ) (shard.State, error) + ReadFromChain(epoch *big.Int, reader ChainReader) (shard.State, error) +} + +// PublicKeys per epoch +type PublicKeys interface { + // If call shardID with StateID then only superCommittee is non-nil, + // otherwise get back the shardSpecific slice as well. + ComputePublicKeys( + epoch *big.Int, reader ChainReader, shardID int, + ) (superCommittee, shardSpecific []*bls.PublicKey) + + ReadPublicKeysFromDB( + hash common.Hash, reader ChainReader, + ) ([]*bls.PublicKey, error) +} + +// Reader .. +type Reader interface { + PublicKeys + MembershipList +} + +// StakingCandidatesReader .. +type StakingCandidatesReader interface { + ValidatorInformation(addr common.Address) (*staking.Validator, error) + ValidatorStakingWithDelegation(addr common.Address) numeric.Dec + ValidatorCandidates() []common.Address +} + +// ChainReader is a subset of Engine.ChainReader, just enough to do assignment +type ChainReader interface { + // ReadShardState retrieves sharding state given the epoch number. + // This api reads the shard state cached or saved on the chaindb. + // Thus, only should be used to read the shard state of the current chain. + ReadShardState(epoch *big.Int) (shard.State, error) + // GetHeader retrieves a block header from the database by hash and number. + GetHeaderByHash(common.Hash) *block.Header + // Config retrieves the blockchain's chain configuration. + Config() *params.ChainConfig +} + +type partialStakingEnabled struct{} + +var ( + // WithStakingEnabled .. + WithStakingEnabled Reader = partialStakingEnabled{} +) + +func preStakingEnabledCommittee(s shardingconfig.Instance) shard.State { + shardNum := int(s.NumShards()) + shardHarmonyNodes := s.NumHarmonyOperatedNodesPerShard() + shardSize := s.NumNodesPerShard() + hmyAccounts := s.HmyAccounts() + fnAccounts := s.FnAccounts() + shardState := shard.State{} + for i := 0; i < shardNum; i++ { + com := shard.Committee{ShardID: uint32(i)} + for j := 0; j < shardHarmonyNodes; j++ { + index := i + j*shardNum // The initial account to use for genesis nodes + pub := &bls.PublicKey{} + pub.DeserializeHexStr(hmyAccounts[index].BlsPublicKey) + pubKey := shard.BlsPublicKey{} + pubKey.FromLibBLSPublicKey(pub) + // TODO: directly read address for bls too + curNodeID := shard.NodeID{ + common2.ParseAddr(hmyAccounts[index].Address), + pubKey, + nil, + } + com.NodeList = append(com.NodeList, curNodeID) + } + // add FN runner's key + for j := shardHarmonyNodes; j < shardSize; j++ { + index := i + (j-shardHarmonyNodes)*shardNum + pub := &bls.PublicKey{} + pub.DeserializeHexStr(fnAccounts[index].BlsPublicKey) + pubKey := shard.BlsPublicKey{} + pubKey.FromLibBLSPublicKey(pub) + // TODO: directly read address for bls too + curNodeID := shard.NodeID{ + common2.ParseAddr(fnAccounts[index].Address), + pubKey, + nil, + } + com.NodeList = append(com.NodeList, curNodeID) + } + shardState = append(shardState, com) + } + return shardState +} + +func with400Stakers( + s shardingconfig.Instance, stakerReader StakingCandidatesReader, +) (shard.State, error) { + // TODO Nervous about this because overtime the list will become quite large + candidates := stakerReader.ValidatorCandidates() + stakers := make([]*staking.Validator, len(candidates)) + for i := range candidates { + // TODO Should be using .ValidatorStakingWithDelegation, not implemented yet + validator, err := stakerReader.ValidatorInformation(candidates[i]) + if err != nil { + return nil, err + } + stakers[i] = validator + } + + sort.SliceStable( + stakers, + func(i, j int) bool { return stakers[i].Stake.Cmp(stakers[j].Stake) >= 0 }, + ) + const sCount = 401 + top := stakers[:sCount] + shardCount := int(s.NumShards()) + superComm := make(shard.State, shardCount) + fillCount := make([]int, shardCount) + // TODO Finish this logic, not correct, need to operate EPoS on slot level, + // not validator level + + for i := 0; i < shardCount; i++ { + superComm[i] = shard.Committee{} + superComm[i].NodeList = make(shard.NodeIDList, s.NumNodesPerShard()) + } + + scratchPad := &bls.PublicKey{} + + for i := range top { + spot := int(top[i].Address.Big().Int64()) % shardCount + fillCount[spot]++ + // scratchPad.DeserializeHexStr() + pubKey := shard.BlsPublicKey{} + pubKey.FromLibBLSPublicKey(scratchPad) + superComm[spot].NodeList = append( + superComm[spot].NodeList, + shard.NodeID{ + top[i].Address, + pubKey, + &shard.StakedMember{big.NewInt(0)}, + }, + ) + } + + utils.Logger().Info().Ints("distribution of Stakers in Shards", fillCount) + return superComm, nil +} + +func (def partialStakingEnabled) ReadPublicKeysFromDB( + h common.Hash, reader ChainReader, +) ([]*bls.PublicKey, error) { + header := reader.GetHeaderByHash(h) + shardID := header.ShardID() + superCommittee, err := reader.ReadShardState(header.Epoch()) + if err != nil { + return nil, err + } + subCommittee := superCommittee.FindCommitteeByID(shardID) + if subCommittee == nil { + return nil, ctxerror.New("cannot find shard in the shard state", + "blockNumber", header.Number(), + "shardID", header.ShardID(), + ) + } + committerKeys := []*bls.PublicKey{} + + for i := range subCommittee.NodeList { + committerKey := new(bls.PublicKey) + err := subCommittee.NodeList[i].BlsPublicKey.ToLibBLSPublicKey(committerKey) + if err != nil { + return nil, ctxerror.New("cannot convert BLS public key", + "blsPublicKey", subCommittee.NodeList[i].BlsPublicKey).WithCause(err) + } + committerKeys = append(committerKeys, committerKey) + } + return committerKeys, nil + + return nil, nil +} + +// ReadPublicKeysFromChain produces publicKeys of entire supercommittee per epoch, optionally providing a +// shard specific subcommittee +func (def partialStakingEnabled) ComputePublicKeys( + epoch *big.Int, reader ChainReader, shardID int, +) ([]*bls.PublicKey, []*bls.PublicKey) { + config := reader.Config() + instance := shard.Schedule.InstanceForEpoch(epoch) + if !config.IsStaking(epoch) { + superComm := preStakingEnabledCommittee(instance) + spot := 0 + allIdentities := make([]*bls.PublicKey, int(instance.NumShards())*instance.NumNodesPerShard()) + for i := range superComm { + for j := range superComm[i].NodeList { + identity := &bls.PublicKey{} + superComm[i].NodeList[j].BlsPublicKey.ToLibBLSPublicKey(identity) + allIdentities[spot] = identity + spot++ + } + } + + if shardID == StateID { + return allIdentities, nil + } + + subCommittee := superComm.FindCommitteeByID(uint32(shardID)) + subCommitteeIdentities := make([]*bls.PublicKey, len(subCommittee.NodeList)) + spot = 0 + for i := range subCommittee.NodeList { + identity := &bls.PublicKey{} + subCommittee.NodeList[i].BlsPublicKey.ToLibBLSPublicKey(identity) + subCommitteeIdentities[spot] = identity + spot++ + } + + return allIdentities, subCommitteeIdentities + } + // TODO Implement for the staked case + return nil, nil +} + +func (def partialStakingEnabled) ReadFromChain( + epoch *big.Int, reader ChainReader, +) (newSuperComm shard.State, err error) { + return reader.ReadShardState(epoch) +} + +// ReadFromComputation is single entry point for reading the State of the network +func (def partialStakingEnabled) ReadFromComputation( + epoch *big.Int, config params.ChainConfig, stakerReader StakingCandidatesReader, +) (newSuperComm shard.State, err error) { + instance := shard.Schedule.InstanceForEpoch(epoch) + if !config.IsStaking(epoch) { + return preStakingEnabledCommittee(instance), nil + } + return with400Stakers(instance, stakerReader) +} diff --git a/shard/shard_state.go b/shard/shard_state.go index d05cb1add..c868c702e 100644 --- a/shard/shard_state.go +++ b/shard/shard_state.go @@ -3,6 +3,8 @@ package shard import ( "bytes" "encoding/hex" + "encoding/json" + "math/big" "sort" "github.com/ethereum/go-ethereum/common" @@ -16,15 +18,71 @@ var ( emptyBlsPubKey = BlsPublicKey{} ) +// PublicKeySizeInBytes .. +const PublicKeySizeInBytes = 48 + // EpochShardState is the shard state of an epoch type EpochShardState struct { Epoch uint64 ShardState State } +// StakedMember is a committee member with stake +type StakedMember struct { + // nil means not active, 0 means our node, >= 0 means staked node + WithDelegationApplied *big.Int `json:"with-delegation-applied,omitempty"` +} + // State is the collection of all committees type State []Committee +// BlsPublicKey defines the bls public key +type BlsPublicKey [PublicKeySizeInBytes]byte + +// NodeID represents node id (BLS address) +type NodeID struct { + EcdsaAddress common.Address `json:"ecdsa_address"` + BlsPublicKey BlsPublicKey `json:"bls_pubkey"` + Validator *StakedMember `json:"staked-validator,omitempty" rlp:"nil"` +} + +// NodeIDList is a list of NodeIDList. +type NodeIDList []NodeID + +// Committee contains the active nodes in one shard +type Committee struct { + ShardID uint32 `json:"shard_id"` + NodeList NodeIDList `json:"node_list"` +} + +// JSON produces a non-pretty printed JSON string of the SuperCommittee +func (ss State) JSON() string { + type V struct { + ECDSAAddress common.Address `json:"ecdsa_address"` + BLSPublicKey string `json:"bls-public-key"` + } + + type T struct { + ShardID uint32 `json:"shard_id"` + Total int `json:"count"` + NodeList []V `json:"entries"` + } + t := []T{} + for i := range ss { + sub := ss[i] + subList := []V{} + for j := range sub.NodeList { + subList = append(subList, V{ + sub.NodeList[j].EcdsaAddress, + sub.NodeList[j].BlsPublicKey.Hex(), + }) + } + t = append(t, T{sub.ShardID, len(sub.NodeList), subList}) + } + buf, _ := json.Marshal(t) + return string(buf) +} + // FindCommitteeByID returns the committee configuration for the given shard, // or nil if the given shard is not found. func (ss State) FindCommitteeByID(shardID uint32) *Committee { @@ -65,9 +123,6 @@ func CompareShardState(s1, s2 State) int { return 0 } -// BlsPublicKey defines the bls public key -type BlsPublicKey [48]byte - // IsEmpty returns whether the bls public key is empty 0 bytes func (pk BlsPublicKey) IsEmpty() bool { return bytes.Compare(pk[:], emptyBlsPubKey[:]) == 0 @@ -100,12 +155,6 @@ func CompareBlsPublicKey(k1, k2 BlsPublicKey) int { return bytes.Compare(k1[:], k2[:]) } -// NodeID represents node id (BLS address) -type NodeID struct { - EcdsaAddress common.Address `json:"ecdsa_address"` - BlsPublicKey BlsPublicKey `json:"bls_pubkey"` -} - // CompareNodeID compares two node IDs. func CompareNodeID(id1, id2 *NodeID) int { if c := bytes.Compare(id1.EcdsaAddress[:], id2.EcdsaAddress[:]); c != 0 { @@ -117,9 +166,6 @@ func CompareNodeID(id1, id2 *NodeID) int { return 0 } -// NodeIDList is a list of NodeIDList. -type NodeIDList []NodeID - // DeepCopy returns a deep copy of the receiver. func (l NodeIDList) DeepCopy() NodeIDList { return append(l[:0:0], l...) @@ -145,12 +191,6 @@ func CompareNodeIDList(l1, l2 NodeIDList) int { return 0 } -// Committee contains the active nodes in one shard -type Committee struct { - ShardID uint32 `json:"shard_id"` - NodeList NodeIDList `json:"node_list"` -} - // DeepCopy returns a deep copy of the receiver. func (c Committee) DeepCopy() Committee { r := Committee{} diff --git a/shard/shard_state_test.go b/shard/shard_state_test.go index 707b1a335..12ce1784d 100644 --- a/shard/shard_state_test.go +++ b/shard/shard_state_test.go @@ -31,14 +31,14 @@ func init() { func TestGetHashFromNodeList(t *testing.T) { l1 := []NodeID{ - {common.Address{0x11}, blsPubKey1}, - {common.Address{0x22}, blsPubKey2}, - {common.Address{0x33}, blsPubKey3}, + {common.Address{0x11}, blsPubKey1, nil}, + {common.Address{0x22}, blsPubKey2, nil}, + {common.Address{0x33}, blsPubKey3, nil}, } l2 := []NodeID{ - {common.Address{0x22}, blsPubKey2}, - {common.Address{0x11}, blsPubKey1}, - {common.Address{0x33}, blsPubKey3}, + {common.Address{0x22}, blsPubKey2, nil}, + {common.Address{0x11}, blsPubKey1, nil}, + {common.Address{0x33}, blsPubKey3, nil}, } h1 := GetHashFromNodeList(l1) h2 := GetHashFromNodeList(l2) @@ -52,17 +52,17 @@ func TestHash(t *testing.T) { com1 := Committee{ ShardID: 22, NodeList: []NodeID{ - {common.Address{0x12}, blsPubKey11}, - {common.Address{0x23}, blsPubKey22}, - {common.Address{0x11}, blsPubKey1}, + {common.Address{0x12}, blsPubKey11, nil}, + {common.Address{0x23}, blsPubKey22, nil}, + {common.Address{0x11}, blsPubKey1, nil}, }, } com2 := Committee{ ShardID: 2, NodeList: []NodeID{ - {common.Address{0x44}, blsPubKey4}, - {common.Address{0x55}, blsPubKey5}, - {common.Address{0x66}, blsPubKey6}, + {common.Address{0x44}, blsPubKey4, nil}, + {common.Address{0x55}, blsPubKey5, nil}, + {common.Address{0x66}, blsPubKey6, nil}, }, } shardState1 := State{com1, com2} @@ -71,17 +71,17 @@ func TestHash(t *testing.T) { com3 := Committee{ ShardID: 2, NodeList: []NodeID{ - {common.Address{0x44}, blsPubKey4}, - {common.Address{0x55}, blsPubKey5}, - {common.Address{0x66}, blsPubKey6}, + {common.Address{0x44}, blsPubKey4, nil}, + {common.Address{0x55}, blsPubKey5, nil}, + {common.Address{0x66}, blsPubKey6, nil}, }, } com4 := Committee{ ShardID: 22, NodeList: []NodeID{ - {common.Address{0x12}, blsPubKey11}, - {common.Address{0x23}, blsPubKey22}, - {common.Address{0x11}, blsPubKey1}, + {common.Address{0x12}, blsPubKey11, nil}, + {common.Address{0x23}, blsPubKey22, nil}, + {common.Address{0x11}, blsPubKey1, nil}, }, } diff --git a/shard/values.go b/shard/values.go new file mode 100644 index 000000000..f925bea90 --- /dev/null +++ b/shard/values.go @@ -0,0 +1,19 @@ +package shard + +import ( + shardingconfig "github.com/harmony-one/harmony/internal/configs/sharding" +) + +const ( + // BeaconChainShardID is the ShardID of the BeaconChain + BeaconChainShardID = 0 +) + +// TODO ek – Schedule should really be part of a general-purpose network +// configuration. We are OK for the time being, +// until the day we should let one node process join multiple networks. +var ( + // Schedule is the sharding configuration schedule. + // Depends on the type of the network. Defaults to the mainnet schedule. + Schedule shardingconfig.Schedule = shardingconfig.MainnetSchedule +) From 55c9386e4c5588f68376ecb5b232e492c530a9b4 Mon Sep 17 00:00:00 2001 From: Edgar Aroutiounian Date: Sun, 10 Nov 2019 21:18:15 -0800 Subject: [PATCH 43/45] [quorum] Staked quorum member (#1819) * [quorum] Factor out single vote & provide for staked quorum vote * [quorum] Pass ShardID to decider via cb * [quorum] Move ShardIDProvider higher, fix nil mistake * [quorum] ReadAllSignatures optimization * [quorum] Safer way to read over map, then to slice * [quorum] Address PR comments - naming changes --- cmd/harmony/main.go | 3 + consensus/consensus_v2.go | 10 +-- consensus/quorum/one-node-one-vote.go | 78 ++++++++++++++++++ consensus/quorum/one-node-staked-vote.go | 75 +++++++++++++++++ consensus/quorum/quorum.go | 100 +++++++++++++---------- consensus/view_change.go | 12 +-- core/blockchain.go | 4 +- internal/chain/engine.go | 4 +- node/node_explorer.go | 2 +- node/node_genesis.go | 4 +- node/node_newblock.go | 4 +- node/node_resharding.go | 2 +- node/worker/worker.go | 2 +- shard/committee/assignment.go | 14 ++-- 14 files changed, 242 insertions(+), 72 deletions(-) create mode 100644 consensus/quorum/one-node-one-vote.go create mode 100644 consensus/quorum/one-node-staked-vote.go diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index efe440d8e..2cb582bcf 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -288,6 +288,9 @@ func setupConsensusAndNode(nodeConfig *nodeconfig.ConfigType) *node.Node { currentConsensus, err := consensus.New( myHost, nodeConfig.ShardID, p2p.Peer{}, nodeConfig.ConsensusPriKey, decider, ) + currentConsensus.Decider.SetShardIDProvider(func() (uint32, error) { + return currentConsensus.ShardID, nil + }) currentConsensus.SelfAddress = common.ParseAddr(initialAccount.Address) if err != nil { diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index cc917476c..6d68c905e 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -385,7 +385,7 @@ func (consensus *Consensus) onPrepare(msg *msg_pb.Message) { } logger = logger.With(). - Int64("NumReceivedSoFar", consensus.Decider.SignatoriesCount(quorum.Prepare)). + Int64("NumReceivedSoFar", consensus.Decider.SignersCount(quorum.Prepare)). Int64("PublicKeys", consensus.Decider.ParticipantsCount()).Logger() logger.Info().Msg("[OnPrepare] Received New Prepare Signature") consensus.Decider.AddSignature(quorum.Prepare, validatorPubKey, &sign) @@ -497,7 +497,7 @@ func (consensus *Consensus) onPrepared(msg *msg_pb.Message) { utils.Logger().Error().Err(err).Msg("ReadSignatureBitmapPayload failed!!") return } - prepareCount := consensus.Decider.SignatoriesCount(quorum.Prepare) + prepareCount := consensus.Decider.SignersCount(quorum.Prepare) if count := utils.CountOneBits(mask.Bitmap); count < prepareCount { utils.Logger().Debug(). Int64("Need", prepareCount). @@ -729,7 +729,7 @@ func (consensus *Consensus) onCommit(msg *msg_pb.Message) { } logger = logger.With(). - Int64("numReceivedSoFar", consensus.Decider.SignatoriesCount(quorum.Commit)). + Int64("numReceivedSoFar", consensus.Decider.SignersCount(quorum.Commit)). Logger() logger.Info().Msg("[OnCommit] Received new commit message") consensus.Decider.AddSignature(quorum.Commit, validatorPubKey, &sign) @@ -761,7 +761,7 @@ func (consensus *Consensus) onCommit(msg *msg_pb.Message) { func (consensus *Consensus) finalizeCommits() { utils.Logger().Info(). - Int64("NumCommits", consensus.Decider.SignatoriesCount(quorum.Commit)). + Int64("NumCommits", consensus.Decider.SignersCount(quorum.Commit)). Msg("[Finalizing] Finalizing Block") beforeCatchupNum := consensus.blockNum @@ -885,7 +885,7 @@ func (consensus *Consensus) onCommitted(msg *msg_pb.Message) { switch consensus.Decider.Policy() { case quorum.SuperMajorityVote: - threshold := consensus.Decider.QuorumThreshold() + threshold := consensus.Decider.QuorumThreshold().Int64() if count := utils.CountOneBits(mask.Bitmap); int64(count) < threshold { utils.Logger().Warn(). Int64("need", threshold). diff --git a/consensus/quorum/one-node-one-vote.go b/consensus/quorum/one-node-one-vote.go new file mode 100644 index 000000000..23a9867ef --- /dev/null +++ b/consensus/quorum/one-node-one-vote.go @@ -0,0 +1,78 @@ +package quorum + +import ( + "math/big" + + "github.com/ethereum/go-ethereum/common" + "github.com/harmony-one/bls/ffi/go/bls" + common2 "github.com/harmony-one/harmony/internal/common" + "github.com/harmony-one/harmony/internal/utils" + // "github.com/harmony-one/harmony/staking/effective" +) + +type uniformVoteWeight struct { + SignatureReader + DependencyInjectionWriter +} + +// Policy .. +func (v *uniformVoteWeight) Policy() Policy { + return SuperMajorityVote +} + +// func (v *uniformVoteWeight) SetShardIDProvider(p func() (uint32, error)) { +// v.p = p +// } + +// IsQuorumAchieved .. +func (v *uniformVoteWeight) IsQuorumAchieved(p Phase) bool { + r := v.SignersCount(p) >= v.QuorumThreshold().Int64() + utils.Logger().Info().Str("phase", p.String()). + Int64("signers-count", v.SignersCount(p)). + Int64("threshold", v.QuorumThreshold().Int64()). + Int64("participants", v.ParticipantsCount()). + Msg("Quorum details") + return r +} + +// QuorumThreshold .. +func (v *uniformVoteWeight) QuorumThreshold() *big.Int { + return big.NewInt(v.ParticipantsCount()*2/3 + 1) +} + +// RewardThreshold .. +func (v *uniformVoteWeight) IsRewardThresholdAchieved() bool { + return v.SignersCount(Commit) >= (v.ParticipantsCount() * 9 / 10) +} + +// func (v *uniformVoteWeight) UpdateVotingPower(effective.StakeKeeper) { +// NO-OP do not add anything here +// } + +// ToggleActive for uniform vote is a no-op, always says that voter is active +func (v *uniformVoteWeight) ToggleActive(*bls.PublicKey) bool { + // NO-OP do not add anything here + return true +} + +// Award .. +func (v *uniformVoteWeight) Award( + // Here hook is the callback which gets the amount the earner is due in just reward + // up to the hook to do side-effects like write the statedb + Pie *big.Int, earners []common2.Address, hook func(earner common.Address, due *big.Int), +) *big.Int { + payout := big.NewInt(0) + last := big.NewInt(0) + count := big.NewInt(int64(len(earners))) + + for i, account := range earners { + cur := big.NewInt(0) + cur.Mul(Pie, big.NewInt(int64(i+1))).Div(cur, count) + diff := big.NewInt(0).Sub(cur, last) + hook(common.Address(account), diff) + payout = big.NewInt(0).Add(payout, diff) + last = cur + } + + return payout +} diff --git a/consensus/quorum/one-node-staked-vote.go b/consensus/quorum/one-node-staked-vote.go new file mode 100644 index 000000000..6af84a7aa --- /dev/null +++ b/consensus/quorum/one-node-staked-vote.go @@ -0,0 +1,75 @@ +package quorum + +import ( + "math/big" + + "github.com/harmony-one/bls/ffi/go/bls" + "github.com/harmony-one/harmony/internal/common" + "github.com/harmony-one/harmony/numeric" + "github.com/harmony-one/harmony/shard" +) + +var ( + twoThirds = numeric.NewDec(2).QuoInt64(3).Int +) + +type stakedVoter struct { + isActive, isHarmonyNode bool + effective numeric.Dec +} + +type stakedVoteWeight struct { + SignatureReader + DependencyInjectionWriter + // EPOS based staking + validatorStakes map[[shard.PublicKeySizeInBytes]byte]stakedVoter + totalEffectiveStakedAmount *big.Int +} + +// Policy .. +func (v *stakedVoteWeight) Policy() Policy { + return SuperMajorityStake +} + +// We must maintain 2/3 quoroum, so whatever is 2/3 staked amount, +// we divide that out & you +// IsQuorumAchieved .. +func (v *stakedVoteWeight) IsQuorumAchieved(p Phase) bool { + // TODO Implement this logic + return true +} + +// QuorumThreshold .. +func (v *stakedVoteWeight) QuorumThreshold() *big.Int { + return new(big.Int).Mul(v.totalEffectiveStakedAmount, twoThirds) +} + +// RewardThreshold .. +func (v *stakedVoteWeight) IsRewardThresholdAchieved() bool { + // TODO Implement + return false +} + +// HACK +var ( + hSentinel = big.NewInt(0) + hEffectiveSentinel = numeric.ZeroDec() +) + +// Award .. +func (v *stakedVoteWeight) Award( + Pie *big.Int, earners []common.Address, hook func(earner common.Address, due *big.Int), +) *big.Int { + // TODO Implement + return nil +} + +// UpdateVotingPower called only at epoch change, prob need to move to CalculateShardState +// func (v *stakedVoteWeight) UpdateVotingPower(keeper effective.StakeKeeper) { +// TODO Implement +// } + +func (v *stakedVoteWeight) ToggleActive(*bls.PublicKey) bool { + // TODO Implement + return true +} diff --git a/consensus/quorum/quorum.go b/consensus/quorum/quorum.go index 6afcf4fc3..61d91cfc6 100644 --- a/consensus/quorum/quorum.go +++ b/consensus/quorum/quorum.go @@ -1,7 +1,12 @@ package quorum import ( + "fmt" + "math/big" + "github.com/harmony-one/bls/ffi/go/bls" + "github.com/harmony-one/harmony/shard" + // "github.com/harmony-one/harmony/staking/effective" ) // Phase is a phase that needs quorum to proceed @@ -16,6 +21,19 @@ const ( ViewChange ) +var phaseNames = map[Phase]string{ + Prepare: "Announce", + Commit: "Prepare", + ViewChange: "Commit", +} + +func (p Phase) String() string { + if name, ok := phaseNames[p]; ok { + return name + } + return fmt.Sprintf("Unknown Quorum Phase %+v", byte(p)) +} + // Policy is the rule we used to decide is quorum achieved type Policy byte @@ -41,7 +59,7 @@ type SignatoryTracker interface { ParticipantTracker AddSignature(p Phase, PubKey *bls.PublicKey, sig *bls.Sign) // Caller assumes concurrency protection - SignatoriesCount(Phase) int64 + SignersCount(Phase) int64 Reset([]Phase) } @@ -52,6 +70,23 @@ type SignatureReader interface { ReadSignature(p Phase, PubKey *bls.PublicKey) *bls.Sign } +// DependencyInjectionWriter .. +type DependencyInjectionWriter interface { + SetShardIDProvider(func() (uint32, error)) +} + +// Decider .. +type Decider interface { + SignatureReader + DependencyInjectionWriter + ToggleActive(*bls.PublicKey) bool + // UpdateVotingPower(keeper effective.StakeKeeper) + Policy() Policy + IsQuorumAchieved(Phase) bool + QuorumThreshold() *big.Int + IsRewardThresholdAchieved() bool +} + // These maps represent the signatories (validators), keys are BLS public keys // and values are BLS private key signed signatures type cIdentities struct { @@ -64,6 +99,14 @@ type cIdentities struct { viewID map[string]*bls.Sign } +type depInject struct { + shardIDProvider func() (uint32, error) +} + +func (d *depInject) SetShardIDProvider(p func() (uint32, error)) { + d.shardIDProvider = p +} + func (s *cIdentities) IndexOf(pubKey *bls.PublicKey) int { idx := -1 for k, v := range s.publicKeys { @@ -104,7 +147,7 @@ func (s *cIdentities) ParticipantsCount() int64 { return int64(len(s.publicKeys)) } -func (s *cIdentities) SignatoriesCount(p Phase) int64 { +func (s *cIdentities) SignersCount(p Phase) int64 { switch p { case Prepare: return int64(len(s.prepare)) @@ -164,9 +207,7 @@ func (s *cIdentities) ReadSignature(p Phase, PubKey *bls.PublicKey) *bls.Sign { } func (s *cIdentities) ReadAllSignatures(p Phase) []*bls.Sign { - sigs := []*bls.Sign{} m := map[string]*bls.Sign{} - switch p { case Prepare: m = s.prepare @@ -175,9 +216,9 @@ func (s *cIdentities) ReadAllSignatures(p Phase) []*bls.Sign { case ViewChange: m = s.viewID } - - for _, sig := range m { - sigs = append(sigs, sig) + sigs := make([]*bls.Sign, 0, len(m)) + for _, value := range m { + sigs = append(sigs, value) } return sigs } @@ -189,47 +230,22 @@ func newMapBackedSignatureReader() SignatureReader { } } -// Decider .. -type Decider interface { - SignatureReader - Policy() Policy - IsQuorumAchieved(Phase) bool - QuorumThreshold() int64 - IsRewardThresholdAchieved() bool -} - -type uniformVoteWeight struct { - SignatureReader -} - // NewDecider .. func NewDecider(p Policy) Decider { + signatureStore := newMapBackedSignatureReader() + dependencies := &depInject{} switch p { case SuperMajorityVote: - return &uniformVoteWeight{newMapBackedSignatureReader()} - // case SuperMajorityStake: + return &uniformVoteWeight{signatureStore, dependencies} + case SuperMajorityStake: + return &stakedVoteWeight{ + signatureStore, + dependencies, + map[[shard.PublicKeySizeInBytes]byte]stakedVoter{}, + big.NewInt(0), + } default: // Should not be possible return nil } } - -// Policy .. -func (v *uniformVoteWeight) Policy() Policy { - return SuperMajorityVote -} - -// IsQuorumAchieved .. -func (v *uniformVoteWeight) IsQuorumAchieved(p Phase) bool { - return v.SignatoriesCount(p) >= v.QuorumThreshold() -} - -// QuorumThreshold .. -func (v *uniformVoteWeight) QuorumThreshold() int64 { - return v.ParticipantsCount()*2/3 + 1 -} - -// RewardThreshold .. -func (v *uniformVoteWeight) IsRewardThresholdAchieved() bool { - return v.SignatoriesCount(Commit) >= (v.ParticipantsCount() * 9 / 10) -} diff --git a/consensus/view_change.go b/consensus/view_change.go index db2aa8e91..b785f3417 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -157,8 +157,8 @@ func (consensus *Consensus) onViewChange(msg *msg_pb.Message) { if consensus.Decider.IsQuorumAchieved(quorum.ViewChange) { utils.Logger().Debug(). - Int64("have", consensus.Decider.SignatoriesCount(quorum.ViewChange)). - Int64("need", consensus.Decider.QuorumThreshold()). + Int64("have", consensus.Decider.SignersCount(quorum.ViewChange)). + Int64("need", consensus.Decider.QuorumThreshold().Int64()). Str("validatorPubKey", recvMsg.SenderPubkey.SerializeToHexStr()). Msg("[onViewChange] Received Enough View Change Messages") return @@ -282,7 +282,7 @@ func (consensus *Consensus) onViewChange(msg *msg_pb.Message) { return } // check has 2f+1 signature in m1 type message - need := consensus.Decider.QuorumThreshold() + need := consensus.Decider.QuorumThreshold().Int64() if count := utils.CountOneBits(mask.Bitmap); count < need { utils.Logger().Debug().Int64("need", need).Int64("have", count). Msg("[onViewChange] M1 Payload Not Have Enough Signature") @@ -345,8 +345,8 @@ func (consensus *Consensus) onViewChange(msg *msg_pb.Message) { // Set the bitmap indicating that this validator signed. consensus.viewIDBitmap.SetKey(recvMsg.SenderPubkey, true) utils.Logger().Debug(). - Int64("numSigs", consensus.Decider.SignatoriesCount(quorum.ViewChange)). - Int64("needed", consensus.Decider.QuorumThreshold()). + Int64("numSigs", consensus.Decider.SignersCount(quorum.ViewChange)). + Int64("needed", consensus.Decider.QuorumThreshold().Int64()). Msg("[onViewChange]") // received enough view change messages, change state to normal consensus @@ -446,7 +446,7 @@ func (consensus *Consensus) onNewView(msg *msg_pb.Message) { viewIDBytes := make([]byte, 8) binary.LittleEndian.PutUint64(viewIDBytes, recvMsg.ViewID) // check total number of sigs >= 2f+1 - need := consensus.Decider.QuorumThreshold() + need := consensus.Decider.QuorumThreshold().Int64() if count := utils.CountOneBits(m3Mask.Bitmap); count < need { utils.Logger().Debug().Int64("need", need).Int64("have", count). Msg("[onNewView] Not Have Enough M3 (ViewID) Signature") diff --git a/core/blockchain.go b/core/blockchain.go index 33fb29f92..1e5f3a9cb 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1935,12 +1935,12 @@ func (bc *BlockChain) GetShardState(epoch *big.Int) (shard.State, error) { } if epoch.Cmp(big.NewInt(GenesisEpoch)) == 0 { - shardState, err = committee.WithStakingEnabled.ReadFromComputation( + shardState, err = committee.WithStakingEnabled.Compute( big.NewInt(GenesisEpoch), *bc.Config(), nil, ) } else { prevEpoch := new(big.Int).Sub(epoch, common.Big1) - shardState, err = committee.WithStakingEnabled.ReadFromChain( + shardState, err = committee.WithStakingEnabled.ReadFromDB( prevEpoch, bc, ) } diff --git a/internal/chain/engine.go b/internal/chain/engine.go index e927e6a65..922987dea 100644 --- a/internal/chain/engine.go +++ b/internal/chain/engine.go @@ -167,7 +167,7 @@ func (e *engineImpl) Finalize( func QuorumForBlock(chain engine.ChainReader, h *block.Header, reCalculate bool) (quorum int, err error) { var ss shard.State if reCalculate { - ss, _ = committee.WithStakingEnabled.ReadFromComputation(h.Epoch(), *chain.Config(), nil) + ss, _ = committee.WithStakingEnabled.Compute(h.Epoch(), *chain.Config(), nil) } else { ss, err = chain.ReadShardState(h.Epoch()) if err != nil { @@ -226,7 +226,7 @@ func GetPublicKeys(chain engine.ChainReader, header *block.Header, reCalculate b var shardState shard.State var err error if reCalculate { - shardState, _ = committee.WithStakingEnabled.ReadFromComputation(header.Epoch(), *chain.Config(), nil) + shardState, _ = committee.WithStakingEnabled.Compute(header.Epoch(), *chain.Config(), nil) } else { shardState, err = chain.ReadShardState(header.Epoch()) if err != nil { diff --git a/node/node_explorer.go b/node/node_explorer.go index 70fa2fe6e..3919c3890 100644 --- a/node/node_explorer.go +++ b/node/node_explorer.go @@ -50,7 +50,7 @@ func (node *Node) ExplorerMessageHandler(payload []byte) { } // check has 2f+1 signatures - need := node.Consensus.Decider.QuorumThreshold() + need := node.Consensus.Decider.QuorumThreshold().Int64() if count := utils.CountOneBits(mask.Bitmap); count < need { utils.Logger().Error().Int64("need", need).Int64("have", count). Msg("[Explorer] not have enough signature") diff --git a/node/node_genesis.go b/node/node_genesis.go index bc9740743..61400eb5b 100644 --- a/node/node_genesis.go +++ b/node/node_genesis.go @@ -8,10 +8,8 @@ import ( "strings" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" - blockfactory "github.com/harmony-one/harmony/block/factory" "github.com/harmony-one/harmony/common/denominations" "github.com/harmony-one/harmony/core" @@ -42,7 +40,7 @@ type genesisInitializer struct { // InitChainDB sets up a new genesis block in the database for the given shard. func (gi *genesisInitializer) InitChainDB(db ethdb.Database, shardID uint32) error { - shardState, _ := committee.WithStakingEnabled.ReadFromComputation( + shardState, _ := committee.WithStakingEnabled.Compute( big.NewInt(core.GenesisEpoch), gi.node.chainConfig, nil, ) if shardID != shard.BeaconChainShardID { diff --git a/node/node_newblock.go b/node/node_newblock.go index f6df9c1bd..515ec4212 100644 --- a/node/node_newblock.go +++ b/node/node_newblock.go @@ -127,7 +127,7 @@ func (node *Node) proposeShardStateWithoutBeaconSync(block *types.Block) shard.S if block == nil || !shard.Schedule.IsLastBlock(block.Number().Uint64()) { return nil } - shardState, _ := committee.WithStakingEnabled.ReadFromComputation( + shardState, _ := committee.WithStakingEnabled.Compute( new(big.Int).Add(block.Header().Epoch(), common.Big1), node.chainConfig, nil, ) return shardState @@ -151,7 +151,7 @@ func (node *Node) proposeBeaconShardState(block *types.Block) error { } // TODO Use ReadFromComputation prevEpoch := new(big.Int).Sub(block.Header().Epoch(), common.Big1) - shardState, err := committee.WithStakingEnabled.ReadFromChain( + shardState, err := committee.WithStakingEnabled.ReadFromDB( prevEpoch, node.Blockchain(), ) if err != nil { diff --git a/node/node_resharding.go b/node/node_resharding.go index 0c55369d9..c15e5bf04 100644 --- a/node/node_resharding.go +++ b/node/node_resharding.go @@ -57,7 +57,7 @@ func (node *Node) validateNewShardState(block *types.Block) error { // TODO ek – this may be called from regular shards, // for vetting beacon chain blocks received during block syncing. // DRand may or or may not get in the way. Test this out. - expected, err := committee.WithStakingEnabled.ReadFromChain( + expected, err := committee.WithStakingEnabled.ReadFromDB( new(big.Int).Sub(block.Header().Epoch(), common.Big1), node.Beaconchain(), ) diff --git a/node/worker/worker.go b/node/worker/worker.go index 8fc3a2054..e3cd959c9 100644 --- a/node/worker/worker.go +++ b/node/worker/worker.go @@ -368,7 +368,7 @@ func (w *Worker) ProposeShardStateWithoutBeaconSync() shard.State { if !shard.Schedule.IsLastBlock(w.current.header.Number().Uint64()) { return nil } - shardState, _ := committee.WithStakingEnabled.ReadFromComputation( + shardState, _ := committee.WithStakingEnabled.Compute( new(big.Int).Add(w.current.header.Epoch(), common.Big1), *w.config, nil, ) return shardState diff --git a/shard/committee/assignment.go b/shard/committee/assignment.go index 69afb9259..11143dd27 100644 --- a/shard/committee/assignment.go +++ b/shard/committee/assignment.go @@ -21,12 +21,12 @@ import ( // a shardID parameter const StateID = -1 -// MembershipList .. -type MembershipList interface { - ReadFromComputation( +// ValidatorList .. +type ValidatorList interface { + Compute( epoch *big.Int, config params.ChainConfig, reader StakingCandidatesReader, ) (shard.State, error) - ReadFromChain(epoch *big.Int, reader ChainReader) (shard.State, error) + ReadFromDB(epoch *big.Int, reader ChainReader) (shard.State, error) } // PublicKeys per epoch @@ -45,7 +45,7 @@ type PublicKeys interface { // Reader .. type Reader interface { PublicKeys - MembershipList + ValidatorList } // StakingCandidatesReader .. @@ -243,14 +243,14 @@ func (def partialStakingEnabled) ComputePublicKeys( return nil, nil } -func (def partialStakingEnabled) ReadFromChain( +func (def partialStakingEnabled) ReadFromDB( epoch *big.Int, reader ChainReader, ) (newSuperComm shard.State, err error) { return reader.ReadShardState(epoch) } // ReadFromComputation is single entry point for reading the State of the network -func (def partialStakingEnabled) ReadFromComputation( +func (def partialStakingEnabled) Compute( epoch *big.Int, config params.ChainConfig, stakerReader StakingCandidatesReader, ) (newSuperComm shard.State, err error) { instance := shard.Schedule.InstanceForEpoch(epoch) From a8163205d13337833653941e010dbd8b033ed346 Mon Sep 17 00:00:00 2001 From: Edgar Aroutiounian Date: Sun, 10 Nov 2019 21:37:05 -0800 Subject: [PATCH 44/45] [reward] Abstract out block-reward, use rewarder, check result correct (#1820) * [reward] Factor out interface for block-reward * [reward] Use factored out block rewarder which is actually same object as quorum.Decider * [quorum] Fix type error because of silly internal/common.Address vs common.Address of ethereum * [testing] Somehow this port being in tandem with previously used one fixes a build timing issue --- consensus/consensus_leader_msg_test.go | 2 +- consensus/engine/consensus_engine.go | 7 ++++ consensus/quorum/one-node-one-vote.go | 7 +--- consensus/quorum/one-node-staked-vote.go | 2 +- consensus/reward/rewarder.go | 16 +++++++ internal/chain/engine.go | 19 +++++++-- internal/chain/reward.go | 53 ++++++++++++++++-------- node/node.go | 2 + 8 files changed, 80 insertions(+), 28 deletions(-) create mode 100644 consensus/reward/rewarder.go diff --git a/consensus/consensus_leader_msg_test.go b/consensus/consensus_leader_msg_test.go index 9abff5be1..46a496ecb 100644 --- a/consensus/consensus_leader_msg_test.go +++ b/consensus/consensus_leader_msg_test.go @@ -46,7 +46,7 @@ func TestConstructAnnounceMessage(test *testing.T) { func TestConstructPreparedMessage(test *testing.T) { leaderPriKey := bls.RandPrivateKey() leaderPubKey := leaderPriKey.GetPublicKey() - leader := p2p.Peer{IP: "127.0.0.1", Port: "6000", ConsensusPubKey: leaderPubKey} + leader := p2p.Peer{IP: "127.0.0.1", Port: "19999", ConsensusPubKey: leaderPubKey} validatorPriKey := bls.RandPrivateKey() validatorPubKey := leaderPriKey.GetPublicKey() diff --git a/consensus/engine/consensus_engine.go b/consensus/engine/consensus_engine.go index 03bbe93d9..88f6dd887 100644 --- a/consensus/engine/consensus_engine.go +++ b/consensus/engine/consensus_engine.go @@ -5,6 +5,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/harmony-one/harmony/block" + "github.com/harmony-one/harmony/consensus/reward" "github.com/harmony-one/harmony/core/state" "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/internal/params" @@ -73,6 +74,12 @@ type Engine interface { // rules of a particular engine. The changes are executed inline. Prepare(chain ChainReader, header *block.Header) error + // Rewarder handles the distribution of block rewards + Rewarder() reward.Distributor + + // SetRewarder assigns the Distributor used in block reward + SetRewarder(reward.Distributor) + // Finalize runs any post-transaction state modifications (e.g. block rewards) // and assembles the final block. // Note: The block header and state database might be updated to reflect any diff --git a/consensus/quorum/one-node-one-vote.go b/consensus/quorum/one-node-one-vote.go index 23a9867ef..8595a9c31 100644 --- a/consensus/quorum/one-node-one-vote.go +++ b/consensus/quorum/one-node-one-vote.go @@ -5,7 +5,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/harmony-one/bls/ffi/go/bls" - common2 "github.com/harmony-one/harmony/internal/common" "github.com/harmony-one/harmony/internal/utils" // "github.com/harmony-one/harmony/staking/effective" ) @@ -20,10 +19,6 @@ func (v *uniformVoteWeight) Policy() Policy { return SuperMajorityVote } -// func (v *uniformVoteWeight) SetShardIDProvider(p func() (uint32, error)) { -// v.p = p -// } - // IsQuorumAchieved .. func (v *uniformVoteWeight) IsQuorumAchieved(p Phase) bool { r := v.SignersCount(p) >= v.QuorumThreshold().Int64() @@ -59,7 +54,7 @@ func (v *uniformVoteWeight) ToggleActive(*bls.PublicKey) bool { func (v *uniformVoteWeight) Award( // Here hook is the callback which gets the amount the earner is due in just reward // up to the hook to do side-effects like write the statedb - Pie *big.Int, earners []common2.Address, hook func(earner common.Address, due *big.Int), + Pie *big.Int, earners []common.Address, hook func(earner common.Address, due *big.Int), ) *big.Int { payout := big.NewInt(0) last := big.NewInt(0) diff --git a/consensus/quorum/one-node-staked-vote.go b/consensus/quorum/one-node-staked-vote.go index 6af84a7aa..efd2b385b 100644 --- a/consensus/quorum/one-node-staked-vote.go +++ b/consensus/quorum/one-node-staked-vote.go @@ -3,8 +3,8 @@ package quorum import ( "math/big" + "github.com/ethereum/go-ethereum/common" "github.com/harmony-one/bls/ffi/go/bls" - "github.com/harmony-one/harmony/internal/common" "github.com/harmony-one/harmony/numeric" "github.com/harmony-one/harmony/shard" ) diff --git a/consensus/reward/rewarder.go b/consensus/reward/rewarder.go new file mode 100644 index 000000000..e13c91cb2 --- /dev/null +++ b/consensus/reward/rewarder.go @@ -0,0 +1,16 @@ +package reward + +import ( + "math/big" + + "github.com/ethereum/go-ethereum/common" +) + +// Distributor .. +type Distributor interface { + Award( + Pie *big.Int, + earners []common.Address, + hook func(earner common.Address, due *big.Int), + ) (payout *big.Int) +} diff --git a/internal/chain/engine.go b/internal/chain/engine.go index 922987dea..20b96e432 100644 --- a/internal/chain/engine.go +++ b/internal/chain/engine.go @@ -8,6 +8,7 @@ import ( "github.com/harmony-one/bls/ffi/go/bls" "github.com/harmony-one/harmony/block" "github.com/harmony-one/harmony/consensus/engine" + "github.com/harmony-one/harmony/consensus/reward" "github.com/harmony-one/harmony/core/state" "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/internal/ctxerror" @@ -19,10 +20,22 @@ import ( "golang.org/x/crypto/sha3" ) -type engineImpl struct{} +type engineImpl struct { + d reward.Distributor +} // Engine is an algorithm-agnostic consensus engine. -var Engine = &engineImpl{} +var Engine = &engineImpl{nil} + +// Rewarder handles the distribution of block rewards +func (e *engineImpl) Rewarder() reward.Distributor { + return e.d +} + +// SetRewarder .. +func (e *engineImpl) SetRewarder(d reward.Distributor) { + e.d = d +} // SealHash returns the hash of a block prior to it being sealed. func (e *engineImpl) SealHash(header *block.Header) (hash common.Hash) { @@ -156,7 +169,7 @@ func (e *engineImpl) Finalize( incxs []*types.CXReceiptsProof, stks []*staking.StakingTransaction) (*types.Block, error) { // Accumulate any block and uncle rewards and commit the final state root // Header seems complete, assemble into a block and return - if err := AccumulateRewards(chain, state, header); err != nil { + if err := AccumulateRewards(chain, state, header, e.Rewarder()); err != nil { return nil, ctxerror.New("cannot pay block reward").WithCause(err) } header.SetRoot(state.IntermediateRoot(chain.Config().IsS3(header.Epoch()))) diff --git a/internal/chain/reward.go b/internal/chain/reward.go index 5ee12a212..c4f585161 100644 --- a/internal/chain/reward.go +++ b/internal/chain/reward.go @@ -8,21 +8,26 @@ import ( "github.com/harmony-one/harmony/block" "github.com/harmony-one/harmony/common/denominations" "github.com/harmony-one/harmony/consensus/engine" + "github.com/harmony-one/harmony/consensus/reward" "github.com/harmony-one/harmony/core/state" bls2 "github.com/harmony-one/harmony/crypto/bls" common2 "github.com/harmony-one/harmony/internal/common" "github.com/harmony-one/harmony/internal/ctxerror" "github.com/harmony-one/harmony/internal/utils" + "github.com/pkg/errors" ) -// BlockReward is the block reward, to be split evenly among block signers. -var BlockReward = new(big.Int).Mul(big.NewInt(24), big.NewInt(denominations.One)) +var ( + // BlockReward is the block reward, to be split evenly among block signers. + BlockReward = new(big.Int).Mul(big.NewInt(24), big.NewInt(denominations.One)) + errPayoutNotEqualBlockReward = errors.New("total payout not equal to blockreward") +) // AccumulateRewards credits the coinbase of the given block with the mining // reward. The total reward consists of the static block reward and rewards for // included uncles. The coinbase of each uncle block is also rewarded. func AccumulateRewards( - bc engine.ChainReader, state *state.DB, header *block.Header, + bc engine.ChainReader, state *state.DB, header *block.Header, rewarder reward.Distributor, ) error { blockNum := header.Number().Uint64() if blockNum == 0 { @@ -72,9 +77,9 @@ func AccumulateRewards( if err := mask.SetMask(header.LastCommitBitmap()); err != nil { return ctxerror.New("cannot set group sig mask bits").WithCause(err) } - totalAmount := big.NewInt(0) - var accounts []common.Address - signers := []string{} + + accounts := []common.Address{} + for idx, member := range parentCommittee.NodeList { if signed, err := mask.IndexEnabled(idx); err != nil { return ctxerror.New("cannot check for committer bit", @@ -85,19 +90,33 @@ func AccumulateRewards( } } - numAccounts := big.NewInt(int64(len(accounts))) - last := new(big.Int) - for i, account := range accounts { - cur := new(big.Int) - cur.Mul(BlockReward, big.NewInt(int64(i+1))).Div(cur, numAccounts) - diff := new(big.Int).Sub(cur, last) - signers = append(signers, common2.MustAddressToBech32(account)) - state.AddBalance(account, diff) - totalAmount = new(big.Int).Add(totalAmount, diff) - last = cur + type t struct { + common.Address + *big.Int } + signers := []string{} + payable := []t{} + + totalAmount := rewarder.Award( + BlockReward, accounts, func(receipient common.Address, amount *big.Int) { + signers = append(signers, common2.MustAddressToBech32(receipient)) + payable = append(payable, t{receipient, amount}) + }) + + if totalAmount.Cmp(BlockReward) != 0 { + utils.Logger().Error(). + Int64("block-reward", BlockReward.Int64()). + Int64("total-amount-paid-out", totalAmount.Int64()). + Msg("Total paid out was not equal to block-reward") + return errors.Wrapf(errPayoutNotEqualBlockReward, "payout "+totalAmount.String()) + } + + for i := range payable { + state.AddBalance(payable[i].Address, payable[i].Int) + } + header.Logger(utils.Logger()).Debug(). - Str("NumAccounts", numAccounts.String()). + Int("NumAccounts", len(accounts)). Str("TotalAmount", totalAmount.String()). Strs("Signers", signers). Msg("[Block Reward] Successfully paid out block reward") diff --git a/node/node.go b/node/node.go index 8e97b23e0..5221ae2c0 100644 --- a/node/node.go +++ b/node/node.go @@ -17,6 +17,7 @@ import ( "github.com/harmony-one/harmony/api/service/syncing/downloader" "github.com/harmony-one/harmony/block" "github.com/harmony-one/harmony/consensus" + "github.com/harmony-one/harmony/consensus/reward" "github.com/harmony-one/harmony/contracts" "github.com/harmony-one/harmony/core" "github.com/harmony-one/harmony/core/types" @@ -514,6 +515,7 @@ func New(host p2p.Host, consensusObj *consensus.Consensus, chainDBFactory shardc node.pendingTransactions = make(map[common.Hash]*types.Transaction) node.pendingStakingTransactions = make(map[common.Hash]*staking.StakingTransaction) node.Consensus.VerifiedNewBlock = make(chan *types.Block) + chain.Engine.SetRewarder(node.Consensus.Decider.(reward.Distributor)) // the sequence number is the next block number to be added in consensus protocol, which is always one more than current chain header block node.Consensus.SetBlockNum(blockchain.CurrentBlock().NumberU64() + 1) From 381a7bd02da4ade46c698d3f757023b969a937d6 Mon Sep 17 00:00:00 2001 From: Edgar Aroutiounian Date: Sun, 10 Nov 2019 22:01:51 -0800 Subject: [PATCH 45/45] [project] Fix two leaking tickers (#1821) --- consensus/consensus_v2.go | 1 + node/node_handler.go | 1 + 2 files changed, 2 insertions(+) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 6d68c905e..591fb0dfe 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -1085,6 +1085,7 @@ func (consensus *Consensus) Start(blockChannel chan *types.Block, stopChan chan utils.Logger().Info().Time("time", time.Now()).Msg("[ConsensusMainLoop] Consensus started") defer close(stoppedChan) ticker := time.NewTicker(3 * time.Second) + defer ticker.Stop() consensus.consensusTimeout[timeoutBootstrap].Start() utils.Logger().Debug(). Uint64("viewID", consensus.viewID). diff --git a/node/node_handler.go b/node/node_handler.go index 947cbfd00..7d54d52ac 100644 --- a/node/node_handler.go +++ b/node/node_handler.go @@ -479,6 +479,7 @@ func (node *Node) pingMessageHandler(msgPayload []byte, sender libp2p_peer.ID) i // bootstrapConsensus is the a goroutine to check number of peers and start the consensus func (node *Node) bootstrapConsensus() { tick := time.NewTicker(5 * time.Second) + defer tick.Stop() lastPeerNum := node.numPeers for { select {