From 92f80b04458d66086f04c5c82e78d0f18ed7cdd6 Mon Sep 17 00:00:00 2001 From: Rongjian Lan Date: Sat, 16 Nov 2019 13:53:44 -0800 Subject: [PATCH 01/14] Add validator snapshot in local db --- consensus/engine/consensus_engine.go | 4 +- core/blockchain.go | 197 ++++++++++++++++++++++----- core/chain_makers.go | 2 +- core/rawdb/accessors_chain.go | 75 ++++++++-- core/rawdb/schema.go | 14 +- core/state_transition.go | 2 +- hmy/api_backend.go | 6 +- internal/chain/engine.go | 2 +- internal/hmyapi/backend.go | 2 +- node/node_handler.go | 2 +- staking/types/validator.go | 3 - 11 files changed, 244 insertions(+), 65 deletions(-) diff --git a/consensus/engine/consensus_engine.go b/consensus/engine/consensus_engine.go index 980b160c6..a2a9ad8b5 100644 --- a/consensus/engine/consensus_engine.go +++ b/consensus/engine/consensus_engine.go @@ -40,8 +40,8 @@ type ChainReader interface { // Thus, only should be used to read the shard state of the current chain. ReadShardState(epoch *big.Int) (shard.State, error) - // CurrentValidatorAddresses retrieves the current list of validators - CurrentValidatorAddresses() []common.Address + // ActiveValidatorAddresses retrieves the list of active validators + ActiveValidatorAddresses() []common.Address } // Engine is an algorithm agnostic consensus engine. diff --git a/core/blockchain.go b/core/blockchain.go index ed12c4d62..4cb8ad040 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -70,9 +70,9 @@ const ( commitsCacheLimit = 10 epochCacheLimit = 10 randomnessCacheLimit = 10 - stakingCacheLimit = 256 - validatorListCacheLimit = 2 - validatorListByDelegatorCacheLimit = 256 + validatorCacheLimit = 1024 + validatorListCacheLimit = 10 + validatorListByDelegatorCacheLimit = 1024 // BlockChainVersion ensures that an incompatible database forces a resync from scratch. BlockChainVersion = 3 @@ -135,7 +135,7 @@ type BlockChain struct { lastCommitsCache *lru.Cache epochCache *lru.Cache // Cache epoch number → first block number randomnessCache *lru.Cache // Cache for vrf/vdf - stakingCache *lru.Cache // Cache for staking validator + validatorCache *lru.Cache // Cache for staking validator validatorListCache *lru.Cache // Cache of validator list validatorListByDelegatorCache *lru.Cache // Cache of validator list by delegator @@ -174,7 +174,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par commitsCache, _ := lru.New(commitsCacheLimit) epochCache, _ := lru.New(epochCacheLimit) randomnessCache, _ := lru.New(randomnessCacheLimit) - stakingCache, _ := lru.New(stakingCacheLimit) + stakingCache, _ := lru.New(validatorCacheLimit) validatorListCache, _ := lru.New(validatorListCacheLimit) validatorListByDelegatorCache, _ := lru.New(validatorListByDelegatorCacheLimit) @@ -195,7 +195,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par lastCommitsCache: commitsCache, epochCache: epochCache, randomnessCache: randomnessCache, - stakingCache: stakingCache, + validatorCache: stakingCache, validatorListCache: validatorListCache, validatorListByDelegatorCache: validatorListByDelegatorCache, engine: engine, @@ -1078,6 +1078,7 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types. batch := bc.db.NewBatch() rawdb.WriteReceipts(batch, block.Hash(), block.NumberU64(), receipts) + //// Cross-shard txns epoch := block.Header().Epoch() if bc.chainConfig.IsCrossTx(block.Epoch()) { shardingConfig := shard.Schedule.InstanceForEpoch(epoch) @@ -1097,6 +1098,7 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types. bc.WriteCXReceiptsProofSpent(block.IncomingReceipts()) } + //// VRF + VDF //check non zero VRF field in header and add to local db if len(block.Vrf()) > 0 { vrfBlockNumbers, _ := bc.ReadEpochVrfBlockNums(block.Header().Epoch()) @@ -1130,16 +1132,45 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types. } } + //// Shard State and Validator Update header := block.Header() if header.ShardStateHash() != (common.Hash{}) { epoch := new(big.Int).Add(header.Epoch(), common.Big1) - err = bc.WriteShardStateBytes(batch, epoch, header.ShardState()) + shardState, err := bc.WriteShardStateBytes(batch, epoch, header.ShardState()) if err != nil { header.Logger(utils.Logger()).Warn().Err(err).Msg("cannot store shard state") return NonStatTy, err } + + processed := make(map[common.Address]struct{}) + allActiveValidators := []common.Address{} + for i := range *shardState { + shard := (*shardState)[i] + for j := range shard.NodeList { + if shard.NodeList[j].StakeWithDelegationApplied != nil { // For external validator + _, ok := processed[shard.NodeList[j].EcdsaAddress] + if !ok { + allActiveValidators = append(allActiveValidators, shard.NodeList[j].EcdsaAddress) + } + } + } + } + bc.UpdateActiveValidatorsSnapshot(allActiveValidators) } + if bc.chainConfig.IsStaking(block.Epoch()) { + for _, tx := range block.StakingTransactions() { + err = bc.UpdateStakingMetaData(tx) + // keep offchain database consistency with onchain we need revert + // but it should not happend unless local database corrupted + if err != nil { + utils.Logger().Debug().Msgf("oops, UpdateStakingMetaData failed, err: %+v", err) + return NonStatTy, err + } + } + } + + //// Cross-links if len(header.CrossLinks()) > 0 { crossLinks := &types.CrossLinks{} err = rlp.DecodeBytes(header.CrossLinks(), crossLinks) @@ -1159,19 +1190,6 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types. bc.WriteShardLastCrossLink(crossLink.ShardID(), crossLink) } } - - if bc.chainConfig.IsStaking(block.Epoch()) { - for _, tx := range block.StakingTransactions() { - err = bc.UpdateStakingMetaData(tx) - // keep offchain database consistency with onchain we need revert - // but it should not happend unless local database corrupted - if err != nil { - utils.Logger().Debug().Msgf("oops, UpdateStakingMetaData failed, err: %+v", err) - return NonStatTy, err - } - } - } - /////////////////////////// END // If the total difficulty is higher than our known, add it to the canonical chain @@ -1870,18 +1888,18 @@ func (bc *BlockChain) WriteShardState( // WriteShardStateBytes saves the given sharding state under the given epoch number. func (bc *BlockChain) WriteShardStateBytes(db rawdb.DatabaseWriter, epoch *big.Int, shardState []byte, -) error { +) (*shard.State, error) { decodeShardState := shard.State{} if err := rlp.DecodeBytes(shardState, &decodeShardState); err != nil { - return err + return nil, err } err := rawdb.WriteShardStateBytes(db, epoch, shardState) if err != nil { - return err + return nil, err } cacheKey := string(epoch.Bytes()) bc.shardStateCache.Add(cacheKey, decodeShardState) - return nil + return &decodeShardState, nil } // ReadLastCommits retrieves last commits. @@ -2276,9 +2294,9 @@ func (bc *BlockChain) ReadTxLookupEntry(txID common.Hash) (common.Hash, uint64, return rawdb.ReadTxLookupEntry(bc.db, txID) } -// ReadStakingValidator reads staking information of given validatorWrapper -func (bc *BlockChain) ReadStakingValidator(addr common.Address) (*staking.ValidatorWrapper, error) { - if cached, ok := bc.stakingCache.Get("staking-" + string(addr.Bytes())); ok { +// ReadValidatorData reads staking information of given validatorWrapper +func (bc *BlockChain) ReadValidatorData(addr common.Address) (*staking.ValidatorWrapper, error) { + if cached, ok := bc.validatorCache.Get("v-" + string(addr.Bytes())); ok { by := cached.([]byte) v := staking.ValidatorWrapper{} if err := rlp.DecodeBytes(by, &v); err != nil { @@ -2287,12 +2305,12 @@ func (bc *BlockChain) ReadStakingValidator(addr common.Address) (*staking.Valida return &v, nil } - return rawdb.ReadStakingValidator(bc.db, addr) + return rawdb.ReadValidatorData(bc.db, addr) } -// WriteStakingValidator reads staking information of given validatorWrapper -func (bc *BlockChain) WriteStakingValidator(v *staking.ValidatorWrapper) error { - err := rawdb.WriteStakingValidator(bc.db, v) +// WriteValidatorData writes staking information of given validatorWrapper +func (bc *BlockChain) WriteValidatorData(v *staking.ValidatorWrapper) error { + err := rawdb.WriteValidatorData(bc.db, v) if err != nil { return err } @@ -2300,7 +2318,68 @@ func (bc *BlockChain) WriteStakingValidator(v *staking.ValidatorWrapper) error { if err != nil { return err } - bc.stakingCache.Add("staking-"+string(v.Address.Bytes()), by) + bc.validatorCache.Add("v-"+string(v.Address.Bytes()), by) + return nil +} + +// ReadValidatorSnapshot reads the snapshot staking information of given validator address +// TODO: put epoch number in to snapshot too. +func (bc *BlockChain) ReadValidatorSnapshot(addr common.Address) (*staking.ValidatorWrapper, error) { + if cached, ok := bc.validatorCache.Get("vs-" + string(addr.Bytes())); ok { + by := cached.([]byte) + v := staking.ValidatorWrapper{} + if err := rlp.DecodeBytes(by, &v); err != nil { + return nil, err + } + return &v, nil + } + + return rawdb.ReadValidatorSnapshot(bc.db, addr) +} + +// WriteValidatorSnapshots writes the snapshot of provided list of validators +func (bc *BlockChain) WriteValidatorsSnapshot(addrs []common.Address) error { + validators := []*staking.ValidatorWrapper{} + for _, addr := range addrs { + validator, err := bc.ReadValidatorData(addr) + if err != nil { + return err + } + validators = append(validators, validator) + } + + batch := bc.db.NewBatch() + for i := range validators { + err := rawdb.WriteValidatorSnapshot(batch, validators[i]) + if err != nil { + return err + } + } + if err := batch.Write(); err != nil { + return err + } + + for i := range validators { + by, err := rlp.EncodeToBytes(validators[i]) + if err == nil { + bc.validatorCache.Add("vs-"+string(validators[i].Address.Bytes()), by) + } + } + return nil +} + +// DeleteValidatorData deletes the snapshot staking information of given validator address +func (bc *BlockChain) DeleteValidatorsSnapshot(addrs []common.Address) error { + batch := bc.db.NewBatch() + for i := range addrs { + rawdb.DeleteValidatorSnapshot(batch, addrs[i]) + } + if err := batch.Write(); err != nil { + return err + } + for i := range addrs { + bc.validatorCache.Remove("vs-" + string(addrs[i].Bytes())) + } return nil } @@ -2314,12 +2393,12 @@ func (bc *BlockChain) ReadValidatorList() ([]common.Address, error) { } return m, nil } - return rawdb.ReadValidatorList(bc.db) + return rawdb.ReadValidatorList(bc.db, false) } // WriteValidatorList writes the list of validator addresses to database func (bc *BlockChain) WriteValidatorList(addrs []common.Address) error { - err := rawdb.WriteValidatorList(bc.db, addrs) + err := rawdb.WriteValidatorList(bc.db, addrs, false) if err != nil { return err } @@ -2330,6 +2409,50 @@ func (bc *BlockChain) WriteValidatorList(addrs []common.Address) error { return nil } +// ReadActiveValidatorList reads the addresses of active validators +func (bc *BlockChain) ReadActiveValidatorList() ([]common.Address, error) { + if cached, ok := bc.validatorListCache.Get("activeValidatorList"); ok { + by := cached.([]byte) + m := []common.Address{} + if err := rlp.DecodeBytes(by, &m); err != nil { + return nil, err + } + return m, nil + } + return rawdb.ReadValidatorList(bc.db, true) +} + +// WriteActiveValidatorList writes the list of active validator addresses to database +func (bc *BlockChain) WriteActiveValidatorList(addrs []common.Address) error { + err := rawdb.WriteValidatorList(bc.db, addrs, true) + if err != nil { + return err + } + bytes, err := rlp.EncodeToBytes(addrs) + if err == nil { + bc.validatorListCache.Add("activeValidatorList", bytes) + } + return nil +} + +// UpdateActiveValidatorsSnapshot updates the list of active validators and updates the content snapshot of the active validators +func (bc *BlockChain) UpdateActiveValidatorsSnapshot(activeValidators []common.Address) error { + prevActiveValidators, err := bc.ReadActiveValidatorList() + if err != nil { + return err + } + + err = bc.DeleteValidatorsSnapshot(prevActiveValidators) + if err != nil { + return err + } + + if err = bc.WriteValidatorsSnapshot(activeValidators); err != nil { + return err + } + return nil +} + // ReadValidatorListByDelegator reads the addresses of validators delegated by a delegator func (bc *BlockChain) ReadValidatorListByDelegator(delegator common.Address) ([]common.Address, error) { if cached, ok := bc.validatorListByDelegatorCache.Get(delegator.Bytes()); ok { @@ -2413,8 +2536,9 @@ func (bc *BlockChain) UpdateStakingMetaData(tx *staking.StakingTransaction) erro return nil } -// CurrentValidatorAddresses returns the address of active validators for current epoch -func (bc *BlockChain) CurrentValidatorAddresses() []common.Address { +// ActiveValidatorAddresses returns the address of active validators for current epoch +// TODO: should only return those that are selected by epos. +func (bc *BlockChain) ActiveValidatorAddresses() []common.Address { list, err := bc.ReadValidatorList() if err != nil { return make([]common.Address, 0) @@ -2428,6 +2552,7 @@ func (bc *BlockChain) CurrentValidatorAddresses() []common.Address { if err != nil { continue } + // TODO: double check this logic here. epoch := shard.Schedule.CalcEpochNumber(val.CreationHeight.Uint64()) if epoch.Cmp(currentEpoch) >= 0 { // wait for next epoch diff --git a/core/chain_makers.go b/core/chain_makers.go index f09bc8028..51f501421 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -270,4 +270,4 @@ func (cr *fakeChainReader) GetHeaderByHash(hash common.Hash) *block.Header func (cr *fakeChainReader) GetHeader(hash common.Hash, number uint64) *block.Header { return nil } func (cr *fakeChainReader) GetBlock(hash common.Hash, number uint64) *types.Block { return nil } func (cr *fakeChainReader) ReadShardState(epoch *big.Int) (shard.State, error) { return nil, nil } -func (cr *fakeChainReader) CurrentValidatorAddresses() []common.Address { return nil } +func (cr *fakeChainReader) ActiveValidatorAddresses() []common.Address { return nil } diff --git a/core/rawdb/accessors_chain.go b/core/rawdb/accessors_chain.go index 97324d79c..db8fd7c71 100644 --- a/core/rawdb/accessors_chain.go +++ b/core/rawdb/accessors_chain.go @@ -614,11 +614,11 @@ func WriteCXReceiptsProofUnspentCheckpoint(db DatabaseWriter, shardID uint32, bl return db.Put(cxReceiptUnspentCheckpointKey(shardID), by) } -// ReadStakingValidator retrieves staking validator by its address -func ReadStakingValidator(db DatabaseReader, addr common.Address) (*staking.ValidatorWrapper, error) { - data, err := db.Get(stakingKey(addr)) +// ReadValidatorData retrieves staking validator by its address +func ReadValidatorData(db DatabaseReader, addr common.Address) (*staking.ValidatorWrapper, error) { + data, err := db.Get(validatorKey(addr)) if len(data) == 0 || err != nil { - utils.Logger().Info().Err(err).Msg("ReadStakingValidator") + utils.Logger().Info().Err(err).Msg("ReadValidatorData") return nil, err } v := staking.ValidatorWrapper{} @@ -629,21 +629,64 @@ func ReadStakingValidator(db DatabaseReader, addr common.Address) (*staking.Vali return &v, nil } -// WriteStakingValidator stores staking validator's information by its address -func WriteStakingValidator(db DatabaseWriter, v *staking.ValidatorWrapper) error { +// WriteValidatorData stores staking validator's information by its address +func WriteValidatorData(db DatabaseWriter, v *staking.ValidatorWrapper) error { bytes, err := rlp.EncodeToBytes(v) if err != nil { - utils.Logger().Error().Msg("[WriteStakingValidator] Failed to encode") + utils.Logger().Error().Msg("[WriteValidatorData] Failed to encode") + return err } - if err := db.Put(stakingKey(v.Address), bytes); err != nil { - utils.Logger().Error().Msg("[WriteStakingValidator] Failed to store to database") + if err := db.Put(validatorKey(v.Address), bytes); err != nil { + utils.Logger().Error().Msg("[WriteValidatorData] Failed to store to database") + return err } return err } +// ReadValidatorSnapshot retrieves staking validator's snapshot by its address +func ReadValidatorSnapshot(db DatabaseReader, addr common.Address) (*staking.ValidatorWrapper, error) { + data, err := db.Get(validatorSnapshotKey(addr)) + if len(data) == 0 || err != nil { + utils.Logger().Info().Err(err).Msg("ReadValidatorSnapshot") + return nil, err + } + v := staking.ValidatorWrapper{} + if err := rlp.DecodeBytes(data, &v); err != nil { + utils.Logger().Error().Err(err).Str("address", addr.Hex()).Msg("Unable to Decode staking validator from database") + return nil, err + } + return &v, nil +} + +// WriteValidatorSnapshot stores staking validator's snapshot by its address +func WriteValidatorSnapshot(db DatabaseWriter, v *staking.ValidatorWrapper) error { + bytes, err := rlp.EncodeToBytes(v) + if err != nil { + utils.Logger().Error().Msg("[WriteValidatorSnapshot] Failed to encode") + return err + } + if err := db.Put(validatorSnapshotKey(v.Address), bytes); err != nil { + utils.Logger().Error().Msg("[WriteValidatorSnapshot] Failed to store to database") + return err + } + return err +} + +// DeleteValidatorSnapshot removes the validator's snapshot by its address +func DeleteValidatorSnapshot(db DatabaseDeleter, addr common.Address) { + if err := db.Delete(validatorSnapshotKey(addr)); err != nil { + utils.Logger().Error().Msg("Failed to delete snapshot of a validator") + } +} + // ReadValidatorList retrieves staking validator by its address -func ReadValidatorList(db DatabaseReader) ([]common.Address, error) { - data, err := db.Get([]byte("validatorList")) +// Return only active validators if isActive==true, otherwise, return all validators +func ReadValidatorList(db DatabaseReader, isActive bool) ([]common.Address, error) { + key := validatorListKey + if isActive { + key = activeValidatorListKey + } + data, err := db.Get(key) if len(data) == 0 || err != nil { return []common.Address{}, nil } @@ -656,12 +699,18 @@ func ReadValidatorList(db DatabaseReader) ([]common.Address, error) { } // WriteValidatorList stores staking validator's information by its address -func WriteValidatorList(db DatabaseWriter, addrs []common.Address) error { +// Writes only for active validators if isActive==true, otherwise, writes for all validators +func WriteValidatorList(db DatabaseWriter, addrs []common.Address, isActive bool) error { + key := validatorListKey + if isActive { + key = activeValidatorListKey + } + bytes, err := rlp.EncodeToBytes(addrs) if err != nil { utils.Logger().Error().Msg("[WriteValidatorList] Failed to encode") } - if err := db.Put([]byte("validatorList"), bytes); err != nil { + if err := db.Put(key, bytes); err != nil { utils.Logger().Error().Msg("[WriteValidatorList] Failed to store to database") } return err diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index 3a4225f61..66c628f0b 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -74,7 +74,10 @@ var ( cxReceiptSpentPrefix = []byte("cxReceiptSpent") // prefix for indicator of unspent of cxReceiptsProof cxReceiptUnspentCheckpointPrefix = []byte("cxReceiptUnspentCheckpoint") // prefix for cxReceiptsProof unspent checkpoint - stakingPrefix = []byte("staking") // prefix for staking validator information + validatorPrefix = []byte("validator-") // prefix for staking validator information + validatorSnapshotPrefix = []byte("validator-snapshot-") // prefix for staking validator's snapshot information + validatorListKey = []byte("validator-list") // key for all validators list + activeValidatorListKey = []byte("active-validator-list") // key for active validators list // epochBlockNumberPrefix + epoch (big.Int.Bytes()) // -> epoch block number (big.Int.Bytes()) @@ -237,7 +240,12 @@ func cxReceiptUnspentCheckpointKey(shardID uint32) []byte { return append(prefix, sKey...) } -func stakingKey(addr common.Address) []byte { - prefix := stakingPrefix +func validatorKey(addr common.Address) []byte { + prefix := validatorPrefix + return append(prefix, addr.Bytes()...) +} + +func validatorSnapshotKey(addr common.Address) []byte { + prefix := validatorSnapshotPrefix return append(prefix, addr.Bytes()...) } diff --git a/core/state_transition.go b/core/state_transition.go index 9b8a92bb8..f076d02b2 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -354,7 +354,7 @@ func (st *StateTransition) applyCreateValidatorTx(createValidator *staking.Creat delegations := []staking.Delegation{} delegations = append(delegations, staking.NewDelegation(v.Address, createValidator.Amount)) - wrapper := staking.ValidatorWrapper{*v, delegations, nil, nil} + wrapper := staking.ValidatorWrapper{*v, delegations} if err := st.state.UpdateStakingInfo(v.Address, &wrapper); err != nil { return err diff --git a/hmy/api_backend.go b/hmy/api_backend.go index f22857630..d7d05b9dd 100644 --- a/hmy/api_backend.go +++ b/hmy/api_backend.go @@ -290,9 +290,9 @@ func (b *APIBackend) SendStakingTx( return nil } -// GetCurrentValidatorAddresses returns the address of active validators for current epoch -func (b *APIBackend) GetCurrentValidatorAddresses() []common.Address { - return b.hmy.BlockChain().CurrentValidatorAddresses() +// GetActiveValidatorAddresses returns the address of active validators for current epoch +func (b *APIBackend) GetActiveValidatorAddresses() []common.Address { + return b.hmy.BlockChain().ActiveValidatorAddresses() } // GetValidatorCandidates returns the up to date validator candidates for next epoch diff --git a/internal/chain/engine.go b/internal/chain/engine.go index ee6bc7289..42dc5a3cc 100644 --- a/internal/chain/engine.go +++ b/internal/chain/engine.go @@ -179,7 +179,7 @@ func (e *engineImpl) Finalize( // Only do such at the last block of an epoch if len(header.ShardState()) > 0 { // TODO: make sure we are using the correct validator list - validators := chain.CurrentValidatorAddresses() + validators := chain.ActiveValidatorAddresses() for _, validator := range validators { wrapper := state.GetStakingInfo(validator) if wrapper != nil { diff --git a/internal/hmyapi/backend.go b/internal/hmyapi/backend.go index deaefbb2a..f8dbdcdb4 100644 --- a/internal/hmyapi/backend.go +++ b/internal/hmyapi/backend.go @@ -73,7 +73,7 @@ type Backend interface { ResendCx(ctx context.Context, txID common.Hash) (uint64, bool) IsLeader() bool SendStakingTx(ctx context.Context, newStakingTx *staking.StakingTransaction) error - GetCurrentValidatorAddresses() []common.Address + GetActiveValidatorAddresses() []common.Address GetValidatorCandidates() []common.Address GetValidatorInformation(addr common.Address) *staking.Validator GetDelegatorsInformation(addr common.Address) []*staking.Delegation diff --git a/node/node_handler.go b/node/node_handler.go index e79ca00e3..89402cb6c 100644 --- a/node/node_handler.go +++ b/node/node_handler.go @@ -411,7 +411,7 @@ func (node *Node) AddNewBlock(newBlock *types.Block) error { } utils.Logger().Debug().Msgf("ValidatorInformation %v: %v", i, val) } - currAddrs := node.Blockchain().CurrentValidatorAddresses() + currAddrs := node.Blockchain().ActiveValidatorAddresses() utils.Logger().Debug().Msgf("CurrentValidators : %v", currAddrs) candidates := node.Blockchain().ValidatorCandidates() utils.Logger().Debug().Msgf("CandidateValidators : %v", candidates) diff --git a/staking/types/validator.go b/staking/types/validator.go index 9d7d93458..2ca0b8a1b 100644 --- a/staking/types/validator.go +++ b/staking/types/validator.go @@ -36,9 +36,6 @@ var ( type ValidatorWrapper struct { Validator `json:"validator" yaml:"validator" rlp:"nil"` Delegations []Delegation `json:"delegations" yaml:"delegations" rlp:"nil"` - // TODO: move snapshot into off-chain db. - SnapshotValidator *Validator `json:"snapshot_validator" yaml:"snaphost_validator" rlp:"nil"` - SnapshotDelegations []Delegation `json:"snapshot_delegations" yaml:"snapshot_delegations" rlp:"nil"` } // Validator - data fields for a validator From f6ffb4327799c740a97b95fc84b415cf51c12a38 Mon Sep 17 00:00:00 2001 From: Rongjian Lan Date: Sat, 16 Nov 2019 14:24:25 -0800 Subject: [PATCH 02/14] Fix bugs and refactor NodeList --- api/service/explorer/service.go | 16 +++---- api/service/explorer/storage_test.go | 8 ++-- consensus/consensus_service.go | 2 +- core/blockchain.go | 64 ++++++++++++++++------------ core/rawdb/accessors_chain.go | 6 +-- internal/chain/engine.go | 4 +- internal/chain/reward.go | 4 +- internal/hmyapi/blockchain.go | 14 +++--- node/node_cross_shard.go | 2 +- node/node_resharding.go | 6 +-- shard/committee/assignment.go | 26 +++++------ shard/shard_state.go | 42 +++++++++--------- shard/shard_state_test.go | 12 +++--- 13 files changed, 107 insertions(+), 99 deletions(-) diff --git a/api/service/explorer/service.go b/api/service/explorer/service.go index dce59d85b..17e5a0476 100644 --- a/api/service/explorer/service.go +++ b/api/service/explorer/service.go @@ -282,8 +282,8 @@ func (s *Service) GetExplorerBlocks(w http.ResponseWriter, r *http.Request) { curEpoch = int64(block.Epoch) } if withSigners { - pubkeys := make([]*bls.PublicKey, len(committee.NodeList)) - for i, validator := range committee.NodeList { + pubkeys := make([]*bls.PublicKey, len(committee.Slots)) + for i, validator := range committee.Slots { pubkeys[i] = new(bls.PublicKey) validator.BlsPublicKey.ToLibBLSPublicKey(pubkeys[i]) } @@ -291,7 +291,7 @@ func (s *Service) GetExplorerBlocks(w http.ResponseWriter, r *http.Request) { if err == nil && accountBlocks[id+1] != nil { err = mask.SetMask(accountBlocks[id+1].Header().LastCommitBitmap()) if err == nil { - for _, validator := range committee.NodeList { + for _, validator := range committee.Slots { oneAddress, err := common2.AddressToBech32(validator.EcdsaAddress) if err != nil { continue @@ -403,8 +403,8 @@ func (s *ServiceAPI) GetExplorerBlocks(ctx context.Context, from, to, page, offs curEpoch = int64(block.Epoch) } if withSigners { - pubkeys := make([]*bls.PublicKey, len(committee.NodeList)) - for i, validator := range committee.NodeList { + pubkeys := make([]*bls.PublicKey, len(committee.Slots)) + for i, validator := range committee.Slots { pubkeys[i] = new(bls.PublicKey) validator.BlsPublicKey.ToLibBLSPublicKey(pubkeys[i]) } @@ -412,7 +412,7 @@ func (s *ServiceAPI) GetExplorerBlocks(ctx context.Context, from, to, page, offs if err == nil && accountBlocks[id+1] != nil { err = mask.SetMask(accountBlocks[id+1].Header().LastCommitBitmap()) if err == nil { - for _, validator := range committee.NodeList { + for _, validator := range committee.Slots { oneAddress, err := common2.AddressToBech32(validator.EcdsaAddress) if err != nil { continue @@ -592,7 +592,7 @@ func (s *Service) GetExplorerCommittee(w http.ResponseWriter, r *http.Request) { return } validators := &Committee{} - for _, validator := range committee.NodeList { + for _, validator := range committee.Slots { validatorBalance := big.NewInt(0) validatorBalance, err := s.GetAccountBalance(validator.EcdsaAddress) if err != nil { @@ -645,7 +645,7 @@ func (s *ServiceAPI) GetExplorerCommittee(ctx context.Context, shardID uint32, e return nil, err } validators := &Committee{} - for _, validator := range committee.NodeList { + for _, validator := range committee.Slots { validatorBalance := big.NewInt(0) validatorBalance, err := s.Service.GetAccountBalance(validator.EcdsaAddress) if err != nil { diff --git a/api/service/explorer/storage_test.go b/api/service/explorer/storage_test.go index ea3832507..bd08e4c9a 100644 --- a/api/service/explorer/storage_test.go +++ b/api/service/explorer/storage_test.go @@ -90,10 +90,10 @@ func TestDumpCommittee(t *testing.T) { BlsPublicKey2 := new(shard.BlsPublicKey) BlsPublicKey1.FromLibBLSPublicKey(blsPubKey1) BlsPublicKey2.FromLibBLSPublicKey(blsPubKey2) - nodeID1 := shard.NodeID{EcdsaAddress: common.HexToAddress("52789f18a342da8023cc401e5d2b14a6b710fba9"), BlsPublicKey: *BlsPublicKey1} - nodeID2 := shard.NodeID{EcdsaAddress: common.HexToAddress("7c41e0668b551f4f902cfaec05b5bdca68b124ce"), BlsPublicKey: *BlsPublicKey2} - nodeIDList := []shard.NodeID{nodeID1, nodeID2} - committee := shard.Committee{ShardID: uint32(0), NodeList: nodeIDList} + nodeID1 := shard.Slot{EcdsaAddress: common.HexToAddress("52789f18a342da8023cc401e5d2b14a6b710fba9"), BlsPublicKey: *BlsPublicKey1} + nodeID2 := shard.Slot{EcdsaAddress: common.HexToAddress("7c41e0668b551f4f902cfaec05b5bdca68b124ce"), BlsPublicKey: *BlsPublicKey2} + nodeIDList := []shard.Slot{nodeID1, nodeID2} + committee := shard.Committee{ShardID: uint32(0), Slots: nodeIDList} shardID := uint32(0) epoch := uint64(0) ins := GetStorageInstance("1.1.1.1", "3333", true) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 278e4aea8..eda4e804e 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -428,7 +428,7 @@ func (consensus *Consensus) getLeaderPubKeyFromCoinbase(header *block.Header) (* ) } committerKey := new(bls.PublicKey) - for _, member := range committee.NodeList { + for _, member := range committee.Slots { if member.EcdsaAddress == header.Coinbase() { err := member.BlsPublicKey.ToLibBLSPublicKey(committerKey) if err != nil { diff --git a/core/blockchain.go b/core/blockchain.go index 4cb8ad040..273f481fb 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1135,6 +1135,7 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types. //// Shard State and Validator Update header := block.Header() if header.ShardStateHash() != (common.Hash{}) { + // Write shard state for the new epoch epoch := new(big.Int).Add(header.Epoch(), common.Big1) shardState, err := bc.WriteShardStateBytes(batch, epoch, header.ShardState()) if err != nil { @@ -1142,15 +1143,18 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types. return NonStatTy, err } - processed := make(map[common.Address]struct{}) + // Find all the active validator addresses and do a snapshot allActiveValidators := []common.Address{} + processed := make(map[common.Address]struct{}) for i := range *shardState { shard := (*shardState)[i] - for j := range shard.NodeList { - if shard.NodeList[j].StakeWithDelegationApplied != nil { // For external validator - _, ok := processed[shard.NodeList[j].EcdsaAddress] + for j := range shard.Slots { + slot := shard.Slots[j] + if slot.StakeWithDelegationApplied != nil { // For external validator + _, ok := processed[slot.EcdsaAddress] if !ok { - allActiveValidators = append(allActiveValidators, shard.NodeList[j].EcdsaAddress) + processed[slot.EcdsaAddress] = struct{}{} + allActiveValidators = append(allActiveValidators, shard.Slots[j].EcdsaAddress) } } } @@ -1158,6 +1162,7 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types. bc.UpdateActiveValidatorsSnapshot(allActiveValidators) } + // Do bookkeeping for new staking txns if bc.chainConfig.IsStaking(block.Epoch()) { for _, tx := range block.StakingTransactions() { err = bc.UpdateStakingMetaData(tx) @@ -2296,7 +2301,7 @@ func (bc *BlockChain) ReadTxLookupEntry(txID common.Hash) (common.Hash, uint64, // ReadValidatorData reads staking information of given validatorWrapper func (bc *BlockChain) ReadValidatorData(addr common.Address) (*staking.ValidatorWrapper, error) { - if cached, ok := bc.validatorCache.Get("v-" + string(addr.Bytes())); ok { + if cached, ok := bc.validatorCache.Get("validator-" + string(addr.Bytes())); ok { by := cached.([]byte) v := staking.ValidatorWrapper{} if err := rlp.DecodeBytes(by, &v); err != nil { @@ -2318,14 +2323,14 @@ func (bc *BlockChain) WriteValidatorData(v *staking.ValidatorWrapper) error { if err != nil { return err } - bc.validatorCache.Add("v-"+string(v.Address.Bytes()), by) + bc.validatorCache.Add("validator-"+string(v.Address.Bytes()), by) return nil } // ReadValidatorSnapshot reads the snapshot staking information of given validator address // TODO: put epoch number in to snapshot too. func (bc *BlockChain) ReadValidatorSnapshot(addr common.Address) (*staking.ValidatorWrapper, error) { - if cached, ok := bc.validatorCache.Get("vs-" + string(addr.Bytes())); ok { + if cached, ok := bc.validatorCache.Get("validator-snapshot-" + string(addr.Bytes())); ok { by := cached.([]byte) v := staking.ValidatorWrapper{} if err := rlp.DecodeBytes(by, &v); err != nil { @@ -2339,6 +2344,7 @@ func (bc *BlockChain) ReadValidatorSnapshot(addr common.Address) (*staking.Valid // WriteValidatorSnapshots writes the snapshot of provided list of validators func (bc *BlockChain) WriteValidatorsSnapshot(addrs []common.Address) error { + // Read all validator's current data validators := []*staking.ValidatorWrapper{} for _, addr := range addrs { validator, err := bc.ReadValidatorData(addr) @@ -2348,6 +2354,7 @@ func (bc *BlockChain) WriteValidatorsSnapshot(addrs []common.Address) error { validators = append(validators, validator) } + // Batch write the current data as snapshot batch := bc.db.NewBatch() for i := range validators { err := rawdb.WriteValidatorSnapshot(batch, validators[i]) @@ -2359,10 +2366,11 @@ func (bc *BlockChain) WriteValidatorsSnapshot(addrs []common.Address) error { return err } + // Update cache for i := range validators { by, err := rlp.EncodeToBytes(validators[i]) if err == nil { - bc.validatorCache.Add("vs-"+string(validators[i].Address.Bytes()), by) + bc.validatorCache.Add("validator-snapshot-"+string(validators[i].Address.Bytes()), by) } } return nil @@ -2378,7 +2386,25 @@ func (bc *BlockChain) DeleteValidatorsSnapshot(addrs []common.Address) error { return err } for i := range addrs { - bc.validatorCache.Remove("vs-" + string(addrs[i].Bytes())) + bc.validatorCache.Remove("validator-snapshot-" + string(addrs[i].Bytes())) + } + return nil +} + +// UpdateActiveValidatorsSnapshot updates the list of active validators and updates the content snapshot of the active validators +func (bc *BlockChain) UpdateActiveValidatorsSnapshot(activeValidators []common.Address) error { + prevActiveValidators, err := bc.ReadActiveValidatorList() + if err != nil { + return err + } + + err = bc.DeleteValidatorsSnapshot(prevActiveValidators) + if err != nil { + return err + } + + if err = bc.WriteValidatorsSnapshot(activeValidators); err != nil { + return err } return nil } @@ -2435,24 +2461,6 @@ func (bc *BlockChain) WriteActiveValidatorList(addrs []common.Address) error { return nil } -// UpdateActiveValidatorsSnapshot updates the list of active validators and updates the content snapshot of the active validators -func (bc *BlockChain) UpdateActiveValidatorsSnapshot(activeValidators []common.Address) error { - prevActiveValidators, err := bc.ReadActiveValidatorList() - if err != nil { - return err - } - - err = bc.DeleteValidatorsSnapshot(prevActiveValidators) - if err != nil { - return err - } - - if err = bc.WriteValidatorsSnapshot(activeValidators); err != nil { - return err - } - return nil -} - // ReadValidatorListByDelegator reads the addresses of validators delegated by a delegator func (bc *BlockChain) ReadValidatorListByDelegator(delegator common.Address) ([]common.Address, error) { if cached, ok := bc.validatorListByDelegatorCache.Get(delegator.Bytes()); ok { diff --git a/core/rawdb/accessors_chain.go b/core/rawdb/accessors_chain.go index db8fd7c71..91486b195 100644 --- a/core/rawdb/accessors_chain.go +++ b/core/rawdb/accessors_chain.go @@ -629,7 +629,7 @@ func ReadValidatorData(db DatabaseReader, addr common.Address) (*staking.Validat return &v, nil } -// WriteValidatorData stores staking validator's information by its address +// WriteValidatorData stores validator's information by its address func WriteValidatorData(db DatabaseWriter, v *staking.ValidatorWrapper) error { bytes, err := rlp.EncodeToBytes(v) if err != nil { @@ -643,7 +643,7 @@ func WriteValidatorData(db DatabaseWriter, v *staking.ValidatorWrapper) error { return err } -// ReadValidatorSnapshot retrieves staking validator's snapshot by its address +// ReadValidatorSnapshot retrieves validator's snapshot by its address func ReadValidatorSnapshot(db DatabaseReader, addr common.Address) (*staking.ValidatorWrapper, error) { data, err := db.Get(validatorSnapshotKey(addr)) if len(data) == 0 || err != nil { @@ -658,7 +658,7 @@ func ReadValidatorSnapshot(db DatabaseReader, addr common.Address) (*staking.Val return &v, nil } -// WriteValidatorSnapshot stores staking validator's snapshot by its address +// WriteValidatorSnapshot stores validator's snapshot by its address func WriteValidatorSnapshot(db DatabaseWriter, v *staking.ValidatorWrapper) error { bytes, err := rlp.EncodeToBytes(v) if err != nil { diff --git a/internal/chain/engine.go b/internal/chain/engine.go index 42dc5a3cc..b3b647377 100644 --- a/internal/chain/engine.go +++ b/internal/chain/engine.go @@ -227,7 +227,7 @@ func QuorumForBlock(chain engine.ChainReader, h *block.Header, reCalculate bool) return 0, errors.Errorf( "cannot find shard %d in shard state", h.ShardID()) } - return (len(c.NodeList))*2/3 + 1, nil + return (len(c.Slots))*2/3 + 1, nil } // Similiar to VerifyHeader, which is only for verifying the block headers of one's own chain, this verification @@ -289,7 +289,7 @@ func GetPublicKeys(chain engine.ChainReader, header *block.Header, reCalculate b ) } var committerKeys []*bls.PublicKey - for _, member := range committee.NodeList { + for _, member := range committee.Slots { committerKey := new(bls.PublicKey) err := member.BlsPublicKey.ToLibBLSPublicKey(committerKey) if err != nil { diff --git a/internal/chain/reward.go b/internal/chain/reward.go index c4f585161..a6a9e952f 100644 --- a/internal/chain/reward.go +++ b/internal/chain/reward.go @@ -61,7 +61,7 @@ func AccumulateRewards( ) } var committerKeys []*bls.PublicKey - for _, member := range parentCommittee.NodeList { + for _, member := range parentCommittee.Slots { committerKey := new(bls.PublicKey) err := member.BlsPublicKey.ToLibBLSPublicKey(committerKey) if err != nil { @@ -80,7 +80,7 @@ func AccumulateRewards( accounts := []common.Address{} - for idx, member := range parentCommittee.NodeList { + for idx, member := range parentCommittee.Slots { if signed, err := mask.IndexEnabled(idx); err != nil { return ctxerror.New("cannot check for committer bit", "committerIndex", idx, diff --git a/internal/hmyapi/blockchain.go b/internal/hmyapi/blockchain.go index ab7ade411..a95f436db 100644 --- a/internal/hmyapi/blockchain.go +++ b/internal/hmyapi/blockchain.go @@ -153,7 +153,7 @@ func (s *PublicBlockChainAPI) GetValidators(ctx context.Context, epoch int64) (m return nil, err } validators := make([]map[string]interface{}, 0) - for _, validator := range committee.NodeList { + for _, validator := range committee.Slots { validatorBalance := new(hexutil.Big) validatorBalance, err = s.b.GetBalance(validator.EcdsaAddress) if err != nil { @@ -193,8 +193,8 @@ func (s *PublicBlockChainAPI) GetBlockSigners(ctx context.Context, blockNr rpc.B if err != nil { return nil, err } - pubkeys := make([]*bls.PublicKey, len(committee.NodeList)) - for i, validator := range committee.NodeList { + pubkeys := make([]*bls.PublicKey, len(committee.Slots)) + for i, validator := range committee.Slots { pubkeys[i] = new(bls.PublicKey) validator.BlsPublicKey.ToLibBLSPublicKey(pubkeys[i]) } @@ -210,7 +210,7 @@ func (s *PublicBlockChainAPI) GetBlockSigners(ctx context.Context, blockNr rpc.B if err != nil { return result, err } - for _, validator := range committee.NodeList { + for _, validator := range committee.Slots { oneAddress, err := internal_common.AddressToBech32(validator.EcdsaAddress) if err != nil { return result, err @@ -241,8 +241,8 @@ func (s *PublicBlockChainAPI) IsBlockSigner(ctx context.Context, blockNr rpc.Blo if err != nil { return false, err } - pubkeys := make([]*bls.PublicKey, len(committee.NodeList)) - for i, validator := range committee.NodeList { + pubkeys := make([]*bls.PublicKey, len(committee.Slots)) + for i, validator := range committee.Slots { pubkeys[i] = new(bls.PublicKey) validator.BlsPublicKey.ToLibBLSPublicKey(pubkeys[i]) } @@ -254,7 +254,7 @@ func (s *PublicBlockChainAPI) IsBlockSigner(ctx context.Context, blockNr rpc.Blo if err != nil { return false, err } - for _, validator := range committee.NodeList { + for _, validator := range committee.Slots { oneAddress, err := internal_common.AddressToBech32(validator.EcdsaAddress) if err != nil { return false, err diff --git a/node/node_cross_shard.go b/node/node_cross_shard.go index ee306c389..0e97e877d 100644 --- a/node/node_cross_shard.go +++ b/node/node_cross_shard.go @@ -298,7 +298,7 @@ func (node *Node) VerifyCrosslinkHeader(prevHeader, header *block.Header) error var committerKeys []*bls.PublicKey parseKeysSuccess := true - for _, member := range committee.NodeList { + for _, member := range committee.Slots { committerKey := new(bls.PublicKey) err = member.BlsPublicKey.ToLibBLSPublicKey(committerKey) if err != nil { diff --git a/node/node_resharding.go b/node/node_resharding.go index 6ac62c807..bd2a6d410 100644 --- a/node/node_resharding.go +++ b/node/node_resharding.go @@ -201,7 +201,7 @@ func (node *Node) transitionIntoNextEpoch(shardState types.State) { for _, c := range shardState { utils.Logger().Debug(). Uint32("shardID", c.ShardID). - Str("nodeList", c.NodeList). + Str("nodeList", c.Slots). Msg("new shard information") } myShardID, isNextLeader := findRoleInShardState( @@ -219,7 +219,7 @@ func (node *Node) transitionIntoNextEpoch(shardState types.State) { // Update public keys var publicKeys []*bls.PublicKey - for idx, nodeID := range myShardState.NodeList { + for idx, nodeID := range myShardState.Slots { key := &bls.PublicKey{} err := key.Deserialize(nodeID.BlsPublicKey[:]) if err != nil { @@ -249,7 +249,7 @@ func findRoleInShardState( ) (shardID uint32, isLeader bool) { keyBytes := key.Serialize() for idx, shard := range state { - for nodeIdx, nodeID := range shard.NodeList { + for nodeIdx, nodeID := range shard.Slots { if bytes.Compare(nodeID.BlsPublicKey[:], keyBytes) == 0 { return uint32(idx), nodeIdx == 0 } diff --git a/shard/committee/assignment.go b/shard/committee/assignment.go index 7abcbaea3..f7cdb75ed 100644 --- a/shard/committee/assignment.go +++ b/shard/committee/assignment.go @@ -85,12 +85,12 @@ func preStakingEnabledCommittee(s shardingconfig.Instance) shard.State { pubKey := shard.BlsPublicKey{} pubKey.FromLibBLSPublicKey(pub) // TODO: directly read address for bls too - curNodeID := shard.NodeID{ + curNodeID := shard.Slot{ common2.ParseAddr(hmyAccounts[index].Address), pubKey, nil, } - com.NodeList = append(com.NodeList, curNodeID) + com.Slots = append(com.Slots, curNodeID) } // add FN runner's key for j := shardHarmonyNodes; j < shardSize; j++ { @@ -100,12 +100,12 @@ func preStakingEnabledCommittee(s shardingconfig.Instance) shard.State { pubKey := shard.BlsPublicKey{} pubKey.FromLibBLSPublicKey(pub) // TODO: directly read address for bls too - curNodeID := shard.NodeID{ + curNodeID := shard.Slot{ common2.ParseAddr(fnAccounts[index].Address), pubKey, nil, } - com.NodeList = append(com.NodeList, curNodeID) + com.Slots = append(com.Slots, curNodeID) } shardState = append(shardState, com) } @@ -137,7 +137,7 @@ func eposStakedCommittee( hAccounts := s.HmyAccounts() for i := 0; i < shardCount; i++ { - superComm[i] = shard.Committee{uint32(i), shard.NodeIDList{}} + superComm[i] = shard.Committee{uint32(i), shard.SlotList{}} } for i := range hAccounts { @@ -146,7 +146,7 @@ func eposStakedCommittee( pub.DeserializeHexStr(hAccounts[i].BlsPublicKey) pubKey := shard.BlsPublicKey{} pubKey.FromLibBLSPublicKey(pub) - superComm[spot].NodeList = append(superComm[spot].NodeList, shard.NodeID{ + superComm[spot].Slots = append(superComm[spot].Slots, shard.Slot{ common2.ParseAddr(hAccounts[i].Address), pubKey, nil, @@ -164,7 +164,7 @@ func eposStakedCommittee( for i := 0; i < stakedSlotsCount; i++ { bucket := int(new(big.Int).Mod(staked[i].BlsPublicKey.Big(), shardBig).Int64()) slot := staked[i] - superComm[bucket].NodeList = append(superComm[bucket].NodeList, shard.NodeID{ + superComm[bucket].Slots = append(superComm[bucket].Slots, shard.Slot{ slot.Address, staked[i].BlsPublicKey, &slot.Dec, @@ -191,10 +191,10 @@ func (def partialStakingEnabled) ComputePublicKeys( allIdentities := make([][]*bls.PublicKey, len(superComm)) for i := range superComm { - allIdentities[i] = make([]*bls.PublicKey, len(superComm[i].NodeList)) - for j := range superComm[i].NodeList { + allIdentities[i] = make([]*bls.PublicKey, len(superComm[i].Slots)) + for j := range superComm[i].Slots { identity := &bls.PublicKey{} - superComm[i].NodeList[j].BlsPublicKey.ToLibBLSPublicKey(identity) + superComm[i].Slots[j].BlsPublicKey.ToLibBLSPublicKey(identity) allIdentities[i][j] = identity } } @@ -220,12 +220,12 @@ func (def partialStakingEnabled) ReadPublicKeysFromDB( } committerKeys := []*bls.PublicKey{} - for i := range subCommittee.NodeList { + for i := range subCommittee.Slots { committerKey := new(bls.PublicKey) - err := subCommittee.NodeList[i].BlsPublicKey.ToLibBLSPublicKey(committerKey) + err := subCommittee.Slots[i].BlsPublicKey.ToLibBLSPublicKey(committerKey) if err != nil { return nil, ctxerror.New("cannot convert BLS public key", - "blsPublicKey", subCommittee.NodeList[i].BlsPublicKey).WithCause(err) + "blsPublicKey", subCommittee.Slots[i].BlsPublicKey).WithCause(err) } committerKeys = append(committerKeys, committerKey) } diff --git a/shard/shard_state.go b/shard/shard_state.go index b8bccccf0..65644b0b0 100644 --- a/shard/shard_state.go +++ b/shard/shard_state.go @@ -34,27 +34,27 @@ type State []Committee // BlsPublicKey defines the bls public key type BlsPublicKey [PublicKeySizeInBytes]byte -// NodeID represents node id (BLS address) -type NodeID struct { +// Slot represents node id (BLS address) +type Slot struct { EcdsaAddress common.Address `json:"ecdsa-address"` BlsPublicKey BlsPublicKey `json:"bls-pubkey"` // nil means not active, 0 means our node, >= 0 means staked node StakeWithDelegationApplied *numeric.Dec `json:"staked-validator" rlp:"nil"` } -// NodeIDList is a list of NodeIDList. -type NodeIDList []NodeID +// SlotList is a list of SlotList. +type SlotList []Slot // Committee contains the active nodes in one shard type Committee struct { - ShardID uint32 `json:"shard-id"` - NodeList NodeIDList `json:"subcommittee"` + ShardID uint32 `json:"shard-id"` + Slots SlotList `json:"subcommittee"` } // JSON produces a non-pretty printed JSON string of the SuperCommittee func (ss State) JSON() string { type t struct { - NodeID + Slot EcdsaAddress string `json:"one-address"` } type v struct { @@ -64,12 +64,12 @@ func (ss State) JSON() string { } dump := make([]v, len(ss)) for i := range ss { - c := len(ss[i].NodeList) + c := len(ss[i].Slots) dump[i].ShardID = ss[i].ShardID dump[i].NodeList = make([]t, c) dump[i].Count = c - for j := range ss[i].NodeList { - n := ss[i].NodeList[j] + for j := range ss[i].Slots { + n := ss[i].Slots[j] dump[i].NodeList[j].BlsPublicKey = n.BlsPublicKey dump[i].NodeList[j].StakeWithDelegationApplied = n.StakeWithDelegationApplied dump[i].NodeList[j].EcdsaAddress = common2.MustAddressToBech32(n.EcdsaAddress) @@ -166,7 +166,7 @@ func CompareBlsPublicKey(k1, k2 BlsPublicKey) int { } // CompareNodeID compares two node IDs. -func CompareNodeID(id1, id2 *NodeID) int { +func CompareNodeID(id1, id2 *Slot) int { if c := bytes.Compare(id1.EcdsaAddress[:], id2.EcdsaAddress[:]); c != 0 { return c } @@ -177,12 +177,12 @@ func CompareNodeID(id1, id2 *NodeID) int { } // DeepCopy returns a deep copy of the receiver. -func (l NodeIDList) DeepCopy() NodeIDList { +func (l SlotList) DeepCopy() SlotList { return append(l[:0:0], l...) } // CompareNodeIDList compares two node ID lists. -func CompareNodeIDList(l1, l2 NodeIDList) int { +func CompareNodeIDList(l1, l2 SlotList) int { commonLen := len(l1) if commonLen > len(l2) { commonLen = len(l2) @@ -205,7 +205,7 @@ func CompareNodeIDList(l1, l2 NodeIDList) int { func (c Committee) DeepCopy() Committee { r := Committee{} r.ShardID = c.ShardID - r.NodeList = c.NodeList.DeepCopy() + r.Slots = c.Slots.DeepCopy() return r } @@ -217,7 +217,7 @@ func CompareCommittee(c1, c2 *Committee) int { case c1.ShardID > c2.ShardID: return +1 } - if c := CompareNodeIDList(c1.NodeList, c2.NodeList); c != 0 { + if c := CompareNodeIDList(c1.Slots, c2.Slots); c != 0 { return c } return 0 @@ -225,7 +225,7 @@ func CompareCommittee(c1, c2 *Committee) int { // GetHashFromNodeList will sort the list, then use Keccak256 to hash the list // NOTE: do not modify the underlining content for hash -func GetHashFromNodeList(nodeList []NodeID) []byte { +func GetHashFromNodeList(nodeList []Slot) []byte { // in general, nodeList should not be empty if nodeList == nil || len(nodeList) == 0 { return []byte{} @@ -248,7 +248,7 @@ func (ss State) Hash() (h common.Hash) { }) d := sha3.NewLegacyKeccak256() for i := range copy { - hash := GetHashFromNodeList(copy[i].NodeList) + hash := GetHashFromNodeList(copy[i].Slots) d.Write(hash) } d.Sum(h[:0]) @@ -256,15 +256,15 @@ func (ss State) Hash() (h common.Hash) { } // CompareNodeIDByBLSKey compares two nodes by their ID; used to sort node list -func CompareNodeIDByBLSKey(n1 NodeID, n2 NodeID) int { +func CompareNodeIDByBLSKey(n1 Slot, n2 Slot) int { return bytes.Compare(n1.BlsPublicKey[:], n2.BlsPublicKey[:]) } -// Serialize serialize NodeID into bytes -func (n NodeID) Serialize() []byte { +// Serialize serialize Slot into bytes +func (n Slot) Serialize() []byte { return append(n.EcdsaAddress[:], n.BlsPublicKey[:]...) } -func (n NodeID) String() string { +func (n Slot) String() string { return "ECDSA: " + common2.MustAddressToBech32(n.EcdsaAddress) + ", BLS: " + hex.EncodeToString(n.BlsPublicKey[:]) } diff --git a/shard/shard_state_test.go b/shard/shard_state_test.go index 12ce1784d..8f55e1782 100644 --- a/shard/shard_state_test.go +++ b/shard/shard_state_test.go @@ -30,12 +30,12 @@ func init() { } func TestGetHashFromNodeList(t *testing.T) { - l1 := []NodeID{ + l1 := []Slot{ {common.Address{0x11}, blsPubKey1, nil}, {common.Address{0x22}, blsPubKey2, nil}, {common.Address{0x33}, blsPubKey3, nil}, } - l2 := []NodeID{ + l2 := []Slot{ {common.Address{0x22}, blsPubKey2, nil}, {common.Address{0x11}, blsPubKey1, nil}, {common.Address{0x33}, blsPubKey3, nil}, @@ -51,7 +51,7 @@ func TestGetHashFromNodeList(t *testing.T) { func TestHash(t *testing.T) { com1 := Committee{ ShardID: 22, - NodeList: []NodeID{ + Slots: []Slot{ {common.Address{0x12}, blsPubKey11, nil}, {common.Address{0x23}, blsPubKey22, nil}, {common.Address{0x11}, blsPubKey1, nil}, @@ -59,7 +59,7 @@ func TestHash(t *testing.T) { } com2 := Committee{ ShardID: 2, - NodeList: []NodeID{ + Slots: []Slot{ {common.Address{0x44}, blsPubKey4, nil}, {common.Address{0x55}, blsPubKey5, nil}, {common.Address{0x66}, blsPubKey6, nil}, @@ -70,7 +70,7 @@ func TestHash(t *testing.T) { com3 := Committee{ ShardID: 2, - NodeList: []NodeID{ + Slots: []Slot{ {common.Address{0x44}, blsPubKey4, nil}, {common.Address{0x55}, blsPubKey5, nil}, {common.Address{0x66}, blsPubKey6, nil}, @@ -78,7 +78,7 @@ func TestHash(t *testing.T) { } com4 := Committee{ ShardID: 22, - NodeList: []NodeID{ + Slots: []Slot{ {common.Address{0x12}, blsPubKey11, nil}, {common.Address{0x23}, blsPubKey22, nil}, {common.Address{0x11}, blsPubKey1, nil}, From f08decbbb4a09a57ce27d1b096627c6fbcd2f8c0 Mon Sep 17 00:00:00 2001 From: Rongjian Lan Date: Sat, 16 Nov 2019 14:27:28 -0800 Subject: [PATCH 03/14] Fix condition check --- core/rawdb/accessors_chain.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/core/rawdb/accessors_chain.go b/core/rawdb/accessors_chain.go index 91486b195..315cc8670 100644 --- a/core/rawdb/accessors_chain.go +++ b/core/rawdb/accessors_chain.go @@ -542,7 +542,7 @@ func WriteShardLastCrossLink(db DatabaseWriter, shardID uint32, data []byte) err // ReadCXReceipts retrieves all the transactions of receipts given destination shardID, number and blockHash func ReadCXReceipts(db DatabaseReader, shardID uint32, number uint64, hash common.Hash, temp bool) (types.CXReceipts, error) { data, err := db.Get(cxReceiptKey(shardID, number, hash, temp)) - if len(data) == 0 || err != nil { + if err != nil || len(data) == 0 { utils.Logger().Info().Err(err).Uint64("number", number).Int("dataLen", len(data)).Msg("ReadCXReceipts") return nil, err } @@ -617,7 +617,7 @@ func WriteCXReceiptsProofUnspentCheckpoint(db DatabaseWriter, shardID uint32, bl // ReadValidatorData retrieves staking validator by its address func ReadValidatorData(db DatabaseReader, addr common.Address) (*staking.ValidatorWrapper, error) { data, err := db.Get(validatorKey(addr)) - if len(data) == 0 || err != nil { + if err != nil || len(data) == 0 { utils.Logger().Info().Err(err).Msg("ReadValidatorData") return nil, err } @@ -646,7 +646,7 @@ func WriteValidatorData(db DatabaseWriter, v *staking.ValidatorWrapper) error { // ReadValidatorSnapshot retrieves validator's snapshot by its address func ReadValidatorSnapshot(db DatabaseReader, addr common.Address) (*staking.ValidatorWrapper, error) { data, err := db.Get(validatorSnapshotKey(addr)) - if len(data) == 0 || err != nil { + if err != nil || len(data) == 0 { utils.Logger().Info().Err(err).Msg("ReadValidatorSnapshot") return nil, err } @@ -687,7 +687,7 @@ func ReadValidatorList(db DatabaseReader, isActive bool) ([]common.Address, erro key = activeValidatorListKey } data, err := db.Get(key) - if len(data) == 0 || err != nil { + if err != nil || len(data) == 0 { return []common.Address{}, nil } addrs := []common.Address{} @@ -719,7 +719,7 @@ func WriteValidatorList(db DatabaseWriter, addrs []common.Address, isActive bool // ReadValidatorListByDelegator retrieves the list of validators delegated by a delegator func ReadValidatorListByDelegator(db DatabaseReader, delegator common.Address) ([]common.Address, error) { data, err := db.Get(delegatorValidatorListKey(delegator)) - if len(data) == 0 || err != nil { + if err != nil || len(data) == 0 { return []common.Address{}, nil } addrs := []common.Address{} From 901785269b44b41c876da364473708619adf72ca Mon Sep 17 00:00:00 2001 From: Rongjian Lan Date: Sat, 16 Nov 2019 15:16:16 -0800 Subject: [PATCH 04/14] fix lint --- core/blockchain.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index 273f481fb..e0e608581 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -2343,7 +2343,7 @@ func (bc *BlockChain) ReadValidatorSnapshot(addr common.Address) (*staking.Valid } // WriteValidatorSnapshots writes the snapshot of provided list of validators -func (bc *BlockChain) WriteValidatorsSnapshot(addrs []common.Address) error { +func (bc *BlockChain) WriteValidatorSnapshots(addrs []common.Address) error { // Read all validator's current data validators := []*staking.ValidatorWrapper{} for _, addr := range addrs { @@ -2376,8 +2376,8 @@ func (bc *BlockChain) WriteValidatorsSnapshot(addrs []common.Address) error { return nil } -// DeleteValidatorData deletes the snapshot staking information of given validator address -func (bc *BlockChain) DeleteValidatorsSnapshot(addrs []common.Address) error { +// DeleteValidatorSnapshots deletes the snapshot staking information of given validator address +func (bc *BlockChain) DeleteValidatorSnapshots(addrs []common.Address) error { batch := bc.db.NewBatch() for i := range addrs { rawdb.DeleteValidatorSnapshot(batch, addrs[i]) @@ -2398,12 +2398,12 @@ func (bc *BlockChain) UpdateActiveValidatorsSnapshot(activeValidators []common.A return err } - err = bc.DeleteValidatorsSnapshot(prevActiveValidators) + err = bc.DeleteValidatorSnapshots(prevActiveValidators) if err != nil { return err } - if err = bc.WriteValidatorsSnapshot(activeValidators); err != nil { + if err = bc.WriteValidatorSnapshots(activeValidators); err != nil { return err } return nil From 59059f4b466fc0ae877c97966de8e53e67b40786 Mon Sep 17 00:00:00 2001 From: Rongjian Lan Date: Sat, 16 Nov 2019 16:07:08 -0800 Subject: [PATCH 05/14] Fix active validator DB api --- consensus/engine/consensus_engine.go | 4 ++-- core/blockchain.go | 31 ++++------------------------ core/chain_makers.go | 2 +- hmy/api_backend.go | 3 ++- internal/chain/engine.go | 5 ++++- 5 files changed, 13 insertions(+), 32 deletions(-) diff --git a/consensus/engine/consensus_engine.go b/consensus/engine/consensus_engine.go index a2a9ad8b5..76ff47b97 100644 --- a/consensus/engine/consensus_engine.go +++ b/consensus/engine/consensus_engine.go @@ -40,8 +40,8 @@ type ChainReader interface { // Thus, only should be used to read the shard state of the current chain. ReadShardState(epoch *big.Int) (shard.State, error) - // ActiveValidatorAddresses retrieves the list of active validators - ActiveValidatorAddresses() []common.Address + // ReadActiveValidatorList retrieves the list of active validators + ReadActiveValidatorList() ([]common.Address, error) } // Engine is an algorithm agnostic consensus engine. diff --git a/core/blockchain.go b/core/blockchain.go index e0e608581..4357454e3 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -2406,6 +2406,10 @@ func (bc *BlockChain) UpdateActiveValidatorsSnapshot(activeValidators []common.A if err = bc.WriteValidatorSnapshots(activeValidators); err != nil { return err } + + if err = bc.WriteActiveValidatorList(activeValidators); err != nil { + return err + } return nil } @@ -2544,33 +2548,6 @@ func (bc *BlockChain) UpdateStakingMetaData(tx *staking.StakingTransaction) erro return nil } -// ActiveValidatorAddresses returns the address of active validators for current epoch -// TODO: should only return those that are selected by epos. -func (bc *BlockChain) ActiveValidatorAddresses() []common.Address { - list, err := bc.ReadValidatorList() - if err != nil { - return make([]common.Address, 0) - } - - currentEpoch := bc.CurrentBlock().Epoch() - - filtered := []common.Address{} - for _, addr := range list { - val, err := bc.ValidatorInformation(addr) - if err != nil { - continue - } - // TODO: double check this logic here. - epoch := shard.Schedule.CalcEpochNumber(val.CreationHeight.Uint64()) - if epoch.Cmp(currentEpoch) >= 0 { - // wait for next epoch - continue - } - filtered = append(filtered, addr) - } - return filtered -} - // ValidatorCandidates returns the up to date validator candidates for next epoch func (bc *BlockChain) ValidatorCandidates() []common.Address { list, err := bc.ReadValidatorList() diff --git a/core/chain_makers.go b/core/chain_makers.go index 51f501421..cdd30e907 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -270,4 +270,4 @@ func (cr *fakeChainReader) GetHeaderByHash(hash common.Hash) *block.Header func (cr *fakeChainReader) GetHeader(hash common.Hash, number uint64) *block.Header { return nil } func (cr *fakeChainReader) GetBlock(hash common.Hash, number uint64) *types.Block { return nil } func (cr *fakeChainReader) ReadShardState(epoch *big.Int) (shard.State, error) { return nil, nil } -func (cr *fakeChainReader) ActiveValidatorAddresses() []common.Address { return nil } +func (cr *fakeChainReader) ReadActiveValidatorList() ([]common.Address, error) { return nil, nil } diff --git a/hmy/api_backend.go b/hmy/api_backend.go index d7d05b9dd..5c44e346c 100644 --- a/hmy/api_backend.go +++ b/hmy/api_backend.go @@ -292,7 +292,8 @@ func (b *APIBackend) SendStakingTx( // GetActiveValidatorAddresses returns the address of active validators for current epoch func (b *APIBackend) GetActiveValidatorAddresses() []common.Address { - return b.hmy.BlockChain().ActiveValidatorAddresses() + list, _ := b.hmy.BlockChain().ReadActiveValidatorList() + return list } // GetValidatorCandidates returns the up to date validator candidates for next epoch diff --git a/internal/chain/engine.go b/internal/chain/engine.go index b3b647377..850a14132 100644 --- a/internal/chain/engine.go +++ b/internal/chain/engine.go @@ -179,7 +179,10 @@ func (e *engineImpl) Finalize( // Only do such at the last block of an epoch if len(header.ShardState()) > 0 { // TODO: make sure we are using the correct validator list - validators := chain.ActiveValidatorAddresses() + validators, err := chain.ReadActiveValidatorList() + if err != nil { + return nil, ctxerror.New("failed to read active validators").WithCause(err) + } for _, validator := range validators { wrapper := state.GetStakingInfo(validator) if wrapper != nil { From b856049aadf798334b10b7024751fae68d780c02 Mon Sep 17 00:00:00 2001 From: Rongjian Lan Date: Sat, 16 Nov 2019 16:53:43 -0800 Subject: [PATCH 06/14] Create snapshot for all validators --- core/blockchain.go | 33 +++++++++++++++++++-------------- core/evm.go | 5 +++++ core/rawdb/accessors_chain.go | 12 ++++++------ core/state_transition.go | 9 +++++++-- 4 files changed, 37 insertions(+), 22 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index 4357454e3..716596c87 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1143,7 +1143,7 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types. return NonStatTy, err } - // Find all the active validator addresses and do a snapshot + // Find all the active validator addresses and store them in db allActiveValidators := []common.Address{} processed := make(map[common.Address]struct{}) for i := range *shardState { @@ -1159,7 +1159,15 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types. } } } - bc.UpdateActiveValidatorsSnapshot(allActiveValidators) + + if err := bc.WriteActiveValidatorList(allActiveValidators); err != nil { + return NonStatTy, err + } + + // Create snapshot for all validators + if err := bc.UpdateValidatorSnapshots(); err != nil { + return NonStatTy, err + } } // Do bookkeeping for new staking txns @@ -2391,23 +2399,20 @@ func (bc *BlockChain) DeleteValidatorSnapshots(addrs []common.Address) error { return nil } -// UpdateActiveValidatorsSnapshot updates the list of active validators and updates the content snapshot of the active validators -func (bc *BlockChain) UpdateActiveValidatorsSnapshot(activeValidators []common.Address) error { - prevActiveValidators, err := bc.ReadActiveValidatorList() +// UpdateValidatorSnapshots updates the content snapshot of all validators +func (bc *BlockChain) UpdateValidatorSnapshots() error { + allValidators, err := bc.ReadValidatorList() if err != nil { return err } - err = bc.DeleteValidatorSnapshots(prevActiveValidators) - if err != nil { - return err - } - - if err = bc.WriteValidatorSnapshots(activeValidators); err != nil { - return err - } + // TODO: enable this once we allow validator to delete itself. + //err = bc.DeleteValidatorSnapshots(allValidators) + //if err != nil { + // return err + //} - if err = bc.WriteActiveValidatorList(activeValidators); err != nil { + if err := bc.WriteValidatorSnapshots(allValidators); err != nil { return err } return nil diff --git a/core/evm.go b/core/evm.go index daa0c59b1..27831aaa5 100644 --- a/core/evm.go +++ b/core/evm.go @@ -19,6 +19,8 @@ package core import ( "math/big" + types2 "github.com/harmony-one/harmony/staking/types" + "github.com/ethereum/go-ethereum/common" "github.com/harmony-one/harmony/block" @@ -38,6 +40,9 @@ type ChainContext interface { // ReadValidatorListByDelegator returns the validators list of a delegator ReadValidatorListByDelegator(common.Address) ([]common.Address, error) + + // ReadValidatorSnapshot returns the snapshot of validator at the beginning of current epoch. + ReadValidatorSnapshot(common.Address) (*types2.ValidatorWrapper, error) } // NewEVMContext creates a new context for use in the EVM. diff --git a/core/rawdb/accessors_chain.go b/core/rawdb/accessors_chain.go index 315cc8670..48889d18a 100644 --- a/core/rawdb/accessors_chain.go +++ b/core/rawdb/accessors_chain.go @@ -680,10 +680,10 @@ func DeleteValidatorSnapshot(db DatabaseDeleter, addr common.Address) { } // ReadValidatorList retrieves staking validator by its address -// Return only active validators if isActive==true, otherwise, return all validators -func ReadValidatorList(db DatabaseReader, isActive bool) ([]common.Address, error) { +// Return only active validators if activeOnly==true, otherwise, return all validators +func ReadValidatorList(db DatabaseReader, activeOnly bool) ([]common.Address, error) { key := validatorListKey - if isActive { + if activeOnly { key = activeValidatorListKey } data, err := db.Get(key) @@ -699,10 +699,10 @@ func ReadValidatorList(db DatabaseReader, isActive bool) ([]common.Address, erro } // WriteValidatorList stores staking validator's information by its address -// Writes only for active validators if isActive==true, otherwise, writes for all validators -func WriteValidatorList(db DatabaseWriter, addrs []common.Address, isActive bool) error { +// Writes only for active validators if activeOnly==true, otherwise, writes for all validators +func WriteValidatorList(db DatabaseWriter, addrs []common.Address, activeOnly bool) error { key := validatorListKey - if isActive { + if activeOnly { key = activeValidatorListKey } diff --git a/core/state_transition.go b/core/state_transition.go index f076d02b2..9da5cec8c 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -378,8 +378,13 @@ func (st *StateTransition) applyEditValidatorTx(editValidator *staking.EditValid } newRate := wrapper.Validator.Rate - // TODO: Use snapshot of validator in this epoch. - rateAtBeginningOfEpoch := wrapper.Validator.Rate + // TODO: make sure we are reading from the correct snapshot + snapshotValidator, err := st.bc.ReadValidatorSnapshot(wrapper.Address) + if err != nil { + return err + } + rateAtBeginningOfEpoch := snapshotValidator.Rate + if rateAtBeginningOfEpoch.IsNil() || (!newRate.IsNil() && !rateAtBeginningOfEpoch.Equal(newRate)) { wrapper.Validator.UpdateHeight = blockNum } From 3b7b70cb1ccfe58356b02a832c91f15310cc727d Mon Sep 17 00:00:00 2001 From: Rongjian Lan Date: Sat, 16 Nov 2019 17:36:40 -0800 Subject: [PATCH 07/14] Fix a bug on 14 epoch comp and refactor undelegation withdrawal --- core/state/statedb.go | 2 +- internal/chain/engine.go | 16 +++------------- staking/types/delegation.go | 17 +++++++++++++++++ 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/core/state/statedb.go b/core/state/statedb.go index eb9415900..10681ff8c 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -684,7 +684,7 @@ func (db *DB) Commit(deleteEmptyObjects bool) (root common.Hash, err error) { return root, err } -// GetStakingInfo update staking information of a given validator (including delegation info) +// GetStakingInfo returns staking information of a given validator (including delegation info) func (db *DB) GetStakingInfo(addr common.Address) *stk.ValidatorWrapper { by := db.GetCode(addr) if len(by) == 0 { diff --git a/internal/chain/engine.go b/internal/chain/engine.go index 850a14132..88f12f375 100644 --- a/internal/chain/engine.go +++ b/internal/chain/engine.go @@ -2,7 +2,6 @@ package chain import ( "encoding/binary" - "math/big" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/rlp" @@ -188,23 +187,14 @@ func (e *engineImpl) Finalize( if wrapper != nil { for i := range wrapper.Delegations { delegation := wrapper.Delegations[i] - totalWithdraw := big.NewInt(0) - count := 0 - for j := range delegation.Entries { - if delegation.Entries[j].Epoch.Cmp(header.Epoch()) > 14 { // need to wait at least 14 epochs to withdraw; - totalWithdraw.Add(totalWithdraw, delegation.Entries[j].Amount) - count++ - } else { - break - } - - } + totalWithdraw := delegation.RemoveUnlockedUndelegations(header.Epoch()) state.AddBalance(delegation.DelegatorAddress, totalWithdraw) - delegation.Entries = delegation.Entries[count:] } if err := state.UpdateStakingInfo(validator, wrapper); err != nil { return nil, ctxerror.New("failed update validator info").WithCause(err) } + } else { + return nil, ctxerror.New("failed getting validator info").WithCause(err) } } } diff --git a/staking/types/delegation.go b/staking/types/delegation.go index 8844e02be..6fc8c5668 100644 --- a/staking/types/delegation.go +++ b/staking/types/delegation.go @@ -93,3 +93,20 @@ func (d *Delegation) DeleteEntry(epoch *big.Int) { d.Entries = entries } } + +// RemoveUnlockedUndelegations removes all fully unlocked undelegations and returns the total sum +func (d *Delegation) RemoveUnlockedUndelegations(curEpoch *big.Int) *big.Int { + totalWithdraw := big.NewInt(0) + count := 0 + for j := range d.Entries { + if curEpoch.Cmp(d.Entries[j].Epoch) > 14 { // need to wait at least 14 epochs to withdraw; + totalWithdraw.Add(totalWithdraw, d.Entries[j].Amount) + count++ + } else { + break + } + + } + d.Entries = d.Entries[count:] + return totalWithdraw +} From 6f1e144a6ef0170eda6de554c6ab92aab65a685a Mon Sep 17 00:00:00 2001 From: Edgar Aroutiounian Date: Mon, 18 Nov 2019 21:26:32 -0800 Subject: [PATCH 08/14] [blockchain] Do not rely on beaconchain for when not on shard0, use shardchain (#1840) --- node/worker/worker.go | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/node/worker/worker.go b/node/worker/worker.go index c4d9635d1..20f43c8f1 100644 --- a/node/worker/worker.go +++ b/node/worker/worker.go @@ -290,26 +290,26 @@ func (w *Worker) SuperCommitteeForNextEpoch( nextCommittee shard.State oops error ) - - switch shardID { - case shard.BeaconChainShardID: - if shard.Schedule.IsLastBlock(w.current.header.Number().Uint64()) { - nextCommittee, oops = committee.WithStakingEnabled.Compute( - new(big.Int).Add(w.current.header.Epoch(), common.Big1), - *w.config, - beacon, - ) - } - default: - // WARN When we first enable staking, this condition may not be robust by itself. - switch beacon.CurrentHeader().Epoch().Cmp(w.current.header.Epoch()) { - case 1: - nextCommittee, oops = committee.WithStakingEnabled.ReadFromDB( - beacon.CurrentHeader().Epoch(), beacon, - ) - } - + // WARN This currently not working and breaks around 15 block + // switch shardID { + // case shard.BeaconChainShardID: + if shard.Schedule.IsLastBlock(w.current.header.Number().Uint64()) { + nextCommittee, oops = committee.WithStakingEnabled.Compute( + new(big.Int).Add(w.current.header.Epoch(), common.Big1), + *w.config, + beacon, + ) } + // default: + // WARN When we first enable staking, this condition may not be robust by itself. + // switch beacon.CurrentHeader().Epoch().Cmp(w.current.header.Epoch()) { + // case 1: + // nextCommittee, oops = committee.WithStakingEnabled.ReadFromDB( + // beacon.CurrentHeader().Epoch(), beacon, + // ) + // } + + // } return nextCommittee, oops } From 77ff9d422d0009b14439afa4f4ef1b6fa992cc61 Mon Sep 17 00:00:00 2001 From: Rongjian Lan Date: Mon, 18 Nov 2019 22:26:08 -0800 Subject: [PATCH 09/14] test code --- block/factory/factory.go | 2 +- cmd/staking/root.go | 15 +++++++++------ consensus/consensus_v2.go | 2 ++ core/blockchain.go | 4 ++-- core/state_processor.go | 2 ++ core/types/block.go | 29 +++++++++++++++++++---------- core/types/bodyfieldsetter.go | 11 ++++++++++- core/types/bodyv0.go | 14 ++++++++++++++ core/types/bodyv1.go | 14 ++++++++++++++ core/types/bodyv2.go | 9 +++++++++ hmy/api_backend.go | 4 ++-- internal/hmyapi/backend.go | 2 +- internal/hmyapi/transactionpool.go | 10 ++++++++++ node/node_handler.go | 30 ++++++++++++++---------------- node/node_newblock.go | 1 + staking/types/transaction.go | 16 ++++++++++++++-- staking/types/validator.go | 22 ++++++++++++++++++++++ 17 files changed, 146 insertions(+), 41 deletions(-) diff --git a/block/factory/factory.go b/block/factory/factory.go index dcef0c142..72c5e4dd8 100644 --- a/block/factory/factory.go +++ b/block/factory/factory.go @@ -30,7 +30,7 @@ func NewFactory(chainConfig *params.ChainConfig) Factory { func (f *factory) NewHeader(epoch *big.Int) *block.Header { var impl blockif.Header switch { - case epoch.Cmp(f.chainConfig.StakingEpoch) >= 0: + case epoch.Cmp(f.chainConfig.StakingEpoch) < 0: // REVERT BEFORE COMMIT impl = v3.NewHeader() case epoch.Cmp(f.chainConfig.CrossLinkEpoch) >= 0: impl = v2.NewHeader() diff --git a/cmd/staking/root.go b/cmd/staking/root.go index 88b605dab..37c81022a 100644 --- a/cmd/staking/root.go +++ b/cmd/staking/root.go @@ -11,6 +11,8 @@ import ( "path" "strconv" + "github.com/harmony-one/harmony/common/denominations" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/rlp" @@ -75,9 +77,9 @@ func (s *staker) run(cmd *cobra.Command, args []string) error { pub := shard.BlsPublicKey{} pub.FromLibBLSPublicKey(p) - ra, _ := numeric.NewDecFromStr("27.27") - maxRate, _ := numeric.NewDecFromStr("150.99") - maxChangeRate, _ := numeric.NewDecFromStr("0.5") + ra, _ := numeric.NewDecFromStr("0.2") + maxRate, _ := numeric.NewDecFromStr("1") + maxChangeRate, _ := numeric.NewDecFromStr("0.05") if cmdType == "create" { return staking.DirectiveCreateValidator, staking.CreateValidator{ Description: &staking.Description{ @@ -92,11 +94,11 @@ func (s *staker) run(cmd *cobra.Command, args []string) error { MaxRate: maxRate, MaxChangeRate: maxChangeRate, }, - MinSelfDelegation: big.NewInt(10), - MaxTotalDelegation: big.NewInt(3000), + MinSelfDelegation: big.NewInt(denominations.One), + MaxTotalDelegation: big.NewInt(0).Mul(big.NewInt(denominations.One), big.NewInt(1000)), ValidatorAddress: common.Address(dAddr), SlotPubKeys: []shard.BlsPublicKey{pub}, - Amount: big.NewInt(100), + Amount: big.NewInt(denominations.One), } } /* @@ -133,6 +135,7 @@ func (s *staker) run(cmd *cobra.Command, args []string) error { if oops1 != nil { return oops1 } + tx := new(staking.StakingTransaction) if err := rlp.DecodeBytes(enc, tx); err != nil { return err diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index d32457a97..2d945e7bf 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -833,6 +833,8 @@ func (consensus *Consensus) finalizeCommits() { Uint64("ViewId", block.Header().ViewID().Uint64()). Str("blockHash", block.Hash().String()). Int("index", consensus.Decider.IndexOf(consensus.PubKey)). + Int("numTxns", len(block.Transactions())). + Int("numStakingTxns", len(block.StakingTransactions())). Msg("HOORAY!!!!!!! CONSENSUS REACHED!!!!!!!") // Send signal to Node so the new block can be added and new round of consensus can be triggered consensus.ReadySignal <- struct{}{} diff --git a/core/blockchain.go b/core/blockchain.go index 716596c87..01905d56d 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1171,7 +1171,7 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types. } // Do bookkeeping for new staking txns - if bc.chainConfig.IsStaking(block.Epoch()) { + //if bc.chainConfig.IsStaking(block.Epoch()) { for _, tx := range block.StakingTransactions() { err = bc.UpdateStakingMetaData(tx) // keep offchain database consistency with onchain we need revert @@ -1181,7 +1181,7 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types. return NonStatTy, err } } - } + //} //// Cross-links if len(header.CrossLinks()) > 0 { diff --git a/core/state_processor.go b/core/state_processor.go index e8d81437d..05b61c131 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -266,7 +266,9 @@ func ApplyIncomingReceipt(config *params.ChainConfig, db *state.DB, header *bloc // requires a signer to derive the sender. // put it here to avoid cyclic import func StakingToMessage(tx *staking.StakingTransaction, blockNum *big.Int) (types.Message, error) { + utils.Logger().Info().Msgf("ApplyStakingMessage: aaaaaa:") payload, err := tx.RLPEncodeStakeMsg() + utils.Logger().Info().Msgf("ApplyStakingMessage: aaaaaa:", err) if err != nil { return types.Message{}, err } diff --git a/core/types/block.go b/core/types/block.go index f450b3824..c2eea6e4d 100644 --- a/core/types/block.go +++ b/core/types/block.go @@ -90,6 +90,10 @@ type BodyInterface interface { // It returns nil if index is out of bounds. TransactionAt(index int) *Transaction + // StakingTransactionAt returns the staking transaction at the given index in this block. + // It returns nil if index is out of bounds. + StakingTransactionAt(index int) *staking.StakingTransaction + // CXReceiptAt returns the CXReceipt given index (calculated from IncomingReceipts) // It returns nil if index is out of bounds CXReceiptAt(index int) *CXReceipt @@ -98,6 +102,10 @@ type BodyInterface interface { // given list. SetTransactions(newTransactions []*Transaction) + // SetStakingTransactions sets the list of staking transactions with a deep copy of the + // given list. + SetStakingTransactions(newStakingTransactions []*staking.StakingTransaction) + // Uncles returns a deep copy of the list of uncle headers of this block. Uncles() []*block.Header @@ -190,11 +198,11 @@ func init() { // Block represents an entire block in the Harmony blockchain. type Block struct { - header *block.Header - uncles []*block.Header - transactions Transactions - stks staking.StakingTransactions - incomingReceipts CXReceiptsProofs + header *block.Header + uncles []*block.Header + transactions Transactions + stakingTransactions staking.StakingTransactions + incomingReceipts CXReceiptsProofs // caches hash atomic.Value @@ -301,8 +309,8 @@ func NewBlock(header *block.Header, txs []*Transaction, receipts []*Receipt, out } if len(stks) > 0 { - b.stks = make(staking.StakingTransactions, len(stks)) - copy(b.stks, stks) + b.stakingTransactions = make(staking.StakingTransactions, len(stks)) + copy(b.stakingTransactions, stks) } return b @@ -332,7 +340,7 @@ func (b *Block) DecodeRLP(s *rlp.Stream) error { } switch eb := eb.(type) { case *extblockV2: - b.header, b.uncles, b.transactions, b.incomingReceipts, b.stks = eb.Header, eb.Uncles, eb.Txs, eb.IncomingReceipts, eb.Stks + b.header, b.uncles, b.transactions, b.incomingReceipts, b.stakingTransactions = eb.Header, eb.Uncles, eb.Txs, eb.IncomingReceipts, eb.Stks case *extblockV1: b.header, b.uncles, b.transactions, b.incomingReceipts = eb.Header, eb.Uncles, eb.Txs, eb.IncomingReceipts case *extblock: @@ -349,7 +357,7 @@ func (b *Block) EncodeRLP(w io.Writer) error { var eb interface{} switch h := b.header.Header.(type) { case *v3.Header: - eb = extblockV2{b.header, b.transactions, b.stks, b.uncles, b.incomingReceipts} + eb = extblockV2{b.header, b.transactions, b.stakingTransactions, b.uncles, b.incomingReceipts} case *v2.Header, *v1.Header: eb = extblockV1{b.header, b.transactions, b.uncles, b.incomingReceipts} case *v0.Header: @@ -376,7 +384,7 @@ func (b *Block) Transactions() Transactions { // StakingTransactions returns stakingTransactions. func (b *Block) StakingTransactions() staking.StakingTransactions { - return b.stks + return b.stakingTransactions } // IncomingReceipts returns verified outgoing receipts @@ -454,6 +462,7 @@ func (b *Block) Body() *Body { } return body.With(). Transactions(b.transactions). + StakingTransactions(b.stakingTransactions). Uncles(b.uncles). IncomingReceipts(b.incomingReceipts). Body() diff --git a/core/types/bodyfieldsetter.go b/core/types/bodyfieldsetter.go index 10032fd8c..7d2f9cb7c 100644 --- a/core/types/bodyfieldsetter.go +++ b/core/types/bodyfieldsetter.go @@ -1,6 +1,9 @@ package types -import "github.com/harmony-one/harmony/block" +import ( + "github.com/harmony-one/harmony/block" + "github.com/harmony-one/harmony/staking/types" +) // BodyFieldSetter is a body field setter. type BodyFieldSetter struct { @@ -13,6 +16,12 @@ func (bfs BodyFieldSetter) Transactions(newTransactions []*Transaction) BodyFiel return bfs } +// StakingTransactions sets the StakingTransactions field of the body. +func (bfs BodyFieldSetter) StakingTransactions(newStakingTransactions []*types.StakingTransaction) BodyFieldSetter { + bfs.b.SetStakingTransactions(newStakingTransactions) + return bfs +} + // Uncles sets the Uncles field of the body. func (bfs BodyFieldSetter) Uncles(newUncles []*block.Header) BodyFieldSetter { bfs.b.SetUncles(newUncles) diff --git a/core/types/bodyv0.go b/core/types/bodyv0.go index 78042af19..4cdecdc56 100644 --- a/core/types/bodyv0.go +++ b/core/types/bodyv0.go @@ -40,6 +40,13 @@ func (b *BodyV0) TransactionAt(index int) *Transaction { return b.f.Transactions[index].Copy() } +// StakingTransactionAt returns the staking transaction at the given index in this block. +// It returns nil if index is out of bounds. (not supported by Body V0) +func (b *BodyV0) StakingTransactionAt(index int) *staking.StakingTransaction { + // not supported + return nil +} + // CXReceiptAt returns the CXReceipt at given index in this block // It returns nil if index is out of bounds // V0 will just return nil because we don't support CXReceipt @@ -57,6 +64,13 @@ func (b *BodyV0) SetTransactions(newTransactions []*Transaction) { b.f.Transactions = txs } +// SetStakingTransactions sets the list of staking transactions with a deep copy of the given +// list. (not supported by Body V0) +func (b *BodyV0) SetStakingTransactions(newTransactions []*staking.StakingTransaction) { + // not supported + return +} + // Uncles returns a deep copy of the list of uncle headers of this block. func (b *BodyV0) Uncles() (uncles []*block.Header) { for _, uncle := range b.f.Uncles { diff --git a/core/types/bodyv1.go b/core/types/bodyv1.go index fcc08b73c..ced6918f5 100644 --- a/core/types/bodyv1.go +++ b/core/types/bodyv1.go @@ -73,6 +73,20 @@ func (b *BodyV1) SetTransactions(newTransactions []*Transaction) { b.f.Transactions = txs } +// SetStakingTransactions sets the list of staking transactions with a deep copy of the given +// list. (not supported by Body V1) +func (b *BodyV1) SetStakingTransactions(newTransactions []*staking.StakingTransaction) { + // not supported + return +} + +// StakingTransactionAt returns the staking transaction at the given index in this block. +// It returns nil if index is out of bounds. (not supported by Body V1) +func (b *BodyV1) StakingTransactionAt(index int) *staking.StakingTransaction { + // not supported + return nil +} + // Uncles returns a deep copy of the list of uncle headers of this block. func (b *BodyV1) Uncles() (uncles []*block.Header) { for _, uncle := range b.f.Uncles { diff --git a/core/types/bodyv2.go b/core/types/bodyv2.go index 1723e913c..67c47f6ef 100644 --- a/core/types/bodyv2.go +++ b/core/types/bodyv2.go @@ -51,6 +51,15 @@ func (b *BodyV2) TransactionAt(index int) *Transaction { return b.f.Transactions[index].Copy() } +// StakingTransactionAt returns the staking transaction at the given index in this block. +// It returns nil if index is out of bounds. +func (b *BodyV2) StakingTransactionAt(index int) *staking.StakingTransaction { + if index < 0 || index >= len(b.f.StakingTransactions) { + return nil + } + return b.f.StakingTransactions[index].Copy() +} + // CXReceiptAt returns the CXReceipt at given index in this block // It returns nil if index is out of bounds func (b *BodyV2) CXReceiptAt(index int) *CXReceipt { diff --git a/hmy/api_backend.go b/hmy/api_backend.go index 5c44e346c..293aac2ac 100644 --- a/hmy/api_backend.go +++ b/hmy/api_backend.go @@ -296,8 +296,8 @@ func (b *APIBackend) GetActiveValidatorAddresses() []common.Address { return list } -// GetValidatorCandidates returns the up to date validator candidates for next epoch -func (b *APIBackend) GetValidatorCandidates() []common.Address { +// GetAllValidatorAddresses returns the up to date validator candidates for next epoch +func (b *APIBackend) GetAllValidatorAddresses() []common.Address { return b.hmy.BlockChain().ValidatorCandidates() } diff --git a/internal/hmyapi/backend.go b/internal/hmyapi/backend.go index f8dbdcdb4..a0dddf638 100644 --- a/internal/hmyapi/backend.go +++ b/internal/hmyapi/backend.go @@ -74,7 +74,7 @@ type Backend interface { IsLeader() bool SendStakingTx(ctx context.Context, newStakingTx *staking.StakingTransaction) error GetActiveValidatorAddresses() []common.Address - GetValidatorCandidates() []common.Address + GetAllValidatorAddresses() []common.Address GetValidatorInformation(addr common.Address) *staking.Validator GetDelegatorsInformation(addr common.Address) []*staking.Delegation GetValidatorStakingWithDelegation(addr common.Address) *big.Int diff --git a/internal/hmyapi/transactionpool.go b/internal/hmyapi/transactionpool.go index 8f52f81f5..8596f5f88 100644 --- a/internal/hmyapi/transactionpool.go +++ b/internal/hmyapi/transactionpool.go @@ -298,3 +298,13 @@ func (s *PublicTransactionPoolAPI) GetCXReceiptByHash(ctx context.Context, hash } return nil } + +// GetAllValidatorAddresses returns ... +func (s *PublicTransactionPoolAPI) GetAllValidatorAddresses() ([]common.Address, error) { + return s.b.GetAllValidatorAddresses(), nil +} + +// GetActiveValidatorAddresses returns ... +func (s *PublicTransactionPoolAPI) GetActiveValidatorAddresses() ([]common.Address, error) { + return s.b.GetActiveValidatorAddresses(), nil +} diff --git a/node/node_handler.go b/node/node_handler.go index 89402cb6c..a582d3178 100644 --- a/node/node_handler.go +++ b/node/node_handler.go @@ -400,23 +400,21 @@ func (node *Node) PostConsensusProcessing(newBlock *types.Block, commitSigAndBit func (node *Node) AddNewBlock(newBlock *types.Block) error { _, err := node.Blockchain().InsertChain([]*types.Block{newBlock}, true /* verifyHeaders */) - /* - // Debug only - addrs, err := node.Blockchain().ReadValidatorList() - utils.Logger().Debug().Msgf("validator list updated, err=%v, len(addrs)=%v", err, len(addrs)) - for i, addr := range addrs { - val, err := node.Blockchain().ValidatorInformation(addr) - if err != nil { - utils.Logger().Debug().Msgf("ValidatorInformation Error %v: err %v", i, err) - } - utils.Logger().Debug().Msgf("ValidatorInformation %v: %v", i, val) + // Debug only + addrs, err := node.Blockchain().ReadValidatorList() + utils.Logger().Debug().Msgf("validator list updated, err=%v, len(addrs)=%v", err, len(addrs)) + for i, addr := range addrs { + val, err := node.Blockchain().ValidatorInformation(addr) + if err != nil { + utils.Logger().Debug().Msgf("ValidatorInformation Error %v: err %v", i, err) } - currAddrs := node.Blockchain().ActiveValidatorAddresses() - utils.Logger().Debug().Msgf("CurrentValidators : %v", currAddrs) - candidates := node.Blockchain().ValidatorCandidates() - utils.Logger().Debug().Msgf("CandidateValidators : %v", candidates) - // Finish debug - */ + utils.Logger().Debug().Msgf("ValidatorInformation %v: %v", i, val) + } + currAddrs, err := node.Blockchain().ReadActiveValidatorList() + utils.Logger().Debug().Msgf("CurrentValidators : %v", currAddrs) + candidates := node.Blockchain().ValidatorCandidates() + utils.Logger().Debug().Msgf("CandidateValidators : %v", candidates) + // Finish debug if err != nil { utils.Logger().Error(). diff --git a/node/node_newblock.go b/node/node_newblock.go index 8f93ca5f4..8344900ef 100644 --- a/node/node_newblock.go +++ b/node/node_newblock.go @@ -91,6 +91,7 @@ func (node *Node) proposeNewBlock() (*types.Block, error) { for _, tx := range node.pendingStakingTransactions { pendingStakingTransactions = append(pendingStakingTransactions, tx) } + node.pendingStakingTransactions = make(map[common.Hash]*types2.StakingTransaction) node.Worker.UpdateCurrent(coinbase) if err := node.Worker.CommitTransactions(pending, pendingStakingTransactions, coinbase); err != nil { diff --git a/staking/types/transaction.go b/staking/types/transaction.go index 49678b45a..d344188d3 100644 --- a/staking/types/transaction.go +++ b/staking/types/transaction.go @@ -4,7 +4,6 @@ import ( "errors" "io" "math/big" - "reflect" "sync/atomic" "github.com/ethereum/go-ethereum/common" @@ -35,7 +34,20 @@ func (d *txdata) CopyFrom(d2 *txdata) { d.AccountNonce = d2.AccountNonce d.Price = new(big.Int).Set(d2.Price) d.GasLimit = d2.GasLimit - d.StakeMsg = reflect.New(reflect.ValueOf(d2.StakeMsg).Elem().Type()).Interface() + //switch d.Directive { // TODO: make these deep copies. + //case DirectiveCreateValidator: + // d.StakeMsg = d2.StakeMsg.(CreateValidator) + //case DirectiveEditValidator: + // d.StakeMsg = d2.StakeMsg.(EditValidator) + //case DirectiveDelegate: + // d.StakeMsg = d2.StakeMsg.(Delegate) + //case DirectiveUndelegate: + // d.StakeMsg = d2.StakeMsg.(Undelegate) + //case DirectiveCollectRewards: + // d.StakeMsg = d2.StakeMsg.(CollectRewards) + //default: + // return + //} d.V = new(big.Int).Set(d2.V) d.R = new(big.Int).Set(d2.R) d.S = new(big.Int).Set(d2.S) diff --git a/staking/types/validator.go b/staking/types/validator.go index 2ca0b8a1b..91368e045 100644 --- a/staking/types/validator.go +++ b/staking/types/validator.go @@ -30,6 +30,8 @@ var ( errInvalidTotalDelegation = errors.New("total delegation can not be bigger than max_total_delegation") errMinSelfDelegationTooSmall = errors.New("min_self_delegation has to be greater than 1 ONE") errInvalidMaxTotalDelegation = errors.New("max_total_delegation can not be less than min_self_delegation") + errCommissionRateTooLarge = errors.New("commission rate and change rate can not be larger than max commission rate") + errInvalidComissionRate = errors.New("commission rate, change rate and max rate should be within 0-100%") ) // ValidatorWrapper contains validator and its delegation information @@ -103,6 +105,26 @@ func (w *ValidatorWrapper) SanityCheck() error { if totalDelegation.Cmp(w.Validator.MaxTotalDelegation) > 0 { return errInvalidTotalDelegation } + + hundredPercent := numeric.NewDec(1) + zeroPercent := numeric.NewDec(0) + if w.Validator.Rate.LT(zeroPercent) || w.Validator.Rate.GT(hundredPercent) { + return errInvalidComissionRate + } + if w.Validator.MaxRate.LT(zeroPercent) || w.Validator.MaxRate.GT(hundredPercent) { + return errInvalidComissionRate + } + if w.Validator.MaxChangeRate.LT(zeroPercent) || w.Validator.MaxChangeRate.GT(hundredPercent) { + return errInvalidComissionRate + } + + if w.Validator.Rate.GT(w.Validator.MaxRate) { + return errCommissionRateTooLarge + } + if w.Validator.MaxChangeRate.GT(w.Validator.MaxRate) { + return errCommissionRateTooLarge + } + return nil } From 4fc6ce57f3d3d734a1acca40a7a82554aa6cb1a4 Mon Sep 17 00:00:00 2001 From: Daniel Van Der Maden Date: Mon, 18 Nov 2019 23:53:01 -0800 Subject: [PATCH 10/14] [test/deploy] Add staking debugging tools to localnet deploy script --- test/deploy.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test/deploy.sh b/test/deploy.sh index 811dac718..395425d88 100755 --- a/test/deploy.sh +++ b/test/deploy.sh @@ -59,6 +59,17 @@ function cleanup_and_result() { [ -e $RESULT_FILE ] && cat $RESULT_FILE } +function debug_staking() { + hmy_one_dir="$(go env GOPATH)/src/github.com/harmony-one" + hmy_bin="${hmy_one_dir}/go-sdk/hmy" + keystore="${hmy_one_dir}/harmony-ops/test-automation/api-tests/LocalnetValidatorKeys" + python3 -m pip install pyhmy + python3 -m pip install requests + python3 "${hmy_one_dir}/harmony-ops/test-automation/api-tests/test.py" --keystore ${keystore} \ + --cli_path ${hmy_bin} --test_dir "${hmy_one_dir}/harmony-ops/test-automation/api-tests/tests/" \ + --rpc_endpoint_src="http://localhost:9500/" --rpc_endpoint_dst="http://localhost:9501/" --ignore_regression_test +} + trap cleanup_and_result SIGINT SIGTERM function usage { @@ -192,6 +203,7 @@ while IFS='' read -r line || [[ -n "$line" ]]; do done < $config if [ "$DOTEST" == "true" ]; then + debug_staking echo "waiting for some block rewards" sleep 60 i=1 From bde9d38ed140dc3457fe96a6da9e903f9f1f0f7b Mon Sep 17 00:00:00 2001 From: Rongjian Lan Date: Mon, 18 Nov 2019 23:53:35 -0800 Subject: [PATCH 11/14] Add missing staking related core code; and add some basic rpcs --- block/factory/factory.go | 2 +- core/blockchain.go | 4 +-- core/state_processor.go | 2 -- internal/hmyapi/blockchain.go | 47 +++++++++++++----------------- internal/hmyapi/transactionpool.go | 10 ------- internal/hmyapi/types.go | 33 +++++++++++++++++++++ node/node_handler.go | 26 ++++++++--------- staking/types/transaction.go | 16 ++-------- staking/types/validator.go | 4 +-- 9 files changed, 73 insertions(+), 71 deletions(-) diff --git a/block/factory/factory.go b/block/factory/factory.go index 72c5e4dd8..dcef0c142 100644 --- a/block/factory/factory.go +++ b/block/factory/factory.go @@ -30,7 +30,7 @@ func NewFactory(chainConfig *params.ChainConfig) Factory { func (f *factory) NewHeader(epoch *big.Int) *block.Header { var impl blockif.Header switch { - case epoch.Cmp(f.chainConfig.StakingEpoch) < 0: // REVERT BEFORE COMMIT + case epoch.Cmp(f.chainConfig.StakingEpoch) >= 0: impl = v3.NewHeader() case epoch.Cmp(f.chainConfig.CrossLinkEpoch) >= 0: impl = v2.NewHeader() diff --git a/core/blockchain.go b/core/blockchain.go index 01905d56d..716596c87 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1171,7 +1171,7 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types. } // Do bookkeeping for new staking txns - //if bc.chainConfig.IsStaking(block.Epoch()) { + if bc.chainConfig.IsStaking(block.Epoch()) { for _, tx := range block.StakingTransactions() { err = bc.UpdateStakingMetaData(tx) // keep offchain database consistency with onchain we need revert @@ -1181,7 +1181,7 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types. return NonStatTy, err } } - //} + } //// Cross-links if len(header.CrossLinks()) > 0 { diff --git a/core/state_processor.go b/core/state_processor.go index 05b61c131..e8d81437d 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -266,9 +266,7 @@ func ApplyIncomingReceipt(config *params.ChainConfig, db *state.DB, header *bloc // requires a signer to derive the sender. // put it here to avoid cyclic import func StakingToMessage(tx *staking.StakingTransaction, blockNum *big.Int) (types.Message, error) { - utils.Logger().Info().Msgf("ApplyStakingMessage: aaaaaa:") payload, err := tx.RLPEncodeStakeMsg() - utils.Logger().Info().Msgf("ApplyStakingMessage: aaaaaa:", err) if err != nil { return types.Message{}, err } diff --git a/internal/hmyapi/blockchain.go b/internal/hmyapi/blockchain.go index a95f436db..337a91c6a 100644 --- a/internal/hmyapi/blockchain.go +++ b/internal/hmyapi/blockchain.go @@ -2,6 +2,7 @@ package hmyapi import ( "context" + "errors" "fmt" "math/big" @@ -298,33 +299,6 @@ func (s *PublicBlockChainAPI) GetLeader(ctx context.Context) string { return s.LatestHeader(ctx).Leader } -// GetValidatorInformation returns full validator info. -func (s *PublicBlockChainAPI) GetValidatorInformation(ctx context.Context, address string) (map[string]interface{}, error) { - validator := s.b.GetValidatorInformation(internal_common.ParseAddr(address)) - slotPubKeys := make([]string, 0) - for _, slotPubKey := range validator.SlotPubKeys { - slotPubKeys = append(slotPubKeys, slotPubKey.Hex()) - } - fields := map[string]interface{}{ - "address": validator.Address.String(), - "stake": hexutil.Uint64(validator.Stake.Uint64()), - "name": validator.Description.Name, - "slotPubKeys": slotPubKeys, - "unbondingHeight": hexutil.Uint64(validator.UnbondingHeight.Uint64()), - "minSelfDelegation": hexutil.Uint64(validator.MinSelfDelegation.Uint64()), - "active": validator.Active, - "identity": validator.Description.Identity, - "commissionRate": hexutil.Uint64(validator.Commission.CommissionRates.Rate.Int.Uint64()), - "commissionUpdateHeight": hexutil.Uint64(validator.Commission.UpdateHeight.Uint64()), - "commissionMaxRate": hexutil.Uint64(validator.Commission.CommissionRates.MaxRate.Uint64()), - "commissionMaxChangeRate": hexutil.Uint64(validator.Commission.CommissionRates.MaxChangeRate.Uint64()), - "website": validator.Description.Website, - "securityContact": validator.Description.SecurityContact, - "details": validator.Description.Details, - } - return fields, nil -} - // GetStake returns validator stake. func (s *PublicBlockChainAPI) GetStake(ctx context.Context, address string) hexutil.Uint64 { validator := s.b.GetValidatorInformation(internal_common.ParseAddr(address)) @@ -529,3 +503,22 @@ func (s *PublicBlockChainAPI) LatestHeader(ctx context.Context) *HeaderInformati header, _ := s.b.HeaderByNumber(context.Background(), rpc.LatestBlockNumber) // latest header should always be available return newHeaderInformation(header) } + +// GetAllValidatorAddresses returns all validator addresses. +func (s *PublicBlockChainAPI) GetAllValidatorAddresses() ([]common.Address, error) { + return s.b.GetAllValidatorAddresses(), nil +} + +// GetActiveValidatorAddresses returns active validator addresses. +func (s *PublicBlockChainAPI) GetActiveValidatorAddresses() ([]common.Address, error) { + return s.b.GetActiveValidatorAddresses(), nil +} + +// GetValidatorInfo returns information about a validator. +func (s *PublicBlockChainAPI) GetValidatorInfo(ctx context.Context, address common.Address) (*RPCValidator, error) { + validator := s.b.GetValidatorInformation(address) + if validator == nil { + return nil, errors.New(fmt.Sprintf("validator not found: %s", address.Hex())) + } + return newRPCValidator(validator), nil +} diff --git a/internal/hmyapi/transactionpool.go b/internal/hmyapi/transactionpool.go index 8596f5f88..8f52f81f5 100644 --- a/internal/hmyapi/transactionpool.go +++ b/internal/hmyapi/transactionpool.go @@ -298,13 +298,3 @@ func (s *PublicTransactionPoolAPI) GetCXReceiptByHash(ctx context.Context, hash } return nil } - -// GetAllValidatorAddresses returns ... -func (s *PublicTransactionPoolAPI) GetAllValidatorAddresses() ([]common.Address, error) { - return s.b.GetAllValidatorAddresses(), nil -} - -// GetActiveValidatorAddresses returns ... -func (s *PublicTransactionPoolAPI) GetActiveValidatorAddresses() ([]common.Address, error) { - return s.b.GetActiveValidatorAddresses(), nil -} diff --git a/internal/hmyapi/types.go b/internal/hmyapi/types.go index ef096314e..6d76ef168 100644 --- a/internal/hmyapi/types.go +++ b/internal/hmyapi/types.go @@ -6,6 +6,10 @@ import ( "strings" "time" + "github.com/harmony-one/harmony/shard" + + types2 "github.com/harmony-one/harmony/staking/types" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" ethtypes "github.com/ethereum/go-ethereum/core/types" @@ -60,6 +64,20 @@ type HeaderInformation struct { LastCommitBitmap string `json:"lastCommitBitmap"` } +// RPCValidator represents a validator +type RPCValidator struct { + Address common.Address `json:"address"` + SlotPubKeys []shard.BlsPublicKey `json:"slot_pub_keys"` + Stake *big.Int `json:"stake" yaml:"stake"` + UnbondingHeight *big.Int `json:"unbonding_height"` + MinSelfDelegation *big.Int `json:"min_self_delegation"` + MaxTotalDelegation *big.Int `json:"min_self_delegation"` + Active bool `json:"active"` + Commission types2.Commission `json:"commission"` + Description types2.Description `json:"description"` + CreationHeight *big.Int `json:"creation_height"` +} + func newHeaderInformation(header *block.Header) *HeaderInformation { if header == nil { return nil @@ -118,6 +136,21 @@ func newRPCCXReceipt(cx *types.CXReceipt, blockHash common.Hash, blockNumber uin return result } +func newRPCValidator(validator *types2.Validator) *RPCValidator { + return &RPCValidator{ + validator.Address, + validator.SlotPubKeys, + validator.Stake, + validator.UnbondingHeight, + validator.MinSelfDelegation, + validator.MaxTotalDelegation, + validator.Active, + validator.Commission, + validator.Description, + validator.CreationHeight, + } +} + // newRPCTransaction returns a transaction that will serialize to the RPC // representation, with the given location metadata set (if available). func newRPCTransaction(tx *types.Transaction, blockHash common.Hash, blockNumber uint64, index uint64) *RPCTransaction { diff --git a/node/node_handler.go b/node/node_handler.go index a582d3178..1f128fba6 100644 --- a/node/node_handler.go +++ b/node/node_handler.go @@ -401,19 +401,19 @@ func (node *Node) AddNewBlock(newBlock *types.Block) error { _, err := node.Blockchain().InsertChain([]*types.Block{newBlock}, true /* verifyHeaders */) // Debug only - addrs, err := node.Blockchain().ReadValidatorList() - utils.Logger().Debug().Msgf("validator list updated, err=%v, len(addrs)=%v", err, len(addrs)) - for i, addr := range addrs { - val, err := node.Blockchain().ValidatorInformation(addr) - if err != nil { - utils.Logger().Debug().Msgf("ValidatorInformation Error %v: err %v", i, err) - } - utils.Logger().Debug().Msgf("ValidatorInformation %v: %v", i, val) - } - currAddrs, err := node.Blockchain().ReadActiveValidatorList() - utils.Logger().Debug().Msgf("CurrentValidators : %v", currAddrs) - candidates := node.Blockchain().ValidatorCandidates() - utils.Logger().Debug().Msgf("CandidateValidators : %v", candidates) + //addrs, err := node.Blockchain().ReadValidatorList() + //utils.Logger().Debug().Msgf("validator list updated, err=%v, len(addrs)=%v", err, len(addrs)) + //for i, addr := range addrs { + // val, err := node.Blockchain().ValidatorInformation(addr) + // if err != nil { + // utils.Logger().Debug().Msgf("ValidatorInformation Error %v: err %v", i, err) + // } + // utils.Logger().Debug().Msgf("ValidatorInformation %v: %v", i, val) + //} + //currAddrs, err := node.Blockchain().ReadActiveValidatorList() + //utils.Logger().Debug().Msgf("CurrentValidators : %v", currAddrs) + //candidates := node.Blockchain().ValidatorCandidates() + //utils.Logger().Debug().Msgf("CandidateValidators : %v", candidates) // Finish debug if err != nil { diff --git a/staking/types/transaction.go b/staking/types/transaction.go index d344188d3..49678b45a 100644 --- a/staking/types/transaction.go +++ b/staking/types/transaction.go @@ -4,6 +4,7 @@ import ( "errors" "io" "math/big" + "reflect" "sync/atomic" "github.com/ethereum/go-ethereum/common" @@ -34,20 +35,7 @@ func (d *txdata) CopyFrom(d2 *txdata) { d.AccountNonce = d2.AccountNonce d.Price = new(big.Int).Set(d2.Price) d.GasLimit = d2.GasLimit - //switch d.Directive { // TODO: make these deep copies. - //case DirectiveCreateValidator: - // d.StakeMsg = d2.StakeMsg.(CreateValidator) - //case DirectiveEditValidator: - // d.StakeMsg = d2.StakeMsg.(EditValidator) - //case DirectiveDelegate: - // d.StakeMsg = d2.StakeMsg.(Delegate) - //case DirectiveUndelegate: - // d.StakeMsg = d2.StakeMsg.(Undelegate) - //case DirectiveCollectRewards: - // d.StakeMsg = d2.StakeMsg.(CollectRewards) - //default: - // return - //} + d.StakeMsg = reflect.New(reflect.ValueOf(d2.StakeMsg).Elem().Type()).Interface() d.V = new(big.Int).Set(d2.V) d.R = new(big.Int).Set(d2.R) d.S = new(big.Int).Set(d2.S) diff --git a/staking/types/validator.go b/staking/types/validator.go index 91368e045..0e51af8c5 100644 --- a/staking/types/validator.go +++ b/staking/types/validator.go @@ -31,7 +31,7 @@ var ( errMinSelfDelegationTooSmall = errors.New("min_self_delegation has to be greater than 1 ONE") errInvalidMaxTotalDelegation = errors.New("max_total_delegation can not be less than min_self_delegation") errCommissionRateTooLarge = errors.New("commission rate and change rate can not be larger than max commission rate") - errInvalidComissionRate = errors.New("commission rate, change rate and max rate should be within 0-100%") + errInvalidComissionRate = errors.New("commission rate, change rate and max rate should be within 0-100 percent") ) // ValidatorWrapper contains validator and its delegation information @@ -45,7 +45,7 @@ type Validator struct { // ECDSA address of the validator Address common.Address `json:"address" yaml:"address"` // The BLS public key of the validator for consensus - SlotPubKeys []shard.BlsPublicKey `json:"validating_pub_key" yaml:"validating_pub_key"` + SlotPubKeys []shard.BlsPublicKey `json:"slot_pub_keys" yaml:"slot_pub_keys"` // The stake put by the validator itself Stake *big.Int `json:"stake" yaml:"stake"` // if unbonding, height at which this validator has begun unbonding From 15d97c1ce22787c1e1064de634db4a3354afeae2 Mon Sep 17 00:00:00 2001 From: Rongjian Lan Date: Tue, 19 Nov 2019 11:07:37 -0800 Subject: [PATCH 12/14] fix lint --- internal/hmyapi/blockchain.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/hmyapi/blockchain.go b/internal/hmyapi/blockchain.go index 337a91c6a..162a5a03e 100644 --- a/internal/hmyapi/blockchain.go +++ b/internal/hmyapi/blockchain.go @@ -2,7 +2,6 @@ package hmyapi import ( "context" - "errors" "fmt" "math/big" @@ -518,7 +517,7 @@ func (s *PublicBlockChainAPI) GetActiveValidatorAddresses() ([]common.Address, e func (s *PublicBlockChainAPI) GetValidatorInfo(ctx context.Context, address common.Address) (*RPCValidator, error) { validator := s.b.GetValidatorInformation(address) if validator == nil { - return nil, errors.New(fmt.Sprintf("validator not found: %s", address.Hex())) + return nil, fmt.Errorf("validator not found: %s", address.Hex()) } return newRPCValidator(validator), nil } From 3106c87bb5920e9b675ae15aac7beb67ec750ad5 Mon Sep 17 00:00:00 2001 From: Rongjian Lan Date: Tue, 19 Nov 2019 12:15:58 -0800 Subject: [PATCH 13/14] Fix a bigInt cmp bug --- staking/types/delegation.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/staking/types/delegation.go b/staking/types/delegation.go index 6fc8c5668..04ddb602a 100644 --- a/staking/types/delegation.go +++ b/staking/types/delegation.go @@ -13,6 +13,11 @@ var ( errInvalidAmount = errors.New("Invalid amount, must be positive") ) +const ( + // The number of epochs a undelegated token needs to be before it's released to the delegator's balance + LockPeriodInEpoch = 14 +) + // Delegation represents the bond with tokens held by an account. It is // owned by one delegator, and is associated with the voting power of one // validator. @@ -99,7 +104,7 @@ func (d *Delegation) RemoveUnlockedUndelegations(curEpoch *big.Int) *big.Int { totalWithdraw := big.NewInt(0) count := 0 for j := range d.Entries { - if curEpoch.Cmp(d.Entries[j].Epoch) > 14 { // need to wait at least 14 epochs to withdraw; + if big.NewInt(0).Sub(curEpoch, d.Entries[j].Epoch).Int64() > LockPeriodInEpoch { // need to wait at least 14 epochs to withdraw; totalWithdraw.Add(totalWithdraw, d.Entries[j].Amount) count++ } else { From 73f60b5c58de42d7e5b3d7fc37640b9ff6acd6aa Mon Sep 17 00:00:00 2001 From: Rongjian Lan Date: Tue, 19 Nov 2019 12:31:31 -0800 Subject: [PATCH 14/14] fix lint --- staking/types/delegation.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/staking/types/delegation.go b/staking/types/delegation.go index 04ddb602a..300834680 100644 --- a/staking/types/delegation.go +++ b/staking/types/delegation.go @@ -14,7 +14,7 @@ var ( ) const ( - // The number of epochs a undelegated token needs to be before it's released to the delegator's balance + // LockPeriodInEpoch is the number of epochs a undelegated token needs to be before it's released to the delegator's balance LockPeriodInEpoch = 14 )