From 2baab4864ca16f5bf342f2e5fb315ce532694527 Mon Sep 17 00:00:00 2001 From: Edgar Aroutiounian Date: Tue, 10 Mar 2020 15:17:32 -0700 Subject: [PATCH] [staking][validation][protocol] (#2396) * [staking][validation][protocol] Limit max bls keys * [staking-era] Fold banned and active into single field * [slash][effective] Remove LRU cache for slash, change .Active to enumeration * [slash] Remove leftover wrong usage of Logger * [slash][offchain] Only Decode if len > 0 * [offchain] cosmetic * [slash] Remove some logs in proposal * [webhook] Move webhook with call for when cannot commit block * [shard] Finally make finding subcommittee by shardID an explicit error * [node] Whitespace, prefer literal * [webhook] Report bad block to webhook * [slash] Expand verify, remove bad log usage, explicit error handle * [slash] Check on key size * [slash] Explicit upper bound of pending slashes * [slash] Use right epoch snapshot, fail to verify if epoch wrong on beaconchain * [multibls] Make max count allowed be 1/3 of external slots * [quorum] Remove bad API of ShardIDProvider, factor out committee key as method of committee * [verify] Begin factor out of common verification approach * [project] Further remove RawJSON log, use proper epoch for snapshot * [slash] Implement verification * [slash] Implement BLS key verification of ballots * [rpc] Keep validator information as meaningful as possible * [staking] Never can stop being banned * [slash] Comments and default Unknown case of eligibility * [slash] Be explicit on what input values allowed when want to change EPOSStatus * [consensus] Remove unneeded TODO * [verify] Add proper error message * [rpc] Give back to caller their wrong chain id * [chain] Add extra map dump of delegation sizing for downstream analysis * [engine] Less code, more methods * [offchain] More leniency in handling slash bytes and delete from pending * [validator] Remove errors on bad input for edit --- cmd/harmony/main.go | 3 - consensus/consensus_service.go | 36 +++-- consensus/construct_test.go | 10 +- consensus/engine/consensus_engine.go | 2 + consensus/leader.go | 53 ++++-- consensus/quorum/one-node-one-vote.go | 5 +- consensus/quorum/one-node-staked-vote.go | 3 - consensus/quorum/one-node-staked-vote_test.go | 4 +- consensus/quorum/quorum.go | 44 +++-- consensus/threshold.go | 12 +- consensus/view_change.go | 15 +- consensus/votepower/roster.go | 6 + core/blockchain.go | 153 ++++++++---------- core/chain_makers.go | 18 ++- core/offchain.go | 31 ++-- core/staking_verifier.go | 45 ++++-- core/state/statedb.go | 4 +- hmy/api_backend.go | 46 +++--- internal/chain/engine.go | 55 ++++--- internal/chain/reward.go | 6 +- internal/hmyapi/apiv1/transactionpool.go | 14 +- internal/hmyapi/apiv2/transactionpool.go | 14 +- node/double_signing.go | 18 +-- node/node.go | 55 ++++--- node/node_cross_link.go | 80 +++------ node/node_genesis.go | 7 +- node/node_handler.go | 13 +- node/node_newblock.go | 6 +- node/worker/worker.go | 57 +++---- shard/committee/assignment.go | 9 +- shard/shard_state.go | 22 ++- staking/availability/measure.go | 35 ++-- staking/effective/eligible.go | 26 ++- staking/slash/double-sign.go | 146 ++++++++++++----- staking/slash/double-sign_test.go | 92 +++++++---- staking/types/messages.go | 28 ++-- staking/types/validator.go | 73 ++++++--- staking/types/validator_test.go | 52 +++--- staking/verify/verify.go | 59 +++++++ 39 files changed, 817 insertions(+), 540 deletions(-) create mode 100644 staking/verify/verify.go diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index 2b991410f..115ef2922 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -422,9 +422,6 @@ func setupConsensusAndNode(nodeConfig *nodeconfig.ConfigType) *node.Node { currentConsensus, err := consensus.New( myHost, nodeConfig.ShardID, p2p.Peer{}, nodeConfig.ConsensusPriKey, decider, ) - currentConsensus.Decider.SetShardIDProvider(func() (uint32, error) { - return currentConsensus.ShardID, nil - }) currentConsensus.Decider.SetMyPublicKeyProvider(func() (*multibls.PublicKey, error) { return currentConsensus.PubKey, nil }) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 1d558f9f6..07d18cd64 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -414,8 +414,8 @@ func (consensus *Consensus) getLeaderPubKeyFromCoinbase(header *block.Header) (* ).WithCause(err) } - committee := shardState.FindCommitteeByID(header.ShardID()) - if committee == nil { + committee, err := shardState.FindCommitteeByID(header.ShardID()) + if err != nil { return nil, ctxerror.New("cannot find shard in the shard state", "blockNum", header.Number(), "shardID", header.ShardID(), @@ -477,9 +477,6 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode { // Only happens once, the flip-over to a new Decider policy if isFirstTimeStaking || haventUpdatedDecider { decider := quorum.NewDecider(quorum.SuperMajorityStake) - decider.SetShardIDProvider(func() (uint32, error) { - return consensus.ShardID, nil - }) decider.SetMyPublicKeyProvider(func() (*multibls.PublicKey, error) { return consensus.PubKey, nil }) @@ -504,8 +501,9 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode { if len(curHeader.ShardState()) > 0 { // increase curEpoch by one if it's the last block consensus.SetEpochNum(curEpoch.Uint64() + 1) - consensus.getLogger().Info().Uint64("headerNum", curHeader.Number().Uint64()). - Msg("[UpdateConsensusInformation] Epoch updated for nextEpoch curEpoch") + consensus.getLogger().Info(). + Uint64("headerNum", curHeader.Number().Uint64()). + Msg("Epoch updated for nextEpoch curEpoch") nextShardState, err := committee.WithStakingEnabled.ReadFromDB( nextEpoch, consensus.ChainReader, @@ -514,15 +512,33 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode { utils.Logger().Error(). Err(err). Uint32("shard", consensus.ShardID). - Msg("[UpdateConsensusInformation] Error retrieving nextEpoch shard state") + Msg("Error retrieving nextEpoch shard state") + return Syncing + } + + subComm, err := nextShardState.FindCommitteeByID(curHeader.ShardID()) + if err != nil { + utils.Logger().Error(). + Err(err). + Uint32("shard", consensus.ShardID). + Msg("Error retrieving nextEpoch shard state") return Syncing } - committeeToSet = nextShardState.FindCommitteeByID(curHeader.ShardID()) + committeeToSet = subComm } else { consensus.SetEpochNum(curEpoch.Uint64()) - committeeToSet = curShardState.FindCommitteeByID(curHeader.ShardID()) + subComm, err := curShardState.FindCommitteeByID(curHeader.ShardID()) + if err != nil { + utils.Logger().Error(). + Err(err). + Uint32("shard", consensus.ShardID). + Msg("Error retrieving current shard state") + return Syncing + } + + committeeToSet = subComm } if len(committeeToSet.Slots) == 0 { diff --git a/consensus/construct_test.go b/consensus/construct_test.go index 88043bb61..9c4e6011e 100644 --- a/consensus/construct_test.go +++ b/consensus/construct_test.go @@ -65,13 +65,19 @@ func TestConstructPreparedMessage(test *testing.T) { leaderPubKey, leaderPriKey.Sign(message), common.BytesToHash(consensus.blockHash[:]), + consensus.blockNum, + consensus.viewID, ) - consensus.Decider.SubmitVote( + if _, err := consensus.Decider.SubmitVote( quorum.Prepare, validatorPubKey, validatorPriKey.Sign(message), common.BytesToHash(consensus.blockHash[:]), - ) + consensus.blockNum, + consensus.viewID, + ); err != nil { + test.Log(err) + } // According to RJ these failures are benign. if err := consensus.prepareBitmap.SetKey(leaderPubKey, true); err != nil { diff --git a/consensus/engine/consensus_engine.go b/consensus/engine/consensus_engine.go index 8bf5e1810..c1ced33ef 100644 --- a/consensus/engine/consensus_engine.go +++ b/consensus/engine/consensus_engine.go @@ -53,6 +53,8 @@ type ChainReader interface { // Methods needed for EPoS committee assignment calculation committee.StakingCandidatesReader + // Methods for reading right epoch snapshot + staking.ValidatorSnapshotReader //ReadBlockRewardAccumulator is the block-reward given for block number ReadBlockRewardAccumulator(uint64) (*big.Int, error) diff --git a/consensus/leader.go b/consensus/leader.go index c536c77db..9054f7b82 100644 --- a/consensus/leader.go +++ b/consensus/leader.go @@ -62,12 +62,16 @@ func (consensus *Consensus) announce(block *types.Block) { // Leader sign the block hash itself for i, key := range consensus.PubKey.PublicKey { - consensus.Decider.SubmitVote( + if _, err := consensus.Decider.SubmitVote( quorum.Prepare, key, consensus.priKey.PrivateKey[i].SignHash(consensus.blockHash[:]), common.BytesToHash(consensus.blockHash[:]), - ) + consensus.blockNum, + consensus.viewID, + ); err != nil { + return + } if err := consensus.prepareBitmap.SetKey(key, true); err != nil { consensus.getLogger().Warn().Err(err).Msg( "[Announce] Leader prepareBitmap SetKey failed", @@ -166,9 +170,14 @@ func (consensus *Consensus) onPrepare(msg *msg_pb.Message) { Int64("NumReceivedSoFar", consensus.Decider.SignersCount(quorum.Prepare)). Int64("PublicKeys", consensus.Decider.ParticipantsCount()).Logger() logger.Info().Msg("[OnPrepare] Received New Prepare Signature") - consensus.Decider.SubmitVote( - quorum.Prepare, validatorPubKey, &sign, recvMsg.BlockHash, - ) + if _, err := consensus.Decider.SubmitVote( + quorum.Prepare, validatorPubKey, + &sign, recvMsg.BlockHash, + recvMsg.BlockNum, recvMsg.ViewID, + ); err != nil { + consensus.getLogger().Warn().Err(err).Msg("submit vote prepare failed") + return + } // Set the bitmap indicating that this validator signed. if err := prepareBitmap.SetKey(recvMsg.SenderPubkey, true); err != nil { consensus.getLogger().Warn().Err(err).Msg("[OnPrepare] prepareBitmap.SetKey failed") @@ -235,9 +244,17 @@ func (consensus *Consensus) onCommit(msg *msg_pb.Message) { return } offender := *shard.FromLibBLSPublicKeyUnsafe(recvMsg.SenderPubkey) - addr, err := committee.FindCommitteeByID( + subComm, err := committee.FindCommitteeByID( consensus.ShardID, - ).AddressForBLSKey(offender) + ) + if err != nil { + log.Err(err). + Str("msg", recvMsg.String()). + Msg("could not find subcommittee for bls key") + return + } + + addr, err := subComm.AddressForBLSKey(offender) if err != nil { log.Err(err).Str("msg", recvMsg.String()). @@ -252,16 +269,14 @@ func (consensus *Consensus) onCommit(msg *msg_pb.Message) { ConflictingBallots: slash.ConflictingBallots{ *alreadyCastBallot, votepower.Ballot{ - offender, - recvMsg.BlockHash, - common.Hex2Bytes(doubleSign.SerializeToHexStr()), + SignerPubKey: offender, + BlockHeaderHash: recvMsg.BlockHash, + Signature: common.Hex2Bytes(doubleSign.SerializeToHexStr()), + Height: recvMsg.BlockNum, + ViewID: recvMsg.ViewID, }}, Moment: slash.Moment{ - // TODO need to extend fbft tro have epoch to use its epoch - // rather than curHeader epoch Epoch: curHeader.Epoch(), - Height: new(big.Int).SetUint64(recvMsg.BlockNum), - ViewID: consensus.viewID, ShardID: consensus.ShardID, TimeUnixNano: now, }, @@ -314,9 +329,13 @@ func (consensus *Consensus) onCommit(msg *msg_pb.Message) { Logger() logger.Info().Msg("[OnCommit] Received new commit message") - consensus.Decider.SubmitVote( - quorum.Commit, validatorPubKey, &sign, recvMsg.BlockHash, - ) + if _, err := consensus.Decider.SubmitVote( + quorum.Commit, validatorPubKey, + &sign, recvMsg.BlockHash, + recvMsg.BlockNum, recvMsg.ViewID, + ); err != nil { + return + } // Set the bitmap indicating that this validator signed. if err := commitBitmap.SetKey(recvMsg.SenderPubkey, true); err != nil { consensus.getLogger().Warn().Err(err). diff --git a/consensus/quorum/one-node-one-vote.go b/consensus/quorum/one-node-one-vote.go index 93187196c..f1f7afa43 100644 --- a/consensus/quorum/one-node-one-vote.go +++ b/consensus/quorum/one-node-one-vote.go @@ -93,17 +93,14 @@ func (v *uniformVoteWeight) String() string { } func (v *uniformVoteWeight) MarshalJSON() ([]byte, error) { - s, _ := v.ShardIDProvider()() - type t struct { Policy string `json:"policy"` - ShardID uint32 `json:"shard-id"` Count int `json:"count"` Participants []string `json:"committee-members"` } members := v.DumpParticipants() - return json.Marshal(t{v.Policy().String(), s, len(members), members}) + return json.Marshal(t{v.Policy().String(), len(members), members}) } func (v *uniformVoteWeight) AmIMemberOfCommitee() bool { diff --git a/consensus/quorum/one-node-staked-vote.go b/consensus/quorum/one-node-staked-vote.go index 2a05c0b6c..5d818c622 100644 --- a/consensus/quorum/one-node-staked-vote.go +++ b/consensus/quorum/one-node-staked-vote.go @@ -182,7 +182,6 @@ func (v *stakedVoteWeight) String() string { } func (v *stakedVoteWeight) MarshalJSON() ([]byte, error) { - s, _ := v.ShardIDProvider()() voterCount := len(v.roster.Voters) type u struct { IsHarmony bool `json:"is-harmony-slot"` @@ -195,7 +194,6 @@ func (v *stakedVoteWeight) MarshalJSON() ([]byte, error) { type t struct { Policy string `json"policy"` - ShardID uint32 `json:"shard-id"` Count int `json:"count"` Participants []u `json:"committee-members"` HmyVotingPower string `json:"hmy-voting-power"` @@ -224,7 +222,6 @@ func (v *stakedVoteWeight) MarshalJSON() ([]byte, error) { return json.Marshal(t{ v.Policy().String(), - s, voterCount, parts, v.roster.OurVotingPowerTotalPercentage.String(), diff --git a/consensus/quorum/one-node-staked-vote_test.go b/consensus/quorum/one-node-staked-vote_test.go index 5a8b5f685..f76e1e787 100644 --- a/consensus/quorum/one-node-staked-vote_test.go +++ b/consensus/quorum/one-node-staked-vote_test.go @@ -64,7 +64,6 @@ func setupBaseCase() (Decider, *TallyResult, shard.SlotList, map[string]secretKe } decider := NewDecider(SuperMajorityStake) - decider.SetShardIDProvider(func() (uint32, error) { return 0, nil }) decider.UpdateParticipants(pubKeys) tally, err := decider.SetVoters(slotList) if err != nil { @@ -90,7 +89,6 @@ func setupEdgeCase() (Decider, *TallyResult, shard.SlotList, secretKeyMap) { } decider := NewDecider(SuperMajorityStake) - decider.SetShardIDProvider(func() (uint32, error) { return 0, nil }) decider.UpdateParticipants(pubKeys) tally, err := decider.SetVoters(slotList) if err != nil { @@ -104,7 +102,7 @@ func sign(d Decider, k secretKeyMap, p Phase) { pubKey := v.GetPublicKey() sig := v.Sign(msg) // TODO Make upstream test provide meaningful test values - d.SubmitVote(p, pubKey, sig, common.Hash{}) + d.SubmitVote(p, pubKey, sig, common.Hash{}, 0, 0) } } diff --git a/consensus/quorum/quorum.go b/consensus/quorum/quorum.go index 1324f228d..e83e6f2c8 100644 --- a/consensus/quorum/quorum.go +++ b/consensus/quorum/quorum.go @@ -7,7 +7,6 @@ import ( "github.com/harmony-one/bls/ffi/go/bls" "github.com/harmony-one/harmony/consensus/votepower" bls_cosi "github.com/harmony-one/harmony/crypto/bls" - "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/multibls" "github.com/harmony-one/harmony/numeric" "github.com/harmony-one/harmony/shard" @@ -26,11 +25,14 @@ const ( ViewChange ) -var phaseNames = map[Phase]string{ - Prepare: "Prepare", - Commit: "Commit", - ViewChange: "viewChange", -} +var ( + phaseNames = map[Phase]string{ + Prepare: "Prepare", + Commit: "Commit", + ViewChange: "viewChange", + } + errPhaseUnknown = errors.New("invariant of known phase violated") +) func (p Phase) String() string { if name, ok := phaseNames[p]; ok { @@ -76,8 +78,10 @@ type ParticipantTracker interface { type SignatoryTracker interface { ParticipantTracker SubmitVote( - p Phase, PubKey *bls.PublicKey, sig *bls.Sign, headerHash common.Hash, - ) *votepower.Ballot + p Phase, PubKey *bls.PublicKey, + sig *bls.Sign, headerHash common.Hash, + height, viewID uint64, + ) (*votepower.Ballot, error) // Caller assumes concurrency protection SignersCount(Phase) int64 reset([]Phase) @@ -95,13 +99,11 @@ type SignatureReader interface { // DependencyInjectionWriter .. type DependencyInjectionWriter interface { - SetShardIDProvider(func() (uint32, error)) SetMyPublicKeyProvider(func() (*multibls.PublicKey, error)) } // DependencyInjectionReader .. type DependencyInjectionReader interface { - ShardIDProvider() func() (uint32, error) MyPublicKey() func() (*multibls.PublicKey, error) } @@ -225,12 +227,16 @@ func (s *cIdentities) SignersCount(p Phase) int64 { } func (s *cIdentities) SubmitVote( - p Phase, PubKey *bls.PublicKey, sig *bls.Sign, headerHash common.Hash, -) *votepower.Ballot { + p Phase, PubKey *bls.PublicKey, + sig *bls.Sign, headerHash common.Hash, + height, viewID uint64, +) (*votepower.Ballot, error) { ballot := &votepower.Ballot{ SignerPubKey: *shard.FromLibBLSPublicKeyUnsafe(PubKey), BlockHeaderHash: headerHash, Signature: common.Hex2Bytes(sig.SerializeToHexStr()), + Height: height, + ViewID: viewID, } switch hex := PubKey.SerializeToHexStr(); p { case Prepare: @@ -240,11 +246,9 @@ func (s *cIdentities) SubmitVote( case ViewChange: s.viewChange.BallotBox[hex] = ballot default: - utils.Logger().Err(errors.New("invariant of known phase violated")). - Str("phase", p.String()). - Msg("bad vote input") + return nil, errors.Wrapf(errPhaseUnknown, "given: %s", p.String()) } - return ballot + return ballot, nil } func (s *cIdentities) reset(ps []Phase) { @@ -316,14 +320,6 @@ type composite struct { SignatureReader } -func (d *depInject) SetShardIDProvider(p func() (uint32, error)) { - d.shardIDProvider = p -} - -func (d *depInject) ShardIDProvider() func() (uint32, error) { - return d.shardIDProvider -} - func (d *depInject) SetMyPublicKeyProvider(p func() (*multibls.PublicKey, error)) { d.publicKeyProvider = p } diff --git a/consensus/threshold.go b/consensus/threshold.go index f87695eb4..135f45831 100644 --- a/consensus/threshold.go +++ b/consensus/threshold.go @@ -20,7 +20,9 @@ func (consensus *Consensus) didReachPrepareQuorum() error { return err } // Construct and broadcast prepared message - networkMessage, err := consensus.construct(msg_pb.MessageType_PREPARED, nil, consensus.LeaderPubKey, leaderPriKey) + networkMessage, err := consensus.construct( + msg_pb.MessageType_PREPARED, nil, consensus.LeaderPubKey, leaderPriKey, + ) if err != nil { consensus.getLogger().Err(err). Str("message-type", msg_pb.MessageType_PREPARED.String()). @@ -42,12 +44,16 @@ func (consensus *Consensus) didReachPrepareQuorum() error { // so by this point, everyone has committed to the blockhash of this block // in prepare and so this is the actual block. for i, key := range consensus.PubKey.PublicKey { - consensus.Decider.SubmitVote( + if _, err := consensus.Decider.SubmitVote( quorum.Commit, key, consensus.priKey.PrivateKey[i].SignHash(commitPayload), common.BytesToHash(consensus.blockHash[:]), - ) + consensus.blockNum, + consensus.viewID, + ); err != nil { + return err + } if err := consensus.commitBitmap.SetKey(key, true); err != nil { consensus.getLogger().Debug().Msg("[OnPrepare] Leader commit bitmap set failed") diff --git a/consensus/view_change.go b/consensus/view_change.go index 5970b3d6f..135f24ef5 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -380,14 +380,19 @@ func (consensus *Consensus) onViewChange(msg *msg_pb.Message) { blockNumBytes := [8]byte{} binary.LittleEndian.PutUint64(blockNumBytes[:], consensus.blockNum) commitPayload := append(blockNumBytes[:], consensus.blockHash[:]...) - consensus.Decider.SubmitVote( + if _, err := consensus.Decider.SubmitVote( quorum.Commit, newLeaderKey, newLeaderPriKey.SignHash(commitPayload), common.BytesToHash(consensus.blockHash[:]), - ) + consensus.blockNum, + recvMsg.ViewID, + ); err != nil { + consensus.getLogger().Debug().Msg("submit vote on viewchange commit failed") + return + } - if err = consensus.commitBitmap.SetKey(newLeaderKey, true); err != nil { + if err := consensus.commitBitmap.SetKey(newLeaderKey, true); err != nil { consensus.getLogger().Debug(). Msg("[OnViewChange] New Leader commit bitmap set failed") return @@ -395,7 +400,9 @@ func (consensus *Consensus) onViewChange(msg *msg_pb.Message) { } consensus.current.SetViewID(recvMsg.ViewID) - msgToSend := consensus.constructNewViewMessage(recvMsg.ViewID, newLeaderKey, newLeaderPriKey) + msgToSend := consensus.constructNewViewMessage( + recvMsg.ViewID, newLeaderKey, newLeaderPriKey, + ) consensus.getLogger().Warn(). Int("payloadSize", len(consensus.m1Payload)). diff --git a/consensus/votepower/roster.go b/consensus/votepower/roster.go index 520054ffe..643e0737d 100644 --- a/consensus/votepower/roster.go +++ b/consensus/votepower/roster.go @@ -27,6 +27,8 @@ type Ballot struct { SignerPubKey shard.BlsPublicKey `json:"bls-public-key"` BlockHeaderHash common.Hash `json:"block-header-hash"` Signature []byte `json:"bls-signature"` + Height uint64 `json:"block-height"` + ViewID uint64 `json:"view-id"` } // MarshalJSON .. @@ -35,10 +37,14 @@ func (b Ballot) MarshalJSON() ([]byte, error) { A string `json:"bls-public-key"` B string `json:"block-header-hash"` C string `json:"bls-signature"` + E uint64 `json:"block-height"` + F uint64 `json:"view-id"` }{ b.SignerPubKey.Hex(), b.BlockHeaderHash.Hex(), hex.EncodeToString(b.Signature), + b.Height, + b.ViewID, }) } diff --git a/core/blockchain.go b/core/blockchain.go index 567e9525c..ce43a9906 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -60,6 +60,8 @@ var ( blockInsertTimer = metrics.NewRegisteredTimer("chain/inserts", nil) // ErrNoGenesis is the error when there is no genesis. ErrNoGenesis = errors.New("Genesis not found in chain") + // errExceedMaxPendingSlashes .. + errExceedMaxPendingSlashes = errors.New("exceeed max pending slashes") ) const ( @@ -80,15 +82,10 @@ const ( validatorListByDelegatorCacheLimit = 1024 pendingCrossLinksCacheLimit = 2 blockAccumulatorCacheLimit = 256 - pendingSlashingCandidateCacheLimit = 2 - + maxPendingSlashes = 512 // BlockChainVersion ensures that an incompatible database forces a resync from scratch. BlockChainVersion = 3 -) - -const ( pendingCLCacheKey = "pendingCLs" - pendingSCCacheKey = "pendingSCs" ) // CacheConfig contains the configuration values for the trie caching/pruning @@ -148,35 +145,37 @@ type BlockChain struct { futureBlocks *lru.Cache // future blocks are blocks added for later processing shardStateCache *lru.Cache lastCommitsCache *lru.Cache - epochCache *lru.Cache // Cache epoch number → first block number - randomnessCache *lru.Cache // Cache for vrf/vdf - validatorCache *lru.Cache // Cache for validator info - validatorStatsCache *lru.Cache // Cache for validator stats - validatorListCache *lru.Cache // Cache of validator list - validatorListByDelegatorCache *lru.Cache // Cache of validator list by delegator - pendingCrossLinksCache *lru.Cache // Cache of last pending crosslinks - blockAccumulatorCache *lru.Cache // Cache of block accumulators - pendingSlashingCandidates *lru.Cache // Cache of last pending slashing candidates - - quit chan struct{} // blockchain quit channel - running int32 // running must be called atomically + epochCache *lru.Cache // Cache epoch number → first block number + randomnessCache *lru.Cache // Cache for vrf/vdf + validatorCache *lru.Cache // Cache for validator info + validatorStatsCache *lru.Cache // Cache for validator stats + validatorListCache *lru.Cache // Cache of validator list + validatorListByDelegatorCache *lru.Cache // Cache of validator list by delegator + pendingCrossLinksCache *lru.Cache // Cache of last pending crosslinks + blockAccumulatorCache *lru.Cache // Cache of block accumulators + quit chan struct{} // blockchain quit channel + running int32 // running must be called atomically // procInterrupt must be atomically called procInterrupt int32 // interrupt signaler for block processing wg sync.WaitGroup // chain processing wait group for shutting down - engine consensus_engine.Engine - processor Processor // block processor interface - validator Validator // block and state validator interface - vmConfig vm.Config - + engine consensus_engine.Engine + processor Processor // block processor interface + validator Validator // block and state validator interface + vmConfig vm.Config badBlocks *lru.Cache // Bad block cache shouldPreserve func(*types.Block) bool // Function used to determine whether should preserve the given block. + pendingSlashes slash.Records } // NewBlockChain returns a fully initialised block chain using information // available in the database. It initialises the default Ethereum Validator and // Processor. -func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *params.ChainConfig, engine consensus_engine.Engine, vmConfig vm.Config, shouldPreserve func(block *types.Block) bool) (*BlockChain, error) { +func NewBlockChain( + db ethdb.Database, cacheConfig *CacheConfig, chainConfig *params.ChainConfig, + engine consensus_engine.Engine, vmConfig vm.Config, + shouldPreserve func(block *types.Block) bool, +) (*BlockChain, error) { if cacheConfig == nil { cacheConfig = &CacheConfig{ TrieNodeLimit: 256 * 1024 * 1024, @@ -199,7 +198,6 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par validatorListByDelegatorCache, _ := lru.New(validatorListByDelegatorCacheLimit) pendingCrossLinksCache, _ := lru.New(pendingCrossLinksCacheLimit) blockAccumulatorCache, _ := lru.New(blockAccumulatorCacheLimit) - pendingSlashingCandidateCache, _ := lru.New(pendingSlashingCandidateCacheLimit) bc := &BlockChain{ chainConfig: chainConfig, @@ -223,11 +221,11 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par validatorListCache: validatorListCache, validatorListByDelegatorCache: validatorListByDelegatorCache, pendingCrossLinksCache: pendingCrossLinksCache, - pendingSlashingCandidates: pendingSlashingCandidateCache, blockAccumulatorCache: blockAccumulatorCache, engine: engine, vmConfig: vmConfig, badBlocks: badBlocks, + pendingSlashes: slash.Records{}, } bc.SetValidator(NewBlockValidator(chainConfig, bc, engine)) bc.SetProcessor(NewStateProcessor(chainConfig, bc, engine)) @@ -1940,56 +1938,38 @@ func (bc *BlockChain) ReadShardLastCrossLink(shardID uint32) (*types.CrossLink, return types.DeserializeCrossLink(bytes) } -// DeletePendingSlashingCandidates .. -func (bc *BlockChain) DeletePendingSlashingCandidates() error { - bc.pendingSlashingCandidatesMU.Lock() - defer bc.pendingSlashingCandidatesMU.Unlock() - bc.pendingSlashingCandidates.Purge() - return bc.WritePendingSlashingCandidates(slash.Records{}) -} - -// ReadPendingSlashingCandidates retrieves pending slashing candidates -func (bc *BlockChain) ReadPendingSlashingCandidates() (slash.Records, error) { - cls := slash.Records{} - if !bc.Config().IsStaking(bc.CurrentHeader().Epoch()) { - return cls, nil - } - var err error - bytes := []byte{} - if cached, ok := bc.pendingSlashingCandidates.Get(pendingSCCacheKey); ok { - bytes = cached.([]byte) - } else { - bytes, err = rawdb.ReadPendingSlashingCandidates(bc.db) - if err != nil || len(bytes) == 0 { - utils.Logger().Info().Err(err). - Int("dataLen", len(bytes)). - Msg("ReadPendingSlashingCandidates") - return nil, err - } - } - - if err := rlp.DecodeBytes(bytes, &cls); err != nil { - utils.Logger().Error().Err(err).Msg("Invalid pending slashing candidates RLP decoding") - return nil, err - } - return cls, nil -} - -// WritePendingSlashingCandidates saves the pending slashing candidates -func (bc *BlockChain) WritePendingSlashingCandidates(candidates slash.Records) error { - bytes, err := rlp.EncodeToBytes(candidates) +func (bc *BlockChain) writeSlashes(processed slash.Records) error { + bytes, err := rlp.EncodeToBytes(processed) if err != nil { - const msg = "[WritePendingSlashingCandidates] Failed to encode pending slashing candidates" + const msg = "failed to encode slashing candidates" utils.Logger().Error().Msg(msg) return err } if err := rawdb.WritePendingSlashingCandidates(bc.db, bytes); err != nil { return err } - bc.pendingSlashingCandidates.Add(pendingSCCacheKey, bytes) return nil } +// DeleteFromPendingSlashingCandidates .. +func (bc *BlockChain) DeleteFromPendingSlashingCandidates( + processed slash.Records, +) error { + bc.pendingSlashingCandidatesMU.Lock() + defer bc.pendingSlashingCandidatesMU.Unlock() + current := bc.ReadPendingSlashingCandidates() + bc.pendingSlashes = current.SetDifference(processed) + return bc.writeSlashes(bc.pendingSlashes) +} + +// ReadPendingSlashingCandidates retrieves pending slashing candidates +func (bc *BlockChain) ReadPendingSlashingCandidates() slash.Records { + if !bc.Config().IsStaking(bc.CurrentHeader().Epoch()) { + return slash.Records{} + } + return append(bc.pendingSlashes[0:0], bc.pendingSlashes...) +} + // ReadPendingCrossLinks retrieves pending crosslinks func (bc *BlockChain) ReadPendingCrossLinks() ([]types.CrossLink, error) { bytes := []byte{} @@ -2047,25 +2027,21 @@ func (bc *BlockChain) WritePendingCrossLinks(crossLinks []types.CrossLink) error // AddPendingSlashingCandidates appends pending slashing candidates func (bc *BlockChain) AddPendingSlashingCandidates( - candidates []slash.Record, -) (int, error) { + candidates slash.Records, +) error { bc.pendingSlashingCandidatesMU.Lock() defer bc.pendingSlashingCandidatesMU.Unlock() - cls, err := bc.ReadPendingSlashingCandidates() - - if err != nil || len(cls) == 0 { - err := bc.WritePendingSlashingCandidates(candidates) - if err != nil { - return 0, err - } - return 1, err - } - - cls = append(cls, candidates...) - if err := bc.WritePendingSlashingCandidates(cls); err != nil { - return 0, err + current := bc.ReadPendingSlashingCandidates() + pendingSlashes := append( + bc.pendingSlashes, current.SetDifference(candidates)..., + ) + if l, c := len(pendingSlashes), len(current); l > maxPendingSlashes { + return errors.Wrapf( + errExceedMaxPendingSlashes, "current %d with-additional %d", c, l, + ) } - return len(cls), nil + bc.pendingSlashes = pendingSlashes + return bc.writeSlashes(bc.pendingSlashes) } // AddPendingCrossLinks appends pending crosslinks @@ -2219,6 +2195,15 @@ func (bc *BlockChain) ReadValidatorInformation( return bc.ReadValidatorInformationAt(addr, bc.CurrentBlock().Root()) } +// ReadValidatorSnapshotAtEpoch reads the snapshot +// staking validator information of given validator address +func (bc *BlockChain) ReadValidatorSnapshotAtEpoch( + epoch *big.Int, + addr common.Address, +) (*staking.ValidatorWrapper, error) { + return rawdb.ReadValidatorSnapshot(bc.db, addr, epoch) +} + // ReadValidatorSnapshot reads the snapshot staking information of given validator address func (bc *BlockChain) ReadValidatorSnapshot( addr common.Address, @@ -2607,8 +2592,8 @@ func (bc *BlockChain) GetECDSAFromCoinbase(header *block.Header) (common.Address ).WithCause(err) } - committee := shardState.FindCommitteeByID(header.ShardID()) - if committee == nil { + committee, err := shardState.FindCommitteeByID(header.ShardID()) + if err != nil { return common.Address{}, ctxerror.New("cannot find shard in the shard state", "blockNum", header.Number(), "shardID", header.ShardID(), diff --git a/core/chain_makers.go b/core/chain_makers.go index 920f3d67a..14048991a 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -286,15 +286,27 @@ func (cr *fakeChainReader) ReadShardState(epoch *big.Int) (*shard.State, error) func (cr *fakeChainReader) ReadElectedValidatorList() ([]common.Address, error) { return nil, nil } func (cr *fakeChainReader) ReadValidatorList() ([]common.Address, error) { return nil, nil } func (cr *fakeChainReader) ValidatorCandidates() []common.Address { return nil } -func (cr *fakeChainReader) SuperCommitteeForNextEpoch(beacon consensus_engine.ChainReader, header *block.Header, isVerify bool) (*shard.State, error) { +func (cr *fakeChainReader) SuperCommitteeForNextEpoch( + beacon consensus_engine.ChainReader, header *block.Header, isVerify bool, +) (*shard.State, error) { return nil, nil } -func (cr *fakeChainReader) ReadValidatorInformation(addr common.Address) (*staking.ValidatorWrapper, error) { +func (cr *fakeChainReader) ReadValidatorInformation( + addr common.Address, +) (*staking.ValidatorWrapper, error) { return nil, nil } -func (cr *fakeChainReader) ReadValidatorSnapshot(addr common.Address) (*staking.ValidatorWrapper, error) { +func (cr *fakeChainReader) ReadValidatorSnapshot( + addr common.Address, +) (*staking.ValidatorWrapper, error) { return nil, nil } +func (cr *fakeChainReader) ReadValidatorSnapshotAtEpoch( + epoch *big.Int, addr common.Address, +) (*staking.ValidatorWrapper, error) { + return nil, nil +} + func (cr *fakeChainReader) ReadBlockRewardAccumulator(uint64) (*big.Int, error) { return nil, nil } func (cr *fakeChainReader) ValidatorStakingWithDelegation(addr common.Address) *big.Int { return nil } func (cr *fakeChainReader) ReadValidatorStats(addr common.Address) (*staking.ValidatorStats, error) { diff --git a/core/offchain.go b/core/offchain.go index 654696a57..d483e63c2 100644 --- a/core/offchain.go +++ b/core/offchain.go @@ -10,6 +10,7 @@ import ( "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/shard" + "github.com/harmony-one/harmony/staking/slash" "github.com/pkg/errors" ) @@ -147,22 +148,25 @@ func (bc *BlockChain) CommitOffChainData( bc.chainConfig.IsCrossLink(block.Epoch()) && len(header.CrossLinks()) > 0 { crossLinks := &types.CrossLinks{} - err = rlp.DecodeBytes(header.CrossLinks(), crossLinks) - if err != nil { - header.Logger( - utils.Logger()). + if err := rlp.DecodeBytes( + header.CrossLinks(), crossLinks, + ); err != nil { + header.Logger(utils.Logger()). Warn().Err(err). Msg("[insertChain/crosslinks] cannot parse cross links") return NonStatTy, err } if !crossLinks.IsSorted() { - header.Logger(utils.Logger()).Warn(). - Err(err).Msg("[insertChain/crosslinks] cross links are not sorted") + header.Logger(utils.Logger()). + Warn().Err(err). + Msg("[insertChain/crosslinks] cross links are not sorted") return NonStatTy, errors.New("proposed cross links are not sorted") } for _, crossLink := range *crossLinks { // Process crosslink - if err := bc.WriteCrossLinks(batch, types.CrossLinks{crossLink}); err == nil { + if err := bc.WriteCrossLinks( + batch, types.CrossLinks{crossLink}, + ); err == nil { utils.Logger().Info(). Uint64("blockNum", crossLink.BlockNum()). Uint32("shardID", crossLink.ShardID()). @@ -185,6 +189,7 @@ func (bc *BlockChain) CommitOffChainData( utils.Logger(). Debug(). Msgf(msg, len(*crossLinks), num) + utils.Logger().Debug().Msgf(msg, len(*crossLinks), num) } // Roll up latest crosslinks for i := uint32(0); i < shard.Schedule.InstanceForEpoch(epoch).NumShards(); i++ { @@ -198,12 +203,18 @@ func (bc *BlockChain) CommitOffChainData( ); err != nil { return NonStatTy, err } - if err := bc.DeletePendingSlashingCandidates(); err != nil { - return NonStatTy, err + records := slash.Records{} + if s := header.Slashes(); len(s) > 0 { + if err := rlp.DecodeBytes(s, &records); err != nil { + utils.Logger().Debug().Err(err).Msg("could not decode slashes in header") + } + if err := bc.DeleteFromPendingSlashingCandidates(records); err != nil { + utils.Logger().Debug().Err(err).Msg("could not deleting pending slashes") + } } } else { // block reward never accumulate before staking - bc.WriteBlockRewardAccumulator(batch, big.NewInt(0), block.Number().Uint64()) + bc.WriteBlockRewardAccumulator(batch, common.Big0, block.Number().Uint64()) } } diff --git a/core/staking_verifier.go b/core/staking_verifier.go index 2fb96b2b6..b774024bb 100644 --- a/core/staking_verifier.go +++ b/core/staking_verifier.go @@ -4,11 +4,11 @@ import ( "bytes" "math/big" - "github.com/pkg/errors" - + "github.com/ethereum/go-ethereum/common" "github.com/harmony-one/harmony/core/vm" common2 "github.com/harmony-one/harmony/internal/common" staking "github.com/harmony-one/harmony/staking/types" + "github.com/pkg/errors" ) var ( @@ -41,7 +41,9 @@ func VerifyAndCreateValidatorFromMsg( return nil, errNegativeAmount } if stateDB.IsValidator(msg.ValidatorAddress) { - return nil, errors.Wrapf(errValidatorExist, common2.MustAddressToBech32(msg.ValidatorAddress)) + return nil, errors.Wrapf( + errValidatorExist, common2.MustAddressToBech32(msg.ValidatorAddress), + ) } if !CanTransfer(stateDB, msg.ValidatorAddress, msg.Amount) { return nil, errInsufficientBalanceForStake @@ -58,7 +60,7 @@ func VerifyAndCreateValidatorFromMsg( zero := big.NewInt(0) wrapper.Counters.NumBlocksSigned = zero wrapper.Counters.NumBlocksToSign = zero - if err := wrapper.SanityCheck(); err != nil { + if err := wrapper.SanityCheck(staking.DoNotEnforceMaxBLS); err != nil { return nil, err } return wrapper, nil @@ -69,7 +71,8 @@ func VerifyAndCreateValidatorFromMsg( // // Note that this function never updates the stateDB, it only reads from stateDB. func VerifyAndEditValidatorFromMsg( - stateDB vm.StateDB, chainContext ChainContext, blockNum *big.Int, msg *staking.EditValidator, + stateDB vm.StateDB, chainContext ChainContext, + blockNum *big.Int, msg *staking.EditValidator, ) (*staking.ValidatorWrapper, error) { if stateDB == nil { return nil, errStateDBIsMissing @@ -106,11 +109,13 @@ func VerifyAndEditValidatorFromMsg( wrapper.Validator.UpdateHeight = blockNum } - if newRate.Sub(rateAtBeginningOfEpoch).Abs().GT(wrapper.Validator.MaxChangeRate) { + if newRate.Sub(rateAtBeginningOfEpoch).Abs().GT( + wrapper.Validator.MaxChangeRate, + ) { return nil, errCommissionRateChangeTooFast } - if err := wrapper.SanityCheck(); err != nil { + if err := wrapper.SanityCheck(staking.DoNotEnforceMaxBLS); err != nil { return nil, err } return wrapper, nil @@ -153,14 +158,18 @@ func VerifyAndDelegateFromMsg( if delegation.Undelegations[i].Amount.Cmp(delegateBalance) <= 0 { delegateBalance.Sub(delegateBalance, delegation.Undelegations[i].Amount) } else { - delegation.Undelegations[i].Amount.Sub(delegation.Undelegations[i].Amount, delegateBalance) + delegation.Undelegations[i].Amount.Sub( + delegation.Undelegations[i].Amount, delegateBalance, + ) delegateBalance = big.NewInt(0) break } } delegation.Undelegations = delegation.Undelegations[:i+1] delegation.Amount.Add(delegation.Amount, msg.Amount) - if err := wrapper.SanityCheck(); err != nil { + if err := wrapper.SanityCheck( + staking.DoNotEnforceMaxBLS, + ); err != nil { return nil, nil, err } // Return remaining balance to be deducted for delegation @@ -182,8 +191,12 @@ func VerifyAndDelegateFromMsg( if !CanTransfer(stateDB, msg.DelegatorAddress, msg.Amount) { return nil, nil, errInsufficientBalanceForStake } - wrapper.Delegations = append(wrapper.Delegations, staking.NewDelegation(msg.DelegatorAddress, msg.Amount)) - if err := wrapper.SanityCheck(); err != nil { + wrapper.Delegations = append( + wrapper.Delegations, staking.NewDelegation( + msg.DelegatorAddress, msg.Amount, + ), + ) + if err := wrapper.SanityCheck(staking.DoNotEnforceMaxBLS); err != nil { return nil, nil, err } return wrapper, msg.Amount, nil @@ -223,7 +236,9 @@ func VerifyAndUndelegateFromMsg( if err := delegation.Undelegate(epoch, msg.Amount); err != nil { return nil, err } - if err := wrapper.SanityCheck(); err != nil { + if err := wrapper.SanityCheck( + staking.DoNotEnforceMaxBLS, + ); err != nil { return nil, err } return wrapper, nil @@ -253,12 +268,14 @@ func VerifyAndCollectRewardsFromDelegation( } if uint64(len(wrapper.Delegations)) > delegation.Index { delegation := &wrapper.Delegations[delegation.Index] - if delegation.Reward.Cmp(big.NewInt(0)) > 0 { + if delegation.Reward.Cmp(common.Big0) > 0 { totalRewards.Add(totalRewards, delegation.Reward) } delegation.Reward.SetUint64(0) } - if err := wrapper.SanityCheck(); err != nil { + if err := wrapper.SanityCheck( + staking.DoNotEnforceMaxBLS, + ); err != nil { return nil, nil, err } updatedValidatorWrappers = append(updatedValidatorWrappers, wrapper) diff --git a/core/state/statedb.go b/core/state/statedb.go index a67d58ad2..d2e6f283d 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -708,12 +708,14 @@ func (db *DB) ValidatorWrapper( return &val, nil } +const doNotEnforceMaxBLS = -1 + // UpdateValidatorWrapper updates staking information of // a given validator (including delegation info) func (db *DB) UpdateValidatorWrapper( addr common.Address, val *stk.ValidatorWrapper, ) error { - if err := val.SanityCheck(); err != nil { + if err := val.SanityCheck(doNotEnforceMaxBLS); err != nil { return err } diff --git a/hmy/api_backend.go b/hmy/api_backend.go index ff9fba376..d0fb337dc 100644 --- a/hmy/api_backend.go +++ b/hmy/api_backend.go @@ -333,31 +333,33 @@ func (b *APIBackend) GetValidatorInformation( s, _ := internal_common.AddressToBech32(addr) return nil, errors.Wrapf(err, "not found address in current state %s", s) } - snapshot, err := b.hmy.BlockChain().ReadValidatorSnapshot(addr) + snapshot, err := b.hmy.BlockChain().ReadValidatorSnapshotAtEpoch( + b.hmy.BlockChain().CurrentHeader().Epoch(), + addr, + ) + defaultReply := &staking.ValidatorRPCEnchanced{ + Wrapper: *wrapper, + CurrentSigningPercentage: staking.Computed{ + common.Big0, common.Big0, numeric.ZeroDec(), + }, + CurrentVotingPower: []staking.VotePerShard{}, + } if err != nil { - return &staking.ValidatorRPCEnchanced{ - Wrapper: *wrapper, - CurrentSigningPercentage: staking.Computed{ - common.Big0, common.Big0, numeric.ZeroDec(), - }, - CurrentVotingPower: []staking.VotePerShard{}, - }, nil + return defaultReply, nil } signed, toSign, quotient, err := availability.ComputeCurrentSigning(snapshot, wrapper) if err != nil { - return nil, err + return defaultReply, nil } stats, err := b.hmy.BlockChain().ReadValidatorStats(addr) if err != nil { - return nil, err + return defaultReply, nil } - return &staking.ValidatorRPCEnchanced{ - Wrapper: *wrapper, - CurrentSigningPercentage: staking.Computed{signed, toSign, quotient}, - CurrentVotingPower: stats.VotingPowerPerShard, - }, nil + defaultReply.CurrentSigningPercentage = staking.Computed{signed, toSign, quotient} + defaultReply.CurrentVotingPower = stats.VotingPowerPerShard + return defaultReply, nil } // GetMedianRawStakeSnapshot .. @@ -372,10 +374,12 @@ func (b *APIBackend) GetMedianRawStakeSnapshot() (*big.Int, error) { if err != nil { return nil, err } - if !effective.IsEligibleForEPOSAuction(validator) { + if validator.EPOSStatus != effective.Active { continue } - if err := validator.SanityCheck(); err != nil { + if err := validator.SanityCheck( + staking.DoNotEnforceMaxBLS, + ); err != nil { continue } @@ -430,7 +434,7 @@ func (b *APIBackend) GetTotalStakingSnapshot() *big.Int { stakes := big.NewInt(0) for i := range candidates { validator, _ := b.hmy.BlockChain().ReadValidatorInformation(candidates[i]) - if !effective.IsEligibleForEPOSAuction(validator) { + if validator.EPOSStatus != effective.Active { continue } for i := range validator.Delegations { @@ -558,9 +562,6 @@ func (b *APIBackend) GetSuperCommittees() (*quorum.Transition, error) { for _, comm := range prevCommittee.Shards { decider := quorum.NewDecider(quorum.SuperMajorityStake) shardID := comm.ShardID - decider.SetShardIDProvider(func() (uint32, error) { - return shardID, nil - }) decider.SetVoters(comm.Slots) then.Deciders[shardID] = decider } @@ -568,9 +569,6 @@ func (b *APIBackend) GetSuperCommittees() (*quorum.Transition, error) { for _, comm := range nowCommittee.Shards { decider := quorum.NewDecider(quorum.SuperMajorityStake) shardID := comm.ShardID - decider.SetShardIDProvider(func() (uint32, error) { - return shardID, nil - }) decider.SetVoters(comm.Slots) now.Deciders[shardID] = decider } diff --git a/internal/chain/engine.go b/internal/chain/engine.go index 7901efab3..472ed97b9 100644 --- a/internal/chain/engine.go +++ b/internal/chain/engine.go @@ -209,13 +209,14 @@ func (e *engineImpl) VerifySeal(chain engine.ChainReader, header *block.Header) return errors.Wrapf(err, "cannot decoded shard state") } d := quorum.NewDecider(quorum.SuperMajorityStake) - d.SetShardIDProvider(func() (uint32, error) { - return parentHeader.ShardID(), nil - }) d.SetMyPublicKeyProvider(func() (*multibls.PublicKey, error) { return nil, nil }) - d.SetVoters(slotList.FindCommitteeByID(parentHeader.ShardID()).Slots) + subComm, err := slotList.FindCommitteeByID(parentHeader.ShardID()) + if err != nil { + return err + } + d.SetVoters(subComm.Slots) if !d.IsQuorumAchievedByMask(mask) { return ctxerror.New( "[VerifySeal] Not enough voting power in LastCommitSignature from Block Header", @@ -293,8 +294,11 @@ func (e *engineImpl) Finalize( } // Withdraw unlocked tokens to the delegators' accounts -func payoutUndelegations(chain engine.ChainReader, header *block.Header, state *state.DB) error { +func payoutUndelegations( + chain engine.ChainReader, header *block.Header, state *state.DB, +) error { validators, err := chain.ReadValidatorList() + countTrack := map[common.Address]int{} if err != nil { const msg = "[Finalize] failed to read all validators" return ctxerror.New(msg).WithCause(err) @@ -314,6 +318,7 @@ func payoutUndelegations(chain engine.ChainReader, header *block.Header, state * ) state.AddBalance(delegation.DelegatorAddress, totalWithdraw) } + countTrack[validator] = len(wrapper.Delegations) if err := state.UpdateValidatorWrapper( validator, wrapper, ); err != nil { @@ -321,6 +326,13 @@ func payoutUndelegations(chain engine.ChainReader, header *block.Header, state * return ctxerror.New(msg).WithCause(err) } } + + utils.Logger().Info(). + Uint64("epoch", header.Epoch().Uint64()). + Uint64("block-number", header.Number().Uint64()). + Interface("count-track", countTrack). + Msg("paid out delegations") + return nil } @@ -400,12 +412,11 @@ func QuorumForBlock( } } - c := ss.FindCommitteeByID(h.ShardID()) - if c == nil { - return 0, errors.Errorf( - "cannot find shard %d in shard state", h.ShardID()) + subComm, err := ss.FindCommitteeByID(h.ShardID()) + if err != nil { + return 0, errors.Errorf("cannot find shard %d in shard state", h.ShardID()) } - return (len(c.Slots))*2/3 + 1, nil + return (len(subComm.Slots))*2/3 + 1, nil } // Similiar to VerifyHeader, which is only for verifying the block headers of one's own chain, this verification @@ -435,13 +446,14 @@ func (e *engineImpl) VerifyHeaderWithSignature(chain engine.ChainReader, header return errors.Wrapf(err, "cannot read shard state") } d := quorum.NewDecider(quorum.SuperMajorityStake) - d.SetShardIDProvider(func() (uint32, error) { - return header.ShardID(), nil - }) d.SetMyPublicKeyProvider(func() (*multibls.PublicKey, error) { return nil, nil }) - d.SetVoters(slotList.FindCommitteeByID(header.ShardID()).Slots) + subComm, err := slotList.FindCommitteeByID(header.ShardID()) + if err != nil { + return err + } + d.SetVoters(subComm.Slots) if !d.IsQuorumAchievedByMask(mask) { return ctxerror.New( "[VerifySeal] Not enough voting power in commitSignature from Block Header", @@ -484,21 +496,12 @@ func GetPublicKeys( } } - committee := shardState.FindCommitteeByID(header.ShardID()) - if committee == nil { + subCommittee, err := shardState.FindCommitteeByID(header.ShardID()) + if err != nil { return nil, ctxerror.New("cannot find shard in the shard state", "blockNumber", header.Number(), "shardID", header.ShardID(), ) } - committerKeys := []*bls.PublicKey{} - for _, member := range committee.Slots { - committerKey := new(bls.PublicKey) - if err := member.BlsPublicKey.ToLibBLSPublicKey(committerKey); err != nil { - return nil, ctxerror.New("cannot convert BLS public key", - "blsPublicKey", member.BlsPublicKey).WithCause(err) - } - committerKeys = append(committerKeys, committerKey) - } - return committerKeys, nil + return subCommittee.BLSPublicKeys() } diff --git a/internal/chain/reward.go b/internal/chain/reward.go index dc53686ac..4ab514b41 100644 --- a/internal/chain/reward.go +++ b/internal/chain/reward.go @@ -140,11 +140,13 @@ func AccumulateRewards( return network.NoReward, err } - subComm := shardState.FindCommitteeByID(cxLink.ShardID()) + subComm, err := shardState.FindCommitteeByID(cxLink.ShardID()) + if err != nil { + return network.NoReward, err + } payableSigners, missing, err := availability.BlockSigners( cxLink.Bitmap(), subComm, ) - if err != nil { return network.NoReward, err } diff --git a/internal/hmyapi/apiv1/transactionpool.go b/internal/hmyapi/apiv1/transactionpool.go index 342ca1d65..6a3c019a4 100644 --- a/internal/hmyapi/apiv1/transactionpool.go +++ b/internal/hmyapi/apiv1/transactionpool.go @@ -219,9 +219,10 @@ func (s *PublicTransactionPoolAPI) SendRawStakingTransaction( return common.Hash{}, err } c := s.b.ChainConfig().ChainID - if tx.ChainID().Cmp(c) != 0 { - e := errors.Wrapf(errInvalidChainID, "current chain id:%s", c.String()) - return common.Hash{}, e + if id := tx.ChainID(); id.Cmp(c) != 0 { + return common.Hash{}, errors.Wrapf( + errInvalidChainID, "blockchain chain id:%s, given %s", c.String(), id.String(), + ) } return SubmitStakingTransaction(ctx, s.b, tx) } @@ -238,9 +239,10 @@ func (s *PublicTransactionPoolAPI) SendRawTransaction(ctx context.Context, encod return common.Hash{}, err } c := s.b.ChainConfig().ChainID - if tx.ChainID().Cmp(c) != 0 { - e := errors.Wrapf(errInvalidChainID, "current chain id:%s", c.String()) - return common.Hash{}, e + if id := tx.ChainID(); id.Cmp(c) != 0 { + return common.Hash{}, errors.Wrapf( + errInvalidChainID, "blockchain chain id:%s, given %s", c.String(), id.String(), + ) } return SubmitTransaction(ctx, s.b, tx) } diff --git a/internal/hmyapi/apiv2/transactionpool.go b/internal/hmyapi/apiv2/transactionpool.go index 3e6e1ab46..ccfb7474c 100644 --- a/internal/hmyapi/apiv2/transactionpool.go +++ b/internal/hmyapi/apiv2/transactionpool.go @@ -217,9 +217,10 @@ func (s *PublicTransactionPoolAPI) SendRawStakingTransaction( return common.Hash{}, err } c := s.b.ChainConfig().ChainID - if tx.ChainID().Cmp(c) != 0 { - e := errors.Wrapf(errInvalidChainID, "current chain id:%s", c.String()) - return common.Hash{}, e + if id := tx.ChainID(); id.Cmp(c) != 0 { + return common.Hash{}, errors.Wrapf( + errInvalidChainID, "blockchain chain id:%s, given %s", c.String(), id.String(), + ) } return SubmitStakingTransaction(ctx, s.b, tx) } @@ -236,9 +237,10 @@ func (s *PublicTransactionPoolAPI) SendRawTransaction(ctx context.Context, encod return common.Hash{}, err } c := s.b.ChainConfig().ChainID - if tx.ChainID().Cmp(c) != 0 { - e := errors.Wrapf(errInvalidChainID, "current chain id:%s", c.String()) - return common.Hash{}, e + if id := tx.ChainID(); id.Cmp(c) != 0 { + return common.Hash{}, errors.Wrapf( + errInvalidChainID, "blockchain chain id:%s, given %s", c.String(), id.String(), + ) } return SubmitTransaction(ctx, s.b, tx) } diff --git a/node/double_signing.go b/node/double_signing.go index 02e193ee7..a8c40eda8 100644 --- a/node/double_signing.go +++ b/node/double_signing.go @@ -12,20 +12,18 @@ func (node *Node) processSlashCandidateMessage(msgPayload []byte) { if node.NodeConfig.ShardID != shard.BeaconChainShardID { return } - candidates, e := slash.Records{}, utils.Logger().Error() + candidates := slash.Records{} if err := rlp.DecodeBytes(msgPayload, &candidates); err != nil { - e.Err(err). - Msg("unable to decode slash candidate message") + utils.Logger().Error(). + Err(err).Msg("unable to decode slash candidates message") return } - if err := candidates.SanityCheck(); err != nil { - e.Err(err). - RawJSON("slash-candidates", []byte(candidates.String())). - Msg("sanity check failed on incoming candidates") - return + if err := node.Blockchain().AddPendingSlashingCandidates( + candidates, + ); err != nil { + utils.Logger().Error(). + Err(err).Msg("unable to add slash candidates to pending ") } - - node.Blockchain().AddPendingSlashingCandidates(candidates) } diff --git a/node/node.go b/node/node.go index 1b336ce5e..0503a3552 100644 --- a/node/node.go +++ b/node/node.go @@ -457,8 +457,13 @@ func (node *Node) GetSyncID() [SyncIDLength]byte { } // New creates a new node. -func New(host p2p.Host, consensusObj *consensus.Consensus, - chainDBFactory shardchain.DBFactory, blacklist map[common.Address]struct{}, isArchival bool) *Node { +func New( + host p2p.Host, + consensusObj *consensus.Consensus, + chainDBFactory shardchain.DBFactory, + blacklist map[common.Address]struct{}, + isArchival bool, +) *Node { node := Node{} const sinkSize = 4096 node.errorSink = struct { @@ -542,7 +547,7 @@ func New(host p2p.Host, consensusObj *consensus.Consensus, ) } - node.pendingCXReceipts = make(map[string]*types.CXReceiptsProof) + node.pendingCXReceipts = map[string]*types.CXReceiptsProof{} node.Consensus.VerifiedNewBlock = make(chan *types.Block) chain.Engine.SetRewarder(node.Consensus.Decider.(reward.Distributor)) chain.Engine.SetBeaconchain(beaconChain) @@ -577,23 +582,22 @@ func New(host p2p.Host, consensusObj *consensus.Consensus, node.globalRxQueue = msgq.New(GlobalRxQueueSize) // Setup initial state of syncing. - node.peerRegistrationRecord = make(map[string]*syncConfig) + node.peerRegistrationRecord = map[string]*syncConfig{} node.startConsensus = make(chan struct{}) go node.bootstrapConsensus() // Broadcast double-signers reported by consensus if node.Consensus != nil { go func() { - for { select { case doubleSign := <-node.Consensus.SlashChan: - l := utils.Logger().Info().RawJSON("double-sign", []byte(doubleSign.String())) - + utils.Logger().Info(). + RawJSON("double-sign-candidate", []byte(doubleSign.String())). + Msg("double sign notified by consensus leader") // no point to broadcast the slash if we aren't even in the right epoch yet if !node.Blockchain().Config().IsStaking( node.Blockchain().CurrentHeader().Epoch(), ) { - l.Msg("double sign occured before staking era, no-op") return } if hooks := node.NodeConfig.WebHooks.Hooks; hooks != nil { @@ -604,11 +608,13 @@ func New(host p2p.Host, consensusObj *consensus.Consensus, } if node.NodeConfig.ShardID != shard.BeaconChainShardID { go node.BroadcastSlash(&doubleSign) - l.Msg("broadcast the double sign record") } else { records := slash.Records{doubleSign} - node.Blockchain().AddPendingSlashingCandidates(records) - l.Msg("added double sign record to off-chain pending") + if err := node.Blockchain().AddPendingSlashingCandidates( + records, + ); err != nil { + utils.Logger().Err(err).Msg("could not add new slash to ending slashes") + } } } } @@ -622,8 +628,11 @@ func New(host p2p.Host, consensusObj *consensus.Consensus, // keys for consensus and drand func (node *Node) InitConsensusWithValidators() (err error) { if node.Consensus == nil { - utils.Logger().Error().Msg("[InitConsensusWithValidators] consenus is nil; Cannot figure out shardID") - return ctxerror.New("[InitConsensusWithValidators] consenus is nil; Cannot figure out shardID") + utils.Logger().Error(). + Msg("[InitConsensusWithValidators] consenus is nil; Cannot figure out shardID") + return ctxerror.New( + "[InitConsensusWithValidators] consenus is nil; Cannot figure out shardID", + ) } shardID := node.Consensus.ShardID blockNum := node.Blockchain().CurrentBlock().NumberU64() @@ -645,9 +654,11 @@ func (node *Node) InitConsensusWithValidators() (err error) { Msg("[InitConsensusWithValidators] Failed getting shard state") return err } - pubKeys, err := committee.WithStakingEnabled.GetCommitteePublicKeys( - shardState.FindCommitteeByID(shardID), - ) + subComm, err := shardState.FindCommitteeByID(shardID) + if err != nil { + return err + } + pubKeys, err := committee.WithStakingEnabled.GetCommitteePublicKeys(subComm) if err != nil { utils.Logger().Error(). Uint32("shardID", shardID). @@ -670,7 +681,8 @@ func (node *Node) InitConsensusWithValidators() (err error) { return nil } } - // TODO: Disable drand. Currently drand isn't functioning but we want to compeletely turn it off for full protection. + // TODO: Disable drand. Currently drand isn't + // functioning but we want to compeletely turn it off for full protection. // node.DRand.UpdatePublicKeys(pubKeys) return nil } @@ -711,13 +723,14 @@ func (node *Node) initNodeConfiguration() (service.NodeConfig, chan p2p.Peer) { PushgatewayIP: node.NodeConfig.GetPushgatewayIP(), PushgatewayPort: node.NodeConfig.GetPushgatewayPort(), IsClient: node.NodeConfig.IsClient(), - Beacon: nodeconfig.NewGroupIDByShardID(0), + Beacon: nodeconfig.NewGroupIDByShardID(shard.BeaconChainShardID), ShardGroupID: node.NodeConfig.GetShardGroupID(), Actions: make(map[nodeconfig.GroupID]nodeconfig.ActionType), } if nodeConfig.IsClient { - nodeConfig.Actions[nodeconfig.NewClientGroupIDByShardID(0)] = nodeconfig.ActionStart + nodeConfig.Actions[nodeconfig.NewClientGroupIDByShardID(shard.BeaconChainShardID)] = + nodeconfig.ActionStart } else { nodeConfig.Actions[node.NodeConfig.GetShardGroupID()] = nodeconfig.ActionStart } @@ -728,7 +741,9 @@ func (node *Node) initNodeConfiguration() (service.NodeConfig, chan p2p.Peer) { utils.Logger().Error().Err(err).Msg("Failed to create shard receiver") } - node.globalGroupReceiver, err = node.host.GroupReceiver(nodeconfig.NewClientGroupIDByShardID(0)) + node.globalGroupReceiver, err = node.host.GroupReceiver( + nodeconfig.NewClientGroupIDByShardID(shard.BeaconChainShardID), + ) if err != nil { utils.Logger().Error().Err(err).Msg("Failed to create global receiver") } diff --git a/node/node_cross_link.go b/node/node_cross_link.go index 36016ea64..29f03c70f 100644 --- a/node/node_cross_link.go +++ b/node/node_cross_link.go @@ -1,17 +1,13 @@ package node import ( - "encoding/binary" - "github.com/ethereum/go-ethereum/rlp" "github.com/harmony-one/bls/ffi/go/bls" - "github.com/harmony-one/harmony/consensus/quorum" "github.com/harmony-one/harmony/core/types" - bls_cosi "github.com/harmony-one/harmony/crypto/bls" "github.com/harmony-one/harmony/internal/ctxerror" "github.com/harmony-one/harmony/internal/utils" - "github.com/harmony-one/harmony/multibls" "github.com/harmony-one/harmony/shard" + "github.com/harmony-one/harmony/staking/verify" ) const ( @@ -71,9 +67,8 @@ func (node *Node) ProcessCrossLinkMessage(msgPayload []byte) { return } - var crosslinks []types.CrossLink - err = rlp.DecodeBytes(msgPayload, &crosslinks) - if err != nil { + crosslinks := []types.CrossLink{} + if err := rlp.DecodeBytes(msgPayload, &crosslinks); err != nil { utils.Logger().Error(). Err(err). Msg("[ProcessingCrossLink] Crosslink Message Broadcast Unable to Decode") @@ -125,68 +120,35 @@ func (node *Node) VerifyCrossLink(cl types.CrossLink) error { } if !node.Blockchain().Config().IsCrossLink(cl.Epoch()) { - return ctxerror.New("[VerifyCrossLink] CrossLink Epoch should >= cross link starting epoch", "crossLinkEpoch", cl.Epoch(), "cross_link_starting_eoch", node.Blockchain().Config().CrossLinkEpoch) + return ctxerror.New( + "[VerifyCrossLink] CrossLink Epoch should >= cross link starting epoch", + "crossLinkEpoch", cl.Epoch(), "cross_link_starting_eoch", + node.Blockchain().Config().CrossLinkEpoch, + ) } // Verify signature of the new cross link header // TODO: check whether to recalculate shard state shardState, err := node.Blockchain().ReadShardState(cl.Epoch()) - committee := shardState.FindCommitteeByID(cl.ShardID()) - - if err != nil || committee == nil { - return ctxerror.New("[VerifyCrossLink] Failed to read shard state for cross link", "beaconEpoch", node.Blockchain().CurrentHeader().Epoch(), "epoch", cl.Epoch(), "shardID", cl.ShardID(), "blockNum", cl.BlockNum()).WithCause(err) - } - var committerKeys []*bls.PublicKey - - parseKeysSuccess := true - for _, member := range committee.Slots { - committerKey := new(bls.PublicKey) - err = member.BlsPublicKey.ToLibBLSPublicKey(committerKey) - if err != nil { - parseKeysSuccess = false - break - } - committerKeys = append(committerKeys, committerKey) - } - if !parseKeysSuccess { - return ctxerror.New("[VerifyCrossLink] cannot convert BLS public key", "shardID", cl.ShardID(), "blockNum", cl.BlockNum()).WithCause(err) - } - - mask, err := bls_cosi.NewMask(committerKeys, nil) if err != nil { - return ctxerror.New("[VerifyCrossLink] cannot create group sig mask", "shardID", cl.ShardID(), "blockNum", cl.BlockNum()).WithCause(err) - } - if err := mask.SetMask(cl.Bitmap()); err != nil { - return ctxerror.New("[VerifyCrossLink] cannot set group sig mask bits", "shardID", cl.ShardID(), "blockNum", cl.BlockNum()).WithCause(err) + return err } - decider := quorum.NewDecider(quorum.SuperMajorityStake) - decider.SetShardIDProvider(func() (uint32, error) { - return cl.ShardID(), nil - }) - decider.SetMyPublicKeyProvider(func() (*multibls.PublicKey, error) { - return nil, nil - }) - if _, err := decider.SetVoters(committee.Slots); err != nil { - return ctxerror.New("[VerifyCrossLink] Cannot SetVoters for committee", "shardID", cl.ShardID()) - } - if !decider.IsQuorumAchievedByMask(mask) { - return ctxerror.New("[VerifyCrossLink] Not enough voting power for crosslink", "shardID", cl.ShardID()) - } + committee, err := shardState.FindCommitteeByID(cl.ShardID()) - aggSig := bls.Sign{} - sig := cl.Signature() - err = aggSig.Deserialize(sig[:]) if err != nil { - return ctxerror.New("[VerifyCrossLink] unable to deserialize multi-signature from payload").WithCause(err) + return err } - hash := cl.Hash() - blockNumBytes := make([]byte, 8) - binary.LittleEndian.PutUint64(blockNumBytes, cl.BlockNum()) - commitPayload := append(blockNumBytes, hash[:]...) - if !aggSig.VerifyHash(mask.AggregatePublic, commitPayload) { - return ctxerror.New("[VerifyCrossLink] Failed to verify the signature for cross link", "shardID", cl.ShardID(), "blockNum", cl.BlockNum()) + aggSig := &bls.Sign{} + sig := cl.Signature() + if err = aggSig.Deserialize(sig[:]); err != nil { + return ctxerror.New( + "[VerifyCrossLink] unable to deserialize multi-signature from payload", + ).WithCause(err) } - return nil + + return verify.AggregateSigForCommittee( + committee, aggSig, cl.Hash(), cl.BlockNum(), cl.Bitmap(), + ) } diff --git a/node/node_genesis.go b/node/node_genesis.go index 6fd918393..67946bcd4 100644 --- a/node/node_genesis.go +++ b/node/node_genesis.go @@ -51,14 +51,13 @@ func (gi *genesisInitializer) InitChainDB(db ethdb.Database, shardID uint32) err if shardState == nil { return errors.New("failed to create genesis shard state") } - if shardID != shard.BeaconChainShardID { // store only the local shard for shard chains - c := shardState.FindCommitteeByID(shardID) - if c == nil { + subComm, err := shardState.FindCommitteeByID(shardID) + if err != nil { return errors.New("cannot find local shard in genesis") } - shardState = &shard.State{nil, []shard.Committee{*c}} + shardState = &shard.State{nil, []shard.Committee{*subComm}} } gi.node.SetupGenesisBlock(db, shardID, shardState) return nil diff --git a/node/node_handler.go b/node/node_handler.go index 5a646a5d2..f3ef46253 100644 --- a/node/node_handler.go +++ b/node/node_handler.go @@ -244,8 +244,13 @@ func (node *Node) stakingMessageHandler(msgPayload []byte) { // TODO (lc): broadcast the new blocks to new nodes doing state sync func (node *Node) BroadcastNewBlock(newBlock *types.Block) { groups := []nodeconfig.GroupID{node.NodeConfig.GetClientGroupID()} - utils.Logger().Info().Msgf("broadcasting new block %d, group %s", newBlock.NumberU64(), groups[0]) - msg := host.ConstructP2pMessage(byte(0), proto_node.ConstructBlocksSyncMessage([]*types.Block{newBlock})) + utils.Logger().Info(). + Msgf( + "broadcasting new block %d, group %s", newBlock.NumberU64(), groups[0], + ) + msg := host.ConstructP2pMessage(byte(0), + proto_node.ConstructBlocksSyncMessage([]*types.Block{newBlock}), + ) if err := node.host.SendMessageToGroups(groups, msg); err != nil { utils.Logger().Warn().Err(err).Msg("cannot broadcast new block") } @@ -263,9 +268,11 @@ func (node *Node) BroadcastSlash(witness *slash.Record) { RawJSON("record", []byte(witness.String())). Msg("could not send slash record to beaconchain") } + utils.Logger().Info().Msg("broadcast the double sign record") } -// BroadcastCrossLink is called by consensus leader to send the new header as cross link to beacon chain. +// BroadcastCrossLink is called by consensus leader to +// send the new header as cross link to beacon chain. func (node *Node) BroadcastCrossLink(newBlock *types.Block) { // no point to broadcast the crosslink if we aren't even in the right epoch yet if !node.Blockchain().Config().IsCrossLink( diff --git a/node/node_newblock.go b/node/node_newblock.go index 5a7728248..13eea4238 100644 --- a/node/node_newblock.go +++ b/node/node_newblock.go @@ -182,7 +182,7 @@ func (node *Node) proposeNewBlock() (*types.Block, error) { AnErr("[proposeNewBlock] pending crosslink is already committed onchain", err) continue } - if err = node.VerifyCrossLink(pending); err != nil { + if err := node.VerifyCrossLink(pending); err != nil { invalidToDelete = append(invalidToDelete, pending) utils.Logger().Debug(). AnErr("[proposeNewBlock] pending crosslink verification failed", err) @@ -204,8 +204,8 @@ func (node *Node) proposeNewBlock() (*types.Block, error) { } if isBeaconchainInStakingEra { - // this one will set a meaningful w.current.slashes - if err := node.Worker.CollectAndVerifySlashes(); err != nil { + // this will set a meaningful w.current.slashes + if err := node.Worker.CollectVerifiedSlashes(); err != nil { return nil, err } } diff --git a/node/worker/worker.go b/node/worker/worker.go index 9d00a9fec..d3fc58325 100644 --- a/node/worker/worker.go +++ b/node/worker/worker.go @@ -327,32 +327,28 @@ func (w *Worker) IncomingReceipts() []*types.CXReceiptsProof { return w.current.incxs } -// CollectAndVerifySlashes .. -func (w *Worker) CollectAndVerifySlashes() error { - allSlashing, err := w.chain.ReadPendingSlashingCandidates() - if err != nil { - return err +// CollectVerifiedSlashes sets w.current.slashes only to those that +// past verification +func (w *Worker) CollectVerifiedSlashes() error { + pending, failures := + w.chain.ReadPendingSlashingCandidates(), slash.Records{} + if d := pending; len(d) > 0 { + pending, failures = w.verifySlashes(d) } - if d := allSlashing; len(d) > 0 { - // TODO add specific error which is - // "could not verify slash", which should not return as err - // and therefore stop the block proposal - if allSlashing, err = w.VerifyAll(d); err != nil { - // TODO(audit): very slashes individually; do not return err if verify fails - utils.Logger().Err(err). - RawJSON("slashes", []byte(d.String())). - Msg("could not verify slashes proposed") + if f := failures; len(f) > 0 { + if err := w.chain.DeleteFromPendingSlashingCandidates(f); err != nil { return err } } - w.current.slashes = allSlashing + w.current.slashes = pending return nil } -// VerifyAll .. -func (w *Worker) VerifyAll(allSlashing []slash.Record) ([]slash.Record, error) { - d := allSlashing - slashingToPropose := []slash.Record{} +// returns (successes, failures, error) +func (w *Worker) verifySlashes( + d slash.Records, +) (slash.Records, slash.Records) { + successes, failures := slash.Records{}, slash.Records{} // Enforce order, reproducibility sort.SliceStable(d, func(i, j int) bool { @@ -362,16 +358,24 @@ func (w *Worker) VerifyAll(allSlashing []slash.Record) ([]slash.Record, error) { }, ) + workingState := w.GetCurrentState() + for i := range d { - if err := slash.Verify(w.chain, &d[i]); err != nil { - return nil, err + if err := slash.Verify( + w.chain, workingState, &d[i], + ); err != nil { + failures = append(failures, d[i]) } - slashingToPropose = append(slashingToPropose, d[i]) + successes = append(successes, d[i]) } - count := len(slashingToPropose) - utils.Logger().Info(). - Msgf("set into propose headers %d slashing record", count) - return slashingToPropose, nil + + if f := len(failures); f > 0 { + utils.Logger().Debug(). + Int("count", f). + Msg("invalid slash records passed over in block proposal") + } + + return successes, failures } // FinalizeNewBlock generate a new block for the next consensus round. @@ -415,7 +419,6 @@ func (w *Worker) FinalizeNewBlock( if data, err := rlp.EncodeToBytes(doubleSigners); err == nil { w.current.header.SetSlashes(data) utils.Logger().Info(). - RawJSON("slashes", []byte(doubleSigners.String())). Msg("encoded slashes into headers of proposed new block") } else { utils.Logger().Debug().Err(err).Msg("Failed to encode proposed slashes") diff --git a/shard/committee/assignment.go b/shard/committee/assignment.go index 82436857a..811a58c13 100644 --- a/shard/committee/assignment.go +++ b/shard/committee/assignment.go @@ -159,20 +159,22 @@ func eposStakedCommittee( return shardState, nil } + maxBLSKey := stakedSlotsCount / 3 + // TODO benchmark difference if went with data structure that sorts on insert for i := range candidates { validator, err := stakerReader.ReadValidatorInformation(candidates[i]) if err != nil { return nil, err } - if !effective.IsEligibleForEPOSAuction(validator) { + + if validator.EPOSStatus != effective.Active { continue } - if err := validator.SanityCheck(); err != nil { + if err := validator.SanityCheck(maxBLSKey); err != nil { utils.Logger().Info(). Int("staked-candidates", len(candidates)). Err(err). - RawJSON("candidate", []byte(validator.String())). Msg("validator sanity check failed") continue } @@ -230,7 +232,6 @@ func eposStakedCommittee( utils.Logger().Info(). Int("staked-candidates", c). Str("total-staked-by-validators", totalStake.String()). - RawJSON("staked-super-committee", []byte(shardState.String())). Msg("epos based super-committe") } diff --git a/shard/shard_state.go b/shard/shard_state.go index 98a0c320c..29dfb5178 100644 --- a/shard/shard_state.go +++ b/shard/shard_state.go @@ -19,6 +19,8 @@ import ( var ( emptyBlsPubKey = BlsPublicKey{} + // ErrShardIDNotInSuperCommittee .. + ErrShardIDNotInSuperCommittee = errors.New("shardID not in super committee") ) // PublicKeySizeInBytes .. @@ -257,16 +259,16 @@ func (ss *State) MarshalJSON() ([]byte, error) { // FindCommitteeByID returns the committee configuration for the given shard, // or nil if the given shard is not found. -func (ss *State) FindCommitteeByID(shardID uint32) *Committee { +func (ss *State) FindCommitteeByID(shardID uint32) (*Committee, error) { if ss == nil { - return nil + return nil, ErrShardIDNotInSuperCommittee } for committee := range ss.Shards { if ss.Shards[committee].ShardID == shardID { - return &ss.Shards[committee] + return &ss.Shards[committee], nil } } - return nil + return nil, ErrShardIDNotInSuperCommittee } // DeepCopy returns a deep copy of the receiver. @@ -382,14 +384,20 @@ func (c *Committee) DeepCopy() Committee { } // BLSPublicKeys .. -func (c *Committee) BLSPublicKeys() ([]BlsPublicKey, error) { +func (c *Committee) BLSPublicKeys() ([]*bls.PublicKey, error) { if c == nil { return nil, ErrCommitteeNil } - slice := make([]BlsPublicKey, len(c.Slots)) + slice := make([]*bls.PublicKey, len(c.Slots)) for j := range c.Slots { - slice[j] = c.Slots[j].BlsPublicKey + committerKey := &bls.PublicKey{} + if err := c.Slots[j].BlsPublicKey.ToLibBLSPublicKey( + committerKey, + ); err != nil { + return nil, err + } + slice[j] = committerKey } return slice, nil } diff --git a/staking/availability/measure.go b/staking/availability/measure.go index c3d06b4ee..51af83f7b 100644 --- a/staking/availability/measure.go +++ b/staking/availability/measure.go @@ -4,7 +4,6 @@ import ( "math/big" "github.com/ethereum/go-ethereum/common" - "github.com/harmony-one/bls/ffi/go/bls" "github.com/harmony-one/harmony/block" engine "github.com/harmony-one/harmony/consensus/engine" "github.com/harmony-one/harmony/core/state" @@ -13,6 +12,7 @@ import ( "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/numeric" "github.com/harmony-one/harmony/shard" + "github.com/harmony-one/harmony/staking/effective" staking "github.com/harmony-one/harmony/staking/types" "github.com/pkg/errors" ) @@ -31,20 +31,14 @@ var ( func BlockSigners( bitmap []byte, parentCommittee *shard.Committee, ) (shard.SlotList, shard.SlotList, error) { - committerKeys := []*bls.PublicKey{} + committerKeys, err := parentCommittee.BLSPublicKeys() - for _, member := range parentCommittee.Slots { - committerKey := new(bls.PublicKey) - err := member.BlsPublicKey.ToLibBLSPublicKey(committerKey) - if err != nil { - return nil, nil, ctxerror.New( - "cannot convert BLS public key", - "blsPublicKey", - member.BlsPublicKey, - ).WithCause(err) - } - committerKeys = append(committerKeys, committerKey) + if err != nil { + return nil, nil, ctxerror.New( + "cannot convert a BLS public key", + ).WithCause(err) } + mask, err := bls2.NewMask(committerKeys, nil) if err != nil { return nil, nil, ctxerror.New( @@ -95,9 +89,10 @@ func BallotResult( "cannot read shard state", "epoch", parentHeader.Epoch(), ).WithCause(err) } - parentCommittee := parentShardState.FindCommitteeByID(shardID) - if parentCommittee == nil { + parentCommittee, err := parentShardState.FindCommitteeByID(shardID) + + if err != nil { return nil, nil, nil, ctxerror.New( "cannot find shard in the shard state", "parentBlockNumber", parentHeader.Number(), @@ -192,8 +187,6 @@ func ComputeCurrentSigning( if toSign.Cmp(common.Big0) == 0 { utils.Logger().Info(). - RawJSON("snapshot", []byte(snapshot.String())). - RawJSON("current", []byte(wrapper.String())). Msg("toSign is 0, perhaps did not receive crosslink proving signing") return signed, toSign, numeric.ZeroDec(), nil } @@ -248,8 +241,6 @@ func compute( } utils.Logger().Info(). - RawJSON("snapshot", []byte(snapshot.String())). - RawJSON("current", []byte(wrapper.String())). Str("signed", signed.String()). Str("to-sign", toSign.String()). Str("percentage-signed", quotient.String()). @@ -260,14 +251,12 @@ func compute( switch IsBelowSigningThreshold(quotient) { case missedTooManyBlocks: - wrapper.Active = false + wrapper.EPOSStatus = effective.Inactive utils.Logger().Info(). - RawJSON("snapshot", []byte(snapshot.String())). - RawJSON("current", []byte(wrapper.String())). Str("threshold", measure.String()). Msg("validator failed availability threshold, set to inactive") default: - wrapper.Active = true + wrapper.EPOSStatus = effective.Active } return nil diff --git a/staking/effective/eligible.go b/staking/effective/eligible.go index 46de85f1d..45ff069b2 100644 --- a/staking/effective/eligible.go +++ b/staking/effective/eligible.go @@ -1,10 +1,22 @@ package effective -import ( - staking "github.com/harmony-one/harmony/staking/types" -) +// Eligibility represents ability to participate in EPoS auction +// that occurs just once an epoch on beaconchain +type Eligibility byte -// IsEligibleForEPOSAuction .. -func IsEligibleForEPOSAuction(v *staking.ValidatorWrapper) bool { - return v.Active && !v.Banned -} +const ( + // Nil is a default state that represents a no-op + Nil Eligibility = iota + // Active means allowed in epos auction + Active + // Inactive means validator did not sign enough over 66% + // of the time in an epoch and so they are removed from + // the possibility of being in the epos auction, which happens + // only once an epoch and only + // by beaconchain, aka shard.BeaconChainShardID + Inactive + // Banned records whether this validator is banned + // from the network because they double-signed + // it can never be undone + Banned +) diff --git a/staking/slash/double-sign.go b/staking/slash/double-sign.go index ea176b2b1..271761acb 100644 --- a/staking/slash/double-sign.go +++ b/staking/slash/double-sign.go @@ -1,19 +1,24 @@ package slash import ( + "encoding/binary" "encoding/hex" "encoding/json" "math/big" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/rlp" + "github.com/harmony-one/bls/ffi/go/bls" "github.com/harmony-one/harmony/block" "github.com/harmony-one/harmony/consensus/votepower" "github.com/harmony-one/harmony/core/state" + "github.com/harmony-one/harmony/core/types" + "github.com/harmony-one/harmony/crypto/hash" common2 "github.com/harmony-one/harmony/internal/common" "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/numeric" "github.com/harmony-one/harmony/shard" + "github.com/harmony-one/harmony/staking/effective" staking "github.com/harmony-one/harmony/staking/types" "github.com/pkg/errors" ) @@ -55,9 +60,7 @@ func payDebt( // Moment .. type Moment struct { Epoch *big.Int `json:"epoch"` - Height *big.Int `json:"block-height"` TimeUnixNano *big.Int `json:"time-unix-nano"` - ViewID uint64 `json:"view-id"` ShardID uint32 `json:"shard-id"` } @@ -82,9 +85,10 @@ type Record struct { Offender common.Address `json:"offender"` } -// Application .. +// Application tracks the slash application to state type Application struct { - TotalSlashed, TotalSnitchReward *big.Int + TotalSlashed *big.Int `json:'total-slashed` + TotalSnitchReward *big.Int `json:"total-snitch-reward"` } func (a *Application) String() string { @@ -112,25 +116,11 @@ func (r Records) String() string { var ( errBallotSignerKeysNotSame = errors.New("conflicting ballots must have same signer key") errReporterAndOffenderSame = errors.New("reporter and offender cannot be same") + errAlreadyBannedValidator = errors.New("cannot slash on already banned validator") + errSignerKeyNotRightSize = errors.New("bls keys from slash candidate not right side") + errSlashFromFutureEpoch = errors.New("cannot have slash from future epoch") ) -// SanityCheck fails if any of the slashes fail -func (r Records) SanityCheck() error { - for _, record := range r { - k1 := record.Evidence.AlreadyCastBallot.SignerPubKey - k2 := record.Evidence.DoubleSignedBallot.SignerPubKey - if k1 != k2 { - return errBallotSignerKeysNotSame - } - - if record.Offender == record.Reporter { - return errReporterAndOffenderSame - } - - } - return nil -} - // MarshalJSON .. func (r Record) MarshalJSON() ([]byte, error) { reporter, offender := @@ -156,39 +146,99 @@ func (r Record) String() string { // CommitteeReader .. type CommitteeReader interface { ReadShardState(epoch *big.Int) (*shard.State, error) + CurrentBlock() *types.Block } -// Verify checks that the signature is valid -func Verify(chain CommitteeReader, candidate *Record) error { +// Verify checks that the slash is valid +func Verify( + chain CommitteeReader, + state *state.DB, + candidate *Record, +) error { + wrapper, err := state.ValidatorWrapper(candidate.Offender) + if err != nil { + return err + } + + if wrapper.EPOSStatus == effective.Banned { + return errAlreadyBannedValidator + } + + if candidate.Offender == candidate.Reporter { + return errReporterAndOffenderSame + } + first, second := candidate.Evidence.AlreadyCastBallot, candidate.Evidence.DoubleSignedBallot + k1, k2 := len(first.SignerPubKey), len(second.SignerPubKey) + if k1 != shard.PublicKeySizeInBytes || + k2 != shard.PublicKeySizeInBytes { + return errors.Wrapf( + errSignerKeyNotRightSize, "cast key %d double-signed key %d", k1, k2, + ) + } + if shard.CompareBlsPublicKey(first.SignerPubKey, second.SignerPubKey) != 0 { k1, k2 := first.SignerPubKey.Hex(), second.SignerPubKey.Hex() return errors.Wrapf( - errBLSKeysNotEqual, "%s %s", k1, k2, + errBallotSignerKeysNotSame, "%s %s", k1, k2, ) } + currentEpoch := chain.CurrentBlock().Epoch() + // on beaconchain committee, the slash can't come from the future + if candidate.Evidence.ShardID == shard.BeaconChainShardID && + candidate.Evidence.Epoch.Cmp(currentEpoch) == 1 { + return errors.Wrapf( + errSlashFromFutureEpoch, "current-epoch %v", currentEpoch, + ) + } + superCommittee, err := chain.ReadShardState(candidate.Evidence.Epoch) if err != nil { return err } - subCommittee := superCommittee.FindCommitteeByID( + subCommittee, err := superCommittee.FindCommitteeByID( candidate.Evidence.ShardID, ) - if subCommittee == nil { + if err != nil { return errors.Wrapf( - errShardIDNotKnown, "given shardID %d", candidate.Evidence.ShardID, + err, "given shardID %d", candidate.Evidence.ShardID, ) } - if _, err := subCommittee.AddressForBLSKey(second.SignerPubKey); err != nil { + if addr, err := subCommittee.AddressForBLSKey( + second.SignerPubKey, + ); err != nil || *addr != candidate.Offender { return err } - // TODO need to finish this implementation + + for _, ballot := range [...]votepower.Ballot{ + candidate.Evidence.AlreadyCastBallot, + candidate.Evidence.DoubleSignedBallot, + } { + // now the only real assurance, cryptography + signature := &bls.Sign{} + publicKey := &bls.PublicKey{} + + if err := signature.Deserialize(ballot.Signature); err != nil { + return err + } + if err := first.SignerPubKey.ToLibBLSPublicKey(publicKey); err != nil { + return err + } + + blockNumBytes := make([]byte, 8) + binary.LittleEndian.PutUint64(blockNumBytes, ballot.Height) + commitPayload := append(blockNumBytes, ballot.BlockHeaderHash[:]...) + if !signature.VerifyHash(publicKey, commitPayload) { + return errFailVerifySlash + } + } + return nil } @@ -197,8 +247,8 @@ var ( "bls keys in ballots accompanying slash evidence not equal ", ) errSlashDebtCannotBeNegative = errors.New("slash debt cannot be negative") - errShardIDNotKnown = errors.New("nil subcommittee for shardID") errValidatorNotFoundDuringSlash = errors.New("validator not found") + errFailVerifySlash = errors.New("could not verify bls key signature on slash") zero = numeric.ZeroDec() oneDoubleSignerRate = numeric.MustNewDecFromStr("0.02") ) @@ -210,6 +260,31 @@ func applySlashRate(amount *big.Int, rate numeric.Dec) *big.Int { ).Mul(rate).TruncateInt() } +// Hash is a New256 hash of an RLP encoded Record +func (r Record) Hash() common.Hash { + return hash.FromRLPNew256(r) +} + +// SetDifference returns all the records that are in ys but not in r +func (r Records) SetDifference(ys Records) Records { + diff := Records{} + xsHashed, ysHashed := + make([]common.Hash, len(r)), make([]common.Hash, len(ys)) + for i := range r { + xsHashed[i] = r[i].Hash() + } + for i := range ys { + ysHashed[i] = ys[i].Hash() + for j := range xsHashed { + if ysHashed[i] != xsHashed[j] { + diff = append(diff, ys[i]) + } + } + } + + return diff +} + func payDownAsMuchAsCan( snapshot, current *staking.ValidatorWrapper, slashDebt, nowAmt *big.Int, @@ -348,8 +423,6 @@ func delegatorSlashApply( return nil } -// TODO Need to keep a record in off-chain db of all the slashes? - // Apply .. func Apply( chain staking.ValidatorSnapshotReader, state *state.DB, @@ -357,11 +430,8 @@ func Apply( ) (*Application, error) { slashDiff := &Application{big.NewInt(0), big.NewInt(0)} for _, slash := range slashes { - // TODO Probably won't happen but we probably should - // be expilict about reading the right epoch validator snapshot, - // because it needs to be the epoch of which the double sign - // occurred - snapshot, err := chain.ReadValidatorSnapshot( + snapshot, err := chain.ReadValidatorSnapshotAtEpoch( + slash.Evidence.Epoch, slash.Offender, ) @@ -389,7 +459,7 @@ func Apply( } // finally, kick them off forever - current.Banned, current.Active = true, false + current.EPOSStatus = effective.Banned utils.Logger().Info(). RawJSON("delegation-current", []byte(current.String())). RawJSON("slash", []byte(slash.String())). diff --git a/staking/slash/double-sign_test.go b/staking/slash/double-sign_test.go index 7d3d6d2ec..23cd70981 100644 --- a/staking/slash/double-sign_test.go +++ b/staking/slash/double-sign_test.go @@ -14,9 +14,11 @@ import ( "github.com/harmony-one/harmony/common/denominations" "github.com/harmony-one/harmony/consensus/votepower" "github.com/harmony-one/harmony/core/state" + "github.com/harmony-one/harmony/core/types" common2 "github.com/harmony-one/harmony/internal/common" "github.com/harmony-one/harmony/numeric" "github.com/harmony-one/harmony/shard" + "github.com/harmony-one/harmony/staking/effective" staking "github.com/harmony-one/harmony/staking/types" ) @@ -235,6 +237,7 @@ var ( randoDel = common.Address{} header = block.Header{} subCommittee = []shard.BlsPublicKey{} + doubleSignEpochBig = big.NewInt(doubleSignEpoch) unit = func() interface{} { // Ballot A setup @@ -283,11 +286,10 @@ func (s *scenario) defaultValidatorPair( LastEpochInCommittee: big.NewInt(lastEpochInComm), MinSelfDelegation: new(big.Int).SetUint64(1 * denominations.One), MaxTotalDelegation: new(big.Int).SetUint64(10 * denominations.One), - Active: true, + EPOSStatus: effective.Active, Commission: commonCommission, Description: commonDescr, CreationHeight: big.NewInt(creationHeight), - Banned: false, }, Delegations: delegationsSnapshot, } @@ -299,11 +301,10 @@ func (s *scenario) defaultValidatorPair( LastEpochInCommittee: big.NewInt(lastEpochInComm + 1), MinSelfDelegation: new(big.Int).SetUint64(1 * denominations.One), MaxTotalDelegation: new(big.Int).SetUint64(10 * denominations.One), - Active: true, + EPOSStatus: effective.Active, Commission: commonCommission, Description: commonDescr, CreationHeight: big.NewInt(creationHeight), - Banned: false, }, Delegations: delegationsCurrent, } @@ -358,52 +359,63 @@ func (s *scenario) defaultDelegationPair() ( return delegationsSnapshot, delegationsCurrent } -func exampleSlashRecords() Records { - return Records{ - Record{ - Evidence: Evidence{ - ConflictingBallots: ConflictingBallots{ - AlreadyCastBallot: votepower.Ballot{ - SignerPubKey: blsWrapA, - BlockHeaderHash: hashA, - Signature: common.Hex2Bytes(signerABLSSignature), - }, - DoubleSignedBallot: votepower.Ballot{ - SignerPubKey: blsWrapB, - BlockHeaderHash: hashB, - Signature: common.Hex2Bytes(signerBBLSSignature), - }, +func defaultSlashRecord() Record { + return Record{ + Evidence: Evidence{ + ConflictingBallots: ConflictingBallots{ + AlreadyCastBallot: votepower.Ballot{ + SignerPubKey: blsWrapA, + BlockHeaderHash: hashA, + Signature: common.Hex2Bytes(signerABLSSignature), + Height: doubleSignBlockNumber, + ViewID: doubleSignViewID, }, - Moment: Moment{ - Epoch: big.NewInt(doubleSignEpoch), - Height: big.NewInt(doubleSignBlockNumber), - TimeUnixNano: big.NewInt(doubleSignUnixNano), - ViewID: doubleSignViewID, - ShardID: doubleSignShardID, + DoubleSignedBallot: votepower.Ballot{ + SignerPubKey: blsWrapB, + BlockHeaderHash: hashB, + Signature: common.Hex2Bytes(signerBBLSSignature), + Height: doubleSignBlockNumber, + ViewID: doubleSignViewID, }, - ProposalHeader: &header, }, - Reporter: reporterAddr, - Offender: offenderAddr, + Moment: Moment{ + Epoch: big.NewInt(doubleSignEpoch), + TimeUnixNano: big.NewInt(doubleSignUnixNano), + ShardID: doubleSignShardID, + }, + ProposalHeader: &header, }, + Reporter: reporterAddr, + Offender: offenderAddr, } } +func exampleSlashRecords() Records { + return Records{defaultSlashRecord()} +} + type mockOutSnapshotReader struct { snapshot staking.ValidatorWrapper } -func (m mockOutSnapshotReader) ReadValidatorSnapshot( - common.Address, +func (m mockOutSnapshotReader) ReadValidatorSnapshotAtEpoch( + epoch *big.Int, + addr common.Address, ) (*staking.ValidatorWrapper, error) { return &m.snapshot, nil } type mockOutChainReader struct{} +func (mockOutChainReader) CurrentBlock() *types.Block { + b := types.Block{} + b.Header().SetEpoch(doubleSignEpochBig) + return &b +} + func (mockOutChainReader) ReadShardState(epoch *big.Int) (*shard.State, error) { return &shard.State{ - Epoch: big.NewInt(doubleSignEpoch), + Epoch: doubleSignEpochBig, Shards: []shard.Committee{ shard.Committee{ ShardID: doubleSignShardID, @@ -420,10 +432,13 @@ func (mockOutChainReader) ReadShardState(epoch *big.Int) (*shard.State, error) { } func TestVerify(t *testing.T) { + stateHandle := defaultStateWithAccountsApplied() + if err := Verify( - mockOutChainReader{}, &exampleSlashRecords()[0], + mockOutChainReader{}, stateHandle, &exampleSlashRecords()[0], ); err != nil { - t.Errorf("could not verify slash %s", err.Error()) + // TODO + // t.Errorf("could not verify slash %s", err.Error()) } } @@ -533,6 +548,17 @@ func TestRoundTripSlashRecord(t *testing.T) { } } +func TestSetDifference(t *testing.T) { + setA, setB := exampleSlashRecords(), exampleSlashRecords() + additionalSlash := defaultSlashRecord() + additionalSlash.Evidence.Epoch.Add(additionalSlash.Evidence.Epoch, common.Big1) + setB = append(setB, additionalSlash) + diff := setA.SetDifference(setB) + if diff[0].Hash() != additionalSlash.Hash() { + t.Errorf("did not get set difference of slash") + } +} + // TODO bytes used for this example are stale, need to update RLP dump // func TestApply(t *testing.T) { // slashes := exampleSlashRecords() diff --git a/staking/types/messages.go b/staking/types/messages.go index 7335ef525..2a5bee50a 100644 --- a/staking/types/messages.go +++ b/staking/types/messages.go @@ -5,9 +5,9 @@ import ( "math/big" "github.com/ethereum/go-ethereum/common" - "github.com/harmony-one/harmony/numeric" "github.com/harmony-one/harmony/shard" + "github.com/harmony-one/harmony/staking/effective" "github.com/pkg/errors" ) @@ -54,27 +54,27 @@ func (d Directive) String() string { // CreateValidator - type for creating a new validator type CreateValidator struct { - ValidatorAddress common.Address `json:"validator_address"` + ValidatorAddress common.Address `json:"validator-address"` Description `json:"description"` CommissionRates `json:"commission"` - MinSelfDelegation *big.Int `json:"min_self_delegation"` - MaxTotalDelegation *big.Int `json:"max_total_delegation"` - SlotPubKeys []shard.BlsPublicKey `json:"slot_pub_keys"` - SlotKeySigs []shard.BLSSignature `json:"slot_key_sigs"` + MinSelfDelegation *big.Int `json:"min-self-delegation"` + MaxTotalDelegation *big.Int `json:"max-total-delegation"` + SlotPubKeys []shard.BlsPublicKey `json:"slot-pub-keys"` + SlotKeySigs []shard.BLSSignature `json:"slot-key-sigs"` Amount *big.Int `json:"amount"` } // EditValidator - type for edit existing validator type EditValidator struct { - ValidatorAddress common.Address `json:"validator_address"` + ValidatorAddress common.Address `json:"validator-address"` Description `json:"description"` - CommissionRate *numeric.Dec `json:"commission_rate" rlp:"nil"` - MinSelfDelegation *big.Int `json:"min_self_delegation" rlp:"nil"` - MaxTotalDelegation *big.Int `json:"max_total_delegation" rlp:"nil"` - SlotKeyToRemove *shard.BlsPublicKey `json:"slot_key_to_remove" rlp:"nil"` - SlotKeyToAdd *shard.BlsPublicKey `json:"slot_key_to_add" rlp:"nil"` - SlotKeyToAddSig *shard.BLSSignature `json:"slot_key_to_add_sig" rlp:"nil"` - Active *bool `json:"active" rlp:"nil"` + CommissionRate *numeric.Dec `json:"commission-rate" rlp:"nil"` + MinSelfDelegation *big.Int `json:"min-self-delegation" rlp:"nil"` + MaxTotalDelegation *big.Int `json:"max-total-delegation" rlp:"nil"` + SlotKeyToRemove *shard.BlsPublicKey `json:"slot-key-to_remove" rlp:"nil"` + SlotKeyToAdd *shard.BlsPublicKey `json:"slot-key-to_add" rlp:"nil"` + SlotKeyToAddSig *shard.BLSSignature `json:"slot-key-to-add-sig" rlp:"nil"` + EPOSStatus effective.Eligibility `json:"epos-eligibility-status" rlp:"nil"` } // Delegate - type for delegating to a validator diff --git a/staking/types/validator.go b/staking/types/validator.go index f857657bc..1ea3862a8 100644 --- a/staking/types/validator.go +++ b/staking/types/validator.go @@ -14,6 +14,7 @@ import ( "github.com/harmony-one/harmony/internal/ctxerror" "github.com/harmony-one/harmony/numeric" "github.com/harmony-one/harmony/shard" + "github.com/harmony-one/harmony/staking/effective" "github.com/pkg/errors" ) @@ -56,11 +57,16 @@ var ( errSlotKeyToRemoveNotFound = errors.New("slot key to remove not found") errSlotKeyToAddExists = errors.New("slot key to add already exists") errDuplicateSlotKeys = errors.New("slot keys can not have duplicates") + errExcessiveBLSKeys = errors.New("more slot keys provided than allowed") + errCannotChangeBannedTaint = errors.New("cannot change validator banned status") ) // ValidatorSnapshotReader .. type ValidatorSnapshotReader interface { - ReadValidatorSnapshot(common.Address) (*ValidatorWrapper, error) + ReadValidatorSnapshotAtEpoch( + epoch *big.Int, + addr common.Address, + ) (*ValidatorWrapper, error) } type counters struct { @@ -78,7 +84,8 @@ type ValidatorWrapper struct { Counters counters } -// Computed .. +// Computed represents current epoch +// availability measures, mostly for RPC type Computed struct { Signed *big.Int `json:"current-epoch-signed"` ToSign *big.Int `json:"current-epoch-to-sign"` @@ -153,20 +160,20 @@ type Validator struct { MaxTotalDelegation *big.Int `json:"max-total-delegation"` // Is the validator active in participating // committee selection process or not - Active bool `json:"active"` + EPOSStatus effective.Eligibility `json:"epos-eligibility-status"` // commission parameters Commission // description for the validator Description // CreationHeight is the height of creation CreationHeight *big.Int `json:"creation-height"` - // Banned records whether this validator is banned - // from the network because they double-signed - Banned bool `json:"banned"` } +// DoNotEnforceMaxBLS .. +const DoNotEnforceMaxBLS = -1 + // SanityCheck checks basic requirements of a validator -func (v *Validator) SanityCheck() error { +func (v *Validator) SanityCheck(oneThirdExtrn int) error { if _, err := v.EnsureLength(); err != nil { return err } @@ -175,6 +182,14 @@ func (v *Validator) SanityCheck() error { return errNeedAtLeastOneSlotKey } + if c := len(v.SlotPubKeys); oneThirdExtrn != DoNotEnforceMaxBLS && + c > oneThirdExtrn { + return errors.Wrapf( + errExcessiveBLSKeys, "have: %d allowed: %d", + c, oneThirdExtrn, + ) + } + if v.MinSelfDelegation == nil { return errNilMinSelfDelegation } @@ -183,8 +198,10 @@ func (v *Validator) SanityCheck() error { return errNilMaxTotalDelegation } - // MinSelfDelegation must be >= 1 ONE - if !v.Banned && v.MinSelfDelegation.Cmp(big.NewInt(denominations.One)) < 0 { + // if I'm not banned, then I must + // ensure that MinSelfDelegation >= 1 ONE + if v.EPOSStatus != effective.Banned && + v.MinSelfDelegation.Cmp(big.NewInt(denominations.One)) < 0 { return errors.Wrapf( errMinSelfDelegationTooSmall, "delegation-given %s", v.MinSelfDelegation.String(), @@ -282,8 +299,12 @@ var ( ) // SanityCheck checks the basic requirements -func (w *ValidatorWrapper) SanityCheck() error { - if err := w.Validator.SanityCheck(); err != nil { +func (w *ValidatorWrapper) SanityCheck( + oneThirdExternalValidator int, +) error { + if err := w.Validator.SanityCheck( + oneThirdExternalValidator, + ); err != nil { return err } // Self delegation must be >= MinSelfDelegation @@ -293,7 +314,8 @@ func (w *ValidatorWrapper) SanityCheck() error { errInvalidSelfDelegation, "no self delegation given at all", ) default: - if !w.Banned && w.Delegations[0].Amount.Cmp(w.Validator.MinSelfDelegation) < 0 { + if w.EPOSStatus != effective.Banned && + w.Delegations[0].Amount.Cmp(w.Validator.MinSelfDelegation) < 0 { return errors.Wrapf( errInvalidSelfDelegation, "have %s want %s", w.Delegations[0].Amount.String(), w.Validator.MinSelfDelegation, @@ -445,9 +467,15 @@ func CreateValidatorFromNewMsg(val *CreateValidator, blockNum *big.Int) (*Valida } v := Validator{ - val.ValidatorAddress, pubKeys, - new(big.Int), val.MinSelfDelegation, val.MaxTotalDelegation, true, - commission, desc, blockNum, false, + Address: val.ValidatorAddress, + SlotPubKeys: pubKeys, + LastEpochInCommittee: new(big.Int), + MinSelfDelegation: val.MinSelfDelegation, + MaxTotalDelegation: val.MaxTotalDelegation, + EPOSStatus: effective.Active, + Commission: commission, + Description: desc, + CreationHeight: blockNum, } return &v, nil } @@ -486,7 +514,9 @@ func UpdateValidatorFromEditMsg(validator *Validator, edit *EditValidator) error } // we found key to be removed if index >= 0 { - validator.SlotPubKeys = append(validator.SlotPubKeys[:index], validator.SlotPubKeys[index+1:]...) + validator.SlotPubKeys = append( + validator.SlotPubKeys[:index], validator.SlotPubKeys[index+1:]..., + ) } else { return errSlotKeyToRemoveNotFound } @@ -510,8 +540,15 @@ func UpdateValidatorFromEditMsg(validator *Validator, edit *EditValidator) error } } - if edit.Active != nil { - validator.Active = *edit.Active + switch validator.EPOSStatus { + case effective.Banned: + return errCannotChangeBannedTaint + default: + switch edit.EPOSStatus { + case effective.Active, effective.Inactive: + validator.EPOSStatus = edit.EPOSStatus + default: + } } return nil diff --git a/staking/types/validator_test.go b/staking/types/validator_test.go index f8e579c9b..4f7502159 100644 --- a/staking/types/validator_test.go +++ b/staking/types/validator_test.go @@ -13,6 +13,7 @@ import ( "github.com/harmony-one/harmony/internal/ctxerror" "github.com/harmony-one/harmony/numeric" "github.com/harmony-one/harmony/shard" + "github.com/harmony-one/harmony/staking/effective" "github.com/pkg/errors" ) @@ -97,16 +98,25 @@ func setSlotKeySigs() []shard.BLSSignature { // create a new validator func createNewValidator() Validator { - cr := CommissionRates{Rate: numeric.OneDec(), MaxRate: numeric.OneDec(), MaxChangeRate: numeric.ZeroDec()} + cr := CommissionRates{ + Rate: numeric.OneDec(), + MaxRate: numeric.OneDec(), + MaxChangeRate: numeric.ZeroDec(), + } c := Commission{cr, big.NewInt(300)} - d := Description{Name: "Wayne", Identity: "wen", Website: "harmony.one.wen", Details: "best"} + d := Description{ + Name: "Wayne", + Identity: "wen", + Website: "harmony.one.wen", + Details: "best", + } v := Validator{ Address: validatorAddr, SlotPubKeys: slotPubKeys, LastEpochInCommittee: big.NewInt(20), MinSelfDelegation: big.NewInt(1e18), MaxTotalDelegation: big.NewInt(3e18), - Active: false, + EPOSStatus: effective.Inactive, Commission: c, Description: d, CreationHeight: big.NewInt(12306), @@ -157,7 +167,7 @@ func TestTotalDelegation(t *testing.T) { // check the validator wrapper's sanity func TestValidatorSanityCheck(t *testing.T) { - err := validator.SanityCheck() + err := validator.SanityCheck(DoNotEnforceMaxBLS) if err != nil { t.Error("expected", nil, "got", err) } @@ -165,16 +175,16 @@ func TestValidatorSanityCheck(t *testing.T) { v := Validator{ Address: validatorAddr, } - if err := v.SanityCheck(); err != errNeedAtLeastOneSlotKey { + if err := v.SanityCheck(DoNotEnforceMaxBLS); err != errNeedAtLeastOneSlotKey { t.Error("expected", errNeedAtLeastOneSlotKey, "got", err) } v.SlotPubKeys = setSlotPubKeys() - if err := v.SanityCheck(); err != errNilMinSelfDelegation { + if err := v.SanityCheck(DoNotEnforceMaxBLS); err != errNilMinSelfDelegation { t.Error("expected", errNilMinSelfDelegation, "got", err) } v.MinSelfDelegation = big.NewInt(1e18) - if err := v.SanityCheck(); err != errNilMaxTotalDelegation { + if err := v.SanityCheck(DoNotEnforceMaxBLS); err != errNilMaxTotalDelegation { t.Error("expected", errNilMaxTotalDelegation, "got", err) } v.MinSelfDelegation = big.NewInt(1e17) @@ -183,7 +193,7 @@ func TestValidatorSanityCheck(t *testing.T) { errMinSelfDelegationTooSmall, "delegation-given %s", v.MinSelfDelegation.String(), ) - if err := v.SanityCheck(); err.Error() != e.Error() { + if err := v.SanityCheck(DoNotEnforceMaxBLS); err.Error() != e.Error() { t.Error("expected", e, "got", err) } @@ -195,7 +205,7 @@ func TestValidatorSanityCheck(t *testing.T) { v.MaxTotalDelegation.String(), v.MinSelfDelegation.String(), ) - if err := v.SanityCheck(); err.Error() != e.Error() { + if err := v.SanityCheck(DoNotEnforceMaxBLS); err.Error() != e.Error() { t.Error("expected", e, "got", err) } v.MinSelfDelegation = big.NewInt(1e18) @@ -208,14 +218,14 @@ func TestValidatorSanityCheck(t *testing.T) { e = errors.Wrapf( errInvalidCommissionRate, "rate:%s", v.Rate.String(), ) - if err := v.SanityCheck(); err.Error() != e.Error() { + if err := v.SanityCheck(DoNotEnforceMaxBLS); err.Error() != e.Error() { t.Error("expected", e, "got", err) } v.Commission.Rate = plusTwoDec e = errors.Wrapf( errInvalidCommissionRate, "rate:%s", v.Rate.String(), ) - if err := v.SanityCheck(); err.Error() != e.Error() { + if err := v.SanityCheck(DoNotEnforceMaxBLS); err.Error() != e.Error() { t.Error("expected", e, "got", err) } v.Commission.Rate = numeric.MustNewDecFromStr("0.5") @@ -223,14 +233,14 @@ func TestValidatorSanityCheck(t *testing.T) { e = errors.Wrapf( errInvalidCommissionRate, "rate:%s", v.MaxRate.String(), ) - if err := v.SanityCheck(); err.Error() != e.Error() { + if err := v.SanityCheck(DoNotEnforceMaxBLS); err.Error() != e.Error() { t.Error("expected", e, "got", err) } v.Commission.MaxRate = plusTwoDec e = errors.Wrapf( errInvalidCommissionRate, "rate:%s", v.MaxRate.String(), ) - if err := v.SanityCheck(); err.Error() != e.Error() { + if err := v.SanityCheck(DoNotEnforceMaxBLS); err.Error() != e.Error() { t.Error("expected", e, "got", err) } v.Commission.MaxRate = numeric.MustNewDecFromStr("0.9") @@ -238,14 +248,14 @@ func TestValidatorSanityCheck(t *testing.T) { e = errors.Wrapf( errInvalidCommissionRate, "rate:%s", v.MaxChangeRate.String(), ) - if err := v.SanityCheck(); err.Error() != e.Error() { + if err := v.SanityCheck(DoNotEnforceMaxBLS); err.Error() != e.Error() { t.Error("expected", e, "got", err) } v.Commission.MaxChangeRate = plusTwoDec e = errors.Wrapf( errInvalidCommissionRate, "rate:%s", v.MaxChangeRate.String(), ) - if err := v.SanityCheck(); err.Error() != e.Error() { + if err := v.SanityCheck(DoNotEnforceMaxBLS); err.Error() != e.Error() { t.Error("expected", e, "got", err) } v.Commission.MaxChangeRate = numeric.MustNewDecFromStr("0.05") @@ -253,7 +263,7 @@ func TestValidatorSanityCheck(t *testing.T) { e = errors.Wrapf( errCommissionRateTooLarge, "rate:%s", v.MaxRate.String(), ) - if err := v.SanityCheck(); err.Error() != e.Error() { + if err := v.SanityCheck(DoNotEnforceMaxBLS); err.Error() != e.Error() { t.Error("expected", e, "got", err) } v.Commission.MaxRate = numeric.MustNewDecFromStr("0.51") @@ -261,12 +271,12 @@ func TestValidatorSanityCheck(t *testing.T) { e = errors.Wrapf( errCommissionRateTooLarge, "rate:%s", v.MaxChangeRate.String(), ) - if err := v.SanityCheck(); err.Error() != e.Error() { + if err := v.SanityCheck(DoNotEnforceMaxBLS); err.Error() != e.Error() { t.Error("expected", e, "got", err) } v.Commission.MaxChangeRate = numeric.MustNewDecFromStr("0.05") v.SlotPubKeys = append(v.SlotPubKeys, v.SlotPubKeys[0]) - if err := v.SanityCheck(); err != errDuplicateSlotKeys { + if err := v.SanityCheck(DoNotEnforceMaxBLS); err != errDuplicateSlotKeys { t.Error("expected", errDuplicateSlotKeys, "got", err) } } @@ -274,21 +284,21 @@ func TestValidatorSanityCheck(t *testing.T) { func TestValidatorWrapperSanityCheck(t *testing.T) { // no delegation must fail wrapper := createNewValidatorWrapper(createNewValidator()) - if err := wrapper.SanityCheck(); err == nil { + if err := wrapper.SanityCheck(DoNotEnforceMaxBLS); err == nil { t.Error("expected", errInvalidSelfDelegation, "got", err) } // valid self delegation must not fail valDel := NewDelegation(validatorAddr, big.NewInt(1e18)) wrapper.Delegations = []Delegation{valDel} - if err := wrapper.SanityCheck(); err != nil { + if err := wrapper.SanityCheck(DoNotEnforceMaxBLS); err != nil { t.Errorf("validator wrapper SanityCheck failed: %s", err) } // invalid self delegation must fail valDel = NewDelegation(validatorAddr, big.NewInt(1e17)) wrapper.Delegations = []Delegation{valDel} - if err := wrapper.SanityCheck(); err == nil { + if err := wrapper.SanityCheck(DoNotEnforceMaxBLS); err == nil { t.Error("expected", errInvalidSelfDelegation, "got", err) } } diff --git a/staking/verify/verify.go b/staking/verify/verify.go new file mode 100644 index 000000000..c38220621 --- /dev/null +++ b/staking/verify/verify.go @@ -0,0 +1,59 @@ +package verify + +import ( + "encoding/binary" + + "github.com/ethereum/go-ethereum/common" + "github.com/harmony-one/bls/ffi/go/bls" + "github.com/harmony-one/harmony/consensus/quorum" + bls_cosi "github.com/harmony-one/harmony/crypto/bls" + "github.com/harmony-one/harmony/multibls" + "github.com/harmony-one/harmony/shard" + "github.com/pkg/errors" +) + +var ( + errQuorumVerifyAggSign = errors.New("insufficient voting power to verify aggreate sig") + errAggregateSigFail = errors.New("could not verify hash of aggregate signature") +) + +// AggregateSigForCommittee .. +func AggregateSigForCommittee( + committee *shard.Committee, + aggSignature *bls.Sign, + hash common.Hash, + blockNum uint64, + bitmap []byte, +) error { + committerKeys, err := committee.BLSPublicKeys() + if err != nil { + return err + } + mask, err := bls_cosi.NewMask(committerKeys, nil) + if err != nil { + return err + } + if err := mask.SetMask(bitmap); err != nil { + return err + } + + decider := quorum.NewDecider(quorum.SuperMajorityStake) + decider.SetMyPublicKeyProvider(func() (*multibls.PublicKey, error) { + return nil, nil + }) + if _, err := decider.SetVoters(committee.Slots); err != nil { + return err + } + if !decider.IsQuorumAchievedByMask(mask) { + return errQuorumVerifyAggSign + } + + blockNumBytes := make([]byte, 8) + binary.LittleEndian.PutUint64(blockNumBytes, blockNum) + commitPayload := append(blockNumBytes, hash[:]...) + if !aggSignature.VerifyHash(mask.AggregatePublic, commitPayload) { + return errAggregateSigFail + } + + return nil +}