From 78c0cf030c8dd70ecf44bbb8b9e75e3d97325736 Mon Sep 17 00:00:00 2001 From: Edgar Aroutiounian Date: Mon, 11 Nov 2019 16:56:21 -0800 Subject: [PATCH 01/26] [staking] Stumble out test case for validator information --- core/blockchain.go | 77 +++++++++++++++++++++++++++++------ internal/params/config.go | 2 +- shard/committee/assignment.go | 7 ++-- 3 files changed, 68 insertions(+), 18 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index ed12c4d62..00838bef3 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -23,6 +23,7 @@ import ( "fmt" "io" "math/big" + "math/rand" "sync" "sync/atomic" "time" @@ -42,6 +43,7 @@ import ( "github.com/harmony-one/harmony/core/state" "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/core/vm" + common2 "github.com/harmony-one/harmony/internal/common" "github.com/harmony-one/harmony/internal/ctxerror" "github.com/harmony-one/harmony/internal/params" "github.com/harmony-one/harmony/internal/utils" @@ -2438,26 +2440,75 @@ func (bc *BlockChain) CurrentValidatorAddresses() []common.Address { return filtered } +const ( + a = "one1x4080kv34a4s3s886vs3rvvhm28pk3r0fnfh6n" + b = "one1pjq82a97vfxvmlm60htyc7kafhyjm59f7wgz22" + c = "one1yc06ghr2p8xnl2380kpfayweguuhxdtupkhqzw" + d = "one1wh4p0kuc7unxez2z8f82zfnhsg4ty6dupqyjt2" + e = "one16qsd5ant9v94jrs89mruzx62h7ekcfxmduh2rx" + f = "one1spshr72utf6rwxseaz339j09ed8p6f8ke370zj" + g = "one1v788ag7ukh6wjujvpempss6h9ugvfxmnsyklr5" + h = "one1hx3gy5e864eycmjtpcvsuz59m3lu7y89q0ddhl" + i = "one1mmvhupm7ejytype664mhs2m95vn9pfdjrqfadv" +) + // ValidatorCandidates returns the up to date validator candidates for next epoch func (bc *BlockChain) ValidatorCandidates() []common.Address { - list, err := bc.ReadValidatorList() - if err != nil { - return make([]common.Address, 0) + // TODO Turn this into 400 length generated slice + return []common.Address{ + common2.ParseAddr(a), + common2.ParseAddr(b), + common2.ParseAddr(c), + common2.ParseAddr(d), + common2.ParseAddr(e), + common2.ParseAddr(f), + common2.ParseAddr(g), + common2.ParseAddr(h), } - return list } +var ( + sequence = rand.New(rand.NewSource(42)) +) + // ValidatorInformation returns the information of validator func (bc *BlockChain) ValidatorInformation(addr common.Address) (*staking.Validator, error) { - state, err := bc.StateAt(bc.CurrentBlock().Root()) - if err != nil || state == nil { - return nil, err - } - wrapper := state.GetStakingInfo(addr) - if wrapper == nil { - return nil, fmt.Errorf("ValidatorInformation not found: %v", addr) - } - return &wrapper.Validator, nil + // EDGAR 0x355E77D991Af6B08C0e7d32111b197da8e1B446f + // EDGAR 0x0C807574BE624CcdFf7A7dD64c7ADd4dC92dd0a9 + // EDGAR 0x261fa45c6A09cD3Faa277d829e91d9473973357C + // EDGAR 0x75eA17DB98F7266C89423A4EA12677822Ab269Bc + // EDGAR 0xD020dA766b2b0b590E072ec7c11b4AbFb36c24DB + // EDGAR 0x806171f95C5a74371a19e8a312c9e5Cb4E1D24f6 + // EDGAR 0x678E7Ea3DCb5f4e9724C0e761843572f10c49B73 + // EDGAR 0xB9A2825327D5724C6E4b0e190E0a85dc7FCf10e5 + someValidator := &staking.Validator{} + someValidator.Address = addr + someValidator.Stake = big.NewInt(int64(sequence.Intn(100))) + switch B := common2.MustAddressToBech32(addr); true { + case B == a: + fmt.Println("Called here:", B, a) + return someValidator, nil + case B == b: + case B == c: + case B == d: + case B == e: + case B == f: + case B == g: + case B == h: + + } + + fmt.Println("EDGAR", addr.String()) + return nil, nil + // state, err := bc.StateAt(bc.CurrentBlock().Root()) + // if err != nil || state == nil { + // return nil, err + // } + // wrapper := state.GetStakingInfo(addr) + // if wrapper == nil { + // return nil, fmt.Errorf("ValidatorInformation not found: %v", addr) + // } + // return &wrapper.Validator, nil } // DelegatorsInformation returns up to date information of delegators of a given validator address diff --git a/internal/params/config.go b/internal/params/config.go index 21af1fda3..c75efc202 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -36,7 +36,7 @@ var ( ChainID: TestnetChainID, CrossTxEpoch: big.NewInt(0), CrossLinkEpoch: big.NewInt(0), - StakingEpoch: EpochTBD, + StakingEpoch: big.NewInt(5), EIP155Epoch: big.NewInt(0), S3Epoch: big.NewInt(0), } diff --git a/shard/committee/assignment.go b/shard/committee/assignment.go index 7abcbaea3..6bc3a78f9 100644 --- a/shard/committee/assignment.go +++ b/shard/committee/assignment.go @@ -1,6 +1,7 @@ package committee import ( + "fmt" "math/big" "github.com/ethereum/go-ethereum/common" @@ -248,8 +249,6 @@ func (def partialStakingEnabled) Compute( if !config.IsStaking(epoch) { return preStakingEnabledCommittee(instance), nil } - stakedSlots := - (instance.NumNodesPerShard() - instance.NumHarmonyOperatedNodesPerShard()) * - int(instance.NumShards()) - return eposStakedCommittee(instance, stakerReader, stakedSlots) + fmt.Println("Staking epoch happened", config.String()) + return with400Stakers(instance, stakerReader) } From 32f4db4c4f8f71504ffa5b13a2896085696c3698 Mon Sep 17 00:00:00 2001 From: Edgar Aroutiounian Date: Tue, 12 Nov 2019 13:19:58 -0800 Subject: [PATCH 02/26] [shard] Provide JSON dump of shard state --- shard/shard_state.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shard/shard_state.go b/shard/shard_state.go index b8bccccf0..33ad597ac 100644 --- a/shard/shard_state.go +++ b/shard/shard_state.go @@ -39,7 +39,7 @@ type NodeID struct { EcdsaAddress common.Address `json:"ecdsa-address"` BlsPublicKey BlsPublicKey `json:"bls-pubkey"` // nil means not active, 0 means our node, >= 0 means staked node - StakeWithDelegationApplied *numeric.Dec `json:"staked-validator" rlp:"nil"` + StakeWithDelegationApplied *big.Int `json:"staked-validator" rlp:"nil"` } // NodeIDList is a list of NodeIDList. @@ -55,7 +55,7 @@ type Committee struct { func (ss State) JSON() string { type t struct { NodeID - EcdsaAddress string `json:"one-address"` + EcdsaAddress string `json:"ecdsa-address"` } type v struct { Committee From 09c427f0806fb55e62e9669ef4115aa0459bff0b Mon Sep 17 00:00:00 2001 From: Edgar Aroutiounian Date: Tue, 12 Nov 2019 16:31:46 -0800 Subject: [PATCH 03/26] [EPoS] Checkpoint w/RJ --- core/blockchain.go | 113 ++++++++--------- internal/configs/sharding/localnet.go | 2 +- internal/params/config.go | 7 +- numeric/decimal.go | 10 -- shard/committee/assignment.go | 171 +++++++++++++++++++------- staking/effective/calculate.go | 91 +++----------- staking/effective/values.go | 20 +++ staking/types/validator.go | 5 +- 8 files changed, 228 insertions(+), 191 deletions(-) create mode 100644 staking/effective/values.go diff --git a/core/blockchain.go b/core/blockchain.go index 00838bef3..54e27dd79 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -37,13 +37,13 @@ import ( "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" + "github.com/harmony-one/bls/ffi/go/bls" "github.com/harmony-one/harmony/block" consensus_engine "github.com/harmony-one/harmony/consensus/engine" "github.com/harmony-one/harmony/core/rawdb" "github.com/harmony-one/harmony/core/state" "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/core/vm" - common2 "github.com/harmony-one/harmony/internal/common" "github.com/harmony-one/harmony/internal/ctxerror" "github.com/harmony-one/harmony/internal/params" "github.com/harmony-one/harmony/internal/utils" @@ -2441,74 +2441,65 @@ func (bc *BlockChain) CurrentValidatorAddresses() []common.Address { } const ( - a = "one1x4080kv34a4s3s886vs3rvvhm28pk3r0fnfh6n" - b = "one1pjq82a97vfxvmlm60htyc7kafhyjm59f7wgz22" - c = "one1yc06ghr2p8xnl2380kpfayweguuhxdtupkhqzw" - d = "one1wh4p0kuc7unxez2z8f82zfnhsg4ty6dupqyjt2" - e = "one16qsd5ant9v94jrs89mruzx62h7ekcfxmduh2rx" - f = "one1spshr72utf6rwxseaz339j09ed8p6f8ke370zj" - g = "one1v788ag7ukh6wjujvpempss6h9ugvfxmnsyklr5" - h = "one1hx3gy5e864eycmjtpcvsuz59m3lu7y89q0ddhl" - i = "one1mmvhupm7ejytype664mhs2m95vn9pfdjrqfadv" + blsKey1 = "9a010ea58079ddcd31d5d10ac85e9b5a7732972ee5478d7296091b2af96fcd673c49d6dd7a4a9e7acd2e4aab0add0e00" + blsKey2 = "9275355e60f8c78337d538eb15e3f8eda120c28635f996d8a13f544d118db32749bcc0ed02a795770141d979a8b9de14" + // Pick big number for interesting looking one addresses + amount = 400 + fixedRandomGen = 98765654323 + fixedRandomGenStakeL = 40 + fixedRandomGenStakeH = 150 ) -// ValidatorCandidates returns the up to date validator candidates for next epoch -func (bc *BlockChain) ValidatorCandidates() []common.Address { - // TODO Turn this into 400 length generated slice - return []common.Address{ - common2.ParseAddr(a), - common2.ParseAddr(b), - common2.ParseAddr(c), - common2.ParseAddr(d), - common2.ParseAddr(e), - common2.ParseAddr(f), - common2.ParseAddr(g), - common2.ParseAddr(h), - } -} +var ( + // By fixing the source, we have predicable sequence + sequenceL = rand.New(rand.NewSource(42)) + sequenceH = rand.New(rand.NewSource(84)) + accountGenerator = rand.New(rand.NewSource(1337)) +) var ( - sequence = rand.New(rand.NewSource(42)) + tempBank map[common.Address]*staking.Validator = map[common.Address]*staking.Validator{} + addrs []common.Address ) +func init() { + addrs = make([]common.Address, amount) + for i := 0; i < amount; i++ { + addr := common.Address{} + addr.SetBytes( + big.NewInt(int64(accountGenerator.Intn(fixedRandomGen))).Bytes(), + ) + addrs[i] = addr + someValidator := &staking.Validator{} + someValidator.Address = addr + low := sequenceL.Intn(fixedRandomGenStakeL) + high := sequenceL.Intn(fixedRandomGenStakeH) + r := sequenceL.Intn(fixedRandomGenStakeH) + 1 + modBy := high - low + 1 + if modBy <= 0 { + modBy *= -1 + modBy++ + } + someValidator.Stake = new(big.Int).Abs(big.NewInt(int64( + (r % modBy) + low, + ))) + k := shard.BlsPublicKey{} + j := bls.PublicKey{} + j.DeserializeHexStr(blsKey1) + k.FromLibBLSPublicKey(&j) + someValidator.SlotPubKeys = []shard.BlsPublicKey{k} + tempBank[addr] = someValidator + } +} + +// ValidatorCandidates returns the up to date validator candidates for next epoch +func (bc *BlockChain) ValidatorCandidates() []common.Address { + return addrs +} + // ValidatorInformation returns the information of validator func (bc *BlockChain) ValidatorInformation(addr common.Address) (*staking.Validator, error) { - // EDGAR 0x355E77D991Af6B08C0e7d32111b197da8e1B446f - // EDGAR 0x0C807574BE624CcdFf7A7dD64c7ADd4dC92dd0a9 - // EDGAR 0x261fa45c6A09cD3Faa277d829e91d9473973357C - // EDGAR 0x75eA17DB98F7266C89423A4EA12677822Ab269Bc - // EDGAR 0xD020dA766b2b0b590E072ec7c11b4AbFb36c24DB - // EDGAR 0x806171f95C5a74371a19e8a312c9e5Cb4E1D24f6 - // EDGAR 0x678E7Ea3DCb5f4e9724C0e761843572f10c49B73 - // EDGAR 0xB9A2825327D5724C6E4b0e190E0a85dc7FCf10e5 - someValidator := &staking.Validator{} - someValidator.Address = addr - someValidator.Stake = big.NewInt(int64(sequence.Intn(100))) - switch B := common2.MustAddressToBech32(addr); true { - case B == a: - fmt.Println("Called here:", B, a) - return someValidator, nil - case B == b: - case B == c: - case B == d: - case B == e: - case B == f: - case B == g: - case B == h: - - } - - fmt.Println("EDGAR", addr.String()) - return nil, nil - // state, err := bc.StateAt(bc.CurrentBlock().Root()) - // if err != nil || state == nil { - // return nil, err - // } - // wrapper := state.GetStakingInfo(addr) - // if wrapper == nil { - // return nil, fmt.Errorf("ValidatorInformation not found: %v", addr) - // } - // return &wrapper.Validator, nil + return tempBank[addr], nil } // DelegatorsInformation returns up to date information of delegators of a given validator address diff --git a/internal/configs/sharding/localnet.go b/internal/configs/sharding/localnet.go index 0bbf18182..0e20ae7b2 100644 --- a/internal/configs/sharding/localnet.go +++ b/internal/configs/sharding/localnet.go @@ -17,7 +17,7 @@ const ( localnetV1Epoch = 1 localnetV2Epoch = 2 - localnetEpochBlock1 = 10 + localnetEpochBlock1 = 5 twoOne = 5 localnetVdfDifficulty = 5000 // This takes about 10s to finish the vdf diff --git a/internal/params/config.go b/internal/params/config.go index c75efc202..7848a9a0c 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -36,9 +36,10 @@ var ( ChainID: TestnetChainID, CrossTxEpoch: big.NewInt(0), CrossLinkEpoch: big.NewInt(0), - StakingEpoch: big.NewInt(5), - EIP155Epoch: big.NewInt(0), - S3Epoch: big.NewInt(0), + // MinEpoch needed is at least 1, crashes on 0 + StakingEpoch: big.NewInt(1), + EIP155Epoch: big.NewInt(0), + S3Epoch: big.NewInt(0), } // PangaeaChainConfig contains the chain parameters for the Pangaea network. diff --git a/numeric/decimal.go b/numeric/decimal.go index d09e3591c..631976d26 100644 --- a/numeric/decimal.go +++ b/numeric/decimal.go @@ -483,11 +483,6 @@ func (d Dec) RoundInt64() int64 { return chopped.Int64() } -// RoundInt round the decimal using bankers rounding -func (d Dec) RoundInt() *big.Int { - return chopPrecisionAndRoundNonMutative(d.Int) -} - //___________________________________________________________________________________ // similar to chopPrecisionAndRound, but always rounds down @@ -509,11 +504,6 @@ func (d Dec) TruncateInt64() int64 { return chopped.Int64() } -// TruncateInt truncates the decimals from the number and returns an Int -func (d Dec) TruncateInt() *big.Int { - return chopPrecisionAndTruncateNonMutative(d.Int) -} - // TruncateDec truncates the decimals from the number and returns a Dec func (d Dec) TruncateDec() Dec { return NewDecFromBigInt(chopPrecisionAndTruncateNonMutative(d.Int)) diff --git a/shard/committee/assignment.go b/shard/committee/assignment.go index 6bc3a78f9..e0c40143c 100644 --- a/shard/committee/assignment.go +++ b/shard/committee/assignment.go @@ -1,6 +1,7 @@ package committee import ( + "encoding/json" "fmt" "math/big" @@ -16,6 +17,10 @@ import ( staking "github.com/harmony-one/harmony/staking/types" ) +// StateID means reading off whole network when using calls that accept +// a shardID parameter +const StateID = -1 + // ValidatorListProvider .. type ValidatorListProvider interface { Compute( @@ -26,7 +31,12 @@ type ValidatorListProvider interface { // PublicKeysProvider per epoch type PublicKeysProvider interface { - ComputePublicKeys(epoch *big.Int, reader DataProvider) [][]*bls.PublicKey + // If call shardID with StateID then only superCommittee is non-nil, + // otherwise get back the shardSpecific slice as well. + ComputePublicKeys( + epoch *big.Int, reader DataProvider, shardID int, + ) (superCommittee, shardSpecific []*bls.PublicKey) + ReadPublicKeysFromDB( hash common.Hash, reader DataProvider, ) ([]*bls.PublicKey, error) @@ -113,13 +123,12 @@ func preStakingEnabledCommittee(s shardingconfig.Instance) shard.State { return shardState } -func eposStakedCommittee( - s shardingconfig.Instance, stakerReader DataProvider, stakedSlotsCount int, +func with400Stakers( + s shardingconfig.Instance, stakerReader DataProvider, ) (shard.State, error) { // TODO Nervous about this because overtime the list will become quite large candidates := stakerReader.ValidatorCandidates() - essentials := map[common.Address]effective.SlotOrder{} - + stakers := make([]*staking.Validator, len(candidates)) // TODO benchmark difference if went with data structure that sorts on insert for i := range candidates { // TODO Should be using .ValidatorStakingWithDelegation, not implemented yet @@ -133,74 +142,152 @@ func eposStakedCommittee( } } + for i := range stakers { + staker := stakers[i] + stakers[i].Stake = new(big.Int).Div( + staker.Stake, big.NewInt(int64(len(staker.SlotPubKeys))), + ) + } + + unsortedStakes := make([]int, len(stakers)) + eposStakes := make([]*big.Int, len(stakers)) + + for i, j := range stakers { + unsortedStakes[i] = int(j.Stake.Int64()) + eposStakes[i] = j.Stake + } + + s3 := effective.Apply(eposStakes) + + sort.SliceStable( + stakers, + func(i, j int) bool { return stakers[i].Stake.Cmp(stakers[j].Stake) >= 0 }, + ) + + // for i, j := range stakers { + // sortedStakes[i] = int(j.Stake.Int64()) + // } + + type t struct { + Stakes []int + } + + t2 := t{make([]int, len(eposStakes))} + for i := range s3 { + t2.Stakes[i] = int(s3[i].TruncateInt64()) + } + + s1, _ := json.Marshal(t{unsortedStakes}) + s2, _ := json.Marshal(t2) + + fmt.Println("Unsorted") + fmt.Println(string(s1)) + fmt.Println("as EPOS") + fmt.Println(string(s2)) + + // fmt.Println("Sorted stakers %+v\n", stakers) + shardCount := int(s.NumShards()) superComm := make(shard.State, shardCount) - hAccounts := s.HmyAccounts() + fillCount := make([]int, shardCount) for i := 0; i < shardCount; i++ { - superComm[i] = shard.Committee{uint32(i), shard.NodeIDList{}} - } - - for i := range hAccounts { - spot := i % shardCount - pub := &bls.PublicKey{} - pub.DeserializeHexStr(hAccounts[i].BlsPublicKey) - pubKey := shard.BlsPublicKey{} - pubKey.FromLibBLSPublicKey(pub) - superComm[spot].NodeList = append(superComm[spot].NodeList, shard.NodeID{ - common2.ParseAddr(hAccounts[i].Address), - pubKey, - nil, - }) + superComm[i] = shard.Committee{ + uint32(i), make(shard.NodeIDList, s.NumNodesPerShard()), + } } - staked := effective.Apply(essentials, stakedSlotsCount) - shardBig := big.NewInt(int64(shardCount)) + shardBig := big.NewInt(int64(s.NumShards())) - if l := len(staked); l < stakedSlotsCount { - // WARN unlikely to happen in production but will happen as we are developing - stakedSlotsCount = l + for i := 0; i < len(s.FnAccounts()); i++ { + bucket := int(new(big.Int).Mod(stakers[i].Address.Big(), shardBig).Int64()) + org := stakers[i].Stake + epos := big.NewInt(s3[i].TruncateInt64()) + fmt.Println("stakes", org, epos) + superComm[bucket].NodeList[fillCount[bucket]] = shard.NodeID{ + stakers[i].Address, + stakers[i].SlotPubKeys[0], + epos, + } + fillCount[bucket]++ } - for i := 0; i < stakedSlotsCount; i++ { - bucket := int(new(big.Int).Mod(staked[i].BlsPublicKey.Big(), shardBig).Int64()) - slot := staked[i] - superComm[bucket].NodeList = append(superComm[bucket].NodeList, shard.NodeID{ - slot.Address, - staked[i].BlsPublicKey, - &slot.Dec, - }) + hAccounts := s.HmyAccounts() + offset := 0 + + for i := range fillCount { + missing := s.NumNodesPerShard() - fillCount[i] + for j := 0; j < missing; j++ { + pub := &bls.PublicKey{} + pub.DeserializeHexStr(hAccounts[offset].BlsPublicKey) + pubKey := shard.BlsPublicKey{} + pubKey.FromLibBLSPublicKey(pub) + superComm[i].NodeList[fillCount[i]+j] = shard.NodeID{ + common2.ParseAddr(hAccounts[offset].Address), + pubKey, + nil, + } + offset++ + } } + fmt.Println("Final", superComm.JSON()) + fmt.Println("stakers", fillCount) return superComm, nil } -// ComputePublicKeys produces publicKeys of entire supercommittee per epoch +// ComputePublicKeys produces publicKeys of entire supercommittee per epoch, optionally providing a +// shard specific subcommittee func (def partialStakingEnabled) ComputePublicKeys( - epoch *big.Int, d DataProvider, -) [][]*bls.PublicKey { - + epoch *big.Int, d DataProvider, shardID int, +) ([]*bls.PublicKey, []*bls.PublicKey) { config := d.Config() instance := shard.Schedule.InstanceForEpoch(epoch) superComm := shard.State{} + if config.IsStaking(epoch) { - superComm, _ = eposStakedCommittee(instance, d, 320) + superComm, _ = with400Stakers(instance, d) } else { superComm = preStakingEnabledCommittee(instance) } - allIdentities := make([][]*bls.PublicKey, len(superComm)) + spot := 0 + shouldBe := int(instance.NumShards()) * instance.NumNodesPerShard() + + total := 0 + for i := range superComm { + total += len(superComm[i].NodeList) + } + + if shouldBe != total { + fmt.Println("Count mismatch", shouldBe, total) + } + allIdentities := make([]*bls.PublicKey, shouldBe) for i := range superComm { - allIdentities[i] = make([]*bls.PublicKey, len(superComm[i].NodeList)) for j := range superComm[i].NodeList { identity := &bls.PublicKey{} superComm[i].NodeList[j].BlsPublicKey.ToLibBLSPublicKey(identity) - allIdentities[i][j] = identity + allIdentities[spot] = identity + spot++ } } - return allIdentities + if shardID == StateID { + return allIdentities, nil + } + + subCommittee := superComm.FindCommitteeByID(uint32(shardID)) + subCommitteeIdentities := make([]*bls.PublicKey, len(subCommittee.NodeList)) + spot = 0 + for i := range subCommittee.NodeList { + identity := &bls.PublicKey{} + subCommittee.NodeList[i].BlsPublicKey.ToLibBLSPublicKey(identity) + subCommitteeIdentities[spot] = identity + spot++ + } + + return allIdentities, subCommitteeIdentities } func (def partialStakingEnabled) ReadPublicKeysFromDB( diff --git a/staking/effective/calculate.go b/staking/effective/calculate.go index 96dca579e..448a492f5 100644 --- a/staking/effective/calculate.go +++ b/staking/effective/calculate.go @@ -1,105 +1,54 @@ package effective import ( - "encoding/json" + "fmt" "math/big" "sort" - "github.com/ethereum/go-ethereum/common" "github.com/harmony-one/harmony/numeric" - "github.com/harmony-one/harmony/shard" ) // medium.com/harmony-one/introducing-harmonys-effective-proof-of-stake-epos-2d39b4b8d58 var ( - two = numeric.NewDecFromBigInt(big.NewInt(2)) c, _ = numeric.NewDecFromStr("0.15") onePlusC = numeric.OneDec().Add(c) oneMinusC = numeric.OneDec().Sub(c) ) -func effectiveStake(median, actual numeric.Dec) numeric.Dec { - left := numeric.MinDec(onePlusC.Mul(median), actual) - right := oneMinusC.Mul(median) +// Stake computes the effective proof of stake as descibed in whitepaper +func Stake(median, actual *big.Int) numeric.Dec { + medianDec := numeric.NewDecFromBigInt(median) + actualDec := numeric.NewDecFromBigInt(actual) + left := numeric.MinDec(onePlusC.Mul(medianDec), actualDec) + right := oneMinusC.Mul(medianDec) return numeric.MaxDec(left, right) } -// SlotPurchase .. -type SlotPurchase struct { - common.Address `json:"slot-owner"` - shard.BlsPublicKey `json:"bls-public-key"` - numeric.Dec `json:"eposed-stake"` -} - -// SlotOrder .. -type SlotOrder struct { - Stake *big.Int - SpreadAmong []shard.BlsPublicKey -} - -// Slots .. -type Slots []SlotPurchase - -// JSON is a plain JSON dump -func (s Slots) JSON() string { - type t struct { - Slots []SlotPurchase `json:"slots"` - } - b, _ := json.Marshal(t{s}) - return string(b) -} - -func median(stakes []SlotPurchase) numeric.Dec { +// Median find the median stake +func Median(stakes []*big.Int) *big.Int { sort.SliceStable( stakes, - func(i, j int) bool { return stakes[i].Dec.LTE(stakes[j].Dec) }, + func(i, j int) bool { return stakes[i].Cmp(stakes[j]) <= 0 }, ) const isEven = 0 switch l := len(stakes); l % 2 { case isEven: - return stakes[(l/2)-1].Dec.Add(stakes[(l/2)+1].Dec).Quo(two) + middle := new(big.Int).Add(stakes[(l/2)-1], stakes[(l/2)+1]) + return new(big.Int).Div(middle, big.NewInt(2)) default: - return stakes[l/2].Dec + return stakes[l/2] } } // Apply .. -func Apply(shortHand map[common.Address]SlotOrder, pull int) Slots { - eposedSlots := Slots{} - if len(shortHand) == 0 { - return eposedSlots - } - // Expand - for staker := range shortHand { - slotsCount := len(shortHand[staker].SpreadAmong) - spread := numeric.NewDecFromBigInt(shortHand[staker].Stake). - QuoInt64(int64(slotsCount)) - for i := 0; i < slotsCount; i++ { - eposedSlots = append(eposedSlots, SlotPurchase{ - staker, - shortHand[staker].SpreadAmong[i], - spread, - }) - } - } - if len(eposedSlots) < len(shortHand) { - // WARN Should never happen - } - - sort.SliceStable( - eposedSlots, - func(i, j int) bool { return eposedSlots[i].Dec.GT(eposedSlots[j].Dec) }, - ) +func Apply(stakes []*big.Int) []numeric.Dec { + asNumeric := make([]numeric.Dec, len(stakes)) + median := Median(stakes) - if l := len(eposedSlots); l < pull { - pull = l - } - picks := eposedSlots[:pull] - median := median(picks) + fmt.Println("Median is:", median.Uint64()) - for i := range picks { - picks[i].Dec = effectiveStake(median, picks[i].Dec) + for i := range stakes { + asNumeric[i] = Stake(median, stakes[i]) } - - return picks + return asNumeric } diff --git a/staking/effective/values.go b/staking/effective/values.go new file mode 100644 index 000000000..6e0840dc4 --- /dev/null +++ b/staking/effective/values.go @@ -0,0 +1,20 @@ +package effective + +import "github.com/harmony-one/harmony/numeric" + +const ( + // ValidatorsPerShard .. + ValidatorsPerShard = 400 + // AllValidatorsCount .. + AllValidatorsCount = ValidatorsPerShard * 4 +) + +// StakeKeeper .. +type StakeKeeper interface { + // Activity + Inventory() struct { + BLSPublicKeys [][48]byte `json:"bls_pubkey"` + WithDelegationApplied []numeric.Dec `json:"with-delegation-applied,omitempty"` + // CurrentlyActive []bool + } +} diff --git a/staking/types/validator.go b/staking/types/validator.go index 9d7d93458..ce387c001 100644 --- a/staking/types/validator.go +++ b/staking/types/validator.go @@ -198,12 +198,11 @@ func CreateValidatorFromNewMsg(val *CreateValidator, blockNum *big.Int) (*Valida commission := Commission{val.CommissionRates, blockNum} pubKeys := []shard.BlsPublicKey{} pubKeys = append(pubKeys, val.SlotPubKeys...) - - // TODO: a new validator should have a minimum of 1 token as self delegation, and that should be added as a delegation entry here. v := Validator{ val.ValidatorAddress, pubKeys, val.Amount, new(big.Int), val.MinSelfDelegation, val.MaxTotalDelegation, false, - commission, desc, blockNum} + commission, desc, big.NewInt(0), + } return &v, nil } From ea7a4927e29cda290158e1ab26e1532e666f9a5f Mon Sep 17 00:00:00 2001 From: Edgar Aroutiounian Date: Thu, 14 Nov 2019 21:08:40 -0800 Subject: [PATCH 04/26] [EPoS] Implement EPoS at slot level --- core/blockchain.go | 35 ++++++--- shard/committee/assignment.go | 137 +++++++++++++-------------------- shard/shard_state.go | 3 +- staking/effective/calculate.go | 71 ++++++++++++----- staking/effective/values.go | 20 ----- 5 files changed, 133 insertions(+), 133 deletions(-) delete mode 100644 staking/effective/values.go diff --git a/core/blockchain.go b/core/blockchain.go index 54e27dd79..a00db112e 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -2441,11 +2441,9 @@ func (bc *BlockChain) CurrentValidatorAddresses() []common.Address { } const ( - blsKey1 = "9a010ea58079ddcd31d5d10ac85e9b5a7732972ee5478d7296091b2af96fcd673c49d6dd7a4a9e7acd2e4aab0add0e00" - blsKey2 = "9275355e60f8c78337d538eb15e3f8eda120c28635f996d8a13f544d118db32749bcc0ed02a795770141d979a8b9de14" // Pick big number for interesting looking one addresses amount = 400 - fixedRandomGen = 98765654323 + fixedRandomGen = 98765654323123134 fixedRandomGenStakeL = 40 fixedRandomGenStakeH = 150 ) @@ -2455,6 +2453,8 @@ var ( sequenceL = rand.New(rand.NewSource(42)) sequenceH = rand.New(rand.NewSource(84)) accountGenerator = rand.New(rand.NewSource(1337)) + blsKeyGen = rand.New(rand.NewSource(4040)) + blsSlotsGen = rand.New(rand.NewSource(8080)) ) var ( @@ -2467,7 +2467,7 @@ func init() { for i := 0; i < amount; i++ { addr := common.Address{} addr.SetBytes( - big.NewInt(int64(accountGenerator.Intn(fixedRandomGen))).Bytes(), + big.NewInt(int64(accountGenerator.Int63n(fixedRandomGen))).Bytes(), ) addrs[i] = addr someValidator := &staking.Validator{} @@ -2483,11 +2483,28 @@ func init() { someValidator.Stake = new(big.Int).Abs(big.NewInt(int64( (r % modBy) + low, ))) - k := shard.BlsPublicKey{} - j := bls.PublicKey{} - j.DeserializeHexStr(blsKey1) - k.FromLibBLSPublicKey(&j) - someValidator.SlotPubKeys = []shard.BlsPublicKey{k} + + slotsCount := blsSlotsGen.Intn(10) + if slotsCount <= 0 { + slotsCount *= -1 + slotsCount++ + } + pubKeys := make([]shard.BlsPublicKey, slotsCount) + + for i := 0; i < slotsCount; i++ { + k := shard.BlsPublicKey{} + j := bls.PublicKey{} + b := bytes.Buffer{} + b.Write(big.NewInt(int64(accountGenerator.Int63n(fixedRandomGen))).Bytes()) + b.Write(big.NewInt(int64(accountGenerator.Int63n(fixedRandomGen))).Bytes()) + b.Write(big.NewInt(int64(accountGenerator.Int63n(fixedRandomGen))).Bytes()) + b.Write(big.NewInt(int64(accountGenerator.Int63n(fixedRandomGen))).Bytes()) + j.Deserialize(b.Bytes()[:shard.PublicKeySizeInBytes]) + k.FromLibBLSPublicKey(&j) + pubKeys[i] = k + } + + someValidator.SlotPubKeys = pubKeys tempBank[addr] = someValidator } } diff --git a/shard/committee/assignment.go b/shard/committee/assignment.go index e0c40143c..5a72901ed 100644 --- a/shard/committee/assignment.go +++ b/shard/committee/assignment.go @@ -1,7 +1,6 @@ package committee import ( - "encoding/json" "fmt" "math/big" @@ -17,9 +16,11 @@ import ( staking "github.com/harmony-one/harmony/staking/types" ) -// StateID means reading off whole network when using calls that accept -// a shardID parameter -const StateID = -1 +const ( + // StateID means reading off whole network when using calls that accept + // a shardID parameter + StateID = -1 +) // ValidatorListProvider .. type ValidatorListProvider interface { @@ -123,12 +124,14 @@ func preStakingEnabledCommittee(s shardingconfig.Instance) shard.State { return shardState } -func with400Stakers( - s shardingconfig.Instance, stakerReader DataProvider, +func eposStakedCommittee( + s shardingconfig.Instance, stakerReader DataProvider, stakedSlotsCount int, ) (shard.State, error) { // TODO Nervous about this because overtime the list will become quite large candidates := stakerReader.ValidatorCandidates() - stakers := make([]*staking.Validator, len(candidates)) + essentials := map[common.Address]effective.SlotOrder{} + slotUsage := map[common.Address]int{} + // TODO benchmark difference if went with data structure that sorts on insert for i := range candidates { // TODO Should be using .ValidatorStakingWithDelegation, not implemented yet @@ -140,56 +143,14 @@ func with400Stakers( validator.Stake, validator.SlotPubKeys, } + slotUsage[validator.Address] = len(validator.SlotPubKeys) } - for i := range stakers { - staker := stakers[i] - stakers[i].Stake = new(big.Int).Div( - staker.Stake, big.NewInt(int64(len(staker.SlotPubKeys))), - ) - } - - unsortedStakes := make([]int, len(stakers)) - eposStakes := make([]*big.Int, len(stakers)) - - for i, j := range stakers { - unsortedStakes[i] = int(j.Stake.Int64()) - eposStakes[i] = j.Stake - } - - s3 := effective.Apply(eposStakes) - - sort.SliceStable( - stakers, - func(i, j int) bool { return stakers[i].Stake.Cmp(stakers[j].Stake) >= 0 }, - ) - - // for i, j := range stakers { - // sortedStakes[i] = int(j.Stake.Int64()) - // } - - type t struct { - Stakes []int - } - - t2 := t{make([]int, len(eposStakes))} - for i := range s3 { - t2.Stakes[i] = int(s3[i].TruncateInt64()) - } - - s1, _ := json.Marshal(t{unsortedStakes}) - s2, _ := json.Marshal(t2) - - fmt.Println("Unsorted") - fmt.Println(string(s1)) - fmt.Println("as EPOS") - fmt.Println(string(s2)) - - // fmt.Println("Sorted stakers %+v\n", stakers) - shardCount := int(s.NumShards()) + maxNodePerShard := s.NumNodesPerShard() superComm := make(shard.State, shardCount) fillCount := make([]int, shardCount) + hAccounts := s.HmyAccounts() for i := 0; i < shardCount; i++ { superComm[i] = shard.Committee{ @@ -197,42 +158,48 @@ func with400Stakers( } } - shardBig := big.NewInt(int64(s.NumShards())) - - for i := 0; i < len(s.FnAccounts()); i++ { - bucket := int(new(big.Int).Mod(stakers[i].Address.Big(), shardBig).Int64()) - org := stakers[i].Stake - epos := big.NewInt(s3[i].TruncateInt64()) - fmt.Println("stakes", org, epos) - superComm[bucket].NodeList[fillCount[bucket]] = shard.NodeID{ - stakers[i].Address, - stakers[i].SlotPubKeys[0], - epos, + for i := range hAccounts { + spot := i % shardCount + pub := &bls.PublicKey{} + pub.DeserializeHexStr(hAccounts[i].BlsPublicKey) + pubKey := shard.BlsPublicKey{} + pubKey.FromLibBLSPublicKey(pub) + superComm[spot].NodeList[fillCount[spot]] = shard.NodeID{ + common2.ParseAddr(hAccounts[i].Address), + pubKey, + nil, } - fillCount[bucket]++ + fillCount[spot]++ } - hAccounts := s.HmyAccounts() - offset := 0 + staked := effective.Apply(essentials) - for i := range fillCount { - missing := s.NumNodesPerShard() - fillCount[i] - for j := 0; j < missing; j++ { - pub := &bls.PublicKey{} - pub.DeserializeHexStr(hAccounts[offset].BlsPublicKey) - pubKey := shard.BlsPublicKey{} - pubKey.FromLibBLSPublicKey(pub) - superComm[i].NodeList[fillCount[i]+j] = shard.NodeID{ - common2.ParseAddr(hAccounts[offset].Address), - pubKey, - nil, + sort.SliceStable( + staked, + func(i, j int) bool { return staked[i].Dec.GTE(staked[j].Dec) }, + ) + + shardBig := big.NewInt(int64(shardCount)) + + for i := 0; i < stakedSlotsCount; i++ { + bucket := int(new(big.Int).Mod(staked[i].Address.Big(), shardBig).Int64()) + slot := staked[i] + pubKey := essentials[slot.Address].SpreadAmong[slotUsage[slot.Address]-1] + slotUsage[slot.Address]-- + // Keep going round till find an open spot + for j := bucket; ; j = (j + 1) % shardCount { + if fillCount[j] != maxNodePerShard { + superComm[j].NodeList[fillCount[j]] = shard.NodeID{ + slot.Address, + pubKey, + &slot.Dec, + } + fillCount[j]++ + break } - offset++ } } - fmt.Println("Final", superComm.JSON()) - fmt.Println("stakers", fillCount) return superComm, nil } @@ -244,9 +211,12 @@ func (def partialStakingEnabled) ComputePublicKeys( config := d.Config() instance := shard.Schedule.InstanceForEpoch(epoch) superComm := shard.State{} + stakedSlots := + (instance.NumNodesPerShard() - instance.NumHarmonyOperatedNodesPerShard()) * + int(instance.NumShards()) if config.IsStaking(epoch) { - superComm, _ = with400Stakers(instance, d) + superComm, _ = eposStakedCommittee(instance, d, stakedSlots) } else { superComm = preStakingEnabledCommittee(instance) } @@ -336,6 +306,9 @@ func (def partialStakingEnabled) Compute( if !config.IsStaking(epoch) { return preStakingEnabledCommittee(instance), nil } + stakedSlots := + (instance.NumNodesPerShard() - instance.NumHarmonyOperatedNodesPerShard()) * + int(instance.NumShards()) fmt.Println("Staking epoch happened", config.String()) - return with400Stakers(instance, stakerReader) + return eposStakedCommittee(instance, stakerReader, stakedSlots) } diff --git a/shard/shard_state.go b/shard/shard_state.go index 33ad597ac..ef52b53b2 100644 --- a/shard/shard_state.go +++ b/shard/shard_state.go @@ -4,7 +4,6 @@ import ( "bytes" "encoding/hex" "encoding/json" - "math/big" "sort" "github.com/ethereum/go-ethereum/common" @@ -39,7 +38,7 @@ type NodeID struct { EcdsaAddress common.Address `json:"ecdsa-address"` BlsPublicKey BlsPublicKey `json:"bls-pubkey"` // nil means not active, 0 means our node, >= 0 means staked node - StakeWithDelegationApplied *big.Int `json:"staked-validator" rlp:"nil"` + StakeWithDelegationApplied *numeric.Dec `json:"staked-validator" rlp:"nil"` } // NodeIDList is a list of NodeIDList. diff --git a/staking/effective/calculate.go b/staking/effective/calculate.go index 448a492f5..6cf173635 100644 --- a/staking/effective/calculate.go +++ b/staking/effective/calculate.go @@ -1,54 +1,85 @@ package effective import ( - "fmt" "math/big" "sort" + "github.com/ethereum/go-ethereum/common" "github.com/harmony-one/harmony/numeric" + "github.com/harmony-one/harmony/shard" ) // medium.com/harmony-one/introducing-harmonys-effective-proof-of-stake-epos-2d39b4b8d58 var ( + two = numeric.NewDecFromBigInt(big.NewInt(2)) c, _ = numeric.NewDecFromStr("0.15") onePlusC = numeric.OneDec().Add(c) oneMinusC = numeric.OneDec().Sub(c) ) -// Stake computes the effective proof of stake as descibed in whitepaper -func Stake(median, actual *big.Int) numeric.Dec { - medianDec := numeric.NewDecFromBigInt(median) - actualDec := numeric.NewDecFromBigInt(actual) - left := numeric.MinDec(onePlusC.Mul(medianDec), actualDec) - right := oneMinusC.Mul(medianDec) +func stake(median, actual numeric.Dec) numeric.Dec { + left := numeric.MinDec(onePlusC.Mul(median), actual) + right := oneMinusC.Mul(median) return numeric.MaxDec(left, right) } -// Median find the median stake -func Median(stakes []*big.Int) *big.Int { +// SlotPurchase .. +type SlotPurchase struct { + common.Address `json:"slot-owner"` + numeric.Dec `json:"eposed-stake"` +} + +// SlotOrder .. +type SlotOrder struct { + Stake *big.Int + SpreadAmong []shard.BlsPublicKey +} + +// Slots .. +type Slots []SlotPurchase + +func median(stakes []SlotPurchase) numeric.Dec { sort.SliceStable( stakes, - func(i, j int) bool { return stakes[i].Cmp(stakes[j]) <= 0 }, + func(i, j int) bool { return stakes[i].Dec.LTE(stakes[j].Dec) }, ) const isEven = 0 switch l := len(stakes); l % 2 { case isEven: - middle := new(big.Int).Add(stakes[(l/2)-1], stakes[(l/2)+1]) - return new(big.Int).Div(middle, big.NewInt(2)) + return stakes[(l/2)-1].Dec.Add(stakes[(l/2)+1].Dec).Quo(two) default: - return stakes[l/2] + return stakes[l/2].Dec } } +// MarshalJSON .. +func (p Slots) MarshalJSON() ([]byte, error) { + return nil, nil +} + // Apply .. -func Apply(stakes []*big.Int) []numeric.Dec { - asNumeric := make([]numeric.Dec, len(stakes)) - median := Median(stakes) +func Apply(shortHand map[common.Address]SlotOrder) Slots { + eposedSlots := Slots{} + + // Expand + for staker := range shortHand { + slotsCount := int64(len(shortHand[staker].SpreadAmong)) + + var i int64 = 0 + spread := numeric.NewDecFromBigInt(shortHand[staker].Stake).QuoInt64(slotsCount) + for ; i < slotsCount; i++ { + eposedSlots = append(eposedSlots, SlotPurchase{staker, spread}) + } + } + if len(eposedSlots) < len(shortHand) { + // WARN Should never happen + } - fmt.Println("Median is:", median.Uint64()) + median := median(eposedSlots) - for i := range stakes { - asNumeric[i] = Stake(median, stakes[i]) + for i := range eposedSlots { + eposedSlots[i].Dec = stake(median, eposedSlots[i].Dec) } - return asNumeric + + return eposedSlots } diff --git a/staking/effective/values.go b/staking/effective/values.go deleted file mode 100644 index 6e0840dc4..000000000 --- a/staking/effective/values.go +++ /dev/null @@ -1,20 +0,0 @@ -package effective - -import "github.com/harmony-one/harmony/numeric" - -const ( - // ValidatorsPerShard .. - ValidatorsPerShard = 400 - // AllValidatorsCount .. - AllValidatorsCount = ValidatorsPerShard * 4 -) - -// StakeKeeper .. -type StakeKeeper interface { - // Activity - Inventory() struct { - BLSPublicKeys [][48]byte `json:"bls_pubkey"` - WithDelegationApplied []numeric.Dec `json:"with-delegation-applied,omitempty"` - // CurrentlyActive []bool - } -} From 3947adf366023186a37d717e56c878a508105885 Mon Sep 17 00:00:00 2001 From: Edgar Aroutiounian Date: Fri, 15 Nov 2019 07:30:40 -0800 Subject: [PATCH 05/26] [shard] Remove two-value loop --- shard/shard_state.go | 2 +- staking/effective/calculate.go | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/shard/shard_state.go b/shard/shard_state.go index ef52b53b2..edf632ded 100644 --- a/shard/shard_state.go +++ b/shard/shard_state.go @@ -54,7 +54,7 @@ type Committee struct { func (ss State) JSON() string { type t struct { NodeID - EcdsaAddress string `json:"ecdsa-address"` + EcdsaAddress string `json:"one-address"` } type v struct { Committee diff --git a/staking/effective/calculate.go b/staking/effective/calculate.go index 6cf173635..f7e20baf8 100644 --- a/staking/effective/calculate.go +++ b/staking/effective/calculate.go @@ -52,11 +52,6 @@ func median(stakes []SlotPurchase) numeric.Dec { } } -// MarshalJSON .. -func (p Slots) MarshalJSON() ([]byte, error) { - return nil, nil -} - // Apply .. func Apply(shortHand map[common.Address]SlotOrder) Slots { eposedSlots := Slots{} From 095be41b562e7123c812d3564b6a5c65b072d12b Mon Sep 17 00:00:00 2001 From: Edgar Aroutiounian Date: Fri, 15 Nov 2019 07:36:02 -0800 Subject: [PATCH 06/26] [epos] Remove test code generation - use commit for testing later --- core/blockchain.go | 88 ++++++++-------------------------------------- 1 file changed, 14 insertions(+), 74 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index a00db112e..0a69d5abb 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -18,12 +18,10 @@ package core import ( - "bytes" "errors" "fmt" "io" "math/big" - "math/rand" "sync" "sync/atomic" "time" @@ -37,7 +35,6 @@ import ( "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" - "github.com/harmony-one/bls/ffi/go/bls" "github.com/harmony-one/harmony/block" consensus_engine "github.com/harmony-one/harmony/consensus/engine" "github.com/harmony-one/harmony/core/rawdb" @@ -2440,83 +2437,26 @@ func (bc *BlockChain) CurrentValidatorAddresses() []common.Address { return filtered } -const ( - // Pick big number for interesting looking one addresses - amount = 400 - fixedRandomGen = 98765654323123134 - fixedRandomGenStakeL = 40 - fixedRandomGenStakeH = 150 -) - -var ( - // By fixing the source, we have predicable sequence - sequenceL = rand.New(rand.NewSource(42)) - sequenceH = rand.New(rand.NewSource(84)) - accountGenerator = rand.New(rand.NewSource(1337)) - blsKeyGen = rand.New(rand.NewSource(4040)) - blsSlotsGen = rand.New(rand.NewSource(8080)) -) - -var ( - tempBank map[common.Address]*staking.Validator = map[common.Address]*staking.Validator{} - addrs []common.Address -) - -func init() { - addrs = make([]common.Address, amount) - for i := 0; i < amount; i++ { - addr := common.Address{} - addr.SetBytes( - big.NewInt(int64(accountGenerator.Int63n(fixedRandomGen))).Bytes(), - ) - addrs[i] = addr - someValidator := &staking.Validator{} - someValidator.Address = addr - low := sequenceL.Intn(fixedRandomGenStakeL) - high := sequenceL.Intn(fixedRandomGenStakeH) - r := sequenceL.Intn(fixedRandomGenStakeH) + 1 - modBy := high - low + 1 - if modBy <= 0 { - modBy *= -1 - modBy++ - } - someValidator.Stake = new(big.Int).Abs(big.NewInt(int64( - (r % modBy) + low, - ))) - - slotsCount := blsSlotsGen.Intn(10) - if slotsCount <= 0 { - slotsCount *= -1 - slotsCount++ - } - pubKeys := make([]shard.BlsPublicKey, slotsCount) - - for i := 0; i < slotsCount; i++ { - k := shard.BlsPublicKey{} - j := bls.PublicKey{} - b := bytes.Buffer{} - b.Write(big.NewInt(int64(accountGenerator.Int63n(fixedRandomGen))).Bytes()) - b.Write(big.NewInt(int64(accountGenerator.Int63n(fixedRandomGen))).Bytes()) - b.Write(big.NewInt(int64(accountGenerator.Int63n(fixedRandomGen))).Bytes()) - b.Write(big.NewInt(int64(accountGenerator.Int63n(fixedRandomGen))).Bytes()) - j.Deserialize(b.Bytes()[:shard.PublicKeySizeInBytes]) - k.FromLibBLSPublicKey(&j) - pubKeys[i] = k - } - - someValidator.SlotPubKeys = pubKeys - tempBank[addr] = someValidator - } -} - // ValidatorCandidates returns the up to date validator candidates for next epoch func (bc *BlockChain) ValidatorCandidates() []common.Address { - return addrs + list, err := bc.ReadValidatorList() + if err != nil { + return make([]common.Address, 0) + } + return list } // ValidatorInformation returns the information of validator func (bc *BlockChain) ValidatorInformation(addr common.Address) (*staking.Validator, error) { - return tempBank[addr], nil + state, err := bc.StateAt(bc.CurrentBlock().Root()) + if err != nil || state == nil { + return nil, err + } + wrapper := state.GetStakingInfo(addr) + if wrapper == nil { + return nil, fmt.Errorf("ValidatorInformation not found: %v", addr) + } + return &wrapper.Validator, nil } // DelegatorsInformation returns up to date information of delegators of a given validator address From 8d76bf13ee923fb82cb132c1360b16ccd90dd323 Mon Sep 17 00:00:00 2001 From: Edgar Aroutiounian Date: Fri, 15 Nov 2019 07:49:44 -0800 Subject: [PATCH 07/26] [epos] Remove debug code, address lint --- internal/configs/sharding/localnet.go | 2 +- internal/params/config.go | 2 +- shard/committee/assignment.go | 5 ++++- staking/effective/calculate.go | 8 ++++---- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/internal/configs/sharding/localnet.go b/internal/configs/sharding/localnet.go index 0e20ae7b2..0bbf18182 100644 --- a/internal/configs/sharding/localnet.go +++ b/internal/configs/sharding/localnet.go @@ -17,7 +17,7 @@ const ( localnetV1Epoch = 1 localnetV2Epoch = 2 - localnetEpochBlock1 = 5 + localnetEpochBlock1 = 10 twoOne = 5 localnetVdfDifficulty = 5000 // This takes about 10s to finish the vdf diff --git a/internal/params/config.go b/internal/params/config.go index 7848a9a0c..c9c2a74d5 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -37,7 +37,7 @@ var ( CrossTxEpoch: big.NewInt(0), CrossLinkEpoch: big.NewInt(0), // MinEpoch needed is at least 1, crashes on 0 - StakingEpoch: big.NewInt(1), + StakingEpoch: EpochTBD, EIP155Epoch: big.NewInt(0), S3Epoch: big.NewInt(0), } diff --git a/shard/committee/assignment.go b/shard/committee/assignment.go index 5a72901ed..2c332e934 100644 --- a/shard/committee/assignment.go +++ b/shard/committee/assignment.go @@ -181,6 +181,10 @@ func eposStakedCommittee( shardBig := big.NewInt(int64(shardCount)) + if len(staked) <= stakedSlotsCount { + // WARN unlikely to happen in production but will happen as we are developing + } + for i := 0; i < stakedSlotsCount; i++ { bucket := int(new(big.Int).Mod(staked[i].Address.Big(), shardBig).Int64()) slot := staked[i] @@ -309,6 +313,5 @@ func (def partialStakingEnabled) Compute( stakedSlots := (instance.NumNodesPerShard() - instance.NumHarmonyOperatedNodesPerShard()) * int(instance.NumShards()) - fmt.Println("Staking epoch happened", config.String()) return eposStakedCommittee(instance, stakerReader, stakedSlots) } diff --git a/staking/effective/calculate.go b/staking/effective/calculate.go index f7e20baf8..742382afb 100644 --- a/staking/effective/calculate.go +++ b/staking/effective/calculate.go @@ -55,13 +55,14 @@ func median(stakes []SlotPurchase) numeric.Dec { // Apply .. func Apply(shortHand map[common.Address]SlotOrder) Slots { eposedSlots := Slots{} - + if len(shortHand) == 0 { + return eposedSlots + } // Expand for staker := range shortHand { slotsCount := int64(len(shortHand[staker].SpreadAmong)) - - var i int64 = 0 spread := numeric.NewDecFromBigInt(shortHand[staker].Stake).QuoInt64(slotsCount) + var i int64 for ; i < slotsCount; i++ { eposedSlots = append(eposedSlots, SlotPurchase{staker, spread}) } @@ -69,7 +70,6 @@ func Apply(shortHand map[common.Address]SlotOrder) Slots { if len(eposedSlots) < len(shortHand) { // WARN Should never happen } - median := median(eposedSlots) for i := range eposedSlots { From ac579e81d83a4ba315855885381923af104a1b42 Mon Sep 17 00:00:00 2001 From: Edgar Aroutiounian Date: Tue, 12 Nov 2019 22:20:42 -0800 Subject: [PATCH 08/26] [slashing] Create Slasher interface, thread through to Accumulate in reward --- consensus/engine/consensus_engine.go | 7 +++++++ consensus/quorum/quorum.go | 28 ++++++++++++++++++++++------ internal/chain/engine.go | 17 ++++++++++++++--- internal/chain/reward.go | 5 ++++- node/node.go | 3 +++ staking/slash/slasher.go | 8 ++++++++ 6 files changed, 58 insertions(+), 10 deletions(-) create mode 100644 staking/slash/slasher.go diff --git a/consensus/engine/consensus_engine.go b/consensus/engine/consensus_engine.go index 980b160c6..4ed63128e 100644 --- a/consensus/engine/consensus_engine.go +++ b/consensus/engine/consensus_engine.go @@ -10,6 +10,7 @@ import ( "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/internal/params" "github.com/harmony-one/harmony/shard" + "github.com/harmony-one/harmony/staking/slash" staking "github.com/harmony-one/harmony/staking/types" ) @@ -83,6 +84,12 @@ type Engine interface { // SetRewarder assigns the Distributor used in block reward SetRewarder(reward.Distributor) + // Slasher handles slashing accounts due to inavailibility or double-signing + Slasher() slash.Slasher + + // SetSlasher assigns the slasher used + SetSlasher(slash.Slasher) + // Finalize runs any post-transaction state modifications (e.g. block rewards) // and assembles the final block. // Note: The block header and state database might be updated to reflect any diff --git a/consensus/quorum/quorum.go b/consensus/quorum/quorum.go index 61d91cfc6..03a86acd0 100644 --- a/consensus/quorum/quorum.go +++ b/consensus/quorum/quorum.go @@ -223,27 +223,43 @@ func (s *cIdentities) ReadAllSignatures(p Phase) []*bls.Sign { return sigs } -func newMapBackedSignatureReader() SignatureReader { - return &cIdentities{ +func newMapBackedSignatureReader() cIdentities { + return cIdentities{ []*bls.PublicKey{}, map[string]*bls.Sign{}, map[string]*bls.Sign{}, map[string]*bls.Sign{}, } } +func (c *composite) ShouldSlash(shard.BlsPublicKey) bool { + s, _ := c.shardIDProvider() + switch s { + case shard.BeaconChainShardID: + return true + default: + return false + } +} + +type composite struct { + cIdentities + depInject +} + // NewDecider .. func NewDecider(p Policy) Decider { signatureStore := newMapBackedSignatureReader() - dependencies := &depInject{} + dependencies := depInject{} + c := &composite{signatureStore, dependencies} switch p { case SuperMajorityVote: - return &uniformVoteWeight{signatureStore, dependencies} + return &uniformVoteWeight{&c.cIdentities, &c.depInject} case SuperMajorityStake: return &stakedVoteWeight{ - signatureStore, - dependencies, + &c.cIdentities, &c.depInject, map[[shard.PublicKeySizeInBytes]byte]stakedVoter{}, big.NewInt(0), } + default: // Should not be possible return nil diff --git a/internal/chain/engine.go b/internal/chain/engine.go index ee6bc7289..0a1df170e 100644 --- a/internal/chain/engine.go +++ b/internal/chain/engine.go @@ -16,6 +16,7 @@ import ( "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/shard" "github.com/harmony-one/harmony/shard/committee" + "github.com/harmony-one/harmony/staking/slash" staking "github.com/harmony-one/harmony/staking/types" "github.com/pkg/errors" "golang.org/x/crypto/sha3" @@ -23,10 +24,11 @@ import ( type engineImpl struct { d reward.Distributor + s slash.Slasher } // Engine is an algorithm-agnostic consensus engine. -var Engine = &engineImpl{nil} +var Engine = &engineImpl{nil, nil} // Rewarder handles the distribution of block rewards func (e *engineImpl) Rewarder() reward.Distributor { @@ -38,6 +40,16 @@ func (e *engineImpl) SetRewarder(d reward.Distributor) { e.d = d } +// Slasher handles slashing accounts due to inavailibility or double-signing +func (e *engineImpl) Slasher() slash.Slasher { + return e.s +} + +// SetSlasher assigns the slasher used +func (e *engineImpl) SetSlasher(s slash.Slasher) { + e.s = s +} + // SealHash returns the hash of a block prior to it being sealed. func (e *engineImpl) SealHash(header *block.Header) (hash common.Hash) { hasher := sha3.NewLegacyKeccak256() @@ -170,8 +182,7 @@ func (e *engineImpl) Finalize( incxs []*types.CXReceiptsProof, stks []*staking.StakingTransaction) (*types.Block, error) { // Accumulate any block and uncle rewards and commit the final state root // Header seems complete, assemble into a block and return - // TODO: Block rewards should be done only in beacon chain based on cross-links - if err := AccumulateRewards(chain, state, header, e.Rewarder()); err != nil { + if err := AccumulateRewards(chain, state, header, e.Rewarder(), e.Slasher()); err != nil { return nil, ctxerror.New("cannot pay block reward").WithCause(err) } diff --git a/internal/chain/reward.go b/internal/chain/reward.go index c4f585161..37523a723 100644 --- a/internal/chain/reward.go +++ b/internal/chain/reward.go @@ -14,6 +14,7 @@ import ( common2 "github.com/harmony-one/harmony/internal/common" "github.com/harmony-one/harmony/internal/ctxerror" "github.com/harmony-one/harmony/internal/utils" + "github.com/harmony-one/harmony/staking/slash" "github.com/pkg/errors" ) @@ -27,7 +28,9 @@ var ( // reward. The total reward consists of the static block reward and rewards for // included uncles. The coinbase of each uncle block is also rewarded. func AccumulateRewards( - bc engine.ChainReader, state *state.DB, header *block.Header, rewarder reward.Distributor, + bc engine.ChainReader, state *state.DB, + header *block.Header, rewarder reward.Distributor, + slasher slash.Slasher, ) error { blockNum := header.Number().Uint64() if blockNum == 0 { diff --git a/node/node.go b/node/node.go index f65962b5b..a76ff6993 100644 --- a/node/node.go +++ b/node/node.go @@ -33,6 +33,7 @@ import ( p2p_host "github.com/harmony-one/harmony/p2p/host" "github.com/harmony-one/harmony/shard" "github.com/harmony-one/harmony/shard/committee" + "github.com/harmony-one/harmony/staking/slash" staking "github.com/harmony-one/harmony/staking/types" ) @@ -441,6 +442,8 @@ func New(host p2p.Host, consensusObj *consensus.Consensus, chainDBFactory shardc node.pendingStakingTransactions = make(map[common.Hash]*staking.StakingTransaction) node.Consensus.VerifiedNewBlock = make(chan *types.Block) chain.Engine.SetRewarder(node.Consensus.Decider.(reward.Distributor)) + chain.Engine.SetSlasher(node.Consensus.Decider.(slash.Slasher)) + // the sequence number is the next block number to be added in consensus protocol, which is always one more than current chain header block node.Consensus.SetBlockNum(blockchain.CurrentBlock().NumberU64() + 1) diff --git a/staking/slash/slasher.go b/staking/slash/slasher.go new file mode 100644 index 000000000..2d6da48e2 --- /dev/null +++ b/staking/slash/slasher.go @@ -0,0 +1,8 @@ +package slash + +import "github.com/harmony-one/harmony/shard" + +// Slasher .. +type Slasher interface { + ShouldSlash(shard.BlsPublicKey) bool +} From 91285cb0c3ef0752d526160074dcfd986d67c410 Mon Sep 17 00:00:00 2001 From: Edgar Aroutiounian Date: Wed, 13 Nov 2019 14:57:32 -0800 Subject: [PATCH 09/26] [slash] UpdateConsensusInformation has Staked based Quorum Decider, provide abstraction for inavailability count --- consensus/consensus_service.go | 19 ++++--- consensus/quorum/one-node-one-vote.go | 9 ++- consensus/quorum/one-node-staked-vote.go | 13 +++++ consensus/quorum/quorum.go | 70 +++++++++++++++--------- internal/chain/engine.go | 14 +++-- internal/chain/reward.go | 48 +++++++++++++--- staking/slash/slasher.go | 11 ++++ 7 files changed, 138 insertions(+), 46 deletions(-) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 278e4aea8..d206aa876 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -457,10 +457,13 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode { hasError := false header := consensus.ChainReader.CurrentHeader() epoch := header.Epoch() - curPubKeys := committee.WithStakingEnabled.ComputePublicKeys( - epoch, consensus.ChainReader, - )[int(header.ShardID())] - + if consensus.Decider.Policy() != quorum.SuperMajorityStake && + consensus.ChainReader.Config().IsStaking(epoch) { + consensus.Decider = quorum.NewDecider(quorum.SuperMajorityStake) + } + _, curPubKeys := committee.WithStakingEnabled.ComputePublicKeys( + epoch, consensus.ChainReader, int(header.ShardID()), + ) consensus.numPrevPubKeys = len(curPubKeys) consensus.getLogger().Info().Msg("[UpdateConsensusInformation] Updating.....") if shard.Schedule.IsLastBlock(header.Number().Uint64()) { @@ -468,9 +471,11 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode { consensus.SetEpochNum(epoch.Uint64() + 1) consensus.getLogger().Info().Uint64("headerNum", header.Number().Uint64()). Msg("[UpdateConsensusInformation] Epoch updated for next epoch") - pubKeys = committee.WithStakingEnabled.ComputePublicKeys( - new(big.Int).Add(epoch, common.Big1), consensus.ChainReader, - )[int(header.ShardID())] + _, pubKeys = committee.WithStakingEnabled.ComputePublicKeys( + new(big.Int).Add(epoch, common.Big1), + consensus.ChainReader, + int(header.ShardID()), + ) } else { consensus.SetEpochNum(epoch.Uint64()) pubKeys = curPubKeys diff --git a/consensus/quorum/one-node-one-vote.go b/consensus/quorum/one-node-one-vote.go index 8595a9c31..e1ca4bbbc 100644 --- a/consensus/quorum/one-node-one-vote.go +++ b/consensus/quorum/one-node-one-vote.go @@ -6,12 +6,13 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/harmony-one/bls/ffi/go/bls" "github.com/harmony-one/harmony/internal/utils" + "github.com/harmony-one/harmony/shard" // "github.com/harmony-one/harmony/staking/effective" ) type uniformVoteWeight struct { - SignatureReader DependencyInjectionWriter + SignatureReader } // Policy .. @@ -71,3 +72,9 @@ func (v *uniformVoteWeight) Award( return payout } + +func (v *uniformVoteWeight) ShouldSlash(k shard.BlsPublicKey) bool { + // No-op, no semantic meaning in one-slot-one-vote + // fmt.Println("Called here for key:", k.Hex()) + return false +} diff --git a/consensus/quorum/one-node-staked-vote.go b/consensus/quorum/one-node-staked-vote.go index efd2b385b..1d6eb9020 100644 --- a/consensus/quorum/one-node-staked-vote.go +++ b/consensus/quorum/one-node-staked-vote.go @@ -7,6 +7,7 @@ import ( "github.com/harmony-one/bls/ffi/go/bls" "github.com/harmony-one/harmony/numeric" "github.com/harmony-one/harmony/shard" + "github.com/harmony-one/harmony/staking/slash" ) var ( @@ -21,6 +22,8 @@ type stakedVoter struct { type stakedVoteWeight struct { SignatureReader DependencyInjectionWriter + DependencyInjectionReader + slash.ThresholdDecider // EPOS based staking validatorStakes map[[shard.PublicKeySizeInBytes]byte]stakedVoter totalEffectiveStakedAmount *big.Int @@ -73,3 +76,13 @@ func (v *stakedVoteWeight) ToggleActive(*bls.PublicKey) bool { // TODO Implement return true } + +func (v *stakedVoteWeight) ShouldSlash(key shard.BlsPublicKey) bool { + s, _ := v.ShardIDProvider()() + switch s { + case shard.BeaconChainShardID: + return v.SlashThresholdMet(key) + default: + return false + } +} diff --git a/consensus/quorum/quorum.go b/consensus/quorum/quorum.go index 03a86acd0..34ac736c7 100644 --- a/consensus/quorum/quorum.go +++ b/consensus/quorum/quorum.go @@ -6,6 +6,7 @@ import ( "github.com/harmony-one/bls/ffi/go/bls" "github.com/harmony-one/harmony/shard" + "github.com/harmony-one/harmony/staking/slash" // "github.com/harmony-one/harmony/staking/effective" ) @@ -75,10 +76,16 @@ type DependencyInjectionWriter interface { SetShardIDProvider(func() (uint32, error)) } +// DependencyInjectionReader .. +type DependencyInjectionReader interface { + ShardIDProvider() func() (uint32, error) +} + // Decider .. type Decider interface { SignatureReader DependencyInjectionWriter + slash.Slasher ToggleActive(*bls.PublicKey) bool // UpdateVotingPower(keeper effective.StakeKeeper) Policy() Policy @@ -96,17 +103,14 @@ type cIdentities struct { commit map[string]*bls.Sign // viewIDSigs: every validator // sign on |viewID|blockHash| in view changing message - viewID map[string]*bls.Sign + viewID map[string]*bls.Sign + seenCounter map[[shard.PublicKeySizeInBytes]byte]int } type depInject struct { shardIDProvider func() (uint32, error) } -func (d *depInject) SetShardIDProvider(p func() (uint32, error)) { - d.shardIDProvider = p -} - func (s *cIdentities) IndexOf(pubKey *bls.PublicKey) int { idx := -1 for k, v := range s.publicKeys { @@ -132,9 +136,23 @@ func (s *cIdentities) Participants() []*bls.PublicKey { } func (s *cIdentities) UpdateParticipants(pubKeys []*bls.PublicKey) { + // TODO - might need to put this in separate method + s.seenCounter = make(map[[shard.PublicKeySizeInBytes]byte]int, len(pubKeys)) + for i := range pubKeys { + k := shard.BlsPublicKey{} + k.FromLibBLSPublicKey(pubKeys[i]) + s.seenCounter[k] = 0 + } s.publicKeys = append(pubKeys[:0:0], pubKeys...) } +func (s *cIdentities) SlashThresholdMet(key shard.BlsPublicKey) bool { + s.seenCounter[key]++ + fmt.Println("Slash Map", s.seenCounter) + return s.seenCounter[key] == slash.UnavailabilityInConsecutiveBlockSigning + +} + func (s *cIdentities) DumpParticipants() []string { keys := make([]string, len(s.publicKeys)) for i := 0; i < len(s.publicKeys); i++ { @@ -174,8 +192,8 @@ func (s *cIdentities) AddSignature(p Phase, PubKey *bls.PublicKey, sig *bls.Sign } func (s *cIdentities) Reset(ps []Phase) { - for _, p := range ps { - switch m := map[string]*bls.Sign{}; p { + for i := range ps { + switch m := map[string]*bls.Sign{}; ps[i] { case Prepare: s.prepare = m case Commit: @@ -223,43 +241,45 @@ func (s *cIdentities) ReadAllSignatures(p Phase) []*bls.Sign { return sigs } -func newMapBackedSignatureReader() cIdentities { - return cIdentities{ +func newMapBackedSignatureReader() *cIdentities { + return &cIdentities{ []*bls.PublicKey{}, map[string]*bls.Sign{}, map[string]*bls.Sign{}, map[string]*bls.Sign{}, + map[[shard.PublicKeySizeInBytes]byte]int{}, } } -func (c *composite) ShouldSlash(shard.BlsPublicKey) bool { - s, _ := c.shardIDProvider() - switch s { - case shard.BeaconChainShardID: - return true - default: - return false - } +type composite struct { + DependencyInjectionWriter + SignatureReader } -type composite struct { - cIdentities - depInject +func (d *depInject) SetShardIDProvider(p func() (uint32, error)) { + d.shardIDProvider = p +} + +func (d *depInject) ShardIDProvider() func() (uint32, error) { + return d.shardIDProvider } // NewDecider .. func NewDecider(p Policy) Decider { signatureStore := newMapBackedSignatureReader() - dependencies := depInject{} - c := &composite{signatureStore, dependencies} + deps := &depInject{} + c := &composite{deps, signatureStore} switch p { case SuperMajorityVote: - return &uniformVoteWeight{&c.cIdentities, &c.depInject} + return &uniformVoteWeight{c.DependencyInjectionWriter, c} case SuperMajorityStake: + fmt.Println("HRS") return &stakedVoteWeight{ - &c.cIdentities, &c.depInject, + c.SignatureReader, + c.DependencyInjectionWriter, + c.DependencyInjectionWriter.(DependencyInjectionReader), + c.SignatureReader.(slash.ThresholdDecider), map[[shard.PublicKeySizeInBytes]byte]stakedVoter{}, big.NewInt(0), } - default: // Should not be possible return nil diff --git a/internal/chain/engine.go b/internal/chain/engine.go index 0a1df170e..75ec56964 100644 --- a/internal/chain/engine.go +++ b/internal/chain/engine.go @@ -177,12 +177,16 @@ func (e *engineImpl) VerifySeal(chain engine.ChainReader, header *block.Header) // Finalize implements Engine, accumulating the block rewards, // setting the final state and assembling the block. func (e *engineImpl) Finalize( - chain engine.ChainReader, header *block.Header, state *state.DB, txs []*types.Transaction, + chain engine.ChainReader, header *block.Header, + state *state.DB, txs []*types.Transaction, receipts []*types.Receipt, outcxs []*types.CXReceipt, - incxs []*types.CXReceiptsProof, stks []*staking.StakingTransaction) (*types.Block, error) { + incxs []*types.CXReceiptsProof, stks []*staking.StakingTransaction, +) (*types.Block, error) { // Accumulate any block and uncle rewards and commit the final state root // Header seems complete, assemble into a block and return - if err := AccumulateRewards(chain, state, header, e.Rewarder(), e.Slasher()); err != nil { + if err := AccumulateRewards( + chain, state, header, e.Rewarder(), e.Slasher(), + ); err != nil { return nil, ctxerror.New("cannot pay block reward").WithCause(err) } @@ -221,7 +225,9 @@ func (e *engineImpl) Finalize( } // QuorumForBlock returns the quorum for the given block header. -func QuorumForBlock(chain engine.ChainReader, h *block.Header, reCalculate bool) (quorum int, err error) { +func QuorumForBlock( + chain engine.ChainReader, h *block.Header, reCalculate bool, +) (quorum int, err error) { var ss shard.State if reCalculate { ss, _ = committee.WithStakingEnabled.Compute(h.Epoch(), *chain.Config(), nil) diff --git a/internal/chain/reward.go b/internal/chain/reward.go index 37523a723..9b56d4260 100644 --- a/internal/chain/reward.go +++ b/internal/chain/reward.go @@ -2,6 +2,7 @@ package chain import ( "math/big" + "sync" "github.com/ethereum/go-ethereum/common" "github.com/harmony-one/bls/ffi/go/bls" @@ -14,6 +15,7 @@ import ( common2 "github.com/harmony-one/harmony/internal/common" "github.com/harmony-one/harmony/internal/ctxerror" "github.com/harmony-one/harmony/internal/utils" + "github.com/harmony-one/harmony/shard" "github.com/harmony-one/harmony/staking/slash" "github.com/pkg/errors" ) @@ -82,29 +84,54 @@ func AccumulateRewards( } accounts := []common.Address{} + missing := shard.NodeIDList{} for idx, member := range parentCommittee.NodeList { - if signed, err := mask.IndexEnabled(idx); err != nil { + switch signed, err := mask.IndexEnabled(idx); true { + case err != nil: return ctxerror.New("cannot check for committer bit", "committerIndex", idx, ).WithCause(err) - } else if signed { + case signed: accounts = append(accounts, member.EcdsaAddress) + default: + missing = append(missing, member) } } - type t struct { + // do it quickly + w := sync.WaitGroup{} + for i := range missing { + w.Add(1) + go func(member int) { + defer w.Add(-1) + // Slash if missing block was long enough + if slasher.ShouldSlash(missing[member].BlsPublicKey) { + // TODO Logic + } + }(i) + } + + w.Wait() + + payable := []struct { + string common.Address *big.Int - } - signers := []string{} - payable := []t{} + }{} totalAmount := rewarder.Award( BlockReward, accounts, func(receipient common.Address, amount *big.Int) { - signers = append(signers, common2.MustAddressToBech32(receipient)) - payable = append(payable, t{receipient, amount}) - }) + payable = append(payable, struct { + string + common.Address + *big.Int + }{ + common2.MustAddressToBech32(receipient), receipient, amount, + }, + ) + }, + ) if totalAmount.Cmp(BlockReward) != 0 { utils.Logger().Error(). @@ -114,7 +141,10 @@ func AccumulateRewards( return errors.Wrapf(errPayoutNotEqualBlockReward, "payout "+totalAmount.String()) } + signers := make([]string, len(payable)) + for i := range payable { + signers[i] = payable[i].string state.AddBalance(payable[i].Address, payable[i].Int) } diff --git a/staking/slash/slasher.go b/staking/slash/slasher.go index 2d6da48e2..c6e1e4047 100644 --- a/staking/slash/slasher.go +++ b/staking/slash/slasher.go @@ -2,7 +2,18 @@ package slash import "github.com/harmony-one/harmony/shard" +const ( + // UnavailabilityInConsecutiveBlockSigning is how many blocks in a row + // before "slashing by unavailability" occurs + UnavailabilityInConsecutiveBlockSigning = 1380 +) + // Slasher .. type Slasher interface { ShouldSlash(shard.BlsPublicKey) bool } + +// ThresholdDecider .. +type ThresholdDecider interface { + SlashThresholdMet(shard.BlsPublicKey) bool +} From c1baaeaa06451d9b69bff8a958c7f9620a9349f1 Mon Sep 17 00:00:00 2001 From: Edgar Aroutiounian Date: Fri, 15 Nov 2019 08:16:36 -0800 Subject: [PATCH 10/26] Revert "[epos] Remove test code generation - use commit for testing later" This reverts commit 37770e3f33a2130383f08adb1ea74e71c1585502. --- core/blockchain.go | 88 ++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 74 insertions(+), 14 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index 0a69d5abb..a00db112e 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -18,10 +18,12 @@ package core import ( + "bytes" "errors" "fmt" "io" "math/big" + "math/rand" "sync" "sync/atomic" "time" @@ -35,6 +37,7 @@ import ( "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" + "github.com/harmony-one/bls/ffi/go/bls" "github.com/harmony-one/harmony/block" consensus_engine "github.com/harmony-one/harmony/consensus/engine" "github.com/harmony-one/harmony/core/rawdb" @@ -2437,26 +2440,83 @@ func (bc *BlockChain) CurrentValidatorAddresses() []common.Address { return filtered } +const ( + // Pick big number for interesting looking one addresses + amount = 400 + fixedRandomGen = 98765654323123134 + fixedRandomGenStakeL = 40 + fixedRandomGenStakeH = 150 +) + +var ( + // By fixing the source, we have predicable sequence + sequenceL = rand.New(rand.NewSource(42)) + sequenceH = rand.New(rand.NewSource(84)) + accountGenerator = rand.New(rand.NewSource(1337)) + blsKeyGen = rand.New(rand.NewSource(4040)) + blsSlotsGen = rand.New(rand.NewSource(8080)) +) + +var ( + tempBank map[common.Address]*staking.Validator = map[common.Address]*staking.Validator{} + addrs []common.Address +) + +func init() { + addrs = make([]common.Address, amount) + for i := 0; i < amount; i++ { + addr := common.Address{} + addr.SetBytes( + big.NewInt(int64(accountGenerator.Int63n(fixedRandomGen))).Bytes(), + ) + addrs[i] = addr + someValidator := &staking.Validator{} + someValidator.Address = addr + low := sequenceL.Intn(fixedRandomGenStakeL) + high := sequenceL.Intn(fixedRandomGenStakeH) + r := sequenceL.Intn(fixedRandomGenStakeH) + 1 + modBy := high - low + 1 + if modBy <= 0 { + modBy *= -1 + modBy++ + } + someValidator.Stake = new(big.Int).Abs(big.NewInt(int64( + (r % modBy) + low, + ))) + + slotsCount := blsSlotsGen.Intn(10) + if slotsCount <= 0 { + slotsCount *= -1 + slotsCount++ + } + pubKeys := make([]shard.BlsPublicKey, slotsCount) + + for i := 0; i < slotsCount; i++ { + k := shard.BlsPublicKey{} + j := bls.PublicKey{} + b := bytes.Buffer{} + b.Write(big.NewInt(int64(accountGenerator.Int63n(fixedRandomGen))).Bytes()) + b.Write(big.NewInt(int64(accountGenerator.Int63n(fixedRandomGen))).Bytes()) + b.Write(big.NewInt(int64(accountGenerator.Int63n(fixedRandomGen))).Bytes()) + b.Write(big.NewInt(int64(accountGenerator.Int63n(fixedRandomGen))).Bytes()) + j.Deserialize(b.Bytes()[:shard.PublicKeySizeInBytes]) + k.FromLibBLSPublicKey(&j) + pubKeys[i] = k + } + + someValidator.SlotPubKeys = pubKeys + tempBank[addr] = someValidator + } +} + // ValidatorCandidates returns the up to date validator candidates for next epoch func (bc *BlockChain) ValidatorCandidates() []common.Address { - list, err := bc.ReadValidatorList() - if err != nil { - return make([]common.Address, 0) - } - return list + return addrs } // ValidatorInformation returns the information of validator func (bc *BlockChain) ValidatorInformation(addr common.Address) (*staking.Validator, error) { - state, err := bc.StateAt(bc.CurrentBlock().Root()) - if err != nil || state == nil { - return nil, err - } - wrapper := state.GetStakingInfo(addr) - if wrapper == nil { - return nil, fmt.Errorf("ValidatorInformation not found: %v", addr) - } - return &wrapper.Validator, nil + return tempBank[addr], nil } // DelegatorsInformation returns up to date information of delegators of a given validator address From f1cf151ba14d9eef323e1a3d70bf92bd899c730c Mon Sep 17 00:00:00 2001 From: Edgar Aroutiounian Date: Fri, 15 Nov 2019 16:01:59 -0800 Subject: [PATCH 11/26] [slashing] Checkpoint to verify changing of committee works --- cmd/client/wallet/main.go | 2 +- cmd/harmony/main.go | 9 +- consensus/consensus.go | 4 + consensus/consensus_service.go | 9 ++ consensus/consensus_v2.go | 27 ++++ consensus/quorum/quorum.go | 1 - core/blockchain.go | 212 +++++++++++++++++++++++--- internal/configs/sharding/localnet.go | 2 +- internal/params/config.go | 2 +- node/node.go | 7 +- node/node_genesis.go | 2 + node/node_handler.go | 40 +++++ node/node_newblock.go | 11 ++ shard/committee/assignment.go | 3 +- shard/shard_state.go | 2 +- 15 files changed, 304 insertions(+), 29 deletions(-) diff --git a/cmd/client/wallet/main.go b/cmd/client/wallet/main.go index a639f64c8..15401e22f 100644 --- a/cmd/client/wallet/main.go +++ b/cmd/client/wallet/main.go @@ -304,7 +304,7 @@ func createWalletNode() *node.Node { panic(err) } chainDBFactory := &shardchain.MemDBFactory{} - w := node.New(host, nil, chainDBFactory, false) + w := node.New(host, nil, chainDBFactory, false, "") w.Client = client.NewClient(w.GetHost(), uint32(shardID)) w.NodeConfig.SetRole(nodeconfig.ClientNode) diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index 7d62c5c05..80e2c58a1 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -310,7 +310,11 @@ func setupConsensusAndNode(nodeConfig *nodeconfig.ConfigType) *node.Node { // Current node. chainDBFactory := &shardchain.LDBFactory{RootDir: nodeConfig.DBDir} - currentNode := node.New(myHost, currentConsensus, chainDBFactory, *isArchival) + // fmt.Println("What is my port at this moment", *port) + + currentNode := node.New( + myHost, currentConsensus, chainDBFactory, *isArchival, *port, + ) switch { case *networkType == nodeconfig.Localnet: @@ -340,7 +344,6 @@ func setupConsensusAndNode(nodeConfig *nodeconfig.ConfigType) *node.Node { currentNode.NodeConfig.SetPushgatewayIP(nodeConfig.PushgatewayIP) currentNode.NodeConfig.SetPushgatewayPort(nodeConfig.PushgatewayPort) currentNode.NodeConfig.SetMetricsFlag(nodeConfig.MetricsFlag) - currentNode.NodeConfig.SetBeaconGroupID(nodeconfig.NewGroupIDByShardID(0)) switch *nodeType { @@ -496,6 +499,8 @@ func main() { currentNode.ServiceManagerSetup() currentNode.RunServices() + // fmt.Println("CurrentRPC-port", *port) + // RPC for SDK not supported for mainnet. if err := currentNode.StartRPC(*port); err != nil { utils.Logger().Warn(). diff --git a/consensus/consensus.go b/consensus/consensus.go index fc59d5a2c..43b9c7e19 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -160,6 +160,10 @@ func (consensus *Consensus) SetCommitDelay(delay time.Duration) { consensus.delayCommit = delay } +func (consensus *Consensus) SetPrivateKey(priKey *bls.SecretKey) { + consensus.priKey = priKey +} + // DisableViewChangeForTestingOnly makes the receiver not propose view // changes when it should, e.g. leader timeout. // diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index d206aa876..ba8b2f1ab 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -119,6 +119,7 @@ func (consensus *Consensus) UpdatePublicKeys(pubKeys []*bls.PublicKey) int64 { for i := range pubKeys { utils.Logger().Info().Int("index", i).Str("BLSPubKey", pubKeys[i].SerializeToHexStr()).Msg("Member") } + consensus.LeaderPubKey = pubKeys[0] utils.Logger().Info(). Str("info", consensus.LeaderPubKey.SerializeToHexStr()).Msg("My Leader") @@ -457,8 +458,16 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode { hasError := false header := consensus.ChainReader.CurrentHeader() epoch := header.Epoch() + + // fmt.Println("update-consensus", + // epoch, + // consensus.Decider.Policy() != quorum.SuperMajorityStake, + // consensus.ChainReader.Config().IsStaking(epoch), + // ) + if consensus.Decider.Policy() != quorum.SuperMajorityStake && consensus.ChainReader.Config().IsStaking(epoch) { + fmt.Println("Hit new decider on quorum") consensus.Decider = quorum.NewDecider(quorum.SuperMajorityStake) } _, curPubKeys := committee.WithStakingEnabled.ComputePublicKeys( diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index d32457a97..0e025a943 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/binary" "encoding/hex" + "fmt" "time" "github.com/ethereum/go-ethereum/common" @@ -315,11 +316,37 @@ func (consensus *Consensus) onPrepare(msg *msg_pb.Message) { } senderKey, err := consensus.verifySenderKey(msg) + // fmt.Println("prepare-sender-key", senderKey.SerializeToHexStr()) + + // for _, v := range consensus.Decider.DumpParticipants() { + // fmt.Println("in-committee-", consensus.ShardID, v) + // } + if err != nil { + fmt.Println("On Prepare is busted =/") + + // type t struct { + // Participants []string `json:"committee-members"` + // ShardID uint32 `json:"shard-id"` + // } + // b, _ := json.Marshal(t{consensus.Decider.DumpParticipants(), consensus.ShardID}) + // fmt.Println(string(b)) + utils.Logger().Error().Err(err).Msg("[OnPrepare] VerifySenderKey failed") return } + if err = verifyMessageSig(senderKey, msg); err != nil { + fmt.Println("unable to verify message sig") + + // p := consensus.Decider.DumpParticipants() + // for _, k := range p { + // if senderKey.SerializeToHexStr() == k { + // fmt.Println("Sender is in my committee quorum", consensus.PubKey.SerializeToHexStr()) + // break + // } + // } + // fmt.Println("Bad sender?", senderKey.SerializeToHexStr()) utils.Logger().Error().Err(err).Msg("[OnPrepare] Failed to verify sender's signature") return } diff --git a/consensus/quorum/quorum.go b/consensus/quorum/quorum.go index 34ac736c7..3e816bbc9 100644 --- a/consensus/quorum/quorum.go +++ b/consensus/quorum/quorum.go @@ -271,7 +271,6 @@ func NewDecider(p Policy) Decider { case SuperMajorityVote: return &uniformVoteWeight{c.DependencyInjectionWriter, c} case SuperMajorityStake: - fmt.Println("HRS") return &stakedVoteWeight{ c.SignatureReader, c.DependencyInjectionWriter, diff --git a/core/blockchain.go b/core/blockchain.go index a00db112e..2d53e650a 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -18,7 +18,6 @@ package core import ( - "bytes" "errors" "fmt" "io" @@ -2442,7 +2441,6 @@ func (bc *BlockChain) CurrentValidatorAddresses() []common.Address { const ( // Pick big number for interesting looking one addresses - amount = 400 fixedRandomGen = 98765654323123134 fixedRandomGenStakeL = 40 fixedRandomGenStakeH = 150 @@ -2457,14 +2455,192 @@ var ( blsSlotsGen = rand.New(rand.NewSource(8080)) ) +type TempTest struct { + Public string `json:"public-key"` + Private string `json:"private-key"` +} + var ( tempBank map[common.Address]*staking.Validator = map[common.Address]*staking.Validator{} addrs []common.Address + + NewKeys map[string]TempTest = map[string]TempTest{ + // Shard0 + "9012": TempTest{"e2c13c84c8f2396cf7180d5026e048e59ec770a2851003f1f2ab764a79c07463681ed7ddfe62bc4d440526a270891c86", "19e9adb6f6a10a160b951c33fee81e53beb7e41ba1b500626a1082215faba010"}, + "9010": TempTest{"1480fca328daaddd3487195c5500969ecccbb806b6bf464734e0e3ad18c64badfae8578d76e2e9281b6a3645d056960a", "8e7874184b1fe76d23b6a3e20b7178928123e7e4e63ae642235c5a955983c26c"}, + "9108": TempTest{"9d657b1854d6477dba8bca9606f6ac884df308558f2b3b545fc76a9ef02abc87d3518cf1134d21acf03036cea2820f02", "f2b677a9cc5b2325843d493dbc1a970d40c7edc032ff848a6116fcfe9a465015"}, + // Shard1 + "9011": TempTest{"72ccee6c5767a21d9b659a510482acca9b2303055d8efe09c2b55bd7d8910de4521036274c8a3a30564f0e9378b05189", "df8c569d2ed038f34408dcf980895de448bd25866192cc994aeb094982d7f769"}, + "9013": TempTest{"249976f984f30306f800ef42fb45272b391cfdd17f966e093a9f711e30f66f77ecda6c367bf79afc9fa31a1789e9ee8e", "ba742bf554447d0a6bb8da47f9134817c3f8a5c3717ed50e6c3fe6bc488fbc67"}, + "9107": TempTest{"ec143fa8ca3febd4b8b8d160b214a7135ab48f13d5f37ffce22607959e5da5b5852c8a8403ba0836fe64645fc1d92f97", "8ac153f6efc02cb4eb7c566a5fe89dd9bbe6da3df0c68b3e5dd5f4887b09725f"}, + } +) + +var ( + accounts = [...][]string{ + {"14cab59f60837090b428a4fe6bca8f82c0bd995322feec4684c444d8f6e7c6c0a6c3f1705b0d285fe0257b266626a316", "1c733e5866e2f615e4d299e493f6593f4f282d10e35ca0362dfc4fba526be863"}, + {"d3c9c227f945c8a7dc0857884710a9904e29b9aeb8ffe1460e978568385cf3a1b6d2a73b47af0c2bb4851aff684ed383", "20667b58e47025b8b4a5524e5fd5fcd094f0126926098d8cba91cf8c57828237"}, + {"ea50cd0a7263d5530b633c5056d183388ac6fc5dfa6e7a8b59ed2c0ca9025d27d54d78e45a817bc3821a5d96d5ae7c97", "353a96f7484637fc5c806b14ed409c89c884f97aec14152be2dc02a95c02a256"}, + {"56b1e97e79fe37e19a5876c1f0e24e12e4252a086088221f73ad43174f477c02b2083e301216791092a8aa2633428883", "255f3197ee303e20a42223ea9dbe1cd2791c6c91d50ff7d2f406d480e133db22"}, + {"b166d48dec6934dbf6c446742de9adee53c7a6872f63e62611634382b163b7a1e0400996674db8c8d6910c4d8d935889", "083d9eb3915564ecaa4997ad4011e42a660287bf0175bd46cbfc5ec766b97d54"}, + {"9cf1e8c8cc7e32cdbdb762d464032a6fa5b74e7dbf916909ca22a57ff0c01856e350e2a7c2b1c74d8871d0c9af70640e", "a506f6895829956c8c514868f7283fcd6833e7cadc4c62e57365678f9417622a"}, + {"e43ac882bdac2bfd79f0645d8a97263ba6aede152400c946ff6670eee1da6154dade87eb43c5fc13316ec3cffbbc8b8f", "2d5ed7ebf50eb0a3586800d691d28b4ba4ff701afd3f1246dfa216b89cc9131a"}, + {"40b33fbbb15a002693bd5a4a7da86e45941f66c82c8c41cbaecadcf3d1956dd4a19208d69908663bcbc606a4c9f67002", "40c57ce993f53c298434a4be3e78dfc35c45c336671e2e8c0cfdb56eda36dc20"}, + {"2b6d32cd1a32f813bb98991002d42703cdb00be6d5f4f319204d2a539da58a14c5a9b3d8bb74a16c054c55339a53ae08", "b52394fc0fc094e71a1a64ef25897db58abc6cd1bb172672e4c0382d5b4a7826"}, + {"444ac7f6f16a4f023c18d7e7acf2cf8ebb96a2473e02a403f2252839b0309851df3abb1d24f7ae4819c7e9283e818897", "3043bbfddc4d950ef65f45e364c5e28b416abc5d5643dd14d59d19410f25f65d"}, + {"dcdc15f99ecd08d6e8e7a8746fc781d0113855ef6866c9d03f445f5b8bf4bde005148b28b75a3c7c3a0f37ac49e77986", "96aedec3f2b3d48c8464f04dcaf4dbf4c27b7108af836d0581d347e569d29344"}, + {"1c4b625ac8424d7e45df5aca7179113af2edeea36457418ae84a1294e54f4b740309fc1b4e4b981b6f51ac0a68dd5791", "15a1b5aa27fe98682db715ad177e8e3c661ad917405a135b18acdfb73a71881e"}, + {"7e72cbeb6152d45903cb476cef4c2e65eb6a5e0fa8d90d46e0bba4ff16853969c73bbaee66d0c3f47a556c502f87a319", "c4cdb2f30ccbad6200fee58cc622c8849139484599cb43b65e52aac55d888a4e"}, + {"1fdf24b10e748798be733d4be7d8c5d18fde8800e3828cf6e7ec66beb87c55aae345bfb5b6321ff059089dcaef67e696", "be1e345b4f85583750c6ed3a912d613d9a47ae06e79492d8a7a63362ed02dd06"}, + {"c29b37816e1ddf0b7736af7ed22f4ddcab26dbb4bdd7a5347474aeffb9d933e81a14b66e98018c4dc125b2b0c1c1f605", "65e2a6aad957d23917ef958e9465b10bf0271122820682bd9360225e0b17c667"}, + {"7990b4f5975e399398b82bdf87eba6852dfff49ce25bcc88a360861a660c1fcee3a1f994a4879f482f467fc53aa54e13", "12deb9915b4b28ea7b822467030910878bfcc1cc27d3f0147dd0cd9213b77718"}, + {"27877ce5cbed77848fa7f894c2b4a7f8fd11d0e31e0c49f88682ef063c44da291a728797e6e5852d0bcec659bf05dd85", "7a84694b1084c2f79d0f61cb46e7ff724a31f3774194e535eedc51df3e89e214"}, + {"73a0feab8fc245419d8459e3d50e5133a835a69050c417e501f826873149c645cf1290619a054d1b4485b9175c69a092", "27ff614de69f7da721c0ec240af78c928ee60418940869b9dbeba6e939f9df70"}, + {"dca05a3e1d080b69496beb131b2c18dfbccf9c5fb470f16bb4a39b5c3aff19fb6aa37b115009ca7724038059dbfa2d02", "ae84b73e496fcf5f5661773ba92125b3d19f53f9539e0ef5f8cabd7c5a739321"}, + {"0a9221c81cfb6340e4e40a46403970d6f929c090bc4169540bbcbe02fb0ad6b06941dcb192677b9e2204a5ef27a24487", "a18c0fbea616989fc31e919c1f8090dc42cf02f410808ab1573d0b1618f3ec00"}, + {"e03d395369fee3c9bc7261bbb17bfb5eaf09e0b88978cb70a6c18f9860c7b68f231db64dd69d04035683762866febe00", "e7ac0c215d3fc055426dfdb4a05b4c48d41c7e850d6c5d7ecb24a6a5061e750c"}, + {"6a58a3ba54421cfd0bf2c8b15b92d1f9015ffc4d6e76852c5609b092a14a5ed2fa49ba1183414640e3c2c43e41b55286", "7d09db6b6fd3f9fa03ef6f788f59c12292250950ebe37e0def2db46c1af8325f"}, + {"adda915c4ca55d78c7dc595bc26452a3cd1a8cc117fbd30bfdd35ab234519566b4ad0aa581ae6ac64812990eabdd1204", "4fd18f56ad25de60b60caedc2c3b14eb8b2cc30e2a258b3f1348fa337b25ae59"}, + {"af1ba6ca3e3ba7436e3054d4b5cb3e19b7b1e0403bfbb76a4e25a1ed0fb75258d9fdf53f609394e002e865bfe525988b", "9cbaa24d59024b19a14f399a8183f4ab8a2558714eb01c3a8ff35890e6750908"}, + {"dcbfe47fdfee29df070c078609be69c412e44b5f7dd10b1ef58ed368b7a8c422f1c3b70debed3441c0cf15ad7d03df05", "82d910e82dd49e79f13829ed74f8562760df0956a7ae7e60ed1cfec9ef2f9727"}, + {"a61d2dfea77bd324d2188181487a99e8afa12b09ebe41e5bd031f24a07eef785fb0ef80f3425ed44ab1023a5a41fac81", "015e5b8a897f4bad686f29b8c1fcc81c1654dff616fd21b1664194369d0c5e62"}, + {"a82bdbd2c017fe51aee694151138a7a1b9df848c7998ad7a9fb1df2003f0d52cc99099d5fc12a3211f91d411c3553b11", "a49de622d95e586b1bf55b53d18364ff46bab263e73e2b03f00fe5d69923d248"}, + {"390851593559be7658ac6c342422f4a447a8291dc517cb1809e150e6b5fd538ab60e1b9bdbe6941ff78fde71ca68f989", "2f5ba8ef8e25999004a5c319cc5a5a4c62ac5c6a0bfc8d25419bfeac91c07f73"}, + {"c04d3fd26431a4d0ebe1ed385b46ad975f6a663365b891dd1b25a731cb7b4295f88b920451337152480571006eafa180", "e8538488afc01b044e01f6eaad582949c792160906fcb7d6047d174ecf2e7b39"}, + {"02e8a378a863080f06f4714161a4b592550e6889e15f296bdb33a156675a37af13a03522973e7329ba0bf8c79dfb4d06", "8b57d172579dbe19c21dfcf00d47ac5eced33a531c298df13038350c5aea391b"}, + {"a39b6cff47d06e9857c94da5229f814ebeaf041a704dcbff8d64224b5fa996e923e015e1c36b9e48031f48948d816b86", "c6bb4e835e4c8949117606d6bdb9610a0ebaf7eafe66ff93e97e8f6d054b8a72"}, + {"226c9493a85f82fd2efd3d933cf0a90436e0d16ad86d06619f01b75d2b89371e51b9c8b9417e3737538f09382a657717", "ae0e6194d4b86bbbfd6a7d557f0b520a37c1d58350801c6b2fe691c8e8b7a617"}, + {"8b0d5f40e3d655f9b8e6e4cef1c005c02ccbee539d7a35e5b148233e2cba8d050a838e69a156c29e416746961513f819", "8539c03e72eb85ff3321593fbb7f8a86806f93756c52b611ccdc0fdc3849ac11"}, + {"72506fb7d6467a9be5dc91051d3639e8eeebf2cf9ef81b3d8a5e95a47c35c2636c74f21ac22ae8bb9e9f43cda4edbf02", "563cce730dab68d1d9a57d8ad092d48b9589b4fb0926a6e2a3d92b110cfc9b6e"}, + {"27614c62a76a38b79dddee0fd9142b1d18b1b2502a411b86a2e43eba3a01b6a87f11192dbc02492647173c77c2cd290d", "e31ad7aed3c46221e36f2bedf1ba30ef4c891d0dfb2ca271a695175224fe2907"}, + {"bd59cbb9514ea7e9a6d8b7d6b5d05497f66aae8d32b3b97657ffe13748963dccdc4d3ff48002dd2b6e05b59e7cd6090c", "65f1a41835f471005c8ab1a65b513372c4a40a68d4c8300313f1368a28d14064"}, + {"5f49373eba2af47b6f0ec9f9ab925b0e4d1769dddc656bdf2adc992eca654495222a1b6e89fd97e86c52b039bb7c3212", "7845b0f43a621204bfbe7d22ca91fdf9abf7eb6ba355bca2f1e650b33d69b20a"}, + {"511329bd1ccce0bb0d86751fb9dbfe7c7e3dd1176112f182145b1123a850e75b13df43c199153de8078b3d2ebbadb086", "0f800ac46d8ce7131f09532ee239ce14a8356375cd374c2d48081a02a3a1de61"}, + {"37df36cc1a203a38264442093e424ff7b56a854ec9ff07e12e316f0f5b0f0ae82262eef41418f74593daafbb3e12780b", "f6d241140bef139880c66042c2a77cd94d5183f982c83359849ded6de196a319"}, + {"6d66cc4174873e1df4c34f56c3d1d01281a2bfd25fbfbd45af64d4aaa5fcbcd889aabeeee6314a0061986074f9937b88", "d9da208d426380be14beee42db5e47f1950927af70bb1f7d30488ce829d28436"}, + {"e965403c1eecd5ed151e844b2d9bf601fb8d8d1fc86169e60346dc9bdfd4c5317312c02b9caada3c9dcecef61b21ba96", "8ff45b01787acd4b2d4a780c1bf3bea1495e6cba6328371028155d4c0fc0d773"}, + {"7bd979db856a697dce7da76feab7f6e3c036984ea766198c695dff95532f8599d4cf6e3c00e2455bfb4b5ce0b26b5192", "c83952ee8ac4b25452a3fcfc75ba07ce58c392878b7c519922fa4200d1169e40"}, + {"4c40089a3eddbef0886fd375302c5dcd7b56f83cf9ff48afa56389d9d7b8b0d6daa0f2e2241773fa7ab7f0cc77621917", "7cd8e8fac117a7e327a9d609f176dfd0445749ec823096ff3f29930047959303"}, + {"2694baa0e0bd4e24c192ae5bc4b3a096b2adb0026823298c4c96bacf062b591a1a3b557d6c81773a0bcd0e1031755b8e", "81ae0d5dd80d47acb9761f3734533fa9543a19ae1a1fd67b10555775f1dcea37"}, + {"68bc09c6277fe5a478a4b036e9ede1934b38e7ba86315360b7c75c1999b848df0bac7fe00f9d1e43ca1fa20c0a200c07", "8a4172905299d69fb005ea2650b42604d8900ce9967304648545e8c0267f6b0f"}, + {"77c4d9e0ec0f1cf6c981481f766b1395ba0912a8020a5ef1e15949ffa95c6844fba6a87cb7dc3a01774ab7e26360cb96", "09a98961b4ca28357fefb75ab183074e8e439a9109d4c8a12e06f1678fcf9c1a"}, + {"7fa504b0d97596010748564f736385fda141c12f1dee6bd1f0c3bf1793f5089a1c78db84ac0ca23dc48b764d356ea188", "26ee79bf3c2391bd9af142058f773b93c9d9b8659cc2899452ba9f3eb017151c"}, + {"ddf52e6c80dc39fa940e85265ef09ade1a44e3c83eb5bc2798555ca3ecd5ec8438db21cbfe79be0f4ccdbffb22699d08", "2350f67df6146af7aef3f36de4a815574aa33db097a22184ef11911014546a4e"}, + {"b0e476eb3d88e44fe6d654b78302009940f5f7ec80e0c236802554f223c8c4011a6732930e7425973210a6d669b92696", "6898e732291fc8fad1eeb76e42be8ac8c15abf6f028bc965bef92acfae389d3a"}, + {"ae7eff87ebc4c3ea47474367a10880139cde747de0be5278c7935f54e245c762294e1fa0094ede65e6f716a0c7ee7680", "d68df81e8b89abf32597524d52240c17d0be7e3a02bc34a2ed3d633307e72f17"}, + {"30b89a0199fb280398f0d65b200b4b45327368a45337f679993ec3e7d38969d08ba3e1387e1c85ab2c4b43ce674bc105", "8340272fa1b8c273102ffed66a3934d8320772dfd5fdffdbfb969fee97364464"}, + {"9542e021b19160efc08a5aca567f7d4cba277e25927b756f3e0ad535b74caa8b3d62216c90304625f3c4293827c3c38c", "01c590a4f2bd80c5411d744b2223f5851d2e1c46731dc3a730902ad017e67c5a"}, + {"7ed1287d91d29e1fbdf43360a494f53d450fab14e3b9cc08a63f06c868d757c32e781622603a37057c909d57f1901088", "f56915dc69dea77304cab132e8bc247e568a0b2b11672a39b84c38c9dcc49468"}, + {"514051f789312e3a1851ad56ae4f64bf798c1147d99065903f5aa5807e938ce6cafa0b4122b0e3b6636f425241b2ce09", "1bb85c34055d3a9e12482136a65a837078119927296ebb5ae6a7ddd0c0aba514"}, + {"f8727e55841bd8ae53db38e67377b58e882072e937f54cdcf70c4c7f151fe7b175110d0962b66167de26c6074b6a8c17", "58847f00751f7d5bf9594957e47a88c773b16257627cd052b7180e1c0721e950"}, + {"b4225d7cbe4a290cf880dce13d65155b8ddb2d27ee495929338780d268ad96747f4013e7455fcd589ceadeecfc138a03", "3e305f7d0941507b251469d4628294582bd9ac3c995067606321422492e3442b"}, + {"82ec676e743abf1265d81b1bab873acc911d533cf9aec5043dbbaa7b29ee1163180cc84bb22d1a567c532e3d676cc281", "da0a4974718fd08ae9cd46a6d0217a67c3fb2758e81a2419d13bb2be56eb134e"}, + {"cce64ff99850372b795fd146688c9c8f54d6aab555cf624623e665ebaffc44b3587d7759a95968b4cc874380be096993", "440bbf61908f54e4801fcf215e62256b1d2bdac6a2c0cd940a20beed04cbb844"}, + {"0b10167f5ca2979864e46125df72aca33956173aee020ca88dec3e65a5e0d41c1f317ba930d66ef9075c9ab20595f58e", "bdd7446112ddf42ce0d468a01a0c1c08e45b6b76239336878922235216002210"}, + {"502ca2a064d45970776140c190ae8ff0472f8c7c3236f43d0d706e72a430ffff4050bb8214de2fcbec7e9c3b1577f688", "184d37b6d7f04f232e5f8262b28a4aa26dc68f2208810d02665383cbfa074835"}, + {"02e95502143d16ab34f082b4368113f49a6b7a4dbe33e10c172a1376add5405a934c2992a4e0755636e3faecc11abd88", "7dbaa1c7760334a0c6b7148fe91d7bde88d212317b45115643e29d0cb8484155"}, + {"9d0892d0e8e2474fcbe225b69b9560571842cbcd8a35f0ecd0d3cb9885a079dfb2d1669f886062f28e2ea347fb7aa014", "b5a7ed451787128bee2b256be308e5b0f73ec1f3504f0400fb801c4ae373c327"}, + {"c4349aa33c18eeda0be31e0a519c5a53cfc6940b995c88b5b34b81bc22310f3eb68a7d6abce6d7e609a6a269a75eeb8d", "5c142f86bbe3fd2f6e446060682587fec7dcef6f360fb1659bd380ecfba9c656"}, + {"ba97d3b920e7590afd420458e4dfbce3328a7890dc150439bc9f9b343d4267851eddad645940a14c14f9c9dd085bab85", "966ca693d191cd6dfe003862122adee65c96d1aeca2ec85aff716c3ffef4ad35"}, + {"6caaa02b4289dff76ddc7f8441bd83821847a7d1d2739303dca02b8510a3cb6037cb2562c2ad5a4528b555ff3128a618", "96bc5bcf9421b96e63c150d2ebdb9a75a85ff3854867462efe0178a7c4855623"}, + {"1467f6aa800e4476846e721d6cd684cd899aa46d87c5b6ba674f9f9565c2fb64cbfec5b4af7b55e86fa2d8604acdb884", "1ef689c2fdcb0e93ed3830311f146f37dc1df26b7027760292a3818b4964671a"}, + {"176f9edf6b38ce85d35f2d314287c03f2939a88230c2a493476a337133f9f2c00d1a69e0d24587b89597c400236d2f10", "c06afc1859c5148fc2c3820614739cc5132d50505fa7dc40e8a914e4b65c9847"}, + {"4a6a00e30b327b8d9a01ebce8b5c765fa1a5ebdd3d7a06207590e5a293ab8b38a2a197e386e678cac86826953f2c5301", "1fda30d5de8a0e8b9d725af8805996f7e8a31f36406dee7e1d1e046265c4de04"}, + {"5c33b153ee8c281b484c20f0a8d9626a1b13aa20d386347250996032df2cc607a307055ac46796de7e65b50e5149648e", "cc1a270c065654d949c5e9d2e158acdd3750655cb94ec94810041dd7a26c9e38"}, + {"0633521bb2c3c27477e97ed506587ab4994072e97f307b87a735a0b77fb48de51975114bd24ea4938063d4edcd1da313", "f555d422832a7f8f8309f61b94cc4dd96d77041684dd08e72396bca00c52b439"}, + {"e4ad16fd499040c0315c91eb645c0fba1b1f922772d39bd894c055735c9b9d483473d8b6321638e007bfcfcc1a67988f", "ba2d5d76f3178b954ad624c6af84467d4b55e43bfdd1add98dc50e9857514c02"}, + {"ea8353454271fc95a17543728255a4c3135ec6a48b82458ff18568b155d30f3449c480882bc77d38cd553195fde5dc12", "964fac9a31467514411d96a44cacf2a5a6f0a2db985a12dcfb87adade0cecb4c"}, + {"e298500b9c972c42cdac74a11a096ba2d64405d015fd5626e07561a297dc6c4f36fb5f1cb1c4d86894aec82d5e2ef691", "08bb76af47e9385c354655e9001ada326e811ae241c4d22f81a2111ad9baba44"}, + {"04e34592e6f062a9558fc09ca379585e15b4ed6131e2766354cce6796233db689c340ed6ee0b7e9f40e85ef7b50e6a13", "efc1f64c2d8cda2d1107c0046703ddf4cf7ecfc6f38adc4631736e83846a145d"}, + {"8c4b304f3a0c7310de817b6e958588cabb4124d0cdea74eecb8113a2daea0dad1ae203bd0d9ce228e2ee89d68965868a", "6e8f06bf1128b4a148b47ba8f2678398add67e0a8b11f81aa7adef4bd7d0c22d"}, + {"02f3d61f931c09d5954878afc08799e1b8de77b1756576e3a11ef836247bdc212bad1fd287712a815228c2b13f5b128f", "468049f68a279e8be38b0d3f0c67758e45cc5bd4b5a236fc7ef3eba78b45176a"}, + {"323245a1027c2a00b9fbb908840601a9bc2fd5808cc313a93b2f14714ebfd234dabe8984bc2169b09ad5327e2a3f6584", "5a65a9fa599fd7ef2140b7b80fd78842dcdbb4084f6e069ef427fee4cb2ca240"}, + {"62a08494a9808ba4fb15431dfb431c8d993afe6a4e2d18214b8ea8ebedcbd46594894d4afc0290739fb17d4f7b71c389", "1d73c784cbd13c5e7987c78540aa21aabc395fd2cbb50a6561677945c2558d39"}, + {"fd46f1477459c863f903ac7daba941e2c91031dc000740ca8d185a86939907bc9893ffe586dacdeabb1a3459c6e8a696", "04ca4066f6f3a49e8ff5ab3680a90bb2e17da973c4fb67022a6ceb3b05b1a152"}, + {"e71d1a01831e052622264ad4d7e7a9123ede454257501c7c60d214904d124dafb023dba7d48dfcc49d50fca5ef2c5098", "de295122f5c2982ad00cda3cf81eba0fb05c087969b01862d7e75569c931b856"}, + {"ea16cc1fd08e907911f247e6837ccbb8ebefbf8986fdbf2cb1e3c29254cf030ea211a279d8e368b61a48e49d1833db17", "1b3a20dfc6f2a25b1da99b2c836b1e8707d897a16313790ad0618f3c952dee10"}, + {"dc90ec2c5c2e82793bcc45aa9ab72fe40b68d63e394e2f8b3b654f1b8bc87cf5e94a2fef0b955acac1018ef526858f84", "0973be2092613e69d8d16d251b74e6ebbb39932f9fecad384cc96f917438b811"}, + {"cb7ce449200b1bbcec236273d2fd79bd6f633cfb21ea4720571d36a4ac0dfbe6d218f74d1093a7888652c2f82ab6820b", "568363bf1acd8ab07feaaf324401eb418883b936c720088a23a1867233083b35"}, + {"277544d9b04150060bf5b53a9e53e64216613e2e5356a2113c7e4706f16a0babe6353a585444495aa175cc157aa40087", "ea0a26d7406f3ecaeb602a129cc6052db891fdec7a11e6ee852e8dbf3520a91d"}, + {"92eeb95ca74d9f4e9deb0afb2f7a1f9973e2edfcd62c0c3d13c66b61bd3cef962198b5b5a2451934ea079ed1182d3591", "5a5fe61ae2e047bc0e8c81d5c5949749f4c419ad921a858e07524f2486f0a12b"}, + {"e307e3b4bc5da252d020a755aba61f3adf35b3bb56b3ec8e0ded021674b620fa14d6071ccbb19f04820955474458bb19", "d44f0e3a91f72df286ae784ea61ec12b7340fafe647ec25b8155d630b55a3b12"}, + {"f59598f47bfde137d5023f865861832949606c6256c235c697f01cfdc41583161ee6674ee89a1766c0a75e623be22509", "f310ef9048ed84f7474802b4ce8778582d1d91c6cb6e03af9741ed3c5759af3e"}, + {"249976f984f30306f800ef42fb45272b391cfdd17f966e093a9f711e30f66f77ecda6c367bf79afc9fa31a1789e9ee8e", "ba742bf554447d0a6bb8da47f9134817c3f8a5c3717ed50e6c3fe6bc488fbc67"}, + {"72ccee6c5767a21d9b659a510482acca9b2303055d8efe09c2b55bd7d8910de4521036274c8a3a30564f0e9378b05189", "df8c569d2ed038f34408dcf980895de448bd25866192cc994aeb094982d7f769"}, + {"08a71ab590ae132ca2e1a7211e0f90498629b55a0a1e2b753b43c5f1cae8cefb4f2494e4fed603f2b725cd893cbb0c90", "14fac9e0979b69f4c9ba3de4eaefa4e26615dfb67430d459d3eda58567849e57"}, + {"a59ffa292bb9162f55a83164c6162dd9b485b156d18c9d701649299da38aa5c3b2713ab9cac64132b98a3bece2061897", "8777f023e07075346c3ea3b2fe95f765284d952b3c04548085519fd9ff09dc61"}, + {"e2c13c84c8f2396cf7180d5026e048e59ec770a2851003f1f2ab764a79c07463681ed7ddfe62bc4d440526a270891c86", "19e9adb6f6a10a160b951c33fee81e53beb7e41ba1b500626a1082215faba010"}, + {"ec143fa8ca3febd4b8b8d160b214a7135ab48f13d5f37ffce22607959e5da5b5852c8a8403ba0836fe64645fc1d92f97", "8ac153f6efc02cb4eb7c566a5fe89dd9bbe6da3df0c68b3e5dd5f4887b09725f"}, + {"dcee54914821c267328842dfa7aa8fc993488505818384169562f1fde35067b3258600e9bec801ffa91b2d4d44b04580", "3c4816103eab7b9e4f279dac808ef79a1754a526682503c9dade0c1b9413a316"}, + {"cc120db9a2d189e58128b6c2c546be5a68c173de71b7f5062c184b3909faeed2699503aea5f2685a1b150685244cfe00", "a409124667a202e4ec48ae124dd1551e2f138090ad5aad51407cbcd5c2a56b5d"}, + {"5005ee176e5040d0cbd2ed7464ea826690e9aa8c846adee42eb438db86be34cac177f50ae66db56f1f2759e06cd4ca86", "c66c665432c441e90bd14746c8b8dcfb0284e16f0214fb4b48da97dd23ba9f0e"}, + {"9d657b1854d6477dba8bca9606f6ac884df308558f2b3b545fc76a9ef02abc87d3518cf1134d21acf03036cea2820f02", "f2b677a9cc5b2325843d493dbc1a970d40c7edc032ff848a6116fcfe9a465015"}, + {"1480fca328daaddd3487195c5500969ecccbb806b6bf464734e0e3ad18c64badfae8578d76e2e9281b6a3645d056960a", "8e7874184b1fe76d23b6a3e20b7178928123e7e4e63ae642235c5a955983c26c"}, + {"5139d2fbd1af771f76ecb93444d15a1a0bdb9706ef8149db9f88d9456e2b616a812f2bc21a320185c4619248950c9582", "3a0032951ccdcca696f82f16b16178f7d239db5444b6203292573ed1109e6466"}, + {"3e15ed2b396c2076d2df067e58dec3c5fc0d5f31ff3b20cd245bfcaa58d73db8373da74ce7b852d95863c1954f1f4986", "8adbbb081aa9885854b8ce9b6d437e30136d065f2835f11ea6a0b89b74509372"}, + {"e99f29a4791a9da645539d154acacb731f15906201e194a77aa7cf2cd4827692c28e7f3cb8d57a3dd0157d79c01e0492", "a68648839687f1bfab70ae1b8d41eab44cfdb00b599f4dfb5aae31cb77c1a60e"}, + {"744d03bb36aec3680fd894d3761d30bd8f9bddabfcc0465e80bb288a43c02801073eaf441a4f99f10776788dc6bcdd8f", "be1fec362830fd7457db13e45cc5e24bf9aa15c54d347385b112bfe18044ff2a"}, + {"a3b437fee6d9c0455b34b70c984e21ee8f1d6c17fb857462c67592b1994c72d72de61afbdc9fdfe5a90d598444b10516", "319f70b9d21899f6a9db38a9262e59a37212004ee8678f143d71bfb1b8bb5b54"}, + {"02dbc59d3d2a173f2799ec5d9010238c762311beb1cf63df03937e416c9c67e491ba5fb912c2e46488c022c009257806", "2c39b8517d58e6f11d71459ad5f074cace01335bffc572741da5845e357b2d26"}, + {"f26d9fe117f382c14128352de40ea9b662168f50a89cc10ccb20a3680c1ebfa5fe1efe84a7b1090e316ab86dbd734094", "f79aea85a3673fc026e1314139cbf9c12fbe5a688623465c966c175b69472f65"}, + {"f3b034ab8643ac72cd29f3151d124348cf459aa7ad69dc0f98782120f69ebc82a21df0bd1db01ff5f9fe304f4c7b0485", "e6fb858d69eff707637c4c1a2c14a9bbc8c7f05ea58b208b119214c236c1512b"}, + {"0a7559029ae550507facd6375a30f70cbc108407bb9c4fab8381fe5b8a30c0a2d99800b21a788b2d2f0e74f3c23c8b12", "25af65b82f969650b0c50d9916fd10798dda73f34382181e238106ebb9649652"}, + {"c4b9926ffc51e45404cecc06dfe0d652c9510651470fbe2b948d6730373d472b3a9a87e748f80785f22544afbb16ec97", "8a2e1d44667d5082350de9b05536de4b3de88becc68b99d4cd922fa006d6481c"}, + {"4a10b2749f91f9d8577f612b374274f8bd90e50d1294ec1bb043e4d0de590ae8adc0e48a7c494cf6559d9ee5bfb2ec82", "f2159355c79d46d5abd239a780d584846bb7f3622306e3dc6aef5d7646f8b037"}, + {"cf03b1d8da5a8b063e278215d8e6677e9cee9a09cd0ededb32e28e623cd43ac7fa72e571e786ac925253e56941170099", "4a4939ded1d7210f01decf8905967a8814d94c7e6901923e49c0c862bda1cb0d"}, + {"eb28831cc5257b4e85f824e4e1c71bd530eb60112f21a8284dc6272e11abc711041b64afebf202086b18917d765a2301", "09cf22643028d316bec1ecedddb2881064132939a7b63f50fc94509b6c5f5433"}, + {"5220fa5b230d78234a25adfefff480a3664405f231eb87c797b33fb9338e5bdcad58e3b80780bf7cfc0705a51fd5d519", "7e985ff6ad71d8968bd7dede437467a9bb120d67cbe26866b81e9df6153e9e73"}, + {"7402dc3baa41314482037cad9ac5487d0af9b8775b82b0cd4265af078a8b6af48358b884c3047c0c46f4bacb658c5b02", "12044340eb632a6ba4e9dc81f9bbcf33fa68792c45773ba51e4d62642c5ff761"}, + {"43f08c35d400b36299cce44c74064da8ad15c2f4896dfa9c5810b09197e8f26e5d0ab11db797f2dc72f3fdd8b8fe4b8b", "00165cab4cea976f26edc80afc9e5a4b15678b20cfbe3d7f38a630fb1824c44a"}, + {"4ad485a064eebbc26fc4c016b8ec979aa6816d1b847fe52111f677e9729f8793576946d67b9aa01265f98d4e437de988", "82e9fed6646457d008201d7cf489e0e308ab54235d1ef59a5ae484ad6e93ec34"}, + {"733cab5ae15056a40cf515acac9a5e283f0dc56e5f7b7a1aed4e258d4ca0477a9804f2ddd59a6c2eef0697538fb8c491", "015db82e40041eeffc1573d93427be038b26d7600cd0651771daf3a97f747701"}, + {"2abd73dd3f286c23e7bc0650a580b6bd29ce3a948067314629898d683b9e4c1d30924c01ff0813f7e912e7e69f07b28e", "c32154e8d23412450a33f46bc61716199d1316a3c3ae00ed7e374b0bc7fb3d0d"}, + {"5ec4b7f1636b7d565addb55c6cf182b044560296dc3c82c846b3faae8e485fef77ce9d8b0e6a04f832454fedbad46a00", "63cae9af8c9adcd402aecce77cf0de3912e4c6e1574cdbf2ed3046c1605f4b30"}, + {"160004f16449ca70b454f6260d8fccfd6b399e4a099ee05c745cea26ba48047373f105bbd426a7035fb582f8e6348291", "09bcf84d83fac97648715d60b7b70eee4e3abc03c0253bc5147913b9ed405717"}, + {"23b33c8771e70d1a84dd2f24b924c50a7c518a6e6f5a073f4c223ef299876689c4164a8f29fc21a531895750166e0b85", "c7c917686f6829fc79cb5020ca7271bee4cd2c3fe8c7b31ddca5e8c1c11e7b1e"}, + {"563a4f6d39adc91bd9033573a517e5a365b181b8b5b6f086f67bce1e09f80489a6b173f0ee02201e5d333dab3b1f9e0c", "452971a484dd203669f330cc5cc7a96febddb72ff8884d63a6ff225728c0ca46"}, + {"d6be59700f6d81ea11cc22481d70b972741d52f9495f1637e7e2a478ed99898a4cbe5a5fbd6fd2648a10ad09796a5811", "52a6e2b5c5da26e75b96894f1a443d948f6235873040a511b4a33202f36cde1e"}, + {"8fce243f7d8f4bb28195c428bc54c4a233d9ac7c204e75cb45771f1d4ba9725b3b9c69c24e390dd932ad930ef4741997", "1b3aab5c04b451cd714d55273f3749aaad4a45247911292ae84f02ccb6a30d1c"}, + {"56318730454846ca6d9d7b23f2de5b3a67b4b62fa057e4556c144d6f543e2e9defc031ba3057be59b5532bb0b2f81c06", "5f434becf1c975090deeda7bf15c5705c03b767db5bd338f400368baffba9825"}, + {"5f64a969b0ca0e06b78c8d6f0f655116dc526efa48115fc33320abfcf73dc1136a5833362dc3f9147a574fc0e5cb2f0c", "98a256ff411ebaacba2d0d1519a1f952b2e143dfe323202fed5046ccb24b2224"}, + {"4ba427d9624cca3f21a76e31793debe6c7adaa871693ee3d88e8e943c4934f941896ca7e1cdac50901eea77998504f99", "39b81fcf87f18e479a6abbe2a7011fe5992fae0beca763782d911cf5dc8e5b0c"}, + {"3a24c5b01460b83f48bc10eb56acb75252d800f3a57d42a04311d195dd63a0b768979c4a7291802d09b5339c3de84a88", "832df5a953bb748109d3346c2d9f1cd2268ae5026cb7d3251cf15e9ae60aad50"}, + {"77f8d0e62982ae95a91b31b2d1ec05035784c4d63c91d213c499a4e6fac839057c53c15eaff6716c150a3888f05a4a13", "1fb813936193004802f311f7b24e034ca52d121c4e715cbe9f3ce6dd3f819356"}, + {"f094a37f1631d595b40aec0ce15822f8a66e7403aad52858dc30e1a740ee83abe4a47d574725f90e9d2149d3afca5309", "b433b444d92f22f05f59b27e150442d1731c95e002610b5da3f265b0c25a6561"}, + {"86d8c9181fc53c83d5c5a59ff71a5552422a4d5f50125c8235605bbed442250ab5eedc2a9af96e72d9df185af0b4620c", "b70c758757675dce385eb2a6019ae4b9768836a29cb8faa50e8a8016c603871d"}, + {"69327dcb847bc70a01fad93968cc746628bf1422ff92b8b32202015ff75d85f8a3edd0a8c738c1d8c6a465ab895a6d13", "cdc4d41677731c59ae715be9a250fa0a674cabfe70b200afc1c11b651f977134"}, + {"7493640a45a5cecbf761ce67b4070d1864e27972ed3ca7c55fd7c1f191be3dc3b5fca52202d4c5fae1156d1659cf4c97", "488a0d363880eaa61904ceadfcdcde361321926b532001e9b33f9e066eb76730"}, + {"441e760d4d1d6fadc9dc4d67194375e14500c7c82d2fa8eb460bc9980dba72f213a34840ff10fd7ebf17cdf8215f7807", "bca8878d4b7da98864ca552843f7b0bb35bbc62b793ef0320b810549e50d0059"}, + {"e44a73e261e0650238743e889e8ba28d77f6d760e4dddc8979b64c2b76716443801b6335700a32d8da477494cafeb301", "d35d194f207d2f1e5032ef4f138c6d732fa371b259f8a954a2c7f4b1bd0cd94e"}, + {"73b97bd2114bd822efdb31fda3bd45d32b11ea2277d88a7e5b7de25e81363aa8a4f2d069111b08e215b933cd96705902", "7760a6f816e7d345554dc4559d9514eb5947226a349d464eda3a75f1af679a26"}, + {"123f4ba326f048e9a9f650e7769db67d17b6a20198aec99fe92c74a8055fc78a9ec28171a1a09ab6d508172100974a06", "097af324bc4757ba78553480f11c0ba001c3cc96590f8537215a1514329f560e"}, + {"0b859a8b28dfe7898f4f4e1cc346cc705da88403157f17813715d5c4e73e99fd2ac0d4c7afd1c87e8268c3f9380fdd05", "313a7f8c831827383a350b6c55a9d24b4532e63f610ac187bc7a92303ec5c604"}, + {"77ce9b523ebc8339e0bce80d249954797e9749be4bf73bdc5bec030a81207be9d76051982c5b80c628de39c4c8e22598", "9d1452e022fd6a280ce7baf4ac3a7151cd71af54f3d0894289f95aa39ad4d439"}, + {"cecdd9c3b88b68ea1a84c7d9e14e6a0112e27a8e7883730ddf129526b513f7630af673f9a89f5f28341163bc3f040e80", "07165b308d2924c30ca2a3e7205f46cd6094c8cfca62e1fb46b0e9ba4290043c"}, + {"da6e4afdafbd87cebe3a7bfa165d118912e2ff180795196f49b0129bd777d671d78486b2660c5528aaee27aa2c314d03", "350fcb263739dc7b718982a9180037da35d067886d7a569854497d6dca56fc39"}, + {"80e59409d68e0caeb5105c02234a861fd8bbc6962928d50f5f244e7ce73fb5425fc0668dd70207a3bcd0135372a8820e", "eb8af67f82be8f3ac0c01f4bf438e4215603cf07e4cf154932f9f4f2e220cd47"}, + {"dfe1f5b9f22293440603171f7b0234ed76a5c3db16de169e94a2d04615e32db6b5099ea9e7607aea9f41694b3ee2a387", "427880ecb5a18008be31af70ac35feba1b7ecb774750509c46fedc6fe87a3c03"}, + {"211c5709ec1d114b427d956a4d4433e3798ec214740ccd4c8dcc8688d8f5e2b54614f1b66c8032d76e8617ada8836e14", "dd00cdc07bc1454de81e99c7bcf08d96f5487f16c6b08c8c25642bf6f671671c"}, + {"b7f66b9f9f023c98b5e4c5ca751561d55914e96d8db25aa1a2d482f4dfa9c38049a1697b432c9500adf9b0a2cc8d1390", "f9679df2366b7076068dcb62478c436d488a1234f46bc5d23a2582c30504ce39"}, + {"24fc0f43dc21bfe7252bc26fd1ecd4c108b5ca6dcd6e213b4497ee3b9f39704a31ce4e4dd4dab8ecc4a0cccbbb9d050f", "818e23447a8c4028d57e3d450c7eb78312fca20a11168df831c629468f3da93e"}, + {"3cf72ba7a2e414f3e3e42591f19b7811cc86abb09b689bcb321bac2280f59e0a052bafbf0dc05f9671ef3d48e6ea0892", "7852d030e30ef02fccddade41dd11dad0761ff3e0de2bf80a220afda46e60a04"}, + {"525cf97855946b70709d2c56313ee51376082681c9a351a1829d49a6d18f337bdca6e5cf2219948b46b733dbe2657e05", "507f1cf227ecc51b58a6cb260a2ad88d0fae52b14df33080ffe634d125420d3a"}, + {"2d43b3290b7d056e9f3d470d5c16a5018dc102b25ab5ccff98c4a7610241c956352d56652466158dd92024d65ad8300a", "dc9a9e84e89c316d7e82f17b6af209f7eecfa0592514b86ed367a0f36e91894f"}, + {"16adbe52c89f86d6746e6135224eb0268db5b82fdc9f3332505daf7a79aab69b274cfc77ab63e0f6ecca5e1bf970fe17", "4464353707dd05051ef67ab31070b44e524c1a31f47681e7a88ae97142a13955"}, + {"1432dc6b2c5c1b546acade2a96482350f2d5369ae7e6eb7cbe5e6abc4680237336ad9715552a110ce7d15c75a5440d80", "93ff03d1944a65e99cbf205094246c19c0269dfd17e75c4094fdb7ad99151b43"}, + {"4f0b413335c109ac5fb56229816bd4aa6969678922125c9ecbbc1b371ed73dab6e743b6bc5ecedfd89a636dec3cb3396", "00de6ebd1259754b4d13117ed98410e6f794de52c91e85f827b2580b6a8e8a28"}, + {"2bd7f336338b4af9ee6ffb24ebd4076b7bc3805bc496325379649356c1d98cb7e1245cd184c885f6271e8c08d765a603", "309ab0f63625e579b9294209fa00b949ed92f1c2203ef7e149d8e9713a45a551"}, + {"7d86fe374d3e108b6988503428984b5a34e5b71746e5371c0f8df0ed29cc5852d89afb395a33503d580f6cef521b9e96", "49e42da63171901d51b62abc234100ecc9c07b22e6f180a747da15da66fa280e"}, + {"95137562374a17ed96a3272a05889e52630a7c7c4c3a7d4c389d9a067805ad246b259e60949260aabc704a89beb10c05", "d0d08d77a535903dcc509f016825a12f6f49219bfef6e16fcc6d29b9b30de30f"}, + } ) +// Public key first, private key second func init() { - addrs = make([]common.Address, amount) - for i := 0; i < amount; i++ { + const a = 100 + addrs = make([]common.Address, a) + + for i := 0; i < a; i++ { addr := common.Address{} addr.SetBytes( big.NewInt(int64(accountGenerator.Int63n(fixedRandomGen))).Bytes(), @@ -2483,30 +2659,26 @@ func init() { someValidator.Stake = new(big.Int).Abs(big.NewInt(int64( (r % modBy) + low, ))) + const amt = 2 + pubKeys := make([]shard.BlsPublicKey, amt) + for j := 0; j < amt; j++ { + pair := accounts[i+j] + priKey := bls.SecretKey{} + pubKey := bls.PublicKey{} - slotsCount := blsSlotsGen.Intn(10) - if slotsCount <= 0 { - slotsCount *= -1 - slotsCount++ - } - pubKeys := make([]shard.BlsPublicKey, slotsCount) + pubKey.DeserializeHexStr(pair[0]) + priKey.DeserializeHexStr(pair[1]) - for i := 0; i < slotsCount; i++ { k := shard.BlsPublicKey{} - j := bls.PublicKey{} - b := bytes.Buffer{} - b.Write(big.NewInt(int64(accountGenerator.Int63n(fixedRandomGen))).Bytes()) - b.Write(big.NewInt(int64(accountGenerator.Int63n(fixedRandomGen))).Bytes()) - b.Write(big.NewInt(int64(accountGenerator.Int63n(fixedRandomGen))).Bytes()) - b.Write(big.NewInt(int64(accountGenerator.Int63n(fixedRandomGen))).Bytes()) - j.Deserialize(b.Bytes()[:shard.PublicKeySizeInBytes]) - k.FromLibBLSPublicKey(&j) - pubKeys[i] = k + k.FromLibBLSPublicKey(&pubKey) + pubKeys[j] = k } someValidator.SlotPubKeys = pubKeys tempBank[addr] = someValidator } + // b, _ := json.Marshal(TestBank) + // fmt.Println("keys", string(b)) } // ValidatorCandidates returns the up to date validator candidates for next epoch diff --git a/internal/configs/sharding/localnet.go b/internal/configs/sharding/localnet.go index 0bbf18182..c656d6035 100644 --- a/internal/configs/sharding/localnet.go +++ b/internal/configs/sharding/localnet.go @@ -17,7 +17,7 @@ const ( localnetV1Epoch = 1 localnetV2Epoch = 2 - localnetEpochBlock1 = 10 + localnetEpochBlock1 = 3 twoOne = 5 localnetVdfDifficulty = 5000 // This takes about 10s to finish the vdf diff --git a/internal/params/config.go b/internal/params/config.go index c9c2a74d5..7848a9a0c 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -37,7 +37,7 @@ var ( CrossTxEpoch: big.NewInt(0), CrossLinkEpoch: big.NewInt(0), // MinEpoch needed is at least 1, crashes on 0 - StakingEpoch: EpochTBD, + StakingEpoch: big.NewInt(1), EIP155Epoch: big.NewInt(0), S3Epoch: big.NewInt(0), } diff --git a/node/node.go b/node/node.go index a76ff6993..89dcc9f74 100644 --- a/node/node.go +++ b/node/node.go @@ -103,6 +103,7 @@ type syncConfig struct { // Node represents a protocol-participating node in the network type Node struct { + myPort string Consensus *consensus.Consensus // Consensus object containing all Consensus related data (e.g. committee members, signatures, commits) BlockChannel chan *types.Block // The channel to send newly proposed blocks ConfirmedBlockChannel chan *types.Block // The channel to send confirmed blocks @@ -381,9 +382,13 @@ func (node *Node) GetSyncID() [SyncIDLength]byte { } // New creates a new node. -func New(host p2p.Host, consensusObj *consensus.Consensus, chainDBFactory shardchain.DBFactory, isArchival bool) *Node { +func New(host p2p.Host, consensusObj *consensus.Consensus, + chainDBFactory shardchain.DBFactory, isArchival bool, port string) *Node { node := Node{} + node.myPort = port + // fmt.Println("as node, my port is", node.myPort) + node.syncFreq = SyncFrequency node.beaconSyncFreq = SyncFrequency diff --git a/node/node_genesis.go b/node/node_genesis.go index 61400eb5b..7c03d4062 100644 --- a/node/node_genesis.go +++ b/node/node_genesis.go @@ -43,6 +43,8 @@ func (gi *genesisInitializer) InitChainDB(db ethdb.Database, shardID uint32) err shardState, _ := committee.WithStakingEnabled.Compute( big.NewInt(core.GenesisEpoch), gi.node.chainConfig, nil, ) + // fmt.Println("initial-shard-state", shardState.JSON()) + if shardID != shard.BeaconChainShardID { // store only the local shard for shard chains c := shardState.FindCommitteeByID(shardID) diff --git a/node/node_handler.go b/node/node_handler.go index e79ca00e3..0aa4f3fd5 100644 --- a/node/node_handler.go +++ b/node/node_handler.go @@ -3,6 +3,7 @@ package node import ( "bytes" "context" + "fmt" "math/big" "math/rand" "sync/atomic" @@ -17,6 +18,7 @@ import ( proto_discovery "github.com/harmony-one/harmony/api/proto/discovery" proto_node "github.com/harmony-one/harmony/api/proto/node" "github.com/harmony-one/harmony/block" + "github.com/harmony-one/harmony/core" "github.com/harmony-one/harmony/core/types" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" "github.com/harmony-one/harmony/internal/ctxerror" @@ -332,6 +334,8 @@ func (node *Node) PostConsensusProcessing(newBlock *types.Block, commitSigAndBit return } + // fmt.Println("Finished consensus->", node.NodeConfig.Port) + // Update last consensus time for metrics // TODO: randomly selected a few validators to broadcast messages instead of only leader broadcast // TODO: refactor the asynchronous calls to separate go routine. @@ -360,8 +364,44 @@ func (node *Node) PostConsensusProcessing(newBlock *types.Block, commitSigAndBit node.BroadcastMissingCXReceipts() // Update consensus keys at last so the change of leader status doesn't mess up normal flow + // Not just at end of epoch, but end of pre-staking now if shard.Schedule.IsLastBlock(newBlock.Number().Uint64()) { + type t struct { + ShardID uint32 `json:"shard-id"` + Count int `json:"count"` + Participants []string `json:"committee-members"` + } + // b1, _ := json.Marshal(t{node.Consensus.ShardID, len(node.Consensus.Decider.DumpParticipants()), node.Consensus.Decider.DumpParticipants()}) + // fmt.Println("before", string(b1)) + node.Consensus.UpdateConsensusInformation() + newKeys := node.Consensus.Decider.DumpParticipants() + myK := node.Consensus.PubKey.SerializeToHexStr() + myKeyInCommittee := false + + for _, k := range newKeys { + if k == myK { + myKeyInCommittee = true + break + } + } + + if myKeyInCommittee == false { + pair := core.NewKeys[node.myPort] + pub, priv := pair.Public, pair.Private + priKey := bls.SecretKey{} + pubKey := bls.PublicKey{} + pubKey.DeserializeHexStr(pub) + priKey.DeserializeHexStr(priv) + fmt.Println("new-pair", pair) + + node.NodeConfig.ConsensusPriKey = &priKey + node.NodeConfig.ConsensusPubKey = &pubKey + + node.Consensus.PubKey = &pubKey + node.Consensus.SetPrivateKey(&priKey) + } + } // TODO chao: uncomment this after beacon syncing is stable diff --git a/node/node_newblock.go b/node/node_newblock.go index 8f93ca5f4..646807232 100644 --- a/node/node_newblock.go +++ b/node/node_newblock.go @@ -1,6 +1,7 @@ package node import ( + "math/big" "sort" "time" @@ -120,6 +121,15 @@ func (node *Node) proposeNewBlock() (*types.Block, error) { node.Consensus.ShardID, node.Beaconchain(), ) + // fmt.Println("Update my keys, right", "on port", node.NodeConfig.Port) + + if node.Beaconchain().CurrentHeader().Epoch().Cmp(big.NewInt(1)) == 0 { + // fmt.Println("Update my keys, right", "on port", node.NodeConfig.Port) + // node.NodeConfig.ConsensusPriKey + // node.NodeConfig.ConsensusPubKey + } + + // fmt.Println("super-comm", shardState.JSON()) if err != nil { return nil, err } @@ -133,6 +143,7 @@ func (node *Node) proposeNewBlock() (*types.Block, error) { return node.Worker.FinalizeNewBlock(sig, mask, node.Consensus.GetViewID(), coinbase, crossLinks, shardState) } +// TODO is this still needed? func (node *Node) proposeLocalShardState(block *types.Block) { logger := block.Logger(utils.Logger()) // TODO ek – read this from beaconchain once BC sync is fixed diff --git a/shard/committee/assignment.go b/shard/committee/assignment.go index 2c332e934..63b0bcfac 100644 --- a/shard/committee/assignment.go +++ b/shard/committee/assignment.go @@ -203,7 +203,7 @@ func eposStakedCommittee( } } } - + // fmt.Println("epos-based-committee", superComm.JSON()) return superComm, nil } @@ -313,5 +313,6 @@ func (def partialStakingEnabled) Compute( stakedSlots := (instance.NumNodesPerShard() - instance.NumHarmonyOperatedNodesPerShard()) * int(instance.NumShards()) + fmt.Println("Hit staking epoch -- compute") return eposStakedCommittee(instance, stakerReader, stakedSlots) } diff --git a/shard/shard_state.go b/shard/shard_state.go index edf632ded..ef52b53b2 100644 --- a/shard/shard_state.go +++ b/shard/shard_state.go @@ -54,7 +54,7 @@ type Committee struct { func (ss State) JSON() string { type t struct { NodeID - EcdsaAddress string `json:"one-address"` + EcdsaAddress string `json:"ecdsa-address"` } type v struct { Committee From e3802cd31b0dda3336b5cff7303908f9a1785ed6 Mon Sep 17 00:00:00 2001 From: Edgar Aroutiounian Date: Tue, 19 Nov 2019 22:47:46 -0800 Subject: [PATCH 12/26] [slashing][reward][committee] Begin wrap up for merge with master, focus on rewards --- consensus/consensus.go | 4 - consensus/consensus_service.go | 28 +-- consensus/consensus_v2.go | 33 ++- consensus/quorum/one-node-one-vote.go | 23 +- consensus/quorum/one-node-staked-vote.go | 141 ++++++++++-- consensus/quorum/quorum.go | 33 ++- core/blockchain.go | 261 ++--------------------- internal/chain/reward.go | 22 ++ node/node_handler.go | 56 ++--- node/worker/worker.go | 38 ++-- numeric/decimal.go | 10 + shard/committee/assignment.go | 110 +++------- shard/shard_state.go | 1 + staking/effective/calculate.go | 53 +++-- 14 files changed, 352 insertions(+), 461 deletions(-) diff --git a/consensus/consensus.go b/consensus/consensus.go index 43b9c7e19..fc59d5a2c 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -160,10 +160,6 @@ func (consensus *Consensus) SetCommitDelay(delay time.Duration) { consensus.delayCommit = delay } -func (consensus *Consensus) SetPrivateKey(priKey *bls.SecretKey) { - consensus.priKey = priKey -} - // DisableViewChangeForTestingOnly makes the receiver not propose view // changes when it should, e.g. leader timeout. // diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index ba8b2f1ab..6347f6ca1 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -263,7 +263,7 @@ func (consensus *Consensus) verifySenderKey(msg *msg_pb.Message) (*bls.PublicKey } if !consensus.IsValidatorInCommittee(senderKey) { - return nil, fmt.Errorf("Validator %s is not in committee", senderKey.SerializeToHexStr()) + return nil, fmt.Errorf("Validator %s is not in committee on shard: %d", senderKey.SerializeToHexStr(), consensus.ShardID) } return senderKey, nil } @@ -458,21 +458,9 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode { hasError := false header := consensus.ChainReader.CurrentHeader() epoch := header.Epoch() - - // fmt.Println("update-consensus", - // epoch, - // consensus.Decider.Policy() != quorum.SuperMajorityStake, - // consensus.ChainReader.Config().IsStaking(epoch), - // ) - - if consensus.Decider.Policy() != quorum.SuperMajorityStake && - consensus.ChainReader.Config().IsStaking(epoch) { - fmt.Println("Hit new decider on quorum") - consensus.Decider = quorum.NewDecider(quorum.SuperMajorityStake) - } - _, curPubKeys := committee.WithStakingEnabled.ComputePublicKeys( - epoch, consensus.ChainReader, int(header.ShardID()), - ) + curPubKeys := committee.WithStakingEnabled.ComputePublicKeys( + epoch, consensus.ChainReader, + )[int(header.ShardID())] consensus.numPrevPubKeys = len(curPubKeys) consensus.getLogger().Info().Msg("[UpdateConsensusInformation] Updating.....") if shard.Schedule.IsLastBlock(header.Number().Uint64()) { @@ -480,11 +468,9 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode { consensus.SetEpochNum(epoch.Uint64() + 1) consensus.getLogger().Info().Uint64("headerNum", header.Number().Uint64()). Msg("[UpdateConsensusInformation] Epoch updated for next epoch") - _, pubKeys = committee.WithStakingEnabled.ComputePublicKeys( - new(big.Int).Add(epoch, common.Big1), - consensus.ChainReader, - int(header.ShardID()), - ) + pubKeys = committee.WithStakingEnabled.ComputePublicKeys( + new(big.Int).Add(epoch, common.Big1), consensus.ChainReader, + )[int(header.ShardID())] } else { consensus.SetEpochNum(epoch.Uint64()) pubKeys = curPubKeys diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 0e025a943..44561a1f6 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -322,16 +322,33 @@ func (consensus *Consensus) onPrepare(msg *msg_pb.Message) { // fmt.Println("in-committee-", consensus.ShardID, v) // } - if err != nil { - fmt.Println("On Prepare is busted =/") + type t struct { + Participants []string `json:"committee-members"` + ShardID uint32 `json:"shard-id"` + MissingID string `json"missing-key"` + CurrentEpoch uint64 `json:"current-epoch"` + CurrentObject uint64 `json:"current-epoch-from-obj"` + PossibleError string `json:"possible-error"` + } - // type t struct { - // Participants []string `json:"committee-members"` - // ShardID uint32 `json:"shard-id"` - // } - // b, _ := json.Marshal(t{consensus.Decider.DumpParticipants(), consensus.ShardID}) - // fmt.Println(string(b)) + // e := "" + // if err != nil { + // e = err.Error() + // } + + // b, _ := json.Marshal(t{ + // consensus.Decider.DumpParticipants(), + // consensus.ShardID, + // senderKey.SerializeToHexStr(), + // consensus.ChainReader.CurrentHeader().Epoch().Uint64(), + // consensus.epoch, + // e, + // }) + // fmt.Println(string(b)) + + if err != nil { + // fmt.Println("On Prepare is busted =/", err, "on shard-", consensus.ShardID) utils.Logger().Error().Err(err).Msg("[OnPrepare] VerifySenderKey failed") return } diff --git a/consensus/quorum/one-node-one-vote.go b/consensus/quorum/one-node-one-vote.go index e1ca4bbbc..1231ff209 100644 --- a/consensus/quorum/one-node-one-vote.go +++ b/consensus/quorum/one-node-one-vote.go @@ -1,6 +1,7 @@ package quorum import ( + "encoding/json" "math/big" "github.com/ethereum/go-ethereum/common" @@ -12,6 +13,7 @@ import ( type uniformVoteWeight struct { DependencyInjectionWriter + DependencyInjectionReader SignatureReader } @@ -41,9 +43,9 @@ func (v *uniformVoteWeight) IsRewardThresholdAchieved() bool { return v.SignersCount(Commit) >= (v.ParticipantsCount() * 9 / 10) } -// func (v *uniformVoteWeight) UpdateVotingPower(effective.StakeKeeper) { -// NO-OP do not add anything here -// } +func (v *uniformVoteWeight) UpdateVotingPower(shard.NodeIDList) { + // NO-OP do not add anything here +} // ToggleActive for uniform vote is a no-op, always says that voter is active func (v *uniformVoteWeight) ToggleActive(*bls.PublicKey) bool { @@ -78,3 +80,18 @@ func (v *uniformVoteWeight) ShouldSlash(k shard.BlsPublicKey) bool { // fmt.Println("Called here for key:", k.Hex()) return false } + +func (v *uniformVoteWeight) JSON() string { + s, _ := v.ShardIDProvider()() + + type t struct { + Policy string `json"policy"` + ShardID uint32 `json:"shard-id"` + Count int `json:"count"` + Participants []string `json:"committee-members"` + } + + members := v.DumpParticipants() + b1, _ := json.Marshal(t{v.Policy().String(), s, len(members), members}) + return string(b1) +} diff --git a/consensus/quorum/one-node-staked-vote.go b/consensus/quorum/one-node-staked-vote.go index 1d6eb9020..22ab7879d 100644 --- a/consensus/quorum/one-node-staked-vote.go +++ b/consensus/quorum/one-node-staked-vote.go @@ -1,21 +1,26 @@ package quorum import ( + "encoding/json" + "fmt" "math/big" "github.com/ethereum/go-ethereum/common" "github.com/harmony-one/bls/ffi/go/bls" + "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/numeric" "github.com/harmony-one/harmony/shard" "github.com/harmony-one/harmony/staking/slash" ) var ( - twoThirds = numeric.NewDec(2).QuoInt64(3).Int + twoThirds = numeric.NewDec(2).QuoInt64(3) + hSentinel = numeric.ZeroDec() ) type stakedVoter struct { isActive, isHarmonyNode bool + earningAccount common.Address effective numeric.Dec } @@ -24,9 +29,8 @@ type stakedVoteWeight struct { DependencyInjectionWriter DependencyInjectionReader slash.ThresholdDecider - // EPOS based staking - validatorStakes map[[shard.PublicKeySizeInBytes]byte]stakedVoter - totalEffectiveStakedAmount *big.Int + validatorStakes map[[shard.PublicKeySizeInBytes]byte]stakedVoter + total numeric.Dec } // Policy .. @@ -34,47 +38,107 @@ func (v *stakedVoteWeight) Policy() Policy { return SuperMajorityStake } -// We must maintain 2/3 quoroum, so whatever is 2/3 staked amount, -// we divide that out & you // IsQuorumAchieved .. func (v *stakedVoteWeight) IsQuorumAchieved(p Phase) bool { // TODO Implement this logic + // fmt.Println("is quorum achieved") + // soFar := numeric.ZeroDec() + w := shard.BlsPublicKey{} + members := v.Participants() + + for i := range members { + w.FromLibBLSPublicKey(members[i]) + // isHMY := v.validatorStakes[w].isHarmonyNode + if v.ReadSignature(p, members[i]) == nil { + // + } + } + return true } // QuorumThreshold .. func (v *stakedVoteWeight) QuorumThreshold() *big.Int { - return new(big.Int).Mul(v.totalEffectiveStakedAmount, twoThirds) + // fmt.Println("check quorum threshold") + return v.total.Mul(twoThirds).Int } // RewardThreshold .. func (v *stakedVoteWeight) IsRewardThresholdAchieved() bool { // TODO Implement - return false + // fmt.Println("check threshold") + return true } -// HACK -var ( - hSentinel = big.NewInt(0) - hEffectiveSentinel = numeric.ZeroDec() -) - // Award .. func (v *stakedVoteWeight) Award( Pie *big.Int, earners []common.Address, hook func(earner common.Address, due *big.Int), ) *big.Int { - // TODO Implement - return nil + payout := big.NewInt(0) + last := big.NewInt(0) + count := big.NewInt(int64(len(earners))) + + s, _ := v.ShardIDProvider()() + fmt.Println("Award called on shard as staked vote", s) + proportional := map[common.Address]numeric.Dec{} + + for _, details := range v.validatorStakes { + if details.isHarmonyNode == false { + proportional[details.earningAccount] = details.effective.QuoTruncate( + v.total, + ) + } + } + + for i := range earners { + cur := big.NewInt(0) + + cur.Mul(Pie, big.NewInt(int64(i+1))).Div(cur, count) + + diff := big.NewInt(0).Sub(cur, last) + + // hook(common.Address(account), diff) + + payout = big.NewInt(0).Add(payout, diff) + + last = cur + } + + return payout } -// UpdateVotingPower called only at epoch change, prob need to move to CalculateShardState -// func (v *stakedVoteWeight) UpdateVotingPower(keeper effective.StakeKeeper) { -// TODO Implement -// } +func (v *stakedVoteWeight) UpdateVotingPower(staked shard.NodeIDList) { + s, _ := v.ShardIDProvider()() -func (v *stakedVoteWeight) ToggleActive(*bls.PublicKey) bool { - // TODO Implement - return true + v.validatorStakes = map[[shard.PublicKeySizeInBytes]byte]stakedVoter{} + v.Reset([]Phase{Prepare, Commit, ViewChange}) + + for i := range staked { + if staked[i].StakeWithDelegationApplied != nil { + v.validatorStakes[staked[i].BlsPublicKey] = stakedVoter{ + true, false, staked[i].EcdsaAddress, *staked[i].StakeWithDelegationApplied, + } + v.total = v.total.Add(*staked[i].StakeWithDelegationApplied) + } else { + v.validatorStakes[staked[i].BlsPublicKey] = stakedVoter{ + true, true, staked[i].EcdsaAddress, hSentinel, + } + } + } + + utils.Logger().Info(). + Uint32("on-shard", s). + Str("Staked", v.total.String()). + Msg("Total staked") +} + +func (v *stakedVoteWeight) ToggleActive(k *bls.PublicKey) bool { + w := shard.BlsPublicKey{} + w.FromLibBLSPublicKey(k) + g := v.validatorStakes[w] + g.isActive = !g.isActive + v.validatorStakes[w] = g + return v.validatorStakes[w].isActive } func (v *stakedVoteWeight) ShouldSlash(key shard.BlsPublicKey) bool { @@ -86,3 +150,34 @@ func (v *stakedVoteWeight) ShouldSlash(key shard.BlsPublicKey) bool { return false } } + +func (v *stakedVoteWeight) JSON() string { + s, _ := v.ShardIDProvider()() + + type t struct { + Policy string `json"policy"` + ShardID uint32 `json:"shard-id"` + Count int `json:"count"` + Participants []string `json:"committee-members"` + TotalStaked string `json:"total-staked"` + } + + members := v.DumpParticipants() + parts := []string{} + for i := range members { + k := bls.PublicKey{} + k.DeserializeHexStr(members[i]) + w := shard.BlsPublicKey{} + w.FromLibBLSPublicKey(&k) + staker := v.validatorStakes[w] + if staker.isHarmonyNode { + parts = append(parts, members[i]) + } else { + parts = append(parts, members[i]+"-"+staker.effective.String()) + } + } + b1, _ := json.Marshal(t{ + v.Policy().String(), s, len(members), parts, v.total.String(), + }) + return string(b1) +} diff --git a/consensus/quorum/quorum.go b/consensus/quorum/quorum.go index 3e816bbc9..80615969f 100644 --- a/consensus/quorum/quorum.go +++ b/consensus/quorum/quorum.go @@ -5,6 +5,7 @@ import ( "math/big" "github.com/harmony-one/bls/ffi/go/bls" + "github.com/harmony-one/harmony/numeric" "github.com/harmony-one/harmony/shard" "github.com/harmony-one/harmony/staking/slash" // "github.com/harmony-one/harmony/staking/effective" @@ -45,6 +46,19 @@ const ( SuperMajorityStake ) +var policyNames = map[Policy]string{ + SuperMajorityStake: "SuperMajorityStake", + SuperMajorityVote: "SuperMajorityVote", +} + +func (p Policy) String() string { + if name, ok := policyNames[p]; ok { + return name + } + return fmt.Sprintf("Unknown Quorum Policy %+v", byte(p)) + +} + // ParticipantTracker .. type ParticipantTracker interface { Participants() []*bls.PublicKey @@ -81,13 +95,19 @@ type DependencyInjectionReader interface { ShardIDProvider() func() (uint32, error) } +//WithJSONDump representation dump +type WithJSONDump interface { + JSON() string +} + // Decider .. type Decider interface { SignatureReader DependencyInjectionWriter slash.Slasher + WithJSONDump ToggleActive(*bls.PublicKey) bool - // UpdateVotingPower(keeper effective.StakeKeeper) + UpdateVotingPower(shard.NodeIDList) Policy() Policy IsQuorumAchieved(Phase) bool QuorumThreshold() *big.Int @@ -155,7 +175,7 @@ func (s *cIdentities) SlashThresholdMet(key shard.BlsPublicKey) bool { func (s *cIdentities) DumpParticipants() []string { keys := make([]string, len(s.publicKeys)) - for i := 0; i < len(s.publicKeys); i++ { + for i := range s.publicKeys { keys[i] = s.publicKeys[i].SerializeToHexStr() } return keys @@ -251,6 +271,7 @@ func newMapBackedSignatureReader() *cIdentities { type composite struct { DependencyInjectionWriter + DependencyInjectionReader SignatureReader } @@ -266,10 +287,12 @@ func (d *depInject) ShardIDProvider() func() (uint32, error) { func NewDecider(p Policy) Decider { signatureStore := newMapBackedSignatureReader() deps := &depInject{} - c := &composite{deps, signatureStore} + c := &composite{deps, deps, signatureStore} switch p { case SuperMajorityVote: - return &uniformVoteWeight{c.DependencyInjectionWriter, c} + return &uniformVoteWeight{ + c.DependencyInjectionWriter, c.DependencyInjectionReader, c, + } case SuperMajorityStake: return &stakedVoteWeight{ c.SignatureReader, @@ -277,7 +300,7 @@ func NewDecider(p Policy) Decider { c.DependencyInjectionWriter.(DependencyInjectionReader), c.SignatureReader.(slash.ThresholdDecider), map[[shard.PublicKeySizeInBytes]byte]stakedVoter{}, - big.NewInt(0), + numeric.ZeroDec(), } default: // Should not be possible diff --git a/core/blockchain.go b/core/blockchain.go index 2d53e650a..ed12c4d62 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -18,11 +18,11 @@ package core import ( + "bytes" "errors" "fmt" "io" "math/big" - "math/rand" "sync" "sync/atomic" "time" @@ -36,7 +36,6 @@ import ( "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" - "github.com/harmony-one/bls/ffi/go/bls" "github.com/harmony-one/harmony/block" consensus_engine "github.com/harmony-one/harmony/consensus/engine" "github.com/harmony-one/harmony/core/rawdb" @@ -2439,256 +2438,26 @@ func (bc *BlockChain) CurrentValidatorAddresses() []common.Address { return filtered } -const ( - // Pick big number for interesting looking one addresses - fixedRandomGen = 98765654323123134 - fixedRandomGenStakeL = 40 - fixedRandomGenStakeH = 150 -) - -var ( - // By fixing the source, we have predicable sequence - sequenceL = rand.New(rand.NewSource(42)) - sequenceH = rand.New(rand.NewSource(84)) - accountGenerator = rand.New(rand.NewSource(1337)) - blsKeyGen = rand.New(rand.NewSource(4040)) - blsSlotsGen = rand.New(rand.NewSource(8080)) -) - -type TempTest struct { - Public string `json:"public-key"` - Private string `json:"private-key"` -} - -var ( - tempBank map[common.Address]*staking.Validator = map[common.Address]*staking.Validator{} - addrs []common.Address - - NewKeys map[string]TempTest = map[string]TempTest{ - // Shard0 - "9012": TempTest{"e2c13c84c8f2396cf7180d5026e048e59ec770a2851003f1f2ab764a79c07463681ed7ddfe62bc4d440526a270891c86", "19e9adb6f6a10a160b951c33fee81e53beb7e41ba1b500626a1082215faba010"}, - "9010": TempTest{"1480fca328daaddd3487195c5500969ecccbb806b6bf464734e0e3ad18c64badfae8578d76e2e9281b6a3645d056960a", "8e7874184b1fe76d23b6a3e20b7178928123e7e4e63ae642235c5a955983c26c"}, - "9108": TempTest{"9d657b1854d6477dba8bca9606f6ac884df308558f2b3b545fc76a9ef02abc87d3518cf1134d21acf03036cea2820f02", "f2b677a9cc5b2325843d493dbc1a970d40c7edc032ff848a6116fcfe9a465015"}, - // Shard1 - "9011": TempTest{"72ccee6c5767a21d9b659a510482acca9b2303055d8efe09c2b55bd7d8910de4521036274c8a3a30564f0e9378b05189", "df8c569d2ed038f34408dcf980895de448bd25866192cc994aeb094982d7f769"}, - "9013": TempTest{"249976f984f30306f800ef42fb45272b391cfdd17f966e093a9f711e30f66f77ecda6c367bf79afc9fa31a1789e9ee8e", "ba742bf554447d0a6bb8da47f9134817c3f8a5c3717ed50e6c3fe6bc488fbc67"}, - "9107": TempTest{"ec143fa8ca3febd4b8b8d160b214a7135ab48f13d5f37ffce22607959e5da5b5852c8a8403ba0836fe64645fc1d92f97", "8ac153f6efc02cb4eb7c566a5fe89dd9bbe6da3df0c68b3e5dd5f4887b09725f"}, - } -) - -var ( - accounts = [...][]string{ - {"14cab59f60837090b428a4fe6bca8f82c0bd995322feec4684c444d8f6e7c6c0a6c3f1705b0d285fe0257b266626a316", "1c733e5866e2f615e4d299e493f6593f4f282d10e35ca0362dfc4fba526be863"}, - {"d3c9c227f945c8a7dc0857884710a9904e29b9aeb8ffe1460e978568385cf3a1b6d2a73b47af0c2bb4851aff684ed383", "20667b58e47025b8b4a5524e5fd5fcd094f0126926098d8cba91cf8c57828237"}, - {"ea50cd0a7263d5530b633c5056d183388ac6fc5dfa6e7a8b59ed2c0ca9025d27d54d78e45a817bc3821a5d96d5ae7c97", "353a96f7484637fc5c806b14ed409c89c884f97aec14152be2dc02a95c02a256"}, - {"56b1e97e79fe37e19a5876c1f0e24e12e4252a086088221f73ad43174f477c02b2083e301216791092a8aa2633428883", "255f3197ee303e20a42223ea9dbe1cd2791c6c91d50ff7d2f406d480e133db22"}, - {"b166d48dec6934dbf6c446742de9adee53c7a6872f63e62611634382b163b7a1e0400996674db8c8d6910c4d8d935889", "083d9eb3915564ecaa4997ad4011e42a660287bf0175bd46cbfc5ec766b97d54"}, - {"9cf1e8c8cc7e32cdbdb762d464032a6fa5b74e7dbf916909ca22a57ff0c01856e350e2a7c2b1c74d8871d0c9af70640e", "a506f6895829956c8c514868f7283fcd6833e7cadc4c62e57365678f9417622a"}, - {"e43ac882bdac2bfd79f0645d8a97263ba6aede152400c946ff6670eee1da6154dade87eb43c5fc13316ec3cffbbc8b8f", "2d5ed7ebf50eb0a3586800d691d28b4ba4ff701afd3f1246dfa216b89cc9131a"}, - {"40b33fbbb15a002693bd5a4a7da86e45941f66c82c8c41cbaecadcf3d1956dd4a19208d69908663bcbc606a4c9f67002", "40c57ce993f53c298434a4be3e78dfc35c45c336671e2e8c0cfdb56eda36dc20"}, - {"2b6d32cd1a32f813bb98991002d42703cdb00be6d5f4f319204d2a539da58a14c5a9b3d8bb74a16c054c55339a53ae08", "b52394fc0fc094e71a1a64ef25897db58abc6cd1bb172672e4c0382d5b4a7826"}, - {"444ac7f6f16a4f023c18d7e7acf2cf8ebb96a2473e02a403f2252839b0309851df3abb1d24f7ae4819c7e9283e818897", "3043bbfddc4d950ef65f45e364c5e28b416abc5d5643dd14d59d19410f25f65d"}, - {"dcdc15f99ecd08d6e8e7a8746fc781d0113855ef6866c9d03f445f5b8bf4bde005148b28b75a3c7c3a0f37ac49e77986", "96aedec3f2b3d48c8464f04dcaf4dbf4c27b7108af836d0581d347e569d29344"}, - {"1c4b625ac8424d7e45df5aca7179113af2edeea36457418ae84a1294e54f4b740309fc1b4e4b981b6f51ac0a68dd5791", "15a1b5aa27fe98682db715ad177e8e3c661ad917405a135b18acdfb73a71881e"}, - {"7e72cbeb6152d45903cb476cef4c2e65eb6a5e0fa8d90d46e0bba4ff16853969c73bbaee66d0c3f47a556c502f87a319", "c4cdb2f30ccbad6200fee58cc622c8849139484599cb43b65e52aac55d888a4e"}, - {"1fdf24b10e748798be733d4be7d8c5d18fde8800e3828cf6e7ec66beb87c55aae345bfb5b6321ff059089dcaef67e696", "be1e345b4f85583750c6ed3a912d613d9a47ae06e79492d8a7a63362ed02dd06"}, - {"c29b37816e1ddf0b7736af7ed22f4ddcab26dbb4bdd7a5347474aeffb9d933e81a14b66e98018c4dc125b2b0c1c1f605", "65e2a6aad957d23917ef958e9465b10bf0271122820682bd9360225e0b17c667"}, - {"7990b4f5975e399398b82bdf87eba6852dfff49ce25bcc88a360861a660c1fcee3a1f994a4879f482f467fc53aa54e13", "12deb9915b4b28ea7b822467030910878bfcc1cc27d3f0147dd0cd9213b77718"}, - {"27877ce5cbed77848fa7f894c2b4a7f8fd11d0e31e0c49f88682ef063c44da291a728797e6e5852d0bcec659bf05dd85", "7a84694b1084c2f79d0f61cb46e7ff724a31f3774194e535eedc51df3e89e214"}, - {"73a0feab8fc245419d8459e3d50e5133a835a69050c417e501f826873149c645cf1290619a054d1b4485b9175c69a092", "27ff614de69f7da721c0ec240af78c928ee60418940869b9dbeba6e939f9df70"}, - {"dca05a3e1d080b69496beb131b2c18dfbccf9c5fb470f16bb4a39b5c3aff19fb6aa37b115009ca7724038059dbfa2d02", "ae84b73e496fcf5f5661773ba92125b3d19f53f9539e0ef5f8cabd7c5a739321"}, - {"0a9221c81cfb6340e4e40a46403970d6f929c090bc4169540bbcbe02fb0ad6b06941dcb192677b9e2204a5ef27a24487", "a18c0fbea616989fc31e919c1f8090dc42cf02f410808ab1573d0b1618f3ec00"}, - {"e03d395369fee3c9bc7261bbb17bfb5eaf09e0b88978cb70a6c18f9860c7b68f231db64dd69d04035683762866febe00", "e7ac0c215d3fc055426dfdb4a05b4c48d41c7e850d6c5d7ecb24a6a5061e750c"}, - {"6a58a3ba54421cfd0bf2c8b15b92d1f9015ffc4d6e76852c5609b092a14a5ed2fa49ba1183414640e3c2c43e41b55286", "7d09db6b6fd3f9fa03ef6f788f59c12292250950ebe37e0def2db46c1af8325f"}, - {"adda915c4ca55d78c7dc595bc26452a3cd1a8cc117fbd30bfdd35ab234519566b4ad0aa581ae6ac64812990eabdd1204", "4fd18f56ad25de60b60caedc2c3b14eb8b2cc30e2a258b3f1348fa337b25ae59"}, - {"af1ba6ca3e3ba7436e3054d4b5cb3e19b7b1e0403bfbb76a4e25a1ed0fb75258d9fdf53f609394e002e865bfe525988b", "9cbaa24d59024b19a14f399a8183f4ab8a2558714eb01c3a8ff35890e6750908"}, - {"dcbfe47fdfee29df070c078609be69c412e44b5f7dd10b1ef58ed368b7a8c422f1c3b70debed3441c0cf15ad7d03df05", "82d910e82dd49e79f13829ed74f8562760df0956a7ae7e60ed1cfec9ef2f9727"}, - {"a61d2dfea77bd324d2188181487a99e8afa12b09ebe41e5bd031f24a07eef785fb0ef80f3425ed44ab1023a5a41fac81", "015e5b8a897f4bad686f29b8c1fcc81c1654dff616fd21b1664194369d0c5e62"}, - {"a82bdbd2c017fe51aee694151138a7a1b9df848c7998ad7a9fb1df2003f0d52cc99099d5fc12a3211f91d411c3553b11", "a49de622d95e586b1bf55b53d18364ff46bab263e73e2b03f00fe5d69923d248"}, - {"390851593559be7658ac6c342422f4a447a8291dc517cb1809e150e6b5fd538ab60e1b9bdbe6941ff78fde71ca68f989", "2f5ba8ef8e25999004a5c319cc5a5a4c62ac5c6a0bfc8d25419bfeac91c07f73"}, - {"c04d3fd26431a4d0ebe1ed385b46ad975f6a663365b891dd1b25a731cb7b4295f88b920451337152480571006eafa180", "e8538488afc01b044e01f6eaad582949c792160906fcb7d6047d174ecf2e7b39"}, - {"02e8a378a863080f06f4714161a4b592550e6889e15f296bdb33a156675a37af13a03522973e7329ba0bf8c79dfb4d06", "8b57d172579dbe19c21dfcf00d47ac5eced33a531c298df13038350c5aea391b"}, - {"a39b6cff47d06e9857c94da5229f814ebeaf041a704dcbff8d64224b5fa996e923e015e1c36b9e48031f48948d816b86", "c6bb4e835e4c8949117606d6bdb9610a0ebaf7eafe66ff93e97e8f6d054b8a72"}, - {"226c9493a85f82fd2efd3d933cf0a90436e0d16ad86d06619f01b75d2b89371e51b9c8b9417e3737538f09382a657717", "ae0e6194d4b86bbbfd6a7d557f0b520a37c1d58350801c6b2fe691c8e8b7a617"}, - {"8b0d5f40e3d655f9b8e6e4cef1c005c02ccbee539d7a35e5b148233e2cba8d050a838e69a156c29e416746961513f819", "8539c03e72eb85ff3321593fbb7f8a86806f93756c52b611ccdc0fdc3849ac11"}, - {"72506fb7d6467a9be5dc91051d3639e8eeebf2cf9ef81b3d8a5e95a47c35c2636c74f21ac22ae8bb9e9f43cda4edbf02", "563cce730dab68d1d9a57d8ad092d48b9589b4fb0926a6e2a3d92b110cfc9b6e"}, - {"27614c62a76a38b79dddee0fd9142b1d18b1b2502a411b86a2e43eba3a01b6a87f11192dbc02492647173c77c2cd290d", "e31ad7aed3c46221e36f2bedf1ba30ef4c891d0dfb2ca271a695175224fe2907"}, - {"bd59cbb9514ea7e9a6d8b7d6b5d05497f66aae8d32b3b97657ffe13748963dccdc4d3ff48002dd2b6e05b59e7cd6090c", "65f1a41835f471005c8ab1a65b513372c4a40a68d4c8300313f1368a28d14064"}, - {"5f49373eba2af47b6f0ec9f9ab925b0e4d1769dddc656bdf2adc992eca654495222a1b6e89fd97e86c52b039bb7c3212", "7845b0f43a621204bfbe7d22ca91fdf9abf7eb6ba355bca2f1e650b33d69b20a"}, - {"511329bd1ccce0bb0d86751fb9dbfe7c7e3dd1176112f182145b1123a850e75b13df43c199153de8078b3d2ebbadb086", "0f800ac46d8ce7131f09532ee239ce14a8356375cd374c2d48081a02a3a1de61"}, - {"37df36cc1a203a38264442093e424ff7b56a854ec9ff07e12e316f0f5b0f0ae82262eef41418f74593daafbb3e12780b", "f6d241140bef139880c66042c2a77cd94d5183f982c83359849ded6de196a319"}, - {"6d66cc4174873e1df4c34f56c3d1d01281a2bfd25fbfbd45af64d4aaa5fcbcd889aabeeee6314a0061986074f9937b88", "d9da208d426380be14beee42db5e47f1950927af70bb1f7d30488ce829d28436"}, - {"e965403c1eecd5ed151e844b2d9bf601fb8d8d1fc86169e60346dc9bdfd4c5317312c02b9caada3c9dcecef61b21ba96", "8ff45b01787acd4b2d4a780c1bf3bea1495e6cba6328371028155d4c0fc0d773"}, - {"7bd979db856a697dce7da76feab7f6e3c036984ea766198c695dff95532f8599d4cf6e3c00e2455bfb4b5ce0b26b5192", "c83952ee8ac4b25452a3fcfc75ba07ce58c392878b7c519922fa4200d1169e40"}, - {"4c40089a3eddbef0886fd375302c5dcd7b56f83cf9ff48afa56389d9d7b8b0d6daa0f2e2241773fa7ab7f0cc77621917", "7cd8e8fac117a7e327a9d609f176dfd0445749ec823096ff3f29930047959303"}, - {"2694baa0e0bd4e24c192ae5bc4b3a096b2adb0026823298c4c96bacf062b591a1a3b557d6c81773a0bcd0e1031755b8e", "81ae0d5dd80d47acb9761f3734533fa9543a19ae1a1fd67b10555775f1dcea37"}, - {"68bc09c6277fe5a478a4b036e9ede1934b38e7ba86315360b7c75c1999b848df0bac7fe00f9d1e43ca1fa20c0a200c07", "8a4172905299d69fb005ea2650b42604d8900ce9967304648545e8c0267f6b0f"}, - {"77c4d9e0ec0f1cf6c981481f766b1395ba0912a8020a5ef1e15949ffa95c6844fba6a87cb7dc3a01774ab7e26360cb96", "09a98961b4ca28357fefb75ab183074e8e439a9109d4c8a12e06f1678fcf9c1a"}, - {"7fa504b0d97596010748564f736385fda141c12f1dee6bd1f0c3bf1793f5089a1c78db84ac0ca23dc48b764d356ea188", "26ee79bf3c2391bd9af142058f773b93c9d9b8659cc2899452ba9f3eb017151c"}, - {"ddf52e6c80dc39fa940e85265ef09ade1a44e3c83eb5bc2798555ca3ecd5ec8438db21cbfe79be0f4ccdbffb22699d08", "2350f67df6146af7aef3f36de4a815574aa33db097a22184ef11911014546a4e"}, - {"b0e476eb3d88e44fe6d654b78302009940f5f7ec80e0c236802554f223c8c4011a6732930e7425973210a6d669b92696", "6898e732291fc8fad1eeb76e42be8ac8c15abf6f028bc965bef92acfae389d3a"}, - {"ae7eff87ebc4c3ea47474367a10880139cde747de0be5278c7935f54e245c762294e1fa0094ede65e6f716a0c7ee7680", "d68df81e8b89abf32597524d52240c17d0be7e3a02bc34a2ed3d633307e72f17"}, - {"30b89a0199fb280398f0d65b200b4b45327368a45337f679993ec3e7d38969d08ba3e1387e1c85ab2c4b43ce674bc105", "8340272fa1b8c273102ffed66a3934d8320772dfd5fdffdbfb969fee97364464"}, - {"9542e021b19160efc08a5aca567f7d4cba277e25927b756f3e0ad535b74caa8b3d62216c90304625f3c4293827c3c38c", "01c590a4f2bd80c5411d744b2223f5851d2e1c46731dc3a730902ad017e67c5a"}, - {"7ed1287d91d29e1fbdf43360a494f53d450fab14e3b9cc08a63f06c868d757c32e781622603a37057c909d57f1901088", "f56915dc69dea77304cab132e8bc247e568a0b2b11672a39b84c38c9dcc49468"}, - {"514051f789312e3a1851ad56ae4f64bf798c1147d99065903f5aa5807e938ce6cafa0b4122b0e3b6636f425241b2ce09", "1bb85c34055d3a9e12482136a65a837078119927296ebb5ae6a7ddd0c0aba514"}, - {"f8727e55841bd8ae53db38e67377b58e882072e937f54cdcf70c4c7f151fe7b175110d0962b66167de26c6074b6a8c17", "58847f00751f7d5bf9594957e47a88c773b16257627cd052b7180e1c0721e950"}, - {"b4225d7cbe4a290cf880dce13d65155b8ddb2d27ee495929338780d268ad96747f4013e7455fcd589ceadeecfc138a03", "3e305f7d0941507b251469d4628294582bd9ac3c995067606321422492e3442b"}, - {"82ec676e743abf1265d81b1bab873acc911d533cf9aec5043dbbaa7b29ee1163180cc84bb22d1a567c532e3d676cc281", "da0a4974718fd08ae9cd46a6d0217a67c3fb2758e81a2419d13bb2be56eb134e"}, - {"cce64ff99850372b795fd146688c9c8f54d6aab555cf624623e665ebaffc44b3587d7759a95968b4cc874380be096993", "440bbf61908f54e4801fcf215e62256b1d2bdac6a2c0cd940a20beed04cbb844"}, - {"0b10167f5ca2979864e46125df72aca33956173aee020ca88dec3e65a5e0d41c1f317ba930d66ef9075c9ab20595f58e", "bdd7446112ddf42ce0d468a01a0c1c08e45b6b76239336878922235216002210"}, - {"502ca2a064d45970776140c190ae8ff0472f8c7c3236f43d0d706e72a430ffff4050bb8214de2fcbec7e9c3b1577f688", "184d37b6d7f04f232e5f8262b28a4aa26dc68f2208810d02665383cbfa074835"}, - {"02e95502143d16ab34f082b4368113f49a6b7a4dbe33e10c172a1376add5405a934c2992a4e0755636e3faecc11abd88", "7dbaa1c7760334a0c6b7148fe91d7bde88d212317b45115643e29d0cb8484155"}, - {"9d0892d0e8e2474fcbe225b69b9560571842cbcd8a35f0ecd0d3cb9885a079dfb2d1669f886062f28e2ea347fb7aa014", "b5a7ed451787128bee2b256be308e5b0f73ec1f3504f0400fb801c4ae373c327"}, - {"c4349aa33c18eeda0be31e0a519c5a53cfc6940b995c88b5b34b81bc22310f3eb68a7d6abce6d7e609a6a269a75eeb8d", "5c142f86bbe3fd2f6e446060682587fec7dcef6f360fb1659bd380ecfba9c656"}, - {"ba97d3b920e7590afd420458e4dfbce3328a7890dc150439bc9f9b343d4267851eddad645940a14c14f9c9dd085bab85", "966ca693d191cd6dfe003862122adee65c96d1aeca2ec85aff716c3ffef4ad35"}, - {"6caaa02b4289dff76ddc7f8441bd83821847a7d1d2739303dca02b8510a3cb6037cb2562c2ad5a4528b555ff3128a618", "96bc5bcf9421b96e63c150d2ebdb9a75a85ff3854867462efe0178a7c4855623"}, - {"1467f6aa800e4476846e721d6cd684cd899aa46d87c5b6ba674f9f9565c2fb64cbfec5b4af7b55e86fa2d8604acdb884", "1ef689c2fdcb0e93ed3830311f146f37dc1df26b7027760292a3818b4964671a"}, - {"176f9edf6b38ce85d35f2d314287c03f2939a88230c2a493476a337133f9f2c00d1a69e0d24587b89597c400236d2f10", "c06afc1859c5148fc2c3820614739cc5132d50505fa7dc40e8a914e4b65c9847"}, - {"4a6a00e30b327b8d9a01ebce8b5c765fa1a5ebdd3d7a06207590e5a293ab8b38a2a197e386e678cac86826953f2c5301", "1fda30d5de8a0e8b9d725af8805996f7e8a31f36406dee7e1d1e046265c4de04"}, - {"5c33b153ee8c281b484c20f0a8d9626a1b13aa20d386347250996032df2cc607a307055ac46796de7e65b50e5149648e", "cc1a270c065654d949c5e9d2e158acdd3750655cb94ec94810041dd7a26c9e38"}, - {"0633521bb2c3c27477e97ed506587ab4994072e97f307b87a735a0b77fb48de51975114bd24ea4938063d4edcd1da313", "f555d422832a7f8f8309f61b94cc4dd96d77041684dd08e72396bca00c52b439"}, - {"e4ad16fd499040c0315c91eb645c0fba1b1f922772d39bd894c055735c9b9d483473d8b6321638e007bfcfcc1a67988f", "ba2d5d76f3178b954ad624c6af84467d4b55e43bfdd1add98dc50e9857514c02"}, - {"ea8353454271fc95a17543728255a4c3135ec6a48b82458ff18568b155d30f3449c480882bc77d38cd553195fde5dc12", "964fac9a31467514411d96a44cacf2a5a6f0a2db985a12dcfb87adade0cecb4c"}, - {"e298500b9c972c42cdac74a11a096ba2d64405d015fd5626e07561a297dc6c4f36fb5f1cb1c4d86894aec82d5e2ef691", "08bb76af47e9385c354655e9001ada326e811ae241c4d22f81a2111ad9baba44"}, - {"04e34592e6f062a9558fc09ca379585e15b4ed6131e2766354cce6796233db689c340ed6ee0b7e9f40e85ef7b50e6a13", "efc1f64c2d8cda2d1107c0046703ddf4cf7ecfc6f38adc4631736e83846a145d"}, - {"8c4b304f3a0c7310de817b6e958588cabb4124d0cdea74eecb8113a2daea0dad1ae203bd0d9ce228e2ee89d68965868a", "6e8f06bf1128b4a148b47ba8f2678398add67e0a8b11f81aa7adef4bd7d0c22d"}, - {"02f3d61f931c09d5954878afc08799e1b8de77b1756576e3a11ef836247bdc212bad1fd287712a815228c2b13f5b128f", "468049f68a279e8be38b0d3f0c67758e45cc5bd4b5a236fc7ef3eba78b45176a"}, - {"323245a1027c2a00b9fbb908840601a9bc2fd5808cc313a93b2f14714ebfd234dabe8984bc2169b09ad5327e2a3f6584", "5a65a9fa599fd7ef2140b7b80fd78842dcdbb4084f6e069ef427fee4cb2ca240"}, - {"62a08494a9808ba4fb15431dfb431c8d993afe6a4e2d18214b8ea8ebedcbd46594894d4afc0290739fb17d4f7b71c389", "1d73c784cbd13c5e7987c78540aa21aabc395fd2cbb50a6561677945c2558d39"}, - {"fd46f1477459c863f903ac7daba941e2c91031dc000740ca8d185a86939907bc9893ffe586dacdeabb1a3459c6e8a696", "04ca4066f6f3a49e8ff5ab3680a90bb2e17da973c4fb67022a6ceb3b05b1a152"}, - {"e71d1a01831e052622264ad4d7e7a9123ede454257501c7c60d214904d124dafb023dba7d48dfcc49d50fca5ef2c5098", "de295122f5c2982ad00cda3cf81eba0fb05c087969b01862d7e75569c931b856"}, - {"ea16cc1fd08e907911f247e6837ccbb8ebefbf8986fdbf2cb1e3c29254cf030ea211a279d8e368b61a48e49d1833db17", "1b3a20dfc6f2a25b1da99b2c836b1e8707d897a16313790ad0618f3c952dee10"}, - {"dc90ec2c5c2e82793bcc45aa9ab72fe40b68d63e394e2f8b3b654f1b8bc87cf5e94a2fef0b955acac1018ef526858f84", "0973be2092613e69d8d16d251b74e6ebbb39932f9fecad384cc96f917438b811"}, - {"cb7ce449200b1bbcec236273d2fd79bd6f633cfb21ea4720571d36a4ac0dfbe6d218f74d1093a7888652c2f82ab6820b", "568363bf1acd8ab07feaaf324401eb418883b936c720088a23a1867233083b35"}, - {"277544d9b04150060bf5b53a9e53e64216613e2e5356a2113c7e4706f16a0babe6353a585444495aa175cc157aa40087", "ea0a26d7406f3ecaeb602a129cc6052db891fdec7a11e6ee852e8dbf3520a91d"}, - {"92eeb95ca74d9f4e9deb0afb2f7a1f9973e2edfcd62c0c3d13c66b61bd3cef962198b5b5a2451934ea079ed1182d3591", "5a5fe61ae2e047bc0e8c81d5c5949749f4c419ad921a858e07524f2486f0a12b"}, - {"e307e3b4bc5da252d020a755aba61f3adf35b3bb56b3ec8e0ded021674b620fa14d6071ccbb19f04820955474458bb19", "d44f0e3a91f72df286ae784ea61ec12b7340fafe647ec25b8155d630b55a3b12"}, - {"f59598f47bfde137d5023f865861832949606c6256c235c697f01cfdc41583161ee6674ee89a1766c0a75e623be22509", "f310ef9048ed84f7474802b4ce8778582d1d91c6cb6e03af9741ed3c5759af3e"}, - {"249976f984f30306f800ef42fb45272b391cfdd17f966e093a9f711e30f66f77ecda6c367bf79afc9fa31a1789e9ee8e", "ba742bf554447d0a6bb8da47f9134817c3f8a5c3717ed50e6c3fe6bc488fbc67"}, - {"72ccee6c5767a21d9b659a510482acca9b2303055d8efe09c2b55bd7d8910de4521036274c8a3a30564f0e9378b05189", "df8c569d2ed038f34408dcf980895de448bd25866192cc994aeb094982d7f769"}, - {"08a71ab590ae132ca2e1a7211e0f90498629b55a0a1e2b753b43c5f1cae8cefb4f2494e4fed603f2b725cd893cbb0c90", "14fac9e0979b69f4c9ba3de4eaefa4e26615dfb67430d459d3eda58567849e57"}, - {"a59ffa292bb9162f55a83164c6162dd9b485b156d18c9d701649299da38aa5c3b2713ab9cac64132b98a3bece2061897", "8777f023e07075346c3ea3b2fe95f765284d952b3c04548085519fd9ff09dc61"}, - {"e2c13c84c8f2396cf7180d5026e048e59ec770a2851003f1f2ab764a79c07463681ed7ddfe62bc4d440526a270891c86", "19e9adb6f6a10a160b951c33fee81e53beb7e41ba1b500626a1082215faba010"}, - {"ec143fa8ca3febd4b8b8d160b214a7135ab48f13d5f37ffce22607959e5da5b5852c8a8403ba0836fe64645fc1d92f97", "8ac153f6efc02cb4eb7c566a5fe89dd9bbe6da3df0c68b3e5dd5f4887b09725f"}, - {"dcee54914821c267328842dfa7aa8fc993488505818384169562f1fde35067b3258600e9bec801ffa91b2d4d44b04580", "3c4816103eab7b9e4f279dac808ef79a1754a526682503c9dade0c1b9413a316"}, - {"cc120db9a2d189e58128b6c2c546be5a68c173de71b7f5062c184b3909faeed2699503aea5f2685a1b150685244cfe00", "a409124667a202e4ec48ae124dd1551e2f138090ad5aad51407cbcd5c2a56b5d"}, - {"5005ee176e5040d0cbd2ed7464ea826690e9aa8c846adee42eb438db86be34cac177f50ae66db56f1f2759e06cd4ca86", "c66c665432c441e90bd14746c8b8dcfb0284e16f0214fb4b48da97dd23ba9f0e"}, - {"9d657b1854d6477dba8bca9606f6ac884df308558f2b3b545fc76a9ef02abc87d3518cf1134d21acf03036cea2820f02", "f2b677a9cc5b2325843d493dbc1a970d40c7edc032ff848a6116fcfe9a465015"}, - {"1480fca328daaddd3487195c5500969ecccbb806b6bf464734e0e3ad18c64badfae8578d76e2e9281b6a3645d056960a", "8e7874184b1fe76d23b6a3e20b7178928123e7e4e63ae642235c5a955983c26c"}, - {"5139d2fbd1af771f76ecb93444d15a1a0bdb9706ef8149db9f88d9456e2b616a812f2bc21a320185c4619248950c9582", "3a0032951ccdcca696f82f16b16178f7d239db5444b6203292573ed1109e6466"}, - {"3e15ed2b396c2076d2df067e58dec3c5fc0d5f31ff3b20cd245bfcaa58d73db8373da74ce7b852d95863c1954f1f4986", "8adbbb081aa9885854b8ce9b6d437e30136d065f2835f11ea6a0b89b74509372"}, - {"e99f29a4791a9da645539d154acacb731f15906201e194a77aa7cf2cd4827692c28e7f3cb8d57a3dd0157d79c01e0492", "a68648839687f1bfab70ae1b8d41eab44cfdb00b599f4dfb5aae31cb77c1a60e"}, - {"744d03bb36aec3680fd894d3761d30bd8f9bddabfcc0465e80bb288a43c02801073eaf441a4f99f10776788dc6bcdd8f", "be1fec362830fd7457db13e45cc5e24bf9aa15c54d347385b112bfe18044ff2a"}, - {"a3b437fee6d9c0455b34b70c984e21ee8f1d6c17fb857462c67592b1994c72d72de61afbdc9fdfe5a90d598444b10516", "319f70b9d21899f6a9db38a9262e59a37212004ee8678f143d71bfb1b8bb5b54"}, - {"02dbc59d3d2a173f2799ec5d9010238c762311beb1cf63df03937e416c9c67e491ba5fb912c2e46488c022c009257806", "2c39b8517d58e6f11d71459ad5f074cace01335bffc572741da5845e357b2d26"}, - {"f26d9fe117f382c14128352de40ea9b662168f50a89cc10ccb20a3680c1ebfa5fe1efe84a7b1090e316ab86dbd734094", "f79aea85a3673fc026e1314139cbf9c12fbe5a688623465c966c175b69472f65"}, - {"f3b034ab8643ac72cd29f3151d124348cf459aa7ad69dc0f98782120f69ebc82a21df0bd1db01ff5f9fe304f4c7b0485", "e6fb858d69eff707637c4c1a2c14a9bbc8c7f05ea58b208b119214c236c1512b"}, - {"0a7559029ae550507facd6375a30f70cbc108407bb9c4fab8381fe5b8a30c0a2d99800b21a788b2d2f0e74f3c23c8b12", "25af65b82f969650b0c50d9916fd10798dda73f34382181e238106ebb9649652"}, - {"c4b9926ffc51e45404cecc06dfe0d652c9510651470fbe2b948d6730373d472b3a9a87e748f80785f22544afbb16ec97", "8a2e1d44667d5082350de9b05536de4b3de88becc68b99d4cd922fa006d6481c"}, - {"4a10b2749f91f9d8577f612b374274f8bd90e50d1294ec1bb043e4d0de590ae8adc0e48a7c494cf6559d9ee5bfb2ec82", "f2159355c79d46d5abd239a780d584846bb7f3622306e3dc6aef5d7646f8b037"}, - {"cf03b1d8da5a8b063e278215d8e6677e9cee9a09cd0ededb32e28e623cd43ac7fa72e571e786ac925253e56941170099", "4a4939ded1d7210f01decf8905967a8814d94c7e6901923e49c0c862bda1cb0d"}, - {"eb28831cc5257b4e85f824e4e1c71bd530eb60112f21a8284dc6272e11abc711041b64afebf202086b18917d765a2301", "09cf22643028d316bec1ecedddb2881064132939a7b63f50fc94509b6c5f5433"}, - {"5220fa5b230d78234a25adfefff480a3664405f231eb87c797b33fb9338e5bdcad58e3b80780bf7cfc0705a51fd5d519", "7e985ff6ad71d8968bd7dede437467a9bb120d67cbe26866b81e9df6153e9e73"}, - {"7402dc3baa41314482037cad9ac5487d0af9b8775b82b0cd4265af078a8b6af48358b884c3047c0c46f4bacb658c5b02", "12044340eb632a6ba4e9dc81f9bbcf33fa68792c45773ba51e4d62642c5ff761"}, - {"43f08c35d400b36299cce44c74064da8ad15c2f4896dfa9c5810b09197e8f26e5d0ab11db797f2dc72f3fdd8b8fe4b8b", "00165cab4cea976f26edc80afc9e5a4b15678b20cfbe3d7f38a630fb1824c44a"}, - {"4ad485a064eebbc26fc4c016b8ec979aa6816d1b847fe52111f677e9729f8793576946d67b9aa01265f98d4e437de988", "82e9fed6646457d008201d7cf489e0e308ab54235d1ef59a5ae484ad6e93ec34"}, - {"733cab5ae15056a40cf515acac9a5e283f0dc56e5f7b7a1aed4e258d4ca0477a9804f2ddd59a6c2eef0697538fb8c491", "015db82e40041eeffc1573d93427be038b26d7600cd0651771daf3a97f747701"}, - {"2abd73dd3f286c23e7bc0650a580b6bd29ce3a948067314629898d683b9e4c1d30924c01ff0813f7e912e7e69f07b28e", "c32154e8d23412450a33f46bc61716199d1316a3c3ae00ed7e374b0bc7fb3d0d"}, - {"5ec4b7f1636b7d565addb55c6cf182b044560296dc3c82c846b3faae8e485fef77ce9d8b0e6a04f832454fedbad46a00", "63cae9af8c9adcd402aecce77cf0de3912e4c6e1574cdbf2ed3046c1605f4b30"}, - {"160004f16449ca70b454f6260d8fccfd6b399e4a099ee05c745cea26ba48047373f105bbd426a7035fb582f8e6348291", "09bcf84d83fac97648715d60b7b70eee4e3abc03c0253bc5147913b9ed405717"}, - {"23b33c8771e70d1a84dd2f24b924c50a7c518a6e6f5a073f4c223ef299876689c4164a8f29fc21a531895750166e0b85", "c7c917686f6829fc79cb5020ca7271bee4cd2c3fe8c7b31ddca5e8c1c11e7b1e"}, - {"563a4f6d39adc91bd9033573a517e5a365b181b8b5b6f086f67bce1e09f80489a6b173f0ee02201e5d333dab3b1f9e0c", "452971a484dd203669f330cc5cc7a96febddb72ff8884d63a6ff225728c0ca46"}, - {"d6be59700f6d81ea11cc22481d70b972741d52f9495f1637e7e2a478ed99898a4cbe5a5fbd6fd2648a10ad09796a5811", "52a6e2b5c5da26e75b96894f1a443d948f6235873040a511b4a33202f36cde1e"}, - {"8fce243f7d8f4bb28195c428bc54c4a233d9ac7c204e75cb45771f1d4ba9725b3b9c69c24e390dd932ad930ef4741997", "1b3aab5c04b451cd714d55273f3749aaad4a45247911292ae84f02ccb6a30d1c"}, - {"56318730454846ca6d9d7b23f2de5b3a67b4b62fa057e4556c144d6f543e2e9defc031ba3057be59b5532bb0b2f81c06", "5f434becf1c975090deeda7bf15c5705c03b767db5bd338f400368baffba9825"}, - {"5f64a969b0ca0e06b78c8d6f0f655116dc526efa48115fc33320abfcf73dc1136a5833362dc3f9147a574fc0e5cb2f0c", "98a256ff411ebaacba2d0d1519a1f952b2e143dfe323202fed5046ccb24b2224"}, - {"4ba427d9624cca3f21a76e31793debe6c7adaa871693ee3d88e8e943c4934f941896ca7e1cdac50901eea77998504f99", "39b81fcf87f18e479a6abbe2a7011fe5992fae0beca763782d911cf5dc8e5b0c"}, - {"3a24c5b01460b83f48bc10eb56acb75252d800f3a57d42a04311d195dd63a0b768979c4a7291802d09b5339c3de84a88", "832df5a953bb748109d3346c2d9f1cd2268ae5026cb7d3251cf15e9ae60aad50"}, - {"77f8d0e62982ae95a91b31b2d1ec05035784c4d63c91d213c499a4e6fac839057c53c15eaff6716c150a3888f05a4a13", "1fb813936193004802f311f7b24e034ca52d121c4e715cbe9f3ce6dd3f819356"}, - {"f094a37f1631d595b40aec0ce15822f8a66e7403aad52858dc30e1a740ee83abe4a47d574725f90e9d2149d3afca5309", "b433b444d92f22f05f59b27e150442d1731c95e002610b5da3f265b0c25a6561"}, - {"86d8c9181fc53c83d5c5a59ff71a5552422a4d5f50125c8235605bbed442250ab5eedc2a9af96e72d9df185af0b4620c", "b70c758757675dce385eb2a6019ae4b9768836a29cb8faa50e8a8016c603871d"}, - {"69327dcb847bc70a01fad93968cc746628bf1422ff92b8b32202015ff75d85f8a3edd0a8c738c1d8c6a465ab895a6d13", "cdc4d41677731c59ae715be9a250fa0a674cabfe70b200afc1c11b651f977134"}, - {"7493640a45a5cecbf761ce67b4070d1864e27972ed3ca7c55fd7c1f191be3dc3b5fca52202d4c5fae1156d1659cf4c97", "488a0d363880eaa61904ceadfcdcde361321926b532001e9b33f9e066eb76730"}, - {"441e760d4d1d6fadc9dc4d67194375e14500c7c82d2fa8eb460bc9980dba72f213a34840ff10fd7ebf17cdf8215f7807", "bca8878d4b7da98864ca552843f7b0bb35bbc62b793ef0320b810549e50d0059"}, - {"e44a73e261e0650238743e889e8ba28d77f6d760e4dddc8979b64c2b76716443801b6335700a32d8da477494cafeb301", "d35d194f207d2f1e5032ef4f138c6d732fa371b259f8a954a2c7f4b1bd0cd94e"}, - {"73b97bd2114bd822efdb31fda3bd45d32b11ea2277d88a7e5b7de25e81363aa8a4f2d069111b08e215b933cd96705902", "7760a6f816e7d345554dc4559d9514eb5947226a349d464eda3a75f1af679a26"}, - {"123f4ba326f048e9a9f650e7769db67d17b6a20198aec99fe92c74a8055fc78a9ec28171a1a09ab6d508172100974a06", "097af324bc4757ba78553480f11c0ba001c3cc96590f8537215a1514329f560e"}, - {"0b859a8b28dfe7898f4f4e1cc346cc705da88403157f17813715d5c4e73e99fd2ac0d4c7afd1c87e8268c3f9380fdd05", "313a7f8c831827383a350b6c55a9d24b4532e63f610ac187bc7a92303ec5c604"}, - {"77ce9b523ebc8339e0bce80d249954797e9749be4bf73bdc5bec030a81207be9d76051982c5b80c628de39c4c8e22598", "9d1452e022fd6a280ce7baf4ac3a7151cd71af54f3d0894289f95aa39ad4d439"}, - {"cecdd9c3b88b68ea1a84c7d9e14e6a0112e27a8e7883730ddf129526b513f7630af673f9a89f5f28341163bc3f040e80", "07165b308d2924c30ca2a3e7205f46cd6094c8cfca62e1fb46b0e9ba4290043c"}, - {"da6e4afdafbd87cebe3a7bfa165d118912e2ff180795196f49b0129bd777d671d78486b2660c5528aaee27aa2c314d03", "350fcb263739dc7b718982a9180037da35d067886d7a569854497d6dca56fc39"}, - {"80e59409d68e0caeb5105c02234a861fd8bbc6962928d50f5f244e7ce73fb5425fc0668dd70207a3bcd0135372a8820e", "eb8af67f82be8f3ac0c01f4bf438e4215603cf07e4cf154932f9f4f2e220cd47"}, - {"dfe1f5b9f22293440603171f7b0234ed76a5c3db16de169e94a2d04615e32db6b5099ea9e7607aea9f41694b3ee2a387", "427880ecb5a18008be31af70ac35feba1b7ecb774750509c46fedc6fe87a3c03"}, - {"211c5709ec1d114b427d956a4d4433e3798ec214740ccd4c8dcc8688d8f5e2b54614f1b66c8032d76e8617ada8836e14", "dd00cdc07bc1454de81e99c7bcf08d96f5487f16c6b08c8c25642bf6f671671c"}, - {"b7f66b9f9f023c98b5e4c5ca751561d55914e96d8db25aa1a2d482f4dfa9c38049a1697b432c9500adf9b0a2cc8d1390", "f9679df2366b7076068dcb62478c436d488a1234f46bc5d23a2582c30504ce39"}, - {"24fc0f43dc21bfe7252bc26fd1ecd4c108b5ca6dcd6e213b4497ee3b9f39704a31ce4e4dd4dab8ecc4a0cccbbb9d050f", "818e23447a8c4028d57e3d450c7eb78312fca20a11168df831c629468f3da93e"}, - {"3cf72ba7a2e414f3e3e42591f19b7811cc86abb09b689bcb321bac2280f59e0a052bafbf0dc05f9671ef3d48e6ea0892", "7852d030e30ef02fccddade41dd11dad0761ff3e0de2bf80a220afda46e60a04"}, - {"525cf97855946b70709d2c56313ee51376082681c9a351a1829d49a6d18f337bdca6e5cf2219948b46b733dbe2657e05", "507f1cf227ecc51b58a6cb260a2ad88d0fae52b14df33080ffe634d125420d3a"}, - {"2d43b3290b7d056e9f3d470d5c16a5018dc102b25ab5ccff98c4a7610241c956352d56652466158dd92024d65ad8300a", "dc9a9e84e89c316d7e82f17b6af209f7eecfa0592514b86ed367a0f36e91894f"}, - {"16adbe52c89f86d6746e6135224eb0268db5b82fdc9f3332505daf7a79aab69b274cfc77ab63e0f6ecca5e1bf970fe17", "4464353707dd05051ef67ab31070b44e524c1a31f47681e7a88ae97142a13955"}, - {"1432dc6b2c5c1b546acade2a96482350f2d5369ae7e6eb7cbe5e6abc4680237336ad9715552a110ce7d15c75a5440d80", "93ff03d1944a65e99cbf205094246c19c0269dfd17e75c4094fdb7ad99151b43"}, - {"4f0b413335c109ac5fb56229816bd4aa6969678922125c9ecbbc1b371ed73dab6e743b6bc5ecedfd89a636dec3cb3396", "00de6ebd1259754b4d13117ed98410e6f794de52c91e85f827b2580b6a8e8a28"}, - {"2bd7f336338b4af9ee6ffb24ebd4076b7bc3805bc496325379649356c1d98cb7e1245cd184c885f6271e8c08d765a603", "309ab0f63625e579b9294209fa00b949ed92f1c2203ef7e149d8e9713a45a551"}, - {"7d86fe374d3e108b6988503428984b5a34e5b71746e5371c0f8df0ed29cc5852d89afb395a33503d580f6cef521b9e96", "49e42da63171901d51b62abc234100ecc9c07b22e6f180a747da15da66fa280e"}, - {"95137562374a17ed96a3272a05889e52630a7c7c4c3a7d4c389d9a067805ad246b259e60949260aabc704a89beb10c05", "d0d08d77a535903dcc509f016825a12f6f49219bfef6e16fcc6d29b9b30de30f"}, - } -) - -// Public key first, private key second -func init() { - const a = 100 - addrs = make([]common.Address, a) - - for i := 0; i < a; i++ { - addr := common.Address{} - addr.SetBytes( - big.NewInt(int64(accountGenerator.Int63n(fixedRandomGen))).Bytes(), - ) - addrs[i] = addr - someValidator := &staking.Validator{} - someValidator.Address = addr - low := sequenceL.Intn(fixedRandomGenStakeL) - high := sequenceL.Intn(fixedRandomGenStakeH) - r := sequenceL.Intn(fixedRandomGenStakeH) + 1 - modBy := high - low + 1 - if modBy <= 0 { - modBy *= -1 - modBy++ - } - someValidator.Stake = new(big.Int).Abs(big.NewInt(int64( - (r % modBy) + low, - ))) - const amt = 2 - pubKeys := make([]shard.BlsPublicKey, amt) - for j := 0; j < amt; j++ { - pair := accounts[i+j] - priKey := bls.SecretKey{} - pubKey := bls.PublicKey{} - - pubKey.DeserializeHexStr(pair[0]) - priKey.DeserializeHexStr(pair[1]) - - k := shard.BlsPublicKey{} - k.FromLibBLSPublicKey(&pubKey) - pubKeys[j] = k - } - - someValidator.SlotPubKeys = pubKeys - tempBank[addr] = someValidator - } - // b, _ := json.Marshal(TestBank) - // fmt.Println("keys", string(b)) -} - // ValidatorCandidates returns the up to date validator candidates for next epoch func (bc *BlockChain) ValidatorCandidates() []common.Address { - return addrs + list, err := bc.ReadValidatorList() + if err != nil { + return make([]common.Address, 0) + } + return list } // ValidatorInformation returns the information of validator func (bc *BlockChain) ValidatorInformation(addr common.Address) (*staking.Validator, error) { - return tempBank[addr], nil + state, err := bc.StateAt(bc.CurrentBlock().Root()) + if err != nil || state == nil { + return nil, err + } + wrapper := state.GetStakingInfo(addr) + if wrapper == nil { + return nil, fmt.Errorf("ValidatorInformation not found: %v", addr) + } + return &wrapper.Validator, nil } // DelegatorsInformation returns up to date information of delegators of a given validator address diff --git a/internal/chain/reward.go b/internal/chain/reward.go index 9b56d4260..f7f056f68 100644 --- a/internal/chain/reward.go +++ b/internal/chain/reward.go @@ -120,6 +120,28 @@ func AccumulateRewards( *big.Int }{} + // fmt.Println("Calling reward", rewarder) + + // hack := + + type t struct { + Count int `json:"count"` + Members []string `json:"members"` + ShardID uint32 `json:"shard-id"` + } + + // g := rewarder.(quorum.Decider).DumpParticipants() + + // if e := parentHeader.Epoch(); e.Cmp(big.NewInt(0)) == 0 { + // b, _ := json.Marshal(t{len(g), g, parentHeader.ShardID()}) + // fmt.Println("epoch", e, string(b), "\n") + // } + + // if e := parentHeader.Epoch(); e.Cmp(big.NewInt(1)) == 0 { + // b, _ := json.Marshal(t{len(g), g, parentHeader.ShardID()}) + // fmt.Println("epoch", e, string(b), "\n") + // } + totalAmount := rewarder.Award( BlockReward, accounts, func(receipient common.Address, amount *big.Int) { payable = append(payable, struct { diff --git a/node/node_handler.go b/node/node_handler.go index 0aa4f3fd5..718a89dac 100644 --- a/node/node_handler.go +++ b/node/node_handler.go @@ -3,12 +3,12 @@ package node import ( "bytes" "context" - "fmt" "math/big" "math/rand" "sync/atomic" "time" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/rlp" "github.com/harmony-one/bls/ffi/go/bls" @@ -18,7 +18,7 @@ import ( proto_discovery "github.com/harmony-one/harmony/api/proto/discovery" proto_node "github.com/harmony-one/harmony/api/proto/node" "github.com/harmony-one/harmony/block" - "github.com/harmony-one/harmony/core" + "github.com/harmony-one/harmony/consensus/quorum" "github.com/harmony-one/harmony/core/types" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" "github.com/harmony-one/harmony/internal/ctxerror" @@ -27,6 +27,7 @@ import ( "github.com/harmony-one/harmony/p2p" "github.com/harmony-one/harmony/p2p/host" "github.com/harmony-one/harmony/shard" + "github.com/harmony-one/harmony/shard/committee" staking "github.com/harmony-one/harmony/staking/types" ) @@ -362,46 +363,35 @@ func (node *Node) PostConsensusProcessing(newBlock *types.Block, commitSigAndBit // Broadcast client requested missing cross shard receipts if there is any node.BroadcastMissingCXReceipts() + next := new(big.Int).Add(newBlock.Epoch(), common.Big1) // Update consensus keys at last so the change of leader status doesn't mess up normal flow - // Not just at end of epoch, but end of pre-staking now if shard.Schedule.IsLastBlock(newBlock.Number().Uint64()) { - type t struct { - ShardID uint32 `json:"shard-id"` - Count int `json:"count"` - Participants []string `json:"committee-members"` + if node.chainConfig.StakingEpoch.Cmp(next) == 0 && + node.Consensus.Decider.Policy() != quorum.SuperMajorityStake { + node.Consensus.Decider = quorum.NewDecider(quorum.SuperMajorityStake) + node.Consensus.Decider.SetShardIDProvider(func() (uint32, error) { + return node.Consensus.ShardID, nil + }) + s, _ := committee.WithStakingEnabled.Compute( + next, node.chainConfig, node.Consensus.ChainReader, + ) + node.Consensus.Decider.UpdateVotingPower( + s.FindCommitteeByID(node.Consensus.ShardID).NodeList, + ) } - // b1, _ := json.Marshal(t{node.Consensus.ShardID, len(node.Consensus.Decider.DumpParticipants()), node.Consensus.Decider.DumpParticipants()}) - // fmt.Println("before", string(b1)) node.Consensus.UpdateConsensusInformation() - newKeys := node.Consensus.Decider.DumpParticipants() - myK := node.Consensus.PubKey.SerializeToHexStr() - myKeyInCommittee := false - for _, k := range newKeys { - if k == myK { - myKeyInCommittee = true - break + if shard.Schedule.IsLastBlock(newBlock.Number().Uint64()) { + if node.chainConfig.StakingEpoch.Cmp(next) == 0 { + // Hit this case again, need after UpdateConsensus + curPubKeys := committee.WithStakingEnabled.ComputePublicKeys( + next, node.Consensus.ChainReader, + )[int(node.Consensus.ShardID)] + node.Consensus.Decider.UpdateParticipants(curPubKeys) } } - - if myKeyInCommittee == false { - pair := core.NewKeys[node.myPort] - pub, priv := pair.Public, pair.Private - priKey := bls.SecretKey{} - pubKey := bls.PublicKey{} - pubKey.DeserializeHexStr(pub) - priKey.DeserializeHexStr(priv) - fmt.Println("new-pair", pair) - - node.NodeConfig.ConsensusPriKey = &priKey - node.NodeConfig.ConsensusPubKey = &pubKey - - node.Consensus.PubKey = &pubKey - node.Consensus.SetPrivateKey(&priKey) - } - } // TODO chao: uncomment this after beacon syncing is stable diff --git a/node/worker/worker.go b/node/worker/worker.go index c4d9635d1..20f43c8f1 100644 --- a/node/worker/worker.go +++ b/node/worker/worker.go @@ -290,26 +290,26 @@ func (w *Worker) SuperCommitteeForNextEpoch( nextCommittee shard.State oops error ) - - switch shardID { - case shard.BeaconChainShardID: - if shard.Schedule.IsLastBlock(w.current.header.Number().Uint64()) { - nextCommittee, oops = committee.WithStakingEnabled.Compute( - new(big.Int).Add(w.current.header.Epoch(), common.Big1), - *w.config, - beacon, - ) - } - default: - // WARN When we first enable staking, this condition may not be robust by itself. - switch beacon.CurrentHeader().Epoch().Cmp(w.current.header.Epoch()) { - case 1: - nextCommittee, oops = committee.WithStakingEnabled.ReadFromDB( - beacon.CurrentHeader().Epoch(), beacon, - ) - } - + // WARN This currently not working and breaks around 15 block + // switch shardID { + // case shard.BeaconChainShardID: + if shard.Schedule.IsLastBlock(w.current.header.Number().Uint64()) { + nextCommittee, oops = committee.WithStakingEnabled.Compute( + new(big.Int).Add(w.current.header.Epoch(), common.Big1), + *w.config, + beacon, + ) } + // default: + // WARN When we first enable staking, this condition may not be robust by itself. + // switch beacon.CurrentHeader().Epoch().Cmp(w.current.header.Epoch()) { + // case 1: + // nextCommittee, oops = committee.WithStakingEnabled.ReadFromDB( + // beacon.CurrentHeader().Epoch(), beacon, + // ) + // } + + // } return nextCommittee, oops } diff --git a/numeric/decimal.go b/numeric/decimal.go index 631976d26..d09e3591c 100644 --- a/numeric/decimal.go +++ b/numeric/decimal.go @@ -483,6 +483,11 @@ func (d Dec) RoundInt64() int64 { return chopped.Int64() } +// RoundInt round the decimal using bankers rounding +func (d Dec) RoundInt() *big.Int { + return chopPrecisionAndRoundNonMutative(d.Int) +} + //___________________________________________________________________________________ // similar to chopPrecisionAndRound, but always rounds down @@ -504,6 +509,11 @@ func (d Dec) TruncateInt64() int64 { return chopped.Int64() } +// TruncateInt truncates the decimals from the number and returns an Int +func (d Dec) TruncateInt() *big.Int { + return chopPrecisionAndTruncateNonMutative(d.Int) +} + // TruncateDec truncates the decimals from the number and returns a Dec func (d Dec) TruncateDec() Dec { return NewDecFromBigInt(chopPrecisionAndTruncateNonMutative(d.Int)) diff --git a/shard/committee/assignment.go b/shard/committee/assignment.go index 63b0bcfac..ed38bca82 100644 --- a/shard/committee/assignment.go +++ b/shard/committee/assignment.go @@ -1,7 +1,6 @@ package committee import ( - "fmt" "math/big" "github.com/ethereum/go-ethereum/common" @@ -16,12 +15,6 @@ import ( staking "github.com/harmony-one/harmony/staking/types" ) -const ( - // StateID means reading off whole network when using calls that accept - // a shardID parameter - StateID = -1 -) - // ValidatorListProvider .. type ValidatorListProvider interface { Compute( @@ -32,12 +25,7 @@ type ValidatorListProvider interface { // PublicKeysProvider per epoch type PublicKeysProvider interface { - // If call shardID with StateID then only superCommittee is non-nil, - // otherwise get back the shardSpecific slice as well. - ComputePublicKeys( - epoch *big.Int, reader DataProvider, shardID int, - ) (superCommittee, shardSpecific []*bls.PublicKey) - + ComputePublicKeys(epoch *big.Int, reader DataProvider) [][]*bls.PublicKey ReadPublicKeysFromDB( hash common.Hash, reader DataProvider, ) ([]*bls.PublicKey, error) @@ -130,7 +118,6 @@ func eposStakedCommittee( // TODO Nervous about this because overtime the list will become quite large candidates := stakerReader.ValidatorCandidates() essentials := map[common.Address]effective.SlotOrder{} - slotUsage := map[common.Address]int{} // TODO benchmark difference if went with data structure that sorts on insert for i := range candidates { @@ -143,19 +130,14 @@ func eposStakedCommittee( validator.Stake, validator.SlotPubKeys, } - slotUsage[validator.Address] = len(validator.SlotPubKeys) } shardCount := int(s.NumShards()) - maxNodePerShard := s.NumNodesPerShard() superComm := make(shard.State, shardCount) - fillCount := make([]int, shardCount) hAccounts := s.HmyAccounts() for i := 0; i < shardCount; i++ { - superComm[i] = shard.Committee{ - uint32(i), make(shard.NodeIDList, s.NumNodesPerShard()), - } + superComm[i] = shard.Committee{uint32(i), shard.NodeIDList{}} } for i := range hAccounts { @@ -164,104 +146,63 @@ func eposStakedCommittee( pub.DeserializeHexStr(hAccounts[i].BlsPublicKey) pubKey := shard.BlsPublicKey{} pubKey.FromLibBLSPublicKey(pub) - superComm[spot].NodeList[fillCount[spot]] = shard.NodeID{ + superComm[spot].NodeList = append(superComm[spot].NodeList, shard.NodeID{ common2.ParseAddr(hAccounts[i].Address), pubKey, nil, - } - fillCount[spot]++ + }) } - staked := effective.Apply(essentials) - - sort.SliceStable( - staked, - func(i, j int) bool { return staked[i].Dec.GTE(staked[j].Dec) }, - ) - + staked := effective.Apply(essentials, stakedSlotsCount) shardBig := big.NewInt(int64(shardCount)) - if len(staked) <= stakedSlotsCount { + if l := len(staked); l < stakedSlotsCount { // WARN unlikely to happen in production but will happen as we are developing + stakedSlotsCount = l } for i := 0; i < stakedSlotsCount; i++ { - bucket := int(new(big.Int).Mod(staked[i].Address.Big(), shardBig).Int64()) + bucket := int(new(big.Int).Mod(staked[i].BlsPublicKey.Big(), shardBig).Int64()) slot := staked[i] - pubKey := essentials[slot.Address].SpreadAmong[slotUsage[slot.Address]-1] - slotUsage[slot.Address]-- - // Keep going round till find an open spot - for j := bucket; ; j = (j + 1) % shardCount { - if fillCount[j] != maxNodePerShard { - superComm[j].NodeList[fillCount[j]] = shard.NodeID{ - slot.Address, - pubKey, - &slot.Dec, - } - fillCount[j]++ - break - } - } + superComm[bucket].NodeList = append(superComm[bucket].NodeList, shard.NodeID{ + slot.Address, + staked[i].BlsPublicKey, + &slot.Dec, + }) } - // fmt.Println("epos-based-committee", superComm.JSON()) + // fmt.Println("epos-comm", superComm.JSON()) return superComm, nil } -// ComputePublicKeys produces publicKeys of entire supercommittee per epoch, optionally providing a -// shard specific subcommittee +// ComputePublicKeys produces publicKeys of entire supercommittee per epoch func (def partialStakingEnabled) ComputePublicKeys( - epoch *big.Int, d DataProvider, shardID int, -) ([]*bls.PublicKey, []*bls.PublicKey) { + epoch *big.Int, d DataProvider, +) [][]*bls.PublicKey { + config := d.Config() instance := shard.Schedule.InstanceForEpoch(epoch) superComm := shard.State{} - stakedSlots := - (instance.NumNodesPerShard() - instance.NumHarmonyOperatedNodesPerShard()) * - int(instance.NumShards()) - if config.IsStaking(epoch) { + stakedSlots := + (instance.NumNodesPerShard() - instance.NumHarmonyOperatedNodesPerShard()) * + int(instance.NumShards()) superComm, _ = eposStakedCommittee(instance, d, stakedSlots) } else { superComm = preStakingEnabledCommittee(instance) } - spot := 0 - shouldBe := int(instance.NumShards()) * instance.NumNodesPerShard() - - total := 0 - for i := range superComm { - total += len(superComm[i].NodeList) - } - - if shouldBe != total { - fmt.Println("Count mismatch", shouldBe, total) - } + allIdentities := make([][]*bls.PublicKey, len(superComm)) - allIdentities := make([]*bls.PublicKey, shouldBe) for i := range superComm { + allIdentities[i] = make([]*bls.PublicKey, len(superComm[i].NodeList)) for j := range superComm[i].NodeList { identity := &bls.PublicKey{} superComm[i].NodeList[j].BlsPublicKey.ToLibBLSPublicKey(identity) - allIdentities[spot] = identity - spot++ + allIdentities[i][j] = identity } } - if shardID == StateID { - return allIdentities, nil - } - - subCommittee := superComm.FindCommitteeByID(uint32(shardID)) - subCommitteeIdentities := make([]*bls.PublicKey, len(subCommittee.NodeList)) - spot = 0 - for i := range subCommittee.NodeList { - identity := &bls.PublicKey{} - subCommittee.NodeList[i].BlsPublicKey.ToLibBLSPublicKey(identity) - subCommitteeIdentities[spot] = identity - spot++ - } - - return allIdentities, subCommitteeIdentities + return allIdentities } func (def partialStakingEnabled) ReadPublicKeysFromDB( @@ -313,6 +254,5 @@ func (def partialStakingEnabled) Compute( stakedSlots := (instance.NumNodesPerShard() - instance.NumHarmonyOperatedNodesPerShard()) * int(instance.NumShards()) - fmt.Println("Hit staking epoch -- compute") return eposStakedCommittee(instance, stakerReader, stakedSlots) } diff --git a/shard/shard_state.go b/shard/shard_state.go index ef52b53b2..515b197e0 100644 --- a/shard/shard_state.go +++ b/shard/shard_state.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/hex" "encoding/json" + "math/big" "sort" "github.com/ethereum/go-ethereum/common" diff --git a/staking/effective/calculate.go b/staking/effective/calculate.go index 742382afb..7b93b9e9a 100644 --- a/staking/effective/calculate.go +++ b/staking/effective/calculate.go @@ -1,6 +1,7 @@ package effective import ( + "encoding/json" "math/big" "sort" @@ -17,7 +18,7 @@ var ( oneMinusC = numeric.OneDec().Sub(c) ) -func stake(median, actual numeric.Dec) numeric.Dec { +func effectiveStake(median, actual numeric.Dec) numeric.Dec { left := numeric.MinDec(onePlusC.Mul(median), actual) right := oneMinusC.Mul(median) return numeric.MaxDec(left, right) @@ -25,8 +26,9 @@ func stake(median, actual numeric.Dec) numeric.Dec { // SlotPurchase .. type SlotPurchase struct { - common.Address `json:"slot-owner"` - numeric.Dec `json:"eposed-stake"` + common.Address `json:"slot-owner"` + shard.BlsPublicKey `json:"bls-public-key"` + numeric.Dec `json:"eposed-stake"` } // SlotOrder .. @@ -38,6 +40,15 @@ type SlotOrder struct { // Slots .. type Slots []SlotPurchase +// JSON is a plain JSON dump +func (s Slots) JSON() string { + type t struct { + Slots []SlotPurchase `json:"slots"` + } + b, _ := json.Marshal(t{s}) + return string(b) +} + func median(stakes []SlotPurchase) numeric.Dec { sort.SliceStable( stakes, @@ -46,35 +57,49 @@ func median(stakes []SlotPurchase) numeric.Dec { const isEven = 0 switch l := len(stakes); l % 2 { case isEven: - return stakes[(l/2)-1].Dec.Add(stakes[(l/2)+1].Dec).Quo(two) + return stakes[(l/2)-1].Dec.Add(stakes[(l / 2)].Dec).Quo(two) default: return stakes[l/2].Dec } } // Apply .. -func Apply(shortHand map[common.Address]SlotOrder) Slots { +func Apply(shortHand map[common.Address]SlotOrder, pull int) Slots { eposedSlots := Slots{} if len(shortHand) == 0 { return eposedSlots } // Expand for staker := range shortHand { - slotsCount := int64(len(shortHand[staker].SpreadAmong)) - spread := numeric.NewDecFromBigInt(shortHand[staker].Stake).QuoInt64(slotsCount) - var i int64 - for ; i < slotsCount; i++ { - eposedSlots = append(eposedSlots, SlotPurchase{staker, spread}) + slotsCount := len(shortHand[staker].SpreadAmong) + spread := numeric.NewDecFromBigInt(shortHand[staker].Stake). + QuoInt64(int64(slotsCount)) + for i := 0; i < slotsCount; i++ { + eposedSlots = append(eposedSlots, SlotPurchase{ + staker, + shortHand[staker].SpreadAmong[i], + spread, + }) } } if len(eposedSlots) < len(shortHand) { // WARN Should never happen } - median := median(eposedSlots) - for i := range eposedSlots { - eposedSlots[i].Dec = stake(median, eposedSlots[i].Dec) + sort.SliceStable( + eposedSlots, + func(i, j int) bool { return eposedSlots[i].Dec.GT(eposedSlots[j].Dec) }, + ) + + if l := len(eposedSlots); l < pull { + pull = l + } + picks := eposedSlots[:pull] + median := median(picks) + + for i := range picks { + picks[i].Dec = effectiveStake(median, picks[i].Dec) } - return eposedSlots + return picks } From 6c2cfae12c45ebe5484e2a381e9555cfde60159e Mon Sep 17 00:00:00 2001 From: Edgar Aroutiounian Date: Wed, 20 Nov 2019 12:41:35 -0800 Subject: [PATCH 13/26] [slashing] Update Naming from Master merge --- consensus/quorum/one-node-one-vote.go | 2 +- consensus/quorum/one-node-staked-vote.go | 2 +- consensus/quorum/quorum.go | 2 +- internal/chain/reward.go | 2 +- node/node_handler.go | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/consensus/quorum/one-node-one-vote.go b/consensus/quorum/one-node-one-vote.go index 1231ff209..bc029a844 100644 --- a/consensus/quorum/one-node-one-vote.go +++ b/consensus/quorum/one-node-one-vote.go @@ -43,7 +43,7 @@ func (v *uniformVoteWeight) IsRewardThresholdAchieved() bool { return v.SignersCount(Commit) >= (v.ParticipantsCount() * 9 / 10) } -func (v *uniformVoteWeight) UpdateVotingPower(shard.NodeIDList) { +func (v *uniformVoteWeight) UpdateVotingPower(shard.SlotList) { // NO-OP do not add anything here } diff --git a/consensus/quorum/one-node-staked-vote.go b/consensus/quorum/one-node-staked-vote.go index 22ab7879d..1ce499d02 100644 --- a/consensus/quorum/one-node-staked-vote.go +++ b/consensus/quorum/one-node-staked-vote.go @@ -107,7 +107,7 @@ func (v *stakedVoteWeight) Award( return payout } -func (v *stakedVoteWeight) UpdateVotingPower(staked shard.NodeIDList) { +func (v *stakedVoteWeight) UpdateVotingPower(staked shard.SlotList) { s, _ := v.ShardIDProvider()() v.validatorStakes = map[[shard.PublicKeySizeInBytes]byte]stakedVoter{} diff --git a/consensus/quorum/quorum.go b/consensus/quorum/quorum.go index 80615969f..333a42f1a 100644 --- a/consensus/quorum/quorum.go +++ b/consensus/quorum/quorum.go @@ -107,7 +107,7 @@ type Decider interface { slash.Slasher WithJSONDump ToggleActive(*bls.PublicKey) bool - UpdateVotingPower(shard.NodeIDList) + UpdateVotingPower(shard.SlotList) Policy() Policy IsQuorumAchieved(Phase) bool QuorumThreshold() *big.Int diff --git a/internal/chain/reward.go b/internal/chain/reward.go index 87faec233..f462d1d4d 100644 --- a/internal/chain/reward.go +++ b/internal/chain/reward.go @@ -84,7 +84,7 @@ func AccumulateRewards( } accounts := []common.Address{} - missing := shard.NodeIDList{} + missing := shard.SlotList{} for idx, member := range parentCommittee.Slots { switch signed, err := mask.IndexEnabled(idx); true { diff --git a/node/node_handler.go b/node/node_handler.go index 06e5d1625..12aaa37c3 100644 --- a/node/node_handler.go +++ b/node/node_handler.go @@ -377,7 +377,7 @@ func (node *Node) PostConsensusProcessing(newBlock *types.Block, commitSigAndBit next, node.chainConfig, node.Consensus.ChainReader, ) node.Consensus.Decider.UpdateVotingPower( - s.FindCommitteeByID(node.Consensus.ShardID).NodeList, + s.FindCommitteeByID(node.Consensus.ShardID).Slots, ) } From c673bd97d927842122368b7e06546bf97cce5a37 Mon Sep 17 00:00:00 2001 From: Edgar Aroutiounian Date: Thu, 21 Nov 2019 07:23:30 -0800 Subject: [PATCH 14/26] [committee] Switch over committee for stake epoch --- cmd/harmony/main.go | 3 --- consensus/consensus_service.go | 3 +-- consensus/engine/consensus_engine.go | 4 ++++ core/chain_makers.go | 6 ++++++ core/state/statedb.go | 1 - internal/chain/engine.go | 7 +++++-- internal/configs/sharding/localnet.go | 2 +- internal/params/config.go | 7 +++---- node/node_genesis.go | 1 + node/node_handler.go | 30 --------------------------- shard/committee/assignment.go | 9 ++++---- staking/types/validator.go | 9 ++++++++ 12 files changed, 34 insertions(+), 48 deletions(-) diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index 80e2c58a1..4d45836d2 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -224,7 +224,6 @@ func setupConsensusKey(nodeConfig *nodeconfig.ConfigType) *bls.PublicKey { os.Exit(100) } pubKey := consensusPriKey.GetPublicKey() - // Consensus keys are the BLS12-381 keys used to sign consensus messages nodeConfig.ConsensusPriKey, nodeConfig.ConsensusPubKey = consensusPriKey, consensusPriKey.GetPublicKey() if nodeConfig.ConsensusPriKey == nil || nodeConfig.ConsensusPubKey == nil { @@ -236,7 +235,6 @@ func setupConsensusKey(nodeConfig *nodeconfig.ConfigType) *bls.PublicKey { func createGlobalConfig() *nodeconfig.ConfigType { var err error - nodeConfig := nodeconfig.GetShardConfig(initialAccount.ShardID) if *nodeType == "validator" { // Set up consensus keys. @@ -310,7 +308,6 @@ func setupConsensusAndNode(nodeConfig *nodeconfig.ConfigType) *node.Node { // Current node. chainDBFactory := &shardchain.LDBFactory{RootDir: nodeConfig.DBDir} - // fmt.Println("What is my port at this moment", *port) currentNode := node.New( myHost, currentConsensus, chainDBFactory, *isArchival, *port, diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 20cbb3cc9..624492934 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -119,7 +119,6 @@ func (consensus *Consensus) UpdatePublicKeys(pubKeys []*bls.PublicKey) int64 { for i := range pubKeys { utils.Logger().Info().Int("index", i).Str("BLSPubKey", pubKeys[i].SerializeToHexStr()).Msg("Member") } - consensus.LeaderPubKey = pubKeys[0] utils.Logger().Info(). Str("info", consensus.LeaderPubKey.SerializeToHexStr()).Msg("My Leader") @@ -263,7 +262,7 @@ func (consensus *Consensus) verifySenderKey(msg *msg_pb.Message) (*bls.PublicKey } if !consensus.IsValidatorInCommittee(senderKey) { - return nil, fmt.Errorf("Validator %s is not in committee on shard: %d", senderKey.SerializeToHexStr(), consensus.ShardID) + return nil, fmt.Errorf("Validator %s is not in committee", senderKey.SerializeToHexStr()) } return senderKey, nil } diff --git a/consensus/engine/consensus_engine.go b/consensus/engine/consensus_engine.go index 6b0d7f07f..f2932c411 100644 --- a/consensus/engine/consensus_engine.go +++ b/consensus/engine/consensus_engine.go @@ -43,6 +43,10 @@ type ChainReader interface { // ReadActiveValidatorList retrieves the list of active validators ReadActiveValidatorList() ([]common.Address, error) + + ValidatorCandidates() []common.Address + ReadValidatorData(addr common.Address) (*staking.ValidatorWrapper, error) + ValidatorStakingWithDelegation(addr common.Address) *big.Int } // Engine is an algorithm agnostic consensus engine. diff --git a/core/chain_makers.go b/core/chain_makers.go index cdd30e907..8d7da2610 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -271,3 +271,9 @@ func (cr *fakeChainReader) GetHeader(hash common.Hash, number uint64) *block.Hea func (cr *fakeChainReader) GetBlock(hash common.Hash, number uint64) *types.Block { return nil } func (cr *fakeChainReader) ReadShardState(epoch *big.Int) (shard.State, error) { return nil, nil } func (cr *fakeChainReader) ReadActiveValidatorList() ([]common.Address, error) { return nil, nil } + +func (cr *fakeChainReader) ValidatorCandidates() []common.Address { return nil } +func (cr *fakeChainReader) ReadValidatorData(addr common.Address) (*staking.ValidatorWrapper, error) { + return nil, nil +} +func (cr *fakeChainReader) ValidatorStakingWithDelegation(addr common.Address) *big.Int { return nil } diff --git a/core/state/statedb.go b/core/state/statedb.go index 10681ff8c..10a38749f 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -30,7 +30,6 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" - "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/staking" stk "github.com/harmony-one/harmony/staking/types" diff --git a/internal/chain/engine.go b/internal/chain/engine.go index 31ecf1501..6dbd7e45c 100644 --- a/internal/chain/engine.go +++ b/internal/chain/engine.go @@ -11,6 +11,7 @@ import ( "github.com/harmony-one/harmony/consensus/reward" "github.com/harmony-one/harmony/core/state" "github.com/harmony-one/harmony/core/types" + common2 "github.com/harmony-one/harmony/internal/common" "github.com/harmony-one/harmony/internal/ctxerror" "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/shard" @@ -199,6 +200,7 @@ func (e *engineImpl) Finalize( } for _, validator := range validators { wrapper := state.GetStakingInfo(validator) + // fmt.Println("In finalize", validator.String(), "wrapper:", wrapper, "on-epoch", header.Epoch(), "on-shard", header.ShardID()) if wrapper != nil { for i := range wrapper.Delegations { delegation := wrapper.Delegations[i] @@ -209,6 +211,7 @@ func (e *engineImpl) Finalize( return nil, ctxerror.New("failed update validator info").WithCause(err) } } else { + err = errors.New("validator came back empty" + common2.MustAddressToBech32(validator)) return nil, ctxerror.New("failed getting validator info").WithCause(err) } } @@ -223,7 +226,7 @@ func QuorumForBlock( ) (quorum int, err error) { var ss shard.State if reCalculate { - ss, _ = committee.WithStakingEnabled.Compute(h.Epoch(), *chain.Config(), nil) + ss, _ = committee.WithStakingEnabled.Compute(h.Epoch(), *chain.Config(), chain) } else { ss, err = chain.ReadShardState(h.Epoch()) if err != nil { @@ -282,7 +285,7 @@ func GetPublicKeys(chain engine.ChainReader, header *block.Header, reCalculate b var shardState shard.State var err error if reCalculate { - shardState, _ = committee.WithStakingEnabled.Compute(header.Epoch(), *chain.Config(), nil) + shardState, _ = committee.WithStakingEnabled.Compute(header.Epoch(), *chain.Config(), chain) } else { shardState, err = chain.ReadShardState(header.Epoch()) if err != nil { diff --git a/internal/configs/sharding/localnet.go b/internal/configs/sharding/localnet.go index c656d6035..0bbf18182 100644 --- a/internal/configs/sharding/localnet.go +++ b/internal/configs/sharding/localnet.go @@ -17,7 +17,7 @@ const ( localnetV1Epoch = 1 localnetV2Epoch = 2 - localnetEpochBlock1 = 3 + localnetEpochBlock1 = 10 twoOne = 5 localnetVdfDifficulty = 5000 // This takes about 10s to finish the vdf diff --git a/internal/params/config.go b/internal/params/config.go index 7848a9a0c..21af1fda3 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -36,10 +36,9 @@ var ( ChainID: TestnetChainID, CrossTxEpoch: big.NewInt(0), CrossLinkEpoch: big.NewInt(0), - // MinEpoch needed is at least 1, crashes on 0 - StakingEpoch: big.NewInt(1), - EIP155Epoch: big.NewInt(0), - S3Epoch: big.NewInt(0), + StakingEpoch: EpochTBD, + EIP155Epoch: big.NewInt(0), + S3Epoch: big.NewInt(0), } // PangaeaChainConfig contains the chain parameters for the Pangaea network. diff --git a/node/node_genesis.go b/node/node_genesis.go index 7c03d4062..0beb04876 100644 --- a/node/node_genesis.go +++ b/node/node_genesis.go @@ -43,6 +43,7 @@ func (gi *genesisInitializer) InitChainDB(db ethdb.Database, shardID uint32) err shardState, _ := committee.WithStakingEnabled.Compute( big.NewInt(core.GenesisEpoch), gi.node.chainConfig, nil, ) + // fmt.Println("initial-shard-state", shardState.JSON()) if shardID != shard.BeaconChainShardID { diff --git a/node/node_handler.go b/node/node_handler.go index 12aaa37c3..1f128fba6 100644 --- a/node/node_handler.go +++ b/node/node_handler.go @@ -8,7 +8,6 @@ import ( "sync/atomic" "time" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/rlp" "github.com/harmony-one/bls/ffi/go/bls" @@ -18,7 +17,6 @@ import ( proto_discovery "github.com/harmony-one/harmony/api/proto/discovery" proto_node "github.com/harmony-one/harmony/api/proto/node" "github.com/harmony-one/harmony/block" - "github.com/harmony-one/harmony/consensus/quorum" "github.com/harmony-one/harmony/core/types" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" "github.com/harmony-one/harmony/internal/ctxerror" @@ -27,7 +25,6 @@ import ( "github.com/harmony-one/harmony/p2p" "github.com/harmony-one/harmony/p2p/host" "github.com/harmony-one/harmony/shard" - "github.com/harmony-one/harmony/shard/committee" staking "github.com/harmony-one/harmony/staking/types" ) @@ -335,8 +332,6 @@ func (node *Node) PostConsensusProcessing(newBlock *types.Block, commitSigAndBit return } - // fmt.Println("Finished consensus->", node.NodeConfig.Port) - // Update last consensus time for metrics // TODO: randomly selected a few validators to broadcast messages instead of only leader broadcast // TODO: refactor the asynchronous calls to separate go routine. @@ -363,35 +358,10 @@ func (node *Node) PostConsensusProcessing(newBlock *types.Block, commitSigAndBit // Broadcast client requested missing cross shard receipts if there is any node.BroadcastMissingCXReceipts() - next := new(big.Int).Add(newBlock.Epoch(), common.Big1) // Update consensus keys at last so the change of leader status doesn't mess up normal flow if shard.Schedule.IsLastBlock(newBlock.Number().Uint64()) { - if node.chainConfig.StakingEpoch.Cmp(next) == 0 && - node.Consensus.Decider.Policy() != quorum.SuperMajorityStake { - node.Consensus.Decider = quorum.NewDecider(quorum.SuperMajorityStake) - node.Consensus.Decider.SetShardIDProvider(func() (uint32, error) { - return node.Consensus.ShardID, nil - }) - s, _ := committee.WithStakingEnabled.Compute( - next, node.chainConfig, node.Consensus.ChainReader, - ) - node.Consensus.Decider.UpdateVotingPower( - s.FindCommitteeByID(node.Consensus.ShardID).Slots, - ) - } - node.Consensus.UpdateConsensusInformation() - - if shard.Schedule.IsLastBlock(newBlock.Number().Uint64()) { - if node.chainConfig.StakingEpoch.Cmp(next) == 0 { - // Hit this case again, need after UpdateConsensus - curPubKeys := committee.WithStakingEnabled.ComputePublicKeys( - next, node.Consensus.ChainReader, - )[int(node.Consensus.ShardID)] - node.Consensus.Decider.UpdateParticipants(curPubKeys) - } - } } // TODO chao: uncomment this after beacon syncing is stable diff --git a/shard/committee/assignment.go b/shard/committee/assignment.go index b6c7498ed..ce0458cdc 100644 --- a/shard/committee/assignment.go +++ b/shard/committee/assignment.go @@ -10,6 +10,7 @@ import ( shardingconfig "github.com/harmony-one/harmony/internal/configs/sharding" "github.com/harmony-one/harmony/internal/ctxerror" "github.com/harmony-one/harmony/internal/params" + "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/shard" "github.com/harmony-one/harmony/staking/effective" staking "github.com/harmony-one/harmony/staking/types" @@ -118,6 +119,7 @@ func eposStakedCommittee( // TODO Nervous about this because overtime the list will become quite large candidates := stakerReader.ValidatorCandidates() essentials := map[common.Address]effective.SlotOrder{} + utils.Logger().Info().Int("candidates-count", len(candidates)).Msg("Preparing EPoS Staked Committee") // TODO benchmark difference if went with data structure that sorts on insert for i := range candidates { @@ -126,6 +128,7 @@ func eposStakedCommittee( if err != nil { return nil, err } + essentials[validator.Address] = effective.SlotOrder{ validator.Stake, validator.SlotPubKeys, @@ -170,7 +173,6 @@ func eposStakedCommittee( &slot.Dec, }) } - // fmt.Println("epos-comm", superComm.JSON()) return superComm, nil } @@ -183,10 +185,7 @@ func (def partialStakingEnabled) ComputePublicKeys( instance := shard.Schedule.InstanceForEpoch(epoch) superComm := shard.State{} if config.IsStaking(epoch) { - stakedSlots := - (instance.NumNodesPerShard() - instance.NumHarmonyOperatedNodesPerShard()) * - int(instance.NumShards()) - superComm, _ = eposStakedCommittee(instance, d, stakedSlots) + superComm, _ = eposStakedCommittee(instance, d, 320) } else { superComm = preStakingEnabledCommittee(instance) } diff --git a/staking/types/validator.go b/staking/types/validator.go index 1d4920201..b97927efb 100644 --- a/staking/types/validator.go +++ b/staking/types/validator.go @@ -11,6 +11,7 @@ import ( "github.com/ethereum/go-ethereum/rlp" common2 "github.com/harmony-one/harmony/internal/common" "github.com/harmony-one/harmony/internal/ctxerror" + "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/numeric" "github.com/harmony-one/harmony/shard" ) @@ -46,6 +47,7 @@ type Validator struct { Address common.Address `json:"address" yaml:"address"` // The BLS public key of the validator for consensus SlotPubKeys []shard.BlsPublicKey `json:"slot_pub_keys" yaml:"slot_pub_keys"` + // TODO Need to remove this .Stake Field // The stake put by the validator itself Stake *big.Int `json:"stake" yaml:"stake"` // if unbonding, height at which this validator has begun unbonding @@ -108,6 +110,13 @@ func (w *ValidatorWrapper) SanityCheck() error { hundredPercent := numeric.NewDec(1) zeroPercent := numeric.NewDec(0) + + utils.Logger().Info(). + Str("rate", w.Validator.Rate.String()). + Str("max-rate", w.Validator.MaxRate.String()). + Str("max-change-rate", w.Validator.MaxChangeRate.String()). + Msg("Sanity check on validator commission rates, should all be in [0, 1]") + if w.Validator.Rate.LT(zeroPercent) || w.Validator.Rate.GT(hundredPercent) { return errInvalidComissionRate } From f348e10a5c3896c956e47f9fbd11006024e242dd Mon Sep 17 00:00:00 2001 From: Edgar Aroutiounian Date: Thu, 21 Nov 2019 07:33:54 -0800 Subject: [PATCH 15/26] [committee] Remove debug code --- cmd/client/wallet/main.go | 2 +- cmd/harmony/main.go | 5 +-- consensus/consensus_v2.go | 43 ------------------------ consensus/quorum/one-node-one-vote.go | 1 - consensus/quorum/one-node-staked-vote.go | 10 ++---- consensus/quorum/quorum.go | 4 +-- internal/chain/engine.go | 1 - node/node.go | 6 +--- node/node_genesis.go | 2 -- node/node_newblock.go | 9 ----- shard/committee/assignment.go | 2 +- 11 files changed, 7 insertions(+), 78 deletions(-) diff --git a/cmd/client/wallet/main.go b/cmd/client/wallet/main.go index 15401e22f..a639f64c8 100644 --- a/cmd/client/wallet/main.go +++ b/cmd/client/wallet/main.go @@ -304,7 +304,7 @@ func createWalletNode() *node.Node { panic(err) } chainDBFactory := &shardchain.MemDBFactory{} - w := node.New(host, nil, chainDBFactory, false, "") + w := node.New(host, nil, chainDBFactory, false) w.Client = client.NewClient(w.GetHost(), uint32(shardID)) w.NodeConfig.SetRole(nodeconfig.ClientNode) diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index 4d45836d2..9f2345669 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -309,9 +309,7 @@ func setupConsensusAndNode(nodeConfig *nodeconfig.ConfigType) *node.Node { // Current node. chainDBFactory := &shardchain.LDBFactory{RootDir: nodeConfig.DBDir} - currentNode := node.New( - myHost, currentConsensus, chainDBFactory, *isArchival, *port, - ) + currentNode := node.New(myHost, currentConsensus, chainDBFactory, *isArchival) switch { case *networkType == nodeconfig.Localnet: @@ -496,7 +494,6 @@ func main() { currentNode.ServiceManagerSetup() currentNode.RunServices() - // fmt.Println("CurrentRPC-port", *port) // RPC for SDK not supported for mainnet. if err := currentNode.StartRPC(*port); err != nil { diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 704fd3776..011c1baef 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -4,7 +4,6 @@ import ( "bytes" "encoding/binary" "encoding/hex" - "fmt" "time" "github.com/ethereum/go-ethereum/common" @@ -316,54 +315,12 @@ func (consensus *Consensus) onPrepare(msg *msg_pb.Message) { } senderKey, err := consensus.verifySenderKey(msg) - // fmt.Println("prepare-sender-key", senderKey.SerializeToHexStr()) - - // for _, v := range consensus.Decider.DumpParticipants() { - // fmt.Println("in-committee-", consensus.ShardID, v) - // } - - type t struct { - Participants []string `json:"committee-members"` - ShardID uint32 `json:"shard-id"` - MissingID string `json"missing-key"` - CurrentEpoch uint64 `json:"current-epoch"` - CurrentObject uint64 `json:"current-epoch-from-obj"` - PossibleError string `json:"possible-error"` - } - - // e := "" - - // if err != nil { - // e = err.Error() - // } - - // b, _ := json.Marshal(t{ - // consensus.Decider.DumpParticipants(), - // consensus.ShardID, - // senderKey.SerializeToHexStr(), - // consensus.ChainReader.CurrentHeader().Epoch().Uint64(), - // consensus.epoch, - // e, - // }) - // fmt.Println(string(b)) - if err != nil { - // fmt.Println("On Prepare is busted =/", err, "on shard-", consensus.ShardID) utils.Logger().Error().Err(err).Msg("[OnPrepare] VerifySenderKey failed") return } if err = verifyMessageSig(senderKey, msg); err != nil { - fmt.Println("unable to verify message sig") - - // p := consensus.Decider.DumpParticipants() - // for _, k := range p { - // if senderKey.SerializeToHexStr() == k { - // fmt.Println("Sender is in my committee quorum", consensus.PubKey.SerializeToHexStr()) - // break - // } - // } - // fmt.Println("Bad sender?", senderKey.SerializeToHexStr()) utils.Logger().Error().Err(err).Msg("[OnPrepare] Failed to verify sender's signature") return } diff --git a/consensus/quorum/one-node-one-vote.go b/consensus/quorum/one-node-one-vote.go index bc029a844..0c065a16e 100644 --- a/consensus/quorum/one-node-one-vote.go +++ b/consensus/quorum/one-node-one-vote.go @@ -77,7 +77,6 @@ func (v *uniformVoteWeight) Award( func (v *uniformVoteWeight) ShouldSlash(k shard.BlsPublicKey) bool { // No-op, no semantic meaning in one-slot-one-vote - // fmt.Println("Called here for key:", k.Hex()) return false } diff --git a/consensus/quorum/one-node-staked-vote.go b/consensus/quorum/one-node-staked-vote.go index 1ce499d02..e776bf9df 100644 --- a/consensus/quorum/one-node-staked-vote.go +++ b/consensus/quorum/one-node-staked-vote.go @@ -2,7 +2,6 @@ package quorum import ( "encoding/json" - "fmt" "math/big" "github.com/ethereum/go-ethereum/common" @@ -40,8 +39,7 @@ func (v *stakedVoteWeight) Policy() Policy { // IsQuorumAchieved .. func (v *stakedVoteWeight) IsQuorumAchieved(p Phase) bool { - // TODO Implement this logic - // fmt.Println("is quorum achieved") + // TODO Implement this logic w/Chao // soFar := numeric.ZeroDec() w := shard.BlsPublicKey{} members := v.Participants() @@ -59,14 +57,12 @@ func (v *stakedVoteWeight) IsQuorumAchieved(p Phase) bool { // QuorumThreshold .. func (v *stakedVoteWeight) QuorumThreshold() *big.Int { - // fmt.Println("check quorum threshold") return v.total.Mul(twoThirds).Int } // RewardThreshold .. func (v *stakedVoteWeight) IsRewardThresholdAchieved() bool { // TODO Implement - // fmt.Println("check threshold") return true } @@ -77,9 +73,7 @@ func (v *stakedVoteWeight) Award( payout := big.NewInt(0) last := big.NewInt(0) count := big.NewInt(int64(len(earners))) - - s, _ := v.ShardIDProvider()() - fmt.Println("Award called on shard as staked vote", s) + // TODO Finish implementing this logic proportional := map[common.Address]numeric.Dec{} for _, details := range v.validatorStakes { diff --git a/consensus/quorum/quorum.go b/consensus/quorum/quorum.go index 333a42f1a..9625898fe 100644 --- a/consensus/quorum/quorum.go +++ b/consensus/quorum/quorum.go @@ -156,7 +156,7 @@ func (s *cIdentities) Participants() []*bls.PublicKey { } func (s *cIdentities) UpdateParticipants(pubKeys []*bls.PublicKey) { - // TODO - might need to put this in separate method + // TODO - might need to put reset of seen counter in separate method s.seenCounter = make(map[[shard.PublicKeySizeInBytes]byte]int, len(pubKeys)) for i := range pubKeys { k := shard.BlsPublicKey{} @@ -168,9 +168,7 @@ func (s *cIdentities) UpdateParticipants(pubKeys []*bls.PublicKey) { func (s *cIdentities) SlashThresholdMet(key shard.BlsPublicKey) bool { s.seenCounter[key]++ - fmt.Println("Slash Map", s.seenCounter) return s.seenCounter[key] == slash.UnavailabilityInConsecutiveBlockSigning - } func (s *cIdentities) DumpParticipants() []string { diff --git a/internal/chain/engine.go b/internal/chain/engine.go index 6dbd7e45c..628843e53 100644 --- a/internal/chain/engine.go +++ b/internal/chain/engine.go @@ -200,7 +200,6 @@ func (e *engineImpl) Finalize( } for _, validator := range validators { wrapper := state.GetStakingInfo(validator) - // fmt.Println("In finalize", validator.String(), "wrapper:", wrapper, "on-epoch", header.Epoch(), "on-shard", header.ShardID()) if wrapper != nil { for i := range wrapper.Delegations { delegation := wrapper.Delegations[i] diff --git a/node/node.go b/node/node.go index 2f88da701..7401e3daf 100644 --- a/node/node.go +++ b/node/node.go @@ -103,7 +103,6 @@ type syncConfig struct { // Node represents a protocol-participating node in the network type Node struct { - myPort string Consensus *consensus.Consensus // Consensus object containing all Consensus related data (e.g. committee members, signatures, commits) BlockChannel chan *types.Block // The channel to send newly proposed blocks ConfirmedBlockChannel chan *types.Block // The channel to send confirmed blocks @@ -375,12 +374,9 @@ func (node *Node) GetSyncID() [SyncIDLength]byte { // New creates a new node. func New(host p2p.Host, consensusObj *consensus.Consensus, - chainDBFactory shardchain.DBFactory, isArchival bool, port string) *Node { + chainDBFactory shardchain.DBFactory, isArchival bool) *Node { node := Node{} - node.myPort = port - // fmt.Println("as node, my port is", node.myPort) - node.syncFreq = SyncFrequency node.beaconSyncFreq = SyncFrequency diff --git a/node/node_genesis.go b/node/node_genesis.go index 0beb04876..501ad738c 100644 --- a/node/node_genesis.go +++ b/node/node_genesis.go @@ -44,8 +44,6 @@ func (gi *genesisInitializer) InitChainDB(db ethdb.Database, shardID uint32) err big.NewInt(core.GenesisEpoch), gi.node.chainConfig, nil, ) - // fmt.Println("initial-shard-state", shardState.JSON()) - if shardID != shard.BeaconChainShardID { // store only the local shard for shard chains c := shardState.FindCommitteeByID(shardID) diff --git a/node/node_newblock.go b/node/node_newblock.go index ec220696e..c814a0527 100644 --- a/node/node_newblock.go +++ b/node/node_newblock.go @@ -1,7 +1,6 @@ package node import ( - "math/big" "sort" "time" @@ -124,14 +123,6 @@ func (node *Node) proposeNewBlock() (*types.Block, error) { node.Consensus.ShardID, node.Beaconchain(), ) - // fmt.Println("Update my keys, right", "on port", node.NodeConfig.Port) - - if node.Beaconchain().CurrentHeader().Epoch().Cmp(big.NewInt(1)) == 0 { - // fmt.Println("Update my keys, right", "on port", node.NodeConfig.Port) - // node.NodeConfig.ConsensusPriKey - // node.NodeConfig.ConsensusPubKey - } - // fmt.Println("super-comm", shardState.JSON()) if err != nil { return nil, err diff --git a/shard/committee/assignment.go b/shard/committee/assignment.go index ce0458cdc..9b75cd9e4 100644 --- a/shard/committee/assignment.go +++ b/shard/committee/assignment.go @@ -128,7 +128,6 @@ func eposStakedCommittee( if err != nil { return nil, err } - essentials[validator.Address] = effective.SlotOrder{ validator.Stake, validator.SlotPubKeys, @@ -173,6 +172,7 @@ func eposStakedCommittee( &slot.Dec, }) } + return superComm, nil } From 5eb5e7a6b07c75572b57ef6e2f866e82bc197d74 Mon Sep 17 00:00:00 2001 From: Edgar Aroutiounian Date: Thu, 21 Nov 2019 07:47:15 -0800 Subject: [PATCH 16/26] [committee] Remove debug code, use shardchain db instead of beaconchain --- cmd/harmony/main.go | 5 +++-- consensus/consensus_v2.go | 1 - consensus/engine/consensus_engine.go | 6 +++--- core/state/statedb.go | 1 + node/node_genesis.go | 1 - node/node_newblock.go | 3 +-- 6 files changed, 8 insertions(+), 9 deletions(-) diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index 9f2345669..7d62c5c05 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -224,6 +224,7 @@ func setupConsensusKey(nodeConfig *nodeconfig.ConfigType) *bls.PublicKey { os.Exit(100) } pubKey := consensusPriKey.GetPublicKey() + // Consensus keys are the BLS12-381 keys used to sign consensus messages nodeConfig.ConsensusPriKey, nodeConfig.ConsensusPubKey = consensusPriKey, consensusPriKey.GetPublicKey() if nodeConfig.ConsensusPriKey == nil || nodeConfig.ConsensusPubKey == nil { @@ -235,6 +236,7 @@ func setupConsensusKey(nodeConfig *nodeconfig.ConfigType) *bls.PublicKey { func createGlobalConfig() *nodeconfig.ConfigType { var err error + nodeConfig := nodeconfig.GetShardConfig(initialAccount.ShardID) if *nodeType == "validator" { // Set up consensus keys. @@ -308,7 +310,6 @@ func setupConsensusAndNode(nodeConfig *nodeconfig.ConfigType) *node.Node { // Current node. chainDBFactory := &shardchain.LDBFactory{RootDir: nodeConfig.DBDir} - currentNode := node.New(myHost, currentConsensus, chainDBFactory, *isArchival) switch { @@ -339,6 +340,7 @@ func setupConsensusAndNode(nodeConfig *nodeconfig.ConfigType) *node.Node { currentNode.NodeConfig.SetPushgatewayIP(nodeConfig.PushgatewayIP) currentNode.NodeConfig.SetPushgatewayPort(nodeConfig.PushgatewayPort) currentNode.NodeConfig.SetMetricsFlag(nodeConfig.MetricsFlag) + currentNode.NodeConfig.SetBeaconGroupID(nodeconfig.NewGroupIDByShardID(0)) switch *nodeType { @@ -494,7 +496,6 @@ func main() { currentNode.ServiceManagerSetup() currentNode.RunServices() - // RPC for SDK not supported for mainnet. if err := currentNode.StartRPC(*port); err != nil { utils.Logger().Warn(). diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 011c1baef..2d945e7bf 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -319,7 +319,6 @@ func (consensus *Consensus) onPrepare(msg *msg_pb.Message) { utils.Logger().Error().Err(err).Msg("[OnPrepare] VerifySenderKey failed") return } - if err = verifyMessageSig(senderKey, msg); err != nil { utils.Logger().Error().Err(err).Msg("[OnPrepare] Failed to verify sender's signature") return diff --git a/consensus/engine/consensus_engine.go b/consensus/engine/consensus_engine.go index f2932c411..a2fd2101a 100644 --- a/consensus/engine/consensus_engine.go +++ b/consensus/engine/consensus_engine.go @@ -10,6 +10,7 @@ import ( "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/internal/params" "github.com/harmony-one/harmony/shard" + "github.com/harmony-one/harmony/shard/committee" "github.com/harmony-one/harmony/staking/slash" staking "github.com/harmony-one/harmony/staking/types" ) @@ -44,9 +45,8 @@ type ChainReader interface { // ReadActiveValidatorList retrieves the list of active validators ReadActiveValidatorList() ([]common.Address, error) - ValidatorCandidates() []common.Address - ReadValidatorData(addr common.Address) (*staking.ValidatorWrapper, error) - ValidatorStakingWithDelegation(addr common.Address) *big.Int + // Methods needed for EPoS committee assignment calculation + committee.StakingCandidatesReader } // Engine is an algorithm agnostic consensus engine. diff --git a/core/state/statedb.go b/core/state/statedb.go index 10a38749f..10681ff8c 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -30,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" + "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/staking" stk "github.com/harmony-one/harmony/staking/types" diff --git a/node/node_genesis.go b/node/node_genesis.go index 501ad738c..61400eb5b 100644 --- a/node/node_genesis.go +++ b/node/node_genesis.go @@ -43,7 +43,6 @@ func (gi *genesisInitializer) InitChainDB(db ethdb.Database, shardID uint32) err shardState, _ := committee.WithStakingEnabled.Compute( big.NewInt(core.GenesisEpoch), gi.node.chainConfig, nil, ) - if shardID != shard.BeaconChainShardID { // store only the local shard for shard chains c := shardState.FindCommitteeByID(shardID) diff --git a/node/node_newblock.go b/node/node_newblock.go index c814a0527..b8dec7191 100644 --- a/node/node_newblock.go +++ b/node/node_newblock.go @@ -120,10 +120,9 @@ func (node *Node) proposeNewBlock() (*types.Block, error) { // Prepare shard state shardState, err := node.Worker.SuperCommitteeForNextEpoch( - node.Consensus.ShardID, node.Beaconchain(), + node.Consensus.ShardID, node.Blockchain(), ) - // fmt.Println("super-comm", shardState.JSON()) if err != nil { return nil, err } From a584baa7601db97df7efc32472f05b92f8f7beb4 Mon Sep 17 00:00:00 2001 From: Edgar Aroutiounian Date: Thu, 21 Nov 2019 08:01:07 -0800 Subject: [PATCH 17/26] [deploy] Temp disable debug_staking command --- test/deploy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/deploy.sh b/test/deploy.sh index 478d322cf..1d47ce1dc 100755 --- a/test/deploy.sh +++ b/test/deploy.sh @@ -212,7 +212,7 @@ while IFS='' read -r line || [[ -n "$line" ]]; do done < $config if [ "$DOTEST" == "true" ]; then - debug_staking + # debug_staking echo "waiting for some block rewards" sleep 60 i=1 From 76dc4cf46cd21bd1d69e2766408df752d099c3fc Mon Sep 17 00:00:00 2001 From: Edgar Aroutiounian Date: Thu, 21 Nov 2019 08:09:26 -0800 Subject: [PATCH 18/26] [core] Add logging for amount of gas needed for a rejected staking transaction --- core/state_transition.go | 1 + 1 file changed, 1 insertion(+) diff --git a/core/state_transition.go b/core/state_transition.go index 9cf9a80d2..58c11842e 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -286,6 +286,7 @@ func (st *StateTransition) StakingTransitionDb() (usedGas uint64, err error) { // TODO: propose staking-specific formula for staking transaction gas, err := IntrinsicGas(st.data, false, homestead) if err != nil { + utils.Logger().Error().Uint64("needed", gas).Msg("Not enough gas in staking transaction") return 0, err } if err = st.useGas(gas); err != nil { From 18b915ed82a639145f9a73451a3c7f1032bc0720 Mon Sep 17 00:00:00 2001 From: Edgar Aroutiounian Date: Thu, 21 Nov 2019 09:48:27 -0800 Subject: [PATCH 19/26] [staking] Exit early in temp bin/staking if not enough balance for staking transaction --- cmd/staking/root.go | 55 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/cmd/staking/root.go b/cmd/staking/root.go index c5a7bf7ed..ca68bbab8 100644 --- a/cmd/staking/root.go +++ b/cmd/staking/root.go @@ -170,6 +170,21 @@ func (s *staker) run(cmd *cobra.Command, args []string) error { rlp.DecodeBytes(enc, tx) hexSignature := hexutil.Encode(enc) param := []interface{}{hexSignature} + + // Check Double check if have enough balance for gas fees + balanceR, _ := baseRequest("hmy_getBalance", "http://localhost:9500", []interface{}{testAccounts[index], "latest"}) + m := map[string]interface{}{} + json.Unmarshal(balanceR, &m) + balance, _ := m["result"].(string) + bln, _ := big.NewInt(0).SetString(balance[2:], 16) + + if bln.Cmp(big.NewInt(0)) == 0 { + fmt.Println("Balance for ", testAccounts[index], "is zero, tx will be rejected b/c not enough for gas fee, exiting") + os.Exit(-1) + } + + fmt.Println("balance", convertBalanceIntoReadableFormat(bln), "\n") + result, reqOops := baseRequest(stakingRPC, "http://localhost:9500", param) fmt.Println(string(result)) return reqOops @@ -245,3 +260,43 @@ var ( }, } ) + +func convertBalanceIntoReadableFormat(balance *big.Int) string { + balance = balance.Div(balance, big.NewInt(denominations.Nano)) + strBalance := fmt.Sprintf("%d", balance.Uint64()) + + bytes := []byte(strBalance) + hasDecimal := false + for i := 0; i < 11; i++ { + if len(bytes)-1-i < 0 { + bytes = append([]byte{'0'}, bytes...) + } + if bytes[len(bytes)-1-i] != '0' && i < 9 { + hasDecimal = true + } + if i == 9 { + newBytes := append([]byte{'.'}, bytes[len(bytes)-i:]...) + bytes = append(bytes[:len(bytes)-i], newBytes...) + } + } + zerosToRemove := 0 + for i := 0; i < len(bytes); i++ { + if hasDecimal { + if bytes[len(bytes)-1-i] == '0' { + bytes = bytes[:len(bytes)-1-i] + i-- + } else { + break + } + } else { + if zerosToRemove < 5 { + bytes = bytes[:len(bytes)-1-i] + i-- + zerosToRemove++ + } else { + break + } + } + } + return string(bytes) +} From 66f6e0a26c9d6eef51a1b5ad72d1af90481f4b91 Mon Sep 17 00:00:00 2001 From: Edgar Aroutiounian Date: Thu, 21 Nov 2019 10:19:50 -0800 Subject: [PATCH 20/26] [committee] Enable Staking on localnet at epoch 3 --- core/state_transition.go | 4 +++- internal/params/config.go | 2 +- shard/committee/assignment.go | 9 +++++++-- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/core/state_transition.go b/core/state_transition.go index 58c11842e..204007d7c 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -285,8 +285,10 @@ func (st *StateTransition) StakingTransitionDb() (usedGas uint64, err error) { // Pay intrinsic gas // TODO: propose staking-specific formula for staking transaction gas, err := IntrinsicGas(st.data, false, homestead) + // TODO Remove this logging + utils.Logger().Info().Uint64("Using", gas).Msg("Gas cost of staking transaction being processed") + if err != nil { - utils.Logger().Error().Uint64("needed", gas).Msg("Not enough gas in staking transaction") return 0, err } if err = st.useGas(gas); err != nil { diff --git a/internal/params/config.go b/internal/params/config.go index 21af1fda3..5892b2258 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -36,7 +36,7 @@ var ( ChainID: TestnetChainID, CrossTxEpoch: big.NewInt(0), CrossLinkEpoch: big.NewInt(0), - StakingEpoch: EpochTBD, + StakingEpoch: big.NewInt(3), EIP155Epoch: big.NewInt(0), S3Epoch: big.NewInt(0), } diff --git a/shard/committee/assignment.go b/shard/committee/assignment.go index 9b75cd9e4..ee24b58c1 100644 --- a/shard/committee/assignment.go +++ b/shard/committee/assignment.go @@ -119,7 +119,8 @@ func eposStakedCommittee( // TODO Nervous about this because overtime the list will become quite large candidates := stakerReader.ValidatorCandidates() essentials := map[common.Address]effective.SlotOrder{} - utils.Logger().Info().Int("candidates-count", len(candidates)).Msg("Preparing EPoS Staked Committee") + + utils.Logger().Info().Int("staked-candidates", len(candidates)).Msg("preparing epos staked committee") // TODO benchmark difference if went with data structure that sorts on insert for i := range candidates { @@ -172,7 +173,11 @@ func eposStakedCommittee( &slot.Dec, }) } - + if c := len(candidates); c != 0 { + utils.Logger().Info().Int("staked-candidates", c). + RawJSON("staked-super-committee", []byte(superComm.JSON())). + Msg("EPoS based super-committe") + } return superComm, nil } From d5d803077579b9e698ce840f15ddb2dd91abb0d2 Mon Sep 17 00:00:00 2001 From: Edgar Aroutiounian Date: Thu, 21 Nov 2019 10:27:52 -0800 Subject: [PATCH 21/26] [committee] Assign staked vote decider when next epoch switches to StakingEpoch --- node/node_handler.go | 37 ++++++++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/node/node_handler.go b/node/node_handler.go index eb2f2a9cb..3f46661bc 100644 --- a/node/node_handler.go +++ b/node/node_handler.go @@ -8,18 +8,17 @@ import ( "sync/atomic" "time" - bls2 "github.com/harmony-one/harmony/crypto/bls" - + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/rlp" "github.com/harmony-one/bls/ffi/go/bls" - libp2p_peer "github.com/libp2p/go-libp2p-core/peer" - "github.com/harmony-one/harmony/api/proto" proto_discovery "github.com/harmony-one/harmony/api/proto/discovery" proto_node "github.com/harmony-one/harmony/api/proto/node" "github.com/harmony-one/harmony/block" + "github.com/harmony-one/harmony/consensus/quorum" "github.com/harmony-one/harmony/core/types" + bls2 "github.com/harmony-one/harmony/crypto/bls" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" "github.com/harmony-one/harmony/internal/ctxerror" "github.com/harmony-one/harmony/internal/utils" @@ -27,7 +26,9 @@ import ( "github.com/harmony-one/harmony/p2p" "github.com/harmony-one/harmony/p2p/host" "github.com/harmony-one/harmony/shard" + "github.com/harmony-one/harmony/shard/committee" staking "github.com/harmony-one/harmony/staking/types" + libp2p_peer "github.com/libp2p/go-libp2p-core/peer" ) const ( @@ -377,10 +378,36 @@ func (node *Node) PostConsensusProcessing(newBlock *types.Block, commitSigAndBit } } } - + next := new(big.Int).Add(newBlock.Epoch(), common.Big1) // Update consensus keys at last so the change of leader status doesn't mess up normal flow if shard.Schedule.IsLastBlock(newBlock.Number().Uint64()) { + + if node.chainConfig.StakingEpoch.Cmp(next) == 0 && + node.Consensus.Decider.Policy() != quorum.SuperMajorityStake { + node.Consensus.Decider = quorum.NewDecider(quorum.SuperMajorityStake) + node.Consensus.Decider.SetShardIDProvider(func() (uint32, error) { + return node.Consensus.ShardID, nil + }) + s, _ := committee.WithStakingEnabled.Compute( + next, node.chainConfig, node.Consensus.ChainReader, + ) + node.Consensus.Decider.UpdateVotingPower( + s.FindCommitteeByID(node.Consensus.ShardID).Slots, + ) + } + node.Consensus.UpdateConsensusInformation() + + if shard.Schedule.IsLastBlock(newBlock.Number().Uint64()) { + if node.chainConfig.StakingEpoch.Cmp(next) == 0 { + // Hit this case again, need after UpdateConsensus + curPubKeys := committee.WithStakingEnabled.ComputePublicKeys( + next, node.Consensus.ChainReader, + )[int(node.Consensus.ShardID)] + node.Consensus.Decider.UpdateParticipants(curPubKeys) + } + } + } // TODO chao: uncomment this after beacon syncing is stable From 043a6e67fe951f1fca0804a98edda66ce1b5aeea Mon Sep 17 00:00:00 2001 From: Edgar Aroutiounian Date: Thu, 21 Nov 2019 10:55:03 -0800 Subject: [PATCH 22/26] [staking] Lint --- cmd/staking/root.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/staking/root.go b/cmd/staking/root.go index ca68bbab8..053fc106a 100644 --- a/cmd/staking/root.go +++ b/cmd/staking/root.go @@ -183,7 +183,7 @@ func (s *staker) run(cmd *cobra.Command, args []string) error { os.Exit(-1) } - fmt.Println("balance", convertBalanceIntoReadableFormat(bln), "\n") + fmt.Println("balance", convertBalanceIntoReadableFormat(bln)) result, reqOops := baseRequest(stakingRPC, "http://localhost:9500", param) fmt.Println(string(result)) From a1f637f99041fdbe47ca441cb0b784e30bed8dbc Mon Sep 17 00:00:00 2001 From: Edgar Aroutiounian Date: Thu, 21 Nov 2019 11:01:20 -0800 Subject: [PATCH 23/26] [staking] Undo bad merge from master --- staking/types/validator.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/staking/types/validator.go b/staking/types/validator.go index 6c87d2620..c2333060f 100644 --- a/staking/types/validator.go +++ b/staking/types/validator.go @@ -236,10 +236,11 @@ func CreateValidatorFromNewMsg(val *CreateValidator, blockNum *big.Int) (*Valida commission := Commission{val.CommissionRates, blockNum} pubKeys := []shard.BlsPublicKey{} pubKeys = append(pubKeys, val.SlotPubKeys...) + // TODO: a new validator should have a minimum of 1 token as self delegation, and that should be added as a delegation entry here. v := Validator{ val.ValidatorAddress, pubKeys, val.Amount, new(big.Int), val.MinSelfDelegation, val.MaxTotalDelegation, false, - commission, desc, big.NewInt(0), + commission, desc, blockNum, } return &v, nil } From 0114330744ffb713aa9ead7839308ab38808b49a Mon Sep 17 00:00:00 2001 From: Edgar Aroutiounian Date: Thu, 21 Nov 2019 11:38:19 -0800 Subject: [PATCH 24/26] [PR] Address PR comments #1 --- consensus/quorum/one-node-staked-vote.go | 6 +++--- consensus/quorum/quorum.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/consensus/quorum/one-node-staked-vote.go b/consensus/quorum/one-node-staked-vote.go index e776bf9df..22b82bb30 100644 --- a/consensus/quorum/one-node-staked-vote.go +++ b/consensus/quorum/one-node-staked-vote.go @@ -28,7 +28,7 @@ type stakedVoteWeight struct { DependencyInjectionWriter DependencyInjectionReader slash.ThresholdDecider - validatorStakes map[[shard.PublicKeySizeInBytes]byte]stakedVoter + validatorStakes map[shard.BlsPublicKey]stakedVoter total numeric.Dec } @@ -48,7 +48,7 @@ func (v *stakedVoteWeight) IsQuorumAchieved(p Phase) bool { w.FromLibBLSPublicKey(members[i]) // isHMY := v.validatorStakes[w].isHarmonyNode if v.ReadSignature(p, members[i]) == nil { - // + // TODO TODO finish this logic } } @@ -104,7 +104,7 @@ func (v *stakedVoteWeight) Award( func (v *stakedVoteWeight) UpdateVotingPower(staked shard.SlotList) { s, _ := v.ShardIDProvider()() - v.validatorStakes = map[[shard.PublicKeySizeInBytes]byte]stakedVoter{} + v.validatorStakes = map[shard.BlsPublicKey]stakedVoter{} v.Reset([]Phase{Prepare, Commit, ViewChange}) for i := range staked { diff --git a/consensus/quorum/quorum.go b/consensus/quorum/quorum.go index 9625898fe..af2633119 100644 --- a/consensus/quorum/quorum.go +++ b/consensus/quorum/quorum.go @@ -297,7 +297,7 @@ func NewDecider(p Policy) Decider { c.DependencyInjectionWriter, c.DependencyInjectionWriter.(DependencyInjectionReader), c.SignatureReader.(slash.ThresholdDecider), - map[[shard.PublicKeySizeInBytes]byte]stakedVoter{}, + map[shard.BlsPublicKey]stakedVoter{}, numeric.ZeroDec(), } default: From ccd0648c73b88cb464e9457b130a5c5505aba71f Mon Sep 17 00:00:00 2001 From: Edgar Aroutiounian Date: Thu, 21 Nov 2019 11:40:09 -0800 Subject: [PATCH 25/26] [PR] Address PR comments #2 --- consensus/quorum/one-node-staked-vote.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consensus/quorum/one-node-staked-vote.go b/consensus/quorum/one-node-staked-vote.go index 22b82bb30..4b97626b5 100644 --- a/consensus/quorum/one-node-staked-vote.go +++ b/consensus/quorum/one-node-staked-vote.go @@ -57,7 +57,7 @@ func (v *stakedVoteWeight) IsQuorumAchieved(p Phase) bool { // QuorumThreshold .. func (v *stakedVoteWeight) QuorumThreshold() *big.Int { - return v.total.Mul(twoThirds).Int + return v.total.Mul(twoThirds).RoundInt() } // RewardThreshold .. From cc8662cb533cb25787051f52301c3fdce9a5cb72 Mon Sep 17 00:00:00 2001 From: Edgar Aroutiounian Date: Thu, 21 Nov 2019 16:02:25 -0800 Subject: [PATCH 26/26] [PR] Address PR comments #3 --- consensus/quorum/one-node-staked-vote.go | 4 ++-- internal/chain/reward.go | 2 +- internal/params/config.go | 2 +- node/node_handler.go | 6 +++--- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/consensus/quorum/one-node-staked-vote.go b/consensus/quorum/one-node-staked-vote.go index 4b97626b5..074f25e89 100644 --- a/consensus/quorum/one-node-staked-vote.go +++ b/consensus/quorum/one-node-staked-vote.go @@ -57,7 +57,7 @@ func (v *stakedVoteWeight) IsQuorumAchieved(p Phase) bool { // QuorumThreshold .. func (v *stakedVoteWeight) QuorumThreshold() *big.Int { - return v.total.Mul(twoThirds).RoundInt() + return v.total.Mul(twoThirds).Ceil().RoundInt() } // RewardThreshold .. @@ -73,7 +73,6 @@ func (v *stakedVoteWeight) Award( payout := big.NewInt(0) last := big.NewInt(0) count := big.NewInt(int64(len(earners))) - // TODO Finish implementing this logic proportional := map[common.Address]numeric.Dec{} for _, details := range v.validatorStakes { @@ -83,6 +82,7 @@ func (v *stakedVoteWeight) Award( ) } } + // TODO Finish implementing this logic w/Chao for i := range earners { cur := big.NewInt(0) diff --git a/internal/chain/reward.go b/internal/chain/reward.go index f462d1d4d..34a5738cc 100644 --- a/internal/chain/reward.go +++ b/internal/chain/reward.go @@ -104,7 +104,7 @@ func AccumulateRewards( for i := range missing { w.Add(1) go func(member int) { - defer w.Add(-1) + defer w.Done() // Slash if missing block was long enough if slasher.ShouldSlash(missing[member].BlsPublicKey) { // TODO Logic diff --git a/internal/params/config.go b/internal/params/config.go index 5892b2258..21af1fda3 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -36,7 +36,7 @@ var ( ChainID: TestnetChainID, CrossTxEpoch: big.NewInt(0), CrossLinkEpoch: big.NewInt(0), - StakingEpoch: big.NewInt(3), + StakingEpoch: EpochTBD, EIP155Epoch: big.NewInt(0), S3Epoch: big.NewInt(0), } diff --git a/node/node_handler.go b/node/node_handler.go index 3f46661bc..cf60f0d34 100644 --- a/node/node_handler.go +++ b/node/node_handler.go @@ -378,10 +378,9 @@ func (node *Node) PostConsensusProcessing(newBlock *types.Block, commitSigAndBit } } } - next := new(big.Int).Add(newBlock.Epoch(), common.Big1) // Update consensus keys at last so the change of leader status doesn't mess up normal flow if shard.Schedule.IsLastBlock(newBlock.Number().Uint64()) { - + next := new(big.Int).Add(newBlock.Epoch(), common.Big1) if node.chainConfig.StakingEpoch.Cmp(next) == 0 && node.Consensus.Decider.Policy() != quorum.SuperMajorityStake { node.Consensus.Decider = quorum.NewDecider(quorum.SuperMajorityStake) @@ -395,7 +394,8 @@ func (node *Node) PostConsensusProcessing(newBlock *types.Block, commitSigAndBit s.FindCommitteeByID(node.Consensus.ShardID).Slots, ) } - + // TODO Need to refactor UpdateConsensusInformation so can fold the following logic + // into UCI - todo because UCI mutates state & called in overloaded contexts node.Consensus.UpdateConsensusInformation() if shard.Schedule.IsLastBlock(newBlock.Number().Uint64()) {