From aadeae045cba9090dd3171ad5675915b3febda53 Mon Sep 17 00:00:00 2001 From: flicker-harmony Date: Wed, 12 Feb 2020 15:09:59 +0300 Subject: [PATCH] Refactor explorer service and add addresses fetch from db --- .gitignore | 1 + Dockerfile | 37 +- api/proto/node/node.go | 4 +- api/service/explorer/service.go | 621 ++---------------- api/service/explorer/storage.go | 115 ++-- api/service/explorer/storage_test.go | 81 +-- api/service/explorer/structs.go | 60 +- api/service/syncing/syncing.go | 19 +- block/header.go | 9 + block/v1/header.go | 2 +- block/v2/header.go | 2 +- cmd/harmony/main.go | 24 +- consensus/checks.go | 37 +- consensus/consensus.go | 3 +- consensus/consensus_service.go | 79 +-- consensus/consensus_service_test.go | 5 +- consensus/consensus_v2.go | 25 +- consensus/construct.go | 31 +- consensus/construct_test.go | 11 +- consensus/engine/consensus_engine.go | 6 +- consensus/fbft_log.go | 14 + consensus/fbft_log_test.go | 25 +- consensus/leader.go | 161 +++-- consensus/quorum/one-node-one-vote.go | 2 +- consensus/quorum/one-node-staked-vote.go | 16 +- consensus/quorum/one-node-staked-vote_test.go | 2 +- consensus/quorum/quorum.go | 35 +- consensus/threshold.go | 21 +- consensus/validator.go | 13 +- consensus/view_change.go | 25 +- consensus/votepower/roster.go | 43 +- core/blockchain.go | 176 ++--- core/chain_makers.go | 4 +- core/genesis.go | 5 + core/offchain.go | 51 +- core/rawdb/accessors_offchain.go | 21 +- core/rawdb/schema.go | 10 +- core/staking_verifier.go | 34 +- core/state/statedb.go | 56 +- core/state_processor.go | 40 +- core/state_transition.go | 14 +- core/tx_list.go | 14 +- core/tx_pool.go | 6 +- core/tx_pool_test.go | 8 + core/types/block.go | 12 + core/types/transaction.go | 4 +- core/types/tx_pool.go | 2 +- core/vm/interface.go | 4 +- crypto/hash/rlp.go | 8 + go.mod | 2 +- hmy/api_backend.go | 119 ++-- hmy/backend.go | 8 - internal/chain/engine.go | 153 +++-- internal/chain/reward.go | 30 +- internal/configs/sharding/testnet.go | 4 +- internal/hmyapi/apiv1/backend.go | 6 +- internal/hmyapi/apiv1/blockchain.go | 18 +- internal/hmyapi/apiv2/backend.go | 6 +- internal/hmyapi/apiv2/blockchain.go | 18 +- internal/hmyapi/backend.go | 6 +- internal/params/config.go | 10 +- internal/params/protocol_params.go | 3 +- internal/utils/singleton.go | 12 +- ...de_double_signing.go => double_signing.go} | 17 +- node/node.go | 25 +- node/node_cross_link.go | 10 +- node/node_cross_shard.go | 10 +- node/node_explorer.go | 4 +- node/node_genesis.go | 5 +- node/node_handler.go | 49 +- node/node_handler_test.go | 4 +- node/node_newblock.go | 64 +- node/service_setup.go | 2 +- node/worker/worker.go | 95 ++- scripts/go_executable_build.sh | 4 + scripts/node.sh | 8 +- shard/committee/assignment.go | 27 +- shard/shard_state.go | 127 +++- staking/availability/apply.go | 56 -- staking/availability/measure.go | 140 ++-- staking/availability/measure_test.go | 66 ++ staking/effective/calculate.go | 20 +- staking/effective/calculate_test.go | 2 +- staking/effective/eligible.go | 10 + staking/network/reward.go | 4 +- staking/slash/double-sign.go | 424 +++++++++++- staking/slash/double-sign_test.go | 495 +++++++++++++- staking/slash/report.go | 63 +- staking/slash/webhook.example.yaml | 4 + staking/types/commission.go | 51 +- staking/types/delegation.go | 86 ++- staking/types/transaction.go | 27 +- staking/types/validator.go | 162 ++--- test/chain/main.go | 4 +- test/configs/local-resharding.txt | 2 +- 95 files changed, 2664 insertions(+), 1796 deletions(-) rename node/{node_double_signing.go => double_signing.go} (53%) delete mode 100644 staking/availability/apply.go create mode 100644 staking/availability/measure_test.go create mode 100644 staking/effective/eligible.go create mode 100644 staking/slash/webhook.example.yaml diff --git a/.gitignore b/.gitignore index 1463f6b98..95752037a 100644 --- a/.gitignore +++ b/.gitignore @@ -79,3 +79,4 @@ db-127.0.0.1-* # dht storage .dht .dht-* +.tern-port diff --git a/Dockerfile b/Dockerfile index 533aa1bd0..7f137779c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,10 +17,12 @@ ENV GIMME_GO_VERSION="1.13.6" ENV PATH="/root/bin:${PATH}" RUN apt install libgmp-dev libssl-dev curl git \ -jq make gcc g++ bash tig tree sudo vim \ -silversearcher-ag unzip emacs-nox bash-completion -y +psmisc dnsutils jq make gcc g++ bash tig tree sudo vim \ +silversearcher-ag unzip emacs-nox nano bash-completion -y -RUN mkdir ~/bin && curl -sL -o ~/bin/gimme https://raw.githubusercontent.com/travis-ci/gimme/master/gimme +RUN mkdir ~/bin && \ + curl -sL -o ~/bin/gimme \ + https://raw.githubusercontent.com/travis-ci/gimme/master/gimme RUN chmod +x ~/bin/gimme @@ -71,9 +73,34 @@ RUN eval "$(~/bin/gimme ${GIMME_GO_VERSION})" ; scripts/go_executable_build.sh RUN cd ${HMY_PATH}/go-sdk && make -j8 && cp hmy /root/bin -RUN echo ". /etc/bash_completion" >> /root/.profile +ARG K1=one1tq4hy947c9gr8qzv06yxz4aeyhc9vn78al4rmu +ARG K2=one1y5gmmzumajkm5mx3g2qsxtza2d3haq0zxyg47r +ARG K3=one1qrqcfek6sc29sachs3glhs4zny72mlad76lqcp -RUN echo ". <(hmy completion)" >> /root/.profile +ARG KS1=8d222cffa99eb1fb86c581d9dfe7d60dd40ec62aa29056b7ff48028385270541 +ARG KS2=da1800da5dedf02717696675c7a7e58383aff90b1014dfa1ab5b7bd1ce3ef535 +ARG KS3=f4267bb5a2f0e65b8f5792bb6992597fac2b35ebfac9885ce0f4152c451ca31a + +RUN hmy keys import-private-key ${KS1} + +RUN hmy keys import-private-key ${KS2} + +RUN hmy keys import-private-key ${KS3} + +RUN hmy keys generate-bls-key > keys.json + +RUN jq '.["encrypted-private-key-path"]' -r keys.json > /root/keypath && cp keys.json /root + +RUN echo "export BLS_KEY_PATH=$(cat /root/keypath)" >> /root/.bashrc + +RUN echo "export BLS_KEY=$(jq '.["public-key"]' -r keys.json)" >> /root/.bashrc + +RUN echo "printf '${K1}, ${K2}, ${K3} are imported accounts in hmy for local dev\n\n'" >> /root/.bashrc + +RUN echo "printf 'test with: hmy blockchain validator information ${K1}\n\n'" >> /root/.bashrc + +RUN echo "echo "$(jq '.["public-key"]' -r keys.json)" is an extern bls key" \ + >> /root/.bashrc RUN echo ". /etc/bash_completion" >> /root/.bashrc diff --git a/api/proto/node/node.go b/api/proto/node/node.go index 147e4b482..8f023f330 100644 --- a/api/proto/node/node.go +++ b/api/proto/node/node.go @@ -180,9 +180,9 @@ func ConstructBlocksSyncMessage(blocks []*types.Block) []byte { } // ConstructSlashMessage .. -func ConstructSlashMessage(witness *slash.Record) []byte { +func ConstructSlashMessage(witnesses slash.Records) []byte { byteBuffer := bytes.NewBuffer(slashH) - slashData, _ := rlp.EncodeToBytes(witness) + slashData, _ := rlp.EncodeToBytes(witnesses) byteBuffer.Write(slashData) return byteBuffer.Bytes() } diff --git a/api/service/explorer/service.go b/api/service/explorer/service.go index 94a2bb5b3..7fc636d3b 100644 --- a/api/service/explorer/service.go +++ b/api/service/explorer/service.go @@ -4,34 +4,27 @@ import ( "context" "encoding/json" "fmt" - "math/big" "net" "net/http" - "os" - "sort" "strconv" - "strings" + "time" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rpc" "github.com/gorilla/mux" - libp2p_peer "github.com/libp2p/go-libp2p-peer" msg_pb "github.com/harmony-one/harmony/api/proto/message" - "github.com/harmony-one/harmony/core/types" - "github.com/harmony-one/harmony/internal/bech32" - common2 "github.com/harmony-one/harmony/internal/common" + "github.com/harmony-one/harmony/consensus/reward" "github.com/harmony-one/harmony/internal/utils" + "github.com/harmony-one/harmony/numeric" "github.com/harmony-one/harmony/p2p" ) // Constants for explorer service. const ( explorerPortDifference = 4000 - paginationOffset = 10 - txViewNone = "NONE" - txViewAll = "ALL" + defaultPageSize = "1000" + maxAddresses = 100000 + totalSupply = 12600000000 ) // HTTPError is an HTTP error. @@ -42,36 +35,17 @@ type HTTPError struct { // Service is the struct for explorer service. type Service struct { - router *mux.Router - IP string - Port string - GetNodeIDs func() []libp2p_peer.ID - ShardID uint32 - Storage *Storage - server *http.Server - messageChan chan *msg_pb.Message - GetAccountBalance func(common.Address) (*big.Int, error) + router *mux.Router + IP string + Port string + Storage *Storage + server *http.Server + messageChan chan *msg_pb.Message } // New returns explorer service. -func New(selfPeer *p2p.Peer, shardID uint32, GetNodeIDs func() []libp2p_peer.ID, GetAccountBalance func(common.Address) (*big.Int, error)) *Service { - return &Service{ - IP: selfPeer.IP, - Port: selfPeer.Port, - ShardID: shardID, - GetNodeIDs: GetNodeIDs, - GetAccountBalance: GetAccountBalance, - } -} - -// ServiceAPI is rpc api. -type ServiceAPI struct { - Service *Service -} - -// NewServiceAPI returns explorer service api. -func NewServiceAPI(explorerService *Service) *ServiceAPI { - return &ServiceAPI{Service: explorerService} +func New(selfPeer *p2p.Peer) *Service { + return &Service{IP: selfPeer.IP, Port: selfPeer.Port} } // StartService starts explorer service. @@ -111,27 +85,20 @@ func (s *Service) Run() *http.Server { addr := net.JoinHostPort("", GetExplorerPort(s.Port)) s.router = mux.NewRouter() - // Set up router for blocks. - // Blocks are divided into pages in consequent groups of offset size. - s.router.Path("/blocks").Queries("from", "{[0-9]*?}", "to", "{[0-9]*?}", "page", "{[0-9]*?}", "offset", "{[0-9]*?}").HandlerFunc(s.GetExplorerBlocks).Methods("GET") - s.router.Path("/blocks").HandlerFunc(s.GetExplorerBlocks) - // Set up router for tx. - s.router.Path("/tx").Queries("id", "{[0-9A-Fa-fx]*?}").HandlerFunc(s.GetExplorerTransaction).Methods("GET") - s.router.Path("/tx").HandlerFunc(s.GetExplorerTransaction) - - // Set up router for address. - // Address transactions are divided into pages in consequent groups of offset size. - s.router.Path("/address").Queries("id", fmt.Sprintf("{([0-9A-Fa-fx]*?)|(t?one1[%s]{38})}", bech32.Charset), "tx_view", "{[A-Z]*?}", "page", "{[0-9]*?}", "offset", "{[0-9]*?}").HandlerFunc(s.GetExplorerAddress).Methods("GET") - s.router.Path("/address").HandlerFunc(s.GetExplorerAddress) + // Set up router for addresses. + // Fetch addresses request, accepts parameter size: how much addresses to read, + // parameter prefix: from which address prefix start + s.router.Path("/addresses").Queries("size", "{[0-9]*?}", "prefix", "{[a-zA-Z0-9]*?}").HandlerFunc(s.GetAddresses).Methods("GET") + s.router.Path("/addresses").HandlerFunc(s.GetAddresses) // Set up router for node count. - s.router.Path("/nodes").HandlerFunc(s.GetExplorerNodeCount) // TODO(ricl): this is going to be replaced by /node-count - s.router.Path("/node-count").HandlerFunc(s.GetExplorerNodeCount) + s.router.Path("/circulating-supply").Queries().HandlerFunc(s.GetCirculatingSupply).Methods("GET") + s.router.Path("/circulating-supply").HandlerFunc(s.GetCirculatingSupply) - // Set up router for shard - s.router.Path("/shard").Queries("id", "{[0-9]*?}").HandlerFunc(s.GetExplorerShard).Methods("GET") - s.router.Path("/shard").HandlerFunc(s.GetExplorerShard) + // Set up router for node count. + s.router.Path("/total-supply").Queries().HandlerFunc(s.GetTotalSupply).Methods("GET") + s.router.Path("/total-supply").HandlerFunc(s.GetTotalSupply) // Do serving now. utils.Logger().Info().Str("port", GetExplorerPort(s.Port)).Msg("Listening") @@ -144,539 +111,54 @@ func (s *Service) Run() *http.Server { return server } -// ReadBlocksFromDB returns a list of types.Block to server blocks end-point. -func (s *Service) ReadBlocksFromDB(from, to int) []*types.Block { - blocks := []*types.Block{} - for i := from - 1; i <= to+1; i++ { - if i < 0 { - blocks = append(blocks, nil) - continue - } - key := GetBlockKey(i) - data, err := s.Storage.db.Get([]byte(key)) - if err != nil { - blocks = append(blocks, nil) - continue - } - block := new(types.Block) - if rlp.DecodeBytes(data, block) != nil { - utils.Logger().Error().Msg("Error on getting from db") - os.Exit(1) - } - blocks = append(blocks, block) - } - return blocks -} - -// GetExplorerBlocks serves end-point /blocks -func (s *Service) GetExplorerBlocks(w http.ResponseWriter, r *http.Request) { +// GetAddresses serves end-point /addresses, returns size of addresses from address with prefix. +func (s *Service) GetAddresses(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") - from := r.FormValue("from") - to := r.FormValue("to") - pageParam := r.FormValue("page") - offsetParam := r.FormValue("offset") - order := r.FormValue("order") - data := &Data{ - Blocks: []*Block{}, - } - defer func() { - if err := json.NewEncoder(w).Encode(data.Blocks); err != nil { - utils.Logger().Warn().Err(err).Msg("cannot JSON-encode blocks") - } - }() - - if from == "" { - utils.Logger().Warn().Msg("Missing from parameter") - w.WriteHeader(http.StatusBadRequest) - return - } - db := s.Storage.GetDB() - fromInt, err := strconv.Atoi(from) - if err != nil { - utils.Logger().Warn().Err(err).Msg("invalid from parameter") - w.WriteHeader(http.StatusBadRequest) - return - } - var toInt int - if to == "" { - toInt, err = func() (int, error) { - bytes, err := db.Get([]byte(BlockHeightKey)) - if err == nil { - return strconv.Atoi(string(bytes)) - } - return toInt, err - }() - } else { - toInt, err = strconv.Atoi(to) - } - if err != nil { - utils.Logger().Warn().Err(err).Msg("invalid to parameter") - w.WriteHeader(http.StatusBadRequest) - return - } - var offset int - if offsetParam != "" { - offset, err = strconv.Atoi(offsetParam) - if err != nil || offset < 1 { - utils.Logger().Warn().Msg("invalid offset parameter") - w.WriteHeader(http.StatusBadRequest) - return - } - } else { - offset = paginationOffset - } - var page int - if pageParam != "" { - page, err = strconv.Atoi(pageParam) - if err != nil { - utils.Logger().Warn().Err(err).Msg("invalid page parameter") - w.WriteHeader(http.StatusBadRequest) - return - } - } else { - page = 0 + sizeStr := r.FormValue("size") + prefix := r.FormValue("prefix") + if sizeStr == "" { + sizeStr = defaultPageSize } - - accountBlocks := s.ReadBlocksFromDB(fromInt, toInt) - for id, accountBlock := range accountBlocks { - if id == 0 || id == len(accountBlocks)-1 || accountBlock == nil { - continue - } - block := NewBlock(accountBlock, id+fromInt-1) - // Populate transactions - for _, tx := range accountBlock.Transactions() { - transaction := GetTransaction(tx, accountBlock) - if transaction != nil { - block.TXs = append(block.TXs, transaction) - } - } - if accountBlocks[id-1] == nil { - block.BlockTime = int64(0) - block.PrevBlock = RefBlock{ - ID: "", - Height: "", - } - } else { - block.BlockTime = accountBlock.Time().Int64() - accountBlocks[id-1].Time().Int64() - block.PrevBlock = RefBlock{ - ID: accountBlocks[id-1].Hash().Hex(), - Height: strconv.Itoa(id + fromInt - 2), - } - } - if accountBlocks[id+1] == nil { - block.NextBlock = RefBlock{ - ID: "", - Height: "", - } - } else { - block.NextBlock = RefBlock{ - ID: accountBlocks[id+1].Hash().Hex(), - Height: strconv.Itoa(id + fromInt), - } - } - data.Blocks = append(data.Blocks, block) - } - if offset*page >= len(data.Blocks) { - data.Blocks = []*Block{} - } else if offset*page+offset > len(data.Blocks) { - data.Blocks = data.Blocks[offset*page:] - } else { - data.Blocks = data.Blocks[offset*page : offset*page+offset] - } - if order == "DESC" { - sort.Slice(data.Blocks[:], func(i, j int) bool { - return data.Blocks[i].Timestamp > data.Blocks[j].Timestamp - }) - } else { - sort.Slice(data.Blocks[:], func(i, j int) bool { - return data.Blocks[i].Timestamp < data.Blocks[j].Timestamp - }) - } -} - -// GetExplorerBlocks rpc end-point. -func (s *ServiceAPI) GetExplorerBlocks(ctx context.Context, from, to, page, offset int, order string) ([]*Block, error) { - if offset == 0 { - offset = paginationOffset - } - db := s.Service.Storage.GetDB() - if to == 0 { - bytes, err := db.Get([]byte(BlockHeightKey)) - if err == nil { - to, err = strconv.Atoi(string(bytes)) - if err != nil { - utils.Logger().Info().Msg("failed to fetch block height") - return nil, err - } - } - } - blocks := make([]*Block, 0) - accountBlocks := s.Service.ReadBlocksFromDB(from, to) - for id, accountBlock := range accountBlocks { - if id == 0 || id == len(accountBlocks)-1 || accountBlock == nil { - continue - } - block := NewBlock(accountBlock, id+from-1) - // Populate transactions - for _, tx := range accountBlock.Transactions() { - transaction := GetTransaction(tx, accountBlock) - if transaction != nil { - block.TXs = append(block.TXs, transaction) - } - } - if accountBlocks[id-1] == nil { - block.BlockTime = int64(0) - block.PrevBlock = RefBlock{ - ID: "", - Height: "", - } - } else { - block.BlockTime = accountBlock.Time().Int64() - accountBlocks[id-1].Time().Int64() - block.PrevBlock = RefBlock{ - ID: accountBlocks[id-1].Hash().Hex(), - Height: strconv.Itoa(id + from - 2), - } - } - if accountBlocks[id+1] == nil { - block.NextBlock = RefBlock{ - ID: "", - Height: "", - } - } else { - block.NextBlock = RefBlock{ - ID: accountBlocks[id+1].Hash().Hex(), - Height: strconv.Itoa(id + from), - } - } - blocks = append(blocks, block) - } - if offset*page >= len(blocks) { - blocks = []*Block{} - } else if offset*page+offset > len(blocks) { - blocks = blocks[offset*page:] - } else { - blocks = blocks[offset*page : offset*page+offset] - } - if order == "DESC" { - sort.Slice(blocks[:], func(i, j int) bool { - return blocks[i].Timestamp > blocks[j].Timestamp - }) - } else { - sort.Slice(blocks[:], func(i, j int) bool { - return blocks[i].Timestamp < blocks[j].Timestamp - }) - } - return blocks, nil -} - -// GetExplorerTransaction servers /tx end-point. -func (s *Service) GetExplorerTransaction(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - id := r.FormValue("id") - data := &Data{} defer func() { - if err := json.NewEncoder(w).Encode(data.TX); err != nil { - utils.Logger().Warn().Err(err).Msg("cannot JSON-encode TX") + if err := json.NewEncoder(w).Encode(data.Addresses); err != nil { + utils.Logger().Warn().Err(err).Msg("cannot JSON-encode addresses") } }() - if id == "" { - utils.Logger().Warn().Msg("invalid id parameter") - w.WriteHeader(http.StatusBadRequest) - return - } - db := s.Storage.GetDB() - bytes, err := db.Get([]byte(GetTXKey(id))) - if err != nil { - utils.Logger().Warn().Err(err).Str("id", id).Msg("cannot read TX") - w.WriteHeader(http.StatusInternalServerError) - return - } - tx := new(Transaction) - if rlp.DecodeBytes(bytes, tx) != nil { - utils.Logger().Warn().Str("id", id).Msg("cannot convert data from DB") - w.WriteHeader(http.StatusInternalServerError) - return - } - data.TX = *tx -} - -// GetExplorerTransaction rpc end-point. -func (s *ServiceAPI) GetExplorerTransaction(ctx context.Context, id string) (*Transaction, error) { - if id == "" { - utils.Logger().Warn().Msg("invalid id parameter") - return nil, nil - } - db := s.Service.Storage.GetDB() - bytes, err := db.Get([]byte(GetTXKey(id))) - if err != nil { - utils.Logger().Warn().Err(err).Str("id", id).Msg("cannot read TX") - return nil, err - } - tx := new(Transaction) - if rlp.DecodeBytes(bytes, tx) != nil { - utils.Logger().Warn().Str("id", id).Msg("cannot convert data from DB") - return nil, err - } - return tx, nil -} -// GetExplorerAddress serves /address end-point. -func (s *Service) GetExplorerAddress(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - id := r.FormValue("id") - if strings.HasPrefix(id, "0x") { - parsedAddr := common2.ParseAddr(id) - oneAddr, err := common2.AddressToBech32(parsedAddr) - if err != nil { - utils.Logger().Warn().Err(err).Msg("unrecognized address format") - w.WriteHeader(http.StatusBadRequest) - return - } - id = oneAddr - } - key := GetAddressKey(id) - txViewParam := r.FormValue("tx_view") - pageParam := r.FormValue("page") - offsetParam := r.FormValue("offset") - order := r.FormValue("order") - txView := txViewNone - if txViewParam != "" { - txView = txViewParam - } - utils.Logger().Info().Str("Address", id).Msg("Querying address") - data := &Data{} - defer func() { - if err := json.NewEncoder(w).Encode(data.Address); err != nil { - utils.Logger().Warn().Err(err).Msg("cannot JSON-encode Address") - } - }() - if id == "" { - utils.Logger().Warn().Msg("missing address id param") + size, err := strconv.Atoi(sizeStr) + if err != nil || size > maxAddresses { w.WriteHeader(http.StatusBadRequest) return } - var err error - var offset int - if offsetParam != "" { - offset, err = strconv.Atoi(offsetParam) - if err != nil || offset < 1 { - utils.Logger().Warn().Msg("invalid offset parameter") - w.WriteHeader(http.StatusBadRequest) - return - } - } else { - offset = paginationOffset - } - var page int - if pageParam != "" { - page, err = strconv.Atoi(pageParam) - if err != nil { - utils.Logger().Warn().Err(err).Msg("invalid page parameter") - w.WriteHeader(http.StatusBadRequest) - return - } - } else { - page = 0 - } - - data.Address.ID = id - // Try to populate the banace by directly calling get balance. - // Check the balance from blockchain rather than local DB dump - balanceAddr := big.NewInt(0) - if s.GetAccountBalance != nil { - address := common2.ParseAddr(id) - balance, err := s.GetAccountBalance(address) - if err == nil { - balanceAddr = balance - } - } - - db := s.Storage.GetDB() - bytes, err := db.Get([]byte(key)) + data.Addresses, err = s.Storage.GetAddresses(size, prefix) if err != nil { - utils.Logger().Warn().Err(err).Str("id", id).Msg("cannot fetch address from db") - data.Address.Balance = balanceAddr - return - } - - if err = rlp.DecodeBytes(bytes, &data.Address); err != nil { - utils.Logger().Warn().Str("id", id).Msg("cannot convert address data") - data.Address.Balance = balanceAddr + w.WriteHeader(http.StatusInternalServerError) + utils.Logger().Warn().Err(err).Msg("wasn't able to fetch addresses from storage") return } - - data.Address.Balance = balanceAddr - - switch txView { - case txViewNone: - data.Address.TXs = nil - case Received: - receivedTXs := make([]*Transaction, 0) - for _, tx := range data.Address.TXs { - if tx.Type == Received { - receivedTXs = append(receivedTXs, tx) - } - } - data.Address.TXs = receivedTXs - case Sent: - sentTXs := make([]*Transaction, 0) - for _, tx := range data.Address.TXs { - if tx.Type == Sent { - sentTXs = append(sentTXs, tx) - } - } - data.Address.TXs = sentTXs - } - if offset*page >= len(data.Address.TXs) { - data.Address.TXs = []*Transaction{} - } else if offset*page+offset > len(data.Address.TXs) { - data.Address.TXs = data.Address.TXs[offset*page:] - } else { - data.Address.TXs = data.Address.TXs[offset*page : offset*page+offset] - } - if order == "DESC" { - sort.Slice(data.Address.TXs[:], func(i, j int) bool { - return data.Address.TXs[i].Timestamp > data.Address.TXs[j].Timestamp - }) - } else { - sort.Slice(data.Address.TXs[:], func(i, j int) bool { - return data.Address.TXs[i].Timestamp < data.Address.TXs[j].Timestamp - }) - } } -// GetExplorerAddress rpc end-point. -func (s *ServiceAPI) GetExplorerAddress(ctx context.Context, id, txView string, page, offset int, order string) (*Address, error) { - if strings.HasPrefix(id, "0x") { - parsedAddr := common2.ParseAddr(id) - oneAddr, err := common2.AddressToBech32(parsedAddr) - if err != nil { - utils.Logger().Warn().Msg("unrecognized address format") - return nil, err - } - id = oneAddr - } - if offset == 0 { - offset = paginationOffset - } - if txView == "" { - txView = txViewNone - } - utils.Logger().Info().Str("Address", id).Msg("Querying address") - if id == "" { - utils.Logger().Warn().Msg("missing address id param") - return nil, nil - } - address := &Address{} - address.ID = id - // Try to populate the banace by directly calling get balance. - // Check the balance from blockchain rather than local DB dump - balanceAddr := big.NewInt(0) - if s.Service.GetAccountBalance != nil { - addr := common2.ParseAddr(id) - balance, err := s.Service.GetAccountBalance(addr) - if err == nil { - balanceAddr = balance - } - } - - key := GetAddressKey(id) - db := s.Service.Storage.GetDB() - bytes, err := db.Get([]byte(key)) - if err != nil { - utils.Logger().Warn().Err(err).Str("id", id).Msg("cannot fetch address from db") - address.Balance = balanceAddr - return address, err - } - if err = rlp.DecodeBytes(bytes, &address); err != nil { - utils.Logger().Warn().Str("id", id).Msg("cannot convert address data") - address.Balance = balanceAddr - return address, err - } - - address.Balance = balanceAddr - - switch txView { - case txViewNone: - address.TXs = nil - case Received: - receivedTXs := make([]*Transaction, 0) - for _, tx := range address.TXs { - if tx.Type == Received { - receivedTXs = append(receivedTXs, tx) - } - } - address.TXs = receivedTXs - case Sent: - sentTXs := make([]*Transaction, 0) - for _, tx := range address.TXs { - if tx.Type == Sent { - sentTXs = append(sentTXs, tx) - } - } - address.TXs = sentTXs - } - - if offset*page >= len(address.TXs) { - address.TXs = []*Transaction{} - } else if offset*page+offset > len(address.TXs) { - address.TXs = address.TXs[offset*page:] - } else { - address.TXs = address.TXs[offset*page : offset*page+offset] - } - if order == "DESC" { - sort.Slice(address.TXs[:], func(i, j int) bool { - return address.TXs[i].Timestamp > address.TXs[j].Timestamp - }) - } else { - sort.Slice(address.TXs[:], func(i, j int) bool { - return address.TXs[i].Timestamp < address.TXs[j].Timestamp - }) - } - return address, nil -} - -// GetExplorerNodeCount serves /nodes end-point. -func (s *Service) GetExplorerNodeCount(w http.ResponseWriter, r *http.Request) { +// GetCirculatingSupply serves /circulating-supply end-point. +func (s *Service) GetCirculatingSupply(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") - if err := json.NewEncoder(w).Encode(len(s.GetNodeIDs())); err != nil { - utils.Logger().Warn().Msg("cannot JSON-encode node count") + timestamp := time.Now().Unix() + circulatingSupply := reward.PercentageForTimeStamp(timestamp).Mul(numeric.NewDec(totalSupply)) + if err := json.NewEncoder(w).Encode(circulatingSupply); err != nil { + utils.Logger().Warn().Msg("cannot JSON-encode circulating supply") w.WriteHeader(http.StatusInternalServerError) } } -// GetExplorerNodeCount rpc end-point. -func (s *ServiceAPI) GetExplorerNodeCount(ctx context.Context) int { - return len(s.Service.GetNodeIDs()) -} - -// GetExplorerShard serves /shard end-point. -func (s *Service) GetExplorerShard(w http.ResponseWriter, r *http.Request) { +// GetTotalSupply serves /total-supply end-point. +func (s *Service) GetTotalSupply(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") - var nodes []Node - for _, nodeID := range s.GetNodeIDs() { - nodes = append(nodes, Node{ - ID: libp2p_peer.IDB58Encode(nodeID), - }) - } - if err := json.NewEncoder(w).Encode(Shard{Nodes: nodes}); err != nil { - utils.Logger().Warn().Msg("cannot JSON-encode shard info") + if err := json.NewEncoder(w).Encode(totalSupply); err != nil { + utils.Logger().Warn().Msg("cannot JSON-encode total supply") w.WriteHeader(http.StatusInternalServerError) } } -// GetExplorerShard rpc end-point. -func (s *ServiceAPI) GetExplorerShard(ctx context.Context) *Shard { - var nodes []Node - for _, nodeID := range s.Service.GetNodeIDs() { - nodes = append(nodes, Node{ - ID: libp2p_peer.IDB58Encode(nodeID), - }) - } - return &Shard{Nodes: nodes} -} - // NotifyService notify service. func (s *Service) NotifyService(params map[string]interface{}) { return @@ -689,12 +171,5 @@ func (s *Service) SetMessageChan(messageChan chan *msg_pb.Message) { // APIs for the services. func (s *Service) APIs() []rpc.API { - return []rpc.API{ - { - Namespace: "explorer", - Version: "1.0", - Service: NewServiceAPI(s), - Public: true, - }, - } + return nil } diff --git a/api/service/explorer/storage.go b/api/service/explorer/storage.go index 9bd3bee03..325802c19 100644 --- a/api/service/explorer/storage.go +++ b/api/service/explorer/storage.go @@ -3,51 +3,34 @@ package explorer import ( "fmt" "os" - "strconv" "sync" - "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/rlp" - "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/internal/utils" + "github.com/syndtr/goleveldb/leveldb" + "github.com/syndtr/goleveldb/leveldb/filter" + "github.com/syndtr/goleveldb/leveldb/opt" + "github.com/syndtr/goleveldb/leveldb/util" ) // Constants for storage. const ( - BlockHeightKey = "bh" - BlockInfoPrefix = "bi" - BlockPrefix = "b" - TXPrefix = "tx" - AddressPrefix = "ad" + AddressPrefix = "ad" + PrefixLen = 3 ) -// GetBlockInfoKey ... -func GetBlockInfoKey(id int) string { - return fmt.Sprintf("%s_%d", BlockInfoPrefix, id) -} - // GetAddressKey ... func GetAddressKey(address string) string { return fmt.Sprintf("%s_%s", AddressPrefix, address) } -// GetBlockKey ... -func GetBlockKey(id int) string { - return fmt.Sprintf("%s_%d", BlockPrefix, id) -} - -// GetTXKey ... -func GetTXKey(hash string) string { - return fmt.Sprintf("%s_%s", TXPrefix, hash) -} - var storage *Storage var once sync.Once // Storage dump the block info into leveldb. type Storage struct { - db *ethdb.LDBDatabase + db *leveldb.DB } // GetStorageInstance returns attack model by using singleton pattern. @@ -69,65 +52,43 @@ func (storage *Storage) Init(ip, port string, remove bool) { utils.Logger().Error().Err(err).Msg("Failed to remove existing database files") } } - if storage.db, err = ethdb.NewLDBDatabase(dbFileName, 0, 0); err != nil { + // https://github.com/ethereum/go-ethereum/blob/master/ethdb/leveldb/leveldb.go#L98 options. + // We had 0 for handles and cache params before, so set 0s for all of them. Filter opt is the same. + options := &opt.Options{ + OpenFilesCacheCapacity: 0, + BlockCacheCapacity: 0, + WriteBuffer: 0, + Filter: filter.NewBloomFilter(10), + } + if storage.db, err = leveldb.OpenFile(dbFileName, options); err != nil { utils.Logger().Error().Err(err).Msg("Failed to create new database") } } // GetDB returns the LDBDatabase of the storage. -func (storage *Storage) GetDB() *ethdb.LDBDatabase { +func (storage *Storage) GetDB() *leveldb.DB { return storage.db } // Dump extracts information from block and index them into lvdb for explorer. func (storage *Storage) Dump(block *types.Block, height uint64) { - //utils.Logger().Debug().Uint64("block height", height).Msg("Dumping block") if block == nil { return } - batch := storage.db.NewBatch() - // Update block height. - if err := batch.Put([]byte(BlockHeightKey), []byte(strconv.Itoa(int(height)))); err != nil { - utils.Logger().Warn().Err(err).Msg("cannot batch block height") - } - - // Store block. - blockData, err := rlp.EncodeToBytes(block) - if err == nil { - if err := batch.Put([]byte(GetBlockKey(int(height))), blockData); err != nil { - utils.Logger().Warn().Err(err).Msg("cannot batch block data") - } - } else { - utils.Logger().Error().Err(err).Msg("Failed to serialize block") - } - + batch := new(leveldb.Batch) // Store txs for _, tx := range block.Transactions() { explorerTransaction := GetTransaction(tx, block) - storage.UpdateTXStorage(batch, explorerTransaction, tx) storage.UpdateAddress(batch, explorerTransaction, tx) } - if err := batch.Write(); err != nil { + if err := storage.db.Write(batch, nil); err != nil { utils.Logger().Warn().Err(err).Msg("cannot write batch") } } -// UpdateTXStorage ... -func (storage *Storage) UpdateTXStorage(batch ethdb.Batch, explorerTransaction *Transaction, tx *types.Transaction) { - if data, err := rlp.EncodeToBytes(explorerTransaction); err == nil { - key := GetTXKey(tx.Hash().Hex()) - if err := batch.Put([]byte(key), data); err != nil { - utils.Logger().Warn().Err(err).Msg("cannot batch TX") - } - } else { - utils.Logger().Error().Msg("EncodeRLP transaction error") - } -} - // UpdateAddress ... -// TODO: deprecate this logic -func (storage *Storage) UpdateAddress(batch ethdb.Batch, explorerTransaction *Transaction, tx *types.Transaction) { +func (storage *Storage) UpdateAddress(batch *leveldb.Batch, explorerTransaction *Transaction, tx *types.Transaction) { explorerTransaction.Type = Received if explorerTransaction.To != "" { storage.UpdateAddressStorage(batch, explorerTransaction.To, explorerTransaction, tx) @@ -137,12 +98,10 @@ func (storage *Storage) UpdateAddress(batch ethdb.Batch, explorerTransaction *Tr } // UpdateAddressStorage updates specific addr Address. -// TODO: deprecate this logic -func (storage *Storage) UpdateAddressStorage(batch ethdb.Batch, addr string, explorerTransaction *Transaction, tx *types.Transaction) { - key := GetAddressKey(addr) - +func (storage *Storage) UpdateAddressStorage(batch *leveldb.Batch, addr string, explorerTransaction *Transaction, tx *types.Transaction) { var address Address - if data, err := storage.db.Get([]byte(key)); err == nil { + key := GetAddressKey(addr) + if data, err := storage.db.Get([]byte(key), nil); err == nil { if err = rlp.DecodeBytes(data, &address); err != nil { utils.Logger().Error().Err(err).Msg("Failed due to error") } @@ -151,10 +110,32 @@ func (storage *Storage) UpdateAddressStorage(batch ethdb.Batch, addr string, exp address.TXs = append(address.TXs, explorerTransaction) encoded, err := rlp.EncodeToBytes(address) if err == nil { - if err := batch.Put([]byte(key), encoded); err != nil { - utils.Logger().Warn().Err(err).Msg("cannot batch address") - } + batch.Put([]byte(key), encoded) } else { utils.Logger().Error().Err(err).Msg("cannot encode address") } } + +// GetAddresses returns size of addresses from address with prefix. +func (storage *Storage) GetAddresses(size int, prefix string) ([]string, error) { + db := storage.GetDB() + key := GetAddressKey(prefix) + iterator := db.NewIterator(&util.Range{Start: []byte(key)}, nil) + addresses := make([]string, 0) + read := 0 + for iterator.Next() && read < size { + address := string(iterator.Key()) + read++ + if len(address) < PrefixLen { + utils.Logger().Info().Msgf("address len < 3 %s", address) + continue + } + addresses = append(addresses, address[PrefixLen:]) + } + iterator.Release() + if err := iterator.Error(); err != nil { + utils.Logger().Error().Err(err).Msg("iterator error") + return nil, err + } + return addresses, nil +} diff --git a/api/service/explorer/storage_test.go b/api/service/explorer/storage_test.go index 54ff6f564..68bd33a71 100644 --- a/api/service/explorer/storage_test.go +++ b/api/service/explorer/storage_test.go @@ -2,98 +2,23 @@ package explorer import ( "bytes" - "math/big" - "strconv" "testing" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/rlp" - - blockfactory "github.com/harmony-one/harmony/block/factory" - "github.com/harmony-one/harmony/core/types" "github.com/stretchr/testify/assert" ) -// Test for GetBlockInfoKey -func TestGetBlockInfoKey(t *testing.T) { - assert.Equal(t, GetBlockInfoKey(3), "bi_3", "error") -} - // Test for GetAddressKey func TestGetAddressKey(t *testing.T) { assert.Equal(t, GetAddressKey("abcd"), "ad_abcd", "error") } -// Test for GetBlockKey -func TestGetBlockKey(t *testing.T) { - assert.Equal(t, GetBlockKey(3), "b_3", "error") -} - -// Test for GetTXKey -func TestGetTXKey(t *testing.T) { - assert.Equal(t, GetTXKey("abcd"), "tx_abcd", "error") -} - +// TestInit .. func TestInit(t *testing.T) { ins := GetStorageInstance("1.1.1.1", "3333", true) - if err := ins.GetDB().Put([]byte{1}, []byte{2}); err != nil { + if err := ins.GetDB().Put([]byte{1}, []byte{2}, nil); err != nil { t.Fatal("(*LDBDatabase).Put failed:", err) } - value, err := ins.GetDB().Get([]byte{1}) + value, err := ins.GetDB().Get([]byte{1}, nil) assert.Equal(t, bytes.Compare(value, []byte{2}), 0, "value should be []byte{2}") assert.Nil(t, err, "error should be nil") } - -func TestDump(t *testing.T) { - tx1 := types.NewTransaction(1, common.BytesToAddress([]byte{0x11}), 0, big.NewInt(111), 1111, big.NewInt(11111), []byte{0x11, 0x11, 0x11}) - tx2 := types.NewTransaction(2, common.BytesToAddress([]byte{0x22}), 0, big.NewInt(222), 2222, big.NewInt(22222), []byte{0x22, 0x22, 0x22}) - tx3 := types.NewTransaction(3, common.BytesToAddress([]byte{0x33}), 0, big.NewInt(333), 3333, big.NewInt(33333), []byte{0x33, 0x33, 0x33}) - txs := []*types.Transaction{tx1, tx2, tx3} - - block := types.NewBlock(blockfactory.NewTestHeader().With().Number(big.NewInt(314)).Header(), txs, types.Receipts{&types.Receipt{}, &types.Receipt{}, &types.Receipt{}}, nil, nil, nil) - ins := GetStorageInstance("1.1.1.1", "3333", true) - ins.Dump(block, uint64(1)) - db := ins.GetDB() - - res, err := db.Get([]byte(BlockHeightKey)) - if err == nil { - toInt, err := strconv.Atoi(string(res)) - assert.Equal(t, toInt, 1, "error") - assert.Nil(t, err, "error") - } else { - t.Error("Error") - } - - data, err := db.Get([]byte(GetBlockKey(1))) - assert.Nil(t, err, "should be nil") - blockData, err := rlp.EncodeToBytes(block) - assert.Nil(t, err, "should be nil") - assert.Equal(t, bytes.Compare(data, blockData), 0, "should be equal") -} - -func TestUpdateAddressStorage(t *testing.T) { - tx1 := types.NewTransaction(1, common.BytesToAddress([]byte{0x11}), 0, big.NewInt(111), 1111, big.NewInt(11111), []byte{0x11, 0x11, 0x11}) - tx2 := types.NewTransaction(2, common.BytesToAddress([]byte{0x22}), 0, big.NewInt(222), 2222, big.NewInt(22222), []byte{0x22, 0x22, 0x22}) - tx3 := types.NewTransaction(3, common.BytesToAddress([]byte{0x33}), 0, big.NewInt(333), 3333, big.NewInt(33333), []byte{0x33, 0x33, 0x33}) - txs := []*types.Transaction{tx1, tx2, tx3} - - block := types.NewBlock(blockfactory.NewTestHeader().With().Number(big.NewInt(314)).Header(), txs, types.Receipts{&types.Receipt{}, &types.Receipt{}, &types.Receipt{}}, nil, nil, nil) - ins := GetStorageInstance("1.1.1.1", "3333", true) - ins.Dump(block, uint64(1)) - db := ins.GetDB() - - res, err := db.Get([]byte(BlockHeightKey)) - if err == nil { - toInt, err := strconv.Atoi(string(res)) - assert.Equal(t, toInt, 1, "error") - assert.Nil(t, err, "error") - } else { - t.Error("Error") - } - - data, err := db.Get([]byte(GetBlockKey(1))) - assert.Nil(t, err, "should be nil") - blockData, err := rlp.EncodeToBytes(block) - assert.Nil(t, err, "should be nil") - assert.Equal(t, bytes.Compare(data, blockData), 0, "should be equal") -} diff --git a/api/service/explorer/structs.go b/api/service/explorer/structs.go index 2560886f6..56c5cd38a 100644 --- a/api/service/explorer/structs.go +++ b/api/service/explorer/structs.go @@ -22,10 +22,7 @@ const ( // Data ... type Data struct { - Blocks []*Block `json:"blocks"` - // Block Block `json:"block"` - Address Address `json:"Address"` - TX Transaction + Addresses []string `json:"Addresses"` } // Address ... @@ -35,12 +32,6 @@ type Address struct { TXs []*Transaction `json:"txs"` } -// Validator contains harmony validator node address and its balance. -type Validator struct { - Address string `json:"address"` - Balance *big.Int `json:"balance"` -} - // Transaction ... type Transaction struct { ID string `json:"id"` @@ -56,55 +47,6 @@ type Transaction struct { Type string `json:"type"` } -// Block ... -type Block struct { - Height string `json:"height"` - ID string `json:"id"` - TXCount string `json:"txCount"` - Timestamp string `json:"timestamp"` - BlockTime int64 `json:"blockTime"` - MerkleRoot string `json:"merkleRoot"` - PrevBlock RefBlock `json:"prevBlock"` - Bytes string `json:"bytes"` - NextBlock RefBlock `json:"nextBlock"` - TXs []*Transaction `json:"txs"` - Signers []string `json:"signers"` - Epoch uint64 `json:"epoch"` - ExtraData string `json:"extra_data"` -} - -// RefBlock ... -type RefBlock struct { - ID string `json:"id"` - Height string `json:"height"` -} - -// Node ... -type Node struct { - ID string `json:"id"` -} - -// Shard ... -type Shard struct { - Nodes []Node `json:"nodes"` -} - -// NewBlock ... -func NewBlock(block *types.Block, height int) *Block { - // TODO(ricl): use block.Header().CommitBitmap and GetPubKeyFromMask - return &Block{ - Height: strconv.Itoa(height), - ID: block.Hash().Hex(), - TXCount: strconv.Itoa(block.Transactions().Len()), - Timestamp: strconv.Itoa(int(block.Time().Int64() * 1000)), - MerkleRoot: block.Root().Hex(), - Bytes: strconv.Itoa(int(block.Size())), - Signers: []string{}, - Epoch: block.Epoch().Uint64(), - ExtraData: string(block.Extra()), - } -} - // GetTransaction ... func GetTransaction(tx *types.Transaction, addressBlock *types.Block) *Transaction { msg, err := tx.AsMessage(types.NewEIP155Signer(tx.ChainID())) diff --git a/api/service/syncing/syncing.go b/api/service/syncing/syncing.go index c03d6db9c..ec8231826 100644 --- a/api/service/syncing/syncing.go +++ b/api/service/syncing/syncing.go @@ -572,9 +572,17 @@ func (ss *StateSync) UpdateBlockAndStatus(block *types.Block, bc *core.BlockChai _, err := bc.InsertChain([]*types.Block{block}, false /* verifyHeaders */) if err != nil { - utils.Logger().Error().Err(err).Msgf("[SYNC] UpdateBlockAndStatus: Error adding new block to blockchain %d %d", block.NumberU64(), block.ShardID()) - - utils.Logger().Debug().Interface("block", bc.CurrentBlock()).Msg("[SYNC] UpdateBlockAndStatus: Rolling back current block!") + utils.Logger().Error(). + Err(err). + Msgf( + "[SYNC] UpdateBlockAndStatus: Error adding new block to blockchain %d %d", + block.NumberU64(), + block.ShardID(), + ) + + utils.Logger().Debug(). + Interface("block", bc.CurrentBlock()). + Msg("[SYNC] UpdateBlockAndStatus: Rolling back current block!") bc.Rollback([]common.Hash{bc.CurrentBlock().Hash()}) return err } @@ -584,7 +592,10 @@ func (ss *StateSync) UpdateBlockAndStatus(block *types.Block, bc *core.BlockChai Str("blockHex", block.Hash().Hex()). Msg("[SYNC] UpdateBlockAndStatus: new block added to blockchain") for i, tx := range block.StakingTransactions() { - utils.Logger().Error().Msgf("StakingTxn %d: %s, %v", i, tx.StakingType().String(), tx.StakingMessage()) + utils.Logger().Info(). + Msgf( + "StakingTxn %d: %s, %v", i, tx.StakingType().String(), tx.StakingMessage(), + ) } return nil } diff --git a/block/header.go b/block/header.go index af72a9ba4..71946bbc4 100644 --- a/block/header.go +++ b/block/header.go @@ -1,6 +1,7 @@ package block import ( + "fmt" "io" "reflect" @@ -50,6 +51,14 @@ func (h *Header) Hash() ethcommon.Hash { return hash.FromRLP(h) } +// String .. +func (h *Header) String() string { + return fmt.Sprintf( + "[ShardID:%v Hash:%s Num:%v View:%v Epoch:%v]", + h.ShardID(), h.Hash().Hex(), h.Number(), h.ViewID(), h.Epoch(), + ) +} + // Logger returns a sub-logger with block contexts added. func (h *Header) Logger(logger *zerolog.Logger) *zerolog.Logger { nlogger := logger.With(). diff --git a/block/v1/header.go b/block/v1/header.go index adc362c12..ab153d4e1 100644 --- a/block/v1/header.go +++ b/block/v1/header.go @@ -367,7 +367,7 @@ func (h *Header) SetCrossLinks(newCrossLinks []byte) { // Slashes .. func (h *Header) Slashes() []byte { - h.Logger(utils.Logger()).Error(). + h.Logger(utils.Logger()).Info(). Msg("No slashes in V1 header") return nil } diff --git a/block/v2/header.go b/block/v2/header.go index 7c293db4b..7e2736ab5 100644 --- a/block/v2/header.go +++ b/block/v2/header.go @@ -366,7 +366,7 @@ func (h *Header) SetCrossLinks(newCrossLinks []byte) { // Slashes .. func (h *Header) Slashes() []byte { - h.Logger(utils.Logger()).Error(). + h.Logger(utils.Logger()).Info(). Msg("No slashes in V2 header") return nil } diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index 6f7e16e54..5b24e19ac 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -291,6 +291,17 @@ func createGlobalConfig() (*nodeconfig.ConfigType, error) { nodeConfig.DBDir = *dbDir + if p := *webHookYamlPath; p != "" { + config, err := slash.NewDoubleSignWebHooksFromPath(p) + if err != nil { + fmt.Fprintf( + os.Stderr, "yaml path is bad: %s", p, + ) + os.Exit(1) + } + nodeConfig.WebHooks.DoubleSigning = config + } + return nodeConfig, nil } @@ -338,18 +349,7 @@ func setupConsensusAndNode(nodeConfig *nodeconfig.ConfigType) *node.Node { // Current node. chainDBFactory := &shardchain.LDBFactory{RootDir: nodeConfig.DBDir} - currentNode := node.New(myHost, currentConsensus, chainDBFactory, blacklist, *isArchival) - - if p := *webHookYamlPath; p != "" { - config, err := slash.NewDoubleSignWebHooksFromPath(p) - if err != nil { - fmt.Fprintf( - os.Stderr, "ERROR provided yaml path %s but yaml found is illegal", p, - ) - os.Exit(1) - } - currentNode.NodeConfig.WebHooks.DoubleSigning = config - } + currentNode := node.New(myHost, currentConsensus, chainDBFactory, blacklist, true) switch { case *networkType == nodeconfig.Localnet: diff --git a/consensus/checks.go b/consensus/checks.go index 75b646b9a..27fe52d99 100644 --- a/consensus/checks.go +++ b/consensus/checks.go @@ -4,12 +4,13 @@ import ( msg_pb "github.com/harmony-one/harmony/api/proto/message" "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/internal/chain" + "github.com/harmony-one/harmony/shard" ) func (consensus *Consensus) validatorSanityChecks(msg *msg_pb.Message) bool { senderKey, err := consensus.verifySenderKey(msg) if err != nil { - if err == errValidNotInCommittee { + if err == shard.ErrValidNotInCommittee { consensus.getLogger().Info(). Msg("sender key not in this slot's subcommittee") } else { @@ -37,7 +38,7 @@ func (consensus *Consensus) validatorSanityChecks(msg *msg_pb.Message) bool { func (consensus *Consensus) leaderSanityChecks(msg *msg_pb.Message) bool { senderKey, err := consensus.verifySenderKey(msg) if err != nil { - if err == errValidNotInCommittee { + if err == shard.ErrValidNotInCommittee { consensus.getLogger().Info().Msg( "[OnAnnounce] sender key not in this slot's subcommittee", ) @@ -56,8 +57,7 @@ func (consensus *Consensus) leaderSanityChecks(msg *msg_pb.Message) bool { return true } -func (consensus *Consensus) onCommitSanityChecks( - recvMsg *FBFTMessage, +func (consensus *Consensus) isRightBlockNumAndViewID(recvMsg *FBFTMessage, ) bool { if recvMsg.ViewID != consensus.viewID || recvMsg.BlockNum != consensus.blockNum { consensus.getLogger().Debug(). @@ -68,24 +68,23 @@ func (consensus *Consensus) onCommitSanityChecks( Msg("[OnCommit] BlockNum/viewID not match") return false } + return true +} - if !consensus.FBFTLog.HasMatchingAnnounce(consensus.blockNum, recvMsg.BlockHash) { - consensus.getLogger().Debug(). - Hex("MsgBlockHash", recvMsg.BlockHash[:]). - Uint64("MsgBlockNum", recvMsg.BlockNum). - Uint64("blockNum", consensus.blockNum). - Msg("[OnCommit] Cannot find matching blockhash") - return false - } - - if !consensus.FBFTLog.HasMatchingPrepared(consensus.blockNum, recvMsg.BlockHash) { +func (consensus *Consensus) couldThisBeADoubleSigner( + recvMsg *FBFTMessage, +) bool { + num, hash, now := consensus.blockNum, recvMsg.BlockHash, consensus.blockNum + suspicious := !consensus.FBFTLog.HasMatchingAnnounce(num, hash) || + !consensus.FBFTLog.HasMatchingPrepared(num, hash) + if suspicious { consensus.getLogger().Debug(). - Hex("blockHash", recvMsg.BlockHash[:]). - Uint64("blockNum", consensus.blockNum). - Msg("[OnCommit] Cannot find matching prepared message") - return false + Str("message", recvMsg.String()). + Uint64("block-on-consensus", now). + Msg("possible double signer") + return true } - return true + return false } func (consensus *Consensus) onAnnounceSanityChecks(recvMsg *FBFTMessage) bool { diff --git a/consensus/consensus.go b/consensus/consensus.go index 510f38cef..8b734d924 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -188,6 +188,7 @@ func New( // FBFT related consensus.FBFTLog = NewFBFTLog() consensus.phase = FBFTAnnounce + // TODO Refactor consensus.block* into State? consensus.current = State{mode: Normal} // FBFT timeout consensus.consensusTimeout = createTimeout() @@ -214,7 +215,7 @@ func New( consensus.SlashChan = make(chan slash.Record) consensus.commitFinishChan = make(chan uint64) consensus.ReadySignal = make(chan struct{}) - consensus.lastBlockReward = big.NewInt(0) + consensus.lastBlockReward = common.Big0 // channel for receiving newly generated VDF consensus.RndChannel = make(chan [vdfAndSeedSize]byte) memprofiling.GetMemProfiling().Add("consensus.FBFTLog", consensus.FBFTLog) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 39fb80e77..c8a94ee4e 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -28,10 +28,6 @@ import ( "github.com/rs/zerolog" ) -var ( - errValidNotInCommittee = errors.New("slot signer not this slot's subcommittee") -) - // WaitForNewRandomness listens to the RndChannel to receive new VDF randomness. func (consensus *Consensus) WaitForNewRandomness() { go func() { @@ -60,40 +56,28 @@ func (consensus *Consensus) GetNextRnd() ([vdFAndProofSize]byte, [32]byte, error return vdfBytes, seed, nil } -// Populates the common basic fields for all consensus message. -func (consensus *Consensus) populateMessageFields(request *msg_pb.ConsensusRequest) { - request.ViewId = consensus.viewID - request.BlockNum = consensus.blockNum - request.ShardId = consensus.ShardID - - // 32 byte block hash - request.BlockHash = consensus.blockHash[:] - - // sender address - request.SenderPubkey = consensus.PubKey.Serialize() - consensus.getLogger().Debug(). - Str("senderKey", consensus.PubKey.SerializeToHexStr()). - Msg("[populateMessageFields]") -} +var ( + empty = []byte{} +) // Signs the consensus message and returns the marshaled message. -func (consensus *Consensus) signAndMarshalConsensusMessage(message *msg_pb.Message) ([]byte, error) { - err := consensus.signConsensusMessage(message) - if err != nil { - return []byte{}, err +func (consensus *Consensus) signAndMarshalConsensusMessage( + message *msg_pb.Message, +) ([]byte, error) { + if err := consensus.signConsensusMessage(message); err != nil { + return empty, err } marshaledMessage, err := protobuf.Marshal(message) if err != nil { - return []byte{}, err + return empty, err } return marshaledMessage, nil } // GetNodeIDs returns Node IDs of all nodes in the same shard func (consensus *Consensus) GetNodeIDs() []libp2p_peer.ID { - nodes := make([]libp2p_peer.ID, 0) - nodes = append(nodes, consensus.host.GetID()) + nodes := []libp2p_peer.ID{consensus.host.GetID()} consensus.validators.Range(func(k, v interface{}) bool { if peer, ok := v.(p2p.Peer); ok { nodes = append(nodes, peer.PeerID) @@ -109,19 +93,17 @@ func (consensus *Consensus) GetViewID() uint64 { return consensus.viewID } -// DebugPrintPublicKeys print all the PublicKeys in string format in Consensus -func (consensus *Consensus) DebugPrintPublicKeys() { - keys := consensus.Decider.DumpParticipants() - utils.Logger().Debug().Strs("PublicKeys", keys).Int("count", len(keys)).Msgf("Debug Public Keys") -} - -// UpdatePublicKeys updates the PublicKeys for quorum on current subcommittee, protected by a mutex +// UpdatePublicKeys updates the PublicKeys for +// quorum on current subcommittee, protected by a mutex func (consensus *Consensus) UpdatePublicKeys(pubKeys []*bls.PublicKey) int64 { consensus.pubKeyLock.Lock() consensus.Decider.UpdateParticipants(pubKeys) utils.Logger().Info().Msg("My Committee updated") for i := range pubKeys { - utils.Logger().Info().Int("index", i).Str("BLSPubKey", pubKeys[i].SerializeToHexStr()).Msg("Member") + utils.Logger().Info(). + Int("index", i). + Str("BLSPubKey", pubKeys[i].SerializeToHexStr()). + Msg("Member") } consensus.LeaderPubKey = pubKeys[0] utils.Logger().Info(). @@ -161,7 +143,7 @@ func (consensus *Consensus) signConsensusMessage(message *msg_pb.Message) error // GetValidatorPeers returns list of validator peers. func (consensus *Consensus) GetValidatorPeers() []p2p.Peer { - validatorPeers := make([]p2p.Peer, 0) + validatorPeers := []p2p.Peer{} consensus.validators.Range(func(k, v interface{}) bool { if peer, ok := v.(p2p.Peer); ok { @@ -213,14 +195,23 @@ func (consensus *Consensus) ResetState() { // Returns a string representation of this consensus func (consensus *Consensus) String() string { - var duty string + duty := "" if consensus.IsLeader() { - duty = "LDR" // leader + duty = "leader" } else { - duty = "VLD" // validator - } - return fmt.Sprintf("[duty:%s, PubKey:%s, ShardID:%v]", - duty, consensus.PubKey.SerializeToHexStr(), consensus.ShardID) + duty = "validator" + } + + return fmt.Sprintf( + "[Duty:%s Pub:%s Header:%s Num:%d View:%d Shard:%d Epoch:%d]", + duty, + consensus.PubKey.SerializeToHexStr(), + hex.EncodeToString(consensus.blockHeader), + consensus.blockNum, + consensus.viewID, + consensus.ShardID, + consensus.epoch, + ) } // ToggleConsensusCheck flip the flag of whether ignore viewID check during consensus process @@ -266,7 +257,7 @@ func (consensus *Consensus) verifySenderKey(msg *msg_pb.Message) (*bls.PublicKey } if !consensus.IsValidatorInCommittee(senderKey) { - return nil, errValidNotInCommittee + return nil, shard.ErrValidNotInCommittee } return senderKey, nil } @@ -279,7 +270,7 @@ func (consensus *Consensus) verifyViewChangeSenderKey(msg *msg_pb.Message) (*bls } if !consensus.IsValidatorInCommittee(senderKey) { - return nil, errValidNotInCommittee + return nil, shard.ErrValidNotInCommittee } return senderKey, nil } @@ -356,7 +347,7 @@ func (consensus *Consensus) SetEpochNum(epoch uint64) { func (consensus *Consensus) ReadSignatureBitmapPayload( recvPayload []byte, offset int, ) (*bls.Sign, *bls_cosi.Mask, error) { - if offset+96 > len(recvPayload) { + if offset+shard.BLSSignatureSizeInBytes > len(recvPayload) { return nil, nil, errors.New("payload not have enough length") } sigAndBitmapPayload := recvPayload[offset:] diff --git a/consensus/consensus_service_test.go b/consensus/consensus_service_test.go index ae5042f5d..d3d77ddc0 100644 --- a/consensus/consensus_service_test.go +++ b/consensus/consensus_service_test.go @@ -37,8 +37,9 @@ func TestPopulateMessageFields(t *testing.T) { Consensus: &msg_pb.ConsensusRequest{}, }, } - consensusMsg := msg.GetConsensus() - consensus.populateMessageFields(consensusMsg) + consensusMsg := consensus.populateMessageFields( + msg.GetConsensus(), consensus.blockHash[:], + ) if consensusMsg.ViewId != 2 { t.Errorf("Consensus ID is not populated correctly") diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 3ab46933e..765ffb475 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -140,7 +140,11 @@ func (consensus *Consensus) finalizeCommits() { return } - consensus.ChainReader.WriteLastCommits(FBFTMsg.Payload) + if err := consensus.ChainReader.WriteLastCommits(FBFTMsg.Payload); err != nil { + consensus.getLogger().Err(err). + Msg("[FinalizeCommits] could not write last commits") + return + } // if leader success finalize the block, send committed message to validators if err := consensus.msgSender.SendWithRetry( @@ -193,7 +197,8 @@ func (consensus *Consensus) LastCommitSig() ([]byte, []byte, error) { return nil, nil, nil } lastCommits, err := consensus.ChainReader.ReadLastCommits() - if err != nil || len(lastCommits) < 96 { + if err != nil || + len(lastCommits) < shard.BLSSignatureSizeInBytes { msgs := consensus.FBFTLog.GetMessagesByTypeSeq(msg_pb.MessageType_COMMITTED, consensus.blockNum-1) if len(msgs) != 1 { consensus.getLogger().Error(). @@ -204,11 +209,11 @@ func (consensus *Consensus) LastCommitSig() ([]byte, []byte, error) { lastCommits = msgs[0].Payload } //#### Read payload data from committed msg - aggSig := make([]byte, 96) - bitmap := make([]byte, len(lastCommits)-96) + aggSig := make([]byte, shard.BLSSignatureSizeInBytes) + bitmap := make([]byte, len(lastCommits)-shard.BLSSignatureSizeInBytes) offset := 0 - copy(aggSig[:], lastCommits[offset:offset+96]) - offset += 96 + copy(aggSig[:], lastCommits[offset:offset+shard.BLSSignatureSizeInBytes]) + offset += shard.BLSSignatureSizeInBytes copy(bitmap[:], lastCommits[offset:]) //#### END Read payload data from committed msg return aggSig, bitmap, nil @@ -486,10 +491,14 @@ func (consensus *Consensus) Start( } // GenerateVrfAndProof generates new VRF/Proof from hash of previous block -func (consensus *Consensus) GenerateVrfAndProof(newBlock *types.Block, vrfBlockNumbers []uint64) []uint64 { +func (consensus *Consensus) GenerateVrfAndProof( + newBlock *types.Block, vrfBlockNumbers []uint64, +) []uint64 { sk := vrf_bls.NewVRFSigner(consensus.priKey) blockHash := [32]byte{} - previousHeader := consensus.ChainReader.GetHeaderByNumber(newBlock.NumberU64() - 1) + previousHeader := consensus.ChainReader.GetHeaderByNumber( + newBlock.NumberU64() - 1, + ) previousHash := previousHeader.Hash() copy(blockHash[:], previousHash[:]) diff --git a/consensus/construct.go b/consensus/construct.go index 866c83a0d..76d85d804 100644 --- a/consensus/construct.go +++ b/consensus/construct.go @@ -20,11 +20,23 @@ type NetworkMessage struct { OptionalAggregateSignature *bls.Sign } +// Populates the common basic fields for all consensus message. +func (consensus *Consensus) populateMessageFields( + request *msg_pb.ConsensusRequest, blockHash []byte, +) *msg_pb.ConsensusRequest { + request.ViewId = consensus.viewID + request.BlockNum = consensus.blockNum + request.ShardId = consensus.ShardID + // 32 byte block hash + request.BlockHash = blockHash + // sender address + request.SenderPubkey = consensus.PubKey.Serialize() + return request +} + // construct is the single creation point of messages intended for the wire. -// The trailing callback provides a hook for custom mutation so call can control -// extra nuance on the network message, it is called after the BFT specific logic func (consensus *Consensus) construct( - p msg_pb.MessageType, payloadForSignOverride []byte, + p msg_pb.MessageType, payloadForSign []byte, ) (*NetworkMessage, error) { message := &msg_pb.Message{ ServiceType: msg_pb.ServiceType_CONSENSUS, @@ -33,11 +45,14 @@ func (consensus *Consensus) construct( Consensus: &msg_pb.ConsensusRequest{}, }, } + var ( + consensusMsg *msg_pb.ConsensusRequest + aggSig *bls.Sign + ) - consensusMsg := message.GetConsensus() - consensus.populateMessageFields(consensusMsg) - - var aggSig *bls.Sign + consensusMsg = consensus.populateMessageFields( + message.GetConsensus(), consensus.blockHash[:], + ) // Do the signing, 96 byte of bls signature switch p { @@ -56,7 +71,7 @@ func (consensus *Consensus) construct( consensusMsg.Payload = s.Serialize() } case msg_pb.MessageType_COMMIT: - if s := consensus.priKey.SignHash(payloadForSignOverride); s != nil { + if s := consensus.priKey.SignHash(payloadForSign); s != nil { consensusMsg.Payload = s.Serialize() } case msg_pb.MessageType_COMMITTED: diff --git a/consensus/construct_test.go b/consensus/construct_test.go index d644fd0e1..5c9c22fe4 100644 --- a/consensus/construct_test.go +++ b/consensus/construct_test.go @@ -3,6 +3,7 @@ package consensus import ( "testing" + "github.com/ethereum/go-ethereum/common" msg_pb "github.com/harmony-one/harmony/api/proto/message" "github.com/harmony-one/harmony/consensus/quorum" "github.com/harmony-one/harmony/crypto/bls" @@ -57,10 +58,16 @@ func TestConstructPreparedMessage(test *testing.T) { message := "test string" consensus.Decider.SubmitVote( - quorum.Prepare, leaderPubKey, leaderPriKey.Sign(message), nil, + quorum.Prepare, + leaderPubKey, + leaderPriKey.Sign(message), + common.BytesToHash(consensus.blockHash[:]), ) consensus.Decider.SubmitVote( - quorum.Prepare, validatorPubKey, validatorPriKey.Sign(message), nil, + quorum.Prepare, + validatorPubKey, + validatorPriKey.Sign(message), + common.BytesToHash(consensus.blockHash[:]), ) // According to RJ these failures are benign. diff --git a/consensus/engine/consensus_engine.go b/consensus/engine/consensus_engine.go index acc3a4931..8bf5e1810 100644 --- a/consensus/engine/consensus_engine.go +++ b/consensus/engine/consensus_engine.go @@ -11,6 +11,7 @@ import ( "github.com/harmony-one/harmony/internal/params" "github.com/harmony-one/harmony/shard" "github.com/harmony-one/harmony/shard/committee" + "github.com/harmony-one/harmony/staking/slash" staking "github.com/harmony-one/harmony/staking/types" ) @@ -44,8 +45,8 @@ type ChainReader interface { // Thus, only should be used to read the shard state of the current chain. ReadShardState(epoch *big.Int) (*shard.State, error) - // ReadActiveValidatorList retrieves the list of active validators - ReadActiveValidatorList() ([]common.Address, error) + // ReadElectedValidatorList retrieves the list of elected validators + ReadElectedValidatorList() ([]common.Address, error) // ReadValidatorList retrieves the list of all validators ReadValidatorList() ([]common.Address, error) @@ -120,6 +121,7 @@ type Engine interface { state *state.DB, txs []*types.Transaction, receipts []*types.Receipt, outcxs []*types.CXReceipt, incxs []*types.CXReceiptsProof, stks []*staking.StakingTransaction, + doubleSigners slash.Records, ) (*types.Block, *big.Int, error) // Seal generates a new sealing request for the given input block and pushes diff --git a/consensus/fbft_log.go b/consensus/fbft_log.go index f8ca92a28..8e517436c 100644 --- a/consensus/fbft_log.go +++ b/consensus/fbft_log.go @@ -39,6 +39,19 @@ type FBFTMessage struct { M3Bitmap *bls_cosi.Mask } +// String .. +func (m *FBFTMessage) String() string { + return fmt.Sprintf( + "[Type:%s ViewID:%d Num:%d BlockHash:%s Sender:%s Leader:%s]", + m.MessageType.String(), + m.ViewID, + m.BlockNum, + m.BlockHash.Hex(), + m.SenderPubkey.SerializeToHexStr(), + m.LeaderPubkey.SerializeToHexStr(), + ) +} + // NewFBFTLog returns new instance of FBFTLog func NewFBFTLog() *FBFTLog { blocks := mapset.NewSet() @@ -220,6 +233,7 @@ func (log *FBFTLog) FindMessageByMaxViewID(msgs []*FBFTMessage) *FBFTMessage { // ParseFBFTMessage parses FBFT message into FBFTMessage structure func ParseFBFTMessage(msg *msg_pb.Message) (*FBFTMessage, error) { + // TODO Have this do sanity checks on the message please pbftMsg := FBFTMessage{} pbftMsg.MessageType = msg.GetType() consensusMsg := msg.GetConsensus() diff --git a/consensus/fbft_log_test.go b/consensus/fbft_log_test.go index 872d36f2d..e1a8269ff 100644 --- a/consensus/fbft_log_test.go +++ b/consensus/fbft_log_test.go @@ -33,31 +33,44 @@ func constructAnnounceMessage(t *testing.T) (*NetworkMessage, error) { func getConsensusMessage(payload []byte) (*msg_pb.Message, error) { msg := &msg_pb.Message{} - err := protobuf.Unmarshal(payload, msg) - if err != nil { + if err := protobuf.Unmarshal(payload, msg); err != nil { return nil, err } return msg, nil } func TestGetMessagesByTypeSeqViewHash(t *testing.T) { - pbftMsg := FBFTMessage{MessageType: msg_pb.MessageType_ANNOUNCE, BlockNum: 2, ViewID: 3, BlockHash: [32]byte{01, 02}} + pbftMsg := FBFTMessage{ + MessageType: msg_pb.MessageType_ANNOUNCE, + BlockNum: 2, + ViewID: 3, + BlockHash: [32]byte{01, 02}, + } log := NewFBFTLog() log.AddMessage(&pbftMsg) - found := log.GetMessagesByTypeSeqViewHash(msg_pb.MessageType_ANNOUNCE, 2, 3, [32]byte{01, 02}) + found := log.GetMessagesByTypeSeqViewHash( + msg_pb.MessageType_ANNOUNCE, 2, 3, [32]byte{01, 02}, + ) if len(found) != 1 { t.Error("cannot find existing message") } - notFound := log.GetMessagesByTypeSeqViewHash(msg_pb.MessageType_ANNOUNCE, 2, 3, [32]byte{01, 03}) + notFound := log.GetMessagesByTypeSeqViewHash( + msg_pb.MessageType_ANNOUNCE, 2, 3, [32]byte{01, 03}, + ) if len(notFound) > 0 { t.Error("find message that not exist") } } func TestHasMatchingAnnounce(t *testing.T) { - pbftMsg := FBFTMessage{MessageType: msg_pb.MessageType_ANNOUNCE, BlockNum: 2, ViewID: 3, BlockHash: [32]byte{01, 02}} + pbftMsg := FBFTMessage{ + MessageType: msg_pb.MessageType_ANNOUNCE, + BlockNum: 2, + ViewID: 3, + BlockHash: [32]byte{01, 02}, + } log := NewFBFTLog() log.AddMessage(&pbftMsg) found := log.HasMatchingViewAnnounce(2, 3, [32]byte{01, 02}) diff --git a/consensus/leader.go b/consensus/leader.go index b3e502d14..9e73bf995 100644 --- a/consensus/leader.go +++ b/consensus/leader.go @@ -1,22 +1,27 @@ package consensus import ( + "bytes" "encoding/binary" + "math/big" "time" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/rlp" "github.com/harmony-one/bls/ffi/go/bls" msg_pb "github.com/harmony-one/harmony/api/proto/message" "github.com/harmony-one/harmony/consensus/quorum" + "github.com/harmony-one/harmony/consensus/votepower" "github.com/harmony-one/harmony/core/types" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" "github.com/harmony-one/harmony/p2p/host" + "github.com/harmony-one/harmony/shard" + "github.com/harmony-one/harmony/staking/slash" ) func (consensus *Consensus) announce(block *types.Block) { blockHash := block.Hash() copy(consensus.blockHash[:], blockHash[:]) - // prepare message and broadcast to validators encodedBlock, err := rlp.EncodeToBytes(block) if err != nil { @@ -31,14 +36,14 @@ func (consensus *Consensus) announce(block *types.Block) { consensus.block = encodedBlock consensus.blockHeader = encodedBlockHeader - network, err := consensus.construct(msg_pb.MessageType_ANNOUNCE, nil) + networkMessage, err := consensus.construct(msg_pb.MessageType_ANNOUNCE, nil) if err != nil { consensus.getLogger().Err(err). Str("message-type", msg_pb.MessageType_ANNOUNCE.String()). Msg("failed constructing message") return } - msgToSend, FPBTMsg := network.Bytes, network.FBFTMsg + msgToSend, FPBTMsg := networkMessage.Bytes, networkMessage.FBFTMsg // TODO(chao): review FPBT log data structure consensus.FBFTLog.AddMessage(FPBTMsg) @@ -54,9 +59,11 @@ func (consensus *Consensus) announce(block *types.Block) { quorum.Prepare, consensus.PubKey, consensus.priKey.SignHash(consensus.blockHash[:]), - nil, + common.BytesToHash(consensus.blockHash[:]), ) - if err := consensus.prepareBitmap.SetKey(consensus.PubKey, true); err != nil { + if err := consensus.prepareBitmap.SetKey( + consensus.PubKey, true, + ); err != nil { consensus.getLogger().Warn().Err(err).Msg( "[Announce] Leader prepareBitmap SetKey failed", ) @@ -155,7 +162,7 @@ func (consensus *Consensus) onPrepare(msg *msg_pb.Message) { Int64("PublicKeys", consensus.Decider.ParticipantsCount()).Logger() logger.Info().Msg("[OnPrepare] Received New Prepare Signature") consensus.Decider.SubmitVote( - quorum.Prepare, validatorPubKey, &sign, nil, + quorum.Prepare, validatorPubKey, &sign, recvMsg.BlockHash, ) // Set the bitmap indicating that this validator signed. if err := prepareBitmap.SetKey(recvMsg.SenderPubkey, true); err != nil { @@ -164,6 +171,7 @@ func (consensus *Consensus) onPrepare(msg *msg_pb.Message) { } if consensus.Decider.IsQuorumAchieved(quorum.Prepare) { + // NOTE Let it handle its own logs if err := consensus.didReachPrepareQuorum(); err != nil { return } @@ -173,73 +181,107 @@ func (consensus *Consensus) onPrepare(msg *msg_pb.Message) { func (consensus *Consensus) onCommit(msg *msg_pb.Message) { recvMsg, err := ParseFBFTMessage(msg) - + log := consensus.getLogger() if err != nil { consensus.getLogger().Debug().Err(err).Msg("[OnCommit] Parse pbft message failed") return } - // let it handle its own logs - // TODO(Edgar)(TEMP DISABLE while testing double sign) - if !consensus.onCommitSanityChecks(recvMsg) { + // NOTE let it handle its own log + if !consensus.isRightBlockNumAndViewID(recvMsg) { return } consensus.mutex.Lock() defer consensus.mutex.Unlock() + if key := (bls.PublicKey{}); consensus.couldThisBeADoubleSigner(recvMsg) { + if alreadyCastBallot := consensus.Decider.ReadBallot( + quorum.Commit, recvMsg.SenderPubkey, + ); alreadyCastBallot != nil { + for _, blk := range consensus.FBFTLog.GetBlocksByNumber(recvMsg.BlockNum) { + alreadyCastBallot.SignerPubKey.ToLibBLSPublicKey(&key) + if recvMsg.SenderPubkey.IsEqual(&key) { + signed := blk.Header() + areHeightsEqual := signed.Number().Uint64() == recvMsg.BlockNum + areViewIDsEqual := signed.ViewID().Uint64() == recvMsg.ViewID + areHeadersEqual := bytes.Compare( + signed.Hash().Bytes(), recvMsg.BlockHash.Bytes(), + ) == 0 + // If signer already signed, and the block height is the same + // and the viewID is the same, then we need to verify the block + // hash, and if block hash is different, then that is a clear + // case of double signing + if areHeightsEqual && areViewIDsEqual && !areHeadersEqual { + var doubleSign bls.Sign + if err := doubleSign.Deserialize(recvMsg.Payload); err != nil { + log.Err(err).Str("msg", recvMsg.String()). + Msg("could not deserialize potential double signer") + return + } + + curHeader := consensus.ChainReader.CurrentHeader() + committee, err := consensus.ChainReader.ReadShardState(curHeader.Epoch()) + if err != nil { + log.Err(err). + Uint32("shard", consensus.ShardID). + Uint64("epoch", curHeader.Epoch().Uint64()). + Msg("could not read shard state") + return + } + offender := *shard.FromLibBLSPublicKeyUnsafe(recvMsg.SenderPubkey) + addr, err := committee.FindCommitteeByID( + consensus.ShardID, + ).AddressForBLSKey(offender) + + if err != nil { + log.Err(err).Str("msg", recvMsg.String()). + Msg("could not find address for bls key") + return + } + + now := big.NewInt(time.Now().UnixNano()) + + go func(reporter common.Address) { + evid := slash.Evidence{ + ConflictingBallots: slash.ConflictingBallots{ + *alreadyCastBallot, + votepower.Ballot{ + offender, + recvMsg.BlockHash, + common.Hex2Bytes(doubleSign.SerializeToHexStr()), + }}, + Moment: slash.Moment{ + // TODO need to extend fbft tro have epoch to use its epoch + // rather than curHeader epoch + Epoch: curHeader.Epoch(), + Height: new(big.Int).SetUint64(recvMsg.BlockNum), + ViewID: consensus.viewID, + ShardID: consensus.ShardID, + TimeUnixNano: now, + }, + ProposalHeader: signed, + } + proof := slash.Record{ + Evidence: evid, + Reporter: reporter, + Offender: *addr, + } + consensus.SlashChan <- proof + }(consensus.SelfAddress) + return + } + } + } + } + return + } + validatorPubKey, commitSig, commitBitmap := recvMsg.SenderPubkey, recvMsg.Payload, consensus.commitBitmap logger := consensus.getLogger().With(). Str("validatorPubKey", validatorPubKey.SerializeToHexStr()).Logger() - if alreadyCastBallot := consensus.Decider.ReadBallot( - quorum.Commit, validatorPubKey, - ); alreadyCastBallot != nil { - logger.Debug().Msg("voter already cast commit message") - return - - // TODO(Edgar) Still working out double sign - // var signed, received *types.Block - // if err := rlp.DecodeBytes( - // alreadyCastBallot.OptSerializedBlock, &signed, - // ); err != nil { - // // TODO handle - // } - - // areHeightsEqual := signed.Header().Number().Uint64() == recvMsg.BlockNum - // areViewIDsEqual := signed.Header().ViewID().Uint64() == recvMsg.ViewID - // areHeadersEqual := bytes.Compare( - // signed.Hash().Bytes(), recvMsg.BlockHash.Bytes(), - // ) == 0 - - // // If signer already signed, and the block height is the same, then we - // // need to verify the block hash, and if block hash is different, then - // // that is a clear case of double signing - // if areHeightsEqual && areViewIDsEqual && !areHeadersEqual { - // // TODO - // if err := rlp.DecodeBytes(recvMsg.Block, &received); err != nil { - // // TODO Some log - // return - // } - - // var doubleSign bls.Sign - // if err := doubleSign.Deserialize(recvMsg.Payload); err != nil { - // return - // } - - // go func() { - // consensus.SlashChan <- slash.NewRecord( - // *shard.FromLibBLSPublicKeyUnsafe(validatorPubKey), - // signed.Header(), received.Header(), - // alreadyCastBallot.Signature, &doubleSign, - // consensus.SelfAddress, - // ) - // }() - // } - // return - } - // has to be called before verifying signature quorumWasMet := consensus.Decider.IsQuorumAchieved(quorum.Commit) // Verify the signature on commitPayload is correct @@ -268,11 +310,12 @@ func (consensus *Consensus) onCommit(msg *msg_pb.Message) { logger.Info().Msg("[OnCommit] Received new commit message") consensus.Decider.SubmitVote( - quorum.Commit, validatorPubKey, &sign, consensus.block, + quorum.Commit, validatorPubKey, &sign, recvMsg.BlockHash, ) // Set the bitmap indicating that this validator signed. if err := commitBitmap.SetKey(recvMsg.SenderPubkey, true); err != nil { - consensus.getLogger().Warn().Err(err).Msg("[OnCommit] commitBitmap.SetKey failed") + consensus.getLogger().Warn().Err(err). + Msg("[OnCommit] commitBitmap.SetKey failed") return } diff --git a/consensus/quorum/one-node-one-vote.go b/consensus/quorum/one-node-one-vote.go index c0dff099d..63c0b2c27 100644 --- a/consensus/quorum/one-node-one-vote.go +++ b/consensus/quorum/one-node-one-vote.go @@ -35,7 +35,7 @@ func (v *uniformVoteWeight) IsQuorumAchieved(p Phase) bool { } // IsQuorumAchivedByMask .. -func (v *uniformVoteWeight) IsQuorumAchievedByMask(mask *bls_cosi.Mask, debug bool) bool { +func (v *uniformVoteWeight) IsQuorumAchievedByMask(mask *bls_cosi.Mask) bool { threshold := v.TwoThirdsSignersCount() currentTotalPower := utils.CountOneBits(mask.Bitmap) if currentTotalPower < threshold { diff --git a/consensus/quorum/one-node-staked-vote.go b/consensus/quorum/one-node-staked-vote.go index 64762b59f..eae22386f 100644 --- a/consensus/quorum/one-node-staked-vote.go +++ b/consensus/quorum/one-node-staked-vote.go @@ -6,6 +6,7 @@ import ( "github.com/harmony-one/bls/ffi/go/bls" "github.com/harmony-one/harmony/consensus/votepower" bls_cosi "github.com/harmony-one/harmony/crypto/bls" + common2 "github.com/harmony-one/harmony/internal/common" "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/numeric" "github.com/harmony-one/harmony/shard" @@ -71,23 +72,12 @@ func (v *stakedVoteWeight) IsQuorumAchieved(p Phase) bool { } // IsQuorumAchivedByMask .. -func (v *stakedVoteWeight) IsQuorumAchievedByMask(mask *bls_cosi.Mask, debug bool) bool { +func (v *stakedVoteWeight) IsQuorumAchievedByMask(mask *bls_cosi.Mask) bool { threshold := v.QuorumThreshold() currentTotalPower := v.computeTotalPowerByMask(mask) if currentTotalPower == nil { - if debug { // temp for remove debug info on crosslink verification - utils.Logger().Warn(). - Msgf("[IsQuorumAchievedByMask] currentTotalPower is nil") - } return false } - if debug { - utils.Logger().Info(). - Str("policy", v.Policy().String()). - Str("threshold", threshold.String()). - Str("total-power-of-signers", currentTotalPower.String()). - Msg("[IsQuorumAchievedByMask] Checking quorum") - } return (*currentTotalPower).GT(threshold) } func (v *stakedVoteWeight) computeCurrentTotalPower(p Phase) (*numeric.Dec, error) { @@ -203,6 +193,7 @@ func (v *stakedVoteWeight) MarshalJSON() ([]byte, error) { voterCount := len(v.roster.Voters) type u struct { IsHarmony bool `json:"is-harmony-slot"` + EarningAccount string `json:"earning-account"` Identity string `json:"bls-public-key"` VotingPower string `json:"voting-power-%"` EffectiveStake string `json:"effective-stake,omitempty"` @@ -224,6 +215,7 @@ func (v *stakedVoteWeight) MarshalJSON() ([]byte, error) { for identity, voter := range v.roster.Voters { member := u{ voter.IsHarmonyNode, + common2.MustAddressToBech32(voter.EarningAccount), identity.Hex(), voter.EffectivePercent.String(), "", diff --git a/consensus/quorum/one-node-staked-vote_test.go b/consensus/quorum/one-node-staked-vote_test.go index e3937853b..5a8b5f685 100644 --- a/consensus/quorum/one-node-staked-vote_test.go +++ b/consensus/quorum/one-node-staked-vote_test.go @@ -104,7 +104,7 @@ func sign(d Decider, k secretKeyMap, p Phase) { pubKey := v.GetPublicKey() sig := v.Sign(msg) // TODO Make upstream test provide meaningful test values - d.SubmitVote(p, pubKey, sig, nil) + d.SubmitVote(p, pubKey, sig, common.Hash{}) } } diff --git a/consensus/quorum/quorum.go b/consensus/quorum/quorum.go index 23c6792b5..18b611b37 100644 --- a/consensus/quorum/quorum.go +++ b/consensus/quorum/quorum.go @@ -3,12 +3,14 @@ package quorum import ( "fmt" + "github.com/ethereum/go-ethereum/common" "github.com/harmony-one/bls/ffi/go/bls" "github.com/harmony-one/harmony/consensus/votepower" bls_cosi "github.com/harmony-one/harmony/crypto/bls" "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/numeric" "github.com/harmony-one/harmony/shard" + "github.com/pkg/errors" ) // Phase is a phase that needs quorum to proceed @@ -73,9 +75,8 @@ type ParticipantTracker interface { type SignatoryTracker interface { ParticipantTracker SubmitVote( - p Phase, PubKey *bls.PublicKey, - sig *bls.Sign, optSerializedBlock []byte, - ) + p Phase, PubKey *bls.PublicKey, sig *bls.Sign, headerHash common.Hash, + ) *votepower.Ballot // Caller assumes concurrency protection SignersCount(Phase) int64 reset([]Phase) @@ -112,7 +113,7 @@ type Decider interface { SetVoters(shard.SlotList) (*TallyResult, error) Policy() Policy IsQuorumAchieved(Phase) bool - IsQuorumAchievedByMask(mask *bls_cosi.Mask, debug bool) bool + IsQuorumAchievedByMask(mask *bls_cosi.Mask) bool QuorumThreshold() numeric.Dec AmIMemberOfCommitee() bool IsRewardThresholdAchieved() bool @@ -157,7 +158,9 @@ func (s *cIdentities) AggregateVotes(p Phase) *bls.Sign { ballots := s.ReadAllBallots(p) sigs := make([]*bls.Sign, 0, len(ballots)) for _, ballot := range ballots { - sigs = append(sigs, ballot.Signature) + sig := &bls.Sign{} + sig.DeserializeHexStr(common.Bytes2Hex(ballot.Signature)) + sigs = append(sigs, sig) } return bls_cosi.AggregateSig(sigs) } @@ -221,20 +224,13 @@ func (s *cIdentities) SignersCount(p Phase) int64 { } func (s *cIdentities) SubmitVote( - p Phase, PubKey *bls.PublicKey, - sig *bls.Sign, optSerializedBlock []byte, -) { - if p != Commit && optSerializedBlock != nil && len(optSerializedBlock) != 0 { - utils.Logger().Debug().Str("phase", p.String()). - Msg("non-commit phase has non-nil, non-empty block") - } - + p Phase, PubKey *bls.PublicKey, sig *bls.Sign, headerHash common.Hash, +) *votepower.Ballot { ballot := &votepower.Ballot{ - SignerPubKey: *shard.FromLibBLSPublicKeyUnsafe(PubKey), - Signature: sig, - OptSerializedBlock: optSerializedBlock[:], + SignerPubKey: *shard.FromLibBLSPublicKeyUnsafe(PubKey), + BlockHeaderHash: headerHash, + Signature: common.Hex2Bytes(sig.SerializeToHexStr()), } - switch hex := PubKey.SerializeToHexStr(); p { case Prepare: s.prepare.BallotBox[hex] = ballot @@ -242,7 +238,12 @@ func (s *cIdentities) SubmitVote( s.commit.BallotBox[hex] = ballot case ViewChange: s.viewChange.BallotBox[hex] = ballot + default: + utils.Logger().Err(errors.New("invariant of known phase violated")). + Str("phase", p.String()). + Msg("bad vote input") } + return ballot } func (s *cIdentities) reset(ps []Phase) { diff --git a/consensus/threshold.go b/consensus/threshold.go index 0249d8172..f44d57d5b 100644 --- a/consensus/threshold.go +++ b/consensus/threshold.go @@ -3,6 +3,7 @@ package consensus import ( "encoding/binary" + "github.com/ethereum/go-ethereum/common" msg_pb "github.com/harmony-one/harmony/api/proto/message" "github.com/harmony-one/harmony/consensus/quorum" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" @@ -14,7 +15,7 @@ func (consensus *Consensus) didReachPrepareQuorum() error { logger := utils.Logger() logger.Debug().Msg("[OnPrepare] Received Enough Prepare Signatures") // Construct and broadcast prepared message - network, err := consensus.construct(msg_pb.MessageType_PREPARED, nil) + networkMessage, err := consensus.construct(msg_pb.MessageType_PREPARED, nil) if err != nil { consensus.getLogger().Err(err). Str("message-type", msg_pb.MessageType_PREPARED.String()). @@ -22,25 +23,21 @@ func (consensus *Consensus) didReachPrepareQuorum() error { return err } msgToSend, FBFTMsg, aggSig := - network.Bytes, - network.FBFTMsg, - network.OptionalAggregateSignature + networkMessage.Bytes, + networkMessage.FBFTMsg, + networkMessage.OptionalAggregateSignature consensus.aggregatedPrepareSig = aggSig consensus.FBFTLog.AddMessage(FBFTMsg) // Leader add commit phase signature - blockNumHash := make([]byte, 8) - binary.LittleEndian.PutUint64(blockNumHash, consensus.blockNum) - commitPayload := append(blockNumHash, consensus.blockHash[:]...) - - // so by this point, everyone has committed to the blockhash of this block - // in prepare and so this is the actual block. - + blockNumHash := [8]byte{} + binary.LittleEndian.PutUint64(blockNumHash[:], consensus.blockNum) + commitPayload := append(blockNumHash[:], consensus.blockHash[:]...) consensus.Decider.SubmitVote( quorum.Commit, consensus.PubKey, consensus.priKey.SignHash(commitPayload), - consensus.block[:], + common.BytesToHash(consensus.blockHash[:]), ) if err := consensus.commitBitmap.SetKey(consensus.PubKey, true); err != nil { diff --git a/consensus/validator.go b/consensus/validator.go index 59a666643..a0ac0acad 100644 --- a/consensus/validator.go +++ b/consensus/validator.go @@ -113,7 +113,7 @@ func (consensus *Consensus) onPrepared(msg *msg_pb.Message) { return } - if !consensus.Decider.IsQuorumAchievedByMask(mask, true) { + if !consensus.Decider.IsQuorumAchievedByMask(mask) { consensus.getLogger().Warn(). Msgf("[OnPrepared] Quorum Not achieved") return @@ -193,7 +193,6 @@ func (consensus *Consensus) onPrepared(msg *msg_pb.Message) { blockNumBytes := make([]byte, 8) binary.LittleEndian.PutUint64(blockNumBytes, consensus.blockNum) networkMessage, _ := consensus.construct( - // TODO: should only sign on block hash msg_pb.MessageType_COMMIT, append(blockNumBytes, consensus.blockHash[:]...), ) @@ -229,12 +228,8 @@ func (consensus *Consensus) onCommitted(msg *msg_pb.Message) { consensus.getLogger().Warn().Msg("[OnCommitted] unable to parse msg") return } - - if recvMsg.BlockNum < consensus.blockNum { - consensus.getLogger().Info(). - Uint64("MsgBlockNum", recvMsg.BlockNum). - Uint64("blockNum", consensus.blockNum). - Msg("[OnCommitted] Received Old Blocks!!") + // NOTE let it handle its own logs + if !consensus.isRightBlockNumCheck(recvMsg) { return } @@ -244,7 +239,7 @@ func (consensus *Consensus) onCommitted(msg *msg_pb.Message) { return } - if !consensus.Decider.IsQuorumAchievedByMask(mask, true) { + if !consensus.Decider.IsQuorumAchievedByMask(mask) { consensus.getLogger().Warn(). Msgf("[OnCommitted] Quorum Not achieved") return diff --git a/consensus/view_change.go b/consensus/view_change.go index b884c17e1..1faf6793a 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -280,7 +280,7 @@ func (consensus *Consensus) onViewChange(msg *msg_pb.Message) { return } - if !consensus.Decider.IsQuorumAchievedByMask(mask, true) { + if !consensus.Decider.IsQuorumAchievedByMask(mask) { consensus.getLogger().Warn(). Msgf("[onViewChange] Quorum Not achieved") return @@ -347,7 +347,7 @@ func (consensus *Consensus) onViewChange(msg *msg_pb.Message) { Msg("[onViewChange]") // received enough view change messages, change state to normal consensus - if consensus.Decider.IsQuorumAchievedByMask(consensus.viewIDBitmap[recvMsg.ViewID], true) { + if consensus.Decider.IsQuorumAchievedByMask(consensus.viewIDBitmap[recvMsg.ViewID]) { consensus.current.SetMode(Normal) consensus.LeaderPubKey = consensus.PubKey consensus.ResetState() @@ -374,14 +374,14 @@ func (consensus *Consensus) onViewChange(msg *msg_pb.Message) { consensus.aggregatedPrepareSig = aggSig consensus.prepareBitmap = mask // Leader sign and add commit message - blockNumBytes := make([]byte, 8) - binary.LittleEndian.PutUint64(blockNumBytes, consensus.blockNum) - commitPayload := append(blockNumBytes, consensus.blockHash[:]...) + blockNumBytes := [8]byte{} + binary.LittleEndian.PutUint64(blockNumBytes[:], consensus.blockNum) + commitPayload := append(blockNumBytes[:], consensus.blockHash[:]...) consensus.Decider.SubmitVote( quorum.Commit, consensus.PubKey, consensus.priKey.SignHash(commitPayload), - nil, + common.BytesToHash(consensus.blockHash[:]), ) if err = consensus.commitBitmap.SetKey(consensus.PubKey, true); err != nil { @@ -398,7 +398,16 @@ func (consensus *Consensus) onViewChange(msg *msg_pb.Message) { Int("payloadSize", len(consensus.m1Payload)). Hex("M1Payload", consensus.m1Payload). Msg("[onViewChange] Sent NewView Message") - consensus.msgSender.SendWithRetry(consensus.blockNum, msg_pb.MessageType_NEWVIEW, []nodeconfig.GroupID{nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(consensus.ShardID))}, host.ConstructP2pMessage(byte(17), msgToSend)) + if err := consensus.msgSender.SendWithRetry( + consensus.blockNum, + msg_pb.MessageType_NEWVIEW, + []nodeconfig.GroupID{ + nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(consensus.ShardID))}, + host.ConstructP2pMessage(byte(17), msgToSend), + ); err != nil { + consensus.getLogger().Err(err). + Msg("could not send out the NEWVIEW message") + } consensus.viewID = recvMsg.ViewID consensus.ResetViewChangeState() @@ -446,7 +455,7 @@ func (consensus *Consensus) onNewView(msg *msg_pb.Message) { viewIDBytes := make([]byte, 8) binary.LittleEndian.PutUint64(viewIDBytes, recvMsg.ViewID) - if !consensus.Decider.IsQuorumAchievedByMask(m3Mask, true) { + if !consensus.Decider.IsQuorumAchievedByMask(m3Mask) { consensus.getLogger().Warn(). Msgf("[onNewView] Quorum Not achieved") return diff --git a/consensus/votepower/roster.go b/consensus/votepower/roster.go index 411d1e374..b2a22bd4b 100644 --- a/consensus/votepower/roster.go +++ b/consensus/votepower/roster.go @@ -7,7 +7,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/harmony-one/bls/ffi/go/bls" - "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/numeric" "github.com/harmony-one/harmony/shard" staking "github.com/harmony-one/harmony/staking/types" @@ -25,20 +24,22 @@ var ( // Ballot is a vote cast by a validator type Ballot struct { - SignerPubKey shard.BlsPublicKey `json:"bls-public-key"` - Signature *bls.Sign `json:"signature"` - OptSerializedBlock []byte `json:"opt-rlp-encoded-block"` + SignerPubKey shard.BlsPublicKey `json:"bls-public-key"` + BlockHeaderHash common.Hash `json:"block-header-hash"` + Signature []byte `json:"bls-signature"` } -// BallotResults are a completed round of votes -type BallotResults struct { - Signature shard.BLSSignature // (aggregated) signature - Bitmap []byte // corresponding bitmap mask for agg signature -} - -// EncodePair returns hex encoded tuple (signature, bitmap) -func (b BallotResults) EncodePair() (string, string) { - return hex.EncodeToString(b.Signature[:]), hex.EncodeToString(b.Bitmap[:]) +// MarshalJSON .. +func (b Ballot) MarshalJSON() ([]byte, error) { + return json.Marshal(struct { + A string `json:"bls-public-key"` + B string `json:"block-header-hash"` + C string `json:"bls-signature"` + }{ + b.SignerPubKey.Hex(), + b.BlockHeaderHash.Hex(), + hex.EncodeToString(b.Signature), + }) } // Round is a round of voting in any FBFT phase @@ -47,6 +48,11 @@ type Round struct { BallotBox map[string]*Ballot } +func (b Ballot) String() string { + data, _ := json.Marshal(b) + return string(data) +} + // NewRound .. func NewRound() *Round { return &Round{AggregatedVote: nil, BallotBox: map[string]*Ballot{}} @@ -195,23 +201,12 @@ func Compute(staked shard.SlotList) (*Roster, error) { if diff := numeric.OneDec().Sub( ourPercentage.Add(theirPercentage), ); !diff.IsZero() && lastStakedVoter != nil { - utils.Logger().Info(). - Str("theirs", theirPercentage.String()). - Str("ours", ourPercentage.String()). - Str("diff", diff.String()). - Str("combined", theirPercentage.Add(diff).Add(ourPercentage).String()). - Str("bls-public-key-of-receipent", lastStakedVoter.Identity.Hex()). - Msg("voting power of hmy & staked slots not sum to 1, giving diff to staked slot") lastStakedVoter.EffectivePercent = lastStakedVoter.EffectivePercent.Add(diff) theirPercentage = theirPercentage.Add(diff) } if lastStakedVoter != nil && !ourPercentage.Add(theirPercentage).Equal(numeric.OneDec()) { - utils.Logger().Error(). - Str("theirs", theirPercentage.String()). - Str("ours", ourPercentage.String()). - Msg("Total voting power not equal 100 percent") return nil, ErrVotingPowerNotEqualOne } diff --git a/core/blockchain.go b/core/blockchain.go index 32a923790..22a7f85db 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -19,7 +19,6 @@ package core import ( "bytes" - "errors" "fmt" "io" "math/big" @@ -43,7 +42,6 @@ import ( "github.com/harmony-one/harmony/core/state" "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/core/vm" - common2 "github.com/harmony-one/harmony/internal/common" "github.com/harmony-one/harmony/internal/ctxerror" "github.com/harmony-one/harmony/internal/params" "github.com/harmony-one/harmony/internal/utils" @@ -53,6 +51,7 @@ import ( "github.com/harmony-one/harmony/staking/slash" staking "github.com/harmony-one/harmony/staking/types" lru "github.com/hashicorp/golang-lru" + "github.com/pkg/errors" ) var ( @@ -259,13 +258,17 @@ func (bc *BlockChain) ValidateNewBlock(block *types.Block) error { // NOTE Order of mutating state here matters. // Process block using the parent state as reference point. - receipts, cxReceipts, _, usedGas, _, err := bc.processor.Process(block, state, bc.vmConfig) + receipts, cxReceipts, _, usedGas, _, err := bc.processor.Process( + block, state, bc.vmConfig, + ) if err != nil { bc.reportBlock(block, receipts, err) return err } - if err := bc.Validator().ValidateState(block, state, receipts, cxReceipts, usedGas); err != nil { + if err := bc.Validator().ValidateState( + block, state, receipts, cxReceipts, usedGas, + ); err != nil { bc.reportBlock(block, receipts, err) return err } @@ -1863,38 +1866,29 @@ func (bc *BlockChain) ReadCrossLink(shardID uint32, blockNum uint64) (*types.Cro // This function will update the latest crosslink in the sense that // any previous block's crosslink is received up to this point // there is no missing hole between genesis to this crosslink of given shardID -func (bc *BlockChain) LastContinuousCrossLink(batch rawdb.DatabaseWriter, cl types.CrossLink) error { - if !bc.Config().IsCrossLink(cl.Epoch()) { +func (bc *BlockChain) LastContinuousCrossLink(batch rawdb.DatabaseWriter, shardID uint32) error { + if !bc.Config().IsCrossLink(bc.CurrentBlock().Epoch()) { return errors.New("Trying to write last continuous cross link with epoch before cross link starting epoch") } - cl0, err := bc.ReadShardLastCrossLink(cl.ShardID()) - if cl0 == nil { - rawdb.WriteShardLastCrossLink(batch, cl.ShardID(), cl.Serialize()) - return nil - } - if err != nil { + oldLink, err := bc.ReadShardLastCrossLink(shardID) + if oldLink == nil || err != nil { return err } - newCheckpoint := uint64(0) - for i := cl0.BlockNum() + 1; i < cl.BlockNum(); i++ { - tmp, err := bc.ReadCrossLink(cl.ShardID(), i) - if err != nil || tmp == nil { - newCheckpoint = i - 1 + newLink := oldLink + // Starting from last checkpoint, keeping reading immediate next crosslink until there is a gap + for i := oldLink.BlockNum() + 1; ; i++ { + tmp, err := bc.ReadCrossLink(shardID, i) + if err == nil && tmp != nil && tmp.BlockNum() == i { + newLink = tmp + } else { break } - if i == cl.BlockNum()-1 { - newCheckpoint = cl.BlockNum() - } } - if newCheckpoint > 0 { - utils.Logger().Debug().Msgf("LastContinuousCrossLink: latest checkpoint blockNum %d", newCheckpoint) - cln, err := bc.ReadCrossLink(cl.ShardID(), newCheckpoint) - if err != nil { - return err - } - return rawdb.WriteShardLastCrossLink(batch, cln.ShardID(), cln.Serialize()) + if newLink.BlockNum() > oldLink.BlockNum() { + utils.Logger().Debug().Msgf("LastContinuousCrossLink: latest checkpoint blockNum %d", newLink.BlockNum()) + return rawdb.WriteShardLastCrossLink(batch, shardID, newLink.Serialize()) } return nil } @@ -1905,22 +1899,29 @@ func (bc *BlockChain) ReadShardLastCrossLink(shardID uint32) (*types.CrossLink, if err != nil { return nil, err } - crossLink, err := types.DeserializeCrossLink(bytes) + return types.DeserializeCrossLink(bytes) +} - return crossLink, err +// DeletePendingSlashingCandidates .. +func (bc *BlockChain) DeletePendingSlashingCandidates() error { + bc.pendingSlashingCandidatesMU.Lock() + defer bc.pendingSlashingCandidatesMU.Unlock() + bc.pendingSlashingCandidates.Purge() + return bc.WritePendingSlashingCandidates(slash.Records{}) } // ReadPendingSlashingCandidates retrieves pending slashing candidates -func (bc *BlockChain) ReadPendingSlashingCandidates() ([]slash.Record, error) { +func (bc *BlockChain) ReadPendingSlashingCandidates() (slash.Records, error) { + cls := slash.Records{} if !bc.Config().IsStaking(bc.CurrentHeader().Epoch()) { - return []slash.Record{}, nil + return cls, nil } - + var err error bytes := []byte{} if cached, ok := bc.pendingSlashingCandidates.Get(pendingSCCacheKey); ok { bytes = cached.([]byte) } else { - bytes, err := rawdb.ReadPendingSlashingCandidates(bc.db) + bytes, err = rawdb.ReadPendingSlashingCandidates(bc.db) if err != nil || len(bytes) == 0 { utils.Logger().Info().Err(err). Int("dataLen", len(bytes)). @@ -1928,7 +1929,7 @@ func (bc *BlockChain) ReadPendingSlashingCandidates() ([]slash.Record, error) { return nil, err } } - cls := []slash.Record{} + if err := rlp.DecodeBytes(bytes, &cls); err != nil { utils.Logger().Error().Err(err).Msg("Invalid pending slashing candidates RLP decoding") return nil, err @@ -1937,12 +1938,7 @@ func (bc *BlockChain) ReadPendingSlashingCandidates() ([]slash.Record, error) { } // WritePendingSlashingCandidates saves the pending slashing candidates -func (bc *BlockChain) WritePendingSlashingCandidates(candidates []slash.Record) error { - if !bc.Config().IsStaking(bc.CurrentHeader().Epoch()) { - utils.Logger().Debug().Msg("Writing slashing candidates in prior to staking epoch") - return nil - } - +func (bc *BlockChain) WritePendingSlashingCandidates(candidates slash.Records) error { bytes, err := rlp.EncodeToBytes(candidates) if err != nil { const msg = "[WritePendingSlashingCandidates] Failed to encode pending slashing candidates" @@ -1952,11 +1948,8 @@ func (bc *BlockChain) WritePendingSlashingCandidates(candidates []slash.Record) if err := rawdb.WritePendingSlashingCandidates(bc.db, bytes); err != nil { return err } - if by, err := rlp.EncodeToBytes(candidates); err == nil { - bc.pendingSlashingCandidates.Add(pendingSCCacheKey, by) - } + bc.pendingSlashingCandidates.Add(pendingSCCacheKey, bytes) return nil - } // ReadPendingCrossLinks retrieves pending crosslinks @@ -2014,20 +2007,27 @@ func (bc *BlockChain) WritePendingCrossLinks(crossLinks []types.CrossLink) error } -// AddPendingSlashingCandidate appends pending slashing candidates -func (bc *BlockChain) AddPendingSlashingCandidate(candidate *slash.Record) (int, error) { +// AddPendingSlashingCandidates appends pending slashing candidates +func (bc *BlockChain) AddPendingSlashingCandidates( + candidates []slash.Record, +) (int, error) { bc.pendingSlashingCandidatesMU.Lock() defer bc.pendingSlashingCandidatesMU.Unlock() - cls, err := bc.ReadPendingSlashingCandidates() + if err != nil || len(cls) == 0 { - err := bc.WritePendingSlashingCandidates([]slash.Record{*candidate}) + err := bc.WritePendingSlashingCandidates(candidates) + if err != nil { + return 0, err + } return 1, err } - cls = append(cls, *candidate) - err = bc.WritePendingSlashingCandidates(cls) - return len(cls), err + cls = append(cls, candidates...) + if err := bc.WritePendingSlashingCandidates(cls); err != nil { + return 0, err + } + return len(cls), nil } // AddPendingCrossLinks appends pending crosslinks @@ -2045,8 +2045,8 @@ func (bc *BlockChain) AddPendingCrossLinks(pendingCLs []types.CrossLink) (int, e return len(cls), err } -// DeleteCommittedFromPendingCrossLinks delete pending crosslinks that already committed (i.e. passed in the params) -func (bc *BlockChain) DeleteCommittedFromPendingCrossLinks(crossLinks []types.CrossLink) (int, error) { +// DeleteFromPendingCrossLinks delete pending crosslinks that already committed (i.e. passed in the params) +func (bc *BlockChain) DeleteFromPendingCrossLinks(crossLinks []types.CrossLink) (int, error) { bc.pendingCrossLinksMutex.Lock() defer bc.pendingCrossLinksMutex.Unlock() @@ -2158,25 +2158,26 @@ func (bc *BlockChain) ReadTxLookupEntry(txID common.Hash) (common.Hash, uint64, return rawdb.ReadTxLookupEntry(bc.db, txID) } -// ReadValidatorInformationAt reads staking information of given validatorWrapper at a specific state root -func (bc *BlockChain) ReadValidatorInformationAt(addr common.Address, root common.Hash) (*staking.ValidatorWrapper, error) { +// ReadValidatorInformationAt reads staking +// information of given validatorWrapper at a specific state root +func (bc *BlockChain) ReadValidatorInformationAt( + addr common.Address, root common.Hash, +) (*staking.ValidatorWrapper, error) { state, err := bc.StateAt(root) if err != nil || state == nil { - return nil, err + return nil, errors.Wrapf(err, "at root: %s", root.Hex()) } - wrapper := state.GetStakingInfo(addr) - if wrapper == nil { - return nil, fmt.Errorf( - "at root: %s, validator info not found: %s", - root.Hex(), - common2.MustAddressToBech32(addr), - ) + wrapper, err := state.ValidatorWrapper(addr) + if err != nil { + return nil, errors.Wrapf(err, "at root: %s", root.Hex()) } return wrapper, nil } // ReadValidatorInformation reads staking information of given validator address -func (bc *BlockChain) ReadValidatorInformation(addr common.Address) (*staking.ValidatorWrapper, error) { +func (bc *BlockChain) ReadValidatorInformation( + addr common.Address, +) (*staking.ValidatorWrapper, error) { return bc.ReadValidatorInformationAt(addr, bc.CurrentBlock().Root()) } @@ -2328,9 +2329,9 @@ func (bc *BlockChain) WriteValidatorList(db rawdb.DatabaseWriter, addrs []common return nil } -// ReadActiveValidatorList reads the addresses of active validators -func (bc *BlockChain) ReadActiveValidatorList() ([]common.Address, error) { - if cached, ok := bc.validatorListCache.Get("activeValidatorList"); ok { +// ReadElectedValidatorList reads the addresses of elected validators +func (bc *BlockChain) ReadElectedValidatorList() ([]common.Address, error) { + if cached, ok := bc.validatorListCache.Get("electedValidatorList"); ok { by := cached.([]byte) m := []common.Address{} if err := rlp.DecodeBytes(by, &m); err != nil { @@ -2341,22 +2342,26 @@ func (bc *BlockChain) ReadActiveValidatorList() ([]common.Address, error) { return rawdb.ReadValidatorList(bc.db, true) } -// WriteActiveValidatorList writes the list of active validator addresses to database +// WriteElectedValidatorList writes the list of +// elected validator addresses to database // Note: this should only be called within the blockchain insert process. -func (bc *BlockChain) WriteActiveValidatorList(batch rawdb.DatabaseWriter, addrs []common.Address) error { - err := rawdb.WriteValidatorList(batch, addrs, true) - if err != nil { +func (bc *BlockChain) WriteElectedValidatorList( + batch rawdb.DatabaseWriter, addrs []common.Address, +) error { + if err := rawdb.WriteValidatorList(batch, addrs, true); err != nil { return err } bytes, err := rlp.EncodeToBytes(addrs) if err == nil { - bc.validatorListCache.Add("activeValidatorList", bytes) + bc.validatorListCache.Add("electedValidatorList", bytes) } return nil } // ReadDelegationsByDelegator reads the addresses of validators delegated by a delegator -func (bc *BlockChain) ReadDelegationsByDelegator(delegator common.Address) ([]staking.DelegationIndex, error) { +func (bc *BlockChain) ReadDelegationsByDelegator( + delegator common.Address, +) ([]staking.DelegationIndex, error) { if cached, ok := bc.validatorListByDelegatorCache.Get(string(delegator.Bytes())); ok { by := cached.([]byte) m := []staking.DelegationIndex{} @@ -2381,9 +2386,12 @@ func (bc *BlockChain) writeDelegationsByDelegator(batch rawdb.DatabaseWriter, de return nil } -// UpdateStakingMetaData updates the validator's and the delegator's meta data according to staking transaction +// UpdateStakingMetaData updates the validator's +// and the delegator's meta data according to staking transaction // Note: this should only be called within the blockchain insert process. -func (bc *BlockChain) UpdateStakingMetaData(batch rawdb.DatabaseWriter, tx *staking.StakingTransaction, root common.Hash) error { +func (bc *BlockChain) UpdateStakingMetaData( + batch rawdb.DatabaseWriter, tx *staking.StakingTransaction, root common.Hash, +) error { // TODO: simply the logic here in staking/types/transaction.go payload, err := tx.RLPEncodeStakeMsg() if err != nil { @@ -2414,23 +2422,27 @@ func (bc *BlockChain) UpdateStakingMetaData(batch rawdb.DatabaseWriter, tx *stak } // Update validator snapshot with the new validator - validator, err := bc.ReadValidatorInformationAt(createValidator.ValidatorAddress, root) + validator, err := bc.ReadValidatorInformationAt( + createValidator.ValidatorAddress, root, + ) if err != nil { return err } - validator.Snapshot.Epoch = epoch - if err := rawdb.WriteValidatorSnapshot(batch, validator, epoch); err != nil { return err } // Add self delegation into the index - return bc.addDelegationIndex(batch, createValidator.ValidatorAddress, createValidator.ValidatorAddress, root) + return bc.addDelegationIndex( + batch, createValidator.ValidatorAddress, createValidator.ValidatorAddress, root, + ) case staking.DirectiveEditValidator: case staking.DirectiveDelegate: delegate := decodePayload.(*staking.Delegate) - return bc.addDelegationIndex(batch, delegate.DelegatorAddress, delegate.ValidatorAddress, root) + return bc.addDelegationIndex( + batch, delegate.DelegatorAddress, delegate.ValidatorAddress, root, + ) case staking.DirectiveUndelegate: case staking.DirectiveCollectRewards: default: @@ -2451,7 +2463,9 @@ func (bc *BlockChain) ReadBlockRewardAccumulator(number uint64) (*big.Int, error // WriteBlockRewardAccumulator directly writes the BlockRewardAccumulator value // Note: this should only be called once during staking launch. -func (bc *BlockChain) WriteBlockRewardAccumulator(batch rawdb.DatabaseWriter, reward *big.Int, number uint64) error { +func (bc *BlockChain) WriteBlockRewardAccumulator( + batch rawdb.DatabaseWriter, reward *big.Int, number uint64, +) error { err := rawdb.WriteBlockRewardAccumulator(batch, reward, number) if err != nil { return err diff --git a/core/chain_makers.go b/core/chain_makers.go index a7b335d29..f39755b9d 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -188,7 +188,7 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse if b.engine != nil { // Finalize and seal the block - block, _, err := b.engine.Finalize(chainreader, b.header, statedb, b.txs, b.receipts, nil, nil, nil) + block, _, err := b.engine.Finalize(chainreader, b.header, statedb, b.txs, b.receipts, nil, nil, nil, nil) if err != nil { panic(err) } @@ -271,7 +271,7 @@ func (cr *fakeChainReader) GetHeaderByHash(hash common.Hash) *block.Header func (cr *fakeChainReader) GetHeader(hash common.Hash, number uint64) *block.Header { return nil } func (cr *fakeChainReader) GetBlock(hash common.Hash, number uint64) *types.Block { return nil } func (cr *fakeChainReader) ReadShardState(epoch *big.Int) (*shard.State, error) { return nil, nil } -func (cr *fakeChainReader) ReadActiveValidatorList() ([]common.Address, error) { return nil, nil } +func (cr *fakeChainReader) ReadElectedValidatorList() ([]common.Address, error) { return nil, nil } func (cr *fakeChainReader) ReadValidatorList() ([]common.Address, error) { return nil, nil } func (cr *fakeChainReader) ValidatorCandidates() []common.Address { return nil } func (cr *fakeChainReader) SuperCommitteeForNextEpoch(beacon consensus_engine.ChainReader, header *block.Header, isVerify bool) (*shard.State, error) { diff --git a/core/genesis.go b/core/genesis.go index 25309cca8..7be3933a3 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -29,8 +29,10 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/rlp" blockfactory "github.com/harmony-one/harmony/block/factory" "github.com/harmony-one/harmony/internal/params" + "github.com/harmony-one/harmony/staking/slash" "github.com/harmony-one/harmony/core/rawdb" "github.com/harmony-one/harmony/core/state" @@ -308,6 +310,9 @@ func (g *Genesis) MustCommit(db ethdb.Database) *types.Block { panic(err) } rawdb.WriteBlockRewardAccumulator(db, big.NewInt(0), 0) + data, _ := rlp.EncodeToBytes([]slash.Record{}) + rawdb.WritePendingSlashingCandidates(db, data) + return block } diff --git a/core/offchain.go b/core/offchain.go index d3656d57f..04586fa92 100644 --- a/core/offchain.go +++ b/core/offchain.go @@ -92,25 +92,10 @@ func (bc *BlockChain) CommitOffChainData( return NonStatTy, err } - // Find all the active validator addresses and store them in db - allActiveValidators := []common.Address{} - processed := make(map[common.Address]struct{}) - for i := range newShardState.Shards { - shard := newShardState.Shards[i] - for j := range shard.Slots { - slot := shard.Slots[j] - if slot.EffectiveStake != nil { // For external validator - _, ok := processed[slot.EcdsaAddress] - if !ok { - processed[slot.EcdsaAddress] = struct{}{} - allActiveValidators = append(allActiveValidators, shard.Slots[j].EcdsaAddress) - } - } - } - } - - // Update active validators - if err := bc.WriteActiveValidatorList(batch, allActiveValidators); err != nil { + // Update elected validators + if err := bc.WriteElectedValidatorList( + batch, newShardState.StakedValidators().Addrs, + ); err != nil { return NonStatTy, err } @@ -164,21 +149,41 @@ func (bc *BlockChain) CommitOffChainData( if err := bc.WriteCrossLinks(batch, types.CrossLinks{crossLink}); err == nil { utils.Logger().Info().Uint64("blockNum", crossLink.BlockNum()).Uint32("shardID", crossLink.ShardID()).Msg("[insertChain/crosslinks] Cross Link Added to Beaconchain") } - bc.LastContinuousCrossLink(batch, crossLink) + + cl0, _ := bc.ReadShardLastCrossLink(crossLink.ShardID()) + if cl0 == nil { + rawdb.WriteShardLastCrossLink(batch, crossLink.ShardID(), crossLink.Serialize()) + } } //clean/update local database cache after crosslink inserted into blockchain - num, err := bc.DeleteCommittedFromPendingCrossLinks(*crossLinks) - utils.Logger().Debug().Msgf("DeleteCommittedFromPendingCrossLinks, crosslinks in header %d, pending crosslinks: %d, error: %+v", len(*crossLinks), num, err) + num, err := bc.DeleteFromPendingCrossLinks(*crossLinks) + if err != nil { + const msg = "DeleteFromPendingCrossLinks, crosslinks in header %d, pending crosslinks: %d, problem: %+v" + utils.Logger().Debug().Msgf(msg, len(*crossLinks), num, err) + } + utils.Logger().Debug().Msgf("DeleteFromPendingCrossLinks, crosslinks in header %d, pending crosslinks: %d", len(*crossLinks), num) + } + // Roll up latest crosslinks + for i := uint32(0); i < shard.Schedule.InstanceForEpoch(epoch).NumShards(); i++ { + bc.LastContinuousCrossLink(batch, i) } if bc.CurrentHeader().ShardID() == shard.BeaconChainShardID { if bc.chainConfig.IsStaking(block.Epoch()) { - bc.UpdateBlockRewardAccumulator(batch, payout, block.Number().Uint64()) + if err := bc.UpdateBlockRewardAccumulator( + batch, payout, block.Number().Uint64(), + ); err != nil { + return NonStatTy, err + } + if err := bc.DeletePendingSlashingCandidates(); err != nil { + return NonStatTy, err + } } else { // block reward never accumulate before staking bc.WriteBlockRewardAccumulator(batch, big.NewInt(0), block.Number().Uint64()) } } + return CanonStatTy, nil } diff --git a/core/rawdb/accessors_offchain.go b/core/rawdb/accessors_offchain.go index f700c37bf..87201fa6f 100644 --- a/core/rawdb/accessors_offchain.go +++ b/core/rawdb/accessors_offchain.go @@ -115,11 +115,6 @@ func WritePendingSlashingCandidates(db DatabaseWriter, bytes []byte) error { return db.Put(pendingSlashingKey, bytes) } -// DeletePendingSlashingCandidates stores last pending slashing candidates into database. -func DeletePendingSlashingCandidates(db DatabaseDeleter) error { - return db.Delete(pendingSlashingKey) -} - // ReadCXReceipts retrieves all the transactions of receipts given destination shardID, number and blockHash func ReadCXReceipts(db DatabaseReader, shardID uint32, number uint64, hash common.Hash) (types.CXReceipts, error) { data, err := db.Get(cxReceiptKey(shardID, number, hash)) @@ -246,11 +241,11 @@ func WriteValidatorStats( } // ReadValidatorList retrieves staking validator by its address -// Return only active validators if activeOnly==true, otherwise, return all validators -func ReadValidatorList(db DatabaseReader, activeOnly bool) ([]common.Address, error) { +// Return only elected validators if electedOnly==true, otherwise, return all validators +func ReadValidatorList(db DatabaseReader, electedOnly bool) ([]common.Address, error) { key := validatorListKey - if activeOnly { - key = activeValidatorListKey + if electedOnly { + key = electedValidatorListKey } data, err := db.Get(key) if err != nil || len(data) == 0 { @@ -265,11 +260,11 @@ func ReadValidatorList(db DatabaseReader, activeOnly bool) ([]common.Address, er } // WriteValidatorList stores staking validator's information by its address -// Writes only for active validators if activeOnly==true, otherwise, writes for all validators -func WriteValidatorList(db DatabaseWriter, addrs []common.Address, activeOnly bool) error { +// Writes only for elected validators if electedOnly==true, otherwise, writes for all validators +func WriteValidatorList(db DatabaseWriter, addrs []common.Address, electedOnly bool) error { key := validatorListKey - if activeOnly { - key = activeValidatorListKey + if electedOnly { + key = electedValidatorListKey } bytes, err := rlp.EncodeToBytes(addrs) diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index 3fc2df222..a9e191c0d 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -71,11 +71,11 @@ var ( cxReceiptSpentPrefix = []byte("cxReceiptSpent") // prefix for indicator of unspent of cxReceiptsProof cxReceiptUnspentCheckpointPrefix = []byte("cxReceiptUnspentCheckpoint") // prefix for cxReceiptsProof unspent checkpoint - validatorPrefix = []byte("validator") // prefix for staking validator information - validatorSnapshotPrefix = []byte("validator-snapshot") // prefix for staking validator's snapshot information - validatorStatsPrefix = []byte("validator-stats") // prefix for staking validator's stats information - validatorListKey = []byte("validator-list") // key for all validators list - activeValidatorListKey = []byte("active-validator-list") // key for active validators list + validatorPrefix = []byte("validator") // prefix for staking validator information + validatorSnapshotPrefix = []byte("validator-snapshot") // prefix for staking validator's snapshot information + validatorStatsPrefix = []byte("validator-stats") // prefix for staking validator's stats information + validatorListKey = []byte("validator-list") // key for all validators list + electedValidatorListKey = []byte("elected-validator-list") // key for elected validators list // epochBlockNumberPrefix + epoch (big.Int.Bytes()) // -> epoch block number (big.Int.Bytes()) diff --git a/core/staking_verifier.go b/core/staking_verifier.go index 9ce48266a..2fb96b2b6 100644 --- a/core/staking_verifier.go +++ b/core/staking_verifier.go @@ -55,9 +55,9 @@ func VerifyAndCreateValidatorFromMsg( wrapper.Delegations = []staking.Delegation{ staking.NewDelegation(v.Address, msg.Amount), } - wrapper.Snapshot.Epoch = epoch - wrapper.Snapshot.NumBlocksSigned = big.NewInt(0) - wrapper.Snapshot.NumBlocksToSign = big.NewInt(0) + zero := big.NewInt(0) + wrapper.Counters.NumBlocksSigned = zero + wrapper.Counters.NumBlocksToSign = zero if err := wrapper.SanityCheck(); err != nil { return nil, err } @@ -83,9 +83,9 @@ func VerifyAndEditValidatorFromMsg( if !stateDB.IsValidator(msg.ValidatorAddress) { return nil, errValidatorNotExist } - wrapper := stateDB.GetStakingInfo(msg.ValidatorAddress) - if wrapper == nil { - return nil, errValidatorNotExist + wrapper, err := stateDB.ValidatorWrapper(msg.ValidatorAddress) + if err != nil { + return nil, err } if err := staking.UpdateValidatorFromEditMsg(&wrapper.Validator, msg); err != nil { return nil, err @@ -133,9 +133,9 @@ func VerifyAndDelegateFromMsg( if !stateDB.IsValidator(msg.ValidatorAddress) { return nil, nil, errValidatorNotExist } - wrapper := stateDB.GetStakingInfo(msg.ValidatorAddress) - if wrapper == nil { - return nil, nil, errValidatorNotExist + wrapper, err := stateDB.ValidatorWrapper(msg.ValidatorAddress) + if err != nil { + return nil, nil, err } // Check for redelegation for i := range wrapper.Delegations { @@ -203,16 +203,20 @@ func VerifyAndUndelegateFromMsg( if epoch == nil { return nil, errEpochMissing } + if msg.Amount.Sign() == -1 { return nil, errNegativeAmount } + if !stateDB.IsValidator(msg.ValidatorAddress) { return nil, errValidatorNotExist } - wrapper := stateDB.GetStakingInfo(msg.ValidatorAddress) - if wrapper == nil { - return nil, errValidatorNotExist + + wrapper, err := stateDB.ValidatorWrapper(msg.ValidatorAddress) + if err != nil { + return nil, err } + for i := range wrapper.Delegations { delegation := &wrapper.Delegations[i] if bytes.Equal(delegation.DelegatorAddress.Bytes(), msg.DelegatorAddress.Bytes()) { @@ -243,9 +247,9 @@ func VerifyAndCollectRewardsFromDelegation( totalRewards := big.NewInt(0) for i := range delegations { delegation := &delegations[i] - wrapper := stateDB.GetStakingInfo(delegation.ValidatorAddress) - if wrapper == nil { - return nil, nil, errValidatorNotExist + wrapper, err := stateDB.ValidatorWrapper(delegation.ValidatorAddress) + if err != nil { + return nil, nil, err } if uint64(len(wrapper.Delegations)) > delegation.Index { delegation := &wrapper.Delegations[delegation.Index] diff --git a/core/state/statedb.go b/core/state/statedb.go index ff1e0b347..a67d58ad2 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -18,22 +18,21 @@ package state import ( - "errors" "fmt" "math/big" "sort" - "github.com/harmony-one/harmony/numeric" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" - "github.com/harmony-one/harmony/core/types" + common2 "github.com/harmony-one/harmony/internal/common" + "github.com/harmony-one/harmony/numeric" "github.com/harmony-one/harmony/staking" stk "github.com/harmony-one/harmony/staking/types" + "github.com/pkg/errors" ) type revision struct { @@ -248,7 +247,9 @@ func (db *DB) GetCodeSize(addr common.Address) int { if stateObject.code != nil { return len(stateObject.code) } - size, err := db.db.ContractCodeSize(stateObject.addrHash, common.BytesToHash(stateObject.CodeHash())) + size, err := db.db.ContractCodeSize( + stateObject.addrHash, common.BytesToHash(stateObject.CodeHash()), + ) if err != nil { db.setError(err) } @@ -684,23 +685,34 @@ func (db *DB) Commit(deleteEmptyObjects bool) (root common.Hash, err error) { return root, err } -// GetStakingInfo returns staking information of a given validator (including delegation info) -func (db *DB) GetStakingInfo(addr common.Address) *stk.ValidatorWrapper { +var ( + errAddressNotPresent = errors.New("address not present in state") +) + +// ValidatorWrapper .. +func (db *DB) ValidatorWrapper( + addr common.Address, +) (*stk.ValidatorWrapper, error) { by := db.GetCode(addr) if len(by) == 0 { - return nil + return nil, errAddressNotPresent } val := stk.ValidatorWrapper{} - err := rlp.DecodeBytes(by, &val) - if err != nil { - fmt.Printf("GetStakingInfo unable to decode: %v\n", err) - return nil + if err := rlp.DecodeBytes(by, &val); err != nil { + return nil, errors.Wrapf( + err, + "could not decode for %s", + common2.MustAddressToBech32(addr), + ) } - return &val + return &val, nil } -// UpdateStakingInfo update staking information of a given validator (including delegation info) -func (db *DB) UpdateStakingInfo(addr common.Address, val *stk.ValidatorWrapper) error { +// UpdateValidatorWrapper updates staking information of +// a given validator (including delegation info) +func (db *DB) UpdateValidatorWrapper( + addr common.Address, val *stk.ValidatorWrapper, +) error { if err := val.SanityCheck(); err != nil { return err } @@ -710,7 +722,6 @@ func (db *DB) UpdateStakingInfo(addr common.Address, val *stk.ValidatorWrapper) return err } db.SetCode(addr, by) - return nil } @@ -737,9 +748,9 @@ func (db *DB) IsValidator(addr common.Address) bool { func (db *DB) AddReward(snapshot *stk.ValidatorWrapper, reward *big.Int) error { rewardPool := big.NewInt(0).Set(reward) - curValidator := db.GetStakingInfo(snapshot.Validator.Address) - if curValidator == nil { - return errors.New("failed to distribute rewards: validator does not exist") + curValidator, err := db.ValidatorWrapper(snapshot.Validator.Address) + if err != nil { + return errors.Wrapf(err, "failed to distribute rewards: validator does not exist") } // Payout commission @@ -762,10 +773,11 @@ func (db *DB) AddReward(snapshot *stk.ValidatorWrapper, reward *big.Int) error { rewardPool.Sub(rewardPool, rewardInt) } - // The last remaining bit belongs to the validator (remember the validator's self delegation is always at index 0) - if rewardPool.Cmp(big.NewInt(0)) > 0 { + // The last remaining bit belongs to the validator (remember the validator's self delegation is + // always at index 0) + if rewardPool.Cmp(common.Big0) > 0 { curValidator.Delegations[0].Reward.Add(curValidator.Delegations[0].Reward, rewardPool) } - return db.UpdateStakingInfo(curValidator.Validator.Address, curValidator) + return db.UpdateValidatorWrapper(curValidator.Validator.Address, curValidator) } diff --git a/core/state_processor.go b/core/state_processor.go index 1af96b3b7..93322b482 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -22,6 +22,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/rlp" "github.com/harmony-one/harmony/block" consensus_engine "github.com/harmony-one/harmony/consensus/engine" "github.com/harmony-one/harmony/core/state" @@ -31,6 +32,7 @@ import ( "github.com/harmony-one/harmony/internal/params" "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/shard" + "github.com/harmony-one/harmony/staking/slash" staking "github.com/harmony-one/harmony/staking/types" ) @@ -66,12 +68,11 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.DB, cfg vm.C var ( receipts types.Receipts outcxs types.CXReceipts - - incxs = block.IncomingReceipts() - usedGas = new(uint64) - header = block.Header() - allLogs []*types.Log - gp = new(GasPool).AddGas(block.GasLimit()) + incxs = block.IncomingReceipts() + usedGas = new(uint64) + header = block.Header() + allLogs []*types.Log + gp = new(GasPool).AddGas(block.GasLimit()) ) beneficiary, err := p.bc.GetECDSAFromCoinbase(header) @@ -107,7 +108,8 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.DB, cfg vm.C allLogs = append(allLogs, receipt.Logs...) } - // incomingReceipts should always be processed after transactions (to be consistent with the block proposal) + // incomingReceipts should always be processed + // after transactions (to be consistent with the block proposal) for _, cx := range block.IncomingReceipts() { err := ApplyIncomingReceipt(p.config, statedb, header, cx) if err != nil { @@ -115,8 +117,18 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.DB, cfg vm.C } } + slashes := slash.Records{} + if s := header.Slashes(); len(s) > 0 { + if err := rlp.DecodeBytes(s, &slashes); err != nil { + return nil, nil, nil, 0, nil, ctxerror.New("cannot finalize block").WithCause(err) + } + } + // Finalize the block, applying any consensus engine specific extras (e.g. block rewards) - _, payout, err := p.engine.Finalize(p.bc, header, statedb, block.Transactions(), receipts, outcxs, incxs, block.StakingTransactions()) + _, payout, err := p.engine.Finalize( + p.bc, header, statedb, block.Transactions(), + receipts, outcxs, incxs, block.StakingTransactions(), slashes, + ) if err != nil { return nil, nil, nil, 0, nil, ctxerror.New("cannot finalize block").WithCause(err) } @@ -125,13 +137,19 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.DB, cfg vm.C } // return true if it is valid -func getTransactionType(config *params.ChainConfig, header *block.Header, tx *types.Transaction) types.TransactionType { - if header.ShardID() == tx.ShardID() && (!config.AcceptsCrossTx(header.Epoch()) || tx.ShardID() == tx.ToShardID()) { +func getTransactionType( + config *params.ChainConfig, header *block.Header, tx *types.Transaction, +) types.TransactionType { + if header.ShardID() == tx.ShardID() && + (!config.AcceptsCrossTx(header.Epoch()) || + tx.ShardID() == tx.ToShardID()) { return types.SameShardTx } numShards := shard.Schedule.InstanceForEpoch(header.Epoch()).NumShards() // Assuming here all the shards are consecutive from 0 to n-1, n is total number of shards - if tx.ShardID() != tx.ToShardID() && header.ShardID() == tx.ShardID() && tx.ToShardID() < numShards { + if tx.ShardID() != tx.ToShardID() && + header.ShardID() == tx.ShardID() && + tx.ToShardID() < numShards { return types.SubtractionOnly } return types.InvalidTx diff --git a/core/state_transition.go b/core/state_transition.go index 6d3ebaf56..b495c4fb6 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -373,7 +373,7 @@ func (st *StateTransition) verifyAndApplyCreateValidatorTx( if err != nil { return err } - if err := st.state.UpdateStakingInfo(wrapper.Validator.Address, wrapper); err != nil { + if err := st.state.UpdateValidatorWrapper(wrapper.Validator.Address, wrapper); err != nil { return err } st.state.SetValidatorFlag(wrapper.Validator.Address) @@ -388,7 +388,7 @@ func (st *StateTransition) verifyAndApplyEditValidatorTx( if err != nil { return err } - return st.state.UpdateStakingInfo(wrapper.Address, wrapper) + return st.state.UpdateValidatorWrapper(wrapper.Address, wrapper) } func (st *StateTransition) verifyAndApplyDelegateTx(delegate *staking.Delegate) error { @@ -396,7 +396,7 @@ func (st *StateTransition) verifyAndApplyDelegateTx(delegate *staking.Delegate) if err != nil { return err } - if err := st.state.UpdateStakingInfo(wrapper.Validator.Address, wrapper); err != nil { + if err := st.state.UpdateValidatorWrapper(wrapper.Validator.Address, wrapper); err != nil { return err } st.state.SubBalance(delegate.DelegatorAddress, balanceToBeDeducted) @@ -408,7 +408,7 @@ func (st *StateTransition) verifyAndApplyUndelegateTx(undelegate *staking.Undele if err != nil { return err } - return st.state.UpdateStakingInfo(wrapper.Validator.Address, wrapper) + return st.state.UpdateValidatorWrapper(wrapper.Validator.Address, wrapper) } func (st *StateTransition) verifyAndApplyCollectRewards(collectRewards *staking.CollectRewards) error { @@ -419,12 +419,14 @@ func (st *StateTransition) verifyAndApplyCollectRewards(collectRewards *staking. if err != nil { return err } - updatedValidatorWrappers, totalRewards, err := VerifyAndCollectRewardsFromDelegation(st.state, delegations) + updatedValidatorWrappers, totalRewards, err := VerifyAndCollectRewardsFromDelegation( + st.state, delegations, + ) if err != nil { return err } for _, wrapper := range updatedValidatorWrappers { - if err := st.state.UpdateStakingInfo(wrapper.Validator.Address, wrapper); err != nil { + if err := st.state.UpdateValidatorWrapper(wrapper.Validator.Address, wrapper); err != nil { return err } } diff --git a/core/tx_list.go b/core/tx_list.go index 611cc9de6..b35d049cf 100644 --- a/core/tx_list.go +++ b/core/tx_list.go @@ -263,7 +263,11 @@ func (l *txList) Add(tx types.PoolTransaction, priceBump uint64) (bool, types.Po } // Otherwise overwrite the old transaction with the current one l.txs.Put(tx) - if cost := tx.Cost(); l.costcap.Cmp(cost) < 0 { + cost, err := tx.Cost() + if err != nil { + return false, nil + } + if l.costcap.Cmp(cost) < 0 { l.costcap = cost } if gas := tx.Gas(); l.gascap < gas { @@ -297,7 +301,13 @@ func (l *txList) Filter(costLimit *big.Int, gasLimit uint64) (types.PoolTransact l.gascap = gasLimit // Filter out all the transactions above the account's funds - removed := l.txs.Filter(func(tx types.PoolTransaction) bool { return tx.Cost().Cmp(costLimit) > 0 || tx.Gas() > gasLimit }) + removed := l.txs.Filter(func(tx types.PoolTransaction) bool { + cost, err := tx.Cost() + if err != nil { + return true // failure should lead to removal of the tx + } + return cost.Cmp(costLimit) > 0 || tx.Gas() > gasLimit + }) // If the list was strict, filter anything above the lowest nonce var invalids types.PoolTransactions diff --git a/core/tx_pool.go b/core/tx_pool.go index 6686aadbf..bb551ae1c 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -707,7 +707,11 @@ func (pool *TxPool) validateTx(tx types.PoolTransaction, local bool) error { } // Transactor should have enough funds to cover the costs // cost == V + GP * GL - if pool.currentState.GetBalance(from).Cmp(tx.Cost()) < 0 { + cost, err := tx.Cost() + if err != nil { + return err + } + if pool.currentState.GetBalance(from).Cmp(cost) < 0 { return ErrInsufficientFunds } intrGas := uint64(0) diff --git a/core/tx_pool_test.go b/core/tx_pool_test.go index 1ace57fed..8e14fb128 100644 --- a/core/tx_pool_test.go +++ b/core/tx_pool_test.go @@ -47,6 +47,10 @@ var ( testTxPoolConfig TxPoolConfig testBLSPubKey = "30b2c38b1316da91e068ac3bd8751c0901ef6c02a1d58bc712104918302c6ed03d5894671d0c816dad2b4d303320f202" testBLSPrvKey = "c6d7603520311f7a4e6aac0b26701fc433b75b38df504cd416ef2b900cd66205" + + gasPrice = big.NewInt(1e9) + gasLimit = big.NewInt(int64(params.TxGasValidatorCreation)) + cost = big.NewInt(1).Mul(gasPrice, gasLimit) ) func init() { @@ -334,6 +338,8 @@ func TestCreateValidatorTransaction(t *testing.T) { } senderAddr, _ := stx.SenderAddress() pool.currentState.AddBalance(senderAddr, big.NewInt(1e18)) + // Add additional create validator tx cost + pool.currentState.AddBalance(senderAddr, cost) err = pool.AddRemote(stx) if err != nil { @@ -358,6 +364,8 @@ func TestMixedTransactions(t *testing.T) { } stxAddr, _ := stx.SenderAddress() pool.currentState.AddBalance(stxAddr, big.NewInt(1e18)) + // Add additional create validator tx cost + pool.currentState.AddBalance(stxAddr, cost) goodFromKey, _ := crypto.GenerateKey() tx := transaction(0, 0, 25000, goodFromKey) diff --git a/core/types/block.go b/core/types/block.go index ad00e2380..c194c62d8 100644 --- a/core/types/block.go +++ b/core/types/block.go @@ -19,6 +19,7 @@ package types import ( "encoding/binary" + "fmt" "io" "math/big" "reflect" @@ -218,6 +219,17 @@ type Block struct { ReceivedFrom interface{} } +func (b *Block) String() string { + m := b.Header() + return fmt.Sprintf( + "[ViewID:%d Num:%d BlockHash:%s]", + m.ViewID(), + m.Number(), + m.Hash().Hex(), + ) + +} + // SetLastCommitSig sets the last block's commit group signature. func (b *Block) SetLastCommitSig(sig []byte, signers []byte) { if len(sig) != len(b.header.LastCommitSignature()) { diff --git a/core/types/transaction.go b/core/types/transaction.go index 42f48f5c7..75049bd85 100644 --- a/core/types/transaction.go +++ b/core/types/transaction.go @@ -414,10 +414,10 @@ func (tx *Transaction) WithSignature(signer Signer, sig []byte) (*Transaction, e } // Cost returns amount + gasprice * gaslimit. -func (tx *Transaction) Cost() *big.Int { +func (tx *Transaction) Cost() (*big.Int, error) { total := new(big.Int).Mul(tx.data.Price, new(big.Int).SetUint64(tx.data.GasLimit)) total.Add(total, tx.data.Amount) - return total + return total, nil } // RawSignatureValues return raw signature values. diff --git a/core/types/tx_pool.go b/core/types/tx_pool.go index 5ed949871..e32512934 100644 --- a/core/types/tx_pool.go +++ b/core/types/tx_pool.go @@ -27,7 +27,7 @@ type PoolTransaction interface { Data() []byte GasPrice() *big.Int Gas() uint64 - Cost() *big.Int + Cost() (*big.Int, error) Value() *big.Int EncodeRLP(w io.Writer) error DecodeRLP(s *rlp.Stream) error diff --git a/core/vm/interface.go b/core/vm/interface.go index 495be10cd..91eaae634 100644 --- a/core/vm/interface.go +++ b/core/vm/interface.go @@ -40,8 +40,8 @@ type StateDB interface { SetCode(common.Address, []byte) GetCodeSize(common.Address) int - GetStakingInfo(common.Address) *staking.ValidatorWrapper - UpdateStakingInfo(common.Address, *staking.ValidatorWrapper) error + ValidatorWrapper(common.Address) (*staking.ValidatorWrapper, error) + UpdateValidatorWrapper(common.Address, *staking.ValidatorWrapper) error SetValidatorFlag(common.Address) UnsetValidatorFlag(common.Address) IsValidator(common.Address) bool diff --git a/crypto/hash/rlp.go b/crypto/hash/rlp.go index d0ba7052f..75140b249 100644 --- a/crypto/hash/rlp.go +++ b/crypto/hash/rlp.go @@ -13,3 +13,11 @@ func FromRLP(x interface{}) (h common.Hash) { hw.Sum(h[:0]) return h } + +// FromRLPNew256 hashes the RLP representation of the given object using New256 +func FromRLPNew256(x interface{}) (h common.Hash) { + hw := sha3.New256() + rlp.Encode(hw, x) + hw.Sum(h[:0]) + return h +} diff --git a/go.mod b/go.mod index 845b42a3d..dd638864d 100644 --- a/go.mod +++ b/go.mod @@ -51,7 +51,7 @@ require ( github.com/prometheus/procfs v0.0.3 // indirect github.com/rjeczalik/notify v0.9.2 github.com/rs/cors v1.7.0 // indirect - github.com/rs/zerolog v1.14.3 + github.com/rs/zerolog v1.18.0 github.com/shirou/gopsutil v2.18.12+incompatible github.com/spf13/cobra v0.0.5 github.com/stretchr/testify v1.4.0 diff --git a/hmy/api_backend.go b/hmy/api_backend.go index 8b6571405..0ab93f8ba 100644 --- a/hmy/api_backend.go +++ b/hmy/api_backend.go @@ -4,7 +4,6 @@ import ( "context" "errors" "math/big" - "sort" "sync" "github.com/ethereum/go-ethereum/common" @@ -23,18 +22,14 @@ import ( "github.com/harmony-one/harmony/core/vm" "github.com/harmony-one/harmony/internal/params" "github.com/harmony-one/harmony/shard" + "github.com/harmony-one/harmony/staking/effective" "github.com/harmony-one/harmony/staking/network" staking "github.com/harmony-one/harmony/staking/types" ) // APIBackend An implementation of internal/hmyapi/Backend. Full client. type APIBackend struct { - hmy *Harmony - MedianStakeCache struct { - sync.Mutex - BlockHeight int64 - MedianRawStake *big.Int - } + hmy *Harmony TotalStakingCache struct { sync.Mutex BlockHeight int64 @@ -316,9 +311,9 @@ func (b *APIBackend) SendStakingTx( return nil } -// GetActiveValidatorAddresses returns the address of active validators for current epoch -func (b *APIBackend) GetActiveValidatorAddresses() []common.Address { - list, _ := b.hmy.BlockChain().ReadActiveValidatorList() +// GetElectedValidatorAddresses returns the address of elected validators for current epoch +func (b *APIBackend) GetElectedValidatorAddresses() []common.Address { + list, _ := b.hmy.BlockChain().ReadElectedValidatorList() return list } @@ -328,68 +323,69 @@ func (b *APIBackend) GetAllValidatorAddresses() []common.Address { } // GetValidatorInformation returns the information of validator -func (b *APIBackend) GetValidatorInformation(addr common.Address) *staking.Validator { +func (b *APIBackend) GetValidatorInformation(addr common.Address) *staking.ValidatorWrapper { val, _ := b.hmy.BlockChain().ReadValidatorInformation(addr) if val != nil { - return &val.Validator + return val } return nil } -var ( - two = big.NewInt(2) -) - // GetMedianRawStakeSnapshot .. -func (b *APIBackend) GetMedianRawStakeSnapshot() *big.Int { - b.MedianStakeCache.Lock() - defer b.MedianStakeCache.Unlock() - if b.MedianStakeCache.BlockHeight != -1 && b.MedianStakeCache.BlockHeight > int64(rpc.LatestBlockNumber)-20 { - return b.MedianStakeCache.MedianRawStake - } - b.MedianStakeCache.BlockHeight = int64(rpc.LatestBlockNumber) +func (b *APIBackend) GetMedianRawStakeSnapshot() (*big.Int, error) { candidates := b.hmy.BlockChain().ValidatorCandidates() - if len(candidates) == 0 { - b.MedianStakeCache.MedianRawStake = big.NewInt(0) - return b.MedianStakeCache.MedianRawStake - } - stakes := []*big.Int{} + essentials := map[common.Address]effective.SlotOrder{} + blsKeys := make(map[shard.BlsPublicKey]struct{}) for i := range candidates { - validator, _ := b.hmy.BlockChain().ReadValidatorInformation(candidates[i]) - stake := big.NewInt(0) + validator, err := b.hmy.BlockChain().ReadValidatorInformation( + candidates[i], + ) + if err != nil { + return nil, err + } + if !effective.IsEligibleForEPOSAuction(validator) { + continue + } + if err := validator.SanityCheck(); err != nil { + continue + } + + validatorStake := big.NewInt(0) for i := range validator.Delegations { - stake.Add(stake, validator.Delegations[i].Amount) + validatorStake.Add( + validatorStake, validator.Delegations[i].Amount, + ) } - stake = stake.Div(stake, big.NewInt(int64(len(validator.SlotPubKeys)))) - for i := 0; i < len(validator.SlotPubKeys); i++ { - stakes = append(stakes, stake) + + found := false + for _, key := range validator.SlotPubKeys { + if _, ok := blsKeys[key]; ok { + found = true + } else { + blsKeys[key] = struct{}{} + } } - } - sort.SliceStable( - stakes, - func(i, j int) bool { return stakes[i].Cmp(stakes[j]) == -1 }, - ) - if l := len(stakes); l > 320 { - stakes = stakes[:320] - } + if found { + continue + } - switch l := len(stakes); l % 2 { - case 0: - left := stakes[(l/2)-1] - right := stakes[l/2] - b.MedianStakeCache.MedianRawStake = new(big.Int).Div(new(big.Int).Add(left, right), two) - default: - b.MedianStakeCache.MedianRawStake = stakes[l/2] + essentials[validator.Address] = effective.SlotOrder{ + validatorStake, + validator.SlotPubKeys, + } } - return b.MedianStakeCache.MedianRawStake + // TODO thread through the right value from shard.Schedule.Instance + median, _ := effective.Compute(essentials, 320) + return median.TruncateInt(), nil } // GetTotalStakingSnapshot .. func (b *APIBackend) GetTotalStakingSnapshot() *big.Int { b.TotalStakingCache.Lock() defer b.TotalStakingCache.Unlock() - if b.TotalStakingCache.BlockHeight != -1 && b.TotalStakingCache.BlockHeight > int64(rpc.LatestBlockNumber)-20 { + if b.TotalStakingCache.BlockHeight != -1 && + b.TotalStakingCache.BlockHeight > int64(rpc.LatestBlockNumber)-20 { return b.TotalStakingCache.TotalStaking } b.TotalStakingCache.BlockHeight = int64(rpc.LatestBlockNumber) @@ -401,6 +397,9 @@ func (b *APIBackend) GetTotalStakingSnapshot() *big.Int { stakes := big.NewInt(0) for i := range candidates { validator, _ := b.hmy.BlockChain().ReadValidatorInformation(candidates[i]) + if !effective.IsEligibleForEPOSAuction(validator) { + continue + } for i := range validator.Delegations { stakes.Add(stakes, validator.Delegations[i].Amount) } @@ -429,7 +428,9 @@ func (b *APIBackend) GetDelegationsByValidator(validator common.Address) []*stak } // GetDelegationsByDelegator returns all delegation information of a delegator -func (b *APIBackend) GetDelegationsByDelegator(delegator common.Address) ([]common.Address, []*staking.Delegation) { +func (b *APIBackend) GetDelegationsByDelegator( + delegator common.Address, +) ([]common.Address, []*staking.Delegation) { addresses := []common.Address{} delegations := []*staking.Delegation{} delegationIndexes, err := b.hmy.BlockChain().ReadDelegationsByDelegator(delegator) @@ -438,7 +439,9 @@ func (b *APIBackend) GetDelegationsByDelegator(delegator common.Address) ([]comm } for i := range delegationIndexes { - wrapper, err := b.hmy.BlockChain().ReadValidatorInformation(delegationIndexes[i].ValidatorAddress) + wrapper, err := b.hmy.BlockChain().ReadValidatorInformation( + delegationIndexes[i].ValidatorAddress, + ) if err != nil || wrapper == nil { return nil, nil } @@ -511,20 +514,22 @@ func (b *APIBackend) GetSuperCommittees() (*quorum.Transition, error) { for _, comm := range prevCommittee.Shards { decider := quorum.NewDecider(quorum.SuperMajorityStake) + shardID := comm.ShardID decider.SetShardIDProvider(func() (uint32, error) { - return comm.ShardID, nil + return shardID, nil }) decider.SetVoters(comm.Slots) - then.Deciders[comm.ShardID] = decider + then.Deciders[shardID] = decider } for _, comm := range nowCommittee.Shards { decider := quorum.NewDecider(quorum.SuperMajorityStake) + shardID := comm.ShardID decider.SetShardIDProvider(func() (uint32, error) { - return comm.ShardID, nil + return shardID, nil }) decider.SetVoters(comm.Slots) - now.Deciders[comm.ShardID] = decider + now.Deciders[shardID] = decider } return &quorum.Transition{then, now}, nil diff --git a/hmy/backend.go b/hmy/backend.go index 28ee91ba0..c465c4352 100644 --- a/hmy/backend.go +++ b/hmy/backend.go @@ -76,14 +76,6 @@ func New( shardID: shardID, } hmy.APIBackend = &APIBackend{hmy: hmy, - MedianStakeCache: struct { - sync.Mutex - BlockHeight int64 - MedianRawStake *big.Int - }{ - BlockHeight: -1, - MedianRawStake: big.NewInt(0), - }, TotalStakingCache: struct { sync.Mutex BlockHeight int64 diff --git a/internal/chain/engine.go b/internal/chain/engine.go index 81c558ece..d85ff3617 100644 --- a/internal/chain/engine.go +++ b/internal/chain/engine.go @@ -15,7 +15,6 @@ import ( "github.com/harmony-one/harmony/consensus/reward" "github.com/harmony-one/harmony/core/state" "github.com/harmony-one/harmony/core/types" - common2 "github.com/harmony-one/harmony/internal/common" "github.com/harmony-one/harmony/internal/ctxerror" "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/shard" @@ -145,7 +144,7 @@ func (e *engineImpl) VerifyShardState(bc engine.ChainReader, beacon engine.Chain } headerShardStateBytes := header.ShardState() // TODO: figure out leader withhold shardState - if headerShardStateBytes == nil || len(headerShardStateBytes) == 0 { + if len(headerShardStateBytes) == 0 { return nil } shardState, err := bc.SuperCommitteeForNextEpoch(beacon, header, true) @@ -217,7 +216,7 @@ func (e *engineImpl) VerifySeal(chain engine.ChainReader, header *block.Header) return nil, nil }) d.SetVoters(slotList.FindCommitteeByID(parentHeader.ShardID()).Slots) - if !d.IsQuorumAchievedByMask(mask, true) { + if !d.IsQuorumAchievedByMask(mask) { return ctxerror.New( "[VerifySeal] Not enough voting power in LastCommitSignature from Block Header", ) @@ -241,7 +240,10 @@ func (e *engineImpl) VerifySeal(chain engine.ChainReader, header *block.Header) lastCommitPayload := append(blockNumHash, parentHash[:]...) if !aggSig.VerifyHash(mask.AggregatePublic, lastCommitPayload) { - return ctxerror.New("[VerifySeal] Unable to verify aggregated signature from last block", "lastBlockNum", header.Number().Uint64()-1, "lastBlockHash", parentHash) + const msg = "[VerifySeal] Unable to verify aggregated signature from last block" + return ctxerror.New( + msg, "lastBlockNum", header.Number().Uint64()-1, "lastBlockHash", parentHash, + ) } return nil } @@ -253,9 +255,9 @@ func (e *engineImpl) Finalize( state *state.DB, txs []*types.Transaction, receipts []*types.Receipt, outcxs []*types.CXReceipt, incxs []*types.CXReceiptsProof, stks []*staking.StakingTransaction, + doubleSigners slash.Records, ) (*types.Block, *big.Int, error) { - - // Accumulate any block and uncle rewards and commit the final state root + // Accumulate block rewards and commit the final state root // Header seems complete, assemble into a block and return payout, err := AccumulateRewards( chain, state, header, e.Rewarder(), e.Beaconchain(), @@ -266,60 +268,116 @@ func (e *engineImpl) Finalize( isBeaconChain := header.ShardID() == shard.BeaconChainShardID isNewEpoch := len(header.ShardState()) > 0 inStakingEra := chain.Config().IsStaking(header.Epoch()) - // Apply the slashes, invariant: assume been verified as legit slash by this point - if isBeaconChain && isNewEpoch && inStakingEra { - if err := slash.Apply(state, header.Slashes()); err != nil { - return nil, nil, ctxerror.New("[Finalize] could not apply slash").WithCause(err) - } - } // Withdraw unlocked tokens to the delegators' accounts // Only do such at the last block of an epoch if isBeaconChain && isNewEpoch && inStakingEra { validators, err := chain.ReadValidatorList() if err != nil { - return nil, nil, ctxerror.New("[Finalize] failed to read active validators").WithCause(err) + const msg = "[Finalize] failed to read all validators" + return nil, nil, ctxerror.New(msg).WithCause(err) } // Payout undelegated/unlocked tokens for _, validator := range validators { - wrapper := state.GetStakingInfo(validator) - if wrapper != nil { - for i := range wrapper.Delegations { - delegation := &wrapper.Delegations[i] - totalWithdraw := delegation.RemoveUnlockedUndelegations( - header.Epoch(), wrapper.LastEpochInCommittee, - ) - state.AddBalance(delegation.DelegatorAddress, totalWithdraw) - } - if err := state.UpdateStakingInfo(validator, wrapper); err != nil { - return nil, nil, ctxerror.New("[Finalize] failed update validator info").WithCause(err) - } - } else { - err = errors.New("[Finalize] validator came back empty " + common2.MustAddressToBech32(validator)) - return nil, nil, ctxerror.New("[Finalize] failed getting validator info").WithCause(err) + wrapper, err := state.ValidatorWrapper(validator) + if err != nil { + return nil, nil, ctxerror.New( + "[Finalize] failed to get validator from state to finalize", + ).WithCause(err) + } + for i := range wrapper.Delegations { + delegation := &wrapper.Delegations[i] + totalWithdraw := delegation.RemoveUnlockedUndelegations( + header.Epoch(), wrapper.LastEpochInCommittee, + ) + state.AddBalance(delegation.DelegatorAddress, totalWithdraw) + } + if err := state.UpdateValidatorWrapper( + validator, wrapper, + ); err != nil { + const msg = "[Finalize] failed update validator info" + return nil, nil, ctxerror.New(msg).WithCause(err) } } + } + + l := utils.Logger().Info(). + Uint64("current-epoch", chain.CurrentHeader().Epoch().Uint64()). + Uint64("finalizing-epoch", header.Epoch().Uint64()). + Uint64("block-number", header.Number().Uint64()) - // Set the LastEpochInCommittee field for all external validators in the upcoming epoch. - newShardState, err := header.GetShardState() + if isBeaconChain && inStakingEra { + nowEpoch := chain.CurrentHeader().Epoch() + superCommittee, err := chain.ReadShardState(nowEpoch) if err != nil { - return nil, nil, ctxerror.New("[Finalize] failed to read shard state").WithCause(err) + return nil, nil, err } - for _, external := range newShardState.ExternalValidators() { - wrapper := state.GetStakingInfo(external) - wrapper.LastEpochInCommittee = newShardState.Epoch - if err := state.UpdateStakingInfo(external, wrapper); err != nil { - return nil, nil, ctxerror.New( - "[Finalize] failed update validator info", - ).WithCause(err) + staked := superCommittee.StakedValidators() + // could happen that only harmony nodes are running, + if isNewEpoch && staked.CountStakedValidator > 0 { + l.Msg("in new epoch (aka last block), apply availability check for activity") + if err := availability.SetInactiveUnavailableValidators( + chain, state, staked.Addrs, + ); err != nil { + return nil, nil, err + } + // Now can reset the counters, do note, only + // after the availability logic runs + newShardState, err := header.GetShardState() + if err != nil { + const msg = "[Finalize] failed to read shard state" + return nil, nil, ctxerror.New(msg).WithCause(err) + } + + if stkd := newShardState.StakedValidators(); stkd.CountStakedValidator > 0 { + for _, addr := range stkd.Addrs { + wrapper, err := state.ValidatorWrapper(addr) + if err != nil { + return nil, nil, err + } + // Set the LastEpochInCommittee field for all + // external validators in the upcoming epoch. + // and set the availability tracking counters to 0 + wrapper.LastEpochInCommittee = newShardState.Epoch + if err := state.UpdateValidatorWrapper(addr, wrapper); err != nil { + return nil, nil, ctxerror.New( + "[Finalize] failed update validator info", + ).WithCause(err) + } + } } } } - if isBeaconChain && isNewEpoch && inStakingEra { - if err := availability.Apply(chain, state); err != nil { - return nil, nil, err + if caught := len( + doubleSigners, + ); isBeaconChain && inStakingEra && caught > 0 { + superCommittee, err := chain.ReadShardState(chain.CurrentHeader().Epoch()) + + if err != nil { + return nil, nil, errors.New("could not read shard state") + } + + staked := superCommittee.StakedValidators() + // Apply the slashes, invariant: assume been verified as legit slash by this point + var slashApplied *slash.Application + rate := slash.Rate(caught, staked.CountStakedBLSKey) + lg := l.Str("rate", rate.String()). + RawJSON("records", []byte(doubleSigners.String())) + + lg.Msg("now applying slash to state during block finalization") + if slashApplied, err = slash.Apply( + chain, + state, + doubleSigners, + rate, + ); err != nil { + return nil, nil, ctxerror.New("[Finalize] could not apply slash").WithCause(err) } + + lg.RawJSON("applied", []byte(slashApplied.String())). + Msg("slash applied successfully") + } header.SetRoot(state.IntermediateRoot(chain.Config().IsS3(header.Epoch()))) @@ -383,7 +441,7 @@ func (e *engineImpl) VerifyHeaderWithSignature(chain engine.ChainReader, header return nil, nil }) d.SetVoters(slotList.FindCommitteeByID(header.ShardID()).Slots) - if !d.IsQuorumAchievedByMask(mask, true) { + if !d.IsQuorumAchievedByMask(mask) { return ctxerror.New( "[VerifySeal] Not enough voting power in commitSignature from Block Header", ) @@ -410,7 +468,9 @@ func (e *engineImpl) VerifyHeaderWithSignature(chain engine.ChainReader, header } // GetPublicKeys finds the public keys of the committee that signed the block header -func GetPublicKeys(chain engine.ChainReader, header *block.Header, reCalculate bool) ([]*bls.PublicKey, error) { +func GetPublicKeys( + chain engine.ChainReader, header *block.Header, reCalculate bool, +) ([]*bls.PublicKey, error) { shardState := new(shard.State) var err error if reCalculate { @@ -430,13 +490,10 @@ func GetPublicKeys(chain engine.ChainReader, header *block.Header, reCalculate b "shardID", header.ShardID(), ) } - var committerKeys []*bls.PublicKey - - utils.Logger().Print(committee.Slots) + committerKeys := []*bls.PublicKey{} for _, member := range committee.Slots { committerKey := new(bls.PublicKey) - err := member.BlsPublicKey.ToLibBLSPublicKey(committerKey) - if err != nil { + if err := member.BlsPublicKey.ToLibBLSPublicKey(committerKey); err != nil { return nil, ctxerror.New("cannot convert BLS public key", "blsPublicKey", member.BlsPublicKey).WithCause(err) } diff --git a/internal/chain/reward.go b/internal/chain/reward.go index 6d6299b21..4fbf1c80c 100644 --- a/internal/chain/reward.go +++ b/internal/chain/reward.go @@ -48,11 +48,11 @@ func AccumulateRewards( //// After staking if bc.Config().IsStaking(header.Epoch()) && bc.CurrentHeader().ShardID() == shard.BeaconChainShardID { - defaultReward := network.BaseStakedReward - // TODO Use cached result in off-chain db instead of full computation - _, percentageStaked, err := network.WhatPercentStakedNow(beaconChain, header.Time().Int64()) + _, percentageStaked, err := network.WhatPercentStakedNow( + beaconChain, header.Time().Int64(), + ) if err != nil { return network.NoReward, err } @@ -73,11 +73,21 @@ func AccumulateRewards( newRewards := big.NewInt(0) // Take care of my own beacon chain committee, _ is missing, for slashing - members, payable, _, err := ballotResultBeaconchain(beaconChain, header) + members, payable, missing, err := ballotResultBeaconchain(beaconChain, header) if err != nil { return network.NoReward, err } + if err := availability.IncrementValidatorSigningCounts( + beaconChain, + shard.Committee{shard.BeaconChainShardID, members}.StakedValidators(), + state, + payable, + missing, + ); err != nil { + return network.NoReward, err + } + votingPower, err := votepower.Compute(members) if err != nil { return network.NoReward, err @@ -130,13 +140,21 @@ func AccumulateRewards( } subComm := shardState.FindCommitteeByID(cxLink.ShardID()) - // _ are the missing signers, later for slashing - payableSigners, _, err := availability.BlockSigners(cxLink.Bitmap(), subComm) + payableSigners, missing, err := availability.BlockSigners( + cxLink.Bitmap(), subComm, + ) if err != nil { return network.NoReward, err } + staked := subComm.StakedValidators() + if err := availability.IncrementValidatorSigningCounts( + beaconChain, staked, state, payableSigners, missing, + ); err != nil { + return network.NoReward, err + } + votingPower, err := votepower.Compute(payableSigners) if err != nil { return network.NoReward, err diff --git a/internal/configs/sharding/testnet.go b/internal/configs/sharding/testnet.go index 7c5b0492c..f6bfad887 100644 --- a/internal/configs/sharding/testnet.go +++ b/internal/configs/sharding/testnet.go @@ -80,5 +80,5 @@ var testnetReshardingEpoch = []*big.Int{ params.TestnetChainConfig.StakingEpoch, } -var testnetV0 = MustNewInstance(3, 10, 7, genesis.TNHarmonyAccounts, genesis.TNFoundationalAccounts, testnetReshardingEpoch, TestnetSchedule.BlocksPerEpoch()) -var testnetV1 = MustNewInstance(3, 20, 7, genesis.TNHarmonyAccounts, genesis.TNFoundationalAccounts, testnetReshardingEpoch, TestnetSchedule.BlocksPerEpoch()) +var testnetV0 = MustNewInstance(4, 25, 25, genesis.TNHarmonyAccounts, genesis.TNFoundationalAccounts, testnetReshardingEpoch, TestnetSchedule.BlocksPerEpoch()) +var testnetV1 = MustNewInstance(4, 50, 25, genesis.TNHarmonyAccounts, genesis.TNFoundationalAccounts, testnetReshardingEpoch, TestnetSchedule.BlocksPerEpoch()) diff --git a/internal/hmyapi/apiv1/backend.go b/internal/hmyapi/apiv1/backend.go index ada12fe6e..bfbe6ab51 100644 --- a/internal/hmyapi/apiv1/backend.go +++ b/internal/hmyapi/apiv1/backend.go @@ -71,9 +71,9 @@ type Backend interface { ResendCx(ctx context.Context, txID common.Hash) (uint64, bool) IsLeader() bool SendStakingTx(ctx context.Context, newStakingTx *staking.StakingTransaction) error - GetActiveValidatorAddresses() []common.Address + GetElectedValidatorAddresses() []common.Address GetAllValidatorAddresses() []common.Address - GetValidatorInformation(addr common.Address) *staking.Validator + GetValidatorInformation(addr common.Address) *staking.ValidatorWrapper GetValidatorStats(addr common.Address) *staking.ValidatorStats GetDelegationsByValidator(validator common.Address) []*staking.Delegation GetDelegationsByDelegator(delegator common.Address) ([]common.Address, []*staking.Delegation) @@ -81,7 +81,7 @@ type Backend interface { GetShardState() (*shard.State, error) GetCurrentStakingErrorSink() []staking.RPCTransactionError GetCurrentTransactionErrorSink() []types.RPCTransactionError - GetMedianRawStakeSnapshot() *big.Int + GetMedianRawStakeSnapshot() (*big.Int, error) GetPendingCXReceipts() []*types.CXReceiptsProof GetCurrentUtilityMetrics() (*network.UtilityMetric, error) GetSuperCommittees() (*quorum.Transition, error) diff --git a/internal/hmyapi/apiv1/blockchain.go b/internal/hmyapi/apiv1/blockchain.go index 4e9ac027e..198279c3c 100644 --- a/internal/hmyapi/apiv1/blockchain.go +++ b/internal/hmyapi/apiv1/blockchain.go @@ -536,7 +536,7 @@ func (s *PublicBlockChainAPI) GetTotalStaking() (*big.Int, error) { // explorer node func (s *PublicBlockChainAPI) GetMedianRawStakeSnapshot() (*big.Int, error) { if s.b.GetShardID() == shard.BeaconChainShardID { - return s.b.GetMedianRawStakeSnapshot(), nil + return s.b.GetMedianRawStakeSnapshot() } return nil, errNotBeaconChainShard } @@ -551,10 +551,10 @@ func (s *PublicBlockChainAPI) GetAllValidatorAddresses() ([]string, error) { return addresses, nil } -// GetActiveValidatorAddresses returns active validator addresses. -func (s *PublicBlockChainAPI) GetActiveValidatorAddresses() ([]string, error) { +// GetElectedValidatorAddresses returns elected validator addresses. +func (s *PublicBlockChainAPI) GetElectedValidatorAddresses() ([]string, error) { addresses := []string{} - for _, addr := range s.b.GetActiveValidatorAddresses() { + for _, addr := range s.b.GetElectedValidatorAddresses() { oneAddr, _ := internal_common.AddressToBech32(addr) addresses = append(addresses, oneAddr) } @@ -573,7 +573,7 @@ func (s *PublicBlockChainAPI) GetValidatorMetrics(ctx context.Context, address s } // GetValidatorInformation returns information about a validator. -func (s *PublicBlockChainAPI) GetValidatorInformation(ctx context.Context, address string) (*staking.Validator, error) { +func (s *PublicBlockChainAPI) GetValidatorInformation(ctx context.Context, address string) (*staking.ValidatorWrapper, error) { validatorAddress := internal_common.ParseAddr(address) validator := s.b.GetValidatorInformation(validatorAddress) if validator == nil { @@ -585,13 +585,13 @@ func (s *PublicBlockChainAPI) GetValidatorInformation(ctx context.Context, addre // GetAllValidatorInformation returns information about all validators. // If page is -1, return all instead of `validatorsPageSize` elements. -func (s *PublicBlockChainAPI) GetAllValidatorInformation(ctx context.Context, page int) ([]*staking.Validator, error) { +func (s *PublicBlockChainAPI) GetAllValidatorInformation(ctx context.Context, page int) ([]*staking.ValidatorWrapper, error) { if page < -1 { - return make([]*staking.Validator, 0), nil + return make([]*staking.ValidatorWrapper, 0), nil } addresses := s.b.GetAllValidatorAddresses() if page != -1 && len(addresses) <= page*validatorsPageSize { - return make([]*staking.Validator, 0), nil + return make([]*staking.ValidatorWrapper, 0), nil } validatorsNum := len(addresses) start := 0 @@ -602,7 +602,7 @@ func (s *PublicBlockChainAPI) GetAllValidatorInformation(ctx context.Context, pa validatorsNum = len(addresses) - start } } - validators := make([]*staking.Validator, validatorsNum) + validators := make([]*staking.ValidatorWrapper, validatorsNum) for i := start; i < start+validatorsNum; i++ { validators[i-start] = s.b.GetValidatorInformation(addresses[i]) if validators[i-start] == nil { diff --git a/internal/hmyapi/apiv2/backend.go b/internal/hmyapi/apiv2/backend.go index 013c52ad7..2eaa438f7 100644 --- a/internal/hmyapi/apiv2/backend.go +++ b/internal/hmyapi/apiv2/backend.go @@ -71,9 +71,9 @@ type Backend interface { ResendCx(ctx context.Context, txID common.Hash) (uint64, bool) IsLeader() bool SendStakingTx(ctx context.Context, newStakingTx *staking.StakingTransaction) error - GetActiveValidatorAddresses() []common.Address + GetElectedValidatorAddresses() []common.Address GetAllValidatorAddresses() []common.Address - GetValidatorInformation(addr common.Address) *staking.Validator + GetValidatorInformation(addr common.Address) *staking.ValidatorWrapper GetValidatorStats(addr common.Address) *staking.ValidatorStats GetDelegationsByValidator(validator common.Address) []*staking.Delegation GetDelegationsByDelegator(delegator common.Address) ([]common.Address, []*staking.Delegation) @@ -81,7 +81,7 @@ type Backend interface { GetShardState() (*shard.State, error) GetCurrentStakingErrorSink() []staking.RPCTransactionError GetCurrentTransactionErrorSink() []types.RPCTransactionError - GetMedianRawStakeSnapshot() *big.Int + GetMedianRawStakeSnapshot() (*big.Int, error) GetPendingCXReceipts() []*types.CXReceiptsProof GetCurrentUtilityMetrics() (*network.UtilityMetric, error) GetSuperCommittees() (*quorum.Transition, error) diff --git a/internal/hmyapi/apiv2/blockchain.go b/internal/hmyapi/apiv2/blockchain.go index 79cea8331..32ea7d4d7 100644 --- a/internal/hmyapi/apiv2/blockchain.go +++ b/internal/hmyapi/apiv2/blockchain.go @@ -501,7 +501,7 @@ func (s *PublicBlockChainAPI) GetTotalStaking() (*big.Int, error) { // explorer node func (s *PublicBlockChainAPI) GetMedianRawStakeSnapshot() (*big.Int, error) { if s.b.GetShardID() == shard.BeaconChainShardID { - return s.b.GetMedianRawStakeSnapshot(), nil + return s.b.GetMedianRawStakeSnapshot() } return nil, errNotBeaconChainShard } @@ -516,10 +516,10 @@ func (s *PublicBlockChainAPI) GetAllValidatorAddresses() ([]string, error) { return addresses, nil } -// GetActiveValidatorAddresses returns active validator addresses. -func (s *PublicBlockChainAPI) GetActiveValidatorAddresses() ([]string, error) { +// GetElectedValidatorAddresses returns elected validator addresses. +func (s *PublicBlockChainAPI) GetElectedValidatorAddresses() ([]string, error) { addresses := []string{} - for _, addr := range s.b.GetActiveValidatorAddresses() { + for _, addr := range s.b.GetElectedValidatorAddresses() { oneAddr, _ := internal_common.AddressToBech32(addr) addresses = append(addresses, oneAddr) } @@ -538,7 +538,7 @@ func (s *PublicBlockChainAPI) GetValidatorMetrics(ctx context.Context, address s } // GetValidatorInformation returns information about a validator. -func (s *PublicBlockChainAPI) GetValidatorInformation(ctx context.Context, address string) (*staking.Validator, error) { +func (s *PublicBlockChainAPI) GetValidatorInformation(ctx context.Context, address string) (*staking.ValidatorWrapper, error) { validatorAddress := internal_common.ParseAddr(address) validator := s.b.GetValidatorInformation(validatorAddress) if validator == nil { @@ -550,13 +550,13 @@ func (s *PublicBlockChainAPI) GetValidatorInformation(ctx context.Context, addre // GetAllValidatorInformation returns information about all validators. // If page is -1, return all else return the pagination. -func (s *PublicBlockChainAPI) GetAllValidatorInformation(ctx context.Context, page int) ([]*staking.Validator, error) { +func (s *PublicBlockChainAPI) GetAllValidatorInformation(ctx context.Context, page int) ([]*staking.ValidatorWrapper, error) { if page < -1 { - return make([]*staking.Validator, 0), nil + return make([]*staking.ValidatorWrapper, 0), nil } addresses := s.b.GetAllValidatorAddresses() if page != -1 && len(addresses) <= page*validatorsPageSize { - return make([]*staking.Validator, 0), nil + return make([]*staking.ValidatorWrapper, 0), nil } validatorsNum := len(addresses) start := 0 @@ -567,7 +567,7 @@ func (s *PublicBlockChainAPI) GetAllValidatorInformation(ctx context.Context, pa validatorsNum = len(addresses) - start } } - validators := make([]*staking.Validator, validatorsNum) + validators := make([]*staking.ValidatorWrapper, validatorsNum) for i := start; i < start+validatorsNum; i++ { validators[i-start] = s.b.GetValidatorInformation(addresses[i]) if validators[i-start] == nil { diff --git a/internal/hmyapi/backend.go b/internal/hmyapi/backend.go index d6fcb0023..1781e33bf 100644 --- a/internal/hmyapi/backend.go +++ b/internal/hmyapi/backend.go @@ -73,9 +73,9 @@ type Backend interface { ResendCx(ctx context.Context, txID common.Hash) (uint64, bool) IsLeader() bool SendStakingTx(ctx context.Context, newStakingTx *staking.StakingTransaction) error - GetActiveValidatorAddresses() []common.Address + GetElectedValidatorAddresses() []common.Address GetAllValidatorAddresses() []common.Address - GetValidatorInformation(addr common.Address) *staking.Validator + GetValidatorInformation(addr common.Address) *staking.ValidatorWrapper GetValidatorStats(addr common.Address) *staking.ValidatorStats GetDelegationsByValidator(validator common.Address) []*staking.Delegation GetDelegationsByDelegator(delegator common.Address) ([]common.Address, []*staking.Delegation) @@ -83,7 +83,7 @@ type Backend interface { GetShardState() (*shard.State, error) GetCurrentStakingErrorSink() []staking.RPCTransactionError GetCurrentTransactionErrorSink() []types.RPCTransactionError - GetMedianRawStakeSnapshot() *big.Int + GetMedianRawStakeSnapshot() (*big.Int, error) GetPendingCXReceipts() []*types.CXReceiptsProof GetCurrentUtilityMetrics() (*network.UtilityMetric, error) GetSuperCommittees() (*quorum.Transition, error) diff --git a/internal/params/config.go b/internal/params/config.go index 2a8dedcc1..0f776058d 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -37,7 +37,7 @@ var ( TestnetChainConfig = &ChainConfig{ ChainID: TestnetChainID, CrossTxEpoch: big.NewInt(0), - CrossLinkEpoch: big.NewInt(3), + CrossLinkEpoch: big.NewInt(4), StakingEpoch: big.NewInt(4), PreStakingEpoch: big.NewInt(2), EIP155Epoch: big.NewInt(0), @@ -50,7 +50,7 @@ var ( PangaeaChainConfig = &ChainConfig{ ChainID: PangaeaChainID, CrossTxEpoch: big.NewInt(0), - CrossLinkEpoch: big.NewInt(2), + CrossLinkEpoch: big.NewInt(3), StakingEpoch: big.NewInt(3), PreStakingEpoch: big.NewInt(0), EIP155Epoch: big.NewInt(0), @@ -62,9 +62,9 @@ var ( LocalnetChainConfig = &ChainConfig{ ChainID: TestnetChainID, CrossTxEpoch: big.NewInt(0), - CrossLinkEpoch: big.NewInt(3), - StakingEpoch: big.NewInt(4), - PreStakingEpoch: big.NewInt(2), + CrossLinkEpoch: big.NewInt(2), + StakingEpoch: big.NewInt(2), + PreStakingEpoch: big.NewInt(0), EIP155Epoch: big.NewInt(0), S3Epoch: big.NewInt(0), ReceiptLogEpoch: big.NewInt(0), diff --git a/internal/params/protocol_params.go b/internal/params/protocol_params.go index 5d087b67f..77cc24b19 100644 --- a/internal/params/protocol_params.go +++ b/internal/params/protocol_params.go @@ -7,7 +7,8 @@ const ( MinGasLimit uint64 = 5000 // Minimum the gas limit may ever be. // GenesisGasLimit ... GenesisGasLimit uint64 = 4712388 // Gas limit of the Genesis block. - + // TestGenesisGasLimit .. + TestGenesisGasLimit uint64 = 10712388 // A Gas limit in testing of the Genesis block. // MaximumExtraDataSize ... MaximumExtraDataSize uint64 = 32 // Maximum size extra data may be after Genesis. // ExpByteGas ... diff --git a/internal/utils/singleton.go b/internal/utils/singleton.go index 19f259067..a71c84a69 100644 --- a/internal/utils/singleton.go +++ b/internal/utils/singleton.go @@ -4,7 +4,6 @@ package utils import ( "fmt" - "io" "os" "path" "sync" @@ -13,6 +12,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/natefinch/lumberjack" "github.com/rs/zerolog" + "github.com/rs/zerolog/diode" ) var ( @@ -73,7 +73,10 @@ func AddLogHandler(handler log.Handler) { // GetLogInstance returns logging singleton. func GetLogInstance() log.Logger { onceForLog.Do(func() { - ostream := log.StreamHandler(io.Writer(os.Stdout), log.TerminalFormat(false)) + writer := diode.NewWriter(os.Stdout, 1000, 10*time.Millisecond, func(missed int) { + fmt.Printf("Logger Dropped %d messages", missed) + }) + ostream := log.StreamHandler(writer, log.TerminalFormat(false)) logHandlers = append(logHandlers, ostream) multiHandler := log.MultiHandler(logHandlers...) glogger = log.NewGlogHandler(multiHandler) @@ -118,7 +121,10 @@ func setZeroLoggerFileOutput(filepath string, maxSize int) error { func Logger() *zerolog.Logger { if zeroLogger == nil { zerolog.TimeFieldFormat = time.RFC3339Nano - logger := zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr}). + writer := diode.NewWriter(os.Stderr, 1000, 10*time.Millisecond, func(missed int) { + fmt.Printf("Logger Dropped %d messages", missed) + }) + logger := zerolog.New(zerolog.ConsoleWriter{Out: writer}). Level(zeroLoggerLevel). With(). Caller(). diff --git a/node/node_double_signing.go b/node/double_signing.go similarity index 53% rename from node/node_double_signing.go rename to node/double_signing.go index 1d777de2f..02e193ee7 100644 --- a/node/node_double_signing.go +++ b/node/double_signing.go @@ -12,13 +12,20 @@ func (node *Node) processSlashCandidateMessage(msgPayload []byte) { if node.NodeConfig.ShardID != shard.BeaconChainShardID { return } + candidates, e := slash.Records{}, utils.Logger().Error() - candidate := slash.Record{} - if err := rlp.DecodeBytes(msgPayload, &candidate); err != nil { - utils.Logger().Error(). - Err(err). + if err := rlp.DecodeBytes(msgPayload, &candidates); err != nil { + e.Err(err). Msg("unable to decode slash candidate message") return } - node.Blockchain().AddPendingSlashingCandidate(&candidate) + + if err := candidates.SanityCheck(); err != nil { + e.Err(err). + RawJSON("slash-candidates", []byte(candidates.String())). + Msg("sanity check failed on incoming candidates") + return + } + + node.Blockchain().AddPendingSlashingCandidates(candidates) } diff --git a/node/node.go b/node/node.go index b1af3b5db..faacc6825 100644 --- a/node/node.go +++ b/node/node.go @@ -36,6 +36,7 @@ import ( p2p_host "github.com/harmony-one/harmony/p2p/host" "github.com/harmony-one/harmony/shard" "github.com/harmony-one/harmony/shard/committee" + "github.com/harmony-one/harmony/staking/slash" staking "github.com/harmony-one/harmony/staking/types" ) @@ -577,14 +578,36 @@ func New(host p2p.Host, consensusObj *consensus.Consensus, // Broadcast double-signers reported by consensus if node.Consensus != nil { go func() { + for { select { case doubleSign := <-node.Consensus.SlashChan: - go node.BroadcastSlash(&doubleSign) + l := utils.Logger().Info().RawJSON("double-sign", []byte(doubleSign.String())) + + // no point to broadcast the slash if we aren't even in the right epoch yet + if !node.Blockchain().Config().IsStaking( + node.Blockchain().CurrentHeader().Epoch(), + ) { + l.Msg("double sign occured before staking era, no-op") + return + } + if hooks := node.NodeConfig.WebHooks.DoubleSigning; hooks != nil { + url := hooks.WebHooks.OnNoticeDoubleSign + go func() { slash.DoPost(url, &doubleSign) }() + } + if node.NodeConfig.ShardID != shard.BeaconChainShardID { + go node.BroadcastSlash(&doubleSign) + l.Msg("broadcast the double sign record") + } else { + records := slash.Records{doubleSign} + node.Blockchain().AddPendingSlashingCandidates(records) + l.Msg("added double sign record to off-chain pending") + } } } }() } + return &node } diff --git a/node/node_cross_link.go b/node/node_cross_link.go index cb0096aa1..b601c19f8 100644 --- a/node/node_cross_link.go +++ b/node/node_cross_link.go @@ -89,21 +89,23 @@ func (node *Node) ProcessCrossLinkMessage(msgPayload []byte) { } exist, err := node.Blockchain().ReadCrossLink(cl.ShardID(), cl.Number().Uint64()) if err == nil && exist != nil { - // TODO: leader add double sign checking utils.Logger().Err(err). Msgf("[ProcessingCrossLink] Cross Link already exists, pass. Beacon Epoch: %d, Block num: %d, Epoch: %d, shardID %d", node.Blockchain().CurrentHeader().Epoch(), cl.Number(), cl.Epoch(), cl.ShardID()) continue } if err = node.VerifyCrossLink(cl); err != nil { - utils.Logger().Err(err). + utils.Logger().Info(). + Str("cross-link-issue", err.Error()). Msgf("[ProcessingCrossLink] Failed to verify new cross link for blockNum %d epochNum %d shard %d skipped: %v", cl.BlockNum(), cl.Epoch().Uint64(), cl.ShardID(), cl) continue } candidates = append(candidates, cl) utils.Logger().Debug(). - Msgf("[ProcessingCrossLink] Committing for shardID %d, blockNum %d", cl.ShardID(), cl.Number().Uint64()) + Msgf("[ProcessingCrossLink] Committing for shardID %d, blockNum %d", + cl.ShardID(), cl.Number().Uint64(), + ) } Len, _ := node.Blockchain().AddPendingCrossLinks(candidates) utils.Logger().Debug(). @@ -167,7 +169,7 @@ func (node *Node) VerifyCrossLink(cl types.CrossLink) error { if _, err := decider.SetVoters(committee.Slots); err != nil { return ctxerror.New("[VerifyCrossLink] Cannot SetVoters for committee", "shardID", cl.ShardID()) } - if !decider.IsQuorumAchievedByMask(mask, false) { + if !decider.IsQuorumAchievedByMask(mask) { return ctxerror.New("[VerifyCrossLink] Not enough voting power for crosslink", "shardID", cl.ShardID()) } diff --git a/node/node_cross_shard.go b/node/node_cross_shard.go index 171ed6f3a..a2138345e 100644 --- a/node/node_cross_shard.go +++ b/node/node_cross_shard.go @@ -46,11 +46,17 @@ func (node *Node) BroadcastCXReceipts(newBlock *types.Block, lastCommits []byte) // BroadcastCXReceiptsWithShardID broadcasts cross shard receipts to given ToShardID func (node *Node) BroadcastCXReceiptsWithShardID(block *types.Block, commitSig []byte, commitBitmap []byte, toShardID uint32) { myShardID := node.Consensus.ShardID - utils.Logger().Info().Uint32("toShardID", toShardID).Uint32("myShardID", myShardID).Uint64("blockNum", block.NumberU64()).Msg("[BroadcastCXReceiptsWithShardID]") + utils.Logger().Info(). + Uint32("toShardID", toShardID). + Uint32("myShardID", myShardID). + Uint64("blockNum", block.NumberU64()). + Msg("[BroadcastCXReceiptsWithShardID]") cxReceipts, err := node.Blockchain().ReadCXReceipts(toShardID, block.NumberU64(), block.Hash()) if err != nil || len(cxReceipts) == 0 { - utils.Logger().Info().Err(err).Uint32("ToShardID", toShardID).Int("numCXReceipts", len(cxReceipts)).Msg("[CXMerkleProof] No receipts found for the destination shard") + utils.Logger().Info().Uint32("ToShardID", toShardID). + Int("numCXReceipts", len(cxReceipts)). + Msg("[CXMerkleProof] No receipts found for the destination shard") return } diff --git a/node/node_explorer.go b/node/node_explorer.go index f9ae2124b..5e4d483c2 100644 --- a/node/node_explorer.go +++ b/node/node_explorer.go @@ -47,7 +47,7 @@ func (node *Node) ExplorerMessageHandler(payload []byte) { return } - if !node.Consensus.Decider.IsQuorumAchievedByMask(mask, false) { + if !node.Consensus.Decider.IsQuorumAchievedByMask(mask) { utils.Logger().Error().Msg("[Explorer] not have enough signature power") return } @@ -150,7 +150,7 @@ func (node *Node) commitBlockForExplorer(block *types.Block) { func (node *Node) GetTransactionsHistory(address, txType, order string) ([]common.Hash, error) { addressData := &explorer.Address{} key := explorer.GetAddressKey(address) - bytes, err := explorer.GetStorageInstance(node.SelfPeer.IP, node.SelfPeer.Port, false).GetDB().Get([]byte(key)) + bytes, err := explorer.GetStorageInstance(node.SelfPeer.IP, node.SelfPeer.Port, false).GetDB().Get([]byte(key), nil) if err != nil { return make([]common.Hash, 0), nil } diff --git a/node/node_genesis.go b/node/node_genesis.go index 374a2239f..25af4b276 100644 --- a/node/node_genesis.go +++ b/node/node_genesis.go @@ -75,6 +75,7 @@ func (node *Node) SetupGenesisBlock(db ethdb.Database, shardID uint32, myShardSt genesisAlloc := make(core.GenesisAlloc) chainConfig := *params.TestnetChainConfig + gasLimit := params.GenesisGasLimit switch node.NodeConfig.GetNetworkType() { case nodeconfig.Mainnet: @@ -89,7 +90,7 @@ func (node *Node) SetupGenesisBlock(db ethdb.Database, shardID uint32, myShardSt default: // all other types share testnet config // Test accounts node.AddTestingAddresses(genesisAlloc, TestAccountNumber) - + gasLimit = params.TestGenesisGasLimit // Smart contract deployer account used to deploy initial smart contract contractDeployerKey, _ := ecdsa.GenerateKey(crypto.S256(), strings.NewReader("Test contract key string stream that is fixed so that generated test key are deterministic every time")) contractDeployerAddress := crypto.PubkeyToAddress(contractDeployerKey.PublicKey) @@ -104,7 +105,7 @@ func (node *Node) SetupGenesisBlock(db ethdb.Database, shardID uint32, myShardSt Factory: blockfactory.NewFactory(&chainConfig), Alloc: genesisAlloc, ShardID: shardID, - GasLimit: params.GenesisGasLimit, + GasLimit: gasLimit, ShardStateHash: myShardState.Hash(), ShardState: *myShardState.DeepCopy(), Timestamp: 1561734000, // GMT: Friday, June 28, 2019 3:00:00 PM. PST: Friday, June 28, 2019 8:00:00 AM diff --git a/node/node_handler.go b/node/node_handler.go index cca72200e..866b3e6f7 100644 --- a/node/node_handler.go +++ b/node/node_handler.go @@ -154,7 +154,8 @@ func (node *Node) HandleMessage(content []byte, sender libp2p_peer.ID) { Msg("block sync") } else { // for non-beaconchain node, subscribe to beacon block broadcast - if node.Blockchain().ShardID() != 0 && node.NodeConfig.Role() != nodeconfig.ExplorerNode { + if node.Blockchain().ShardID() != shard.BeaconChainShardID && + node.NodeConfig.Role() != nodeconfig.ExplorerNode { for _, block := range blocks { if block.ShardID() == 0 { utils.Logger().Info(). @@ -241,23 +242,15 @@ func (node *Node) BroadcastNewBlock(newBlock *types.Block) { // BroadcastSlash .. func (node *Node) BroadcastSlash(witness *slash.Record) { - // no point to broadcast the crosslink if we aren't even in the right epoch yet - if !node.Blockchain().Config().IsCrossLink( - node.Blockchain().CurrentHeader().Epoch(), - ) { - return - } - - // Send it to beaconchain if I'm shardchain, otherwise just add it to pending - if node.NodeConfig.ShardID != shard.BeaconChainShardID { - node.host.SendMessageToGroups( - []nodeconfig.GroupID{nodeconfig.NewGroupIDByShardID(shard.BeaconChainShardID)}, - host.ConstructP2pMessage( - byte(0), - proto_node.ConstructSlashMessage(witness)), - ) - } else { - node.Blockchain().AddPendingSlashingCandidate(witness) + if err := node.host.SendMessageToGroups( + []nodeconfig.GroupID{nodeconfig.NewGroupIDByShardID(shard.BeaconChainShardID)}, + host.ConstructP2pMessage( + byte(0), + proto_node.ConstructSlashMessage(slash.Records{*witness})), + ); err != nil { + utils.Logger().Err(err). + RawJSON("record", []byte(witness.String())). + Msg("could not send slash record to beaconchain") } } @@ -322,8 +315,7 @@ func (node *Node) BroadcastCrossLink(newBlock *types.Block) { // VerifyNewBlock is called by consensus participants to verify the block (account model) they are // running consensus on func (node *Node) VerifyNewBlock(newBlock *types.Block) error { - err := node.Blockchain().Validator().ValidateHeader(newBlock, true) - if err != nil { + if err := node.Blockchain().Validator().ValidateHeader(newBlock, true); err != nil { utils.Logger().Error(). Str("blockHash", newBlock.Hash().Hex()). Err(err). @@ -334,6 +326,7 @@ func (node *Node) VerifyNewBlock(newBlock *types.Block) error { newBlock.Hash(), ).WithCause(err) } + if newBlock.ShardID() != node.Blockchain().ShardID() { utils.Logger().Error(). Uint32("my shard ID", node.Blockchain().ShardID()). @@ -341,13 +334,13 @@ func (node *Node) VerifyNewBlock(newBlock *types.Block) error { Msg("wrong shard ID") return ctxerror.New("wrong shard ID", "my shard ID", node.Blockchain().ShardID(), - "new block's shard ID", newBlock.ShardID()) + "new block's shard ID", newBlock.ShardID(), + ) } - err = node.Blockchain().Engine().VerifyShardState( + if err := node.Blockchain().Engine().VerifyShardState( node.Blockchain(), node.Beaconchain(), newBlock.Header(), - ) - if err != nil { + ); err != nil { utils.Logger().Error(). Str("blockHash", newBlock.Hash().Hex()). Err(err). @@ -358,8 +351,7 @@ func (node *Node) VerifyNewBlock(newBlock *types.Block) error { ).WithCause(err) } - err = node.Blockchain().ValidateNewBlock(newBlock) - if err != nil { + if err := node.Blockchain().ValidateNewBlock(newBlock); err != nil { utils.Logger().Error(). Str("blockHash", newBlock.Hash().Hex()). Int("numTx", len(newBlock.Transactions())). @@ -374,7 +366,7 @@ func (node *Node) VerifyNewBlock(newBlock *types.Block) error { // Verify cross links // TODO: move into ValidateNewBlock - if node.NodeConfig.ShardID == 0 { + if node.NodeConfig.ShardID == shard.BeaconChainShardID { err := node.VerifyBlockCrossLinks(newBlock) if err != nil { utils.Logger().Debug().Err(err).Msg("ops2 VerifyBlockCrossLinks Failed") @@ -383,8 +375,7 @@ func (node *Node) VerifyNewBlock(newBlock *types.Block) error { } // TODO: move into ValidateNewBlock - err = node.verifyIncomingReceipts(newBlock) - if err != nil { + if err := node.verifyIncomingReceipts(newBlock); err != nil { utils.Logger().Error(). Str("blockHash", newBlock.Hash().Hex()). Int("numIncomingReceipts", len(newBlock.IncomingReceipts())). diff --git a/node/node_handler_test.go b/node/node_handler_test.go index 21ec9fb1d..2bf7188b2 100644 --- a/node/node_handler_test.go +++ b/node/node_handler_test.go @@ -41,7 +41,7 @@ func TestAddNewBlock(t *testing.T) { txs, stks, common.Address{}, ) block, _ := node.Worker.FinalizeNewBlock( - []byte{}, []byte{}, 0, common.Address{}, nil, nil, nil, + []byte{}, []byte{}, 0, common.Address{}, nil, nil, ) _, err = node.Blockchain().InsertChain([]*types.Block{block}, true) @@ -78,7 +78,7 @@ func TestVerifyNewBlock(t *testing.T) { txs, stks, common.Address{}, ) block, _ := node.Worker.FinalizeNewBlock( - []byte{}, []byte{}, 0, common.Address{}, nil, nil, nil, + []byte{}, []byte{}, 0, common.Address{}, nil, nil, ) if err := node.VerifyNewBlock(block); err != nil { diff --git a/node/node_newblock.go b/node/node_newblock.go index 94f88ac12..5084bbd42 100644 --- a/node/node_newblock.go +++ b/node/node_newblock.go @@ -10,13 +10,12 @@ import ( "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/shard" - "github.com/harmony-one/harmony/staking/slash" staking "github.com/harmony-one/harmony/staking/types" ) // Constants of proposing a new block const ( - PeriodicBlock = 200 * time.Millisecond + PeriodicBlock = 20 * time.Millisecond IncomingReceiptsLimit = 6000 // 2000 * (numShards - 1) ) @@ -49,6 +48,9 @@ func (node *Node) WaitForConsensusReadyV2(readySignal chan struct{}, stopChan ch continue } + // Start counting for block time before block proposal. + tmpDeadline := time.Now().Add(node.BlockPeriod) + utils.Logger().Debug(). Uint64("blockNum", node.Blockchain().CurrentBlock().NumberU64()+1). Msg("PROPOSING NEW BLOCK ------------------------------------------------") @@ -62,8 +64,9 @@ func (node *Node) WaitForConsensusReadyV2(readySignal chan struct{}, stopChan ch Int("crossShardReceipts", newBlock.IncomingReceipts().Len()). Msg("=========Successfully Proposed New Block==========") - // Set deadline will be BlockPeriod from now at this place. Announce stage happens right after this. - deadline = time.Now().Add(node.BlockPeriod) + // Set deadline only if block proposal is successful, otherwise, we should + // immediately start retrying block proposal + deadline = tmpDeadline // Send the new block to Consensus so it can be confirmed. node.BlockChannel <- newBlock break @@ -121,7 +124,8 @@ func (node *Node) proposeNewBlock() (*types.Block, error) { pendingStakingTxs = append(pendingStakingTxs, stakingTx) } } else { - utils.Logger().Err(types.ErrUnknownPoolTxType).Msg("Failed to parse pending transactions") + utils.Logger().Err(types.ErrUnknownPoolTxType). + Msg("Failed to parse pending transactions") return nil, types.ErrUnknownPoolTxType } } @@ -145,30 +149,52 @@ func (node *Node) proposeNewBlock() (*types.Block, error) { } } - // Prepare cross links - var ( - slashingToPropose []slash.Record - crossLinksToPropose types.CrossLinks - ) + // Prepare cross links and slashings messages + var crossLinksToPropose types.CrossLinks - if node.NodeConfig.ShardID == shard.BeaconChainShardID && - node.Blockchain().Config().IsCrossLink(node.Worker.GetCurrentHeader().Epoch()) { - allPending, err := node.Blockchain().ReadPendingCrossLinks() + isBeaconchainInCrossLinkEra := node.NodeConfig.ShardID == shard.BeaconChainShardID && + node.Blockchain().Config().IsCrossLink(node.Worker.GetCurrentHeader().Epoch()) + + isBeaconchainInStakingEra := node.NodeConfig.ShardID == shard.BeaconChainShardID && + node.Blockchain().Config().IsStaking(node.Worker.GetCurrentHeader().Epoch()) + if isBeaconchainInCrossLinkEra { + allPending, err := node.Blockchain().ReadPendingCrossLinks() + invalidToDelete := []types.CrossLink{} if err == nil { for _, pending := range allPending { - if err = node.VerifyCrossLink(pending); err != nil { - continue - } exist, err := node.Blockchain().ReadCrossLink(pending.ShardID(), pending.BlockNum()) if err == nil || exist != nil { + invalidToDelete = append(invalidToDelete, pending) + utils.Logger().Debug(). + AnErr("[proposeNewBlock] pending crosslink is already committed onchain", err) + continue + } + if err = node.VerifyCrossLink(pending); err != nil { + invalidToDelete = append(invalidToDelete, pending) + utils.Logger().Debug(). + AnErr("[proposeNewBlock] pending crosslink verification failed", err) continue } crossLinksToPropose = append(crossLinksToPropose, pending) } - utils.Logger().Debug().Msgf("[proposeNewBlock] Proposed %d crosslinks from %d pending crosslinks", len(crossLinksToPropose), len(allPending)) + utils.Logger().Debug(). + Msgf("[proposeNewBlock] Proposed %d crosslinks from %d pending crosslinks", + len(crossLinksToPropose), len(allPending), + ) } else { - utils.Logger().Error().Err(err).Msgf("[proposeNewBlock] Unable to Read PendingCrossLinks, number of crosslinks: %d", len(allPending)) + utils.Logger().Error().Err(err).Msgf( + "[proposeNewBlock] Unable to Read PendingCrossLinks, number of crosslinks: %d", + len(allPending), + ) + } + node.Blockchain().DeleteFromPendingCrossLinks(invalidToDelete) + } + + if isBeaconchainInStakingEra { + // this one will set a meaningful w.current.slashes + if err := node.Worker.CollectAndVerifySlashes(); err != nil { + return nil, err } } @@ -186,10 +212,10 @@ func (node *Node) proposeNewBlock() (*types.Block, error) { utils.Logger().Error().Err(err).Msg("[proposeNewBlock] Cannot get commit signatures from last block") return nil, err } + return node.Worker.FinalizeNewBlock( sig, mask, node.Consensus.GetViewID(), coinbase, crossLinksToPropose, shardState, - slashingToPropose, ) } diff --git a/node/service_setup.go b/node/service_setup.go index 1818481df..caf472423 100644 --- a/node/service_setup.go +++ b/node/service_setup.go @@ -72,7 +72,7 @@ func (node *Node) setupForExplorerNode() { // Register networkinfo service. node.serviceManager.RegisterService(service.NetworkInfo, networkinfo.MustNew(node.host, node.NodeConfig.GetShardGroupID(), chanPeer, nil, node.networkInfoDHTPath())) // Register explorer service. - node.serviceManager.RegisterService(service.SupportExplorer, explorer.New(&node.SelfPeer, node.NodeConfig.GetShardID(), node.Consensus.GetNodeIDs, node.GetBalanceOfAddress)) + node.serviceManager.RegisterService(service.SupportExplorer, explorer.New(&node.SelfPeer)) // Register explorer service. } diff --git a/node/worker/worker.go b/node/worker/worker.go index 4dd6a9751..29df5449b 100644 --- a/node/worker/worker.go +++ b/node/worker/worker.go @@ -33,10 +33,11 @@ type environment struct { gasPool *core.GasPool // available gas used to pack transactions header *block.Header txs []*types.Transaction - stakingTxs staking.StakingTransactions + stakingTxs []*staking.StakingTransaction receipts []*types.Receipt outcxs []*types.CXReceipt // cross shard transaction receipts (source shard) incxs []*types.CXReceiptsProof // cross shard receipts and its proof (desitinatin shard) + slashes slash.Records } // Worker is the main object which takes care of submitting new work to consensus engine @@ -208,7 +209,9 @@ func (w *Worker) CommitReceipts(receiptsList []*types.CXReceiptsProof) error { if len(receiptsList) == 0 { w.current.header.SetIncomingReceiptHash(types.EmptyRootHash) } else { - w.current.header.SetIncomingReceiptHash(types.DeriveSha(types.CXReceiptsProofs(receiptsList))) + w.current.header.SetIncomingReceiptHash( + types.DeriveSha(types.CXReceiptsProofs(receiptsList)), + ) } for _, cx := range receiptsList { @@ -273,8 +276,11 @@ func (w *Worker) GetNewEpoch() *big.Int { epoch := new(big.Int).Set(parent.Header().Epoch()) shardState, err := parent.Header().GetShardState() - if err == nil && shardState.Epoch != nil && w.config.IsStaking(shardState.Epoch) { - // For shard state of staking epochs, the shard state will have an epoch and it will decide the next epoch for following blocks + if err == nil && + shardState.Epoch != nil && + w.config.IsStaking(shardState.Epoch) { + // For shard state of staking epochs, the shard state will + // have an epoch and it will decide the next epoch for following blocks epoch = new(big.Int).Set(shardState.Epoch) } else { if len(parent.Header().ShardState()) > 0 && parent.NumberU64() != 0 { @@ -300,10 +306,56 @@ func (w *Worker) IncomingReceipts() []*types.CXReceiptsProof { return w.current.incxs } +// CollectAndVerifySlashes .. +func (w *Worker) CollectAndVerifySlashes() error { + allSlashing, err := w.chain.ReadPendingSlashingCandidates() + if err != nil { + return err + } + if d := allSlashing; len(d) > 0 { + // TODO add specific error which is + // "could not verify slash", which should not return as err + // and therefore stop the block proposal + if allSlashing, err = w.VerifyAll(d); err != nil { + utils.Logger().Err(err). + RawJSON("slashes", []byte(d.String())). + Msg("could not verify slashes proposed") + return err + } + } + w.current.slashes = allSlashing + return nil +} + +// VerifyAll .. +func (w *Worker) VerifyAll(allSlashing []slash.Record) ([]slash.Record, error) { + d := allSlashing + slashingToPropose := []slash.Record{} + // Enforce order, reproducibility + sort.SliceStable(d, + func(i, j int) bool { + return bytes.Compare( + d[i].Reporter.Bytes(), d[j].Reporter.Bytes(), + ) == -1 + }, + ) + + for i := range d { + if err := slash.Verify(w.chain, &d[i]); err != nil { + return nil, err + } + slashingToPropose = append(slashingToPropose, d[i]) + } + count := len(slashingToPropose) + utils.Logger().Info(). + Msgf("set into propose headers %d slashing record", count) + return slashingToPropose, nil +} + // FinalizeNewBlock generate a new block for the next consensus round. func (w *Worker) FinalizeNewBlock( sig []byte, signers []byte, viewID uint64, coinbase common.Address, - crossLinks types.CrossLinks, shardState *shard.State, doubleSigners []slash.Record, + crossLinks types.CrossLinks, shardState *shard.State, ) (*types.Block, error) { if len(sig) > 0 && len(signers) > 0 { sig2 := w.current.header.LastCommitSignature() @@ -314,23 +366,8 @@ func (w *Worker) FinalizeNewBlock( w.current.header.SetCoinbase(coinbase) w.current.header.SetViewID(new(big.Int).SetUint64(viewID)) - // Slashes - if d := doubleSigners; d != nil && len(d) != 0 { - // Enforce order, reproducibility - sort.SliceStable(d, - func(i, j int) bool { - return bytes.Compare( - d[i].Beneficiary.Bytes(), d[j].Beneficiary.Bytes(), - ) == -1 - }, - ) - if rlpBytes, err := rlp.EncodeToBytes(d); err == nil { - w.current.header.SetSlashes(rlpBytes) - } - } - // Cross Links - if crossLinks != nil && len(crossLinks) != 0 { + if len(crossLinks) > 0 { crossLinks.Sort() crossLinkData, err := rlp.EncodeToBytes(crossLinks) if err == nil { @@ -347,6 +384,21 @@ func (w *Worker) FinalizeNewBlock( utils.Logger().Debug().Msg("Zero crosslinks to finalize") } + if w.config.IsStaking(w.current.header.Epoch()) { + doubleSigners := w.current.slashes + if len(doubleSigners) > 0 { + if data, err := rlp.EncodeToBytes(doubleSigners); err == nil { + w.current.header.SetSlashes(data) + utils.Logger().Info(). + RawJSON("slashes", []byte(doubleSigners.String())). + Msg("encoded slashes into headers of proposed new block") + } else { + utils.Logger().Debug().Err(err).Msg("Failed to encode proposed slashes") + return nil, err + } + } + } + // Shard State if shardState != nil && len(shardState.Shards) != 0 { //we store shardstatehash in header only before prestaking epoch (header v0,v1,v2) @@ -373,6 +425,7 @@ func (w *Worker) FinalizeNewBlock( block, _, err := w.engine.Finalize( w.chain, copyHeader, state, w.current.txs, w.current.receipts, w.current.outcxs, w.current.incxs, w.current.stakingTxs, + w.current.slashes, ) if err != nil { return nil, ctxerror.New("cannot finalize block").WithCause(err) diff --git a/scripts/go_executable_build.sh b/scripts/go_executable_build.sh index 089036adb..02462f55b 100755 --- a/scripts/go_executable_build.sh +++ b/scripts/go_executable_build.sh @@ -137,6 +137,10 @@ function build_only done $MD5 "${!SRC[@]}" "${!LIB[@]}" > md5sum.txt + # hardcode the prebuilt libcrypto to md5sum.txt + if [ "$(uname -s)" == "Linux" ]; then + echo '771150db04267126823190c873a96e48 libcrypto.so.10' >> md5sum.txt + fi fi popd } diff --git a/scripts/node.sh b/scripts/node.sh index 29cc5dbf6..0a9403f2a 100755 --- a/scripts/node.sh +++ b/scripts/node.sh @@ -125,6 +125,7 @@ options: -M support multi-key mode (default: off) -A enable archival node mode (default: off) -B blacklist specify file containing blacklisted accounts as a newline delimited file (default: ./.hmy/blacklist.txt) + -I use statically linked Harmony binary examples: @@ -177,12 +178,13 @@ staking_mode=false multi_key=false archival=false blacklist=./.hmy/blacklist.txt +static=false verify=false ${BLSKEYFILE=} unset OPTIND OPTARG opt OPTIND=1 -while getopts :1chk:sSp:dDmN:tT:i:ba:U:PvVyzn:MAB:Y opt +while getopts :1chk:sSp:dDmN:tT:i:ba:U:PvVyzn:MAIB:Y opt do case "${opt}" in '?') usage "unrecognized option -${OPTARG}";; @@ -204,6 +206,7 @@ do t) network=devnet;; T) node_type="${OPTARG}";; i) shard_id="${OPTARG}";; + I) static=true;; a) db_file_to_dl="${OPTARG}";; U) upgrade_rel="${OPTARG}";; P) public_rpc=true;; @@ -289,6 +292,9 @@ if [ "$OS" == "Darwin" ]; then fi if [ "$OS" == "Linux" ]; then FOLDER=release/linux-x86_64/$REL/ + if [ "$static" == "true" ]; then + FOLDER=release/linux-x86_64/$REL/static/ + fi fi extract_checksum() { diff --git a/shard/committee/assignment.go b/shard/committee/assignment.go index aab8ac74a..981d12e91 100644 --- a/shard/committee/assignment.go +++ b/shard/committee/assignment.go @@ -117,23 +117,25 @@ func eposStakedCommittee( // TODO Nervous about this because overtime the list will become quite large candidates := stakerReader.ValidatorCandidates() essentials := map[common.Address]effective.SlotOrder{} - - utils.Logger().Info().Int("staked-candidates", len(candidates)).Msg("preparing epos staked committee") - + l := utils.Logger().Info().Int("staked-candidates", len(candidates)) + l.Msg("preparing epos staked committee") blsKeys := make(map[shard.BlsPublicKey]struct{}) // TODO benchmark difference if went with data structure that sorts on insert for i := range candidates { validator, err := stakerReader.ReadValidatorInformation(candidates[i]) - if err != nil { return nil, err } - + if !effective.IsEligibleForEPOSAuction(validator) { + l.RawJSON("candidate", []byte(validator.String())). + Msg("validator not eligible for epos") + continue + } if err := validator.SanityCheck(); err != nil { - utils.Logger().Error(). - Str("failure", validator.String()). - Msg("Sanity check of validator failed") + l.Err(err). + RawJSON("candidate", []byte(validator.String())). + Msg("validator sanity check failed") continue } validatorStake := big.NewInt(0) @@ -152,7 +154,8 @@ func eposStakedCommittee( } } if found { - utils.Logger().Info().Msgf("[eposStakedCommittee] Duplicate bls key found %x, in validator %+v. Ignoring", dupKey, validator) + const m = "Duplicate bls key found %x, in validator %+v. Ignoring" + l.Msgf(m, dupKey, validator) continue } @@ -185,7 +188,7 @@ func eposStakedCommittee( } if stakedSlotsCount == 0 { - utils.Logger().Info().Int("slots-for-epos", stakedSlotsCount). + l.Int("slots-for-epos", stakedSlotsCount). Msg("committe composed only of harmony node") return shardState, nil } @@ -212,10 +215,10 @@ func eposStakedCommittee( } if c := len(candidates); c != 0 { - utils.Logger().Info().Int("staked-candidates", c). + l.Int("staked-candidates", c). Str("total-staked-by-validators", totalStake.String()). RawJSON("staked-super-committee", []byte(shardState.String())). - Msg("EPoS based super-committe") + Msg("epos based super-committe") } return shardState, nil diff --git a/shard/shard_state.go b/shard/shard_state.go index f8e8d7676..5ef5383d1 100644 --- a/shard/shard_state.go +++ b/shard/shard_state.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/hex" "encoding/json" + "errors" "math/big" "sort" @@ -55,6 +56,15 @@ type Committee struct { Slots SlotList `json:"subcommittee"` } +func (l SlotList) String() string { + blsKeys := make([]string, len(l)) + for i, k := range l { + blsKeys[i] = k.BlsPublicKey.Hex() + } + s, _ := json.Marshal(blsKeys) + return string(s) +} + /* Legacy These are the pre-staking used data-structures, needed to maintain compatibilty for RLP decode/encode @@ -135,32 +145,84 @@ func EncodeWrapper(shardState State, isStaking bool) ([]byte, error) { return data, err } -// ExternalValidators returns only the staking era, -// external validators aka non-harmony nodes -func (ss *State) ExternalValidators() []common.Address { - processed := make(map[common.Address]struct{}) +// StakedSlots gives overview of subset of shard state that is +// coming via an stake, that is, view epos +type StakedSlots struct { + CountStakedValidator int + CountStakedBLSKey int + Addrs []common.Address + LookupSet map[common.Address]struct{} +} + +// StakedValidators filters for non-harmony operated nodes, +// returns ( +// totalStakedValidatorsCount, totalStakedBLSKeys, +// addrsOnNetworkSlice, addrsOnNetworkSet, +// ) +func (c Committee) StakedValidators() *StakedSlots { + countStakedValidator, countStakedBLSKey := 0, 0 + networkWideSlice, networkWideSet := + []common.Address{}, map[common.Address]struct{}{} + for _, slot := range c.Slots { + + // an external validator, + // non-nil EffectiveStake is how we known + if addr := slot.EcdsaAddress; slot.EffectiveStake != nil { + countStakedBLSKey++ + if _, seen := networkWideSet[addr]; !seen { + countStakedValidator++ + networkWideSet[addr] = struct{}{} + networkWideSlice = append(networkWideSlice, addr) + } + } + } + + return &StakedSlots{ + CountStakedValidator: countStakedValidator, + CountStakedBLSKey: countStakedBLSKey, + Addrs: networkWideSlice, + LookupSet: networkWideSet, + } +} + +// StakedValidators filters for non-harmony operated nodes, +// returns ( +// totalStakedValidatorsCount, totalStakedBLSKeys, +// addrsOnNetworkSlice, addrsOnNetworkSet, +// ) +func (ss *State) StakedValidators() *StakedSlots { + countStakedValidator, countStakedBLSKey := 0, 0 + networkWideSlice, networkWideSet := + []common.Address{}, + map[common.Address]struct{}{} + for i := range ss.Shards { shard := ss.Shards[i] for j := range shard.Slots { - slot := shard.Slots[j] - if slot.EffectiveStake != nil { // For external validator - _, ok := processed[slot.EcdsaAddress] - if !ok { - processed[slot.EcdsaAddress] = struct{}{} + slot := shard.Slots[j] + // an external validator, + // non-nil EffectiveStake is how we known + if addr := slot.EcdsaAddress; slot.EffectiveStake != nil { + countStakedBLSKey++ + if _, seen := networkWideSet[addr]; !seen { + countStakedValidator++ + networkWideSet[addr] = struct{}{} + networkWideSlice = append(networkWideSlice, addr) } } } } - slice, i := make([]common.Address, len(processed)), 0 - for key := range processed { - slice[i] = key - i++ + + return &StakedSlots{ + CountStakedValidator: countStakedValidator, + CountStakedBLSKey: countStakedBLSKey, + Addrs: networkWideSlice, + LookupSet: networkWideSet, } - return slice } -// String .. +// String produces a non-pretty printed JSON string of the SuperCommittee func (ss *State) String() string { s, _ := json.Marshal(ss) return string(s) @@ -312,13 +374,46 @@ func CompareNodeIDList(l1, l2 SlotList) int { } // DeepCopy returns a deep copy of the receiver. -func (c Committee) DeepCopy() Committee { +func (c *Committee) DeepCopy() Committee { r := Committee{} r.ShardID = c.ShardID r.Slots = c.Slots.DeepCopy() return r } +// BLSPublicKeys .. +func (c *Committee) BLSPublicKeys() ([]BlsPublicKey, error) { + if c == nil { + return nil, errCommitteeNil + } + + slice := make([]BlsPublicKey, len(c.Slots)) + for j := range c.Slots { + slice[j] = c.Slots[j].BlsPublicKey + } + return slice, nil +} + +var ( + // ErrValidNotInCommittee .. + ErrValidNotInCommittee = errors.New("slot signer not this slot's subcommittee") + errCommitteeNil = errors.New("subcommittee is nil pointer") +) + +// AddressForBLSKey .. +func (c *Committee) AddressForBLSKey(key BlsPublicKey) (*common.Address, error) { + if c == nil { + return nil, errCommitteeNil + } + + for _, slot := range c.Slots { + if CompareBlsPublicKey(slot.BlsPublicKey, key) == 0 { + return &slot.EcdsaAddress, nil + } + } + return nil, ErrValidNotInCommittee +} + // CompareCommittee compares two committees and their leader/node list. func CompareCommittee(c1, c2 *Committee) int { switch { diff --git a/staking/availability/apply.go b/staking/availability/apply.go deleted file mode 100644 index a2ed6b584..000000000 --- a/staking/availability/apply.go +++ /dev/null @@ -1,56 +0,0 @@ -package availability - -import ( - "github.com/ethereum/go-ethereum/common" - engine "github.com/harmony-one/harmony/consensus/engine" - "github.com/harmony-one/harmony/core/state" - "github.com/harmony-one/harmony/shard" -) - -// Apply .. -func Apply(bc engine.ChainReader, state *state.DB) error { - header := bc.CurrentHeader() - if epoch := header.Epoch(); bc.Config().IsStaking(epoch) { - if header.ShardID() == shard.BeaconChainShardID { - superCommittee, err := bc.ReadShardState(header.Epoch()) - processed := make(map[common.Address]struct{}) - - if err != nil { - return err - } - - for j := range superCommittee.Shards { - shard := superCommittee.Shards[j] - for j := range shard.Slots { - slot := shard.Slots[j] - if slot.EffectiveStake != nil { // For external validator - _, ok := processed[slot.EcdsaAddress] - if !ok { - processed[slot.EcdsaAddress] = struct{}{} - } - } - } - } - - if err := IncrementValidatorSigningCounts( - bc, header, header.ShardID(), state, processed, - ); err != nil { - return err - } - - // // kick out the inactive validators so they won't come up in the auction as possible - // // candidates in the following call to SuperCommitteeForNextEpoch - if shard.Schedule.IsLastBlock(header.Number().Uint64()) { - if err := SetInactiveUnavailableValidators( - bc, state, processed, - ); err != nil { - return err - } - } - } else { - // TODO Handle shard chain - } - } - - return nil -} diff --git a/staking/availability/measure.go b/staking/availability/measure.go index 81d32336b..895f6970e 100644 --- a/staking/availability/measure.go +++ b/staking/availability/measure.go @@ -10,12 +10,15 @@ import ( "github.com/harmony-one/harmony/core/state" bls2 "github.com/harmony-one/harmony/crypto/bls" "github.com/harmony-one/harmony/internal/ctxerror" + "github.com/harmony-one/harmony/internal/utils" + "github.com/harmony-one/harmony/numeric" "github.com/harmony-one/harmony/shard" + staking "github.com/harmony-one/harmony/staking/types" "github.com/pkg/errors" ) var ( - measure = new(big.Int).Div(big.NewInt(2), big.NewInt(3)) + measure = numeric.NewDec(2).Quo(numeric.NewDec(3)) errValidatorEpochDeviation = errors.New( "validator snapshot epoch not exactly one epoch behind", ) @@ -109,30 +112,46 @@ func BallotResult( } func bumpCount( - chain engine.ChainReader, - state *state.DB, onlyConsider map[common.Address]struct{}, - signers shard.SlotList, didSign bool, + bc Reader, + state *state.DB, + signers shard.SlotList, + didSign bool, + stakedAddrSet map[common.Address]struct{}, ) error { - epoch := chain.CurrentHeader().Epoch() + l := utils.Logger().Info() for i := range signers { addr := signers[i].EcdsaAddress - if _, ok := onlyConsider[addr]; !ok { + // NOTE if the signer address is not part of the staked addrs, + // then it must be a harmony operated node running, + // hence keep on going + if _, isAddrForStaked := stakedAddrSet[addr]; !isAddrForStaked { continue } - wrapper, err := chain.ReadValidatorInformation(addr) + + wrapper, err := state.ValidatorWrapper(addr) if err != nil { return err } - wrapper.Snapshot.NumBlocksToSign.Add( - wrapper.Snapshot.NumBlocksToSign, common.Big1, + + l.RawJSON("validator", []byte(wrapper.String())). + Msg("about to adjust counters") + + wrapper.Counters.NumBlocksToSign.Add( + wrapper.Counters.NumBlocksToSign, common.Big1, ) + if didSign { - wrapper.Snapshot.NumBlocksSigned.Add( - wrapper.Snapshot.NumBlocksSigned, common.Big1, + wrapper.Counters.NumBlocksSigned.Add( + wrapper.Counters.NumBlocksSigned, common.Big1, ) } - wrapper.Snapshot.Epoch = epoch - if err := state.UpdateStakingInfo(addr, wrapper); err != nil { + + l.RawJSON("validator", []byte(wrapper.String())). + Msg("bumped signing counters") + + if err := state.UpdateValidatorWrapper( + addr, wrapper, + ); err != nil { return err } } @@ -141,21 +160,29 @@ func bumpCount( // IncrementValidatorSigningCounts .. func IncrementValidatorSigningCounts( - chain engine.ChainReader, header *block.Header, - shardID uint32, state *state.DB, onlyConsider map[common.Address]struct{}, + bc Reader, + staked *shard.StakedSlots, + state *state.DB, + signers, missing shard.SlotList, ) error { - _, signers, missing, err := BallotResult(chain, header, shardID) + l := utils.Logger().Info() + l.RawJSON("missing", []byte(missing.String())). + Msg("signers that did sign") - if err != nil { - return err - } - if err := bumpCount(chain, state, onlyConsider, signers, true); err != nil { - return err - } - if err := bumpCount(chain, state, onlyConsider, missing, false); err != nil { + l.Msg("bumping signing counters for non-missing signers") + + if err := bumpCount( + bc, state, signers, true, staked.LookupSet, + ); err != nil { return err } - return nil + l.Msg("bumping missing signers counters") + return bumpCount(bc, state, missing, false, staked.LookupSet) +} + +// Reader .. +type Reader interface { + ReadValidatorSnapshot(addr common.Address) (*staking.ValidatorWrapper, error) } // SetInactiveUnavailableValidators sets the validator to @@ -164,72 +191,73 @@ func IncrementValidatorSigningCounts( // whenever committee selection happens in future, the // signing threshold is 66% func SetInactiveUnavailableValidators( - bc engine.ChainReader, state *state.DB, - onlyConsider map[common.Address]struct{}, + bc Reader, state *state.DB, addrs []common.Address, ) error { - addrs, err := bc.ReadActiveValidatorList() - if err != nil { - return err - } - - now := bc.CurrentHeader().Epoch() for i := range addrs { - - if _, ok := onlyConsider[addrs[i]]; !ok { - continue - } - snapshot, err := bc.ReadValidatorSnapshot(addrs[i]) - if err != nil { return err } - wrapper, err := bc.ReadValidatorInformation(addrs[i]) + wrapper, err := state.ValidatorWrapper(addrs[i]) if err != nil { return err } - stats := wrapper.Snapshot - snapEpoch := snapshot.Snapshot.Epoch - snapSigned := snapshot.Snapshot.NumBlocksSigned - snapToSign := snapshot.Snapshot.NumBlocksToSign + statsNow, snapSigned, snapToSign := + wrapper.Counters, + snapshot.Counters.NumBlocksSigned, + snapshot.Counters.NumBlocksToSign - if d := new(big.Int).Sub(now, snapEpoch); d.Cmp(common.Big1) != 0 { - return errors.Wrapf( - errValidatorEpochDeviation, "bc %s, snapshot %s", - now.String(), snapEpoch.String(), - ) - } + l := utils.Logger().Info(). + RawJSON("snapshot", []byte(snapshot.String())). + RawJSON("current", []byte(wrapper.String())) + + l.Msg("begin checks for availability") - signed := new(big.Int).Sub(stats.NumBlocksSigned, snapSigned) - toSign := new(big.Int).Sub(stats.NumBlocksToSign, snapToSign) + signed, toSign := + new(big.Int).Sub(statsNow.NumBlocksSigned, snapSigned), + new(big.Int).Sub(statsNow.NumBlocksToSign, snapToSign) if signed.Sign() == -1 { return errors.Wrapf( errNegativeSign, "diff for signed period wrong: stat %s, snapshot %s", - stats.NumBlocksSigned.String(), snapSigned.String(), + statsNow.NumBlocksSigned.String(), snapSigned.String(), ) } if toSign.Sign() == -1 { return errors.Wrapf( errNegativeSign, "diff for toSign period wrong: stat %s, snapshot %s", - stats.NumBlocksToSign.String(), snapToSign.String(), + statsNow.NumBlocksToSign.String(), snapToSign.String(), ) } if toSign.Cmp(common.Big0) == 0 { - return ErrDivByZero + l.Msg("toSign is 0, perhaps did not receive crosslink proving signing") + continue } - if r := new(big.Int).Div(signed, toSign); r.Cmp(measure) == -1 { + s1, s2 := + numeric.NewDecFromBigInt(signed), numeric.NewDecFromBigInt(toSign) + quotient := s1.Quo(s2) + + l.Str("signed", s1.String()). + Str("to-sign", s2.String()). + Str("percentage-signed", quotient.String()). + Bool("meets-threshold", quotient.LTE(measure)). + Msg("check if signing percent is meeting required threshold") + + if quotient.LTE(measure) { wrapper.Active = false - if err := state.UpdateStakingInfo(addrs[i], wrapper); err != nil { + l.Str("threshold", measure.String()). + Msg("validator failed availability threshold, set to inactive") + if err := state.UpdateValidatorWrapper(addrs[i], wrapper); err != nil { return err } } } + return nil } diff --git a/staking/availability/measure_test.go b/staking/availability/measure_test.go new file mode 100644 index 000000000..26fd309fe --- /dev/null +++ b/staking/availability/measure_test.go @@ -0,0 +1,66 @@ +package availability + +import ( + "math/big" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/harmony-one/harmony/core/state" + common2 "github.com/harmony-one/harmony/internal/common" + staking "github.com/harmony-one/harmony/staking/types" +) + +type fakerAuctioneer struct{} + +const ( + to0 = "one1zyxauxquys60dk824p532jjdq753pnsenrgmef" + to2 = "one14438psd5vrjes7qm97jrj3t0s5l4qff5j5cn4h" +) + +var ( + validatorS0Addr, validatorS2Addr = common.Address{}, common.Address{} + addrs = []common.Address{} + validatorS0, validatorS2 = &staking.ValidatorWrapper{}, &staking.ValidatorWrapper{} +) + +func init() { + validatorS0Addr, _ = common2.Bech32ToAddress(to0) + validatorS2Addr, _ = common2.Bech32ToAddress(to2) + addrs = []common.Address{validatorS0Addr, validatorS2Addr} +} + +func (fakerAuctioneer) ReadValidatorSnapshot( + addr common.Address, +) (*staking.ValidatorWrapper, error) { + switch addr { + case validatorS0Addr: + return validatorS0, nil + case validatorS2Addr: + return validatorS0, nil + default: + panic("bad input in test case") + } +} + +func defaultStateWithAccountsApplied() *state.DB { + st := ethdb.NewMemDatabase() + stateHandle, _ := state.New(common.Hash{}, state.NewDatabase(st)) + for _, addr := range addrs { + stateHandle.CreateAccount(addr) + } + stateHandle.SetBalance(validatorS0Addr, big.NewInt(0).SetUint64(1994680320000000000)) + stateHandle.SetBalance(validatorS2Addr, big.NewInt(0).SetUint64(1999975592000000000)) + return stateHandle +} + +func TestSetInactiveUnavailableValidators(t *testing.T) { + state := defaultStateWithAccountsApplied() + if err := SetInactiveUnavailableValidators( + fakerAuctioneer{}, state, addrs, + ); err != nil { + // + } + + t.Log("Unimplemented") +} diff --git a/staking/effective/calculate.go b/staking/effective/calculate.go index 0d4a5ee05..7b0894921 100644 --- a/staking/effective/calculate.go +++ b/staking/effective/calculate.go @@ -66,7 +66,8 @@ func (s Slots) JSON() string { return string(b) } -func median(stakes []SlotPurchase) numeric.Dec { +// Median .. +func Median(stakes []SlotPurchase) numeric.Dec { if len(stakes) == 0 { utils.Logger().Error().Int("non-zero", len(stakes)). @@ -90,11 +91,13 @@ func median(stakes []SlotPurchase) numeric.Dec { } } -// Apply .. -func Apply(shortHand map[common.Address]SlotOrder, pull int) Slots { +// Compute .. +func Compute( + shortHand map[common.Address]SlotOrder, pull int, +) (numeric.Dec, Slots) { eposedSlots := Slots{} if len(shortHand) == 0 { - return eposedSlots + return numeric.ZeroDec(), eposedSlots } type t struct { @@ -144,11 +147,16 @@ func Apply(shortHand map[common.Address]SlotOrder, pull int) Slots { picks := eposedSlots[:pull] if len(picks) == 0 { - return Slots{} + return numeric.ZeroDec(), Slots{} } - median := median(picks) + return Median(picks), picks + +} +// Apply .. +func Apply(shortHand map[common.Address]SlotOrder, pull int) Slots { + median, picks := Compute(shortHand, pull) for i := range picks { picks[i].Dec = effectiveStake(median, picks[i].Dec) } diff --git a/staking/effective/calculate_test.go b/staking/effective/calculate_test.go index 99c665e15..fc4b21043 100644 --- a/staking/effective/calculate_test.go +++ b/staking/effective/calculate_test.go @@ -78,7 +78,7 @@ func TestMedian(t *testing.T) { } else { expectedMedian = copyPurchases[numPurchases].Dec } - med := median(testingPurchases) + med := Median(testingPurchases) if !med.Equal(expectedMedian) { t.Errorf("Expected: %s, Got: %s", expectedMedian.String(), med.String()) } diff --git a/staking/effective/eligible.go b/staking/effective/eligible.go new file mode 100644 index 000000000..46de85f1d --- /dev/null +++ b/staking/effective/eligible.go @@ -0,0 +1,10 @@ +package effective + +import ( + staking "github.com/harmony-one/harmony/staking/types" +) + +// IsEligibleForEPOSAuction .. +func IsEligibleForEPOSAuction(v *staking.ValidatorWrapper) bool { + return v.Active && !v.Banned +} diff --git a/staking/network/reward.go b/staking/network/reward.go index c6199b9ca..0e6c95534 100644 --- a/staking/network/reward.go +++ b/staking/network/reward.go @@ -52,8 +52,8 @@ func WhatPercentStakedNow( timestamp int64, ) (*big.Int, *numeric.Dec, error) { stakedNow := numeric.ZeroDec() - // Only active validators' stake is counted in stake ratio because only their stake is under slashing risk - active, err := beaconchain.ReadActiveValidatorList() + // Only elected validators' stake is counted in stake ratio because only their stake is under slashing risk + active, err := beaconchain.ReadElectedValidatorList() if err != nil { return nil, nil, err } diff --git a/staking/slash/double-sign.go b/staking/slash/double-sign.go index 5a1f54820..e056bc6e2 100644 --- a/staking/slash/double-sign.go +++ b/staking/slash/double-sign.go @@ -1,52 +1,408 @@ package slash import ( + "encoding/hex" + "encoding/json" + "math/big" + "github.com/ethereum/go-ethereum/common" - "github.com/harmony-one/bls/ffi/go/bls" + "github.com/ethereum/go-ethereum/rlp" "github.com/harmony-one/harmony/block" + "github.com/harmony-one/harmony/consensus/votepower" "github.com/harmony-one/harmony/core/state" + common2 "github.com/harmony-one/harmony/internal/common" + "github.com/harmony-one/harmony/internal/utils" + "github.com/harmony-one/harmony/numeric" "github.com/harmony-one/harmony/shard" + staking "github.com/harmony-one/harmony/staking/types" + "github.com/pkg/errors" +) + +const ( + haveEnoughToPayOff = 1 + paidOffExact = 0 + debtCollectionsRepoUndelegations = -1 + validatorsOwnDel = 0 ) +// invariant assumes snapshot, current can be rlp.EncodeToBytes +func payDebt( + snapshot, current *staking.ValidatorWrapper, + slashDebt, payment *big.Int, + slashDiff *Application, +) error { + utils.Logger().Info(). + RawJSON("snapshot", []byte(snapshot.String())). + RawJSON("current", []byte(current.String())). + Uint64("slash-debt", slashDebt.Uint64()). + Uint64("payment", payment.Uint64()). + RawJSON("slash-track", []byte(slashDiff.String())). + Msg("slash debt payment before application") + slashDiff.TotalSlashed.Add(slashDiff.TotalSlashed, payment) + slashDebt.Sub(slashDebt, payment) + if slashDebt.Cmp(common.Big0) == -1 { + x1, _ := rlp.EncodeToBytes(snapshot) + x2, _ := rlp.EncodeToBytes(current) + utils.Logger().Info(). + Str("snapshot-rlp", hex.EncodeToString(x1)). + Str("current-rlp", hex.EncodeToString(x2)). + Msg("slashdebt balance cannot go below zero") + return errSlashDebtCannotBeNegative + } + return nil +} + +// Moment .. +type Moment struct { + Epoch *big.Int `json:"epoch"` + Height *big.Int `json:"block-height"` + TimeUnixNano *big.Int `json:"time-unix-nano"` + ViewID uint64 `json:"view-id"` + ShardID uint32 `json:"shard-id"` +} + +// Evidence .. +type Evidence struct { + Moment + ConflictingBallots + ProposalHeader *block.Header `json:"header"` +} + +// ConflictingBallots .. +type ConflictingBallots struct { + AlreadyCastBallot votepower.Ballot `json:"already-cast-vote"` + DoubleSignedBallot votepower.Ballot `json:"double-signed-vote"` +} + // Record is an proof of a slashing made by a witness of a double-signing event type Record struct { - Offender shard.BlsPublicKey - Signed struct { - Header *block.Header - Signature *bls.Sign - } `json:"signed"` - DoubleSigned struct { - Header *block.Header - Signature *bls.Sign - } `json:"double-signed"` - Beneficiary common.Address // the reporter who will get rewarded -} - -// NewRecord .. -func NewRecord( - offender shard.BlsPublicKey, - signedHeader, doubleSignedHeader *block.Header, - signedSignature, doubleSignedSignature *bls.Sign, - beneficiary common.Address, -) Record { - r := Record{} - r.Offender = offender - r.Signed.Header = signedHeader - r.Signed.Signature = signedSignature - r.DoubleSigned.Header = doubleSignedHeader - r.DoubleSigned.Signature = doubleSignedSignature - r.Beneficiary = beneficiary - return r -} - -// TODO(Edgar) Implement Verify and Apply + // the reporter who will get rewarded + Evidence Evidence `json:"evidence"` + Reporter common.Address `json:"reporter"` + Offender common.Address `json:"offender"` +} + +// Application .. +type Application struct { + TotalSlashed, TotalSnitchReward *big.Int +} + +func (a *Application) String() string { + s, _ := json.Marshal(a) + return string(s) +} + +// MarshalJSON .. +func (e Evidence) MarshalJSON() ([]byte, error) { + return json.Marshal(struct { + Moment + ConflictingBallots + ProposalHeader string `json:"header"` + }{e.Moment, e.ConflictingBallots, e.ProposalHeader.String()}) +} + +// Records .. +type Records []Record + +func (r Records) String() string { + s, _ := json.Marshal(r) + return string(s) +} + +var ( + errBallotSignerKeysNotSame = errors.New("conflicting ballots must have same signer key") + errReporterAndOffenderSame = errors.New("reporter and offender cannot be same") +) + +// SanityCheck fails if any of the slashes fail +func (r Records) SanityCheck() error { + for _, record := range r { + k1 := record.Evidence.AlreadyCastBallot.SignerPubKey + k2 := record.Evidence.DoubleSignedBallot.SignerPubKey + if k1 != k2 { + return errBallotSignerKeysNotSame + } + + if record.Offender == record.Reporter { + return errReporterAndOffenderSame + } + + } + return nil +} + +// MarshalJSON .. +func (r Record) MarshalJSON() ([]byte, error) { + reporter, offender := + common2.MustAddressToBech32(r.Reporter), + common2.MustAddressToBech32(r.Offender) + return json.Marshal(struct { + Evidence Evidence `json:"evidence"` + Beneficiary string `json:"beneficiary"` + AddressForBLSKey string `json:"offender"` + }{r.Evidence, reporter, offender}) +} + +func (e Evidence) String() string { + s, _ := json.Marshal(e) + return string(s) +} + +func (r Record) String() string { + s, _ := json.Marshal(r) + return string(s) +} + +// CommitteeReader .. +type CommitteeReader interface { + ReadShardState(epoch *big.Int) (*shard.State, error) +} // Verify checks that the signature is valid -func Verify(candidate *Record) error { +func Verify(chain CommitteeReader, candidate *Record) error { + first, second := + candidate.Evidence.AlreadyCastBallot, + candidate.Evidence.DoubleSignedBallot + if shard.CompareBlsPublicKey(first.SignerPubKey, second.SignerPubKey) != 0 { + k1, k2 := first.SignerPubKey.Hex(), second.SignerPubKey.Hex() + return errors.Wrapf( + errBLSKeysNotEqual, "%s %s", k1, k2, + ) + } + superCommittee, err := chain.ReadShardState(candidate.Evidence.Epoch) + + if err != nil { + return err + } + + subCommittee := superCommittee.FindCommitteeByID( + candidate.Evidence.ShardID, + ) + + if subCommittee == nil { + return errors.Wrapf( + errShardIDNotKnown, "given shardID %d", candidate.Evidence.ShardID, + ) + } + + if _, err := subCommittee.AddressForBLSKey(second.SignerPubKey); err != nil { + return err + } + // TODO need to finish this implementation return nil } -// Apply .. -func Apply(state *state.DB, slashes []byte) error { +var ( + errBLSKeysNotEqual = errors.New( + "bls keys in ballots accompanying slash evidence not equal ", + ) + errSlashDebtCannotBeNegative = errors.New("slash debt cannot be negative") + errShardIDNotKnown = errors.New("nil subcommittee for shardID") + errValidatorNotFoundDuringSlash = errors.New("validator not found") + zero = numeric.ZeroDec() + oneDoubleSignerRate = numeric.MustNewDecFromStr("0.02") +) + +// applySlashRate returns (amountPostSlash, amountOfReduction, amountOfReduction / 2) +func applySlashRate(amount *big.Int, rate numeric.Dec) *big.Int { + return numeric.NewDecFromBigInt( + amount, + ).Mul(rate).TruncateInt() +} + +func payDownAsMuchAsCan( + snapshot, current *staking.ValidatorWrapper, + slashDebt, nowAmt *big.Int, + slashDiff *Application, +) error { + if nowAmt.Cmp(common.Big0) == 1 && slashDebt.Cmp(common.Big0) == 1 { + // 0.50_amount > 0.06_debt => slash == 0.0, nowAmt == 0.44 + if nowAmt.Cmp(slashDebt) >= 0 { + nowAmt.Sub(nowAmt, slashDebt) + if err := payDebt( + snapshot, current, slashDebt, slashDebt, slashDiff, + ); err != nil { + return err + } + } else { + // 0.50_amount < 2.4_debt =>, slash == 1.9, nowAmt == 0.0 + if err := payDebt( + snapshot, current, slashDebt, nowAmt, slashDiff, + ); err != nil { + return err + } + nowAmt.Sub(nowAmt, nowAmt) + } + } + return nil } + +func delegatorSlashApply( + snapshot, current *staking.ValidatorWrapper, + rate numeric.Dec, + state *state.DB, + reporter common.Address, + doubleSignEpoch *big.Int, + slashTrack *Application, +) error { + + for _, delegationSnapshot := range snapshot.Delegations { + slashDebt := applySlashRate(delegationSnapshot.Amount, rate) + slashDiff := &Application{big.NewInt(0), big.NewInt(0)} + snapshotAddr := delegationSnapshot.DelegatorAddress + for _, delegationNow := range current.Delegations { + if nowAmt := delegationNow.Amount; delegationNow.DelegatorAddress == snapshotAddr { + l := utils.Logger().Info().RawJSON("delegation-snapshot", []byte(delegationSnapshot.String())). + RawJSON("delegation-current", []byte(delegationNow.String())) + + l.Uint64("initial-slash-debt", slashDebt.Uint64()). + Str("rate", rate.String()). + Msg("attempt to apply slashing based on snapshot amount to current state") + // Current delegation has some money and slashdebt is still not paid off + // so contribute as much as can with current delegation amount + if err := payDownAsMuchAsCan( + snapshot, current, slashDebt, nowAmt, slashDiff, + ); err != nil { + return err + } + + // NOTE Assume did as much as could above, now check the undelegations + for _, undelegate := range delegationNow.Undelegations { + // the epoch matters, only those undelegation + // such that epoch>= doubleSignEpoch should be slashable + if undelegate.Epoch.Cmp(doubleSignEpoch) >= 0 { + if slashDebt.Cmp(common.Big0) <= 0 { + l.Msg("paid off the slash debt") + break + } + nowAmt := undelegate.Amount + if err := payDownAsMuchAsCan( + snapshot, current, slashDebt, nowAmt, slashDiff, + ); err != nil { + return err + } + + if nowAmt.Cmp(common.Big0) == 0 { + // TODO need to remove the undelegate + l.Msg("delegation amount after paying slash debt is 0") + } + } + } + + // if we still have a slashdebt + // even after taking away from delegation amount + // and even after taking away from undelegate, + // then we need to take from their pending rewards + if slashDebt.Cmp(common.Big0) == 1 { + nowAmt := delegationNow.Reward + l.Uint64("slash-debt", slashDebt.Uint64()). + Uint64("now-amount-reward", nowAmt.Uint64()). + Msg("needed to dig into reward to pay off slash debt") + if err := payDownAsMuchAsCan( + snapshot, current, slashDebt, nowAmt, slashDiff, + ); err != nil { + return err + } + } + + // NOTE only need to pay snitch here, + // they only get half of what was actually dispersed + halfOfSlashDebt := new(big.Int).Div(slashDiff.TotalSlashed, common.Big2) + slashDiff.TotalSnitchReward.Add(slashDiff.TotalSnitchReward, halfOfSlashDebt) + l.Uint64("reporter-reward", halfOfSlashDebt.Uint64()). + RawJSON("application", []byte(slashDiff.String())). + Msg("completed an application of slashing") + state.AddBalance(reporter, halfOfSlashDebt) + slashTrack.TotalSnitchReward.Add( + slashTrack.TotalSnitchReward, slashDiff.TotalSnitchReward, + ) + slashTrack.TotalSlashed.Add( + slashTrack.TotalSlashed, slashDiff.TotalSlashed, + ) + } + } + // after the loops, paid off as much as could + if slashDebt.Cmp(common.Big0) == -1 { + x1, _ := rlp.EncodeToBytes(snapshot) + x2, _ := rlp.EncodeToBytes(current) + utils.Logger().Error().Str("slash-rate", rate.String()). + Str("snapshot-rlp", hex.EncodeToString(x1)). + Str("current-rlp", hex.EncodeToString(x2)). + Msg("slash debt not paid off") + return errors.Wrapf(errSlashDebtCannotBeNegative, "amt %v", slashDebt) + } + } + return nil +} + +// TODO Need to keep a record in off-chain db of all the slashes? + +// Apply .. +func Apply( + chain staking.ValidatorSnapshotReader, state *state.DB, + slashes Records, rate numeric.Dec, +) (*Application, error) { + slashDiff := &Application{big.NewInt(0), big.NewInt(0)} + for _, slash := range slashes { + // TODO Probably won't happen but we probably should + // be expilict about reading the right epoch validator snapshot, + // because it needs to be the epoch of which the double sign + // occurred + snapshot, err := chain.ReadValidatorSnapshot( + slash.Offender, + ) + + if err != nil { + return nil, errors.Errorf( + "could not find validator %s", + common2.MustAddressToBech32(slash.Offender), + ) + } + + current, err := state.ValidatorWrapper(slash.Offender) + if err != nil { + return nil, errors.Wrapf( + errValidatorNotFoundDuringSlash, " %s ", err.Error(), + ) + } + // NOTE invariant: first delegation is the validators own + // stake, rest are external delegations. + // Bottom line: everyone will be slashed under the same rule. + if err := delegatorSlashApply( + snapshot, current, rate, state, + slash.Reporter, slash.Evidence.Epoch, slashDiff, + ); err != nil { + return nil, err + } + + // finally, kick them off forever + current.Banned, current.Active = true, false + utils.Logger().Info(). + RawJSON("delegation-current", []byte(current.String())). + RawJSON("slash", []byte(slash.String())). + Msg("about to update staking info for a validator after a slash") + + if err := state.UpdateValidatorWrapper( + snapshot.Address, current, + ); err != nil { + return nil, err + } + } + return slashDiff, nil +} + +// Rate is the slashing % rate +func Rate(doubleSignerCount, committeeSize int) numeric.Dec { + if doubleSignerCount == 0 || committeeSize == 0 { + return zero + } + switch doubleSignerCount { + case 1: + return oneDoubleSignerRate + default: + return numeric.NewDec( + int64(doubleSignerCount), + ).Quo(numeric.NewDec(int64(committeeSize))) + } +} diff --git a/staking/slash/double-sign_test.go b/staking/slash/double-sign_test.go index 3b66e9837..2b6415932 100644 --- a/staking/slash/double-sign_test.go +++ b/staking/slash/double-sign_test.go @@ -1,12 +1,499 @@ package slash import ( + "encoding/hex" + "fmt" + "math/big" "testing" - "github.com/harmony-one/harmony/consensus/quorum" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/rlp" + "github.com/harmony-one/bls/ffi/go/bls" + "github.com/harmony-one/harmony/block" + "github.com/harmony-one/harmony/common/denominations" + "github.com/harmony-one/harmony/consensus/votepower" + "github.com/harmony-one/harmony/core/state" + common2 "github.com/harmony-one/harmony/internal/common" + "github.com/harmony-one/harmony/numeric" + "github.com/harmony-one/harmony/shard" + staking "github.com/harmony-one/harmony/staking/types" ) -func TestDidAnyoneDoubleSign(t *testing.T) { - d := quorum.NewDecider(quorum.SuperMajorityStake) - t.Log("Unimplemented", d) +var ( + commonCommission = staking.Commission{ + CommissionRates: staking.CommissionRates{ + Rate: numeric.MustNewDecFromStr("0.167983520183826780"), + MaxRate: numeric.MustNewDecFromStr("0.179184469782137200"), + MaxChangeRate: numeric.MustNewDecFromStr("0.152212761523253600"), + }, + UpdateHeight: big.NewInt(10), + } + + commonDescr = staking.Description{ + Name: "someoneA", + Identity: "someoneB", + Website: "someoneC", + SecurityContact: "someoneD", + Details: "someoneE", + } +) + +const ( + // ballot A hex values + signerABLSPublicHex = "be23bc3c93fe14a25f3533" + + "feee1cff1c60706845a4907" + + "c5df58bc19f5d1760bfff06fe7c9d1f596b18fdf529e0508e0a" + signerAHeaderHashHex = "0x68bf572c03e36b4b7a4f268797d18" + + "7027b288bc69725084bab5bfe6214cb8ddf" + signerABLSSignature = "ab14e519485b70d8af76ae83205" + + "290792d679a8a11eb9a12d21787cc0b73" + + "96e340d88b2a0c39888d0c9cec1" + + "2c4a09b06b2eec3e851f08f3070f3" + + "804b35fe4a2033725f073623e3870756141ebc" + + "2a6495478930c428f6e6b25f292dab8552d30c" + // ballot B hex values + signerBBLSPublicHex = "be23bc3c93fe14a25f3533" + + "feee1cff1c60706845a490" + + "7c5df58bc19f5d1760bfff" + + "06fe7c9d1f596b18fdf529e0508e0a" + signerBHeaderHashHex = "0x1dbf572c03e36b4b7a4f268797d187" + + "027b288bc69725084bab5bfe6214cb8ddf" + signerBBLSSignature = "0894d55a541a90ada11535866e5a848d9d6a2b5c30" + + "932a95f48133f886140cefbe4d690eddd0540d246df1fec" + + "8b4f719ad9de0bc822f0a1bf70e78b321a5e4462ba3e3efd" + + "cd24c21b9cb24ed6b26f02785a2cdbd168696c5f4a49b6c00f00994" + // RLP encoded header + proposalHeaderRLPBytes = "f9039187486d6e79546764827633f90383a080532768867d8c1a96" + + "6ae9df80d4efc9a9a83559b0e2d13b2aa819a6a7627c7294" + + "6911b75b2560be9a8f71164a33086be4511fc99aa0eb514ec3bc1" + + "8e67ad7d9641768b7a9b9655ff78e54970465aeb037c2d81c5987a05" + + "6e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc0016" + + "22fb5e363b421a056e81f171bcc55a6ff8345e692c0f86e5b" + + "48e01b996cadc001622fb5e363b421a056e81f171bcc55a6" + + "ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a05" + + "6e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc0" + + "01622fb5e363b421b901000000000000000000000000000" + + "00000000000000000000000000000000000000000000000000" + + "00000000000000000000000000000000000000000000000000000" + + "000000000000000000000000000000000000000000000000000000" + + "00000000000000000000000000000000000000000000000000000000" + + "000000000000000000000000000000000000000000000000000000000" + + "0000000000000000000000000000000000000000000000000000000" + + "0000000000000000000000000000000000000000000000000000" + + "00000000000000000000000000000000000000000000000000000" + + "00000000000000000000000000000000000000000000000000000" + + "00002383a9235180845e4c602680a00000000000000000000000" + + "000000000000000000000000000000000000000000230680b860c" + + "4db93981d5870f0f77bb8487dde189e4579e3866ae3b598fbed0f5" + + "d87e53d21a039e427d2a206eeca4ff1f1fab2ed0d2171b9c1f12" + + "1e42c1a1456e6e1ba4c5e2fb3ddda5d34873bcdf01f14d1da1bc5" + + "0e70dc6f0bdfda3f36f9dafe1e29880e7fb8807b05ee6062476f5a43" + + "6b29876d726491fb76bfe2cf8f6d9e3a7361e0ca95bd66c4ef4593c" + + "2ac616c4ca171af74a93f5ef9e4d1b74421e789f07e0a36e4b00b7a9" + + "13fb6a296e20ee4dcd2d74088ea9711b8b7693af18d3f6ab925d26a0b" + + "e30d8899f18e4b7f9e8c6937e78863b828fa172e8edef106cca814294" + + "eee7146eec7018080b88bf889f887a0a1cc8366aa9c8acac219a229b2" + + "eb88deee3bc9b133baa610bd88af294422ab0020b860e5a234c8bce79" + + "24c4f934edf7ce9f8f4b10d61d1e72becdda95c5fb3195598cc9ae334" + + "8a00c493fb77a87a094dbf08130c3f93793337be78167b8b241d7ef9e" + + "1fab1f8097dda339a6ca2737c291d8a7733c9b5b540ea30544c46" + + "a0426a34f60d3f010580" + // trailing bech32 info + reporterBech32 = "one1pdv9lrdwl0rg5vglh4xtyrv3wjk3wsqket7zxy" + offenderBech32 = "one1zyxauxquys60dk824p532jjdq753pnsenrgmef" + // some rando delegator + randoDelegatorBech32 = "one1nqevvacj3y5ltuef05my4scwy5wuqteur72jk5" + // rest of the committee + commK1 = "65f55eb3052f9e9f632b2923be594ba77c55543f5c58ee1454b9cfd658d25e06373b0f7d42a19c84768139ea294f6204" + commK2 = "02c8ff0b88f313717bc3a627d2f8bb172ba3ad3bb9ba3ecb8eed4b7c878653d3d4faf769876c528b73f343967f74a917" + commK3 = "e751ec995defe4931273aaebcb2cd14bf37e629c554a57d3f334c37881a34a6188a93e76113c55ef3481da23b7d7ab09" + commK4 = "2d61379e44a772e5757e27ee2b3874254f56073e6bd226eb8b160371cc3c18b8c4977bd3dcb71fd57dc62bf0e143fd08" + commK5 = "86dc2fdc2ceec18f6923b99fd86a68405c132e1005cf1df72dca75db0adfaeb53d201d66af37916d61f079f34f21fb96" + commK6 = "95117937cd8c09acd2dfae847d74041a67834ea88662a7cbed1e170350bc329e53db151e5a0ef3e712e35287ae954818" + // double signing info + doubleSignShardID = 0 + doubleSignEpoch = 3 + doubleSignBlockNumber = 37 + doubleSignViewID = 38 + doubleSignUnixNano = 1582049233802498300 + // validator creation parameters + lastEpochInComm = 5 + creationHeight = 33 +) + +type scenario struct { + snapshot, current *staking.ValidatorWrapper + slashRate float64 + result *Application +} + +func defaultFundingScenario() *scenario { + return &scenario{ + snapshot: nil, + current: nil, + slashRate: 0.02, + result: nil, + } +} + +func scenariorealWorldSample1() *scenario { + const ( + snapshotBytes = "f90108f8be94110dde181c2434f6d8eaa869154a4d07a910ce19f1b0be23bc3c93fe14a25f3533feee1cff1c60706845a4907c5df58bc19f5d1760bfff06fe7c9d1f596b18fdf529e0508e0a06880de0b6b3a764000088b469471f8014000001e0dec9880254cc1f20ad395cc988027c97536ea18970c988021cc4b33cdff1601df83e945f546573745f6b65795f76616c696461746f72308c746573745f6163636f756e748b6861726d6f6e792e6f6e658a44616e69656c2d56444d846e6f6e651d80f842e094110dde181c2434f6d8eaa869154a4d07a910ce19880f43fc2c04ee000080c0e0949832c677128929f5f3297d364ac30e251dc02f3c8814d1120d7b16000080c0c3060180" + + currentBytes = "f90118f8be94110dde181c2434f6d8eaa869154a4d07a910ce19f1b0be23bc3c93fe14a25f3533feee1cff1c60706845a4907c5df58bc19f5d1760bfff06fe7c9d1f596b18fdf529e0508e0a06880de0b6b3a764000088b469471f8014000001e0dec9880254cc1f20ad395cc988027c97536ea18970c988021cc4b33cdff1601df83e945f546573745f6b65795f76616c696461746f72308c746573745f6163636f756e748b6861726d6f6e792e6f6e658a44616e69656c2d56444d846e6f6e651d80f852e894110dde181c2434f6d8eaa869154a4d07a910ce19880f43fc2c04ee000088e6ec131ed55ec404c0e8949832c677128929f5f3297d364ac30e251dc02f3c8814d1120d7b16000088d52ac35139f06a2cc0c3070280" + ) + + snapshotData, _ := hex.DecodeString(snapshotBytes) + currentData, _ := hex.DecodeString(currentBytes) + + var snapshot, current staking.ValidatorWrapper + + if err := rlp.DecodeBytes(snapshotData, &snapshot); err != nil { + panic("test case has bad input") + } + + if err := rlp.DecodeBytes(currentData, ¤t); err != nil { + panic("test case has bad input") + } + + return &scenario{ + slashRate: 0.02, + result: &Application{ + TotalSlashed: big.NewInt(0.052 * denominations.One), + TotalSnitchReward: big.NewInt(0.026 * denominations.One), + }, + snapshot: &snapshot, + current: ¤t, + } +} + +var ( + scenarioTwoPercent = defaultFundingScenario() + scenarioEightyPercent = defaultFundingScenario() +) + +func init() { + { + s := scenarioTwoPercent + s.result = &Application{ + TotalSlashed: big.NewInt(0.1 * denominations.One), + TotalSnitchReward: big.NewInt(0.05 * denominations.One), + } + s.snapshot, s.current = s.defaultValidatorPair(s.defaultDelegationPair()) + } + { + s := scenarioEightyPercent + s.slashRate = 0.80 + s.result = &Application{ + TotalSlashed: big.NewInt(4 * denominations.One), + TotalSnitchReward: big.NewInt(2 * denominations.One), + } + s.snapshot, s.current = s.defaultValidatorPair(s.defaultDelegationPair()) + } +} + +var ( + signerA, signerB = &bls.PublicKey{}, &bls.PublicKey{} + hashA, hashB = common.Hash{}, common.Hash{} + reporterAddr, offenderAddr = common.Address{}, common.Address{} + randoDel = common.Address{} + header = block.Header{} + subCommittee = []shard.BlsPublicKey{} + + unit = func() interface{} { + // Ballot A setup + signerA.DeserializeHexStr(signerABLSPublicHex) + headerHashA, _ := hex.DecodeString(signerAHeaderHashHex) + hashA = common.BytesToHash(headerHashA) + // Ballot B setup + signerB.DeserializeHexStr(signerBBLSPublicHex) + headerHashB, _ := hex.DecodeString(signerBHeaderHashHex) + hashB = common.BytesToHash(headerHashB) + // address setup + reporterAddr, _ = common2.Bech32ToAddress(reporterBech32) + offenderAddr, _ = common2.Bech32ToAddress(offenderBech32) + randoDel, _ = common2.Bech32ToAddress(randoDelegatorBech32) + headerData, err := hex.DecodeString(proposalHeaderRLPBytes) + if err != nil { + panic("test case has bad input") + } + if err := rlp.DecodeBytes(headerData, &header); err != nil { + panic("test case has bad input") + } + for _, hexK := range [...]string{ + commK1, commK2, commK3, commK4, commK5, commK6, + } { + k := &bls.PublicKey{} + k.DeserializeHexStr(hexK) + subCommittee = append(subCommittee, *shard.FromLibBLSPublicKeyUnsafe(k)) + } + return nil + }() + + blsWrapA, blsWrapB = *shard.FromLibBLSPublicKeyUnsafe(signerA), + *shard.FromLibBLSPublicKeyUnsafe(signerB) +) + +func (s *scenario) defaultValidatorPair( + delegationsSnapshot, delegationsCurrent staking.Delegations, +) ( + *staking.ValidatorWrapper, *staking.ValidatorWrapper, +) { + + validatorSnapshot := &staking.ValidatorWrapper{ + Validator: staking.Validator{ + Address: offenderAddr, + SlotPubKeys: []shard.BlsPublicKey{blsWrapA}, + LastEpochInCommittee: big.NewInt(lastEpochInComm), + MinSelfDelegation: new(big.Int).SetUint64(1 * denominations.One), + MaxTotalDelegation: new(big.Int).SetUint64(10 * denominations.One), + Active: true, + Commission: commonCommission, + Description: commonDescr, + CreationHeight: big.NewInt(creationHeight), + Banned: false, + }, + Delegations: delegationsSnapshot, + } + + validatorCurrent := &staking.ValidatorWrapper{ + Validator: staking.Validator{ + Address: offenderAddr, + SlotPubKeys: []shard.BlsPublicKey{blsWrapA}, + LastEpochInCommittee: big.NewInt(lastEpochInComm + 1), + MinSelfDelegation: new(big.Int).SetUint64(1 * denominations.One), + MaxTotalDelegation: new(big.Int).SetUint64(10 * denominations.One), + Active: true, + Commission: commonCommission, + Description: commonDescr, + CreationHeight: big.NewInt(creationHeight), + Banned: false, + }, + Delegations: delegationsCurrent, + } + return validatorSnapshot, validatorCurrent +} + +func (s *scenario) defaultDelegationPair() ( + staking.Delegations, staking.Delegations, +) { + delegationsSnapshot := staking.Delegations{ + // NOTE delegation is the validator themselves + staking.Delegation{ + DelegatorAddress: offenderAddr, + Amount: new(big.Int).SetUint64(2 * denominations.One), + Reward: common.Big0, + Undelegations: staking.Undelegations{}, + }, + staking.Delegation{ + DelegatorAddress: randoDel, + Amount: new(big.Int).SetUint64(3 * denominations.One), + Reward: common.Big0, + Undelegations: staking.Undelegations{}, + }, + } + + delegationsCurrent := staking.Delegations{ + staking.Delegation{ + DelegatorAddress: offenderAddr, + Amount: new(big.Int).SetUint64(1.96 * denominations.One), + Reward: common.Big0, + Undelegations: staking.Undelegations{ + staking.Undelegation{ + Amount: new(big.Int).SetUint64(1 * denominations.One), + Epoch: big.NewInt(doubleSignEpoch + 2), + }, + }, + }, + // some external delegator + staking.Delegation{ + DelegatorAddress: randoDel, + Amount: new(big.Int).SetUint64(0.5 * denominations.One), + Reward: common.Big0, + Undelegations: staking.Undelegations{ + staking.Undelegation{ + Amount: new(big.Int).SetUint64(2.5 * denominations.One), + Epoch: big.NewInt(doubleSignEpoch + 2), + }, + }, + }, + } + + return delegationsSnapshot, delegationsCurrent +} + +func exampleSlashRecords() Records { + return Records{ + Record{ + Evidence: Evidence{ + ConflictingBallots: ConflictingBallots{ + AlreadyCastBallot: votepower.Ballot{ + SignerPubKey: blsWrapA, + BlockHeaderHash: hashA, + Signature: common.Hex2Bytes(signerABLSSignature), + }, + DoubleSignedBallot: votepower.Ballot{ + SignerPubKey: blsWrapB, + BlockHeaderHash: hashB, + Signature: common.Hex2Bytes(signerBBLSSignature), + }, + }, + Moment: Moment{ + Epoch: big.NewInt(doubleSignEpoch), + Height: big.NewInt(doubleSignBlockNumber), + TimeUnixNano: big.NewInt(doubleSignUnixNano), + ViewID: doubleSignViewID, + ShardID: doubleSignShardID, + }, + ProposalHeader: &header, + }, + Reporter: reporterAddr, + Offender: offenderAddr, + }, + } +} + +type mockOutSnapshotReader struct { + snapshot staking.ValidatorWrapper +} + +func (m mockOutSnapshotReader) ReadValidatorSnapshot( + common.Address, +) (*staking.ValidatorWrapper, error) { + return &m.snapshot, nil +} + +type mockOutChainReader struct{} + +func (mockOutChainReader) ReadShardState(epoch *big.Int) (*shard.State, error) { + return &shard.State{ + Epoch: big.NewInt(doubleSignEpoch), + Shards: []shard.Committee{ + shard.Committee{ + ShardID: doubleSignShardID, + Slots: shard.SlotList{ + shard.Slot{ + EcdsaAddress: offenderAddr, + BlsPublicKey: blsWrapA, + EffectiveStake: nil, + }, + }, + }, + }, + }, nil +} + +func TestVerify(t *testing.T) { + if err := Verify( + mockOutChainReader{}, &exampleSlashRecords()[0], + ); err != nil { + t.Errorf("could not verify slash %s", err.Error()) + } +} + +func testScenario( + t *testing.T, stateHandle *state.DB, slashes Records, s *scenario, +) { + if err := stateHandle.UpdateValidatorWrapper( + offenderAddr, s.snapshot, + ); err != nil { + t.Fatalf("creation of validator failed %s", err.Error()) + } + + stateHandle.IntermediateRoot(false) + stateHandle.Commit(false) + + if err := stateHandle.UpdateValidatorWrapper( + offenderAddr, s.current, + ); err != nil { + t.Fatalf("update of validator failed %s", err.Error()) + } + + stateHandle.IntermediateRoot(false) + stateHandle.Commit(false) + // NOTE See dump.json to see what account + // state looks like as of this point + + slashResult, err := Apply( + mockOutSnapshotReader{*s.snapshot}, + stateHandle, + slashes, + numeric.MustNewDecFromStr( + fmt.Sprintf("%f", s.slashRate), + ), + ) + + if err != nil { + t.Fatalf("rate: %v, slash application failed %s", s.slashRate, err.Error()) + } + + if sn := slashResult.TotalSlashed; sn.Cmp( + s.result.TotalSlashed, + ) != 0 { + t.Errorf( + "total slash incorrect have %v want %v", + sn, + s.result.TotalSlashed, + ) + } + + if sn := slashResult.TotalSnitchReward; sn.Cmp( + s.result.TotalSnitchReward, + ) != 0 { + t.Errorf( + "total snitch incorrect have %v want %v", + sn, + s.result.TotalSnitchReward, + ) + } +} + +func defaultStateWithAccountsApplied() *state.DB { + st := ethdb.NewMemDatabase() + stateHandle, _ := state.New(common.Hash{}, state.NewDatabase(st)) + for _, addr := range []common.Address{reporterAddr, offenderAddr, randoDel} { + stateHandle.CreateAccount(addr) + } + stateHandle.SetBalance(offenderAddr, big.NewInt(0).SetUint64(1994680320000000000)) + stateHandle.SetBalance(randoDel, big.NewInt(0).SetUint64(1999975592000000000)) + return stateHandle } + +func TestTwoPercentSlashed(t *testing.T) { + slashes := exampleSlashRecords() + stateHandle := defaultStateWithAccountsApplied() + testScenario(t, stateHandle, slashes, scenarioTwoPercent) +} + +func TestEightyPercentSlashed(t *testing.T) { + slashes := exampleSlashRecords() + stateHandle := defaultStateWithAccountsApplied() + testScenario(t, stateHandle, slashes, scenarioEightyPercent) +} + +func TestRoundTripSlashRecord(t *testing.T) { + slashes := exampleSlashRecords() + serializedA := slashes.String() + data, err := rlp.EncodeToBytes(slashes) + if err != nil { + t.Errorf("encoding slash records failed %s", err.Error()) + } + roundTrip := Records{} + if err := rlp.DecodeBytes(data, &roundTrip); err != nil { + t.Errorf("decoding slash records failed %s", err.Error()) + } + serializedB := roundTrip.String() + if serializedA != serializedB { + t.Error("rlp encode/decode round trip records failed") + } +} + +// TODO bytes used for this example are stale, need to update RLP dump +// func TestApply(t *testing.T) { +// slashes := exampleSlashRecords() +// { +// stateHandle := defaultStateWithAccountsApplied() +// testScenario(t, stateHandle, slashes, scenariorealWorldSample1()) +// } +// } diff --git a/staking/slash/report.go b/staking/slash/report.go index aa11d8e10..7f0c27eb9 100644 --- a/staking/slash/report.go +++ b/staking/slash/report.go @@ -1,17 +1,43 @@ package slash import ( + "bytes" + "encoding/json" "io/ioutil" + "net/http" + "github.com/harmony-one/bls/ffi/go/bls" "gopkg.in/yaml.v2" ) +const ( + // DefaultWebHookPath .. + DefaultWebHookPath = "staking/slash/webhook.example.yaml" +) + // DoubleSignWebHooks .. type DoubleSignWebHooks struct { - WebHooks struct { + WebHooks *struct { OnNoticeDoubleSign string `yaml:"notice-double-sign"` OnThisNodeDoubleSigned string `yaml:"this-node-double-signed"` } `yaml:"web-hooks"` + Malicious *struct { + Trigger *struct { + PublicKeys []string `yaml:"list"` + DoubleSignNodeURL string `yaml:"double-sign"` + } `yaml:"trigger"` + } `yaml:"malicious"` +} + +// Contains .. +func (h *DoubleSignWebHooks) Contains(key *bls.PublicKey) bool { + hex := key.SerializeToHexStr() + for _, key := range h.Malicious.Trigger.PublicKeys { + if hex == key { + return true + } + } + return false } // NewDoubleSignWebHooksFromPath .. @@ -26,3 +52,38 @@ func NewDoubleSignWebHooksFromPath(yamlPath string) (*DoubleSignWebHooks, error) } return &t, nil } + +// ReportResult .. +type ReportResult struct { + Result string `json:"result"` + Payload string `json:"payload"` +} + +// NewSuccess .. +func NewSuccess(payload string) *ReportResult { + return &ReportResult{"success", payload} +} + +// NewFailure .. +func NewFailure(payload string) *ReportResult { + return &ReportResult{"failure", payload} +} + +// DoPost is a fire and forget helper +func DoPost(url string, record *Record) (*ReportResult, error) { + payload, err := json.Marshal(record) + if err != nil { + return nil, err + } + resp, err := http.Post(url, "application/json", bytes.NewBuffer(payload)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + result, err := ioutil.ReadAll(resp.Body) + anon := ReportResult{} + if err := json.Unmarshal(result, &anon); err != nil { + return nil, err + } + return &anon, nil +} diff --git a/staking/slash/webhook.example.yaml b/staking/slash/webhook.example.yaml new file mode 100644 index 000000000..7f5764c04 --- /dev/null +++ b/staking/slash/webhook.example.yaml @@ -0,0 +1,4 @@ +web-hooks: + # node will call these webhooks + notice-double-sign: http://localhost:5430/on-notice-double-sign + this-node-double-signed: http://localhost:5430/on-this-node-double-signed diff --git a/staking/types/commission.go b/staking/types/commission.go index 2eee82070..3277007b5 100644 --- a/staking/types/commission.go +++ b/staking/types/commission.go @@ -1,8 +1,6 @@ package types import ( - "encoding/json" - "fmt" "math/big" "github.com/harmony-one/harmony/numeric" @@ -12,52 +10,17 @@ type ( // Commission defines a commission parameters for a given validator. Commission struct { CommissionRates - UpdateHeight *big.Int + UpdateHeight *big.Int `json:"update-height"` } // CommissionRates defines the initial commission rates to be used for creating a // validator. CommissionRates struct { - Rate numeric.Dec // the commission rate charged to delegators, as a fraction - MaxRate numeric.Dec // maximum commission rate which validator can ever charge, as a fraction - MaxChangeRate numeric.Dec // maximum increase of the validator commission every epoch, as a fraction + // the commission rate charged to delegators, as a fraction + Rate numeric.Dec `json:"rate"` + // maximum commission rate which validator can ever charge, as a fraction + MaxRate numeric.Dec `json:"max-rate"` + // maximum increase of the validator commission every epoch, as a fraction + MaxChangeRate numeric.Dec `json:"max-change-rate"` } ) - -// MarshalJSON .. -func (c Commission) MarshalJSON() ([]byte, error) { - type t struct { - CommissionRates `json:"commision-rates"` - UpdateHeight string `json:"update-height"` - } - return json.Marshal(t{ - CommissionRates: c.CommissionRates, - UpdateHeight: c.UpdateHeight.String(), - }) -} - -// MarshalJSON .. -func (cr CommissionRates) MarshalJSON() ([]byte, error) { - type t struct { - Rate string `json:"rate"` - MaxRate string `json:"max-rate"` - MaxChangeRate string `json:"max-change-rate"` - } - return json.Marshal(t{ - Rate: cr.Rate.String(), - MaxRate: cr.MaxRate.String(), - MaxChangeRate: cr.MaxChangeRate.String(), - }) -} - -// String returns a human readable string representation of a validator. -func (c Commission) String() string { - return fmt.Sprintf(` - Commission: - Rate: %s - MaxRate: %s - MaxChangeRate: %s - UpdateHeight: %v`, - c.Rate, c.MaxRate, c.MaxChangeRate, - c.UpdateHeight) -} diff --git a/staking/types/delegation.go b/staking/types/delegation.go index 929e68f67..c69d10f3d 100644 --- a/staking/types/delegation.go +++ b/staking/types/delegation.go @@ -1,11 +1,14 @@ package types import ( + "encoding/json" "errors" "math/big" "sort" "github.com/ethereum/go-ethereum/common" + "github.com/harmony-one/harmony/crypto/hash" + common2 "github.com/harmony-one/harmony/internal/common" ) var ( @@ -22,16 +25,76 @@ const ( // owned by one delegator, and is associated with the voting power of one // validator. type Delegation struct { - DelegatorAddress common.Address `json:"delegator_address"` - Amount *big.Int `json:"amount"` - Reward *big.Int `json:"reward"` - Undelegations []Undelegation `json:"undelegations"` + DelegatorAddress common.Address + Amount *big.Int + Reward *big.Int + Undelegations Undelegations +} + +// Delegations .. +type Delegations []Delegation + +// String .. +func (d Delegations) String() string { + s, _ := json.Marshal(d) + return string(s) +} + +// MarshalJSON .. +func (d Delegation) MarshalJSON() ([]byte, error) { + return json.Marshal(struct { + DelegatorAddress string `json:"delegator-address"` + Amount *big.Int `json:"amount"` + Reward *big.Int `json:"reward"` + Undelegations Undelegations `json:"undelegations"` + }{common2.MustAddressToBech32(d.DelegatorAddress), d.Amount, + d.Reward, d.Undelegations, + }) +} + +func (d Delegation) String() string { + s, _ := json.Marshal(d) + return string(s) +} + +// Hash is a New256 hash of an RLP encoded Delegation +func (d Delegation) Hash() common.Hash { + return hash.FromRLPNew256(d) +} + +// SetDifference .. +func SetDifference(xs, ys []Delegation) []Delegation { + diff := []Delegation{} + xsHashed, ysHashed := + make([]common.Hash, len(xs)), make([]common.Hash, len(ys)) + for i := range xs { + xsHashed[i] = xs[i].Hash() + } + for i := range ys { + ysHashed[i] = ys[i].Hash() + for j := range xsHashed { + if ysHashed[j] != xsHashed[j] { + diff = append(diff, ys[j]) + } + } + } + + return diff } // Undelegation represents one undelegation entry type Undelegation struct { - Amount *big.Int - Epoch *big.Int + Amount *big.Int `json:"amount"` + Epoch *big.Int `json:"epoch"` +} + +// Undelegations .. +type Undelegations []Undelegation + +// String .. +func (u Undelegations) String() string { + s, _ := json.Marshal(u) + return string(s) } // DelegationIndex stored the index of a delegation in the validator's delegation list @@ -85,8 +148,8 @@ func (d *Delegation) Undelegate(epoch *big.Int, amt *big.Int) error { // TotalInUndelegation - return the total amount of token in undelegation (locking period) func (d *Delegation) TotalInUndelegation() *big.Int { total := big.NewInt(0) - for _, entry := range d.Undelegations { - total.Add(total, entry.Amount) + for i := range d.Undelegations { + total.Add(total, d.Undelegations[i].Amount) } return total } @@ -105,8 +168,11 @@ func (d *Delegation) DeleteEntry(epoch *big.Int) { } } -// RemoveUnlockedUndelegations removes all fully unlocked undelegations and returns the total sum -func (d *Delegation) RemoveUnlockedUndelegations(curEpoch, lastEpochInCommittee *big.Int) *big.Int { +// RemoveUnlockedUndelegations removes all fully unlocked +// undelegations and returns the total sum +func (d *Delegation) RemoveUnlockedUndelegations( + curEpoch, lastEpochInCommittee *big.Int, +) *big.Int { totalWithdraw := big.NewInt(0) count := 0 for j := range d.Undelegations { diff --git a/staking/types/transaction.go b/staking/types/transaction.go index 6e1ed4f39..9a4369be2 100644 --- a/staking/types/transaction.go +++ b/staking/types/transaction.go @@ -172,8 +172,31 @@ func (tx *StakingTransaction) GasPrice() *big.Int { } // Cost .. -func (tx *StakingTransaction) Cost() *big.Int { - return new(big.Int).Mul(tx.data.Price, new(big.Int).SetUint64(tx.data.GasLimit)) +func (tx *StakingTransaction) Cost() (*big.Int, error) { + total := new(big.Int).Mul(tx.data.Price, new(big.Int).SetUint64(tx.data.GasLimit)) + switch tx.StakingType() { + case DirectiveCreateValidator: + msg, err := RLPDecodeStakeMsg(tx.Data(), DirectiveCreateValidator) + if err != nil { + return nil, err + } + stkMsg, ok := msg.(*CreateValidator) + if !ok { + return nil, errStakingTransactionTypeCastErr + } + total.Add(total, stkMsg.Amount) + case DirectiveDelegate: + msg, err := RLPDecodeStakeMsg(tx.Data(), DirectiveDelegate) + if err != nil { + return nil, err + } + stkMsg, ok := msg.(*Delegate) + if !ok { + return nil, errStakingTransactionTypeCastErr + } + total.Add(total, stkMsg.Amount) + } + return total, nil } // ChainID is what chain this staking transaction for diff --git a/staking/types/validator.go b/staking/types/validator.go index ec90625f9..b03132efa 100644 --- a/staking/types/validator.go +++ b/staking/types/validator.go @@ -28,34 +28,74 @@ const ( ) var ( - errAddressNotMatch = errors.New("Validator key not match") - errInvalidSelfDelegation = errors.New("self delegation can not be less than min_self_delegation") - errInvalidTotalDelegation = errors.New("total delegation can not be bigger than max_total_delegation") - errMinSelfDelegationTooSmall = errors.New("min_self_delegation has to be greater than 1 ONE") - errInvalidMaxTotalDelegation = errors.New("max_total_delegation can not be less than min_self_delegation") - errCommissionRateTooLarge = errors.New("commission rate and change rate can not be larger than max commission rate") - errInvalidCommissionRate = errors.New("commission rate, change rate and max rate should be within 0-100 percent") - errNeedAtLeastOneSlotKey = errors.New("need at least one slot key") - errBLSKeysNotMatchSigs = errors.New("bls keys and corresponding signatures could not be verified") - errNilMinSelfDelegation = errors.New("MinSelfDelegation can not be nil") - errNilMaxTotalDelegation = errors.New("MaxTotalDelegation can not be nil") - errSlotKeyToRemoveNotFound = errors.New("slot key to remove not found") - errSlotKeyToAddExists = errors.New("slot key to add already exists") - errDuplicateSlotKeys = errors.New("slot keys can not have duplicates") + errAddressNotMatch = errors.New("Validator key not match") + errInvalidSelfDelegation = errors.New( + "self delegation can not be less than min_self_delegation", + ) + errInvalidTotalDelegation = errors.New( + "total delegation can not be bigger than max_total_delegation", + ) + errMinSelfDelegationTooSmall = errors.New( + "min_self_delegation has to be greater than 1 ONE", + ) + errInvalidMaxTotalDelegation = errors.New( + "max_total_delegation can not be less than min_self_delegation", + ) + errCommissionRateTooLarge = errors.New( + "commission rate and change rate can not be larger than max commission rate", + ) + errInvalidCommissionRate = errors.New( + "commission rate, change rate and max rate should be within 0-100 percent", + ) + errNeedAtLeastOneSlotKey = errors.New("need at least one slot key") + errBLSKeysNotMatchSigs = errors.New( + "bls keys and corresponding signatures could not be verified", + ) + errNilMinSelfDelegation = errors.New("MinSelfDelegation can not be nil") + errNilMaxTotalDelegation = errors.New("MaxTotalDelegation can not be nil") + errSlotKeyToRemoveNotFound = errors.New("slot key to remove not found") + errSlotKeyToAddExists = errors.New("slot key to add already exists") + errDuplicateSlotKeys = errors.New("slot keys can not have duplicates") ) +// ValidatorSnapshotReader .. +type ValidatorSnapshotReader interface { + ReadValidatorSnapshot(common.Address) (*ValidatorWrapper, error) +} + +type counters struct { + // The number of blocks the validator + // should've signed when in active mode (selected in committee) + NumBlocksToSign *big.Int `json:"num-blocks-to-sign",rlp:"nil"` + // The number of blocks the validator actually signed + NumBlocksSigned *big.Int `json:"num-blocks-signed",rlp:"nil"` +} + // ValidatorWrapper contains validator and its delegation information type ValidatorWrapper struct { - Validator `json:"validator"` - Delegations []Delegation `json:"delegations"` + Validator + Delegations Delegations + Counters counters +} - Snapshot struct { - Epoch *big.Int - // The number of blocks the validator should've signed when in active mode (selected in committee) - NumBlocksToSign *big.Int `rlp:"nil"` - // The number of blocks the validator actually signed - NumBlocksSigned *big.Int `rlp:"nil"` - } +func (w ValidatorWrapper) String() string { + s, _ := json.Marshal(w) + return string(s) +} + +// MarshalJSON .. +func (w ValidatorWrapper) MarshalJSON() ([]byte, error) { + return json.Marshal(struct { + Validator + Address string `json:"address"` + Delegations Delegations `json:"delegations"` + Counters counters `json:"availability"` + }{ + w.Validator, + common2.MustAddressToBech32(w.Address), + w.Delegations, + w.Counters, + }) } // VotePerShard .. @@ -85,25 +125,28 @@ type ValidatorStats struct { // Validator - data fields for a validator type Validator struct { // ECDSA address of the validator - Address common.Address + Address common.Address `json:"address"` // The BLS public key of the validator for consensus - SlotPubKeys []shard.BlsPublicKey - // The number of the last epoch this validator is selected in committee (0 means never selected) - LastEpochInCommittee *big.Int + SlotPubKeys []shard.BlsPublicKey `json:"bls-public-keys"` + // The number of the last epoch this validator is + // selected in committee (0 means never selected) + LastEpochInCommittee *big.Int `json:"last-epoch-in-committee"` // validator's self declared minimum self delegation - MinSelfDelegation *big.Int + MinSelfDelegation *big.Int `json:"min-self-delegation"` // maximum total delegation allowed - MaxTotalDelegation *big.Int - // Is the validator active in participating committee selection process or not - Active bool + MaxTotalDelegation *big.Int `json:"max-total-delegation"` + // Is the validator active in participating + // committee selection process or not + Active bool `json:"active"` // commission parameters Commission // description for the validator Description // CreationHeight is the height of creation - CreationHeight *big.Int - // Banned records whether this validator is banned from the network because they double-signed - Banned bool + CreationHeight *big.Int `json:"creation-height"` + // Banned records whether this validator is banned + // from the network because they double-signed + Banned bool `json:"banned"` } // SanityCheck checks basic requirements of a validator @@ -125,7 +168,7 @@ func (v *Validator) SanityCheck() error { } // MinSelfDelegation must be >= 1 ONE - if v.MinSelfDelegation.Cmp(big.NewInt(denominations.One)) < 0 { + if !v.Banned && v.MinSelfDelegation.Cmp(big.NewInt(denominations.One)) < 0 { return errors.Wrapf( errMinSelfDelegationTooSmall, "delegation-given %s", v.MinSelfDelegation.String(), @@ -197,35 +240,6 @@ func (v *ValidatorStats) MarshalJSON() ([]byte, error) { VotingPowerPerShard: v.VotingPowerPerShard, BLSKeyPerShard: v.BLSKeyPerShard, }) - -} - -// MarshalJSON .. -func (v *Validator) MarshalJSON() ([]byte, error) { - type t struct { - Address string `json:"one-address"` - SlotPubKeys []string `json:"bls-public-keys"` - MinSelfDelegation string `json:"min-self-delegation"` - MaxTotalDelegation string `json:"max-total-delegation"` - Active bool `json:"active"` - Commission Commission `json:"commission"` - Description Description `json:"description"` - CreationHeight uint64 `json:"creation-height"` - } - slots := make([]string, len(v.SlotPubKeys)) - for i := range v.SlotPubKeys { - slots[i] = v.SlotPubKeys[i].Hex() - } - return json.Marshal(t{ - Address: common2.MustAddressToBech32(v.Address), - SlotPubKeys: slots, - MinSelfDelegation: v.MinSelfDelegation.String(), - MaxTotalDelegation: v.MaxTotalDelegation.String(), - Active: v.Active, - Commission: v.Commission, - Description: v.Description, - CreationHeight: v.CreationHeight.Uint64(), - }) } func printSlotPubKeys(pubKeys []shard.BlsPublicKey) string { @@ -263,7 +277,7 @@ func (w *ValidatorWrapper) SanityCheck() error { errInvalidSelfDelegation, "no self delegation given at all", ) default: - if w.Delegations[0].Amount.Cmp(w.Validator.MinSelfDelegation) < 0 { + if !w.Banned && w.Delegations[0].Amount.Cmp(w.Validator.MinSelfDelegation) < 0 { return errors.Wrapf( errInvalidSelfDelegation, "have %s want %s", w.Delegations[0].Amount.String(), w.Validator.MinSelfDelegation, @@ -288,7 +302,7 @@ type Description struct { Name string `json:"name"` // name Identity string `json:"identity"` // optional identity signature (ex. UPort or Keybase) Website string `json:"website"` // optional website link - SecurityContact string `json:"security_contact"` // optional security contact info + SecurityContact string `json:"security-contact"` // optional security contact info Details string `json:"details"` // optional details } @@ -488,17 +502,7 @@ func UpdateValidatorFromEditMsg(validator *Validator, edit *EditValidator) error } // String returns a human readable string representation of a validator. -func (v *Validator) String() string { - return fmt.Sprintf(`Validator - Address: %s - SlotPubKeys: %s - LastEpochInCommittee: %v - Minimum Self Delegation: %v - Maximum Total Delegation: %v - Description: %v - Commission: %v`, - common2.MustAddressToBech32(v.Address), printSlotPubKeys(v.SlotPubKeys), - v.LastEpochInCommittee, - v.MinSelfDelegation, v.MaxTotalDelegation, v.Description, v.Commission, - ) +func (v Validator) String() string { + s, _ := json.Marshal(v) + return string(s) } diff --git a/test/chain/main.go b/test/chain/main.go index d01a75f2d..fb56189e5 100644 --- a/test/chain/main.go +++ b/test/chain/main.go @@ -134,7 +134,7 @@ func fundFaucetContract(chain *core.BlockChain) { fmt.Println(err) } block, _ := contractworker. - FinalizeNewBlock([]byte{}, []byte{}, 0, common.Address{}, nil, nil, nil) + FinalizeNewBlock([]byte{}, []byte{}, 0, common.Address{}, nil, nil) _, err = chain.InsertChain(types.Blocks{block}, true /* verifyHeaders */) if err != nil { fmt.Println(err) @@ -179,7 +179,7 @@ func callFaucetContractToFundAnAddress(chain *core.BlockChain) { fmt.Println(err) } block, _ := contractworker.FinalizeNewBlock( - []byte{}, []byte{}, 0, common.Address{}, nil, nil, nil, + []byte{}, []byte{}, 0, common.Address{}, nil, nil, ) _, err = chain.InsertChain(types.Blocks{block}, true /* verifyHeaders */) if err != nil { diff --git a/test/configs/local-resharding.txt b/test/configs/local-resharding.txt index 39cab2b2b..718332f51 100644 --- a/test/configs/local-resharding.txt +++ b/test/configs/local-resharding.txt @@ -22,4 +22,4 @@ 127.0.0.1 9107 validator one1d7jfnr6yraxnrycgaemyktkmhmajhp8kl0yahv f47238daef97d60deedbde5302d05dea5de67608f11f406576e363661f7dcbc4a1385948549b31a6c70f6fde8a391486 127.0.0.1 9108 validator one1r4zyyjqrulf935a479sgqlpa78kz7zlcg2jfen fc4b9c535ee91f015efff3f32fbb9d32cdd9bfc8a837bb3eee89b8fff653c7af2050a4e147ebe5c7233dc2d5df06ee0a 127.0.0.1 9109 validator one1p7ht2d4kl8ve7a8jxw746yfnx4wnfxtp8jqxwe ca86e551ee42adaaa6477322d7db869d3e203c00d7b86c82ebee629ad79cb6d57b8f3db28336778ec2180e56a8e07296 -127.0.0.1 9099 explorer \ No newline at end of file +127.0.0.1 9099 explorer