[project][rpc][log][config] OSTN improvements (#2420)

* [assignment] Make error case be explicit on public keys, log out important committee size info

* [committee] Do hmy nodes first, exit early if 0 externals to do

* [genesis] Never ignore errors

* [slash][availability] Remove log, remove leftover yaml

* [committee] Maintain same API as before

* [rpc] Enchance RPC of validator information for current epoch signing percent

* [availability] Remove excessive log

* [availability][slash] Unify yaml hook config

* [availability][webhooks] Call webhook when reach missing signing threshold

cmd/harmony/main.go

@@ -38,7 +38,7 @@ import (
 	"github.com/harmony-one/harmony/p2p/p2pimpl"
 	p2putils "github.com/harmony-one/harmony/p2p/utils"
 	"github.com/harmony-one/harmony/shard"
-	"github.com/harmony-one/harmony/staking/slash"
+	"github.com/harmony-one/harmony/staking/webhooks"
 	golog "github.com/ipfs/go-log"
 	"github.com/pkg/errors"
 	gologging "github.com/whyrusleeping/go-logging"
@@ -358,14 +358,14 @@ func createGlobalConfig() (*nodeconfig.ConfigType, error) {
 	nodeConfig.DBDir = *dbDir
 
 	if p := *webHookYamlPath; p != "" {
-		config, err := slash.NewDoubleSignWebHooksFromPath(p)
+		config, err := webhooks.NewWebHooksFromPath(p)
 		if err != nil {
 			fmt.Fprintf(
 				os.Stderr, "yaml path is bad: %s", p,
 			)
 			os.Exit(1)
 		}
-		nodeConfig.WebHooks.DoubleSigning = config
+		nodeConfig.WebHooks.Hooks = config
 	}
 
 	return nodeConfig, nil
@@ -442,7 +442,7 @@ func setupConsensusAndNode(nodeConfig *nodeconfig.ConfigType) *node.Node {
 
 	// TODO: refactor the creation of blockchain out of node.New()
 	currentConsensus.ChainReader = currentNode.Blockchain()
-
+	currentNode.NodeConfig.DNSZone = *dnsZone
 	// Set up prometheus pushgateway for metrics monitoring serivce.
 	currentNode.NodeConfig.SetPushgatewayIP(nodeConfig.PushgatewayIP)
 	currentNode.NodeConfig.SetPushgatewayPort(nodeConfig.PushgatewayPort)

consensus/consensus_service.go

@@ -533,7 +533,7 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode {
 
 	// update public keys in the committee
 	oldLeader := consensus.LeaderPubKey
-	pubKeys := committee.WithStakingEnabled.GetCommitteePublicKeys(
+	pubKeys, _ := committee.WithStakingEnabled.GetCommitteePublicKeys(
 		committeeToSet,
 	)
 	consensus.getLogger().Info().

core/genesis.go

@@ -310,9 +310,13 @@ func (g *Genesis) MustCommit(db ethdb.Database) *types.Block {
 		panic(err)
 	}
 	rawdb.WriteBlockRewardAccumulator(db, big.NewInt(0), 0)
-	data, _ := rlp.EncodeToBytes([]slash.Record{})
+	data, err := rlp.EncodeToBytes(slash.Records{})
-	rawdb.WritePendingSlashingCandidates(db, data)
+	if err != nil {
-
+		panic(err)
+	}
+	if err := rawdb.WritePendingSlashingCandidates(db, data); err != nil {
+		panic(err)
+	}
 	return block
 }
 

hmy/api_backend.go

@@ -2,7 +2,6 @@ package hmy
 
 import (
 	"context"
-	"errors"
 	"math/big"
 	"sync"
 
@@ -20,11 +19,14 @@ import (
 	"github.com/harmony-one/harmony/core/state"
 	"github.com/harmony-one/harmony/core/types"
 	"github.com/harmony-one/harmony/core/vm"
+	internal_common "github.com/harmony-one/harmony/internal/common"
 	"github.com/harmony-one/harmony/internal/params"
 	"github.com/harmony-one/harmony/shard"
+	"github.com/harmony-one/harmony/staking/availability"
 	"github.com/harmony-one/harmony/staking/effective"
 	"github.com/harmony-one/harmony/staking/network"
 	staking "github.com/harmony-one/harmony/staking/types"
+	"github.com/pkg/errors"
 )
 
 // APIBackend An implementation of internal/hmyapi/Backend. Full client.
@@ -323,12 +325,25 @@ func (b *APIBackend) GetAllValidatorAddresses() []common.Address {
 }
 
 // GetValidatorInformation returns the information of validator
-func (b *APIBackend) GetValidatorInformation(addr common.Address) *staking.ValidatorWrapper {
+func (b *APIBackend) GetValidatorInformation(
-	val, _ := b.hmy.BlockChain().ReadValidatorInformation(addr)
+	addr common.Address,
-	if val != nil {
+) (*staking.ValidatorRPCEnchanced, error) {
-		return val
+	wrapper, err := b.hmy.BlockChain().ReadValidatorInformation(addr)
+	if err != nil {
+		s, _ := internal_common.AddressToBech32(addr)
+		return nil, errors.Wrapf(err, "not found address in current state %s", s)
 	}
-	return nil
+	snapshot, err := b.hmy.BlockChain().ReadValidatorSnapshot(addr)
+	if err != nil {
+		s, _ := internal_common.AddressToBech32(addr)
+		return nil, errors.Wrapf(err, "not found address in snapshot %s", s)
+	}
+	signed, toSign, quotient, err := availability.ComputeCurrentSigning(snapshot, wrapper)
+	return &staking.ValidatorRPCEnchanced{
+			ValidatorWrapper:         *wrapper,
+			CurrentSigningPercentage: staking.Computed{signed, toSign, quotient},
+		},
+		nil
 }
 
 // GetMedianRawStakeSnapshot ..

internal/configs/node/config.go

@@ -13,7 +13,7 @@ import (
 	"github.com/harmony-one/harmony/internal/params"
 	"github.com/harmony-one/harmony/multibls"
 	"github.com/harmony-one/harmony/shard"
-	"github.com/harmony-one/harmony/staking/slash"
+	"github.com/harmony-one/harmony/staking/webhooks"
 	p2p_crypto "github.com/libp2p/go-libp2p-crypto"
 	"github.com/pkg/errors"
 )
@@ -91,8 +91,9 @@ type ConfigType struct {
 	networkType      NetworkType
 	shardingSchedule shardingconfig.Schedule
 	WebHooks         struct {
-		DoubleSigning *slash.DoubleSignWebHooks
+		Hooks *webhooks.Hooks
 	}
+	DNSZone string
 }
 
 // configs is a list of node configuration.

internal/hmyapi/apiv1/backend.go

@@ -73,7 +73,7 @@ type Backend interface {
 	SendStakingTx(ctx context.Context, newStakingTx *staking.StakingTransaction) error
 	GetElectedValidatorAddresses() []common.Address
 	GetAllValidatorAddresses() []common.Address
-	GetValidatorInformation(addr common.Address) *staking.ValidatorWrapper
+	GetValidatorInformation(addr common.Address) (*staking.ValidatorRPCEnchanced, error)
 	GetValidatorStats(addr common.Address) *staking.ValidatorStats
 	GetDelegationsByValidator(validator common.Address) []*staking.Delegation
 	GetDelegationsByDelegator(delegator common.Address) ([]common.Address, []*staking.Delegation)

internal/hmyapi/apiv1/blockchain.go

@@ -2,7 +2,6 @@ package apiv1
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"math/big"
 	"time"
@@ -26,6 +25,7 @@ import (
 	"github.com/harmony-one/harmony/shard"
 	"github.com/harmony-one/harmony/staking/network"
 	staking "github.com/harmony-one/harmony/staking/types"
+	"github.com/pkg/errors"
 )
 
 const (
@@ -573,21 +573,20 @@ func (s *PublicBlockChainAPI) GetValidatorMetrics(ctx context.Context, address s
 }
 
 // GetValidatorInformation returns information about a validator.
-func (s *PublicBlockChainAPI) GetValidatorInformation(ctx context.Context, address string) (*staking.ValidatorWrapper, error) {
+func (s *PublicBlockChainAPI) GetValidatorInformation(
+	ctx context.Context, address string,
+) (*staking.ValidatorRPCEnchanced, error) {
 	validatorAddress := internal_common.ParseAddr(address)
-	validator := s.b.GetValidatorInformation(validatorAddress)
+	return s.b.GetValidatorInformation(validatorAddress)
-	if validator == nil {
-		addr, _ := internal_common.AddressToBech32(validatorAddress)
-		return nil, fmt.Errorf("validator not found: %s", addr)
-	}
-	return validator, nil
 }
 
 // GetAllValidatorInformation returns information about all validators.
 // If page is -1, return all instead of `validatorsPageSize` elements.
-func (s *PublicBlockChainAPI) GetAllValidatorInformation(ctx context.Context, page int) ([]*staking.ValidatorWrapper, error) {
+func (s *PublicBlockChainAPI) GetAllValidatorInformation(
+	ctx context.Context, page int,
+) ([]*staking.ValidatorWrapper, error) {
 	if page < -1 {
-		return make([]*staking.ValidatorWrapper, 0), nil
+		return nil, errors.Errorf("page given %d cannot be less than -1", page)
 	}
 	addresses := s.b.GetAllValidatorAddresses()
 	if page != -1 && len(addresses) <= page*validatorsPageSize {
@@ -604,11 +603,11 @@ func (s *PublicBlockChainAPI) GetAllValidatorInformation(ctx context.Context, pa
 	}
 	validators := make([]*staking.ValidatorWrapper, validatorsNum)
 	for i := start; i < start+validatorsNum; i++ {
-		validators[i-start] = s.b.GetValidatorInformation(addresses[i])
+		information, err := s.b.GetValidatorInformation(addresses[i])
-		if validators[i-start] == nil {
+		if err != nil {
-			addr, _ := internal_common.AddressToBech32(addresses[i])
+			return nil, err
-			return nil, fmt.Errorf("error when getting validator info of %s", addr)
 		}
+		validators[i-start] = &information.ValidatorWrapper
 	}
 	return validators, nil
 }

internal/hmyapi/apiv1/harmony.go

@@ -56,6 +56,7 @@ type NodeMetadata struct {
 	CurrentEpoch   uint64             `json:"current-epoch"`
 	BlocksPerEpoch *uint64            `json:"blocks-per-epoch,omitempty"`
 	Role           string             `json:"role"`
+	DNSZone        string             `json:"dns-zone"`
 }
 
 // GetNodeMetadata produces a NodeMetadata record, data is from the answering RPC node
@@ -80,5 +81,6 @@ func (s *PublicHarmonyAPI) GetNodeMetadata() NodeMetadata {
 		header.Epoch().Uint64(),
 		blockEpoch,
 		cfg.Role().String(),
+		cfg.DNSZone,
 	}
 }

internal/hmyapi/apiv2/backend.go

@@ -73,7 +73,7 @@ type Backend interface {
 	SendStakingTx(ctx context.Context, newStakingTx *staking.StakingTransaction) error
 	GetElectedValidatorAddresses() []common.Address
 	GetAllValidatorAddresses() []common.Address
-	GetValidatorInformation(addr common.Address) *staking.ValidatorWrapper
+	GetValidatorInformation(addr common.Address) (*staking.ValidatorRPCEnchanced, error)
 	GetValidatorStats(addr common.Address) *staking.ValidatorStats
 	GetDelegationsByValidator(validator common.Address) []*staking.Delegation
 	GetDelegationsByDelegator(delegator common.Address) ([]common.Address, []*staking.Delegation)

internal/hmyapi/apiv2/blockchain.go

@@ -2,7 +2,6 @@ package apiv2
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"math/big"
 	"time"
@@ -26,6 +25,7 @@ import (
 	"github.com/harmony-one/harmony/shard"
 	"github.com/harmony-one/harmony/staking/network"
 	staking "github.com/harmony-one/harmony/staking/types"
+	"github.com/pkg/errors"
 )
 
 const (
@@ -537,22 +537,20 @@ func (s *PublicBlockChainAPI) GetValidatorMetrics(ctx context.Context, address s
 	return stats, nil
 }
 
-// GetValidatorInformation returns information about a validator.
+// GetValidatorInformation ..
-func (s *PublicBlockChainAPI) GetValidatorInformation(ctx context.Context, address string) (*staking.ValidatorWrapper, error) {
+func (s *PublicBlockChainAPI) GetValidatorInformation(
-	validatorAddress := internal_common.ParseAddr(address)
+	ctx context.Context, address string,
-	validator := s.b.GetValidatorInformation(validatorAddress)
+) (*staking.ValidatorRPCEnchanced, error) {
-	if validator == nil {
+	return s.GetValidatorInformation(ctx, address)
-		addr, _ := internal_common.AddressToBech32(validatorAddress)
-		return nil, fmt.Errorf("validator not found: %s", addr)
-	}
-	return validator, nil
 }
 
 // GetAllValidatorInformation returns information about all validators.
 // If page is -1, return all else return the pagination.
-func (s *PublicBlockChainAPI) GetAllValidatorInformation(ctx context.Context, page int) ([]*staking.ValidatorWrapper, error) {
+func (s *PublicBlockChainAPI) GetAllValidatorInformation(
+	ctx context.Context, page int,
+) ([]*staking.ValidatorWrapper, error) {
 	if page < -1 {
-		return make([]*staking.ValidatorWrapper, 0), nil
+		return nil, errors.Errorf("page given %d cannot be less than -1", page)
 	}
 	addresses := s.b.GetAllValidatorAddresses()
 	if page != -1 && len(addresses) <= page*validatorsPageSize {
@@ -569,11 +567,11 @@ func (s *PublicBlockChainAPI) GetAllValidatorInformation(ctx context.Context, pa
 	}
 	validators := make([]*staking.ValidatorWrapper, validatorsNum)
 	for i := start; i < start+validatorsNum; i++ {
-		validators[i-start] = s.b.GetValidatorInformation(addresses[i])
+		information, err := s.b.GetValidatorInformation(addresses[i])
-		if validators[i-start] == nil {
+		if err != nil {
-			addr, _ := internal_common.AddressToBech32(addresses[i])
+			return nil, err
-			return nil, fmt.Errorf("error when getting validator info of %s", addr)
 		}
+		validators[i-start] = &information.ValidatorWrapper
 	}
 	return validators, nil
 }

internal/hmyapi/backend.go

@@ -75,7 +75,7 @@ type Backend interface {
 	SendStakingTx(ctx context.Context, newStakingTx *staking.StakingTransaction) error
 	GetElectedValidatorAddresses() []common.Address
 	GetAllValidatorAddresses() []common.Address
-	GetValidatorInformation(addr common.Address) *staking.ValidatorWrapper
+	GetValidatorInformation(addr common.Address) (*staking.ValidatorRPCEnchanced, error)
 	GetValidatorStats(addr common.Address) *staking.ValidatorStats
 	GetDelegationsByValidator(validator common.Address) []*staking.Delegation
 	GetDelegationsByDelegator(delegator common.Address) ([]common.Address, []*staking.Delegation)

node/node.go

@@ -38,6 +38,7 @@ import (
 	"github.com/harmony-one/harmony/shard/committee"
 	"github.com/harmony-one/harmony/staking/slash"
 	staking "github.com/harmony-one/harmony/staking/types"
+	"github.com/harmony-one/harmony/staking/webhooks"
 )
 
 // State is a state of a node.
@@ -591,9 +592,11 @@ func New(host p2p.Host, consensusObj *consensus.Consensus,
 						l.Msg("double sign occured before staking era, no-op")
 						return
 					}
-					if hooks := node.NodeConfig.WebHooks.DoubleSigning; hooks != nil {
+					if hooks := node.NodeConfig.WebHooks.Hooks; hooks != nil {
-						url := hooks.WebHooks.OnNoticeDoubleSign
+						if s := hooks.Slashing; s != nil {
-						go func() { slash.DoPost(url, &doubleSign) }()
+							url := s.OnNoticeDoubleSign
+							go func() { webhooks.DoPost(url, &doubleSign) }()
+						}
 					}
 					if node.NodeConfig.ShardID != shard.BeaconChainShardID {
 						go node.BroadcastSlash(&doubleSign)
@@ -638,10 +641,10 @@ func (node *Node) InitConsensusWithValidators() (err error) {
 			Msg("[InitConsensusWithValidators] Failed getting shard state")
 		return err
 	}
-	pubKeys := committee.WithStakingEnabled.GetCommitteePublicKeys(
+	pubKeys, err := committee.WithStakingEnabled.GetCommitteePublicKeys(
 		shardState.FindCommitteeByID(shardID),
 	)
-	if len(pubKeys) == 0 {
+	if err != nil {
 		utils.Logger().Error().
 			Uint32("shardID", shardID).
 			Uint64("blockNum", blockNum).

node/node_genesis.go

@@ -92,10 +92,15 @@ func (node *Node) SetupGenesisBlock(db ethdb.Database, shardID uint32, myShardSt
 		node.AddTestingAddresses(genesisAlloc, TestAccountNumber)
 		gasLimit = params.TestGenesisGasLimit
 		// Smart contract deployer account used to deploy initial smart contract
-		contractDeployerKey, _ := ecdsa.GenerateKey(crypto.S256(), strings.NewReader("Test contract key string stream that is fixed so that generated test key are deterministic every time"))
+		contractDeployerKey, _ := ecdsa.GenerateKey(
+			crypto.S256(),
+			strings.NewReader("Test contract key string stream that is fixed so that generated test key are deterministic every time"),
+		)
 		contractDeployerAddress := crypto.PubkeyToAddress(contractDeployerKey.PublicKey)
 		contractDeployerFunds := big.NewInt(ContractDeployerInitFund)
-		contractDeployerFunds = contractDeployerFunds.Mul(contractDeployerFunds, big.NewInt(denominations.One))
+		contractDeployerFunds = contractDeployerFunds.Mul(
+			contractDeployerFunds, big.NewInt(denominations.One),
+		)
 		genesisAlloc[contractDeployerAddress] = core.GenesisAccount{Balance: contractDeployerFunds}
 		node.ContractDeployerKey = contractDeployerKey
 	}

node/node_handler.go

@@ -7,6 +7,7 @@ import (
 	"math/rand"
 	"time"
 
+	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/harmony-one/bls/ffi/go/bls"
 	"github.com/harmony-one/harmony/api/proto"
@@ -22,8 +23,10 @@ import (
 	"github.com/harmony-one/harmony/p2p"
 	"github.com/harmony-one/harmony/p2p/host"
 	"github.com/harmony-one/harmony/shard"
+	"github.com/harmony-one/harmony/staking/availability"
 	"github.com/harmony-one/harmony/staking/slash"
 	staking "github.com/harmony-one/harmony/staking/types"
+	"github.com/harmony-one/harmony/staking/webhooks"
 	libp2p_peer "github.com/libp2p/go-libp2p-core/peer"
 )
 
@@ -463,9 +466,29 @@ func (node *Node) PostConsensusProcessing(
 	if len(newBlock.Header().ShardState()) > 0 {
 		node.Consensus.UpdateConsensusInformation()
 	}
+	if h := node.NodeConfig.WebHooks.Hooks; h != nil {
+		if h.Availability != nil {
+			// TODO ask ganesh
+			addr := common.Address{}
+			wrapper, err := node.Beaconchain().ReadValidatorInformation(addr)
+			if err != nil {
+				return
+			}
+			snapshot, err := node.Beaconchain().ReadValidatorSnapshot(addr)
+			if err != nil {
+				return
+			}
+			signed, toSign, quotient, err :=
+				availability.ComputeCurrentSigning(snapshot, wrapper)
+			if availability.IsBelowSigningThreshold(quotient) {
+				url := h.Availability.DroppedBelowThreshold
+				go func() {
+					webhooks.DoPost(url, staking.Computed{signed, toSign, quotient})
+				}()
 
-	// TODO chao: uncomment this after beacon syncing is stable
+			}
-	// node.Blockchain().UpdateCXReceiptsCheckpointsByBlock(newBlock)
+		}
+	}
 }
 
 func (node *Node) pingMessageHandler(msgPayload []byte, sender libp2p_peer.ID) int {

shard/committee/assignment.go

@@ -23,7 +23,9 @@ type ValidatorListProvider interface {
 		epoch *big.Int, reader DataProvider,
 	) (*shard.State, error)
 	ReadFromDB(epoch *big.Int, reader DataProvider) (*shard.State, error)
-	GetCommitteePublicKeys(committee *shard.Committee) []*bls.PublicKey
+	GetCommitteePublicKeys(
+		committee *shard.Committee,
+	) ([]*bls.PublicKey, error)
 }
 
 // Reader is committee.Reader and it is the API that committee membership assignment needs
@@ -123,6 +125,40 @@ func eposStakedCommittee(
 		Int("staked-candidates", len(candidates)).
 		Msg("preparing epos staked committee")
 
+	shardCount := int(s.NumShards())
+	shardState := &shard.State{}
+	shardState.Shards = make([]shard.Committee, shardCount)
+	hAccounts := s.HmyAccounts()
+	shardHarmonyNodes := s.NumHarmonyOperatedNodesPerShard()
+
+	for i := 0; i < shardCount; i++ {
+		shardState.Shards[i] = shard.Committee{uint32(i), shard.SlotList{}}
+		for j := 0; j < shardHarmonyNodes; j++ {
+			index := i + j*shardCount
+			pub := &bls.PublicKey{}
+			if err := pub.DeserializeHexStr(hAccounts[index].BlsPublicKey); err != nil {
+				return nil, err
+			}
+			pubKey := shard.BlsPublicKey{}
+			if err := pubKey.FromLibBLSPublicKey(pub); err != nil {
+				return nil, err
+			}
+			shardState.Shards[i].Slots = append(shardState.Shards[i].Slots, shard.Slot{
+				common2.ParseAddr(hAccounts[index].Address),
+				pubKey,
+				nil,
+			})
+		}
+	}
+
+	if stakedSlotsCount == 0 {
+		utils.Logger().Info().
+			Int("staked-candidates", len(candidates)).
+			Int("slots-for-epos", stakedSlotsCount).
+			Msg("committe composed only of harmony node")
+		return shardState, nil
+	}
+
 	// TODO benchmark difference if went with data structure that sorts on insert
 	for i := range candidates {
 		validator, err := stakerReader.ReadValidatorInformation(candidates[i])
@@ -130,10 +166,6 @@ func eposStakedCommittee(
 			return nil, err
 		}
 		if !effective.IsEligibleForEPOSAuction(validator) {
-			utils.Logger().Info().
-				Int("staked-candidates", len(candidates)).
-				RawJSON("candidate", []byte(validator.String())).
-				Msg("validator not eligible for epos")
 			continue
 		}
 		if err := validator.SanityCheck(); err != nil {
@@ -173,36 +205,6 @@ func eposStakedCommittee(
 		}
 	}
 
-	shardCount := int(s.NumShards())
-	shardState := &shard.State{}
-	shardState.Shards = make([]shard.Committee, shardCount)
-	hAccounts := s.HmyAccounts()
-	shardHarmonyNodes := s.NumHarmonyOperatedNodesPerShard()
-
-	for i := 0; i < shardCount; i++ {
-		shardState.Shards[i] = shard.Committee{uint32(i), shard.SlotList{}}
-		for j := 0; j < shardHarmonyNodes; j++ {
-			index := i + j*shardCount
-			pub := &bls.PublicKey{}
-			pub.DeserializeHexStr(hAccounts[index].BlsPublicKey)
-			pubKey := shard.BlsPublicKey{}
-			pubKey.FromLibBLSPublicKey(pub)
-			shardState.Shards[i].Slots = append(shardState.Shards[i].Slots, shard.Slot{
-				common2.ParseAddr(hAccounts[index].Address),
-				pubKey,
-				nil,
-			})
-		}
-	}
-
-	if stakedSlotsCount == 0 {
-		utils.Logger().Info().
-			Int("staked-candidates", len(candidates)).
-			Int("slots-for-epos", stakedSlotsCount).
-			Msg("committe composed only of harmony node")
-		return shardState, nil
-	}
-
 	staked := effective.Apply(essentials, stakedSlotsCount)
 	shardBig := big.NewInt(int64(shardCount))
 
@@ -236,28 +238,35 @@ func eposStakedCommittee(
 }
 
 // GetCommitteePublicKeys returns the public keys of a shard
-func (def partialStakingEnabled) GetCommitteePublicKeys(committee *shard.Committee) []*bls.PublicKey {
+func (def partialStakingEnabled) GetCommitteePublicKeys(
+	committee *shard.Committee,
+) ([]*bls.PublicKey, error) {
 	if committee == nil {
-		utils.Logger().Error().Msg("[GetCommitteePublicKeys] Committee is nil")
+		return []*bls.PublicKey{}, nil
-		return []*bls.PublicKey{}
 	}
 	allIdentities := make([]*bls.PublicKey, len(committee.Slots))
 	for i := range committee.Slots {
 		identity := &bls.PublicKey{}
-		committee.Slots[i].BlsPublicKey.ToLibBLSPublicKey(identity)
+		if err := committee.Slots[i].BlsPublicKey.ToLibBLSPublicKey(
+			identity,
+		); err != nil {
+			return nil, err
+		}
 		allIdentities[i] = identity
 	}
 
-	return allIdentities
+	return allIdentities, nil
 }
 
+// ReadFromDB is a wrapper on ReadShardState
 func (def partialStakingEnabled) ReadFromDB(
 	epoch *big.Int, reader DataProvider,
 ) (newSuperComm *shard.State, err error) {
 	return reader.ReadShardState(epoch)
 }
 
-// ReadFromComputation is single entry point for reading the State of the network
+// Compute is single entry point for
+// computing a new super committee, aka new shard state
 func (def partialStakingEnabled) Compute(
 	epoch *big.Int, stakerReader DataProvider,
 ) (newSuperComm *shard.State, err error) {
@@ -291,5 +300,12 @@ func (def partialStakingEnabled) Compute(
 	}
 	// Set the epoch of shard state
 	shardState.Epoch = big.NewInt(0).Set(epoch)
+	staked := shardState.StakedValidators()
+	utils.Logger().Info().
+		Int("bls-key-count", staked.CountStakedBLSKey).
+		Int("validator-one-addr-count", staked.CountStakedValidator).
+		Int("max-staked-slots-count", stakedSlots).
+		Uint64("computed-for-epoch", epoch.Uint64()).
+		Msg("computed new super committee")
 	return shardState, nil
 }

shard/shard_state.go

@@ -384,7 +384,7 @@ func (c *Committee) DeepCopy() Committee {
 // BLSPublicKeys ..
 func (c *Committee) BLSPublicKeys() ([]BlsPublicKey, error) {
 	if c == nil {
-		return nil, errCommitteeNil
+		return nil, ErrCommitteeNil
 	}
 
 	slice := make([]BlsPublicKey, len(c.Slots))
@@ -397,13 +397,14 @@ func (c *Committee) BLSPublicKeys() ([]BlsPublicKey, error) {
 var (
 	// ErrValidNotInCommittee ..
 	ErrValidNotInCommittee = errors.New("slot signer not this slot's subcommittee")
-	errCommitteeNil        = errors.New("subcommittee is nil pointer")
+	// ErrCommitteeNil ..
+	ErrCommitteeNil = errors.New("subcommittee is nil pointer")
 )
 
 // AddressForBLSKey ..
 func (c *Committee) AddressForBLSKey(key BlsPublicKey) (*common.Address, error) {
 	if c == nil {
-		return nil, errCommitteeNil
+		return nil, ErrCommitteeNil
 	}
 
 	for _, slot := range c.Slots {

staking/availability/measure.go

@@ -132,9 +132,6 @@ func bumpCount(
 			return err
 		}
 
-		utils.Logger().Info().RawJSON("validator", []byte(wrapper.String())).
-			Msg("about to adjust counters")
-
 		wrapper.Counters.NumBlocksToSign.Add(
 			wrapper.Counters.NumBlocksToSign, common.Big1,
 		)
@@ -145,9 +142,6 @@ func bumpCount(
 			)
 		}
 
-		utils.Logger().Info().RawJSON("validator", []byte(wrapper.String())).
-			Msg("bumped signing counters")
-
 		if err := compute(bc, state, wrapper); err != nil {
 			return err
 		}
@@ -168,71 +162,78 @@ func IncrementValidatorSigningCounts(
 	state *state.DB,
 	signers, missing shard.SlotList,
 ) error {
-	utils.Logger().Info().
-		RawJSON("missing", []byte(missing.String())).
-		Msg("signers that did sign")
-
-	utils.Logger().Info().
-		Msg("bumping signing counters for non-missing signers")
-
 	if err := bumpCount(
 		bc, state, signers, true, staked.LookupSet,
 	); err != nil {
 		return err
 	}
-	utils.Logger().Info().
-		Msg("bumping missing signers counters")
 	return bumpCount(bc, state, missing, false, staked.LookupSet)
 }
 
 // Reader ..
 type Reader interface {
-	ReadValidatorSnapshot(addr common.Address) (*staking.ValidatorWrapper, error)
+	ReadValidatorSnapshot(
+		addr common.Address,
+	) (*staking.ValidatorWrapper, error)
 }
 
-// compute sets the validator to
+// ComputeCurrentSigning returns (signed, toSign, quotient, error)
-// inactive and thereby keeping it out of
+func ComputeCurrentSigning(
-// consideration in the pool of validators for
+	snapshot, wrapper *staking.ValidatorWrapper,
-// whenever committee selection happens in future, the
+) (*big.Int, *big.Int, numeric.Dec, error) {
-// signing threshold is 66%
-func compute(
-	bc Reader,
-	state *state.DB,
-	wrapper *staking.ValidatorWrapper,
-) error {
-	snapshot, err := bc.ReadValidatorSnapshot(wrapper.Address)
-	if err != nil {
-		return err
-	}
-
 	statsNow, snapSigned, snapToSign :=
 		wrapper.Counters,
 		snapshot.Counters.NumBlocksSigned,
 		snapshot.Counters.NumBlocksToSign
 
-	utils.Logger().Info().
-		RawJSON("snapshot", []byte(snapshot.String())).
-		RawJSON("current", []byte(wrapper.String())).
-		Msg("begin checks for availability")
-
 	signed, toSign :=
 		new(big.Int).Sub(statsNow.NumBlocksSigned, snapSigned),
 		new(big.Int).Sub(statsNow.NumBlocksToSign, snapToSign)
 
 	if signed.Sign() == -1 {
-		return errors.Wrapf(
+		return nil, nil, numeric.ZeroDec(), errors.Wrapf(
 			errNegativeSign, "diff for signed period wrong: stat %s, snapshot %s",
 			statsNow.NumBlocksSigned.String(), snapSigned.String(),
 		)
 	}
 
 	if toSign.Sign() == -1 {
-		return errors.Wrapf(
+		return nil, nil, numeric.ZeroDec(), errors.Wrapf(
 			errNegativeSign, "diff for toSign period wrong: stat %s, snapshot %s",
 			statsNow.NumBlocksToSign.String(), snapToSign.String(),
 		)
 	}
 
+	s1, s2 :=
+		numeric.NewDecFromBigInt(signed), numeric.NewDecFromBigInt(toSign)
+	quotient := s1.Quo(s2)
+	return signed, toSign, quotient, nil
+}
+
+// IsBelowSigningThreshold ..
+func IsBelowSigningThreshold(quotient numeric.Dec) bool {
+	return quotient.LTE(measure)
+}
+
+// compute sets the validator to
+// inactive and thereby keeping it out of
+// consideration in the pool of validators for
+// whenever committee selection happens in future, the
+// signing threshold is 66%
+func compute(
+	bc Reader,
+	state *state.DB,
+	wrapper *staking.ValidatorWrapper,
+) error {
+	utils.Logger().Info().Msg("begin compute for availability")
+
+	snapshot, err := bc.ReadValidatorSnapshot(wrapper.Address)
+	if err != nil {
+		return err
+	}
+
+	signed, toSign, quotient, err := ComputeCurrentSigning(snapshot, wrapper)
+
 	if toSign.Cmp(common.Big0) == 0 {
 		utils.Logger().Info().
 			RawJSON("snapshot", []byte(snapshot.String())).
@@ -241,22 +242,22 @@ func compute(
 		return nil
 	}
 
-	s1, s2 :=
+	if err != nil {
-		numeric.NewDecFromBigInt(signed), numeric.NewDecFromBigInt(toSign)
+		return err
-	quotient := s1.Quo(s2)
+	}
 
 	utils.Logger().Info().
 		RawJSON("snapshot", []byte(snapshot.String())).
 		RawJSON("current", []byte(wrapper.String())).
-		Str("signed", s1.String()).
+		Str("signed", signed.String()).
-		Str("to-sign", s2.String()).
+		Str("to-sign", toSign.String()).
 		Str("percentage-signed", quotient.String()).
 		Bool("meets-threshold", quotient.LTE(measure)).
 		Msg("check if signing percent is meeting required threshold")
 
 	const missedTooManyBlocks = true
 
-	switch quotient.LTE(measure) {
+	switch IsBelowSigningThreshold(quotient) {
 	case missedTooManyBlocks:
 		wrapper.Active = false
 		utils.Logger().Info().

staking/types/validator.go

@@ -78,6 +78,30 @@ type ValidatorWrapper struct {
 	Counters    counters
 }
 
+// Computed ..
+type Computed struct {
+	Signed     *big.Int    `json:"current-epoch-signed"`
+	ToSign     *big.Int    `json:"current-epoch-to-sign"`
+	Percentage numeric.Dec `json:"percentage"`
+}
+
+// ValidatorRPCEnchanced contains extra information for RPC consumer
+type ValidatorRPCEnchanced struct {
+	ValidatorWrapper
+	CurrentSigningPercentage Computed
+}
+
+// MarshalJSON ..
+func (w ValidatorRPCEnchanced) MarshalJSON() ([]byte, error) {
+	return json.Marshal(struct {
+		ValidatorWrapper
+		CurrentSigningPercentage Computed `json:"current-epoch-signing-percent"`
+	}{
+		w.ValidatorWrapper,
+		w.CurrentSigningPercentage,
+	})
+}
+
 func (w ValidatorWrapper) String() string {
 	s, _ := json.Marshal(w)
 	return string(s)

staking/webhooks/yaml.go

@@ -1,4 +1,4 @@
-package slash
+package webhooks
 
 import (
 	"bytes"
@@ -6,51 +6,28 @@ import (
 	"io/ioutil"
 	"net/http"
 
-	"github.com/harmony-one/bls/ffi/go/bls"
 	"gopkg.in/yaml.v2"
 )
 
 const (
 	// DefaultWebHookPath ..
-	DefaultWebHookPath = "staking/slash/webhook.example.yaml"
+	DefaultWebHookPath = "staking/webhooks/webhook.example.yaml"
 )
 
+// AvailabilityHooks ..
+type AvailabilityHooks struct {
+	DroppedBelowThreshold string `yaml:"dropped-below-threshold"`
+}
+
 // DoubleSignWebHooks ..
 type DoubleSignWebHooks struct {
-	WebHooks *struct {
+	OnNoticeDoubleSign string `yaml:"notice-double-sign"`
-		OnNoticeDoubleSign     string `yaml:"notice-double-sign"`
-		OnThisNodeDoubleSigned string `yaml:"this-node-double-signed"`
-	} `yaml:"web-hooks"`
-	Malicious *struct {
-		Trigger *struct {
-			PublicKeys        []string `yaml:"list"`
-			DoubleSignNodeURL string   `yaml:"double-sign"`
-		} `yaml:"trigger"`
-	} `yaml:"malicious"`
 }
 
-// Contains ..
+// Hooks ..
-func (h *DoubleSignWebHooks) Contains(key *bls.PublicKey) bool {
+type Hooks struct {
-	hex := key.SerializeToHexStr()
+	Slashing     *DoubleSignWebHooks `yaml:"slashing-hooks"`
-	for _, key := range h.Malicious.Trigger.PublicKeys {
+	Availability *AvailabilityHooks  `yaml:"availability-hooks"`
-		if hex == key {
-			return true
-		}
-	}
-	return false
-}
-
-// NewDoubleSignWebHooksFromPath ..
-func NewDoubleSignWebHooksFromPath(yamlPath string) (*DoubleSignWebHooks, error) {
-	rawYAML, err := ioutil.ReadFile(yamlPath)
-	if err != nil {
-		return nil, err
-	}
-	t := DoubleSignWebHooks{}
-	if err := yaml.UnmarshalStrict(rawYAML, &t); err != nil {
-		return nil, err
-	}
-	return &t, nil
 }
 
 // ReportResult ..
@@ -70,7 +47,7 @@ func NewFailure(payload string) *ReportResult {
 }
 
 // DoPost is a fire and forget helper
-func DoPost(url string, record *Record) (*ReportResult, error) {
+func DoPost(url string, record interface{}) (*ReportResult, error) {
 	payload, err := json.Marshal(record)
 	if err != nil {
 		return nil, err
@@ -87,3 +64,16 @@ func DoPost(url string, record *Record) (*ReportResult, error) {
 	}
 	return &anon, nil
 }
+
+// NewWebHooksFromPath ..
+func NewWebHooksFromPath(yamlPath string) (*Hooks, error) {
+	rawYAML, err := ioutil.ReadFile(yamlPath)
+	if err != nil {
+		return nil, err
+	}
+	t := Hooks{}
+	if err := yaml.UnmarshalStrict(rawYAML, &t); err != nil {
+		return nil, err
+	}
+	return &t, nil
+}