[availability] Count availability for shard chain, more log, remove error of div by zero as error condition (#2342)

* [availability] More log, remove error of div by zero

* [availability] Apply increment of count in accumulate since need crosslink

* [availability] More log

* [staking] VC easy flow of create validator and delegation, restart watchdog

* [availability] Use method instead of inline, fix Cmp mistake in measure

* [availability] Add explicit log for measure

* [availability] Begin unit test impl

* [availability] Factor out subset of methods that availability needs, expand test

* [availability] Do not increment count when LastEpochInCommittee is 0

* [availability] Remove dev scripts

* [availability] Adjust logs

* [availability] Remove test BLS keys

* [availability] Remove check on LastEpoch in bumpCount, remove spurious log

* [availability] Further simplify, current shard state is the elected validator

* [consensus] Remove log

* [project] Remove noisy, hard to know where happened log; whitespace on style

consensus/votepower/roster.go

@@ -7,7 +7,6 @@ import (
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/harmony-one/bls/ffi/go/bls"
-	"github.com/harmony-one/harmony/internal/utils"
 	"github.com/harmony-one/harmony/numeric"
 	"github.com/harmony-one/harmony/shard"
 	staking "github.com/harmony-one/harmony/staking/types"
@@ -202,23 +201,12 @@ func Compute(staked shard.SlotList) (*Roster, error) {
 	if diff := numeric.OneDec().Sub(
 		ourPercentage.Add(theirPercentage),
 	); !diff.IsZero() && lastStakedVoter != nil {
-		utils.Logger().Info().
-			Str("theirs", theirPercentage.String()).
-			Str("ours", ourPercentage.String()).
-			Str("diff", diff.String()).
-			Str("combined", theirPercentage.Add(diff).Add(ourPercentage).String()).
-			Str("bls-public-key-of-receipent", lastStakedVoter.Identity.Hex()).
-			Msg("voting power of hmy & staked slots not sum to 1, giving diff to staked slot")
 		lastStakedVoter.EffectivePercent = lastStakedVoter.EffectivePercent.Add(diff)
 		theirPercentage = theirPercentage.Add(diff)
 	}
 
 	if lastStakedVoter != nil &&
 		!ourPercentage.Add(theirPercentage).Equal(numeric.OneDec()) {
-		utils.Logger().Error().
-			Str("theirs", theirPercentage.String()).
-			Str("ours", ourPercentage.String()).
-			Msg("Total voting power not equal 100 percent")
 		return nil, ErrVotingPowerNotEqualOne
 	}
 

core/offchain.go

@@ -92,25 +92,10 @@ func (bc *BlockChain) CommitOffChainData(
 			return NonStatTy, err
 		}
 
-		// Find all the elected validator addresses and store them in db
-		allElectedValidators := []common.Address{}
-		processed := make(map[common.Address]struct{})
-		for i := range newShardState.Shards {
-			shard := newShardState.Shards[i]
-			for j := range shard.Slots {
-				slot := shard.Slots[j]
-				if slot.EffectiveStake != nil { // For external validator
-					_, ok := processed[slot.EcdsaAddress]
-					if !ok {
-						processed[slot.EcdsaAddress] = struct{}{}
-						allElectedValidators = append(allElectedValidators, shard.Slots[j].EcdsaAddress)
-					}
-				}
-			}
-		}
-
 		// Update elected validators
-		if err := bc.WriteElectedValidatorList(batch, allElectedValidators); err != nil {
+		if err := bc.WriteElectedValidatorList(
+			batch, newShardState.StakedValidators().Addrs,
+		); err != nil {
 			return NonStatTy, err
 		}
 

core/state_processor.go

@@ -68,12 +68,11 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.DB, cfg vm.C
 	var (
 		receipts types.Receipts
 		outcxs   types.CXReceipts
-
-		incxs   = block.IncomingReceipts()
-		usedGas = new(uint64)
-		header  = block.Header()
-		allLogs []*types.Log
-		gp      = new(GasPool).AddGas(block.GasLimit())
+		incxs    = block.IncomingReceipts()
+		usedGas  = new(uint64)
+		header   = block.Header()
+		allLogs  []*types.Log
+		gp       = new(GasPool).AddGas(block.GasLimit())
 	)
 	beneficiary, err := p.bc.GetECDSAFromCoinbase(header)
 

internal/chain/engine.go

@@ -144,7 +144,7 @@ func (e *engineImpl) VerifyShardState(bc engine.ChainReader, beacon engine.Chain
 	}
 	headerShardStateBytes := header.ShardState()
 	// TODO: figure out leader withhold shardState
-	if headerShardStateBytes == nil || len(headerShardStateBytes) == 0 {
+	if len(headerShardStateBytes) == 0 {
 		return nil
 	}
 	shardState, err := bc.SuperCommitteeForNextEpoch(beacon, header, true)
@@ -240,7 +240,10 @@ func (e *engineImpl) VerifySeal(chain engine.ChainReader, header *block.Header)
 	lastCommitPayload := append(blockNumHash, parentHash[:]...)
 
 	if !aggSig.VerifyHash(mask.AggregatePublic, lastCommitPayload) {
-		return ctxerror.New("[VerifySeal] Unable to verify aggregated signature from last block", "lastBlockNum", header.Number().Uint64()-1, "lastBlockHash", parentHash)
+		const msg = "[VerifySeal] Unable to verify aggregated signature from last block"
+		return ctxerror.New(
+			msg, "lastBlockNum", header.Number().Uint64()-1, "lastBlockHash", parentHash,
+		)
 	}
 	return nil
 }
@@ -254,7 +257,6 @@ func (e *engineImpl) Finalize(
 	incxs []*types.CXReceiptsProof, stks []*staking.StakingTransaction,
 	doubleSigners slash.Records,
 ) (*types.Block, *big.Int, error) {
-
 	// Accumulate block rewards and commit the final state root
 	// Header seems complete, assemble into a block and return
 	payout, err := AccumulateRewards(
@@ -272,7 +274,8 @@ func (e *engineImpl) Finalize(
 	if isBeaconChain && isNewEpoch && inStakingEra {
 		validators, err := chain.ReadValidatorList()
 		if err != nil {
-			return nil, nil, ctxerror.New("[Finalize] failed to read all validators").WithCause(err)
+			const msg = "[Finalize] failed to read all validators"
+			return nil, nil, ctxerror.New(msg).WithCause(err)
 		}
 		// Payout undelegated/unlocked tokens
 		for _, validator := range validators {
@@ -304,56 +307,42 @@ func (e *engineImpl) Finalize(
 		Uint64("block-number", header.Number().Uint64())
 
 	if isBeaconChain && inStakingEra {
-		superCommittee, err := chain.ReadShardState(
-			chain.CurrentHeader().Epoch(),
-		)
+		nowEpoch := chain.CurrentHeader().Epoch()
+		superCommittee, err := chain.ReadShardState(nowEpoch)
+		if err != nil {
+			return nil, nil, err
+		}
 		staked := superCommittee.StakedValidators()
 		// could happen that only harmony nodes are running,
-		if staked.CountStakedValidator > 0 {
-			l.RawJSON("external", []byte(staked.StateSubset.String())).
-				Msg("have non-zero external")
-
-			if err != nil {
-				return nil, nil, err
-			}
-
-			l.Msg("bumping validator signing counts")
-			if err := availability.IncrementValidatorSigningCounts(
-				chain, header, header.ShardID(), state, staked.LookupSet,
+		if isNewEpoch && staked.CountStakedValidator > 0 {
+			l.Msg("in new epoch (aka last block), apply availability check for activity")
+			if err := availability.SetInactiveUnavailableValidators(
+				chain, state, staked.Addrs,
 			); err != nil {
 				return nil, nil, err
 			}
+			// Now can reset the counters, do note, only
+			// after the availability logic runs
+			newShardState, err := header.GetShardState()
+			if err != nil {
+				const msg = "[Finalize] failed to read shard state"
+				return nil, nil, ctxerror.New(msg).WithCause(err)
+			}
 
-			if isNewEpoch {
-				l.Msg("in new epoch (aka last block), apply availability check for activity")
-				if err := availability.SetInactiveUnavailableValidators(
-					chain, state,
-				); err != nil {
-					return nil, nil, err
-				}
-				// Now can reset the counters, do note, only
-				// after the availability logic runs
-				newShardState, err := header.GetShardState()
-				if err != nil {
-					const msg = "[Finalize] failed to read shard state"
-					return nil, nil, ctxerror.New(msg).WithCause(err)
-				}
-
-				if stkd := newShardState.StakedValidators(); stkd.CountStakedValidator > 0 {
-					for _, addr := range stkd.Addrs {
-						wrapper, err := state.ValidatorWrapper(addr)
-						if err != nil {
-							return nil, nil, err
-						}
-						// Set the LastEpochInCommittee field for all
-						// external validators in the upcoming epoch.
-						// and set the availability tracking counters to 0
-						wrapper.LastEpochInCommittee = newShardState.Epoch
-						if err := state.UpdateValidatorWrapper(addr, wrapper); err != nil {
-							return nil, nil, ctxerror.New(
-								"[Finalize] failed update validator info",
-							).WithCause(err)
-						}
+			if stkd := newShardState.StakedValidators(); stkd.CountStakedValidator > 0 {
+				for _, addr := range stkd.Addrs {
+					wrapper, err := state.ValidatorWrapper(addr)
+					if err != nil {
+						return nil, nil, err
+					}
+					// Set the LastEpochInCommittee field for all
+					// external validators in the upcoming epoch.
+					// and set the availability tracking counters to 0
+					wrapper.LastEpochInCommittee = newShardState.Epoch
+					if err := state.UpdateValidatorWrapper(addr, wrapper); err != nil {
+						return nil, nil, ctxerror.New(
+							"[Finalize] failed update validator info",
+						).WithCause(err)
 					}
 				}
 			}
@@ -479,7 +468,9 @@ func (e *engineImpl) VerifyHeaderWithSignature(chain engine.ChainReader, header
 }
 
 // GetPublicKeys finds the public keys of the committee that signed the block header
-func GetPublicKeys(chain engine.ChainReader, header *block.Header, reCalculate bool) ([]*bls.PublicKey, error) {
+func GetPublicKeys(
+	chain engine.ChainReader, header *block.Header, reCalculate bool,
+) ([]*bls.PublicKey, error) {
 	shardState := new(shard.State)
 	var err error
 	if reCalculate {
@@ -499,13 +490,10 @@ func GetPublicKeys(chain engine.ChainReader, header *block.Header, reCalculate b
 			"shardID", header.ShardID(),
 		)
 	}
-	var committerKeys []*bls.PublicKey
-
-	utils.Logger().Print(committee.Slots)
+	committerKeys := []*bls.PublicKey{}
 	for _, member := range committee.Slots {
 		committerKey := new(bls.PublicKey)
-		err := member.BlsPublicKey.ToLibBLSPublicKey(committerKey)
-		if err != nil {
+		if err := member.BlsPublicKey.ToLibBLSPublicKey(committerKey); err != nil {
 			return nil, ctxerror.New("cannot convert BLS public key",
 				"blsPublicKey", member.BlsPublicKey).WithCause(err)
 		}

internal/chain/reward.go

@@ -48,11 +48,11 @@ func AccumulateRewards(
 	//// After staking
 	if bc.Config().IsStaking(header.Epoch()) &&
 		bc.CurrentHeader().ShardID() == shard.BeaconChainShardID {
-
 		defaultReward := network.BaseStakedReward
-
 		// TODO Use cached result in off-chain db instead of full computation
-		_, percentageStaked, err := network.WhatPercentStakedNow(beaconChain, header.Time().Int64())
+		_, percentageStaked, err := network.WhatPercentStakedNow(
+			beaconChain, header.Time().Int64(),
+		)
 		if err != nil {
 			return network.NoReward, err
 		}
@@ -73,11 +73,21 @@ func AccumulateRewards(
 		newRewards := big.NewInt(0)
 
 		// Take care of my own beacon chain committee, _ is missing, for slashing
-		members, payable, _, err := ballotResultBeaconchain(beaconChain, header)
+		members, payable, missing, err := ballotResultBeaconchain(beaconChain, header)
 		if err != nil {
 			return network.NoReward, err
 		}
 
+		if err := availability.IncrementValidatorSigningCounts(
+			beaconChain,
+			shard.Committee{shard.BeaconChainShardID, members}.StakedValidators(),
+			state,
+			payable,
+			missing,
+		); err != nil {
+			return network.NoReward, err
+		}
+
 		votingPower, err := votepower.Compute(members)
 		if err != nil {
 			return network.NoReward, err
@@ -130,13 +140,21 @@ func AccumulateRewards(
 				}
 
 				subComm := shardState.FindCommitteeByID(cxLink.ShardID())
-				// _ are the missing signers, later for slashing
-				payableSigners, _, err := availability.BlockSigners(cxLink.Bitmap(), subComm)
+				payableSigners, missing, err := availability.BlockSigners(
+					cxLink.Bitmap(), subComm,
+				)
 
 				if err != nil {
 					return network.NoReward, err
 				}
 
+				staked := subComm.StakedValidators()
+				if err := availability.IncrementValidatorSigningCounts(
+					beaconChain, staked, state, payableSigners, missing,
+				); err != nil {
+					return network.NoReward, err
+				}
+
 				votingPower, err := votepower.Compute(payableSigners)
 				if err != nil {
 					return network.NoReward, err

node/node_handler.go

@@ -315,8 +315,7 @@ func (node *Node) BroadcastCrossLink(newBlock *types.Block) {
 // VerifyNewBlock is called by consensus participants to verify the block (account model) they are
 // running consensus on
 func (node *Node) VerifyNewBlock(newBlock *types.Block) error {
-	err := node.Blockchain().Validator().ValidateHeader(newBlock, true)
-	if err != nil {
+	if err := node.Blockchain().Validator().ValidateHeader(newBlock, true); err != nil {
 		utils.Logger().Error().
 			Str("blockHash", newBlock.Hash().Hex()).
 			Err(err).
@@ -327,6 +326,7 @@ func (node *Node) VerifyNewBlock(newBlock *types.Block) error {
 			newBlock.Hash(),
 		).WithCause(err)
 	}
+
 	if newBlock.ShardID() != node.Blockchain().ShardID() {
 		utils.Logger().Error().
 			Uint32("my shard ID", node.Blockchain().ShardID()).
@@ -334,13 +334,13 @@ func (node *Node) VerifyNewBlock(newBlock *types.Block) error {
 			Msg("wrong shard ID")
 		return ctxerror.New("wrong shard ID",
 			"my shard ID", node.Blockchain().ShardID(),
-			"new block's shard ID", newBlock.ShardID())
+			"new block's shard ID", newBlock.ShardID(),
+		)
 	}
 
-	err = node.Blockchain().Engine().VerifyShardState(
+	if err := node.Blockchain().Engine().VerifyShardState(
 		node.Blockchain(), node.Beaconchain(), newBlock.Header(),
-	)
-	if err != nil {
+	); err != nil {
 		utils.Logger().Error().
 			Str("blockHash", newBlock.Hash().Hex()).
 			Err(err).
@@ -351,8 +351,7 @@ func (node *Node) VerifyNewBlock(newBlock *types.Block) error {
 		).WithCause(err)
 	}
 
-	err = node.Blockchain().ValidateNewBlock(newBlock)
-	if err != nil {
+	if err := node.Blockchain().ValidateNewBlock(newBlock); err != nil {
 		utils.Logger().Error().
 			Str("blockHash", newBlock.Hash().Hex()).
 			Int("numTx", len(newBlock.Transactions())).
@@ -367,7 +366,7 @@ func (node *Node) VerifyNewBlock(newBlock *types.Block) error {
 
 	// Verify cross links
 	// TODO: move into ValidateNewBlock
-	if node.NodeConfig.ShardID == 0 {
+	if node.NodeConfig.ShardID == shard.BeaconChainShardID {
 		err := node.VerifyBlockCrossLinks(newBlock)
 		if err != nil {
 			utils.Logger().Debug().Err(err).Msg("ops2 VerifyBlockCrossLinks Failed")
@@ -376,8 +375,7 @@ func (node *Node) VerifyNewBlock(newBlock *types.Block) error {
 	}
 
 	// TODO: move into ValidateNewBlock
-	err = node.verifyIncomingReceipts(newBlock)
-	if err != nil {
+	if err := node.verifyIncomingReceipts(newBlock); err != nil {
 		utils.Logger().Error().
 			Str("blockHash", newBlock.Hash().Hex()).
 			Int("numIncomingReceipts", len(newBlock.IncomingReceipts())).

shard/shard_state.go

@@ -56,6 +56,15 @@ type Committee struct {
 	Slots   SlotList `json:"subcommittee"`
 }
 
+func (l SlotList) String() string {
+	blsKeys := make([]string, len(l))
+	for i, k := range l {
+		blsKeys[i] = k.BlsPublicKey.Hex()
+	}
+	s, _ := json.Marshal(blsKeys)
+	return string(s)
+}
+
 /* Legacy
 These are the pre-staking used data-structures, needed to maintain
 compatibilty for RLP decode/encode
@@ -141,7 +150,6 @@ func EncodeWrapper(shardState State, isStaking bool) ([]byte, error) {
 type StakedSlots struct {
 	CountStakedValidator int
 	CountStakedBLSKey    int
-	StateSubset          *State
 	Addrs                []common.Address
 	LookupSet            map[common.Address]struct{}
 }
@@ -149,14 +157,40 @@ type StakedSlots struct {
 // StakedValidators filters for non-harmony operated nodes,
 // returns (
 //   totalStakedValidatorsCount, totalStakedBLSKeys,
-//   stateSubset, addrsOnNetworkSlice, addrsOnNetworkSet,
+//   addrsOnNetworkSlice, addrsOnNetworkSet,
 // )
-func (ss *State) StakedValidators() *StakedSlots {
-	onlyExternal := &State{
-		Epoch:  ss.Epoch,
-		Shards: make([]Committee, len(ss.Shards)),
+func (c Committee) StakedValidators() *StakedSlots {
+	countStakedValidator, countStakedBLSKey := 0, 0
+	networkWideSlice, networkWideSet :=
+		[]common.Address{}, map[common.Address]struct{}{}
+	for _, slot := range c.Slots {
+
+		// an external validator,
+		// non-nil EffectiveStake is how we known
+		if addr := slot.EcdsaAddress; slot.EffectiveStake != nil {
+			countStakedBLSKey++
+			if _, seen := networkWideSet[addr]; !seen {
+				countStakedValidator++
+				networkWideSet[addr] = struct{}{}
+				networkWideSlice = append(networkWideSlice, addr)
+			}
+		}
+	}
+
+	return &StakedSlots{
+		CountStakedValidator: countStakedValidator,
+		CountStakedBLSKey:    countStakedBLSKey,
+		Addrs:                networkWideSlice,
+		LookupSet:            networkWideSet,
 	}
+}
 
+// StakedValidators filters for non-harmony operated nodes,
+// returns (
+//   totalStakedValidatorsCount, totalStakedBLSKeys,
+//   addrsOnNetworkSlice, addrsOnNetworkSet,
+// )
+func (ss *State) StakedValidators() *StakedSlots {
 	countStakedValidator, countStakedBLSKey := 0, 0
 	networkWideSlice, networkWideSet :=
 		[]common.Address{},
@@ -165,17 +199,12 @@ func (ss *State) StakedValidators() *StakedSlots {
 	for i := range ss.Shards {
 		shard := ss.Shards[i]
 		for j := range shard.Slots {
-			onlyExternal.Shards[i].ShardID = shard.ShardID
-			onlyExternal.Shards[i].Slots = SlotList{}
+
 			slot := shard.Slots[j]
 			// an external validator,
 			// non-nil EffectiveStake is how we known
 			if addr := slot.EcdsaAddress; slot.EffectiveStake != nil {
 				countStakedBLSKey++
-				onlyExternal.Shards[i].Slots = append(
-					onlyExternal.Shards[i].Slots, slot,
-				)
-
 				if _, seen := networkWideSet[addr]; !seen {
 					countStakedValidator++
 					networkWideSet[addr] = struct{}{}
@@ -188,7 +217,6 @@ func (ss *State) StakedValidators() *StakedSlots {
 	return &StakedSlots{
 		CountStakedValidator: countStakedValidator,
 		CountStakedBLSKey:    countStakedBLSKey,
-		StateSubset:          onlyExternal,
 		Addrs:                networkWideSlice,
 		LookupSet:            networkWideSet,
 	}

staking/availability/measure.go

@@ -11,12 +11,14 @@ import (
 	bls2 "github.com/harmony-one/harmony/crypto/bls"
 	"github.com/harmony-one/harmony/internal/ctxerror"
 	"github.com/harmony-one/harmony/internal/utils"
+	"github.com/harmony-one/harmony/numeric"
 	"github.com/harmony-one/harmony/shard"
+	staking "github.com/harmony-one/harmony/staking/types"
 	"github.com/pkg/errors"
 )
 
 var (
-	measure                    = new(big.Int).Div(common.Big2, common.Big3)
+	measure                    = numeric.NewDec(2).Quo(numeric.NewDec(3))
 	errValidatorEpochDeviation = errors.New(
 		"validator snapshot epoch not exactly one epoch behind",
 	)
@@ -110,7 +112,7 @@ func BallotResult(
 }
 
 func bumpCount(
-	chain engine.ChainReader,
+	bc Reader,
 	state *state.DB,
 	signers shard.SlotList,
 	didSign bool,
@@ -158,23 +160,29 @@ func bumpCount(
 
 // IncrementValidatorSigningCounts ..
 func IncrementValidatorSigningCounts(
-	chain engine.ChainReader, header *block.Header,
-	shardID uint32, state *state.DB,
-	stakedAddrSet map[common.Address]struct{},
+	bc Reader,
+	staked *shard.StakedSlots,
+	state *state.DB,
+	signers, missing shard.SlotList,
 ) error {
-	l := utils.Logger().Info().Str("candidate-header", header.String())
-	_, signers, missing, err := BallotResult(chain, header, shardID)
-	if err != nil {
-		return err
-	}
+	l := utils.Logger().Info()
+	l.RawJSON("missing", []byte(missing.String())).
+		Msg("signers that did sign")
+
 	l.Msg("bumping signing counters for non-missing signers")
+
 	if err := bumpCount(
-		chain, state, signers, true, stakedAddrSet,
+		bc, state, signers, true, staked.LookupSet,
 	); err != nil {
 		return err
 	}
-	l.Msg("bumping missed signing counter ")
-	return bumpCount(chain, state, missing, false, stakedAddrSet)
+	l.Msg("bumping missing signers counters")
+	return bumpCount(bc, state, missing, false, staked.LookupSet)
+}
+
+// Reader ..
+type Reader interface {
+	ReadValidatorSnapshot(addr common.Address) (*staking.ValidatorWrapper, error)
 }
 
 // SetInactiveUnavailableValidators sets the validator to
@@ -183,14 +191,8 @@ func IncrementValidatorSigningCounts(
 // whenever committee selection happens in future, the
 // signing threshold is 66%
 func SetInactiveUnavailableValidators(
-	bc engine.ChainReader, state *state.DB,
+	bc Reader, state *state.DB, addrs []common.Address,
 ) error {
-	addrs, err := bc.ReadElectedValidatorList()
-	if err != nil {
-		return err
-	}
-
-	now := bc.CurrentHeader().Epoch()
 	for i := range addrs {
 		snapshot, err := bc.ReadValidatorSnapshot(addrs[i])
 		if err != nil {
@@ -203,9 +205,8 @@ func SetInactiveUnavailableValidators(
 			return err
 		}
 
-		statsNow, snapEpoch, snapSigned, snapToSign :=
+		statsNow, snapSigned, snapToSign :=
 			wrapper.Counters,
-			snapshot.LastEpochInCommittee,
 			snapshot.Counters.NumBlocksSigned,
 			snapshot.Counters.NumBlocksToSign
 
@@ -215,18 +216,6 @@ func SetInactiveUnavailableValidators(
 
 		l.Msg("begin checks for availability")
 
-		if snapEpoch.Cmp(common.Big0) == 0 {
-			l.Msg("pass newly joined validator for inactivity check")
-			continue
-		}
-
-		if d := new(big.Int).Sub(now, snapEpoch); d.Cmp(common.Big1) != 0 {
-			return errors.Wrapf(
-				errValidatorEpochDeviation, "bc %s, snapshot %s",
-				now.String(), snapEpoch.String(),
-			)
-		}
-
 		signed, toSign :=
 			new(big.Int).Sub(statsNow.NumBlocksSigned, snapSigned),
 			new(big.Int).Sub(statsNow.NumBlocksToSign, snapToSign)
@@ -246,10 +235,21 @@ func SetInactiveUnavailableValidators(
 		}
 
 		if toSign.Cmp(common.Big0) == 0 {
-			return ErrDivByZero
+			l.Msg("toSign is 0, perhaps did not receive crosslink proving signing")
+			continue
 		}
 
-		if r := new(big.Int).Div(signed, toSign); r.Cmp(measure) == -1 {
+		s1, s2 :=
+			numeric.NewDecFromBigInt(signed), numeric.NewDecFromBigInt(toSign)
+		quotient := s1.Quo(s2)
+
+		l.Str("signed", s1.String()).
+			Str("to-sign", s2.String()).
+			Str("percentage-signed", quotient.String()).
+			Bool("meets-threshold", quotient.LTE(measure)).
+			Msg("check if signing percent is meeting required threshold")
+
+		if quotient.LTE(measure) {
 			wrapper.Active = false
 			l.Str("threshold", measure.String()).
 				Msg("validator failed availability threshold, set to inactive")

staking/availability/measure_test.go

@@ -0,0 +1,66 @@
+package availability
+
+import (
+	"math/big"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/harmony-one/harmony/core/state"
+	common2 "github.com/harmony-one/harmony/internal/common"
+	staking "github.com/harmony-one/harmony/staking/types"
+)
+
+type fakerAuctioneer struct{}
+
+const (
+	to0 = "one1zyxauxquys60dk824p532jjdq753pnsenrgmef"
+	to2 = "one14438psd5vrjes7qm97jrj3t0s5l4qff5j5cn4h"
+)
+
+var (
+	validatorS0Addr, validatorS2Addr = common.Address{}, common.Address{}
+	addrs                            = []common.Address{}
+	validatorS0, validatorS2         = &staking.ValidatorWrapper{}, &staking.ValidatorWrapper{}
+)
+
+func init() {
+	validatorS0Addr, _ = common2.Bech32ToAddress(to0)
+	validatorS2Addr, _ = common2.Bech32ToAddress(to2)
+	addrs = []common.Address{validatorS0Addr, validatorS2Addr}
+}
+
+func (fakerAuctioneer) ReadValidatorSnapshot(
+	addr common.Address,
+) (*staking.ValidatorWrapper, error) {
+	switch addr {
+	case validatorS0Addr:
+		return validatorS0, nil
+	case validatorS2Addr:
+		return validatorS0, nil
+	default:
+		panic("bad input in test case")
+	}
+}
+
+func defaultStateWithAccountsApplied() *state.DB {
+	st := ethdb.NewMemDatabase()
+	stateHandle, _ := state.New(common.Hash{}, state.NewDatabase(st))
+	for _, addr := range addrs {
+		stateHandle.CreateAccount(addr)
+	}
+	stateHandle.SetBalance(validatorS0Addr, big.NewInt(0).SetUint64(1994680320000000000))
+	stateHandle.SetBalance(validatorS2Addr, big.NewInt(0).SetUint64(1999975592000000000))
+	return stateHandle
+}
+
+func TestSetInactiveUnavailableValidators(t *testing.T) {
+	state := defaultStateWithAccountsApplied()
+	if err := SetInactiveUnavailableValidators(
+		fakerAuctioneer{}, state, addrs,
+	); err != nil {
+		//
+	}
+
+	t.Log("Unimplemented")
+}