Make sure crosslink is verified before proposal

core/blockchain.go

@@ -1156,28 +1156,6 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types.
 			return NonStatTy, err
 		}
 
-		// Staking epoch migration. Only Execute once:
-		// Normally the last block of an epoch should have the same epoch as this block
-		// In the case beacon chain catch up, it may have a epoch larger than the current epoch
-		// We need to write the same shard state for this one-off epoch so other readers doesn't break
-		parentHeader := bc.GetHeaderByHash(header.ParentHash())
-		if parentHeader != nil && parentHeader.Epoch().Cmp(header.Epoch()) != 0 {
-			curShardState, err := bc.ReadShardState(parentHeader.Epoch())
-			if err != nil {
-				header.Logger(utils.Logger()).Warn().Err(err).Msg("cannot read current shard state")
-				return NonStatTy, err
-			}
-			data, err := rlp.EncodeToBytes(curShardState)
-			if err != nil {
-				return NonStatTy, err
-			}
-			_, err = bc.WriteShardStateBytes(batch, header.Epoch(), data)
-			if err != nil {
-				header.Logger(utils.Logger()).Warn().Err(err).Msg("cannot store shard state")
-				return NonStatTy, err
-			}
-		}
-
 		// Find all the active validator addresses and store them in db
 		allActiveValidators := []common.Address{}
 		processed := make(map[common.Address]struct{})
@@ -1223,10 +1201,10 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types.
 
 		if shardState, err = shard.DecodeWrapper(block.Header().ShardState()); err == nil {
 			if err = bc.UpdateValidatorVotingPower(shardState); err != nil {
-				utils.Logger().Err(err)
+				utils.Logger().Err(err).Msg("[UpdateValidatorVotingPower] Failed to update voting power")
 			}
 		} else {
-			utils.Logger().Err(err)
+			utils.Logger().Err(err).Msg("[UpdateValidatorVotingPower] Failed to decode shard state")
 		}
 	}
 

internal/chain/reward.go

@@ -4,8 +4,6 @@ import (
 	"math/big"
 	"sort"
 
-	"github.com/harmony-one/harmony/shard/committee"
-
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/harmony-one/bls/ffi/go/bls"
@@ -258,9 +256,6 @@ func AccumulateRewards(
 			for i := range crossLinks {
 				cxLink := crossLinks[i]
 				shardState, err := bc.ReadShardState(cxLink.Epoch())
-				if !bc.Config().IsStaking(cxLink.Epoch()) {
-					shardState, err = committee.WithStakingEnabled.Compute(cxLink.Epoch(), bc)
-				}
 
 				if err != nil {
 					return err

node/node_cross_shard.go

@@ -157,11 +157,6 @@ func (node *Node) ProcessCrossLinkMessage(msgPayload []byte) {
 			Msgf("[ProcessingCrossLink] Crosslink going to propose: %d", len(crosslinks))
 
 		for _, cl := range crosslinks {
-			if cl.Number() == nil || cl.Epoch().Cmp(node.Blockchain().Config().CrossLinkEpoch) < 0 {
-				utils.Logger().Debug().
-					Msgf("[ProcessingCrossLink] Crosslink blockNum %d epochNum %d shard %d skipped: %v", cl.BlockNum(), cl.Epoch().Uint64(), cl.ShardID(), cl)
-				continue
-			}
 			exist, err := node.Blockchain().ReadCrossLink(cl.ShardID(), cl.Number().Uint64())
 			if err == nil && exist != nil {
 				// TODO: leader add double sign checking
@@ -170,7 +165,12 @@ func (node *Node) ProcessCrossLinkMessage(msgPayload []byte) {
 				continue
 			}
 
-			err = node.VerifyCrossLink(cl)
+			if err = node.VerifyCrossLink(cl); err != nil {
+				utils.Logger().Debug().
+					Msgf("[ProcessingCrossLink] Crosslink blockNum %d epochNum %d shard %d skipped: %v", cl.BlockNum(), cl.Epoch().Uint64(), cl.ShardID(), cl)
+				continue
+			}
+
 			if err != nil {
 				utils.Logger().Error().
 					Err(err).
@@ -228,6 +228,10 @@ func (node *Node) verifyIncomingReceipts(block *types.Block) error {
 
 // VerifyCrossLink verifies the header is valid
 func (node *Node) VerifyCrossLink(cl types.CrossLink) error {
+	if node.Blockchain().ShardID() != shard.BeaconChainShardID {
+		return ctxerror.New("Shard chains should not verify cross links")
+	}
+
 	if cl.BlockNum() <= 1 {
 		return ctxerror.New("CrossLink BlockNumber should greater than 1")
 	}

node/node_newblock.go

@@ -5,6 +5,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/harmony-one/harmony/shard"
+
 	"github.com/harmony-one/harmony/core/rawdb"
 
 	types2 "github.com/harmony-one/harmony/staking/types"
@@ -80,13 +82,6 @@ func (node *Node) WaitForConsensusReadyV2(readySignal chan struct{}, stopChan ch
 func (node *Node) proposeNewBlock() (*types.Block, error) {
 	node.Worker.UpdateCurrent()
 
-	// Prepare shard state
-	// NOTE: this will potentially override shard chain's epoch to beacon chain's epoch during staking migration period.
-	// So this needs to be executed early on.
-	shardState, err := node.Worker.SuperCommitteeForNextEpoch(
-		node.Consensus.ShardID, node.Beaconchain(),
-	)
-
 	// Update worker's current header and state data in preparation to propose/process new transactions
 	coinbase := node.Consensus.SelfAddress
 
@@ -139,6 +134,9 @@ func (node *Node) proposeNewBlock() (*types.Block, error) {
 
 		if err == nil {
 			for _, pending := range allPending {
+				if err = node.VerifyCrossLink(pending); err != nil {
+					continue
+				}
 				exist, err := node.Blockchain().ReadCrossLink(pending.ShardID(), pending.BlockNum())
 				if err == nil || exist != nil {
 					continue
@@ -151,7 +149,11 @@ func (node *Node) proposeNewBlock() (*types.Block, error) {
 		}
 	}
 
-	if err != nil {
+	// Prepare shard state
+	shardState := new(shard.State)
+	if shardState, err = node.Worker.SuperCommitteeForNextEpoch(
+		node.Consensus.ShardID, node.Beaconchain(),
+	); err != nil {
 		return nil, err
 	}
 

node/worker/worker.go

@@ -262,11 +262,16 @@ func (w *Worker) GetNewEpoch() *big.Int {
 	parent := w.chain.CurrentBlock()
 	epoch := new(big.Int).Set(parent.Header().Epoch())
 
-	// TODO: Don't depend on sharding state for epoch change.
+	shardState, err := parent.Header().GetShardState()
+	if err == nil && shardState.Epoch != nil {
+		// For shard state of staking epochs, the shard state will have an epoch and it will decide the next epoch for following blocks
+		epoch = new(big.Int).Set(shardState.Epoch)
+	} else {
 		if len(parent.Header().ShardState()) > 0 && parent.NumberU64() != 0 {
-		// ... except if parent has a resharding assignment it increases by 1.
+			// if parent has proposed a new shard state it increases by 1, except for genesis block.
 			epoch = epoch.Add(epoch, common.Big1)
 		}
+	}
 	return epoch
 }
 
@@ -315,15 +320,11 @@ func (w *Worker) SuperCommitteeForNextEpoch(
 					beaconEpoch, beacon,
 				)
 
-				// Set this block's epoch to be beaconEpoch - 1, so the next block will have beaconEpoch
-				// This shouldn't be exactly beaconEpoch because the next block will have beaconEpoch + 1
-				blockEpoch := big.NewInt(0).Set(beaconEpoch).Sub(beaconEpoch, big.NewInt(1))
 				utils.Logger().Debug().
 					Uint64("blockNum", w.current.header.Number().Uint64()).
-					Uint64("myPrevEpoch", w.current.header.Epoch().Uint64()).
+					Uint64("myCurEpoch", w.current.header.Epoch().Uint64()).
-					Uint64("myCurEpoch", blockEpoch.Uint64()).
+					Uint64("beaconEpoch", beaconEpoch.Uint64()).
 					Msg("Propose new epoch as beacon chain's epoch")
-				w.current.header.SetEpoch(blockEpoch)
 			case 0:
 				// If it's same epoch, no need to propose new shard state (new epoch change)
 			case -1:
@@ -337,20 +338,17 @@ func (w *Worker) SuperCommitteeForNextEpoch(
 					beaconEpoch, beacon,
 				)
 
-				blockEpoch := big.NewInt(0).Set(beaconEpoch).Sub(beaconEpoch, big.NewInt(1))
 				utils.Logger().Debug().
 					Uint64("blockNum", w.current.header.Number().Uint64()).
-					Uint64("myPrevEpoch", w.current.header.Epoch().Uint64()).
+					Uint64("myCurEpoch", w.current.header.Epoch().Uint64()).
-					Uint64("myCurEpoch", blockEpoch.Uint64()).
+					Uint64("beaconEpoch", beaconEpoch.Uint64()).
-					Msg("Propose one-time catch up with beacon chain's epoch")
+					Msg("Propose entering staking along with beacon chain's epoch")
-				// Set this block's epoch to be beaconEpoch - 1, so the next block will have beaconEpoch
-				w.current.header.SetEpoch(blockEpoch)
 			} else {
 				// If I are not in staking nor has beacon chain proposed a staking-based shard state,
 				// do pre-staking committee calculation
 				if shard.Schedule.IsLastBlock(w.current.header.Number().Uint64()) {
 					nextCommittee, err = committee.WithStakingEnabled.Compute(
-						new(big.Int).Add(w.current.header.Epoch(), common.Big1),
+						nextEpoch,
 						w.chain,
 					)
 				}

shard/committee/assignment.go

@@ -117,6 +117,11 @@ func eposStakedCommittee(
 	// TODO benchmark difference if went with data structure that sorts on insert
 	for i := range candidates {
 		validator, err := stakerReader.ReadValidatorInformation(candidates[i])
+
+		if err != nil {
+			return nil, err
+		}
+
 		if err := validator.SanityCheck(); err != nil {
 			continue
 		}
@@ -124,9 +129,6 @@ func eposStakedCommittee(
 		for _, delegation := range validator.Delegations {
 			validatorStake.Add(validatorStake, delegation.Amount)
 		}
-		if err != nil {
-			return nil, err
-		}
 		essentials[validator.Address] = effective.SlotOrder{
 			validatorStake,
 			validator.SlotPubKeys,
@@ -221,10 +223,18 @@ func (def partialStakingEnabled) Compute(
 
 	instance := shard.Schedule.InstanceForEpoch(epoch)
 	if preStaking {
+		// Pre-staking shard state doesn't need to set epoch (backward compatible)
 		return preStakingEnabledCommittee(instance), nil
 	}
 	stakedSlots :=
 		(instance.NumNodesPerShard() - instance.NumHarmonyOperatedNodesPerShard()) *
 			int(instance.NumShards())
-	return eposStakedCommittee(instance, stakerReader, stakedSlots)
+	shardState, err := eposStakedCommittee(instance, stakerReader, stakedSlots)
+
+	if err != nil {
+		return nil, err
+	}
+	// Set the epoch of shard state
+	shardState.Epoch = big.NewInt(0).Set(epoch)
+	return shardState, nil
 }

shard/shard_state.go

@@ -96,6 +96,7 @@ func DecodeWrapper(shardState []byte) (*State, error) {
 				})
 			}
 		}
+		newSS.Epoch = nil // Make sure for legacy state, the epoch is nil
 		return &newSS, nil
 	}
 	return nil, err2