diff --git a/consensus/engine/consensus_engine.go b/consensus/engine/consensus_engine.go index 5ac2c776b..e77bd66df 100644 --- a/consensus/engine/consensus_engine.go +++ b/consensus/engine/consensus_engine.go @@ -143,5 +143,5 @@ type Engine interface { receipts []*types.Receipt, outcxs []*types.CXReceipt, incxs []*types.CXReceiptsProof, stks staking.StakingTransactions, doubleSigners slash.Records, sigsReady chan bool, viewID func() uint64, - ) (*types.Block, reward.Reader, error) + ) (*types.Block, map[common.Address][]common.Address, reward.Reader, error) } diff --git a/core/blockchain.go b/core/blockchain.go index 45140349a..0347db79d 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -108,6 +108,7 @@ type BlockChain interface { block *types.Block, receipts []*types.Receipt, cxReceipts []*types.CXReceipt, stakeMsgs []types2.StakeMsg, + delegationsToRemove map[common.Address][]common.Address, paid reward.Reader, state *state.DB, ) (status WriteStatus, err error) @@ -299,6 +300,7 @@ type BlockChain interface { UpdateStakingMetaData( batch rawdb.DatabaseWriter, block *types.Block, stakeMsgs []types2.StakeMsg, + delegationsToRemove map[common.Address][]common.Address, state *state.DB, epoch, newEpoch *big.Int, ) (newValidators []common.Address, err error) // ReadBlockRewardAccumulator must only be called on beaconchain @@ -339,6 +341,7 @@ type BlockChain interface { receipts []*types.Receipt, cxReceipts []*types.CXReceipt, stakeMsgs []types2.StakeMsg, + delegationsToRemove map[common.Address][]common.Address, payout reward.Reader, state *state.DB, ) (status WriteStatus, err error) diff --git a/core/blockchain_impl.go b/core/blockchain_impl.go index 20b323339..26613b821 100644 --- a/core/blockchain_impl.go +++ b/core/blockchain_impl.go @@ -539,7 +539,7 @@ func (bc *BlockChainImpl) validateNewBlock(block *types.Block) error { // NOTE Order of mutating state here matters. // Process block using the parent state as reference point. // Do not read cache from processor. - receipts, cxReceipts, _, _, usedGas, _, _, err := bc.processor.Process( + receipts, cxReceipts, _, _, _, usedGas, _, _, err := bc.processor.Process( block, state, bc.vmConfig, false, ) if err != nil { @@ -1460,6 +1460,7 @@ func (bc *BlockChainImpl) WriteBlockWithState( block *types.Block, receipts []*types.Receipt, cxReceipts []*types.CXReceipt, stakeMsgs []staking.StakeMsg, + delegationsToRemove map[common.Address][]common.Address, paid reward.Reader, state *state.DB, ) (status WriteStatus, err error) { @@ -1560,7 +1561,7 @@ func (bc *BlockChainImpl) WriteBlockWithState( // Write offchain data if status, err := bc.CommitOffChainData( batch, block, receipts, - cxReceipts, stakeMsgs, + cxReceipts, stakeMsgs, delegationsToRemove, paid, state, ); err != nil { return status, err @@ -1883,7 +1884,7 @@ func (bc *BlockChainImpl) insertChain(chain types.Blocks, verifyHeaders bool) (i } // Process block using the parent state as reference point. substart := time.Now() - receipts, cxReceipts, stakeMsgs, logs, usedGas, payout, newState, err := bc.processor.Process( + receipts, cxReceipts, stakeMsgs, delegationsToRemove, logs, usedGas, payout, newState, err := bc.processor.Process( block, state, vmConfig, true, ) state = newState // update state in case the new state is cached. @@ -1920,7 +1921,7 @@ func (bc *BlockChainImpl) insertChain(chain types.Blocks, verifyHeaders bool) (i // Write the block to the chain and get the status. substart = time.Now() status, err := bc.WriteBlockWithState( - block, receipts, cxReceipts, stakeMsgs, payout, state, + block, receipts, cxReceipts, stakeMsgs, delegationsToRemove, payout, state, ) if err != nil { return i, events, coalescedLogs, err @@ -3065,6 +3066,7 @@ func (bc *BlockChainImpl) writeDelegationsByDelegator( func (bc *BlockChainImpl) UpdateStakingMetaData( batch rawdb.DatabaseWriter, block *types.Block, stakeMsgs []staking.StakeMsg, + delegationsToRemove map[common.Address][]common.Address, state *state.DB, epoch, newEpoch *big.Int, ) (newValidators []common.Address, err error) { newValidators, newDelegations, err := bc.prepareStakingMetaData(block, stakeMsgs, state) @@ -3119,6 +3121,16 @@ func (bc *BlockChainImpl) UpdateStakingMetaData( if err := bc.writeDelegationsByDelegator(batch, addr, delegations); err != nil { return newValidators, err } + + + + for delegatorAddress, validatorAddresses := range delegationsToRemove { + if err := bc.RemoveDelegationsFromDelegator(batch, delegatorAddress, validatorAddresses); err != nil { + return newValidators, err + } + } + + } return newValidators, nil } @@ -3149,7 +3161,7 @@ func (bc *BlockChainImpl) prepareStakingMetaData( return nil, nil, err } } else { - panic("Only *staking.Delegate stakeMsgs are supported at the moment") + return nil, nil, errors.New("Only *staking.Delegate stakeMsgs are supported at the moment") } } for _, txn := range block.StakingTransactions() { @@ -3277,7 +3289,7 @@ func (bc *BlockChainImpl) addDelegationIndex( } } - // Found the delegation from state and add the delegation index + // Find the delegation from state and add the delegation index (the position in validator) // Note this should read from the state of current block in concern wrapper, err := state.ValidatorWrapper(validatorAddress, true, false) if err != nil { @@ -3293,6 +3305,11 @@ func (bc *BlockChainImpl) addDelegationIndex( Index: uint64(i), BlockNum: blockNum, }) + + // wrapper.Delegations will not have another delegator + // with the same address, so we are done + break + } } return delegations, nil diff --git a/core/blockchain_stub.go b/core/blockchain_stub.go index 5d83149a6..f3148f8c9 100644 --- a/core/blockchain_stub.go +++ b/core/blockchain_stub.go @@ -113,7 +113,7 @@ func (a Stub) WriteBlockWithoutState(block *types.Block, td *big.Int) (err error return errors.Errorf("method WriteBlockWithoutState not implemented for %s", a.Name) } -func (a Stub) WriteBlockWithState(block *types.Block, receipts []*types.Receipt, cxReceipts []*types.CXReceipt, stakeMsgs []staking.StakeMsg, paid reward.Reader, state *state.DB) (status WriteStatus, err error) { +func (a Stub) WriteBlockWithState(block *types.Block, receipts []*types.Receipt, cxReceipts []*types.CXReceipt, stakeMsgs []staking.StakeMsg, delegationsToRemove map[common.Address][]common.Address, paid reward.Reader, state *state.DB) (status WriteStatus, err error) { return 0, errors.Errorf("method WriteBlockWithState not implemented for %s", a.Name) } @@ -373,7 +373,7 @@ func (a Stub) ReadDelegationsByDelegatorAt(delegator common.Address, blockNum *b return nil, errors.Errorf("method ReadDelegationsByDelegatorAt not implemented for %s", a.Name) } -func (a Stub) UpdateStakingMetaData(batch rawdb.DatabaseWriter, block *types.Block, stakeMsgs []staking.StakeMsg, state *state.DB, epoch, newEpoch *big.Int) (newValidators []common.Address, err error) { +func (a Stub) UpdateStakingMetaData(batch rawdb.DatabaseWriter, block *types.Block, stakeMsgs []staking.StakeMsg, delegationsToRemove map[common.Address][]common.Address, state *state.DB, epoch, newEpoch *big.Int) (newValidators []common.Address, err error) { return nil, errors.Errorf("method UpdateStakingMetaData not implemented for %s", a.Name) } @@ -412,7 +412,7 @@ func (a Stub) IsEnablePruneBeaconChainFeature() bool { return false } -func (a Stub) CommitOffChainData(batch rawdb.DatabaseWriter, block *types.Block, receipts []*types.Receipt, cxReceipts []*types.CXReceipt, stakeMsgs []staking.StakeMsg, payout reward.Reader, state *state.DB) (status WriteStatus, err error) { +func (a Stub) CommitOffChainData(batch rawdb.DatabaseWriter, block *types.Block, receipts []*types.Receipt, cxReceipts []*types.CXReceipt, stakeMsgs []staking.StakeMsg, delegationsToRemove map[common.Address][]common.Address, payout reward.Reader, state *state.DB) (status WriteStatus, err error) { return 0, errors.Errorf("method CommitOffChainData not implemented for %s", a.Name) } diff --git a/core/evm.go b/core/evm.go index e11726a56..c8a661919 100644 --- a/core/evm.go +++ b/core/evm.go @@ -243,7 +243,7 @@ func DelegateFn(ref *block.Header, chain ChainContext) vm.DelegateFunc { func UndelegateFn(ref *block.Header, chain ChainContext) vm.UndelegateFunc { // moved from state_transition.go to here, with some modifications return func(db vm.StateDB, rosettaTracer vm.RosettaTracer, undelegate *stakingTypes.Undelegate) error { - wrapper, err := VerifyAndUndelegateFromMsg(db, ref.Epoch(), undelegate) + wrapper, err := VerifyAndUndelegateFromMsg(db, ref.Epoch(), chain.Config(), undelegate) if err != nil { return err } diff --git a/core/evm_test.go b/core/evm_test.go index 04399ff14..3305d972f 100644 --- a/core/evm_test.go +++ b/core/evm_test.go @@ -32,7 +32,7 @@ func getTestEnvironment(testBankKey ecdsa.PrivateKey) (*BlockChainImpl, *state.D var ( testBankAddress = crypto.PubkeyToAddress(testBankKey.PublicKey) testBankFunds = new(big.Int).Mul(big.NewInt(denominations.One), big.NewInt(40000)) - chainConfig = params.TestChainConfig + chainConfig = params.LocalnetChainConfig blockFactory = blockfactory.ForTest database = rawdb.NewMemoryDatabase() gspec = Genesis{ @@ -50,7 +50,7 @@ func getTestEnvironment(testBankKey ecdsa.PrivateKey) (*BlockChainImpl, *state.D chain, _ := NewBlockChain(database, nil, nil, cacheConfig, gspec.Config, engine, vm.Config{}) db, _ := chain.StateAt(genesis.Root()) - // make a fake block header (use epoch 1 so that locked tokens can be tested) + // make a fake block header header := blockFactory.NewHeader(common.Big0) return chain, db, header, database @@ -119,6 +119,51 @@ func TestEVMStaking(t *testing.T) { t.Errorf("Got error %v in Undelegate", err) } + // undelegate test - epoch 3 to test NoNilDelegationsEpoch case + delegatorKey, _ := crypto.GenerateKey() + ctx3 := NewEVMContext(msg, blockfactory.ForTest.NewHeader(common.Big3), chain, nil) + delegate = sampleDelegate(*delegatorKey) // 1000 ONE + db.AddBalance(delegate.DelegatorAddress, delegate.Amount) + delegate.ValidatorAddress = wrapper.Address + err = ctx.Delegate(db, nil, &delegate) + if err != nil { + t.Errorf("Got error %v in Delegate for new delegator", err) + } + undelegate = sampleUndelegate(*delegatorKey) + // try undelegating such that remaining < minimum (100 ONE) + undelegate.ValidatorAddress = wrapper.Address + undelegate.Amount = new(big.Int).Mul(big.NewInt(denominations.One), big.NewInt(901)) + err = ctx3.Undelegate(db, nil, &undelegate) + if err == nil { + t.Errorf("Got no error in Undelegate for new delegator") + } else { + if err.Error() != "Minimum: 100000000000000000000, Remaining: 99000000000000000000: remaining delegation must be 0 or >= 100 ONE" { + t.Errorf("Got error %v but expected %v", err, staking.ErrUndelegationRemaining) + } + } + // now undelegate such that remaining == minimum (100 ONE) + undelegate.Amount = new(big.Int).Mul(big.NewInt(denominations.One), big.NewInt(900)) + err = ctx3.Undelegate(db, nil, &undelegate) + if err != nil { + t.Errorf("Got error %v in Undelegate for new delegator", err) + } + // remaining < 100 ONE after remaining = minimum + undelegate.Amount = new(big.Int).Mul(big.NewInt(denominations.One), big.NewInt(1)) + err = ctx3.Undelegate(db, nil, &undelegate) + if err == nil { + t.Errorf("Got no error in Undelegate for new delegator") + } else { + if err.Error() != "Minimum: 100000000000000000000, Remaining: 99000000000000000000: remaining delegation must be 0 or >= 100 ONE" { + t.Errorf("Got error %v but expected %v", err, staking.ErrUndelegationRemaining) + } + } + // remaining == 0 + undelegate.Amount = new(big.Int).Mul(big.NewInt(denominations.One), big.NewInt(100)) + err = ctx3.Undelegate(db, nil, &undelegate) + if err != nil { + t.Errorf("Got error %v in Undelegate for new delegator", err) + } + // collectRewards test collectRewards := sampleCollectRewards(*key) // add block rewards to make sure there are some to collect diff --git a/core/offchain.go b/core/offchain.go index 6ce7794e4..341a6b7f6 100644 --- a/core/offchain.go +++ b/core/offchain.go @@ -28,6 +28,7 @@ func (bc *BlockChainImpl) CommitOffChainData( receipts []*types.Receipt, cxReceipts []*types.CXReceipt, stakeMsgs []staking.StakeMsg, + delegationsToRemove map[common.Address][]common.Address, payout reward.Reader, state *state.DB, ) (status WriteStatus, err error) { @@ -118,7 +119,7 @@ func (bc *BlockChainImpl) CommitOffChainData( // Do bookkeeping for new staking txns newVals, err := bc.UpdateStakingMetaData( - batch, block, stakeMsgs, state, epoch, nextBlockEpoch, + batch, block, stakeMsgs, delegationsToRemove, state, epoch, nextBlockEpoch, ) if err != nil { utils.Logger().Err(err).Msg("UpdateStakingMetaData failed") @@ -327,3 +328,33 @@ func (bc *BlockChainImpl) getNextBlockEpoch(header *block.Header) (*big.Int, err } return nextBlockEpoch, nil } + +func (bc *BlockChainImpl) RemoveDelegationsFromDelegator( + batch rawdb.DatabaseWriter, + delegatorAddress common.Address, + validatorAddresses []common.Address, +) error { + delegationIndexes, err := bc.ReadDelegationsByDelegator(delegatorAddress) + if err != nil { + return err + } + finalDelegationIndexes := delegationIndexes[:0] + for _, validatorAddress := range validatorAddresses { + // TODO: can this be sped up from O(vd) to something shorter? + for _, delegationIndex := range delegationIndexes { + if bytes.Equal( + validatorAddress.Bytes(), + delegationIndex.ValidatorAddress.Bytes(), + ) { + // do nothing + break + } + finalDelegationIndexes = append( + finalDelegationIndexes, + delegationIndex, + ) + } + } + bc.writeDelegationsByDelegator(batch, delegatorAddress, finalDelegationIndexes) + return nil +} diff --git a/core/staking_verifier.go b/core/staking_verifier.go index c36d24e68..a99f85a98 100644 --- a/core/staking_verifier.go +++ b/core/staking_verifier.go @@ -367,7 +367,7 @@ func VerifyAndDelegateFromMsg( // // Note that this function never updates the stateDB, it only reads from stateDB. func VerifyAndUndelegateFromMsg( - stateDB vm.StateDB, epoch *big.Int, msg *staking.Undelegate, + stateDB vm.StateDB, epoch *big.Int, chainConfig *params.ChainConfig, msg *staking.Undelegate, ) (*staking.ValidatorWrapper, error) { if stateDB == nil { return nil, errStateDBIsMissing @@ -389,10 +389,15 @@ func VerifyAndUndelegateFromMsg( return nil, err } + var minimumRemainingDelegation *big.Int + if chainConfig.IsNoNilDelegations(epoch) { + minimumRemainingDelegation = minimumDelegationV2 // 100 ONE + } + for i := range wrapper.Delegations { delegation := &wrapper.Delegations[i] if bytes.Equal(delegation.DelegatorAddress.Bytes(), msg.DelegatorAddress.Bytes()) { - if err := delegation.Undelegate(epoch, msg.Amount); err != nil { + if err := delegation.Undelegate(epoch, msg.Amount, minimumRemainingDelegation); err != nil { return nil, err } if err := wrapper.SanityCheck(); err != nil { diff --git a/core/staking_verifier_test.go b/core/staking_verifier_test.go index 53d47a478..38e641a41 100644 --- a/core/staking_verifier_test.go +++ b/core/staking_verifier_test.go @@ -1194,8 +1194,9 @@ func TestVerifyAndUndelegateFromMsg(t *testing.T) { epoch *big.Int msg staking.Undelegate - expVWrapper staking.ValidatorWrapper - expErr error + expVWrapper staking.ValidatorWrapper + expErr error + noNilDelegationsEpoch *big.Int }{ { // 0: Undelegate at delegation with an entry already exist at the same epoch. @@ -1362,9 +1363,70 @@ func TestVerifyAndUndelegateFromMsg(t *testing.T) { expErr: errNoDelegationToUndelegate, }, + + { + // 12: Undelegate with NoNilDelegationsEpoch set + // such that remaining < minimum + sdb: makeDefaultStateForUndelegate(t), // delegatorAddr has 15k ones delegated + epoch: big.NewInt(defaultEpoch), + msg: func() staking.Undelegate { + msg := defaultMsgUndelegate() + msg.Amount = new(big.Int).Mul(oneBig, big.NewInt(14901)) + return msg + }(), + expVWrapper: makeDefaultSnapVWrapperForUndelegate(t), + expErr: staking.ErrUndelegationRemaining, + noNilDelegationsEpoch: big.NewInt(defaultEpoch), + }, + { + // 13: Undelegate with NoNilDelegationsEpoch set + // such that remaining == minimum + // delegatorAddr has 15k ones delegated and 5k in Undelegations at defaultEpoch + sdb: makeDefaultStateForUndelegate(t), + epoch: big.NewInt(defaultEpoch), + msg: func() staking.Undelegate { + msg := defaultMsgUndelegate() + msg.Amount = new(big.Int).Mul(oneBig, big.NewInt(14900)) + return msg + }(), + expVWrapper: func(t *testing.T) staking.ValidatorWrapper { + w := makeDefaultSnapVWrapperForUndelegate(t) + w.Delegations[1].Amount = new(big.Int).Mul(oneBig, big.NewInt(100)) + w.Delegations[1].Undelegations[0].Amount = new(big.Int).Mul(oneBig, big.NewInt(19900)) + return w + }(t), + noNilDelegationsEpoch: big.NewInt(defaultEpoch), + }, + { + // 14: Undelegate with NoNilDelegationsEpoch set + // such that remaining == 0 + // delegatorAddr has 15k ones delegated and 5k in Undelegations at defaultEpoch + sdb: makeDefaultStateForUndelegate(t), + epoch: big.NewInt(defaultEpoch), + msg: func() staking.Undelegate { + msg := defaultMsgUndelegate() + msg.Amount = fifteenKOnes + return msg + }(), + expVWrapper: func(t *testing.T) staking.ValidatorWrapper { + w := makeDefaultSnapVWrapperForUndelegate(t) + w.Delegations[1].Amount = common.Big0 + w.Delegations[1].Undelegations[0].Amount = twentyKOnes + return w + }(t), + noNilDelegationsEpoch: big.NewInt(defaultEpoch), + }, + + } for i, test := range tests { - w, err := VerifyAndUndelegateFromMsg(test.sdb, test.epoch, &test.msg) + config := ¶ms.ChainConfig{} + if test.noNilDelegationsEpoch != nil { + config.NoNilDelegationsEpoch = test.noNilDelegationsEpoch + } else { + config.NoNilDelegationsEpoch = big.NewInt(10000000) // EpochTBD + } + w, err := VerifyAndUndelegateFromMsg(test.sdb, test.epoch, config, &test.msg) if assErr := assertError(err, test.expErr); assErr != nil { t.Errorf("Test %v: %v", i, assErr) @@ -1383,7 +1445,7 @@ func makeDefaultSnapVWrapperForUndelegate(t *testing.T) staking.ValidatorWrapper w := makeVWrapperByIndex(validatorIndex) newDelegation := staking.NewDelegation(delegatorAddr, new(big.Int).Set(twentyKOnes)) - if err := newDelegation.Undelegate(big.NewInt(defaultEpoch), fiveKOnes); err != nil { + if err := newDelegation.Undelegate(big.NewInt(defaultEpoch), fiveKOnes, nil); err != nil { t.Fatal(err) } w.Delegations = append(w.Delegations, newDelegation) diff --git a/core/state_processor.go b/core/state_processor.go index 49e22461e..962314b73 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -63,6 +63,7 @@ type ProcessorResult struct { UsedGas uint64 Reward reward.Reader State *state.DB + DelegationsToRemove map[common.Address][]common.Address } // NewStateProcessor initialises a new StateProcessor. @@ -96,6 +97,7 @@ func (p *StateProcessor) Process( block *types.Block, statedb *state.DB, cfg vm.Config, readCache bool, ) ( types.Receipts, types.CXReceipts, []staking.StakeMsg, + map[common.Address][]common.Address, []*types.Log, UsedGas, reward.Reader, *state.DB, error, ) { cacheKey := block.Hash() @@ -105,7 +107,7 @@ func (p *StateProcessor) Process( // Only the successful results are cached in case for retry. result := cached.(*ProcessorResult) utils.Logger().Info().Str("block num", block.Number().String()).Msg("result cache hit.") - return result.Receipts, result.CxReceipts, result.StakeMsgs, result.Logs, result.UsedGas, result.Reward, result.State, nil + return result.Receipts, result.CxReceipts, result.StakeMsgs, result.DelegationsToRemove, result.Logs, result.UsedGas, result.Reward, result.State, nil } } @@ -122,7 +124,7 @@ func (p *StateProcessor) Process( beneficiary, err := p.bc.GetECDSAFromCoinbase(header) if err != nil { - return nil, nil, nil, nil, 0, nil, statedb, err + return nil, nil, nil, nil, nil, 0, nil, statedb, err } startTime := time.Now() @@ -133,7 +135,7 @@ func (p *StateProcessor) Process( p.bc, &beneficiary, gp, statedb, header, tx, usedGas, cfg, ) if err != nil { - return nil, nil, nil, nil, 0, nil, statedb, err + return nil, nil, nil, nil, nil, 0, nil, statedb, err } receipts = append(receipts, receipt) if cxReceipt != nil { @@ -155,7 +157,7 @@ func (p *StateProcessor) Process( p.bc, &beneficiary, gp, statedb, header, tx, usedGas, cfg, ) if err != nil { - return nil, nil, nil, nil, 0, nil, statedb, err + return nil, nil, nil, nil, nil, 0, nil, statedb, err } receipts = append(receipts, receipt) allLogs = append(allLogs, receipt.Logs...) @@ -168,7 +170,7 @@ func (p *StateProcessor) Process( if err := ApplyIncomingReceipt( p.bc.Config(), statedb, header, cx, ); err != nil { - return nil, nil, + return nil, nil, nil, nil, nil, 0, nil, statedb, errors.New("[Process] Cannot apply incoming receipts") } } @@ -176,14 +178,13 @@ func (p *StateProcessor) Process( slashes := slash.Records{} if s := header.Slashes(); len(s) > 0 { if err := rlp.DecodeBytes(s, &slashes); err != nil { - return nil, nil, nil, nil, 0, nil, statedb, errors.New( - "[Process] Cannot finalize block", - ) + return nil, nil, nil, nil, nil, 0, nil, statedb, errors.Wrap(err, + "[Process] Cannot finalize block") } } if err := MayTestnetShardReduction(p.bc, statedb, header); err != nil { - return nil, nil, nil, nil, 0, nil, statedb, err + return nil, nil, nil, nil, nil, 0, nil, statedb, err } // Finalize the block, applying any consensus engine specific extras (e.g. block rewards) @@ -192,14 +193,14 @@ func (p *StateProcessor) Process( // Block processing don't need to block on reward computation as in block proposal sigsReady <- true }() - _, payout, err := p.bc.Engine().Finalize( + _, delegationsToRemove, payout, err := p.bc.Engine().Finalize( p.bc, p.beacon, header, statedb, block.Transactions(), receipts, outcxs, incxs, block.StakingTransactions(), slashes, sigsReady, func() uint64 { return header.ViewID().Uint64() }, ) if err != nil { - return nil, nil, nil, nil, 0, nil, statedb, errors.WithMessage(err, "[Process] Cannot finalize block") + return nil, nil, nil, nil, nil, 0, nil, statedb, errors.WithMessage(err, "[Process] Cannot finalize block") } result := &ProcessorResult{ @@ -210,9 +211,10 @@ func (p *StateProcessor) Process( UsedGas: *usedGas, Reward: payout, State: statedb, + DelegationsToRemove: delegationsToRemove, } p.resultCache.Add(cacheKey, result) - return receipts, outcxs, blockStakeMsgs, allLogs, *usedGas, payout, statedb, nil + return receipts, outcxs, blockStakeMsgs, delegationsToRemove, allLogs, *usedGas, payout, statedb, nil } // CacheProcessorResult caches the process result on the cache key. diff --git a/core/tx_pool.go b/core/tx_pool.go index d91849022..7cd3cd9b9 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -933,7 +933,7 @@ func (pool *TxPool) validateStakingTx(tx *staking.StakingTransaction) error { return errors.WithMessagef(ErrInvalidSender, "staking transaction sender is %s", b32) } - _, err = VerifyAndUndelegateFromMsg(pool.currentState, pool.pendingEpoch(), stkMsg) + _, err = VerifyAndUndelegateFromMsg(pool.currentState, pool.pendingEpoch(), pool.chainconfig, stkMsg) return err case staking.DirectiveCollectRewards: msg, err := staking.RLPDecodeStakeMsg(tx.Data(), staking.DirectiveCollectRewards) diff --git a/core/types.go b/core/types.go index 4dfd24164..a5e97e078 100644 --- a/core/types.go +++ b/core/types.go @@ -17,6 +17,7 @@ package core import ( + "github.com/ethereum/go-ethereum/common" "github.com/harmony-one/harmony/consensus/reward" "github.com/harmony-one/harmony/core/state" "github.com/harmony-one/harmony/core/types" @@ -55,6 +56,7 @@ type Validator interface { type Processor interface { Process(block *types.Block, statedb *state.DB, cfg vm.Config, readCache bool) ( types.Receipts, types.CXReceipts, []stakingTypes.StakeMsg, + map[common.Address][]common.Address, []*types.Log, uint64, reward.Reader, *state.DB, error, ) CacheProcessorResult(cacheKey interface{}, result *ProcessorResult) diff --git a/core/vm/interface.go b/core/vm/interface.go index 0d600ca16..f29c93977 100644 --- a/core/vm/interface.go +++ b/core/vm/interface.go @@ -49,6 +49,7 @@ type StateDB interface { UnsetValidatorFlag(common.Address) IsValidator(common.Address) bool GetValidatorFirstElectionEpoch(addr common.Address) *big.Int + //AddReward(*staking.ValidatorWrapper, *big.Int, map[common.Address]numeric.Dec, bool) error AddReward(*staking.ValidatorWrapper, *big.Int, map[common.Address]numeric.Dec) error AddRefund(uint64) diff --git a/hmy/downloader/adapter_test.go b/hmy/downloader/adapter_test.go index 692ed8ad7..335a4710b 100644 --- a/hmy/downloader/adapter_test.go +++ b/hmy/downloader/adapter_test.go @@ -156,8 +156,8 @@ func (e *dummyEngine) Finalize( receipts []*types.Receipt, outcxs []*types.CXReceipt, incxs []*types.CXReceiptsProof, stks staking.StakingTransactions, doubleSigners slash.Records, sigsReady chan bool, viewID func() uint64, -) (*types.Block, reward.Reader, error) { - return nil, nil, nil +) (*types.Block, map[common.Address][]common.Address, reward.Reader, error) { + return nil, nil, nil, nil } type testInsertHelper struct { diff --git a/hmy/tracer.go b/hmy/tracer.go index 2529e7a2f..45c9b690c 100644 --- a/hmy/tracer.go +++ b/hmy/tracer.go @@ -282,7 +282,7 @@ func (hmy *Harmony) TraceChain(ctx context.Context, start, end *types.Block, con traced += uint64(len(txs)) } // Generate the next state snapshot fast without tracing - _, _, _, _, _, _, _, err := hmy.BlockChain.Processor().Process(block, statedb, vm.Config{}, false) + _, _, _, _, _, _, _, _, err := hmy.BlockChain.Processor().Process(block, statedb, vm.Config{}, false) if err != nil { failed = err break @@ -677,7 +677,7 @@ func (hmy *Harmony) ComputeStateDB(block *types.Block, reexec uint64) (*state.DB if block = hmy.BlockChain.GetBlockByNumber(block.NumberU64() + 1); block == nil { return nil, fmt.Errorf("block #%d not found", block.NumberU64()+1) } - _, _, _, _, _, _, _, err := hmy.BlockChain.Processor().Process(block, statedb, vm.Config{}, false) + _, _, _, _, _, _, _, _, err := hmy.BlockChain.Processor().Process(block, statedb, vm.Config{}, false) if err != nil { return nil, fmt.Errorf("processing block %d failed: %v", block.NumberU64(), err) } diff --git a/internal/chain/engine.go b/internal/chain/engine.go index 4f3aac9ff..f357790d7 100644 --- a/internal/chain/engine.go +++ b/internal/chain/engine.go @@ -269,7 +269,7 @@ func (e *engineImpl) Finalize( receipts []*types.Receipt, outcxs []*types.CXReceipt, incxs []*types.CXReceiptsProof, stks staking.StakingTransactions, doubleSigners slash.Records, sigsReady chan bool, viewID func() uint64, -) (*types.Block, reward.Reader, error) { +) (*types.Block, map[common.Address][]common.Address, reward.Reader, error) { isBeaconChain := header.ShardID() == shard.BeaconChainShardID inStakingEra := chain.Config().IsStaking(header.Epoch()) @@ -279,22 +279,22 @@ func (e *engineImpl) Finalize( if IsCommitteeSelectionBlock(chain, header) { startTime := time.Now() if err := payoutUndelegations(chain, header, state); err != nil { - return nil, nil, err + return nil, nil, nil, err } - utils.Logger().Debug().Int64("elapsed time", time.Now().Sub(startTime).Milliseconds()).Msg("PayoutUndelegations") + utils.Logger().Debug().Int64("elapsed time", time.Since(startTime).Milliseconds()).Msg("PayoutUndelegations") // Needs to be after payoutUndelegations because payoutUndelegations // depends on the old LastEpochInCommittee startTime = time.Now() if err := setElectionEpochAndMinFee(chain, header, state, chain.Config()); err != nil { - return nil, nil, err + return nil, nil, nil, err } - utils.Logger().Debug().Int64("elapsed time", time.Now().Sub(startTime).Milliseconds()).Msg("SetElectionEpochAndMinFee") + utils.Logger().Debug().Int64("elapsed time", time.Since(startTime).Milliseconds()).Msg("SetElectionEpochAndMinFee") curShardState, err := chain.ReadShardState(chain.CurrentBlock().Epoch()) if err != nil { - return nil, nil, err + return nil, nil, nil, err } startTime = time.Now() // Needs to be before AccumulateRewardsAndCountSigs because @@ -305,7 +305,7 @@ func (e *engineImpl) Finalize( if err := availability.ComputeAndMutateEPOSStatus( chain, state, addr, ); err != nil { - return nil, nil, err + return nil, nil, nil, err } } utils.Logger().Debug().Int64("elapsed time", time.Now().Sub(startTime).Milliseconds()).Msg("ComputeAndMutateEPOSStatus") @@ -317,16 +317,16 @@ func (e *engineImpl) Finalize( chain, state, header, beacon, sigsReady, ) if err != nil { - return nil, nil, err + return nil, nil, nil, err } // Apply slashes if isBeaconChain && inStakingEra && len(doubleSigners) > 0 { if err := applySlashes(chain, header, state, doubleSigners); err != nil { - return nil, nil, err + return nil, nil, nil, err } } else if len(doubleSigners) > 0 { - return nil, nil, errors.New("slashes proposed in non-beacon chain or non-staking epoch") + return nil, nil, nil, errors.New("slashes proposed in non-beacon chain or non-staking epoch") } // ViewID setting needs to happen after commig sig reward logic for pipelining reason. @@ -350,9 +350,37 @@ func (e *engineImpl) Finalize( remainderOne, ) } + + // **** clear stale delegations + // must be done after slashing and paying out rewards + // to avoid conflicts with snapshots + // any nil (un)delegations are eligible to be removed + // bulk pruning happens at the first block of the NoNilDelegationsEpoch + 1 + // and each epoch thereafter + delegationsToRemove := make(map[common.Address][]common.Address, 0) + if shouldPruneStaleStakingData(chain, header) { + startTime := time.Now() + // this will modify wrappers in-situ, which will be used by UpdateValidatorSnapshots + // which occurs in the next epoch at the second to last block + err = pruneStaleStakingData(chain, header, state, delegationsToRemove) + if err != nil { + utils.Logger().Error().AnErr("err", err). + Uint64("blockNum", header.Number().Uint64()). + Uint64("epoch", header.Epoch().Uint64()). + Msg("pruneStaleStakingData error") + return nil, nil, nil, err + } + utils.Logger().Info(). + Int64("elapsed time", time.Since(startTime).Milliseconds()). + Uint64("blockNum", header.Number().Uint64()). + Uint64("epoch", header.Epoch().Uint64()). + Msg("pruneStaleStakingData") + } + + // Finalize the state root header.SetRoot(state.IntermediateRoot(chain.Config().IsS3(header.Epoch()))) - return types.NewBlock(header, txs, receipts, outcxs, incxs, stks), payout, nil + return types.NewBlock(header, txs, receipts, outcxs, incxs, stks), delegationsToRemove, payout, nil } // Withdraw unlocked tokens to the delegators' accounts @@ -409,6 +437,86 @@ func IsCommitteeSelectionBlock(chain engine.ChainReader, header *block.Header) b return isBeaconChain && header.IsLastBlockInEpoch() && inPreStakingEra } +// shouldPruneStaleStakingData checks that all of the following are true +// (1) we are in the beacon chain +// (2) it is the first block of the epoch +// (3) the chain is hard forked to no nil delegations epoch +func shouldPruneStaleStakingData( + chain engine.ChainReader, + header *block.Header, +) bool { + firstBlockInEpoch := false + // if not first epoch + if header.Epoch().Cmp(common.Big0) > 0 { + // calculate the last block of prior epoch + targetEpoch := new(big.Int).Sub(header.Epoch(), common.Big1) + lastBlockNumber := shard.Schedule.EpochLastBlock(targetEpoch.Uint64()) + // add 1 to it + firstBlockInEpoch = header.Number().Uint64() == lastBlockNumber+1 + } else { + // otherwise gensis block + firstBlockInEpoch = header.Number().Cmp(common.Big0) == 0 + } + return header.ShardID() == shard.BeaconChainShardID && + firstBlockInEpoch && + chain.Config().IsNoNilDelegations(header.Epoch()) +} + +// pruneStaleStakingData prunes any stale staking data +// must be called only if shouldPruneStaleStakingData is true +// here and not in staking package to avoid import cycle for state +func pruneStaleStakingData( + chain engine.ChainReader, + header *block.Header, + state *state.DB, + delegationsToRemove map[common.Address][]common.Address, +) error { + validators, err := chain.ReadValidatorList() + if err != nil { + return err + } + for _, validator := range validators { + wrapper, err := state.ValidatorWrapper(validator, true, false) + if err != nil { + return errors.New( + "[pruneStaleStakingData] failed to get validator from state to finalize", + ) + } + delegationsFinal := wrapper.Delegations[:0] + for i := range wrapper.Delegations { + delegation := &wrapper.Delegations[i] + shouldRemove := i != 0 && // never remove the (inactive) validator + len(delegation.Undelegations) == 0 && + delegation.Amount.Cmp(common.Big0) == 0 && + delegation.Reward.Cmp(common.Big0) == 0 + if !shouldRemove { + // append it to final delegations + delegationsFinal = append( + delegationsFinal, + *delegation, + ) + } else { + // in this delegator's delegationIndexes, remove the one which has this validatorAddress + // since a validatorAddress is enough information to uniquely identify the delegationIndex + delegationsToRemove[delegation.DelegatorAddress] = append( + delegationsToRemove[delegation.DelegatorAddress], + wrapper.Address, + ) + } + } + if len(wrapper.Delegations) != len(delegationsFinal) { + utils.Logger().Info(). + Str("ValidatorAddress", wrapper.Address.Hex()). + Uint64("epoch", header.Epoch().Uint64()). + Int("count", len(wrapper.Delegations)-len(delegationsFinal)). + Msg("pruneStaleStakingData pruned count") + } + // now re-assign the delegations + wrapper.Delegations = delegationsFinal + } + return nil +} + func setElectionEpochAndMinFee(chain engine.ChainReader, header *block.Header, state *state.DB, config *params.ChainConfig) error { newShardState, err := header.GetShardState() if err != nil { diff --git a/internal/params/config.go b/internal/params/config.go index 4a0715a47..6e3ec1d28 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -74,6 +74,7 @@ var ( FeeCollectEpoch: big.NewInt(1535), // 2023-07-20 05:51:07+00:00 ValidatorCodeFixEpoch: big.NewInt(1535), // 2023-07-20 05:51:07+00:00 HIP30Epoch: EpochTBD, + NoNilDelegationsEpoch: EpochTBD, } // TestnetChainConfig contains the chain parameters to run a node on the harmony test network. @@ -116,6 +117,7 @@ var ( FeeCollectEpoch: big.NewInt(1296), // 2023-04-28 07:14:20+00:00 ValidatorCodeFixEpoch: big.NewInt(1296), // 2023-04-28 07:14:20+00:00 HIP30Epoch: EpochTBD, + NoNilDelegationsEpoch: EpochTBD, } // PangaeaChainConfig contains the chain parameters for the Pangaea network. // All features except for CrossLink are enabled at launch. @@ -158,6 +160,7 @@ var ( FeeCollectEpoch: EpochTBD, ValidatorCodeFixEpoch: EpochTBD, HIP30Epoch: EpochTBD, + NoNilDelegationsEpoch: EpochTBD, } // PartnerChainConfig contains the chain parameters for the Partner network. @@ -201,6 +204,7 @@ var ( FeeCollectEpoch: big.NewInt(848), // 2023-04-28 04:33:33+00:00 ValidatorCodeFixEpoch: big.NewInt(848), HIP30Epoch: EpochTBD, + NoNilDelegationsEpoch: EpochTBD, } // StressnetChainConfig contains the chain parameters for the Stress test network. @@ -244,6 +248,7 @@ var ( LeaderRotationExternalBeaconLeaders: EpochTBD, ValidatorCodeFixEpoch: EpochTBD, HIP30Epoch: EpochTBD, + NoNilDelegationsEpoch: big.NewInt(2), } // LocalnetChainConfig contains the chain parameters to run for local development. @@ -286,6 +291,7 @@ var ( FeeCollectEpoch: big.NewInt(2), ValidatorCodeFixEpoch: big.NewInt(2), HIP30Epoch: EpochTBD, + NoNilDelegationsEpoch: big.NewInt(2), } // AllProtocolChanges ... @@ -330,6 +336,7 @@ var ( big.NewInt(0), // FeeCollectEpoch big.NewInt(0), // ValidatorCodeFixEpoch big.NewInt(0), // HIP30Epoch + big.NewInt(0), // NoNilDelegationsEpoch } // TestChainConfig ... @@ -374,6 +381,7 @@ var ( big.NewInt(0), // FeeCollectEpoch big.NewInt(0), // ValidatorCodeFixEpoch big.NewInt(0), // HIP30Epoch + big.NewInt(0), // NoNilDelegationsEpoch } // TestRules ... @@ -513,6 +521,9 @@ type ChainConfig struct { // AllowlistEpoch is the first epoch to support allowlist of HIP18 AllowlistEpoch *big.Int + // The first epoch at the end of which stale delegations are removed + NoNilDelegationsEpoch *big.Int `json:"no-nil-delegations-epoch,omitempty"` + LeaderRotationExternalNonBeaconLeaders *big.Int `json:"leader-rotation-external-non-beacon-leaders,omitempty"` LeaderRotationExternalBeaconLeaders *big.Int `json:"leader-rotation-external-beacon-leaders,omitempty"` @@ -541,7 +552,19 @@ type ChainConfig struct { // String implements the fmt.Stringer interface. func (c *ChainConfig) String() string { - return fmt.Sprintf("{ChainID: %v EthCompatibleChainID: %v EIP155: %v CrossTx: %v Staking: %v CrossLink: %v ReceiptLog: %v SHA3Epoch: %v StakingPrecompileEpoch: %v ChainIdFixEpoch: %v CrossShardXferPrecompileEpoch: %v}", + // use string1 + string2 here instead of concatening in the end + return fmt.Sprintf("{ChainID: %v "+ + "EthCompatibleChainID: %v "+ + "EIP155: %v "+ + "CrossTx: %v "+ + "Staking: %v "+ + "CrossLink: %v "+ + "ReceiptLog: %v "+ + "SHA3Epoch: %v "+ + "StakingPrecompileEpoch: %v "+ + "ChainIdFixEpoch: %v "+ + "CrossShardXferPrecompileEpoch: %v "+ + "NoNilDelegationsEpoch: %v}", c.ChainID, c.EthCompatibleChainID, c.EIP155Epoch, @@ -553,6 +576,7 @@ func (c *ChainConfig) String() string { c.StakingPrecompileEpoch, c.ChainIdFixEpoch, c.CrossShardXferPrecompileEpoch, + c.NoNilDelegationsEpoch, ) } @@ -746,12 +770,19 @@ func (c *ChainConfig) IsHIP6And8Epoch(epoch *big.Int) bool { return isForked(c.HIP6And8Epoch, epoch) } -// IsStakingPrecompileEpoch determines whether staking +// IsStakingPrecompile determines whether staking // precompiles are available in the EVM func (c *ChainConfig) IsStakingPrecompile(epoch *big.Int) bool { return isForked(c.StakingPrecompileEpoch, epoch) } +// IsNoNilDelegations determines whether to clear +// nil delegations regularly (and of course also once) +func (c *ChainConfig) IsNoNilDelegations(epoch *big.Int) bool { + return isForked(c.NoNilDelegationsEpoch, epoch) +} + + // IsCrossShardXferPrecompile determines whether the // Cross Shard Transfer Precompile is available in the EVM func (c *ChainConfig) IsCrossShardXferPrecompile(epoch *big.Int) bool { @@ -859,6 +890,7 @@ type Rules struct { // eip-155 chain id fix IsChainIdFix bool IsValidatorCodeFix bool + IsNoNilDelegations bool } // Rules ensures c's ChainID is not nil. @@ -884,5 +916,6 @@ func (c *ChainConfig) Rules(epoch *big.Int) Rules { IsCrossShardXferPrecompile: c.IsCrossShardXferPrecompile(epoch), IsChainIdFix: c.IsChainIdFix(epoch), IsValidatorCodeFix: c.IsValidatorCodeFix(epoch), + IsNoNilDelegations: c.IsNoNilDelegations(epoch), } } diff --git a/node/worker/worker.go b/node/worker/worker.go index 8029d4bb3..362f9cddf 100644 --- a/node/worker/worker.go +++ b/node/worker/worker.go @@ -48,6 +48,7 @@ type environment struct { incxs []*types.CXReceiptsProof // cross shard receipts and its proof (desitinatin shard) slashes slash.Records stakeMsgs []staking.StakeMsg + delegationsToRemove map[common.Address][]common.Address } // Worker is the main object which takes care of submitting new work to consensus engine @@ -344,6 +345,7 @@ func (w *Worker) GetCurrentResult() *core.ProcessorResult { Reward: w.current.reward, State: w.current.state, StakeMsgs: w.current.stakeMsgs, + DelegationsToRemove: w.current.delegationsToRemove, } } @@ -555,7 +557,7 @@ func (w *Worker) FinalizeNewBlock( } }() - block, payout, err := w.chain.Engine().Finalize( + block, delegationsToRemove, payout, err := w.chain.Engine().Finalize( w.chain, w.beacon, copyHeader, state, w.current.txs, w.current.receipts, @@ -566,6 +568,7 @@ func (w *Worker) FinalizeNewBlock( return nil, errors.Wrapf(err, "cannot finalize block") } w.current.reward = payout + w.current.delegationsToRemove = delegationsToRemove return block, nil } diff --git a/staking/types/delegation.go b/staking/types/delegation.go index 9f0a0c622..0ec742481 100644 --- a/staking/types/delegation.go +++ b/staking/types/delegation.go @@ -2,18 +2,21 @@ package types import ( "encoding/json" - "errors" + //"errors" + "fmt" "math/big" "sort" "github.com/ethereum/go-ethereum/common" "github.com/harmony-one/harmony/crypto/hash" common2 "github.com/harmony-one/harmony/internal/common" + "github.com/pkg/errors" ) var ( - errInsufficientBalance = errors.New("insufficient balance to undelegate") - errInvalidAmount = errors.New("invalid amount, must be positive") + errInsufficientBalance = errors.New("insufficient balance to undelegate") + errInvalidAmount = errors.New("invalid amount, must be positive") + ErrUndelegationRemaining = errors.New("remaining delegation must be 0 or >= 100 ONE") ) const ( @@ -120,13 +123,25 @@ func NewDelegation(delegatorAddr common.Address, } // Undelegate - append entry to the undelegation -func (d *Delegation) Undelegate(epoch *big.Int, amt *big.Int) error { +func (d *Delegation) Undelegate( + epoch *big.Int, + amt *big.Int, + minimumRemainingDelegation *big.Int, +) error { if amt.Sign() <= 0 { return errInvalidAmount } if d.Amount.Cmp(amt) < 0 { return errInsufficientBalance } + if minimumRemainingDelegation != nil { + finalAmount := big.NewInt(0).Sub(d.Amount, amt) + if finalAmount.Cmp(minimumRemainingDelegation) < 0 && finalAmount.Cmp(common.Big0) != 0 { + return errors.Wrapf(ErrUndelegationRemaining, + fmt.Sprintf("Minimum: %d, Remaining: %d", minimumRemainingDelegation, finalAmount), + ) + } + } d.Amount.Sub(d.Amount, amt) exist := false