package core
import (
"errors"
"math/big"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/rlp"
"github.com/harmony-one/harmony/core/rawdb"
"github.com/harmony-one/harmony/core/state"
"github.com/harmony-one/harmony/core/types"
"github.com/harmony-one/harmony/internal/utils"
"github.com/harmony-one/harmony/shard"
)
// CommitOffChainData write off chain data of a block onto db writer.
func ( bc * BlockChain ) CommitOffChainData (
batch rawdb . DatabaseWriter , block * types . Block , receipts [ ] * types . Receipt ,
cxReceipts [ ] * types . CXReceipt , payout * big . Int , state * state . DB , root common . Hash ) ( status WriteStatus , err error ) {
//// Write receipts of the block
rawdb . WriteReceipts ( batch , block . Hash ( ) , block . NumberU64 ( ) , receipts )
//// Cross-shard txns
epoch := block . Header ( ) . Epoch ( )
if bc . chainConfig . HasCrossTxFields ( block . Epoch ( ) ) {
shardingConfig := shard . Schedule . InstanceForEpoch ( epoch )
shardNum := int ( shardingConfig . NumShards ( ) )
for i := 0 ; i < shardNum ; i ++ {
if i == int ( block . ShardID ( ) ) {
continue
}
shardReceipts := types . CXReceipts ( cxReceipts ) . GetToShardReceipts ( uint32 ( i ) )
err := rawdb . WriteCXReceipts ( batch , uint32 ( i ) , block . NumberU64 ( ) , block . Hash ( ) , shardReceipts )
if err != nil {
utils . Logger ( ) . Error ( ) . Err ( err ) . Interface ( "shardReceipts" , shardReceipts ) . Int ( "toShardID" , i ) . Msg ( "WriteCXReceipts cannot write into database" )
return NonStatTy , err
}
}
// Mark incomingReceipts in the block as spent
bc . WriteCXReceiptsProofSpent ( batch , block . IncomingReceipts ( ) )
}
//// VRF + VDF
//check non zero VRF field in header and add to local db
//if len(block.Vrf()) > 0 {
// vrfBlockNumbers, _ := bc.ReadEpochVrfBlockNums(block.Header().Epoch())
// if (len(vrfBlockNumbers) > 0) && (vrfBlockNumbers[len(vrfBlockNumbers)-1] == block.NumberU64()) {
// utils.Logger().Error().
// Str("number", block.Number().String()).
// Str("epoch", block.Header().Epoch().String()).
// Msg("VRF block number is already in local db")
// } else {
// vrfBlockNumbers = append(vrfBlockNumbers, block.NumberU64())
// err = bc.WriteEpochVrfBlockNums(block.Header().Epoch(), vrfBlockNumbers)
// if err != nil {
// utils.Logger().Error().
// Str("number", block.Number().String()).
// Str("epoch", block.Header().Epoch().String()).
// Msg("failed to write VRF block number to local db")
// return NonStatTy, err
// }
// }
//}
//
////check non zero Vdf in header and add to local db
//if len(block.Vdf()) > 0 {
// err = bc.WriteEpochVdfBlockNum(block.Header().Epoch(), block.Number())
// if err != nil {
// utils.Logger().Error().
// Str("number", block.Number().String()).
// Str("epoch", block.Header().Epoch().String()).
// Msg("failed to write VDF block number to local db")
// return NonStatTy, err
// }
//}
//// Shard State and Validator Update
header := block . Header ( )
if len ( header . ShardState ( ) ) > 0 {
// Write shard state for the new epoch
epoch := new ( big . Int ) . Add ( header . Epoch ( ) , common . Big1 )
shardState , err := block . Header ( ) . GetShardState ( )
if err == nil && shardState . Epoch != nil && bc . chainConfig . IsStaking ( shardState . Epoch ) {
// After staking, the epoch will be decided by the epoch in the shard state.
epoch = new ( big . Int ) . Set ( shardState . Epoch )
}
newShardState , err := bc . WriteShardStateBytes ( batch , epoch , header . ShardState ( ) )
if err != nil {
header . Logger ( utils . Logger ( ) ) . Warn ( ) . Err ( err ) . Msg ( "cannot store shard state" )
return NonStatTy , err
}
// Find all the elected validator addresses and store them in db
allElectedValidators := [ ] common . Address { }
processed := make ( map [ common . Address ] struct { } )
for i := range newShardState . Shards {
shard := newShardState . Shards [ i ]
for j := range shard . Slots {
slot := shard . Slots [ j ]
if slot . EffectiveStake != nil { // For external validator
_ , ok := processed [ slot . EcdsaAddress ]
if ! ok {
processed [ slot . EcdsaAddress ] = struct { } { }
allElectedValidators = append ( allElectedValidators , shard . Slots [ j ] . EcdsaAddress )
}
}
}
}
// Update elected validators
if err := bc . WriteElectedValidatorList ( batch , allElectedValidators ) ; err != nil {
return NonStatTy , err
}
// Update snapshots for all validators
if err := bc . UpdateValidatorSnapshots ( batch , epoch ) ; err != nil {
return NonStatTy , err
}
}
// Do bookkeeping for new staking txns
for _ , tx := range block . StakingTransactions ( ) {
err = bc . UpdateStakingMetaData ( batch , tx , root )
// keep offchain database consistency with onchain we need revert
// but it should not happend unless local database corrupted
if err != nil {
utils . Logger ( ) . Debug ( ) . Msgf ( "oops, UpdateStakingMetaData failed, err: %+v" , err )
return NonStatTy , err
}
}
// Update voting power of validators for all shards
if block . ShardID ( ) == shard . BeaconChainShardID &&
len ( block . Header ( ) . ShardState ( ) ) > 0 {
shardState := new ( shard . State )
if shardState , err = shard . DecodeWrapper ( block . Header ( ) . ShardState ( ) ) ; err == nil {
if err = bc . UpdateValidatorVotingPower ( batch , shardState ) ; err != nil {
utils . Logger ( ) . Err ( err ) . Msg ( "[UpdateValidatorVotingPower] Failed to update voting power" )
}
} else {
utils . Logger ( ) . Err ( err ) . Msg ( "[UpdateValidatorVotingPower] Failed to decode shard state" )
}
}
//// Writing beacon chain cross links
if header . ShardID ( ) == shard . BeaconChainShardID &&
bc . chainConfig . IsCrossLink ( block . Epoch ( ) ) &&
len ( header . CrossLinks ( ) ) > 0 {
crossLinks := & types . CrossLinks { }
err = rlp . DecodeBytes ( header . CrossLinks ( ) , crossLinks )
if err != nil {
header . Logger ( utils . Logger ( ) ) . Warn ( ) . Err ( err ) . Msg ( "[insertChain/crosslinks] cannot parse cross links" )
return NonStatTy , err
}
if ! crossLinks . IsSorted ( ) {
header . Logger ( utils . Logger ( ) ) . Warn ( ) . Err ( err ) . Msg ( "[insertChain/crosslinks] cross links are not sorted" )
return NonStatTy , errors . New ( "proposed cross links are not sorted" )
}
for _ , crossLink := range * crossLinks {
// Process crosslink
if err := bc . WriteCrossLinks ( batch , types . CrossLinks { crossLink } ) ; err == nil {
utils . Logger ( ) . Info ( ) . Uint64 ( "blockNum" , crossLink . BlockNum ( ) ) . Uint32 ( "shardID" , crossLink . ShardID ( ) ) . Msg ( "[insertChain/crosslinks] Cross Link Added to Beaconchain" )
}
bc . LastContinuousCrossLink ( batch , crossLink )
}
//clean/update local database cache after crosslink inserted into blockchain
num , err := bc . DeleteCommittedFromPendingCrossLinks ( * crossLinks )
utils . Logger ( ) . Debug ( ) . Msgf ( "DeleteCommittedFromPendingCrossLinks, crosslinks in header %d, pending crosslinks: %d, error: %+v" , len ( * crossLinks ) , num , err )
}
if bc . CurrentHeader ( ) . ShardID ( ) == shard . BeaconChainShardID {
if bc . chainConfig . IsStaking ( block . Epoch ( ) ) {
[double-sign] Provide proof of double sign in slash record sent to beaconchain (#2253)
* [double-sign] Commit changes in consensus needed for double-sign
* [double-sign] Leader captures when valdator double signs, broadcasts to beaconchain
* [slash] Add quick iteration tool for testing double-signing
* [slash] Add webhook example
* [slash] Add http server for hook to trigger double sign behavior
* [double-sign] Use bin/trigger-double-sign to cause a double-sign
* [double-sign] Full feedback loop working
* [slash] Thread through the slash records in the block proposal step
* [slash] Compute the slashing rate
* [double-sign] Generalize yaml malicious for many keys
* [double-sign][slash] Modify data structures, verify via webhook handler
* [slash][double-sign] Find one address of bls public key signer, seemingly settle on data structures
* [slash] Apply to state slashing for double signing
* [slash][double-sign] Checkpoint for working code that slashes on beaconchain
* [slash] Keep track of the total slash and total reporters reward
* [slash] Dump account state before and after the slash
* [slash] Satisfy Travis
* [slash][state] Apply slash to the snapshot at beginning of epoch, now need to capture also the new delegates
* [slash] Capture the unique new delegations since snapshot as well
* [slash] Filter undelegation by epoch of double sign
* [slash] Add TODO of correctness needed in slash needs on off-chain data
* [rpc] Fix closure issue on shardID
* [slash] Add delegator to double-sign testing script
* [slash] Expand crt-validator.sh with commenting printfs and make delegation
* [slash] Finish track payment of leftover slash debt after undelegation runs out
* [slash] Now be explicit about error wrt delegatorSlashApply
* [slash] Capture specific sanity check on slash paidoff
* [slash] Track slash from undelegation piecemeal
* [slash][delegation] Named slice types, .String()
* [slash] Do no RLP encode twice, once is enough
* [slash] Remove special case of validators own delegation
* [slash] Refactor approach to slash state application
* [slash] Begin expanding out Verify
* [slash] Slash on snapshot delegations, not current
* [slash] Fix Epoch Cmp
* [slash] Third iteration on slash logic
* [slash] Use full slash amount
* [slash] More log, whitespace
* [slash] Remove Println, add log
* [slash] Remove debug Println
* [slash] Add record in unit test
* [slash] Build Validator snapshot, current. Fill out slash record
* [slash] Need to get RLP dump of a header to use in test
* [slash] Factor out double sign test constants
* [slash] Factor out common for validator, stub out slash application, finish out deserialization setup
* [slash] Factor out data structure creation because of var lexical scoping
* [slash] Seem to have pipeline of unit test e2e executing
* [slash] Add expected snitch, slash amounts
* [slash] Checkpoint
* [slash] Unit test correctly checks case of validator own stake which could drop below 1 ONE in slashing
* [config] add double-sign testnet config (#1)
Signed-off-by: Leo Chen <leo@harmony.one>
* [slash] Commit for as is code & data of current dump.json
* [slash] Order of state operation not correct in test, hence bad results, thank you dlv
* [slash] Add snapshot state dump
* [slash] Pay off slash of validator own delegation correctly
* [slash] Pay off slash debt with special case for min-self
* [slash] Pass first scenario conclusively
* [slash] 2% slash passes unit test for own delegation and external
* [slash] Parameterize unit test to easily test .02 vs .80 slash
* [slash] Handle own delegation correctly at 80% slash
* [slash] Have 80% slash working with external delegator
* [slash] Remove debug code from slash
* [slash] Adjust Apply signature, test again for 2% slash
* [slash] Factor out scenario in testing so can test 2% and 80% at same time
* [slash] Correct balance deduction on plan delegation
* [slash] Mock out ChainReader for TestVerify
* [slash] Small surface area interface, now feedback loop for verify
* [slash] Remove development json
* [slash] trigger-double-sign consumes yaml
* [slash] Remove dead code
* [slash][test] Factor ValidatorWrapper into scenario
* [slash][test] Add example from local-testing dump - caution might be off
* [slash] Factor out mutation of slashDebt
* [slash][test] Factor out tests so can easily load test-case from bytes
* [slash] Fix payment mistake in validator own delegation wrt min-self-delgation respected
* [slash] Satisfy Travis
* [slash] Begin cleanup of PR
* [slash] Apply slash from header to Finalize via state processor
* [slash] Productionize code, Println => logs; adjust slash picked in newblock
* [slash] Need pointer for rlp.Decode
* [slash] ValidatorInformation use full wrapper
* Fix median stake
* [staking] Adjust MarshalJSON for Validator, Wrapper
* Refactor offchain data commit; Make block onchain/offchain commit atomic (#2279)
* Refactor offchain data; Add epoch to ValidatorSnapshot
* Make block onchain/offchain data commit atomically
* [slash][committee] Set .Active to false on double sign, do not consider banned or inactive for committee assignment
* [effective] VC eligible.go
* [consensus] Redundant field in printf
* [docker] import-ks for a dev account
* [slash] Create BLS key for dockerfile and crt-validator.sh
* [slash][docker] Easy deployment of double-sign testing
* [docker] Have slash work as single docker command
* [rpc] Fix median-stake RPC
* [slash] Update webhook with default docker BLS key
* [docker][slash] Fresh yaml copy for docker build, remove dev code in main.go
* [slash] Remove helper binary, commented out code, change to local config
* [params] Factor out test genesis value
* Add shard checking to Tx-Pool & correct blacklist (#2301)
* [core] Fix blacklist & add shardID check
* [staking + node + cmd] Fix blacklist & add shardID check
* [slash] Adjust to PR comments part 1
* [docker] Use different throw away funded account
* [docker] Create easier testing for delegation with private keys
* [docker] Update yaml
* [slash] Remove special case for slashing validator own delegation wrt min-self-delegate
* [docker] Install nano as well
* [slash] Early error if banned
* [quorum] Expose earning account in decider marshal json
* Revert "Refactor offchain data commit; Make block onchain/offchain commit atomic (#2279)"
This reverts commit 9ffbf682c075b49188923c65a0bbf39ac188be00.
* [slash] Add non-sanity check way to update validator
* [reward] Increase percision on percentage in schedule
* [slash] Adjust logs
* [committee] Check eligibility of validator before doing sanity check
* [slash] Update docker
* [slash] Move create validator script to test
* [slash] More log
* [param] Make things faster
* [slash][off-chain] Clear out slashes from pending in writeblockwithstate
* [cross-link] Log is not error, just info
* [blockchain] Not necessary to guard DeletePendingSlashingCandidates
* [slash][consensus] Use plain []byte for signature b/c bls.Sign has private impl fields, rlp does not encode that
* [slash][test] Use faucet as sender, assume user imported
* [slash] Test setup
* [slash] reserve error for real error in logs
* [slash][availability] Apply availability correct, bump signing count each block
* [slash][staking] Consider banned field in sanity check, pay snitch only half of what was actually slashed
* [slash] Pay as much as can
* [slash] use right nowAmt
* [slash] Take away from rewards as well
* [slash] iterate faster
* [slash] Remove dev based timing
* [slash] Add more log, sanity check incoming slash records, only count external for slash rate
* [availability][state] Adjust signature of ValidatorWrapper wrt state, filter out for staked validators, correct availaibility measure on running counters
* [availability] More log
* [slash] Simply pre slash erra slashing
* [slash] Remove development code
* [slash] Use height from recvMsg, todo on epoch
* [staking] Not necessary to touch LastEpochInCommittee in staking_verifier
* [slash] Undo ds in endpoint pattern config
* [slash] Add TODO and log when delegation becomes 0 b/c slash debt payment
* [slash] Abstract staked validators from shard.State into type, set slash rate based BLSKey count
Co-authored-by: Leo Chen <leo@harmony.one>
Co-authored-by: flicker-harmony <52401354+flicker-harmony@users.noreply.github.com>
Co-authored-by: Rongjian Lan <rongjian@harmony.one>
Co-authored-by: Daniel Van Der Maden <daniel@harmony.one>
5 years ago
if err := bc . UpdateBlockRewardAccumulator (
batch , payout , block . Number ( ) . Uint64 ( ) ,
) ; err != nil {
return NonStatTy , err
}
if err := bc . DeletePendingSlashingCandidates ( ) ; err != nil {
return NonStatTy , err
}
} else {
// block reward never accumulate before staking
bc . WriteBlockRewardAccumulator ( batch , big . NewInt ( 0 ) , block . Number ( ) . Uint64 ( ) )
}
}
[double-sign] Provide proof of double sign in slash record sent to beaconchain (#2253)
* [double-sign] Commit changes in consensus needed for double-sign
* [double-sign] Leader captures when valdator double signs, broadcasts to beaconchain
* [slash] Add quick iteration tool for testing double-signing
* [slash] Add webhook example
* [slash] Add http server for hook to trigger double sign behavior
* [double-sign] Use bin/trigger-double-sign to cause a double-sign
* [double-sign] Full feedback loop working
* [slash] Thread through the slash records in the block proposal step
* [slash] Compute the slashing rate
* [double-sign] Generalize yaml malicious for many keys
* [double-sign][slash] Modify data structures, verify via webhook handler
* [slash][double-sign] Find one address of bls public key signer, seemingly settle on data structures
* [slash] Apply to state slashing for double signing
* [slash][double-sign] Checkpoint for working code that slashes on beaconchain
* [slash] Keep track of the total slash and total reporters reward
* [slash] Dump account state before and after the slash
* [slash] Satisfy Travis
* [slash][state] Apply slash to the snapshot at beginning of epoch, now need to capture also the new delegates
* [slash] Capture the unique new delegations since snapshot as well
* [slash] Filter undelegation by epoch of double sign
* [slash] Add TODO of correctness needed in slash needs on off-chain data
* [rpc] Fix closure issue on shardID
* [slash] Add delegator to double-sign testing script
* [slash] Expand crt-validator.sh with commenting printfs and make delegation
* [slash] Finish track payment of leftover slash debt after undelegation runs out
* [slash] Now be explicit about error wrt delegatorSlashApply
* [slash] Capture specific sanity check on slash paidoff
* [slash] Track slash from undelegation piecemeal
* [slash][delegation] Named slice types, .String()
* [slash] Do no RLP encode twice, once is enough
* [slash] Remove special case of validators own delegation
* [slash] Refactor approach to slash state application
* [slash] Begin expanding out Verify
* [slash] Slash on snapshot delegations, not current
* [slash] Fix Epoch Cmp
* [slash] Third iteration on slash logic
* [slash] Use full slash amount
* [slash] More log, whitespace
* [slash] Remove Println, add log
* [slash] Remove debug Println
* [slash] Add record in unit test
* [slash] Build Validator snapshot, current. Fill out slash record
* [slash] Need to get RLP dump of a header to use in test
* [slash] Factor out double sign test constants
* [slash] Factor out common for validator, stub out slash application, finish out deserialization setup
* [slash] Factor out data structure creation because of var lexical scoping
* [slash] Seem to have pipeline of unit test e2e executing
* [slash] Add expected snitch, slash amounts
* [slash] Checkpoint
* [slash] Unit test correctly checks case of validator own stake which could drop below 1 ONE in slashing
* [config] add double-sign testnet config (#1)
Signed-off-by: Leo Chen <leo@harmony.one>
* [slash] Commit for as is code & data of current dump.json
* [slash] Order of state operation not correct in test, hence bad results, thank you dlv
* [slash] Add snapshot state dump
* [slash] Pay off slash of validator own delegation correctly
* [slash] Pay off slash debt with special case for min-self
* [slash] Pass first scenario conclusively
* [slash] 2% slash passes unit test for own delegation and external
* [slash] Parameterize unit test to easily test .02 vs .80 slash
* [slash] Handle own delegation correctly at 80% slash
* [slash] Have 80% slash working with external delegator
* [slash] Remove debug code from slash
* [slash] Adjust Apply signature, test again for 2% slash
* [slash] Factor out scenario in testing so can test 2% and 80% at same time
* [slash] Correct balance deduction on plan delegation
* [slash] Mock out ChainReader for TestVerify
* [slash] Small surface area interface, now feedback loop for verify
* [slash] Remove development json
* [slash] trigger-double-sign consumes yaml
* [slash] Remove dead code
* [slash][test] Factor ValidatorWrapper into scenario
* [slash][test] Add example from local-testing dump - caution might be off
* [slash] Factor out mutation of slashDebt
* [slash][test] Factor out tests so can easily load test-case from bytes
* [slash] Fix payment mistake in validator own delegation wrt min-self-delgation respected
* [slash] Satisfy Travis
* [slash] Begin cleanup of PR
* [slash] Apply slash from header to Finalize via state processor
* [slash] Productionize code, Println => logs; adjust slash picked in newblock
* [slash] Need pointer for rlp.Decode
* [slash] ValidatorInformation use full wrapper
* Fix median stake
* [staking] Adjust MarshalJSON for Validator, Wrapper
* Refactor offchain data commit; Make block onchain/offchain commit atomic (#2279)
* Refactor offchain data; Add epoch to ValidatorSnapshot
* Make block onchain/offchain data commit atomically
* [slash][committee] Set .Active to false on double sign, do not consider banned or inactive for committee assignment
* [effective] VC eligible.go
* [consensus] Redundant field in printf
* [docker] import-ks for a dev account
* [slash] Create BLS key for dockerfile and crt-validator.sh
* [slash][docker] Easy deployment of double-sign testing
* [docker] Have slash work as single docker command
* [rpc] Fix median-stake RPC
* [slash] Update webhook with default docker BLS key
* [docker][slash] Fresh yaml copy for docker build, remove dev code in main.go
* [slash] Remove helper binary, commented out code, change to local config
* [params] Factor out test genesis value
* Add shard checking to Tx-Pool & correct blacklist (#2301)
* [core] Fix blacklist & add shardID check
* [staking + node + cmd] Fix blacklist & add shardID check
* [slash] Adjust to PR comments part 1
* [docker] Use different throw away funded account
* [docker] Create easier testing for delegation with private keys
* [docker] Update yaml
* [slash] Remove special case for slashing validator own delegation wrt min-self-delegate
* [docker] Install nano as well
* [slash] Early error if banned
* [quorum] Expose earning account in decider marshal json
* Revert "Refactor offchain data commit; Make block onchain/offchain commit atomic (#2279)"
This reverts commit 9ffbf682c075b49188923c65a0bbf39ac188be00.
* [slash] Add non-sanity check way to update validator
* [reward] Increase percision on percentage in schedule
* [slash] Adjust logs
* [committee] Check eligibility of validator before doing sanity check
* [slash] Update docker
* [slash] Move create validator script to test
* [slash] More log
* [param] Make things faster
* [slash][off-chain] Clear out slashes from pending in writeblockwithstate
* [cross-link] Log is not error, just info
* [blockchain] Not necessary to guard DeletePendingSlashingCandidates
* [slash][consensus] Use plain []byte for signature b/c bls.Sign has private impl fields, rlp does not encode that
* [slash][test] Use faucet as sender, assume user imported
* [slash] Test setup
* [slash] reserve error for real error in logs
* [slash][availability] Apply availability correct, bump signing count each block
* [slash][staking] Consider banned field in sanity check, pay snitch only half of what was actually slashed
* [slash] Pay as much as can
* [slash] use right nowAmt
* [slash] Take away from rewards as well
* [slash] iterate faster
* [slash] Remove dev based timing
* [slash] Add more log, sanity check incoming slash records, only count external for slash rate
* [availability][state] Adjust signature of ValidatorWrapper wrt state, filter out for staked validators, correct availaibility measure on running counters
* [availability] More log
* [slash] Simply pre slash erra slashing
* [slash] Remove development code
* [slash] Use height from recvMsg, todo on epoch
* [staking] Not necessary to touch LastEpochInCommittee in staking_verifier
* [slash] Undo ds in endpoint pattern config
* [slash] Add TODO and log when delegation becomes 0 b/c slash debt payment
* [slash] Abstract staked validators from shard.State into type, set slash rate based BLSKey count
Co-authored-by: Leo Chen <leo@harmony.one>
Co-authored-by: flicker-harmony <52401354+flicker-harmony@users.noreply.github.com>
Co-authored-by: Rongjian Lan <rongjian@harmony.one>
Co-authored-by: Daniel Van Der Maden <daniel@harmony.one>
5 years ago
return CanonStatTy , nil
}