[sync] ease the verification on dns client side, enable server forward compatibility

pull/3628/head
Jacky Wang 4 years ago
parent b1a71c3e2c
commit e8266f5c86
No known key found for this signature in database
GPG Key ID: 1085CE5F4FF5842C
  1. 2
      api/service/legacysync/downloader/client.go
  2. 75
      api/service/legacysync/downloader/proto/downloader.pb.go
  3. 2
      api/service/legacysync/downloader/proto/downloader.proto
  4. 34
      api/service/legacysync/syncing.go
  5. 37
      node/node_syncing.go

@ -95,7 +95,7 @@ func (client *Client) GetBlocks(hashes [][]byte) *pb.DownloaderResponse {
func (client *Client) GetBlocksAndSigs(hashes [][]byte) *pb.DownloaderResponse {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
request := &pb.DownloaderRequest{Type: pb.DownloaderRequest_BLOCKWITHSIG}
request := &pb.DownloaderRequest{Type: pb.DownloaderRequest_BLOCK, GetBlocksWithSig: true}
request.Hashes = make([][]byte, len(hashes))
for i := range hashes {
request.Hashes[i] = make([]byte, len(hashes[i]))

@ -40,7 +40,6 @@ const (
DownloaderRequest_REGISTERTIMEOUT DownloaderRequest_RequestType = 5
DownloaderRequest_UNKNOWN DownloaderRequest_RequestType = 6
DownloaderRequest_BLOCKHEADER DownloaderRequest_RequestType = 7
DownloaderRequest_BLOCKWITHSIG DownloaderRequest_RequestType = 8
)
// Enum value maps for DownloaderRequest_RequestType.
@ -54,7 +53,6 @@ var (
5: "REGISTERTIMEOUT",
6: "UNKNOWN",
7: "BLOCKHEADER",
8: "BLOCKWITHSIG",
}
DownloaderRequest_RequestType_value = map[string]int32{
"BLOCKHASH": 0,
@ -65,7 +63,6 @@ var (
"REGISTERTIMEOUT": 5,
"UNKNOWN": 6,
"BLOCKHEADER": 7,
"BLOCKWITHSIG": 8,
}
)
@ -161,6 +158,7 @@ type DownloaderRequest struct {
Port string `protobuf:"bytes,6,opt,name=port,proto3" json:"port,omitempty"`
Size uint32 `protobuf:"varint,7,opt,name=size,proto3" json:"size,omitempty"`
RegisterWithSig bool `protobuf:"varint,8,opt,name=registerWithSig,proto3" json:"registerWithSig,omitempty"` // Expect to have NEWBLOCK response of block along with current signature
GetBlocksWithSig bool `protobuf:"varint,9,opt,name=getBlocksWithSig,proto3" json:"getBlocksWithSig,omitempty"` // Have block along with signature for BLOCK request.
}
func (x *DownloaderRequest) Reset() {
@ -251,6 +249,13 @@ func (x *DownloaderRequest) GetRegisterWithSig() bool {
return false
}
func (x *DownloaderRequest) GetGetBlocksWithSig() bool {
if x != nil {
return x.GetBlocksWithSig
}
return false
}
// DownloaderResponse is the generic response of DownloaderRequest.
type DownloaderResponse struct {
state protoimpl.MessageState
@ -321,7 +326,7 @@ var File_downloader_proto protoreflect.FileDescriptor
var file_downloader_proto_rawDesc = []byte{
0x0a, 0x10, 0x64, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x65, 0x72, 0x2e, 0x70, 0x72, 0x6f,
0x74, 0x6f, 0x12, 0x0a, 0x64, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x65, 0x72, 0x22, 0xa2,
0x74, 0x6f, 0x12, 0x0a, 0x64, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x65, 0x72, 0x22, 0xbc,
0x03, 0x0a, 0x11, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x65, 0x72, 0x52, 0x65, 0x71,
0x75, 0x65, 0x73, 0x74, 0x12, 0x3d, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01,
0x28, 0x0e, 0x32, 0x29, 0x2e, 0x64, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x65, 0x72, 0x2e,
@ -338,36 +343,38 @@ var file_downloader_proto_rawDesc = []byte{
0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x12, 0x28, 0x0a,
0x0f, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x57, 0x69, 0x74, 0x68, 0x53, 0x69, 0x67,
0x18, 0x08, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72,
0x57, 0x69, 0x74, 0x68, 0x53, 0x69, 0x67, 0x22, 0x99, 0x01, 0x0a, 0x0b, 0x52, 0x65, 0x71, 0x75,
0x65, 0x73, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0d, 0x0a, 0x09, 0x42, 0x4c, 0x4f, 0x43, 0x4b,
0x48, 0x41, 0x53, 0x48, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x42, 0x4c, 0x4f, 0x43, 0x4b, 0x10,
0x01, 0x12, 0x0c, 0x0a, 0x08, 0x4e, 0x45, 0x57, 0x42, 0x4c, 0x4f, 0x43, 0x4b, 0x10, 0x02, 0x12,
0x0f, 0x0a, 0x0b, 0x42, 0x4c, 0x4f, 0x43, 0x4b, 0x48, 0x45, 0x49, 0x47, 0x48, 0x54, 0x10, 0x03,
0x12, 0x0c, 0x0a, 0x08, 0x52, 0x45, 0x47, 0x49, 0x53, 0x54, 0x45, 0x52, 0x10, 0x04, 0x12, 0x13,
0x0a, 0x0f, 0x52, 0x45, 0x47, 0x49, 0x53, 0x54, 0x45, 0x52, 0x54, 0x49, 0x4d, 0x45, 0x4f, 0x55,
0x54, 0x10, 0x05, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x06,
0x12, 0x0f, 0x0a, 0x0b, 0x42, 0x4c, 0x4f, 0x43, 0x4b, 0x48, 0x45, 0x41, 0x44, 0x45, 0x52, 0x10,
0x07, 0x12, 0x10, 0x0a, 0x0c, 0x42, 0x4c, 0x4f, 0x43, 0x4b, 0x57, 0x49, 0x54, 0x48, 0x53, 0x49,
0x47, 0x10, 0x08, 0x22, 0xd4, 0x01, 0x0a, 0x12, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64,
0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x61,
0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x07, 0x70, 0x61, 0x79,
0x6c, 0x6f, 0x61, 0x64, 0x12, 0x47, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01,
0x28, 0x0e, 0x32, 0x33, 0x2e, 0x64, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x65, 0x72, 0x2e,
0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e,
0x73, 0x65, 0x2e, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f,
0x6e, 0x73, 0x65, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x20, 0x0a,
0x0b, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x48, 0x65, 0x69, 0x67, 0x68, 0x74, 0x18, 0x03, 0x20, 0x01,
0x28, 0x04, 0x52, 0x0b, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x48, 0x65, 0x69, 0x67, 0x68, 0x74, 0x22,
0x39, 0x0a, 0x14, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f,
0x6e, 0x73, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x53, 0x55, 0x43, 0x43, 0x45,
0x53, 0x53, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x46, 0x41, 0x49, 0x4c, 0x10, 0x01, 0x12, 0x0a,
0x0a, 0x06, 0x49, 0x4e, 0x53, 0x59, 0x4e, 0x43, 0x10, 0x02, 0x32, 0x56, 0x0a, 0x0a, 0x44, 0x6f,
0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x65, 0x72, 0x12, 0x48, 0x0a, 0x05, 0x51, 0x75, 0x65, 0x72,
0x79, 0x12, 0x1d, 0x2e, 0x64, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x65, 0x72, 0x2e, 0x44,
0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
0x1a, 0x1e, 0x2e, 0x64, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x65, 0x72, 0x2e, 0x44, 0x6f,
0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
0x22, 0x00, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
0x57, 0x69, 0x74, 0x68, 0x53, 0x69, 0x67, 0x12, 0x2a, 0x0a, 0x10, 0x67, 0x65, 0x74, 0x42, 0x6c,
0x6f, 0x63, 0x6b, 0x73, 0x57, 0x69, 0x74, 0x68, 0x53, 0x69, 0x67, 0x18, 0x09, 0x20, 0x01, 0x28,
0x08, 0x52, 0x10, 0x67, 0x65, 0x74, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x73, 0x57, 0x69, 0x74, 0x68,
0x53, 0x69, 0x67, 0x22, 0x87, 0x01, 0x0a, 0x0b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x54,
0x79, 0x70, 0x65, 0x12, 0x0d, 0x0a, 0x09, 0x42, 0x4c, 0x4f, 0x43, 0x4b, 0x48, 0x41, 0x53, 0x48,
0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x42, 0x4c, 0x4f, 0x43, 0x4b, 0x10, 0x01, 0x12, 0x0c, 0x0a,
0x08, 0x4e, 0x45, 0x57, 0x42, 0x4c, 0x4f, 0x43, 0x4b, 0x10, 0x02, 0x12, 0x0f, 0x0a, 0x0b, 0x42,
0x4c, 0x4f, 0x43, 0x4b, 0x48, 0x45, 0x49, 0x47, 0x48, 0x54, 0x10, 0x03, 0x12, 0x0c, 0x0a, 0x08,
0x52, 0x45, 0x47, 0x49, 0x53, 0x54, 0x45, 0x52, 0x10, 0x04, 0x12, 0x13, 0x0a, 0x0f, 0x52, 0x45,
0x47, 0x49, 0x53, 0x54, 0x45, 0x52, 0x54, 0x49, 0x4d, 0x45, 0x4f, 0x55, 0x54, 0x10, 0x05, 0x12,
0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x06, 0x12, 0x0f, 0x0a, 0x0b,
0x42, 0x4c, 0x4f, 0x43, 0x4b, 0x48, 0x45, 0x41, 0x44, 0x45, 0x52, 0x10, 0x07, 0x22, 0xd4, 0x01,
0x0a, 0x12, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70,
0x6f, 0x6e, 0x73, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18,
0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x47,
0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x33, 0x2e, 0x64,
0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x65, 0x72, 0x2e, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f,
0x61, 0x64, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x67,
0x69, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x54, 0x79, 0x70,
0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x62, 0x6c, 0x6f, 0x63, 0x6b,
0x48, 0x65, 0x69, 0x67, 0x68, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0b, 0x62, 0x6c,
0x6f, 0x63, 0x6b, 0x48, 0x65, 0x69, 0x67, 0x68, 0x74, 0x22, 0x39, 0x0a, 0x14, 0x52, 0x65, 0x67,
0x69, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x54, 0x79, 0x70,
0x65, 0x12, 0x0b, 0x0a, 0x07, 0x53, 0x55, 0x43, 0x43, 0x45, 0x53, 0x53, 0x10, 0x00, 0x12, 0x08,
0x0a, 0x04, 0x46, 0x41, 0x49, 0x4c, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x49, 0x4e, 0x53, 0x59,
0x4e, 0x43, 0x10, 0x02, 0x32, 0x56, 0x0a, 0x0a, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64,
0x65, 0x72, 0x12, 0x48, 0x0a, 0x05, 0x51, 0x75, 0x65, 0x72, 0x79, 0x12, 0x1d, 0x2e, 0x64, 0x6f,
0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x65, 0x72, 0x2e, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61,
0x64, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x64, 0x6f, 0x77,
0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x65, 0x72, 0x2e, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64,
0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x62, 0x06, 0x70, 0x72,
0x6f, 0x74, 0x6f, 0x33,
}
var (

@ -18,7 +18,6 @@ message DownloaderRequest {
REGISTERTIMEOUT = 5;
UNKNOWN = 6;
BLOCKHEADER = 7;
BLOCKWITHSIG = 8;
}
// Request type.
@ -32,6 +31,7 @@ message DownloaderRequest {
string port = 6;
uint32 size = 7;
bool registerWithSig = 8; // Expect to have NEWBLOCK response of block along with current signature
bool getBlocksWithSig = 9; // Have block along with signature for BLOCK request.
}
// DownloaderResponse is the generic response of DownloaderRequest.

@ -628,8 +628,9 @@ func (ss *StateSync) handleBlockSyncResult(payload [][]byte, tasks syncBlockTask
}
for i, blockBytes := range payload {
var bws BlockWithSig
if err := rlp.DecodeBytes(blockBytes, &bws); err != nil {
// For forward compatibility at server side, it can be types.block or BlockWithSig
blockObj, err := RlpDecodeBlockOrBlockWithSig(blockBytes)
if err != nil {
utils.Logger().Warn().
Err(err).
Int("taskIndex", tasks[i].index).
@ -638,8 +639,6 @@ func (ss *StateSync) handleBlockSyncResult(payload [][]byte, tasks syncBlockTask
failedTasks = append(failedTasks, tasks[i])
continue
}
blockObj := bws.Block
blockObj.SetCurrentCommitSig(bws.CommitSigAndBitmap)
gotHash := blockObj.Hash()
if !bytes.Equal(gotHash[:], tasks[i].blockHash) {
utils.Logger().Warn().
@ -656,6 +655,24 @@ func (ss *StateSync) handleBlockSyncResult(payload [][]byte, tasks syncBlockTask
return failedTasks
}
// RlpDecodeBlockOrBlockWithSig decode payload to types.Block or BlockWithSig.
// Return the block with commitSig if set.
func RlpDecodeBlockOrBlockWithSig(payload []byte) (*types.Block, error) {
var block *types.Block
if err := rlp.DecodeBytes(payload, &block); err == nil {
// received payload as *types.Block
return block, nil
}
var bws BlockWithSig
if err := rlp.DecodeBytes(payload, &bws); err == nil {
block := bws.Block
block.SetCurrentCommitSig(bws.CommitSigAndBitmap)
return block, nil
}
return nil, errors.New("failed to decode to either types.Block or BlockWithSig")
}
// downloadTaskQueue is wrapper around Queue with item to be SyncBlockTask
type downloadTaskQueue struct {
q *queue.Queue
@ -802,12 +819,13 @@ func (ss *StateSync) getBlockFromLastMileBlocksByParentHash(parentHash common.Ha
}
// UpdateBlockAndStatus ...
func (ss *StateSync) UpdateBlockAndStatus(block *types.Block, bc *core.BlockChain, verifyAllSig, haveCurrentSig bool) error {
func (ss *StateSync) UpdateBlockAndStatus(block *types.Block, bc *core.BlockChain, verifyAllSig bool) error {
if block.NumberU64() != bc.CurrentBlock().NumberU64()+1 {
utils.Logger().Info().Uint64("curBlockNum", bc.CurrentBlock().NumberU64()).Uint64("receivedBlockNum", block.NumberU64()).Msg("[SYNC] Inappropriate block number, ignore!")
return nil
}
haveCurrentSig := len(block.GetCurrentCommitSig()) != 0
// Verify block signatures
if block.NumberU64() > 1 {
// Verify signature every 100 blocks
@ -883,7 +901,7 @@ func (ss *StateSync) generateNewState(bc *core.BlockChain, worker *worker.Worker
}
// Enforce sig check for the last block in a batch
enforceSigCheck := !commonIter.HasNext()
err = ss.UpdateBlockAndStatus(block, bc, enforceSigCheck, true)
err = ss.UpdateBlockAndStatus(block, bc, enforceSigCheck)
if err != nil {
break
}
@ -900,7 +918,7 @@ func (ss *StateSync) generateNewState(bc *core.BlockChain, worker *worker.Worker
if block == nil {
break
}
err = ss.UpdateBlockAndStatus(block, bc, true, true)
err = ss.UpdateBlockAndStatus(block, bc, true)
if err != nil {
break
}
@ -921,7 +939,7 @@ func (ss *StateSync) generateNewState(bc *core.BlockChain, worker *worker.Worker
if block == nil {
break
}
err = ss.UpdateBlockAndStatus(block, bc, false, false)
err = ss.UpdateBlockAndStatus(block, bc, false)
if err != nil {
break
}

@ -200,7 +200,7 @@ func (node *Node) doBeaconSyncing() {
for beaconBlock := range node.BeaconBlockChannel {
if node.beaconSync != nil {
err := node.beaconSync.UpdateBlockAndStatus(
beaconBlock, node.Beaconchain(), true, false,
beaconBlock, node.Beaconchain(), true,
)
if err != nil {
node.beaconSync.AddLastMileBlock(beaconBlock)
@ -449,26 +449,24 @@ func (node *Node) CalculateResponse(request *downloader_pb.DownloaderRequest, in
case downloader_pb.DownloaderRequest_BLOCK:
var hash common.Hash
for _, bytes := range request.Hashes {
hash.SetBytes(bytes)
encodedBlock, err := node.getEncodedBlockByHash(hash)
if err == nil {
response.Payload = append(response.Payload, encodedBlock)
}
}
case downloader_pb.DownloaderRequest_BLOCKWITHSIG:
var hash common.Hash
withSig := request.GetBlocksWithSig
for _, bytes := range request.Hashes {
hash.SetBytes(bytes)
encoded, err := node.getEncodedBlockWithSigByHash(hash)
if err != nil {
utils.Logger().Info().Err(err).Str("hash", hash.String()).Msg("failed to get block with sig")
continue
var (
encoded []byte
err error
)
if withSig {
encoded, err = node.getEncodedBlockWithSigByHash(hash)
} else {
encoded, err = node.getEncodedBlockByHash(hash)
}
if err == nil {
response.Payload = append(response.Payload, encoded)
}
}
case downloader_pb.DownloaderRequest_BLOCKHEIGHT:
response.BlockHeight = node.Blockchain().CurrentBlock().NumberU64()
@ -479,15 +477,12 @@ func (node *Node) CalculateResponse(request *downloader_pb.DownloaderRequest, in
response.Type = downloader_pb.DownloaderResponse_INSYNC
return response, nil
}
var bws legacysync.BlockWithSig
err := rlp.DecodeBytes(request.BlockHash, &bws)
block, err := legacysync.RlpDecodeBlockOrBlockWithSig(request.BlockHash)
if err != nil {
utils.Logger().Warn().Msg("[SYNC] unable to decode received new block")
utils.Logger().Warn().Err(err).Msg("[SYNC] unable to decode received new block")
return response, err
}
blockObj := bws.Block
blockObj.SetCurrentCommitSig(bws.CommitSigAndBitmap)
node.stateSync.AddNewBlock(request.PeerHash, blockObj)
node.stateSync.AddNewBlock(request.PeerHash, block)
case downloader_pb.DownloaderRequest_REGISTER:
peerID := string(request.PeerHash[:])

Loading…
Cancel
Save