PoS_EVM
/
woop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Remove UTXO model

Browse Source
pull/171/head
Rongjian Lan 6 years ago
parent 6c0f566a14
commit a73a9a9f09
33 changed files with 240 additions and 3451 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 32
      benchmark.go
  2. 172
      blockchain/block.go
  3. 21
      blockchain/block_test.go
  4. 265
      blockchain/blockchain.go
  5. 92
      blockchain/blockchain_test.go
  6. 1
      blockchain/identity.go
  7. 69
      blockchain/merkle_tree.go
  8. 82
      blockchain/merkle_tree_test.go
  9. 281
      blockchain/transaction.go
  10. 11
      blockchain/transaction_test.go
  11. 564
      blockchain/utxopool.go
  12. 68
      blockchain/utxopool_test.go
  13. 125
      client/client.go
  14. 123
      client/txgen/main.go
  15. 214
      client/txgen/txgen/utxo_txs_generator.go
  16. 25
      client/wallet/main.go
  17. 394
      client/wallet_v2/main.go
  18. 5
      consensus/consensus.go
  19. 97
      consensus/consensus_leader.go
  20. 34
      consensus/consensus_validator.go
  21. 3
      core/types/block.go
  22. 159
      node/node.go
  23. 319
      node/node_handler.go
  24. 24
      node/node_test.go
  25. 40
      node/node_utils.go
  26. 55
      proto/client/client.go
  27. 4
      proto/consensus/consensus.go
  28. 57
      proto/node/node.go
  29. 15
      services/explorer/storage.go
  30. 111
      services/syncing/downloader/server_test.go
  31. 4
      services/syncing/interface.go
  32. 34
      services/syncing/syncing.go
  33. 191
      services/syncing/syncing_test.go

32
benchmark.go
Unescape View File

@ -76,9 +76,7 @@ func loggingInit(logFolder, role, ip, port string, onlyLogTps bool) {
} }
func main() { func main() {
accountModel := flag.Bool("account_model", true, "Whether to use account model")
// TODO: use http://getmyipaddress.org/ or http://www.get-myip.com/ to retrieve my IP address // TODO: use http://getmyipaddress.org/ or http://www.get-myip.com/ to retrieve my IP address
ip := flag.String("ip", "127.0.0.1", "IP of the node") ip := flag.String("ip", "127.0.0.1", "IP of the node")
port := flag.String("port", "9000", "port of the node.") port := flag.String("port", "9000", "port of the node.")
configFile := flag.String("config_file", "config.txt", "file containing all ip addresses") configFile := flag.String("config_file", "config.txt", "file containing all ip addresses")
@ -191,32 +189,18 @@ func main() {
consensus.BlockVerifier = currentNode.VerifyNewBlock consensus.BlockVerifier = currentNode.VerifyNewBlock
consensus.OnConsensusDone = currentNode.PostConsensusProcessing consensus.OnConsensusDone = currentNode.PostConsensusProcessing
// Temporary testing code, to be removed.
currentNode.AddTestingAddresses(10000)
currentNode.State = node.NodeWaitToJoin currentNode.State = node.NodeWaitToJoin
if consensus.IsLeader { if consensus.IsLeader {
currentNode.State = node.NodeLeader currentNode.State = node.NodeLeader
if *accountModel { // Let consensus run
// Let consensus run go func() {
go func() { consensus.WaitForNewBlock(currentNode.BlockChannel)
consensus.WaitForNewBlockAccount(currentNode.BlockChannelAccount) }()
}() // Node waiting for consensus readiness to create new block
// Node waiting for consensus readiness to create new block go func() {
go func() { currentNode.WaitForConsensusReady(consensus.ReadySignal)
currentNode.WaitForConsensusReadyAccount(consensus.ReadySignal) }()
}()
} else {
// Let consensus run
go func() {
consensus.WaitForNewBlock(currentNode.BlockChannel)
}()
// Node waiting for consensus readiness to create new block
go func() {
currentNode.WaitForConsensusReady(consensus.ReadySignal)
}()
}
} else { } else {
if *peerDiscovery { if *peerDiscovery {
go currentNode.JoinShard(leader) go currentNode.JoinShard(leader)

172
blockchain/block.go
Unescape View File

@ -1,172 +0,0 @@
package blockchain
import (
"bytes"
"crypto/sha256"
"encoding/gob"
"fmt"
"log"
"time"
"github.com/harmony-one/harmony/db"
"github.com/harmony-one/harmony/utils"
)
const (
// TimeStampForGenesisBlock is the constant timestamp for the genesis block.
TimeStampForGenesisBlock = 0
)
// Block is a block in the blockchain that contains block headers, transactions and signature etc.
type Block struct {
// Header
Timestamp int64
PrevBlockHash [32]byte
NumTransactions int32
TransactionIds [][32]byte
Transactions []*Transaction // Transactions.
ShardID uint32
Hash [32]byte
MerkleRootData []byte
State *State // If present, this block is state block.
// Signature...
Bitmap []byte // Contains which validator signed the block.
Signature [66]byte // Schnorr collective signature.
AccountBlock []byte // Temporary piggy-back.
}
// State is used in Block to indicate that block is a state block.
type State struct {
NumBlocks int32 // Total number of blocks.
NumTransactions int32 // Total number of transactions.
}
// IsStateBlock is used to check if a block is a state block.
func (b *Block) IsStateBlock() bool {
// TODO: think of a better indicator to check.
return b.State != nil && bytes.Equal(b.PrevBlockHash[:], (&[32]byte{})[:])
}
// Serialize serializes the block.
func (b *Block) Serialize() []byte {
var result bytes.Buffer
encoder := gob.NewEncoder(&result)
err := encoder.Encode(b)
if err != nil {
log.Panic(err)
}
return result.Bytes()
}
// DeserializeBlock deserializes a block.
func DeserializeBlock(d []byte) (*Block, error) {
var block Block
decoder := gob.NewDecoder(bytes.NewReader(d))
err := decoder.Decode(&block)
if err != nil {
log.Panic(err)
}
return &block, err
}
// Used for debuging.
func (b *Block) String() string {
res := fmt.Sprintf("Block created at %v\n", b.Timestamp)
for id, tx := range b.Transactions {
res += fmt.Sprintf("Transaction %v: %v\n", id, *tx)
}
res += fmt.Sprintf("previous blockhash: %v\n", b.PrevBlockHash)
res += fmt.Sprintf("hash: %v\n", b.Hash)
return res
}
func (b *Block) generateMerkleRootData() {
var data [][]byte
for _, txID := range b.TransactionIds {
data = append(data, txID[:])
}
merkleTre := NewMerkleTree(data)
b.MerkleRootData = merkleTre.RootNode.Data
}
func (b *Block) Write(db db.Database, key string) error {
return db.Put([]byte(key), b.Serialize())
}
// Delete deletes the given key in the given databse.
func Delete(db db.Database, key string) error {
return db.Delete([]byte(key))
}
// CalculateBlockHash returns a hash of the block
func (b *Block) CalculateBlockHash() []byte {
var hashes [][]byte
var blockHash [32]byte
hashes = append(hashes, utils.ConvertFixedDataIntoByteArray(b.Timestamp))
hashes = append(hashes, b.PrevBlockHash[:])
for _, id := range b.TransactionIds {
hashes = append(hashes, id[:])
}
b.generateMerkleRootData()
hashes = append(hashes, b.MerkleRootData)
hashes = append(hashes, utils.ConvertFixedDataIntoByteArray(b.ShardID))
blockHash = sha256.Sum256(bytes.Join(hashes, []byte{}))
return blockHash[:]
}
// NewBlock creates and returns a new block.
func NewBlock(transactions []*Transaction, prevBlockHash [32]byte, shardID uint32, isGenesisBlock bool) *Block {
numTxs := int32(len(transactions))
var txIDs [][32]byte
for _, tx := range transactions {
txIDs = append(txIDs, tx.ID)
}
timestamp := time.Now().Unix()
if isGenesisBlock {
timestamp = TimeStampForGenesisBlock
}
block := &Block{Timestamp: timestamp, PrevBlockHash: prevBlockHash, NumTransactions: numTxs, TransactionIds: txIDs, Transactions: transactions, ShardID: shardID, Hash: [32]byte{}}
copy(block.Hash[:], block.CalculateBlockHash()[:])
return block
}
// NewGenesisBlock creates and returns genesis Block.
func NewGenesisBlock(coinbase *Transaction, shardID uint32) *Block {
return NewBlock([]*Transaction{coinbase}, [32]byte{}, shardID, true)
}
// NewStateBlock creates and returns a state Block based on utxo pool.
// TODO(RJ): take care of dangling cross shard transaction
func NewStateBlock(utxoPool *UTXOPool, numBlocks, numTxs int32) *Block {
stateTransactions := []*Transaction{}
stateTransactionIds := [][32]byte{}
for address, txHash2Vout2AmountMap := range utxoPool.UtxoMap {
stateTransaction := &Transaction{}
for txHash, vout2AmountMap := range txHash2Vout2AmountMap {
for index, amount := range vout2AmountMap {
txHashBytes, err := utils.Get32BytesFromString(txHash)
if err == nil {
stateTransaction.TxInput = append(stateTransaction.TxInput, *NewTXInput(NewOutPoint(&txHashBytes, index), address, utxoPool.ShardID))
stateTransaction.TxOutput = append(stateTransaction.TxOutput, TXOutput{Amount: amount, Address: address, ShardID: utxoPool.ShardID})
} else {
return nil
}
}
}
if len(stateTransaction.TxOutput) != 0 {
stateTransaction.SetID()
stateTransactionIds = append(stateTransactionIds, stateTransaction.ID)
stateTransactions = append(stateTransactions, stateTransaction)
}
}
newBlock := NewBlock(stateTransactions, [32]byte{}, utxoPool.ShardID, false)
newBlock.State = &State{NumBlocks: numBlocks, NumTransactions: numTxs}
return newBlock
}

21
blockchain/block_test.go
Unescape View File

@ -1,21 +0,0 @@
package blockchain
import (
"reflect"
"testing"
)
func TestBlockSerialize(t *testing.T) {
cbtx := NewCoinbaseTX(TestAddressOne, genesisCoinbaseData, 0)
if cbtx == nil {
t.Errorf("Failed to create a coinbase transaction.")
}
block := NewGenesisBlock(cbtx, 0)
serializedValue := block.Serialize()
deserializedBlock, _ := DeserializeBlock(serializedValue)
if !reflect.DeepEqual(block, deserializedBlock) {
t.Errorf("Original block and the deserialized block not equal.")
}
}

265
blockchain/blockchain.go
Unescape View File

@ -1,265 +0,0 @@
package blockchain
import (
"bytes"
"encoding/hex"
"github.com/dedis/kyber"
"github.com/harmony-one/harmony/crypto/pki"
)
// Blockchain keeps a sequence of Blocks
type Blockchain struct {
Blocks []*Block
}
const genesisCoinbaseData = "The Times 03/Jan/2009 Chancellor on brink of second bailout for banks"
// FindBlock finds a block with given blockHash.
func (bc *Blockchain) FindBlock(blockHash []byte) *Block {
if len(blockHash) != 32 {
return nil
}
for _, block := range bc.Blocks {
if bytes.Equal(block.Hash[:], blockHash[:]) {
return block
}
}
return nil
}
// FindBlockWithPrevHash fins a block with given prevHash.
func (bc *Blockchain) FindBlockWithPrevHash(prevHash []byte) *Block {
if len(prevHash) != 32 {
return nil
}
for _, block := range bc.Blocks {
if bytes.Equal(block.PrevBlockHash[:], prevHash[:]) {
return block
}
}
return nil
}
// GetLatestBlock gests the latest block at the end of the chain
func (bc *Blockchain) GetLatestBlock() *Block {
if len(bc.Blocks) == 0 {
return nil
}
return bc.Blocks[len(bc.Blocks)-1]
}
// GetBlockHashes returns array of hashes of the given blockchain.
func (bc *Blockchain) GetBlockHashes() [][32]byte {
res := [][32]byte{}
for _, block := range bc.Blocks {
res = append(res, block.Hash)
}
return res
}
// FindUnspentUtxos returns a list of transactions containing unspent outputs.
func (bc *Blockchain) FindUnspentUtxos(address [20]byte) map[TxID]map[uint32]TXOutput {
spentTXOs := make(map[string][]uint32)
result := make(map[TxID]map[uint32]TXOutput)
for index := len(bc.Blocks) - 1; index >= 0; index-- {
block := bc.Blocks[index]
for _, tx := range block.Transactions {
txID := hex.EncodeToString(tx.ID[:])
for outIdx, txOutput := range tx.TxOutput {
shouldContinue := false
for index := range spentTXOs[txID] {
if spentTXOs[txID][index] == uint32(outIdx) {
shouldContinue = true
break
}
}
if shouldContinue {
continue
}
if txOutput.Address == address {
_, ok := result[tx.ID]
if !ok {
result[tx.ID] = make(map[uint32]TXOutput)
}
result[tx.ID][uint32(outIdx)] = txOutput
}
}
for _, txInput := range tx.TxInput {
if address == txInput.Address {
ID := hex.EncodeToString(txInput.PreviousOutPoint.TxID[:])
spentTXOs[ID] = append(spentTXOs[ID], txInput.PreviousOutPoint.Index)
}
}
}
}
return result
}
// FindUTXO finds and returns all unspent transaction outputs.
func (bc *Blockchain) FindUTXO(address [20]byte) []TXOutput {
var UTXOs []TXOutput
unspentTXs := bc.FindUnspentUtxos(address)
for _, utxos := range unspentTXs {
for _, txOutput := range utxos {
if txOutput.Address == address {
UTXOs = append(UTXOs, txOutput)
break
}
}
}
return UTXOs
}
// FindSpendableOutputs finds and returns unspent outputs to reference in inputs.
func (bc *Blockchain) FindSpendableOutputs(address [20]byte, amount int) (int, map[string][]uint32) {
unspentOutputs := make(map[string][]uint32)
unspentUtxos := bc.FindUnspentUtxos(address)
accumulated := 0
Work:
for txID, txOutputs := range unspentUtxos {
txID := hex.EncodeToString(txID[:])
for outIdx, txOutput := range txOutputs {
if txOutput.Address == address && accumulated < amount {
accumulated += txOutput.Amount
unspentOutputs[txID] = append(unspentOutputs[txID], uint32(outIdx))
if accumulated >= amount {
break Work
}
}
}
}
return accumulated, unspentOutputs
}
// NewUTXOTransaction creates a new transaction
func (bc *Blockchain) NewUTXOTransaction(priKey kyber.Scalar, from, to [20]byte, amount int, shardID uint32) *Transaction {
var inputs []TXInput
var outputs []TXOutput
acc, validOutputs := bc.FindSpendableOutputs(from, amount)
if acc < amount {
return nil
}
// Build a list of inputs
for txid, outs := range validOutputs {
id, err := hex.DecodeString(txid)
if err != nil {
return nil
}
txID := TxID{}
copy(txID[:], id[:])
for _, out := range outs {
input := NewTXInput(NewOutPoint(&txID, out), from, shardID)
inputs = append(inputs, *input)
}
}
// Build a list of outputs
outputs = append(outputs, TXOutput{amount, to, shardID})
if acc > amount {
outputs = append(outputs, TXOutput{acc - amount, from, shardID}) // a change
}
tx := Transaction{ID: [32]byte{}, TxInput: inputs, TxOutput: outputs, Proofs: nil}
tx.SetID()
pubKey := pki.GetPublicKeyFromScalar(priKey)
bytes, err := pubKey.MarshalBinary()
if err == nil {
copy(tx.PublicKey[:], bytes)
} else {
panic("Failed to serialize public key")
}
tx.SetID() // TODO(RJ): figure out the correct way to set Tx ID.
tx.Sign(priKey)
return &tx
}
// AddNewUserTransfer creates a new transaction and a block of that transaction.
// Mostly used for testing.
func (bc *Blockchain) AddNewUserTransfer(utxoPool *UTXOPool, priKey kyber.Scalar, from, to [20]byte, amount int, shardID uint32) bool {
tx := bc.NewUTXOTransaction(priKey, from, to, amount, shardID)
if tx != nil {
newBlock := NewBlock([]*Transaction{tx}, bc.Blocks[len(bc.Blocks)-1].Hash, shardID, false)
if bc.VerifyNewBlockAndUpdate(utxoPool, newBlock) {
return true
}
}
return false
}
// VerifyNewBlockAndUpdate verifies if the new coming block is valid for the current blockchain.
func (bc *Blockchain) VerifyNewBlockAndUpdate(utxopool *UTXOPool, block *Block) bool {
length := len(bc.Blocks)
if !bytes.Equal(block.PrevBlockHash[:], bc.Blocks[length-1].Hash[:]) {
return false
}
if block.Timestamp < bc.Blocks[length-1].Timestamp {
return false
}
if utxopool != nil && !utxopool.VerifyAndUpdate(block.Transactions) {
return false
}
bc.Blocks = append(bc.Blocks, block)
return true
}
// CreateBlockchain creates a new blockchain DB
// TODO(minhdoan): This func is not used, consider to remove.
func CreateBlockchain(address [20]byte, shardID uint32) *Blockchain {
// TODO: We assume we have not created any blockchain before.
// In current bitcoin, we can check if we created a blockchain before accessing local db.
cbtx := NewCoinbaseTX(address, genesisCoinbaseData, shardID)
genesis := NewGenesisBlock(cbtx, shardID)
bc := Blockchain{[]*Block{genesis}}
return &bc
}
// CreateBlockchainWithMoreBlocks is used for syncing testing.
func CreateBlockchainWithMoreBlocks(addresses [][20]byte, shardID uint32) *Blockchain {
return &Blockchain{CreateMoreBlocks(addresses, shardID)}
}
// CreateMoreBlocks is used for syncing testing.
func CreateMoreBlocks(addresses [][20]byte, shardID uint32) []*Block {
blocks := []*Block{}
for _, address := range addresses {
cbtx := NewCoinbaseTX(address, genesisCoinbaseData, shardID)
blocks = append(blocks, NewGenesisBlock(cbtx, shardID))
}
return blocks
}
// CreateStateBlock creates state block based on the utxos.
func (bc *Blockchain) CreateStateBlock(utxoPool *UTXOPool) *Block {
var numBlocks int32
var numTxs int32
for _, block := range bc.Blocks {
if block.IsStateBlock() {
numBlocks += block.State.NumBlocks
numTxs += block.State.NumTransactions
} else {
numBlocks++
numTxs += block.NumTransactions
}
}
return NewStateBlock(utxoPool, numBlocks, numTxs)
}

92
blockchain/blockchain_test.go
Unescape View File

@ -1,92 +0,0 @@
package blockchain
import (
"testing"
"github.com/harmony-one/harmony/crypto/pki"
)
var (
PriIntOne = 111
PriIntTwo = 2
PriIntThree = 3
PriIntFour = 4
PriKeyOne = pki.GetPrivateKeyScalarFromInt(PriIntOne)
PriKeyTwo = pki.GetPrivateKeyScalarFromInt(PriIntTwo)
PriKeyThree = pki.GetPrivateKeyScalarFromInt(PriIntThree)
PriKeyFour = pki.GetPrivateKeyScalarFromInt(PriIntFour)
TestAddressOne = pki.GetAddressFromInt(PriIntOne)
TestAddressTwo = pki.GetAddressFromInt(PriIntTwo)
TestAddressThree = pki.GetAddressFromInt(PriIntThree)
TestAddressFour = pki.GetAddressFromInt(PriIntFour)
)
func TestCreateBlockchain(t *testing.T) {
if bc := CreateBlockchain(TestAddressOne, 0); bc == nil {
t.Errorf("failed to create a blockchain")
}
}
func TestFindSpendableOutputs(t *testing.T) {
requestAmount := 3
bc := CreateBlockchain(TestAddressOne, 0)
accumulated, unspentOutputs := bc.FindSpendableOutputs(TestAddressOne, requestAmount)
if accumulated < DefaultCoinbaseValue {
t.Error("Failed to find enough unspent ouptuts")
}
if len(unspentOutputs) <= 0 {
t.Error("Failed to find enough unspent ouptuts")
}
}
func TestFindUTXO(t *testing.T) {
bc := CreateBlockchain(TestAddressOne, 0)
utxo := bc.FindUTXO(TestAddressOne)
total := 0
for _, value := range utxo {
total += value.Amount
if value.Address != TestAddressOne {
t.Error("FindUTXO failed.")
}
}
if total != DefaultCoinbaseValue {
t.Error("FindUTXO failed.")
}
}
func TestAddNewUserTransfer(t *testing.T) {
bc := CreateBlockchain(TestAddressOne, 0)
utxoPool := CreateUTXOPoolFromGenesisBlock(bc.Blocks[0])
if !bc.AddNewUserTransfer(utxoPool, PriKeyOne, TestAddressOne, TestAddressThree, 3, 0) {
t.Error("Failed to add new transfer to alok.")
}
if !bc.AddNewUserTransfer(utxoPool, PriKeyOne, TestAddressOne, TestAddressTwo, 3, 0) {
t.Error("Failed to add new transfer to rj.")
}
if bc.AddNewUserTransfer(utxoPool, PriKeyOne, TestAddressOne, TestAddressFour, 100, 0) {
t.Error("minh should not have enough fun to make the transfer.")
}
}
func TestVerifyNewBlock(t *testing.T) {
bc := CreateBlockchain(TestAddressOne, 0)
utxoPool := CreateUTXOPoolFromGenesisBlock(bc.Blocks[0])
bc.AddNewUserTransfer(utxoPool, PriKeyOne, TestAddressOne, TestAddressThree, 3, 0)
bc.AddNewUserTransfer(utxoPool, PriKeyOne, TestAddressOne, TestAddressTwo, 10, 0)
tx := bc.NewUTXOTransaction(PriKeyOne, TestAddressOne, TestAddressFour, 10, 0)
if tx == nil {
t.Error("failed to create a new transaction.")
}
newBlock := NewBlock([]*Transaction{tx}, bc.Blocks[len(bc.Blocks)-1].Hash, 0, false)
if !bc.VerifyNewBlockAndUpdate(utxoPool, newBlock) {
t.Error("failed to add a new valid block.")
}
}

1
blockchain/identity.go
Unescape View File

@ -1 +0,0 @@
package blockchain

69
blockchain/merkle_tree.go
Unescape View File

@ -1,69 +0,0 @@
package blockchain
import (
"crypto/sha256"
)
// MerkleTree represent a Merkle tree
type MerkleTree struct {
RootNode *MerkleNode
}
// MerkleNode represent a Merkle tree node
type MerkleNode struct {
Left *MerkleNode
Right *MerkleNode
Data []byte
}
// NewMerkleTree creates a new Merkle tree from a sequence of data
func NewMerkleTree(data [][]byte) *MerkleTree {
if len(data) == 0 {
return nil
}
var nodes []*MerkleNode
for _, datum := range data {
node := NewMerkleNode(nil, nil, datum)
nodes = append(nodes, node)
}
for len(nodes) > 1 {
var newLevel []*MerkleNode
if len(nodes)%2 != 0 {
nodes = append(nodes, nodes[len(nodes)-1])
}
for j := 0; j < len(nodes); j += 2 {
node := NewMerkleNode(nodes[j], nodes[j+1], nil)
newLevel = append(newLevel, node)
}
nodes = newLevel
}
mTree := MerkleTree{nodes[0]}
return &mTree
}
// NewMerkleNode creates a new Merkle tree node
func NewMerkleNode(left, right *MerkleNode, data []byte) *MerkleNode {
mNode := MerkleNode{}
prevHashes := []byte{}
if left != nil {
prevHashes = append(prevHashes, left.Data...)
}
if right != nil {
prevHashes = append(prevHashes, right.Data...)
}
prevHashes = append(prevHashes, data...)
hash := sha256.Sum256(prevHashes)
mNode.Data = hash[:]
mNode.Left = left
mNode.Right = right
return &mNode
}

82
blockchain/merkle_tree_test.go
Unescape View File

@ -1,82 +0,0 @@
package blockchain
import (
"encoding/hex"
"fmt"
"testing"
)
func TestNewMerkleNode(t *testing.T) {
data := [][]byte{
[]byte("node1"),
[]byte("node2"),
[]byte("node3"),
}
fmt.Println("Testing")
// Level 1
n1 := NewMerkleNode(nil, nil, data[0])
n2 := NewMerkleNode(nil, nil, data[1])
n3 := NewMerkleNode(nil, nil, data[2])
n4 := NewMerkleNode(nil, nil, data[2])
// Level 2
n5 := NewMerkleNode(n1, n2, nil)
n6 := NewMerkleNode(n3, n4, nil)
// Level 3
n7 := NewMerkleNode(n5, n6, nil)
if hex.EncodeToString(n7.Data) != "4e3e44e55926330ab6c31892f980f8bfd1a6e910ff1ebc3f778211377f35227e" {
t.Errorf("merkle tree is not built correctly.")
}
}
func TestNewMerkleTree(t *testing.T) {
data := [][]byte{
[]byte("node1"),
[]byte("node2"),
[]byte("node3"),
[]byte("node4"),
}
// Level 1
n1 := NewMerkleNode(nil, nil, data[0])
n2 := NewMerkleNode(nil, nil, data[1])
n3 := NewMerkleNode(nil, nil, data[2])
n4 := NewMerkleNode(nil, nil, data[3])
// Level 2
n5 := NewMerkleNode(n1, n2, nil)
n6 := NewMerkleNode(n3, n4, nil)
// Level 3
n7 := NewMerkleNode(n5, n6, nil)
rootHash := fmt.Sprintf("%x", n7.Data)
mTree := NewMerkleTree(data)
if rootHash != fmt.Sprintf("%x", mTree.RootNode.Data) {
t.Errorf("Merkle tree root hash is incorrect")
}
}
func TestNewMerkleTree2(t *testing.T) {
data := [][]byte{
[]byte("node1"),
[]byte("node2"),
}
// Level 1
n1 := NewMerkleNode(nil, nil, data[0])
n2 := NewMerkleNode(nil, nil, data[1])
// Level 2
n3 := NewMerkleNode(n1, n2, nil)
rootHash := fmt.Sprintf("%x", n3.Data)
mTree := NewMerkleTree(data)
if rootHash != fmt.Sprintf("%x", mTree.RootNode.Data) {
t.Errorf("Merkle tree root hash is incorrect")
}
}

281
blockchain/transaction.go
Unescape View File

@ -1,281 +0,0 @@
package blockchain
import (
"bytes"
"crypto/sha256"
"encoding/binary"
"encoding/gob"
"encoding/hex"
"fmt"
"log"
"math"
"github.com/dedis/kyber"
"github.com/dedis/kyber/sign/schnorr"
"github.com/harmony-one/harmony/crypto"
)
var (
// zeroHash is the zero value for a Hash and is defined as
// a package level variable to avoid the need to create a new instance
// every time a check is needed.
zeroHash TxID
)
const (
// DefaultCoinbaseValue is the default value of coinbase transaction.
DefaultCoinbaseValue = 1
// DefaultNumUtxos is the default value of number Utxos.
DefaultNumUtxos = 100
)
// Transaction is the struct of a Transaction.
type Transaction struct {
ID [32]byte // 32 byte hash
TxInput []TXInput
TxOutput []TXOutput
PublicKey [32]byte
Signature [64]byte
Proofs []CrossShardTxProof // The proofs for crossShard tx unlock-to-commit/abort
}
// TXOutput is the struct of transaction output in a transaction.
type TXOutput struct {
Amount int // TODO: Switch to big int or uint32
Address [20]byte // last 20 bytes of the hash of public key
ShardID uint32 // The Id of the shard where this UTXO belongs
}
// TxID structure type.
type TxID = [32]byte
// OutPoint defines a data type that is used to track previous
// transaction outputs.
// TxID is the transaction id
// Index is the index of the transaction ouput in the previous transaction
type OutPoint struct {
TxID TxID
Index uint32
}
// NewOutPoint returns a new transaction outpoint point with the
// provided txID and index.
func NewOutPoint(txID *TxID, index uint32) *OutPoint {
return &OutPoint{
TxID: *txID,
Index: index,
}
}
// TXInput is the struct of transaction input (a UTXO) in a transaction.
type TXInput struct {
PreviousOutPoint OutPoint
Address [20]byte // TODO: @minh do we really need this?
ShardID uint32 // The Id of the shard where this UTXO belongs
}
// NewTXInput returns a new transaction input with the provided
// previous outpoint point, output address and shardID
func NewTXInput(prevOut *OutPoint, address [20]byte, shardID uint32) *TXInput {
return &TXInput{
PreviousOutPoint: *prevOut,
Address: address,
ShardID: shardID,
}
}
// CrossShardTxProof is the proof of accept or reject in the cross shard transaction locking phase.
// This is created by the shard leader, filled with proof signatures after consensus, and returned back to the client.
// One proof structure is only tied to one shard. Therefore, the utxos in the proof are all with the same shard.
type CrossShardTxProof struct {
Accept bool // false means proof-of-reject, true means proof-of-accept
TxID [32]byte // Id of the transaction which this proof is on
TxInput []TXInput // The list of Utxo that this proof is on. They should be in the same shard.
BlockHash [32]byte // The hash of the block where the proof is registered
// Signatures
}
// CrossShardTxAndProof is the proof of accept or reject in the cross shard transaction locking phase.
// This is a internal data structure that doesn't go across network
type CrossShardTxAndProof struct {
Transaction *Transaction // The cross shard tx
Proof *CrossShardTxProof // The proof
}
// SetID sets ID of a transaction (32 byte hash of the whole transaction)
func (tx *Transaction) SetID() {
var encoded bytes.Buffer
var hash [32]byte
enc := gob.NewEncoder(&encoded)
err := enc.Encode(tx)
if err != nil {
log.Panic(err)
}
hash = sha256.Sum256(encoded.Bytes())
tx.ID = hash
}
// Sign signs the given transaction with a private key.
func (tx *Transaction) Sign(priKey kyber.Scalar) error {
signature, err := schnorr.Sign(crypto.Ed25519Curve, priKey, tx.GetContentToVerify())
if err != nil {
log.Panic(err)
}
copy(tx.Signature[:], signature)
return err
}
// IsCrossShard returns if the transaction is a cross transation.
func (tx *Transaction) IsCrossShard() bool {
shardIDs := make(map[uint32]bool)
for _, value := range tx.TxInput {
shardIDs[value.ShardID] = true
}
for _, value := range tx.TxOutput {
shardIDs[value.ShardID] = true
}
return len(shardIDs) > 1
}
// GetContentToVerify gets content to verify.
func (tx *Transaction) GetContentToVerify() []byte {
tempTx := *tx
tempTx.Signature = [64]byte{}
tempTx.Proofs = []CrossShardTxProof{}
return tempTx.Serialize()
}
// NewCoinbaseTX creates a new coinbase transaction
func NewCoinbaseTX(toAddress [20]byte, data string, shardID uint32) *Transaction {
if data == "" {
data = fmt.Sprintf("Reward to '%b'", toAddress)
}
txin := NewTXInput(NewOutPoint(&TxID{}, math.MaxUint32), toAddress, shardID)
outputs := []TXOutput{}
for i := 0; i < DefaultNumUtxos; i++ {
outputs = append(outputs, TXOutput{DefaultCoinbaseValue, toAddress, shardID})
}
tx := Transaction{ID: [32]byte{}, TxInput: []TXInput{*txin}, TxOutput: outputs, Proofs: nil}
// TODO: take care of the signature of coinbase transaction.
tx.SetID()
return &tx
}
// Used for debuging.
func (txInput *TXInput) String() string {
res := fmt.Sprintf("TxID: %v, ", hex.EncodeToString(txInput.PreviousOutPoint.TxID[:]))
res += fmt.Sprintf("TxOutputIndex: %v, ", txInput.PreviousOutPoint.Index)
res += fmt.Sprintf("Address: %v, ", txInput.Address)
res += fmt.Sprintf("ShardID: %v", txInput.ShardID)
return res
}
// Used for debuging.
func (txOutput *TXOutput) String() string {
res := fmt.Sprintf("Amount: %v, ", txOutput.Amount)
res += fmt.Sprintf("Address: %v", txOutput.Address)
res += fmt.Sprintf("ShardID: %v", txOutput.ShardID)
return res
}
// Used for debuging.
func (proof *CrossShardTxProof) String() string {
res := fmt.Sprintf("Accept: %v, ", proof.Accept)
res += fmt.Sprintf("TxId: %v, ", hex.EncodeToString(proof.TxID[:]))
res += fmt.Sprintf("BlockHash: %v, ", hex.EncodeToString(proof.BlockHash[:]))
res += fmt.Sprintf("TxInput:\n")
for id, value := range proof.TxInput {
res += fmt.Sprintf("%v: %v\n", id, value.String())
}
return res
}
// Used for debuging.
func (tx *Transaction) String() string {
res := fmt.Sprintf("ID: %v\n", hex.EncodeToString(tx.ID[:]))
res += fmt.Sprintf("TxInput:\n")
for id, value := range tx.TxInput {
res += fmt.Sprintf("%v: %v\n", id, value.String())
}
res += fmt.Sprintf("TxOutput:\n")
for id, value := range tx.TxOutput {
res += fmt.Sprintf("%v: %v\n", id, value.String())
}
for id, value := range tx.Proofs {
res += fmt.Sprintf("Proof:\n")
res += fmt.Sprintf("%v: %v\n", id, value.String())
}
res += fmt.Sprintf("PublicKey: %v\n", hex.EncodeToString(tx.PublicKey[:]))
res += fmt.Sprintf("Sig: %v\n", hex.EncodeToString(tx.Signature[:]))
return res
}
// Serialize return serialized bytes of the transaction.
func (tx *Transaction) Serialize() []byte {
buffer := bytes.NewBuffer([]byte{})
buffer.Write(tx.ID[:])
for _, value := range tx.TxInput {
buffer.Write(value.Serialize())
}
for _, value := range tx.TxOutput {
buffer.Write(value.Serialize())
}
for _, value := range tx.Proofs {
buffer.Write(value.Serialize())
}
buffer.Write(tx.PublicKey[:])
buffer.Write(tx.Signature[:])
return buffer.Bytes()
}
// Serialize return serialized bytes of the TXInput.
func (txInput *TXInput) Serialize() []byte {
buffer := bytes.NewBuffer([]byte{})
buffer.Write(txInput.Address[:])
fourBytes := make([]byte, 4)
binary.BigEndian.PutUint32(fourBytes, txInput.ShardID)
buffer.Write(fourBytes)
binary.BigEndian.PutUint32(fourBytes, txInput.PreviousOutPoint.Index)
buffer.Write(fourBytes)
buffer.Write(txInput.PreviousOutPoint.TxID[:])
return buffer.Bytes()
}
// Serialize return serialized bytes of the TXOutput.
func (txOutput *TXOutput) Serialize() []byte {
buffer := bytes.NewBuffer([]byte{})
buffer.Write(txOutput.Address[:])
fourBytes := make([]byte, 4)
binary.BigEndian.PutUint32(fourBytes, txOutput.ShardID)
buffer.Write(fourBytes)
binary.BigEndian.PutUint32(fourBytes, uint32(txOutput.Amount)) // TODO(RJ): make amount a bigInt
buffer.Write(fourBytes)
return buffer.Bytes()
}
// Serialize returns serialized bytes of the CrossShardTxProof.
func (proof *CrossShardTxProof) Serialize() []byte {
buffer := bytes.NewBuffer([]byte{})
buffer.Write(proof.TxID[:])
buffer.Write(proof.BlockHash[:])
for _, value := range proof.TxInput {
buffer.Write(value.Serialize())
}
if proof.Accept {
buffer.WriteByte(byte(1))
} else {
buffer.WriteByte(byte(0))
}
return buffer.Bytes()
}

11
blockchain/transaction_test.go
Unescape View File

@ -1,11 +0,0 @@
package blockchain
import (
"testing"
)
func TestNewCoinbaseTX(t *testing.T) {
if cbtx := NewCoinbaseTX(TestAddressOne, genesisCoinbaseData, 0); cbtx == nil {
t.Errorf("failed to create a coinbase transaction.")
}
}

564
blockchain/utxopool.go
Unescape View File

@ -1,564 +0,0 @@
package blockchain
import (
"bytes"
"encoding/gob"
"encoding/hex"
"errors"
"fmt"
"sync"
"github.com/dedis/kyber/sign/schnorr"
"github.com/harmony-one/harmony/crypto"
"github.com/harmony-one/harmony/log"
)
// Vout2AmountMap is a TODO type.
type Vout2AmountMap = map[uint32]int
// TXHash2Vout2AmountMap is a TODO type.
type TXHash2Vout2AmountMap = map[string]Vout2AmountMap
// UtxoMap is a TODO type.
type UtxoMap = map[[20]byte]TXHash2Vout2AmountMap
// UTXOPool stores transactions and balance associated with each address.
type UTXOPool struct {
// Mapping from address to a map of transaction id to a map of the index of output
// array in that transaction to that balance.
/*
The 3-d map's structure:
address - [
txID1 - [
outputIndex1 - value1
outputIndex2 - value2
]
txID2 - [
outputIndex1 - value1
outputIndex2 - value2
]
]
*/
UtxoMap UtxoMap
LockedUtxoMap UtxoMap
ShardID uint32
mutex sync.Mutex
}
// MergeUtxoMap merges the utxoMap into that of the UtxoPool
func (utxoPool *UTXOPool) MergeUtxoMap(utxoMap UtxoMap) {
for address, txHash2Vout2AmountMap := range utxoMap {
clientTxHashMap, ok := utxoPool.UtxoMap[address]
if ok {
for txHash, vout2AmountMap := range txHash2Vout2AmountMap {
clientVout2AmountMap, ok := clientTxHashMap[txHash]
if ok {
for vout, amount := range vout2AmountMap {
clientVout2AmountMap[vout] = amount
}
} else {
clientTxHashMap[txHash] = vout2AmountMap
}
}
} else {
utxoPool.UtxoMap[address] = txHash2Vout2AmountMap
}
}
}
// GetUtxoMapByAddresses gets the Utxo map for specific addresses
func (utxoPool *UTXOPool) GetUtxoMapByAddresses(addresses [][20]byte) UtxoMap {
result := make(UtxoMap)
for _, address := range addresses {
utxos, ok := utxoPool.UtxoMap[address]
if ok {
result[address] = utxos
}
}
return result
}
// VerifyTransactions verifies if a list of transactions valid for this shard.
func (utxoPool *UTXOPool) VerifyTransactions(transactions []*Transaction) bool {
spentTXOs := make(map[[20]byte]map[string]map[uint32]bool)
if utxoPool != nil {
for _, tx := range transactions {
if crossShard, err := utxoPool.VerifyOneTransaction(tx, &spentTXOs); !crossShard && err != nil {
return false
}
}
}
return true
}
// VerifyStateBlock verifies if the given state block matches the current utxo pool.
func (utxoPool *UTXOPool) VerifyStateBlock(stateBlock *Block) bool {
accountBalanceInUtxoPool := make(map[[20]byte]int)
for address, txHash2Vout2AmountMap := range utxoPool.UtxoMap {
for _, vout2AmountMap := range txHash2Vout2AmountMap {
for _, amount := range vout2AmountMap {
accountBalanceInUtxoPool[address] = accountBalanceInUtxoPool[address] + amount
}
}
}
for _, transaction := range stateBlock.Transactions {
for _, txOutput := range transaction.TxOutput {
if txOutput.ShardID != utxoPool.ShardID {
return false
}
accountBalanceInUtxoPool[txOutput.Address] = accountBalanceInUtxoPool[txOutput.Address] - txOutput.Amount
}
}
for _, amount := range accountBalanceInUtxoPool {
if amount != 0 {
return false
}
}
return true
}
// VerifyOneTransaction verifies if a list of transactions valid.
// Add another sanity check function (e.g. spending the same utxo) called before this one.
func (utxoPool *UTXOPool) VerifyOneTransaction(tx *Transaction, spentTXOs *map[[20]byte]map[string]map[uint32]bool) (crossShard bool, err error) {
var nilPubKey [32]byte
// TODO(ricl): remove. just for btc replay.
if tx.PublicKey == nilPubKey {
return false, nil
}
if len(tx.Proofs) > 1 {
return utxoPool.VerifyUnlockTransaction(tx)
}
if spentTXOs == nil {
spentTXOs = &map[[20]byte]map[string]map[uint32]bool{}
}
inTotal := 0
// Calculate the sum of TxInput
for _, in := range tx.TxInput {
// Only check the input for my own shard.
if in.ShardID != utxoPool.ShardID {
crossShard = true
continue
}
inTxID := hex.EncodeToString(in.PreviousOutPoint.TxID[:])
index := in.PreviousOutPoint.Index
// Check if the transaction with the address is spent or not.
if val, ok := (*spentTXOs)[in.Address][inTxID][index]; ok {
if val {
return crossShard, errors.New("TxInput is already spent")
}
}
// Mark the transactions with the address and index spent.
if _, ok := (*spentTXOs)[in.Address]; !ok {
(*spentTXOs)[in.Address] = make(map[string]map[uint32]bool)
}
if _, ok := (*spentTXOs)[in.Address][inTxID]; !ok {
(*spentTXOs)[in.Address][inTxID] = make(map[uint32]bool)
}
(*spentTXOs)[in.Address][inTxID][index] = true
// Sum the balance up to the inTotal.
utxoPool.mutex.Lock()
if val, ok := utxoPool.UtxoMap[in.Address][inTxID][index]; ok {
inTotal += val
} else {
utxoPool.mutex.Unlock()
return crossShard, errors.New("Specified TxInput does not exist in utxo pool")
}
utxoPool.mutex.Unlock()
}
outTotal := 0
// Calculate the sum of TxOutput
for _, out := range tx.TxOutput {
outTotal += out.Amount
if out.ShardID != utxoPool.ShardID {
crossShard = true
}
}
// TODO: improve this checking logic
if (crossShard && inTotal > outTotal) || (!crossShard && inTotal != outTotal) {
return crossShard, errors.New("Input and output amount doesn't match")
}
if inTotal == 0 {
return false, errors.New("Input amount is 0") // Here crossShard is false, because if there is no business for this shard, it's effectively not crossShard no matter what.
}
// Verify the signature
pubKey := crypto.Ed25519Curve.Point()
tempErr := pubKey.UnmarshalBinary(tx.PublicKey[:])
if tempErr != nil {
log.Error("Failed to deserialize public key", "error", tempErr)
}
tempErr = schnorr.Verify(crypto.Ed25519Curve, pubKey, tx.GetContentToVerify(), tx.Signature[:])
if tempErr != nil {
log.Error("Failed to verify signature", "error", tempErr, "public key", pubKey, "pubKey in bytes", tx.PublicKey[:])
return crossShard, errors.New("Invalid signature")
}
return crossShard, nil
}
// VerifyUnlockTransaction verifies a cross shard transaction that contains proofs for unlock-to-commit/abort.
func (utxoPool *UTXOPool) VerifyUnlockTransaction(tx *Transaction) (crossShard bool, err error) {
err = nil
crossShard = false // unlock transaction is treated as crossShard=false because it will be finalized now (doesn't need more steps)
txInputs := make(map[TXInput]bool)
for _, curProof := range tx.Proofs {
for _, txInput := range curProof.TxInput {
txInputs[txInput] = true
}
}
for _, txInput := range tx.TxInput {
val, ok := txInputs[txInput]
if !ok || !val {
err = errors.New("Invalid unlock transaction: not all proofs are provided for tx inputs")
}
}
return
}
// Update updates Utxo balances with a list of new transactions.
func (utxoPool *UTXOPool) Update(transactions []*Transaction) {
if utxoPool != nil {
for _, tx := range transactions {
utxoPool.UpdateOneTransaction(tx)
}
}
}
// UpdateOneTransaction updates utxoPool in respect to the new Transaction.
func (utxoPool *UTXOPool) UpdateOneTransaction(tx *Transaction) {
isUnlockTx := len(tx.Proofs) > 1
unlockToCommit := true
if isUnlockTx {
for _, proof := range tx.Proofs {
if !proof.Accept {
unlockToCommit = false // if any proof is a rejection, they it's a unlock-to-abort tx. Otherwise, it's unlock-to-commit
}
}
}
isCrossShard := false
// check whether it's a cross shard tx.
for _, in := range tx.TxInput {
if in.ShardID != utxoPool.ShardID {
isCrossShard = true
break
}
}
for _, out := range tx.TxOutput {
if out.ShardID != utxoPool.ShardID {
isCrossShard = true
break
}
}
isValidCrossShard := true
if isCrossShard {
// Check whether for this cross shard transaction is valid or not.
for _, in := range tx.TxInput {
// Only check the input for my own shard.
if in.ShardID != utxoPool.ShardID {
continue
}
inTxID := hex.EncodeToString(in.PreviousOutPoint.TxID[:])
if _, ok := utxoPool.UtxoMap[in.Address][inTxID][in.PreviousOutPoint.Index]; !ok {
isValidCrossShard = false
}
}
}
utxoPool.mutex.Lock()
defer utxoPool.mutex.Unlock()
if utxoPool != nil {
txID := hex.EncodeToString(tx.ID[:])
// Remove
if !isUnlockTx {
if isValidCrossShard {
for _, in := range tx.TxInput {
// Only check the input for my own shard.
if in.ShardID != utxoPool.ShardID {
continue
}
// NOTE: for the locking phase of cross tx, the utxo is simply removed from the pool.
inTxID := hex.EncodeToString(in.PreviousOutPoint.TxID[:])
value := utxoPool.UtxoMap[in.Address][inTxID][in.PreviousOutPoint.Index]
utxoPool.DeleteOneUtxo(in.Address, inTxID, in.PreviousOutPoint.Index)
if isCrossShard {
// put the delete (locked) utxo into a separate locked utxo pool
inTxID := hex.EncodeToString(in.PreviousOutPoint.TxID[:])
if _, ok := utxoPool.LockedUtxoMap[in.Address]; !ok {
utxoPool.LockedUtxoMap[in.Address] = make(TXHash2Vout2AmountMap)
}
if _, ok := utxoPool.LockedUtxoMap[in.Address][inTxID]; !ok {
utxoPool.LockedUtxoMap[in.Address][inTxID] = make(Vout2AmountMap)
}
utxoPool.LockedUtxoMap[in.Address][inTxID][in.PreviousOutPoint.Index] = value
}
}
}
}
// Update
if !isCrossShard || isUnlockTx {
if !unlockToCommit {
// unlock-to-abort, bring back (unlock) the utxo input
for _, in := range tx.TxInput {
// Only unlock the input for my own shard.
if in.ShardID != utxoPool.ShardID {
continue
}
inTxID := hex.EncodeToString(in.PreviousOutPoint.TxID[:])
if utxoPool.LockedUtxoExists(in.Address, inTxID, in.PreviousOutPoint.Index) {
// bring back the locked (removed) utxo
if _, ok := utxoPool.UtxoMap[in.Address]; !ok {
utxoPool.UtxoMap[in.Address] = make(TXHash2Vout2AmountMap)
utxoPool.UtxoMap[in.Address][inTxID] = make(Vout2AmountMap)
}
if _, ok := utxoPool.UtxoMap[in.Address][inTxID]; !ok {
utxoPool.UtxoMap[in.Address][inTxID] = make(Vout2AmountMap)
}
value := utxoPool.LockedUtxoMap[in.Address][inTxID][in.PreviousOutPoint.Index]
utxoPool.UtxoMap[in.Address][inTxID][in.PreviousOutPoint.Index] = value
utxoPool.DeleteOneLockedUtxo(in.Address, inTxID, in.PreviousOutPoint.Index)
}
}
} else {
// normal utxo output update
for index, out := range tx.TxOutput {
// Only check the input for my own shard.
if out.ShardID != utxoPool.ShardID {
continue
}
if _, ok := utxoPool.UtxoMap[out.Address]; !ok {
utxoPool.UtxoMap[out.Address] = make(TXHash2Vout2AmountMap)
utxoPool.UtxoMap[out.Address][txID] = make(Vout2AmountMap)
}
if _, ok := utxoPool.UtxoMap[out.Address][txID]; !ok {
utxoPool.UtxoMap[out.Address][txID] = make(Vout2AmountMap)
}
utxoPool.UtxoMap[out.Address][txID][uint32(index)] = out.Amount
}
if isUnlockTx { // for unlock-to-commit transaction, also need to delete the locked utxo
for _, in := range tx.TxInput {
// Only unlock the input for my own shard.
if in.ShardID != utxoPool.ShardID {
continue
}
inTxID := hex.EncodeToString(in.PreviousOutPoint.TxID[:])
utxoPool.DeleteOneLockedUtxo(in.Address, inTxID, in.PreviousOutPoint.Index)
}
}
}
} // If it's a cross shard locking Tx, then don't update so the input UTXOs are locked (removed), and the money is not spendable until unlock-to-commit or unlock-to-abort
}
}
// VerifyOneTransactionAndUpdate verifies and update a valid transaction.
// Return false if the transaction is not valid.
func (utxoPool *UTXOPool) VerifyOneTransactionAndUpdate(tx *Transaction) bool {
if _, err := utxoPool.VerifyOneTransaction(tx, nil); err == nil {
utxoPool.UpdateOneTransaction(tx)
return true
}
return false
}
// VerifyAndUpdate verifies a list of transactions and update utxoPool.
func (utxoPool *UTXOPool) VerifyAndUpdate(transactions []*Transaction) bool {
if utxoPool.VerifyTransactions(transactions) {
utxoPool.Update(transactions)
return true
}
return false
}
// CreateUTXOPoolFromGenesisBlock a Utxo pool from a genesis block.
func CreateUTXOPoolFromGenesisBlock(block *Block) *UTXOPool {
shardID := block.ShardID
var utxoPool UTXOPool
utxoPool.UtxoMap = make(UtxoMap)
utxoPool.LockedUtxoMap = make(UtxoMap)
for _, tx := range block.Transactions {
txID := hex.EncodeToString(tx.ID[:])
for index, out := range tx.TxOutput {
_, ok := utxoPool.UtxoMap[out.Address]
if !ok {
utxoPool.UtxoMap[out.Address] = make(TXHash2Vout2AmountMap)
}
_, ok = utxoPool.UtxoMap[out.Address][txID]
if !ok {
utxoPool.UtxoMap[out.Address][txID] = make(Vout2AmountMap)
}
utxoPool.UtxoMap[out.Address][txID][uint32(index)] = out.Amount
}
}
utxoPool.ShardID = shardID
return &utxoPool
}
// SelectTransactionsForNewBlock returns a list of index of valid transactions for the new block.
func (utxoPool *UTXOPool) SelectTransactionsForNewBlock(transactions []*Transaction, maxNumTxs int) ([]*Transaction, []*Transaction, []*Transaction, []*CrossShardTxAndProof) {
selected, unselected, invalid, crossShardTxs := []*Transaction{}, []*Transaction{}, []*Transaction{}, []*CrossShardTxAndProof{}
spentTXOs := make(map[[20]byte]map[string]map[uint32]bool)
for _, tx := range transactions {
crossShard, err := utxoPool.VerifyOneTransaction(tx, &spentTXOs)
if len(selected) < maxNumTxs {
//if err != nil && rand.Intn(10) < 1 {
// log.Warn("Invalid Transaction", "Reason", err)
//}
if err == nil || crossShard {
if crossShard {
proof := CrossShardTxProof{Accept: err == nil, TxID: tx.ID, TxInput: getShardTxInput(tx, utxoPool.ShardID)}
txAndProof := CrossShardTxAndProof{tx, &proof}
crossShardTxs = append(crossShardTxs, &txAndProof)
tx.Proofs = append(tx.Proofs, proof)
}
selected = append(selected, tx)
} else {
invalid = append(invalid, tx)
}
} else {
unselected = append(unselected, tx)
}
}
return selected, unselected, invalid, crossShardTxs
}
func getShardTxInput(transaction *Transaction, shardID uint32) []TXInput {
result := []TXInput{}
for _, txInput := range transaction.TxInput {
if txInput.ShardID == shardID {
result = append(result, txInput)
}
}
return result
}
// DeleteOneUtxo deletes TODO.
func (utxoPool *UTXOPool) DeleteOneUtxo(address [20]byte, txID string, index uint32) {
delete(utxoPool.UtxoMap[address][txID], index)
if len(utxoPool.UtxoMap[address][txID]) == 0 {
delete(utxoPool.UtxoMap[address], txID)
if len(utxoPool.UtxoMap[address]) == 0 {
delete(utxoPool.UtxoMap, address)
}
}
}
// LockedUtxoExists checks if the looked utxo exists.
func (utxoPool *UTXOPool) LockedUtxoExists(address [20]byte, txID string, index uint32) bool {
_, ok := utxoPool.LockedUtxoMap[address]
if !ok {
return false
}
_, ok = utxoPool.LockedUtxoMap[address][txID]
if !ok {
return false
}
_, ok = utxoPool.LockedUtxoMap[address][txID][index]
return ok
}
// DeleteOneLockedUtxo deletes one balance item of UTXOPool and clean up if possible.
func (utxoPool *UTXOPool) DeleteOneLockedUtxo(address [20]byte, txID string, index uint32) {
delete(utxoPool.LockedUtxoMap[address][txID], index)
if len(utxoPool.LockedUtxoMap[address][txID]) == 0 {
delete(utxoPool.LockedUtxoMap[address], txID)
if len(utxoPool.LockedUtxoMap[address]) == 0 {
delete(utxoPool.LockedUtxoMap, address)
}
}
}
// CleanUp cleans up UTXOPool.
func (utxoPool *UTXOPool) CleanUp() {
for address, txMap := range utxoPool.UtxoMap {
for txid, outIndexes := range txMap {
for index, value := range outIndexes {
if value == 0 {
delete(utxoPool.UtxoMap[address][txid], index)
}
}
if len(utxoPool.UtxoMap[address][txid]) == 0 {
delete(utxoPool.UtxoMap[address], txid)
}
}
if len(utxoPool.UtxoMap[address]) == 0 {
delete(utxoPool.UtxoMap, address)
}
}
}
// Used for debugging.
func (utxoPool *UTXOPool) String() string {
return printUtxos(&utxoPool.UtxoMap)
}
// StringOfLockedUtxos is used for debugging.
func (utxoPool *UTXOPool) StringOfLockedUtxos() string {
return printUtxos(&utxoPool.LockedUtxoMap)
}
func printUtxos(utxos *UtxoMap) string {
res := ""
for address, v1 := range *utxos {
for txid, v2 := range v1 {
for index, value := range v2 {
res += fmt.Sprintf("address: %v, tx id: %v, index: %v, value: %v\n", address, txid, index, value)
}
}
}
return res
}
// GetSizeInByteOfUtxoMap gets a snapshot copy of the current pool
func (utxoPool *UTXOPool) GetSizeInByteOfUtxoMap() int {
utxoPool.mutex.Lock()
defer utxoPool.mutex.Unlock()
byteBuffer := bytes.NewBuffer([]byte{})
encoder := gob.NewEncoder(byteBuffer)
encoder.Encode(utxoPool.UtxoMap)
return len(byteBuffer.Bytes())
}
// CountNumOfUtxos counts the total number of utxos in a pool.
func (utxoPool *UTXOPool) CountNumOfUtxos() int {
return countNumOfUtxos(&utxoPool.UtxoMap)
}
// CountNumOfLockedUtxos counts the total number of locked utxos in a pool.
func (utxoPool *UTXOPool) CountNumOfLockedUtxos() int {
return countNumOfUtxos(&utxoPool.LockedUtxoMap)
}
func countNumOfUtxos(utxos *UtxoMap) int {
countAll := 0
for _, utxoMap := range *utxos {
for txIDStr, val := range utxoMap {
_, err := hex.DecodeString(txIDStr)
if err != nil {
continue
}
countAll += len(val)
}
}
return countAll
}

68
blockchain/utxopool_test.go
Unescape View File

@ -1,68 +0,0 @@
package blockchain
import (
"testing"
)
func TestVerifyOneTransactionAndUpdate(t *testing.T) {
bc := CreateBlockchain(TestAddressOne, 0)
utxoPool := CreateUTXOPoolFromGenesisBlock(bc.Blocks[0])
bc.AddNewUserTransfer(utxoPool, PriKeyOne, TestAddressOne, TestAddressThree, 3, 0)
bc.AddNewUserTransfer(utxoPool, PriKeyOne, TestAddressOne, TestAddressTwo, 100, 0)
tx := bc.NewUTXOTransaction(PriKeyOne, TestAddressOne, TestAddressFour, 10, 0)
if tx == nil {
t.Error("failed to create a new transaction.")
}
if _, err := utxoPool.VerifyOneTransaction(tx, nil); err != nil {
t.Error("failed to verify a valid transaction.")
}
utxoPool.VerifyOneTransactionAndUpdate(tx)
}
func TestVerifyOneTransactionFail(t *testing.T) {
bc := CreateBlockchain(TestAddressOne, 0)
utxoPool := CreateUTXOPoolFromGenesisBlock(bc.Blocks[0])
bc.AddNewUserTransfer(utxoPool, PriKeyOne, TestAddressOne, TestAddressThree, 3, 0)
bc.AddNewUserTransfer(utxoPool, PriKeyOne, TestAddressOne, TestAddressTwo, 100, 0)
tx := bc.NewUTXOTransaction(PriKeyOne, TestAddressOne, TestAddressFour, 10, 0)
if tx == nil {
t.Error("failed to create a new transaction.")
}
tx.TxInput = append(tx.TxInput, tx.TxInput[0])
if _, err := utxoPool.VerifyOneTransaction(tx, nil); err == nil {
t.Error("Tx with multiple identical TxInput shouldn't be valid")
}
}
func TestDeleteOneBalanceItem(t *testing.T) {
bc := CreateBlockchain(TestAddressOne, 0)
utxoPool := CreateUTXOPoolFromGenesisBlock(bc.Blocks[0])
bc.AddNewUserTransfer(utxoPool, PriKeyOne, TestAddressOne, TestAddressThree, 3, 0)
bc.AddNewUserTransfer(utxoPool, PriKeyThree, TestAddressThree, TestAddressTwo, 3, 0)
if _, ok := utxoPool.UtxoMap[TestAddressThree]; ok {
t.Errorf("alok should not be contained in the balance map")
}
}
func TestCleanUp(t *testing.T) {
var utxoPool UTXOPool
utxoPool.UtxoMap = make(UtxoMap)
utxoPool.UtxoMap[TestAddressOne] = make(TXHash2Vout2AmountMap)
utxoPool.UtxoMap[TestAddressTwo] = TXHash2Vout2AmountMap{
"abcd": {
0: 1,
},
}
utxoPool.CleanUp()
if _, ok := utxoPool.UtxoMap[TestAddressOne]; ok {
t.Errorf("minh should not be contained in the balance map")
}
}

125
client/client.go
Unescape View File

@ -1,25 +1,19 @@
package client package client
import ( import (
"bytes" "github.com/harmony-one/harmony/core/types"
"encoding/gob" "github.com/simple-rules/harmony-benchmark/blockchain"
"sync" "sync"
"github.com/harmony-one/harmony/p2p/host"
"github.com/harmony-one/harmony/proto/node"
"github.com/harmony-one/harmony/blockchain"
"github.com/harmony-one/harmony/log" "github.com/harmony-one/harmony/log"
"github.com/harmony-one/harmony/p2p" "github.com/harmony-one/harmony/p2p"
client_proto "github.com/harmony-one/harmony/proto/client" "github.com/harmony-one/harmony/p2p/host"
) )
// Client represents a node (e.g. a wallet) which sends transactions and receives responses from the harmony network // Client represents a node (e.g. a wallet) which sends transactions and receives responses from the harmony network
type Client struct { type Client struct {
PendingCrossTxs map[[32]byte]*blockchain.Transaction // Map of TxId to pending cross shard txs. Pending means the proof-of-accept/rejects are not complete Leaders *map[uint32]p2p.Peer // Map of shard Id and corresponding leader
PendingCrossTxsMutex sync.Mutex // Mutex for the pending txs list UpdateBlocks func([]*types.Block) // Closure function used to sync new block with the leader. Once the leader finishes the consensus on a new block, it will send it to the clients. Clients use this method to update their blockchain
Leaders *map[uint32]p2p.Peer // Map of shard Id and corresponding leader
UpdateBlocks func([]*blockchain.Block) // Closure function used to sync new block with the leader. Once the leader finishes the consensus on a new block, it will send it to the clients. Clients use this method to update their blockchain
ShardUtxoMap map[uint32]blockchain.UtxoMap ShardUtxoMap map[uint32]blockchain.UtxoMap
ShardUtxoMapMutex sync.Mutex // Mutex for the UTXO maps ShardUtxoMapMutex sync.Mutex // Mutex for the UTXO maps
@ -29,109 +23,9 @@ type Client struct {
host host.Host host host.Host
} }
// TransactionMessageHandler is the message handler for Client/Transaction messages.
func (client *Client) TransactionMessageHandler(msgPayload []byte) {
messageType := client_proto.TransactionMessageType(msgPayload[0])
switch messageType {
case client_proto.ProofOfLock:
// Decode the list of blockchain.CrossShardTxProof
txDecoder := gob.NewDecoder(bytes.NewReader(msgPayload[1:])) // skip the ProofOfLock messge type
proofs := new([]blockchain.CrossShardTxProof)
err := txDecoder.Decode(proofs)
if err != nil {
client.log.Error("Failed deserializing cross transaction proof list")
}
client.handleProofOfLockMessage(proofs)
case client_proto.UtxoResponse:
txDecoder := gob.NewDecoder(bytes.NewReader(msgPayload[1:])) // skip the ProofOfLock messge type
fetchUtxoResponse := new(client_proto.FetchUtxoResponseMessage)
err := txDecoder.Decode(fetchUtxoResponse)
client.log.Debug("UtxoResponse")
if err != nil {
client.log.Error("Failed deserializing utxo response")
}
client.handleFetchUtxoResponseMessage(*fetchUtxoResponse)
}
}
// handleProofOfLockMessage handles the followings:
// Client once receives a list of proofs from a leader, for each proof:
// 1) retreive the pending cross shard transaction
// 2) add the proof to the transaction
// 3) checks whether all input utxos of the transaction have a corresponding proof.
// 4) for all transactions with full proofs, broadcast them back to the leaders
func (client *Client) handleProofOfLockMessage(proofs *[]blockchain.CrossShardTxProof) {
txsToSend := []*blockchain.Transaction{}
//fmt.Printf("PENDING CLIENT TX - %d\n", len(client.PendingCrossTxs))
// Loop through the newly received list of proofs
client.PendingCrossTxsMutex.Lock()
log.Info("CLIENT PENDING CROSS TX", "num", len(client.PendingCrossTxs))
for _, proof := range *proofs {
// Find the corresponding pending cross tx
txAndProofs, ok := client.PendingCrossTxs[proof.TxID]
readyToUnlock := true // A flag used to mark whether whether this pending cross tx have all the proofs for its utxo input
if ok {
// Add the new proof to the cross tx's proof list
txAndProofs.Proofs = append(txAndProofs.Proofs, proof)
// Check whether this pending cross tx have all the proofs for its utxo inputs
txInputs := make(map[blockchain.TXInput]bool)
for _, curProof := range txAndProofs.Proofs {
for _, txInput := range curProof.TxInput {
txInputs[txInput] = true
}
}
for _, txInput := range txAndProofs.TxInput {
val, ok := txInputs[txInput]
if !ok || !val {
readyToUnlock = false
}
}
} else {
readyToUnlock = false
}
if readyToUnlock {
txsToSend = append(txsToSend, txAndProofs)
}
}
// Delete all the transactions with full proofs from the pending cross txs
for _, txToSend := range txsToSend {
delete(client.PendingCrossTxs, txToSend.ID)
}
client.PendingCrossTxsMutex.Unlock()
// Broadcast the cross txs with full proofs for unlock-to-commit/abort
if len(txsToSend) != 0 {
client.sendCrossShardTxUnlockMessage(txsToSend)
}
}
func (client *Client) handleFetchUtxoResponseMessage(utxoResponse client_proto.FetchUtxoResponseMessage) {
client.ShardUtxoMapMutex.Lock()
defer client.ShardUtxoMapMutex.Unlock()
_, ok := client.ShardUtxoMap[utxoResponse.ShardID]
if ok {
return
}
client.ShardUtxoMap[utxoResponse.ShardID] = utxoResponse.UtxoMap
}
func (client *Client) sendCrossShardTxUnlockMessage(txsToSend []*blockchain.Transaction) {
for shardID, txs := range BuildOutputShardTransactionMap(txsToSend) {
host.SendMessage(client.host, (*client.Leaders)[shardID], node.ConstructUnlockToCommitOrAbortMessage(txs), nil)
}
}
// NewClient creates a new Client // NewClient creates a new Client
func NewClient(host host.Host, leaders *map[uint32]p2p.Peer) *Client { func NewClient(host host.Host, leaders *map[uint32]p2p.Peer) *Client {
client := Client{} client := Client{}
client.PendingCrossTxs = make(map[[32]byte]*blockchain.Transaction)
client.Leaders = leaders client.Leaders = leaders
client.host = host client.host = host
// Logger // Logger
@ -178,12 +72,3 @@ func (client *Client) GetLeaders() []p2p.Peer {
} }
return leaders return leaders
} }
//// GetLeaders returns leader peers.
//func (client *Client) GetShardLeader(uint32 shardID) p2p.Peer {
// leaders := []p2p.Peer{}
// for _, leader := range *client.Leaders {
// leaders = append(leaders, leader)
// }
// return leaders
//}

123
client/txgen/main.go
Unescape View File

@ -9,8 +9,6 @@ import (
"sync" "sync"
"time" "time"
"github.com/ethereum/go-ethereum/rlp"
"github.com/harmony-one/harmony/blockchain"
"github.com/harmony-one/harmony/client" "github.com/harmony-one/harmony/client"
client_config "github.com/harmony-one/harmony/client/config" client_config "github.com/harmony-one/harmony/client/config"
@ -43,11 +41,9 @@ func main() {
ip := flag.String("ip", "127.0.0.1", "IP of the node") ip := flag.String("ip", "127.0.0.1", "IP of the node")
port := flag.String("port", "9999", "port of the node.") port := flag.String("port", "9999", "port of the node.")
accountModel := flag.Bool("account_model", true, "Whether to use account model")
configFile := flag.String("config_file", "local_config.txt", "file containing all ip addresses and config") configFile := flag.String("config_file", "local_config.txt", "file containing all ip addresses and config")
maxNumTxsPerBatch := flag.Int("max_num_txs_per_batch", 20000, "number of transactions to send per message") maxNumTxsPerBatch := flag.Int("max_num_txs_per_batch", 20000, "number of transactions to send per message")
logFolder := flag.String("log_folder", "latest", "the folder collecting the logs of this execution") logFolder := flag.String("log_folder", "latest", "the folder collecting the logs of this execution")
numSubset := flag.Int("numSubset", 3, "the number of subsets of utxos to process separately")
duration := flag.Int("duration", 10, "duration of the tx generation in second. If it's negative, the experiment runs forever.") duration := flag.Int("duration", 10, "duration of the tx generation in second. If it's negative, the experiment runs forever.")
versionFlag := flag.Bool("version", false, "Output version info") versionFlag := flag.Bool("version", false, "Output version info")
crossShardRatio := flag.Int("cross_shard_ratio", 30, "The percentage of cross shard transactions.") crossShardRatio := flag.Int("cross_shard_ratio", 30, "The percentage of cross shard transactions.")
@ -117,7 +113,6 @@ func main() {
host := p2pimpl.NewHost(*clientPeer) host := p2pimpl.NewHost(*clientPeer)
node := node.New(host, &consensus.Consensus{ShardID: shardID}, nil) node := node.New(host, &consensus.Consensus{ShardID: shardID}, nil)
// Assign many fake addresses so we have enough address to play with at first // Assign many fake addresses so we have enough address to play with at first
node.AddTestingAddresses(setting.NumOfAddress)
nodes = append(nodes, node) nodes = append(nodes, node)
} }
@ -134,23 +129,18 @@ func main() {
readySignal <- i readySignal <- i
} }
}() }()
updateBlocksFunc := func(blocks []*blockchain.Block) { updateBlocksFunc := func(blocks []*types.Block) {
log.Info("RECEIVED BLOCK", "block", blocks)
for _, block := range blocks { for _, block := range blocks {
for _, node := range nodes { for _, node := range nodes {
shardID := block.ShardID shardID := block.ShardID()
accountBlock := new(types.Block)
err := rlp.DecodeBytes(block.AccountBlock, accountBlock)
if err == nil {
shardID = accountBlock.ShardID()
}
if node.Consensus.ShardID == shardID { if node.Consensus.ShardID == shardID {
// Add it to blockchain // Add it to blockchain
log.Info("Current Block", "hash", node.Chain.CurrentBlock().Hash().Hex()) log.Info("Current Block", "hash", node.Blockchain().CurrentBlock().Hash().Hex())
log.Info("Adding block from leader", "txNum", len(accountBlock.Transactions()), "shardID", shardID, "preHash", accountBlock.ParentHash().Hex()) log.Info("Adding block from leader", "txNum", len(block.Transactions()), "shardID", shardID, "preHash", block.ParentHash().Hex())
node.AddNewBlock(block) node.AddNewBlock(block)
utxoPoolMutex.Lock() utxoPoolMutex.Lock()
node.UpdateUtxoAndState(block)
node.Worker.UpdateCurrent() node.Worker.UpdateCurrent()
utxoPoolMutex.Unlock() utxoPoolMutex.Unlock()
readySignal <- shardID readySignal <- shardID
@ -182,93 +172,37 @@ func main() {
totalTime := float64(*duration) totalTime := float64(*duration)
client.InitLookUpIntPriKeyMap() client.InitLookUpIntPriKeyMap()
subsetCounter := 0
for {
if *accountModel { t := time.Now()
for { if totalTime > 0 && t.Sub(start).Seconds() >= totalTime {
t := time.Now() log.Debug("Generator timer ended.", "duration", (int(t.Sub(start))), "startTime", start, "totalTime", totalTime)
if totalTime > 0 && t.Sub(start).Seconds() >= totalTime { break
log.Debug("Generator timer ended.", "duration", (int(t.Sub(start))), "startTime", start, "totalTime", totalTime)
break
}
select {
case shardID := <-readySignal:
shardIDTxsMap := make(map[uint32]types.Transactions)
lock := sync.Mutex{}
utxoPoolMutex.Lock()
log.Warn("STARTING TX GEN", "gomaxprocs", runtime.GOMAXPROCS(0))
txs, _ := txgen.GenerateSimulatedTransactionsAccount(int(shardID), nodes, setting)
// TODO: Put cross shard tx into a pending list waiting for proofs from leaders
lock.Lock()
// Put txs into corresponding shards
shardIDTxsMap[shardID] = append(shardIDTxsMap[shardID], txs...)
lock.Unlock()
utxoPoolMutex.Unlock()
lock.Lock()
for shardID, txs := range shardIDTxsMap { // Send the txs to corresponding shards
go func(shardID uint32, txs types.Transactions) {
SendTxsToLeaderAccount(clientNode, shardIDLeaderMap[shardID], txs)
}(shardID, txs)
}
lock.Unlock()
}
} }
} else { select {
for { case shardID := <-readySignal:
t := time.Now() shardIDTxsMap := make(map[uint32]types.Transactions)
if totalTime > 0 && t.Sub(start).Seconds() >= totalTime {
log.Debug("Generator timer ended.", "duration", (int(t.Sub(start))), "startTime", start, "totalTime", totalTime)
break
}
shardIDTxsMap := make(map[uint32][]*blockchain.Transaction)
lock := sync.Mutex{} lock := sync.Mutex{}
var wg sync.WaitGroup
wg.Add(len(shardIDLeaderMap))
utxoPoolMutex.Lock() utxoPoolMutex.Lock()
log.Warn("STARTING TX GEN", "gomaxprocs", runtime.GOMAXPROCS(0)) log.Warn("STARTING TX GEN", "gomaxprocs", runtime.GOMAXPROCS(0))
for shardID := range shardIDLeaderMap { // Generate simulated transactions txs, _ := txgen.GenerateSimulatedTransactionsAccount(int(shardID), nodes, setting)
go func(shardID uint32) {
txs, crossTxs := txgen.GenerateSimulatedTransactions(subsetCounter, *numSubset, int(shardID), nodes, setting) // TODO: Put cross shard tx into a pending list waiting for proofs from leaders
// Put cross shard tx into a pending list waiting for proofs from leaders lock.Lock()
if clientPeer != nil { // Put txs into corresponding shards
clientNode.Client.PendingCrossTxsMutex.Lock() shardIDTxsMap[shardID] = append(shardIDTxsMap[shardID], txs...)
for _, tx := range crossTxs { lock.Unlock()
clientNode.Client.PendingCrossTxs[tx.ID] = tx
}
clientNode.Client.PendingCrossTxsMutex.Unlock()
}
lock.Lock()
// Put txs into corresponding shards
shardIDTxsMap[shardID] = append(shardIDTxsMap[shardID], txs...)
for _, crossTx := range crossTxs {
for curShardID := range client.GetInputShardIDsOfCrossShardTx(crossTx) {
shardIDTxsMap[curShardID] = append(shardIDTxsMap[curShardID], crossTx)
}
}
lock.Unlock()
wg.Done()
}(shardID)
}
wg.Wait()
utxoPoolMutex.Unlock() utxoPoolMutex.Unlock()
lock.Lock() lock.Lock()
for shardID, txs := range shardIDTxsMap { // Send the txs to corresponding shards for shardID, txs := range shardIDTxsMap { // Send the txs to corresponding shards
go func(shardID uint32, txs []*blockchain.Transaction) { go func(shardID uint32, txs types.Transactions) {
SendTxsToLeader(clientNode, shardIDLeaderMap[shardID], txs) SendTxsToLeader(clientNode, shardIDLeaderMap[shardID], txs)
}(shardID, txs) }(shardID, txs)
} }
lock.Unlock() lock.Unlock()
subsetCounter++
time.Sleep(10000 * time.Millisecond)
} }
} }
@ -278,15 +212,8 @@ func main() {
time.Sleep(3000 * time.Millisecond) time.Sleep(3000 * time.Millisecond)
} }
// SendTxsToLeader sends txs to leader. // SendTxsToLeader sends txs to leader account.
func SendTxsToLeader(clientNode *node.Node, leader p2p.Peer, txs []*blockchain.Transaction) { func SendTxsToLeader(clientNode *node.Node, leader p2p.Peer, txs types.Transactions) {
log.Debug("[Generator] Sending txs to...", "leader", leader, "numTxs", len(txs))
msg := proto_node.ConstructTransactionListMessage(txs)
clientNode.SendMessage(leader, msg)
}
// SendTxsToLeaderAccount sends txs to leader account.
func SendTxsToLeaderAccount(clientNode *node.Node, leader p2p.Peer, txs types.Transactions) {
log.Debug("[Generator] Sending account-based txs to...", "leader", leader, "numTxs", len(txs)) log.Debug("[Generator] Sending account-based txs to...", "leader", leader, "numTxs", len(txs))
msg := proto_node.ConstructTransactionListMessageAccount(txs) msg := proto_node.ConstructTransactionListMessageAccount(txs)
clientNode.SendMessage(leader, msg) clientNode.SendMessage(leader, msg)

214
client/txgen/txgen/utxo_txs_generator.go
Unescape View File

@ -1,214 +0,0 @@
package txgen
import (
"encoding/binary"
"encoding/hex"
"math/rand"
"github.com/harmony-one/harmony/blockchain"
"github.com/harmony-one/harmony/client"
"github.com/harmony-one/harmony/crypto/pki"
"github.com/harmony-one/harmony/log"
"github.com/harmony-one/harmony/node"
)
// TxInfo is the transaction info.
type TxInfo struct {
// Global Input
shardID int
dataNodes []*node.Node
// Temp Input
id [32]byte
index uint32
value int
address [20]byte
// Output
txs []*blockchain.Transaction
crossTxs []*blockchain.Transaction
txCount int
}
// GenerateSimulatedTransactions generates at most "maxNumTxs" number of simulated transactions based on the current UtxoPools of all shards.
// The transactions are generated by going through the existing utxos and
// randomly select a subset of them as the input for each new transaction. The output
// address of the new transaction are randomly selected from [0 - N), where N is the total number of fake addresses.
//
// When crossShard=true, besides the selected utxo input, select another valid utxo as input from the same address in a second shard.
// Similarly, generate another utxo output in that second shard.
//
// NOTE: the genesis block should contain N coinbase transactions which add
// token (1000) to each address in [0 - N). See node.AddTestingAddresses()
//
// Params:
// subsetID - the which subset of the utxo to work on (used to select addresses)
// shardID - the shardID for current shard
// dataNodes - nodes containing utxopools of all shards
// Returns:
// all single-shard txs
// all cross-shard txs
func GenerateSimulatedTransactions(subsetID, numSubset int, shardID int, dataNodes []*node.Node, setting Settings) ([]*blockchain.Transaction, []*blockchain.Transaction) {
/*
UTXO map structure:
address - [
txID1 - [
outputIndex1 - value1
outputIndex2 - value2
]
txID2 - [
outputIndex1 - value1
outputIndex2 - value2
]
]
*/
txInfo := TxInfo{}
txInfo.shardID = shardID
txInfo.dataNodes = dataNodes
txInfo.txCount = 0
UTXOLOOP:
// Loop over all addresses
for address, txMap := range dataNodes[shardID].UtxoPool.UtxoMap {
if int(binary.BigEndian.Uint32(address[:]))%numSubset == subsetID%numSubset { // Work on one subset of utxo at a time
txInfo.address = address
// Loop over all txIDs for the address
for txIDStr, utxoMap := range txMap {
// Parse TxId
id, err := hex.DecodeString(txIDStr)
if err != nil {
continue
}
copy(txInfo.id[:], id[:])
// Loop over all utxos for the txID
utxoSize := len(utxoMap)
batchSize := utxoSize / numSubset
i := subsetID % numSubset
counter := 0
for index, value := range utxoMap {
counter++
if batchSize*i < counter && counter > batchSize*(i+1) {
continue
}
txInfo.index = index
txInfo.value = value
randNum := rand.Intn(100)
subsetRatio := 100 // / numSubset
if randNum < subsetRatio { // Sample based on batch size
if setting.CrossShard && randNum < subsetRatio*setting.CrossShardRatio/100 { // 30% cross shard transactions: add another txinput from another shard
generateCrossShardTx(&txInfo, setting)
} else {
generateSingleShardTx(&txInfo, setting)
}
if txInfo.txCount >= setting.MaxNumTxsPerBatch {
break UTXOLOOP
}
}
}
}
}
}
log.Info("UTXO CLIENT", "numUtxo", dataNodes[shardID].UtxoPool.CountNumOfUtxos(), "shardID", shardID)
log.Debug("[Generator] generated transations", "single-shard", len(txInfo.txs), "cross-shard", len(txInfo.crossTxs))
return txInfo.txs, txInfo.crossTxs
}
func generateCrossShardTx(txInfo *TxInfo, setting Settings) {
nodeShardID := txInfo.dataNodes[txInfo.shardID].Consensus.ShardID
crossShardID := nodeShardID
// a random shard to spend money to
for {
crossShardID = uint32(rand.Intn(len(txInfo.dataNodes)))
if crossShardID != nodeShardID {
break
}
}
//crossShardNode := txInfo.dataNodes[crossShardID]
//crossShardUtxosMap := crossShardNode.UtxoPool.UtxoMap[txInfo.address]
//
//// Get the cross shard utxo from another shard
//var crossTxin *blockchain.TXInput
//crossUtxoValue := 0
//// Loop over utxos for the same address from the other shard and use the first utxo as the second cross tx input
//for crossTxIdStr, crossShardUtxos := range crossShardUtxosMap {
// // Parse TxId
// id, err := hex.DecodeString(crossTxIdStr)
// if err != nil {
// continue
// }
// crossTxId := [32]byte{}
// copy(crossTxId[:], id[:])
//
// for crossShardIndex, crossShardValue := range crossShardUtxos {
// crossUtxoValue = crossShardValue
// crossTxin = blockchain.NewTXInput(blockchain.NewOutPoint(&crossTxId, crossShardIndex), txInfo.address, crossShardID)
// break
// }
// if crossTxin != nil {
// break
// }
//}
// Add the utxo from current shard
txIn := blockchain.NewTXInput(blockchain.NewOutPoint(&txInfo.id, txInfo.index), txInfo.address, nodeShardID)
txInputs := []blockchain.TXInput{*txIn}
// Add the utxo from the other shard, if any
//if crossTxin != nil { // This means the ratio of cross shard tx could be lower than 1/3
// txInputs = append(txInputs, *crossTxin)
//}
// Spend the utxo from the current shard to a random address in [0 - N)
txout := blockchain.TXOutput{Amount: txInfo.value, Address: pki.GetAddressFromInt(rand.Intn(setting.NumOfAddress) + 1), ShardID: crossShardID}
txOutputs := []blockchain.TXOutput{txout}
// Spend the utxo from the other shard, if any, to a random address in [0 - N)
//if crossTxin != nil {
// crossTxout := blockchain.TXOutput{Amount: crossUtxoValue, Address: pki.GetAddressFromInt(rand.Intn(setting.numOfAddress) + 1), ShardID: crossShardID}
// txOutputs = append(txOutputs, crossTxout)
//}
// Construct the new transaction
tx := blockchain.Transaction{ID: [32]byte{}, TxInput: txInputs, TxOutput: txOutputs, Proofs: nil}
priKeyInt, ok := client.LookUpIntPriKey(txInfo.address)
if ok {
tx.PublicKey = pki.GetBytesFromPublicKey(pki.GetPublicKeyFromScalar(pki.GetPrivateKeyScalarFromInt(priKeyInt)))
tx.SetID() // TODO(RJ): figure out the correct way to set Tx ID.
tx.Sign(pki.GetPrivateKeyScalarFromInt(priKeyInt))
} else {
log.Error("Failed to look up the corresponding private key from address", "Address", txInfo.address)
return
}
txInfo.crossTxs = append(txInfo.crossTxs, &tx)
txInfo.txCount++
}
func generateSingleShardTx(txInfo *TxInfo, setting Settings) {
nodeShardID := txInfo.dataNodes[txInfo.shardID].Consensus.ShardID
// Add the utxo as new tx input
txin := blockchain.NewTXInput(blockchain.NewOutPoint(&txInfo.id, txInfo.index), txInfo.address, nodeShardID)
// Spend the utxo to a random address in [0 - N)
txout := blockchain.TXOutput{Amount: txInfo.value, Address: pki.GetAddressFromInt(rand.Intn(setting.NumOfAddress) + 1), ShardID: nodeShardID}
tx := blockchain.Transaction{ID: [32]byte{}, TxInput: []blockchain.TXInput{*txin}, TxOutput: []blockchain.TXOutput{txout}, Proofs: nil}
priKeyInt, ok := client.LookUpIntPriKey(txInfo.address)
if ok {
tx.PublicKey = pki.GetBytesFromPublicKey(pki.GetPublicKeyFromScalar(pki.GetPrivateKeyScalarFromInt(priKeyInt)))
tx.SetID() // TODO(RJ): figure out the correct way to set Tx ID.
tx.Sign(pki.GetPrivateKeyScalarFromInt(priKeyInt))
} else {
log.Error("Failed to look up the corresponding private key from address", "Address", txInfo.address)
return
}
txInfo.txs = append(txInfo.txs, &tx)
txInfo.txCount++
}

25
client/wallet/main.go
Unescape View File

@ -4,7 +4,6 @@ import (
"crypto/ecdsa" "crypto/ecdsa"
"crypto/rand" "crypto/rand"
"encoding/hex" "encoding/hex"
"errors"
"flag" "flag"
"fmt" "fmt"
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
@ -19,7 +18,6 @@ import (
"math/big" "math/big"
"strings" "strings"
"github.com/harmony-one/harmony/blockchain"
"github.com/harmony-one/harmony/client" "github.com/harmony-one/harmony/client"
"github.com/harmony-one/harmony/node" "github.com/harmony-one/harmony/node"
"github.com/harmony-one/harmony/p2p" "github.com/harmony-one/harmony/p2p"
@ -356,29 +354,6 @@ func GetFreeToken(address common.Address, walletNode *node.Node) {
} }
} }
// FetchUtxos fetches utxos of specified address from the Harmony network
func FetchUtxos(addresses [][20]byte, walletNode *node.Node) (map[uint32]blockchain.UtxoMap, error) {
walletNode.Client.ShardUtxoMap = make(map[uint32]blockchain.UtxoMap)
walletNode.BroadcastMessage(walletNode.Client.GetLeaders(), proto_node.ConstructFetchUtxoMessage(*walletNode.ClientPeer, addresses))
doneSignal := make(chan int)
go func() {
for {
if len(walletNode.Client.ShardUtxoMap) == len(*walletNode.Client.Leaders) {
doneSignal <- 0
break
}
}
}()
select {
case <-doneSignal:
return walletNode.Client.ShardUtxoMap, nil
case <-time.After(3 * time.Second):
return nil, errors.New("Utxo fetch timed out")
}
}
// ReadAddresses reads the addresses stored in local keystore // ReadAddresses reads the addresses stored in local keystore
func ReadAddresses() []common.Address { func ReadAddresses() []common.Address {
priKeys := ReadPrivateKeys() priKeys := ReadPrivateKeys()

394
client/wallet_v2/main.go
Unescape View File

@ -1,394 +0,0 @@
package main
import (
"crypto/rand"
"encoding/hex"
"errors"
"flag"
"fmt"
"log"
"strings"
"github.com/harmony-one/harmony/p2p/p2pimpl"
"io"
"io/ioutil"
math_rand "math/rand"
"os"
"strconv"
"time"
"github.com/dedis/kyber"
"github.com/harmony-one/harmony/blockchain"
"github.com/harmony-one/harmony/client"
client_config "github.com/harmony-one/harmony/client/config"
"github.com/harmony-one/harmony/crypto"
"github.com/harmony-one/harmony/crypto/pki"
"github.com/harmony-one/harmony/node"
"github.com/harmony-one/harmony/p2p"
proto_node "github.com/harmony-one/harmony/proto/node"
"github.com/harmony-one/harmony/utils"
)
func main() {
// Account subcommands
accountImportCommand := flag.NewFlagSet("import", flag.ExitOnError)
accountImportPtr := accountImportCommand.String("privateKey", "", "Specify the private key to import")
// Transfer subcommands
transferCommand := flag.NewFlagSet("transfer", flag.ExitOnError)
transferSenderPtr := transferCommand.String("sender", "0", "Specify the sender account address or index")
transferReceiverPtr := transferCommand.String("receiver", "", "Specify the receiver account")
transferAmountPtr := transferCommand.Int("amount", 0, "Specify the amount to transfer")
// Verify that a subcommand has been provided
// os.Arg[0] is the main command
// os.Arg[1] will be the subcommand
if len(os.Args) < 2 {
fmt.Println("account or transfer subcommand is required")
os.Exit(1)
}
// Switch on the subcommand
switch os.Args[1] {
case "account":
switch os.Args[2] {
case "new":
randomBytes := [32]byte{}
_, err := io.ReadFull(rand.Reader, randomBytes[:])
if err != nil {
fmt.Println("Failed to create a new private key...")
return
}
priKey := crypto.Ed25519Curve.Scalar().SetBytes(randomBytes[:])
priKeyBytes, err := priKey.MarshalBinary()
if err != nil {
panic("Failed to serialize the private key")
}
pubKey := pki.GetPublicKeyFromScalar(priKey)
address := pki.GetAddressFromPublicKey(pubKey)
StorePrivateKey(priKeyBytes)
fmt.Printf("New account created:\nAddress: {%x}\n", address)
case "list":
for i, address := range ReadAddresses() {
fmt.Printf("Account %d:\n {%x}\n", i+1, address)
}
case "clearAll":
ClearKeystore()
fmt.Println("All existing accounts deleted...")
case "import":
accountImportCommand.Parse(os.Args[3:])
priKey := *accountImportPtr
if priKey == "" {
fmt.Println("Error: --privateKey is required")
return
}
if !accountImportCommand.Parsed() {
fmt.Println("Failed to parse flags")
}
priKeyBytes, err := hex.DecodeString(priKey)
if err != nil {
panic("Failed to parse the private key into bytes")
}
StorePrivateKey(priKeyBytes)
fmt.Println("Private key imported...")
case "showBalance":
walletNode := CreateWalletServerNode()
go walletNode.StartServer()
shardUtxoMap, err := FetchUtxos(ReadAddresses(), walletNode)
if err != nil {
fmt.Println(err)
}
PrintUtxoBalance(shardUtxoMap)
case "test":
// Testing code
priKey := pki.GetPrivateKeyScalarFromInt(444)
address := pki.GetAddressFromPrivateKey(priKey)
priKeyBytes, err := priKey.MarshalBinary()
if err != nil {
panic("Failed to deserialize private key scalar.")
}
fmt.Printf("Private Key :\n {%x}\n", priKeyBytes)
fmt.Printf("Address :\n {%x}\n", address)
}
case "transfer":
transferCommand.Parse(os.Args[2:])
if !transferCommand.Parsed() {
fmt.Println("Failed to parse flags")
}
sender := *transferSenderPtr
receiver := *transferReceiverPtr
amount := *transferAmountPtr
if amount <= 0 {
fmt.Println("Please specify positive amount to transfer")
}
priKeys := ReadPrivateKeys()
if len(priKeys) == 0 {
fmt.Println("No imported account to use.")
return
}
senderIndex, err := strconv.Atoi(sender)
senderAddress := ""
addresses := ReadAddresses()
if err != nil {
senderIndex = -1
for i, address := range addresses {
if fmt.Sprintf("%x", address) == senderAddress {
senderIndex = i
break
}
}
if senderIndex == -1 {
fmt.Println("The specified sender account is not imported yet.")
break
}
}
if senderIndex >= len(priKeys) {
fmt.Println("Sender account index out of bounds.")
return
}
receiverAddress, err := hex.DecodeString(receiver)
if err != nil || len(receiverAddress) != 20 {
fmt.Println("The receiver address is not a valid.")
return
}
// Generate transaction
trimmedReceiverAddress := [20]byte{}
copy(trimmedReceiverAddress[:], receiverAddress[:20])
senderPriKey := priKeys[senderIndex]
senderAddressBytes := pki.GetAddressFromPrivateKey(senderPriKey)
// Start client server
walletNode := CreateWalletServerNode()
go walletNode.StartServer()
shardUtxoMap, err := FetchUtxos([][20]byte{senderAddressBytes}, walletNode)
if err != nil {
fmt.Printf("Failed to fetch utxos: %s\n", err)
}
cummulativeBalance := 0
txInputs := []blockchain.TXInput{}
LOOP:
for shardID, utxoMap := range shardUtxoMap {
for txID, vout2AmountMap := range utxoMap[senderAddressBytes] {
txIDBytes, err := utils.Get32BytesFromString(txID)
if err != nil {
fmt.Println("Failed to parse txID")
continue
}
for voutIndex, utxoAmount := range vout2AmountMap {
cummulativeBalance += utxoAmount
txIn := blockchain.NewTXInput(blockchain.NewOutPoint(&txIDBytes, voutIndex), senderAddressBytes, shardID)
txInputs = append(txInputs, *txIn)
if cummulativeBalance >= amount {
break LOOP
}
}
}
}
txout := blockchain.TXOutput{Amount: amount, Address: trimmedReceiverAddress, ShardID: uint32(math_rand.Intn(len(shardUtxoMap)))}
txOutputs := []blockchain.TXOutput{txout}
if cummulativeBalance > amount {
changeTxOut := blockchain.TXOutput{Amount: cummulativeBalance - amount, Address: senderAddressBytes, ShardID: uint32(math_rand.Intn(len(shardUtxoMap)))}
txOutputs = append(txOutputs, changeTxOut)
}
tx := blockchain.Transaction{ID: [32]byte{}, PublicKey: pki.GetBytesFromPublicKey(pki.GetPublicKeyFromScalar(senderPriKey)), TxInput: txInputs, TxOutput: txOutputs, Proofs: nil}
tx.SetID() // TODO(RJ): figure out the correct way to set Tx ID.
tx.Sign(senderPriKey)
pubKey := crypto.Ed25519Curve.Point()
err = pubKey.UnmarshalBinary(tx.PublicKey[:])
if err != nil {
fmt.Println("Failed to deserialize public key", "error", err)
}
err = ExecuteTransaction(tx, walletNode)
if err != nil {
fmt.Println(err)
} else {
fmt.Println("Transaction submitted successfully")
}
default:
flag.PrintDefaults()
os.Exit(1)
}
}
func getShardIDToLeaderMap() map[uint32]p2p.Peer {
// TODO(ricl): Later use data.harmony.one for API.
str, _ := client.DownloadURLAsString("https://s3-us-west-2.amazonaws.com/unique-bucket-bin/leaders.txt")
lines := strings.Split(str, "\n")
shardIDLeaderMap := map[uint32]p2p.Peer{}
log.Print(lines)
for _, line := range lines {
if line == "" {
continue
}
parts := strings.Split(line, " ")
shardID := parts[3]
id, err := strconv.Atoi(shardID)
if err == nil {
shardIDLeaderMap[uint32(id)] = p2p.Peer{IP: parts[0], Port: parts[1]}
} else {
log.Print("[Generator] Error parsing the shard Id ", shardID)
}
}
return shardIDLeaderMap
}
// CreateWalletServerNode creates wallet server node.
func CreateWalletServerNode() *node.Node {
configr := client_config.NewConfig()
var shardIDLeaderMap map[uint32]p2p.Peer
var clientPeer *p2p.Peer
if true {
configr.ReadConfigFile("local_config2.txt")
shardIDLeaderMap = configr.GetShardIDToLeaderMap()
clientPeer = configr.GetClientPeer()
} else {
shardIDLeaderMap = getShardIDToLeaderMap()
clientPeer = &p2p.Peer{Port: "127.0.0.1", IP: "1234"}
}
host := p2pimpl.NewHost(*clientPeer)
walletNode := node.New(host, nil, nil)
walletNode.Client = client.NewClient(walletNode.GetHost(), &shardIDLeaderMap)
return walletNode
}
// ExecuteTransaction issues the transaction to the Harmony network
func ExecuteTransaction(tx blockchain.Transaction, walletNode *node.Node) error {
if tx.IsCrossShard() {
walletNode.Client.PendingCrossTxsMutex.Lock()
walletNode.Client.PendingCrossTxs[tx.ID] = &tx
walletNode.Client.PendingCrossTxsMutex.Unlock()
}
msg := proto_node.ConstructTransactionListMessage([]*blockchain.Transaction{&tx})
walletNode.BroadcastMessage(walletNode.Client.GetLeaders(), msg)
doneSignal := make(chan int)
go func() {
for {
if len(walletNode.Client.PendingCrossTxs) == 0 {
doneSignal <- 0
break
}
}
}()
select {
case <-doneSignal:
time.Sleep(100 * time.Millisecond)
return nil
case <-time.After(5 * time.Second):
return errors.New("Cross-shard Transaction processing timed out")
}
}
// FetchUtxos fetches utxos of specified address from the Harmony network
func FetchUtxos(addresses [][20]byte, walletNode *node.Node) (map[uint32]blockchain.UtxoMap, error) {
fmt.Println("Fetching account balance...")
walletNode.Client.ShardUtxoMap = make(map[uint32]blockchain.UtxoMap)
walletNode.BroadcastMessage(walletNode.Client.GetLeaders(), proto_node.ConstructFetchUtxoMessage(*walletNode.ClientPeer, addresses))
doneSignal := make(chan int)
go func() {
for {
if len(walletNode.Client.ShardUtxoMap) == len(*walletNode.Client.Leaders) {
doneSignal <- 0
break
}
}
}()
select {
case <-doneSignal:
return walletNode.Client.ShardUtxoMap, nil
case <-time.After(3 * time.Second):
return nil, errors.New("Utxo fetch timed out")
}
}
// PrintUtxoBalance prints UTXO balance.
func PrintUtxoBalance(shardUtxoMap map[uint32]blockchain.UtxoMap) {
addressBalance := make(map[[20]byte]int)
for _, utxoMap := range shardUtxoMap {
for address, txHash2Vout2AmountMap := range utxoMap {
for _, vout2AmountMap := range txHash2Vout2AmountMap {
for _, amount := range vout2AmountMap {
value, ok := addressBalance[address]
if ok {
addressBalance[address] = value + amount
} else {
addressBalance[address] = amount
}
}
}
}
}
for address, balance := range addressBalance {
fmt.Printf("Address: {%x}\n", address)
fmt.Printf("Balance: %d\n", balance)
}
}
// ReadAddresses reads the addresses stored in local keystore
func ReadAddresses() [][20]byte {
priKeys := ReadPrivateKeys()
addresses := [][20]byte{}
for _, key := range priKeys {
addresses = append(addresses, pki.GetAddressFromPrivateKey(key))
}
return addresses
}
// StorePrivateKey stores the specified private key in local keystore
func StorePrivateKey(priKey []byte) {
for _, address := range ReadAddresses() {
if address == pki.GetAddressFromPrivateKey(crypto.Ed25519Curve.Scalar().SetBytes(priKey)) {
fmt.Println("The key already exists in the keystore")
return
}
}
f, err := os.OpenFile("keystore", os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
panic("Failed to open keystore")
}
_, err = f.Write(priKey)
if err != nil {
panic("Failed to write to keystore")
}
f.Close()
}
// ClearKeystore deletes all data in the local keystore
func ClearKeystore() {
ioutil.WriteFile("keystore", []byte{}, 0644)
}
// ReadPrivateKeys reads all the private key stored in local keystore
func ReadPrivateKeys() []kyber.Scalar {
keys, err := ioutil.ReadFile("keystore")
if err != nil {
return []kyber.Scalar{}
}
keyScalars := []kyber.Scalar{}
for i := 0; i < len(keys); i += 32 {
priKey := crypto.Ed25519Curve.Scalar()
priKey.UnmarshalBinary(keys[i : i+32])
keyScalars = append(keyScalars, priKey)
}
return keyScalars
}

5
consensus/consensus.go
Unescape View File

@ -13,7 +13,6 @@ import (
"github.com/ethereum/go-ethereum/crypto/sha3" "github.com/ethereum/go-ethereum/crypto/sha3"
"github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rlp"
"github.com/harmony-one/harmony/blockchain"
"github.com/harmony-one/harmony/core/state" "github.com/harmony-one/harmony/core/state"
"github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/core/types"
"github.com/harmony-one/harmony/crypto" "github.com/harmony-one/harmony/crypto"
@ -91,10 +90,10 @@ type Consensus struct {
// Signal channel for starting a new consensus process // Signal channel for starting a new consensus process
ReadySignal chan struct{} ReadySignal chan struct{}
// The verifier func passed from Node object // The verifier func passed from Node object
BlockVerifier func(*blockchain.Block) bool BlockVerifier func(*types.Block) bool
// The post-consensus processing func passed from Node object // The post-consensus processing func passed from Node object
// Called when consensus on a new block is done // Called when consensus on a new block is done
OnConsensusDone func(*blockchain.Block) OnConsensusDone func(*types.Block)
Log log.Logger Log log.Logger

97
consensus/consensus_leader.go
Unescape View File

@ -3,13 +3,12 @@ package consensus
import ( import (
"bytes" "bytes"
"encoding/binary" "encoding/binary"
"encoding/gob"
"encoding/hex" "encoding/hex"
"errors" "errors"
"github.com/ethereum/go-ethereum/rlp"
"strconv" "strconv"
"time" "time"
"github.com/ethereum/go-ethereum/rlp"
"github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/core/types"
"github.com/harmony-one/harmony/p2p/host" "github.com/harmony-one/harmony/p2p/host"
"github.com/harmony-one/harmony/services/explorer" "github.com/harmony-one/harmony/services/explorer"
@ -18,7 +17,6 @@ import (
"github.com/dedis/kyber" "github.com/dedis/kyber"
"github.com/dedis/kyber/sign/schnorr" "github.com/dedis/kyber/sign/schnorr"
"github.com/harmony-one/harmony/blockchain"
"github.com/harmony-one/harmony/crypto" "github.com/harmony-one/harmony/crypto"
"github.com/harmony-one/harmony/log" "github.com/harmony-one/harmony/log"
"github.com/harmony-one/harmony/p2p" "github.com/harmony-one/harmony/p2p"
@ -34,35 +32,7 @@ var (
) )
// WaitForNewBlock waits for the next new block to run consensus on // WaitForNewBlock waits for the next new block to run consensus on
func (consensus *Consensus) WaitForNewBlock(blockChannel chan blockchain.Block) { func (consensus *Consensus) WaitForNewBlock(blockChannel chan *types.Block) {
consensus.Log.Debug("Waiting for block", "consensus", consensus)
for { // keep waiting for new blocks
newBlock := <-blockChannel
c := consensus.RemovePeers(consensus.OfflinePeerList)
if c > 0 {
consensus.Log.Debug("WaitForNewBlock", "removed peers", c)
}
for !consensus.HasEnoughValidators() {
consensus.Log.Debug("Not enough validators", "# Validators", len(consensus.PublicKeys))
time.Sleep(waitForEnoughValidators * time.Millisecond)
}
// TODO: think about potential race condition
startTime = time.Now()
consensus.Log.Debug("STARTING CONSENSUS", "consensus", consensus, "startTime", startTime, "publicKeys", len(consensus.PublicKeys))
for consensus.state == Finished {
// time.Sleep(500 * time.Millisecond)
consensus.ResetState()
consensus.startConsensus(&newBlock)
break
}
}
}
// WaitForNewBlockAccount waits for the next new block to run consensus on
func (consensus *Consensus) WaitForNewBlockAccount(blockChannel chan *types.Block) {
consensus.Log.Debug("Waiting for block", "consensus", consensus) consensus.Log.Debug("Waiting for block", "consensus", consensus)
for { // keep waiting for new blocks for { // keep waiting for new blocks
newBlock := <-blockChannel newBlock := <-blockChannel
@ -82,13 +52,8 @@ func (consensus *Consensus) WaitForNewBlockAccount(blockChannel chan *types.Bloc
consensus.Log.Debug("STARTING CONSENSUS", "numTxs", len(newBlock.Transactions()), "consensus", consensus, "startTime", startTime, "publicKeys", len(consensus.PublicKeys)) consensus.Log.Debug("STARTING CONSENSUS", "numTxs", len(newBlock.Transactions()), "consensus", consensus, "startTime", startTime, "publicKeys", len(consensus.PublicKeys))
for consensus.state == Finished { for consensus.state == Finished {
// time.Sleep(500 * time.Millisecond) // time.Sleep(500 * time.Millisecond)
data, err := rlp.EncodeToBytes(newBlock) consensus.ResetState()
if err == nil { consensus.startConsensus(newBlock)
consensus.ResetState()
consensus.startConsensus(&blockchain.Block{Hash: newBlock.Hash(), AccountBlock: data})
} else {
consensus.Log.Error("Failed encoding the block with RLP")
}
break break
} }
} }
@ -107,8 +72,6 @@ func (consensus *Consensus) ProcessMessageLeader(message []byte) {
} }
switch msgType { switch msgType {
case proto_consensus.StartConsensus:
consensus.processStartConsensusMessage(payload)
case proto_consensus.Commit: case proto_consensus.Commit:
consensus.processCommitMessage(payload, ChallengeDone) consensus.processCommitMessage(payload, ChallengeDone)
case proto_consensus.Response: case proto_consensus.Response:
@ -122,25 +85,20 @@ func (consensus *Consensus) ProcessMessageLeader(message []byte) {
} }
} }
// processStartConsensusMessage is the handler for message which triggers consensus process.
// TODO(minh): clean-up. this function is never called.
func (consensus *Consensus) processStartConsensusMessage(payload []byte) {
// TODO: remove these method after testnet
tx := blockchain.NewCoinbaseTX([20]byte{0}, "y", 0)
consensus.startConsensus(blockchain.NewGenesisBlock(tx, 0))
}
// startConsensus starts a new consensus for a block by broadcast a announce message to the validators // startConsensus starts a new consensus for a block by broadcast a announce message to the validators
func (consensus *Consensus) startConsensus(newBlock *blockchain.Block) { func (consensus *Consensus) startConsensus(newBlock *types.Block) {
// Copy over block hash and block header data // Copy over block hash and block header data
copy(consensus.blockHash[:], newBlock.Hash[:]) blockHash := newBlock.Hash()
copy(consensus.blockHash[:], blockHash[:])
consensus.Log.Debug("Start encoding block") consensus.Log.Debug("Start encoding block")
// prepare message and broadcast to validators // prepare message and broadcast to validators
byteBuffer := bytes.NewBuffer([]byte{}) encodedBlock, err := rlp.EncodeToBytes(newBlock)
encoder := gob.NewEncoder(byteBuffer) if err != nil {
encoder.Encode(newBlock) consensus.Log.Debug("Failed encoding block")
consensus.blockHeader = byteBuffer.Bytes() return
}
consensus.blockHeader = encodedBlock
consensus.Log.Debug("Stop encoding block") consensus.Log.Debug("Stop encoding block")
msgToSend := consensus.constructAnnounceMessage() msgToSend := consensus.constructAnnounceMessage()
@ -448,21 +406,20 @@ func (consensus *Consensus) processResponseMessage(payload []byte, targetState S
} else { } else {
// TODO: reconstruct the whole block from header and transactions // TODO: reconstruct the whole block from header and transactions
// For now, we used the stored whole block already stored in consensus.blockHeader // For now, we used the stored whole block already stored in consensus.blockHeader
txDecoder := gob.NewDecoder(bytes.NewReader(consensus.blockHeader)) var blockHeaderObj types.Block
var blockHeaderObj blockchain.Block err = rlp.DecodeBytes(consensus.blockHeader, &blockHeaderObj)
err = txDecoder.Decode(&blockHeaderObj)
if err != nil { if err != nil {
consensus.Log.Debug("failed to construct the new block after consensus") consensus.Log.Debug("failed to construct the new block after consensus")
} }
// Sign the block // Sign the block
copy(blockHeaderObj.Signature[:], collectiveSig[:]) copy(blockHeaderObj.Header().Signature[:], collectiveSig[:])
copy(blockHeaderObj.Bitmap[:], bitmap) copy(blockHeaderObj.Header().Bitmap[:], bitmap)
consensus.OnConsensusDone(&blockHeaderObj) consensus.OnConsensusDone(&blockHeaderObj)
consensus.reportMetrics(blockHeaderObj) consensus.reportMetrics(blockHeaderObj)
// Dump new block into level db. // Dump new block into level db.
explorer.GetStorageInstance(consensus.leader.IP, consensus.leader.Port, true).Dump(blockHeaderObj.AccountBlock, consensus.consensusID) explorer.GetStorageInstance(consensus.leader.IP, consensus.leader.Port, true).Dump(&blockHeaderObj, consensus.consensusID)
// Claim new consensus reached. // Claim new consensus reached.
consensus.Log.Debug("Consensus reached with signatures.", "numOfSignatures", len(*responses)) consensus.Log.Debug("Consensus reached with signatures.", "numOfSignatures", len(*responses))
// Reset state to Finished, and clear other data. // Reset state to Finished, and clear other data.
@ -501,19 +458,10 @@ func (consensus *Consensus) verifyResponse(commitments *map[uint16]kyber.Point,
return nil return nil
} }
func (consensus *Consensus) reportMetrics(block blockchain.Block) { func (consensus *Consensus) reportMetrics(block types.Block) {
if block.IsStateBlock() { // Skip state block stats
return
}
endTime := time.Now() endTime := time.Now()
timeElapsed := endTime.Sub(startTime) timeElapsed := endTime.Sub(startTime)
numOfTxs := int(block.NumTransactions) numOfTxs := len(block.Transactions())
if block.AccountBlock != nil {
accountBlock := new(types.Block)
rlp.DecodeBytes(block.AccountBlock, accountBlock)
numOfTxs = len(accountBlock.Transactions())
}
tps := float64(numOfTxs) / timeElapsed.Seconds() tps := float64(numOfTxs) / timeElapsed.Seconds()
consensus.Log.Info("TPS Report", consensus.Log.Info("TPS Report",
"numOfTXs", numOfTxs, "numOfTXs", numOfTxs,
@ -530,8 +478,9 @@ func (consensus *Consensus) reportMetrics(block blockchain.Block) {
} }
txHashes := []string{} txHashes := []string{}
for i, end := 0, len(block.TransactionIds); i < 3 && i < end; i++ { for i, end := 0, len(block.Transactions()); i < 3 && i < end; i++ {
txHashes = append(txHashes, hex.EncodeToString(block.TransactionIds[end-1-i][:])) txHash := block.Transactions()[end-1-i].Hash()
txHashes = append(txHashes, hex.EncodeToString(txHash[:]))
} }
metrics := map[string]interface{}{ metrics := map[string]interface{}{
"key": consensus.pubKey.String(), "key": consensus.pubKey.String(),

34
consensus/consensus_validator.go
Unescape View File

@ -3,10 +3,10 @@ package consensus
import ( import (
"bytes" "bytes"
"encoding/binary" "encoding/binary"
"encoding/gob"
"github.com/dedis/kyber/sign/schnorr" "github.com/dedis/kyber/sign/schnorr"
"github.com/ethereum/go-ethereum/rlp"
"github.com/harmony-one/harmony/attack" "github.com/harmony-one/harmony/attack"
"github.com/harmony-one/harmony/blockchain" "github.com/harmony-one/harmony/core/types"
"github.com/harmony-one/harmony/crypto" "github.com/harmony-one/harmony/crypto"
"github.com/harmony-one/harmony/log" "github.com/harmony-one/harmony/log"
proto_consensus "github.com/harmony-one/harmony/proto/consensus" proto_consensus "github.com/harmony-one/harmony/proto/consensus"
@ -83,11 +83,10 @@ func (consensus *Consensus) processAnnounceMessage(payload []byte) {
} }
// check block header is valid // check block header is valid
txDecoder := gob.NewDecoder(bytes.NewReader(blockHeader)) var blockHeaderObj types.Block // TODO: separate header from block. Right now, this blockHeader data is actually the whole block
var blockHeaderObj blockchain.Block // TODO: separate header from block. Right now, this blockHeader data is actually the whole block err := rlp.DecodeBytes(blockHeader, &blockHeaderObj)
err := txDecoder.Decode(&blockHeaderObj)
if err != nil { if err != nil {
consensus.Log.Warn("Unparseable block header data", "consensus", consensus) consensus.Log.Warn("Unparseable block header data", "error", err)
return return
} }
consensus.blockHeader = blockHeader // TODO: think about remove this field and use blocksReceived instead consensus.blockHeader = blockHeader // TODO: think about remove this field and use blocksReceived instead
@ -102,14 +101,10 @@ func (consensus *Consensus) processAnnounceMessage(payload []byte) {
} }
// check block hash // check block hash
if blockHeaderObj.AccountBlock != nil { hash := blockHeaderObj.Hash()
// TODO: deal with block hash of account model if !bytes.Equal(blockHash[:], hash[:]) {
} else { consensus.Log.Warn("Block hash doesn't match", "consensus", consensus)
// TODO: totally switch to account model. return
if !bytes.Equal(blockHash[:], blockHeaderObj.CalculateBlockHash()[:]) || !bytes.Equal(blockHeaderObj.Hash[:], blockHeaderObj.CalculateBlockHash()[:]) {
consensus.Log.Warn("Block hash doesn't match", "consensus", consensus)
return
}
} }
// check block data (transactions // check block data (transactions
@ -257,9 +252,12 @@ func (consensus *Consensus) processChallengeMessage(payload []byte, targetState
// TODO: reconstruct the whole block from header and transactions // TODO: reconstruct the whole block from header and transactions
// For now, we used the stored whole block in consensus.blockHeader // For now, we used the stored whole block in consensus.blockHeader
txDecoder := gob.NewDecoder(bytes.NewReader(val.blockHeader)) var blockHeaderObj types.Block // TODO: separate header from block. Right now, this blockHeader data is actually the whole block
var blockHeaderObj blockchain.Block err := rlp.DecodeBytes(val.blockHeader, &blockHeaderObj)
err := txDecoder.Decode(&blockHeaderObj) if err != nil {
consensus.Log.Warn("Unparseable block header data", "error", err)
return
}
if err != nil { if err != nil {
consensus.Log.Debug("failed to construct the new block after consensus") consensus.Log.Debug("failed to construct the new block after consensus")
} }
@ -268,7 +266,7 @@ func (consensus *Consensus) processChallengeMessage(payload []byte, targetState
consensus.Log.Debug("[WARNING] Block content is not verified successfully", "consensusID", consensus.consensusID) consensus.Log.Debug("[WARNING] Block content is not verified successfully", "consensusID", consensus.consensusID)
return return
} }
consensus.Log.Info("Finished Response. Adding block to chain", "numTx", len(blockHeaderObj.Transactions)) consensus.Log.Info("Finished Response. Adding block to chain", "numTx", len(blockHeaderObj.Transactions()))
consensus.OnConsensusDone(&blockHeaderObj) consensus.OnConsensusDone(&blockHeaderObj)
} else { } else {
break break

3
core/types/block.go
Unescape View File

@ -92,6 +92,9 @@ type Header struct {
MixDigest common.Hash `json:"mixHash" gencodec:"required"` MixDigest common.Hash `json:"mixHash" gencodec:"required"`
Nonce BlockNonce `json:"nonce" gencodec:"required"` Nonce BlockNonce `json:"nonce" gencodec:"required"`
ShardID ShardID `json:"shardID" gencodec:"required"` ShardID ShardID `json:"shardID" gencodec:"required"`
Bitmap []byte `json:"bitmap" gencodec:"required"` // Contains which validator signed the block.
Signature [66]byte `json:"signature" gencodec:"required"` // Schnorr collective signature.
// TODO(RJ): add epoch info
} }
// field type overrides for gencodec // field type overrides for gencodec

159
node/node.go
Unescape View File

@ -6,8 +6,8 @@ import (
"encoding/gob" "encoding/gob"
"encoding/hex" "encoding/hex"
"fmt" "fmt"
"github.com/ethereum/go-ethereum/rlp"
"math/big" "math/big"
"math/rand"
"os" "os"
"strconv" "strconv"
"strings" "strings"
@ -23,7 +23,6 @@ import (
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/params"
"github.com/harmony-one/harmony/blockchain"
bft "github.com/harmony-one/harmony/consensus" bft "github.com/harmony-one/harmony/consensus"
"github.com/harmony-one/harmony/core" "github.com/harmony-one/harmony/core"
"github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/core/types"
@ -97,16 +96,13 @@ type NetworkNode struct {
// Node represents a program (machine) participating in the network // Node represents a program (machine) participating in the network
// TODO(minhdoan, rj): consider using BlockChannel *chan blockchain.Block for efficiency. // TODO(minhdoan, rj): consider using BlockChannel *chan blockchain.Block for efficiency.
type Node struct { type Node struct {
Consensus *bft.Consensus // Consensus object containing all Consensus related data (e.g. committee members, signatures, commits) Consensus *bft.Consensus // Consensus object containing all Consensus related data (e.g. committee members, signatures, commits)
BlockChannel chan blockchain.Block // The channel to receive new blocks from Node BlockChannel chan *types.Block // The channel to receive new blocks from Node
pendingTransactions []*blockchain.Transaction // All the transactions received but not yet processed for Consensus pendingTransactions types.Transactions // All the transactions received but not yet processed for Consensus
transactionInConsensus []*blockchain.Transaction // The transactions selected into the new block and under Consensus process transactionInConsensus []*types.Transaction // The transactions selected into the new block and under Consensus process
blockchain *blockchain.Blockchain // The blockchain for the shard where this node belongs blockchain *core.BlockChain // The blockchain for the shard where this node belongs
db *hdb.LDBDatabase // LevelDB to store blockchain. db *hdb.LDBDatabase // LevelDB to store blockchain.
UtxoPool *blockchain.UTXOPool // The corresponding UTXO pool of the current blockchain log log.Logger // Log utility
CrossTxsInConsensus []*blockchain.CrossShardTxAndProof // The cross shard txs that is under consensus, the proof is not filled yet.
CrossTxsToReturn []*blockchain.CrossShardTxAndProof // The cross shard txs and proof that needs to be sent back to the user client.
log log.Logger // Log utility
pendingTxMutex sync.Mutex pendingTxMutex sync.Mutex
crossTxToReturnMutex sync.Mutex crossTxToReturnMutex sync.Mutex
ClientPeer *p2p.Peer // The peer for the benchmark tx generator client, used for leaders to return proof-of-accept ClientPeer *p2p.Peer // The peer for the benchmark tx generator client, used for leaders to return proof-of-accept
@ -114,17 +110,11 @@ type Node struct {
SelfPeer p2p.Peer // TODO(minhdoan): it could be duplicated with Self below whose is Alok work. SelfPeer p2p.Peer // TODO(minhdoan): it could be duplicated with Self below whose is Alok work.
IDCPeer p2p.Peer IDCPeer p2p.Peer
chain *core.BlockChain // Account Model Neighbors sync.Map // All the neighbor nodes, key is the sha256 of Peer IP/Port, value is the p2p.Peer
Neighbors sync.Map // All the neighbor nodes, key is the sha256 of Peer IP/Port, value is the p2p.Peer State State // State of the Node
State State // State of the Node
// Account Model TxPool *core.TxPool
pendingTransactionsAccount types.Transactions // TODO: replace with txPool Worker *worker.Worker
pendingTxMutexAccount sync.Mutex
Chain *core.BlockChain
TxPool *core.TxPool
BlockChannelAccount chan *types.Block // The channel to receive new blocks from Node
Worker *worker.Worker
// Client server (for wallet requests) // Client server (for wallet requests)
clientServer *clientService.Server clientServer *clientService.Server
@ -148,54 +138,31 @@ type Node struct {
OfflinePeers chan p2p.Peer OfflinePeers chan p2p.Peer
} }
// Add new crossTx and proofs to the list of crossTx that needs to be sent back to client // Blockchain returns the blockchain from node
func (node *Node) addCrossTxsToReturn(crossTxs []*blockchain.CrossShardTxAndProof) { func (node *Node) Blockchain() *core.BlockChain {
node.crossTxToReturnMutex.Lock() return node.blockchain
node.CrossTxsToReturn = append(node.CrossTxsToReturn, crossTxs...)
node.crossTxToReturnMutex.Unlock()
node.log.Debug("Got more cross transactions to return", "num", len(crossTxs), len(node.pendingTransactions), "node", node)
} }
// Add new transactions to the pending transaction list // Add new transactions to the pending transaction list
func (node *Node) addPendingTransactions(newTxs []*blockchain.Transaction) { func (node *Node) addPendingTransactions(newTxs types.Transactions) {
node.pendingTxMutex.Lock() node.pendingTxMutex.Lock()
node.pendingTransactions = append(node.pendingTransactions, newTxs...) node.pendingTransactions = append(node.pendingTransactions, newTxs...)
node.pendingTxMutex.Unlock() node.pendingTxMutex.Unlock()
} node.log.Debug("Got more transactions", "num", len(newTxs), "totalPending", len(node.pendingTransactions))
// Add new transactions to the pending transaction list
func (node *Node) addPendingTransactionsAccount(newTxs types.Transactions) {
node.pendingTxMutexAccount.Lock()
node.pendingTransactionsAccount = append(node.pendingTransactionsAccount, newTxs...)
node.pendingTxMutexAccount.Unlock()
node.log.Debug("Got more transactions (account model)", "num", len(newTxs), "totalPending", len(node.pendingTransactionsAccount))
} }
// Take out a subset of valid transactions from the pending transaction list // Take out a subset of valid transactions from the pending transaction list
// Note the pending transaction list will then contain the rest of the txs // Note the pending transaction list will then contain the rest of the txs
func (node *Node) getTransactionsForNewBlock(maxNumTxs int) ([]*blockchain.Transaction, []*blockchain.CrossShardTxAndProof) { func (node *Node) getTransactionsForNewBlock(maxNumTxs int) types.Transactions {
node.pendingTxMutex.Lock() node.pendingTxMutex.Lock()
selected, unselected, invalid, crossShardTxs := node.UtxoPool.SelectTransactionsForNewBlock(node.pendingTransactions, maxNumTxs) selected, unselected, invalid := node.Worker.SelectTransactionsForNewBlock(node.pendingTransactions, maxNumTxs)
_ = invalid // invalid txs are discard _ = invalid // invalid txs are discard
node.log.Debug("Invalid transactions discarded", "number", len(invalid)) node.log.Debug("Invalid transactions discarded", "number", len(invalid))
node.pendingTransactions = unselected node.pendingTransactions = unselected
node.log.Debug("Remaining pending transactions", "number", len(node.pendingTransactions))
node.pendingTxMutex.Unlock() node.pendingTxMutex.Unlock()
return selected, crossShardTxs return selected
}
// Take out a subset of valid transactions from the pending transaction list
// Note the pending transaction list will then contain the rest of the txs
func (node *Node) getTransactionsForNewBlockAccount(maxNumTxs int) (types.Transactions, []*blockchain.CrossShardTxAndProof) {
node.pendingTxMutexAccount.Lock()
selected, unselected, invalid := node.Worker.SelectTransactionsForNewBlock(node.pendingTransactionsAccount, maxNumTxs)
_ = invalid // invalid txs are discard
node.log.Debug("Invalid transactions discarded", "number", len(invalid))
node.pendingTransactionsAccount = unselected
node.log.Debug("Remaining pending transactions", "number", len(node.pendingTransactionsAccount))
node.pendingTxMutexAccount.Unlock()
return selected, nil //TODO: replace cross-shard proofs for account model
} }
// StartServer starts a server and process the request by a handler. // StartServer starts a server and process the request by a handler.
@ -217,19 +184,8 @@ func (node *Node) String() string {
// Currently used for stats reporting purpose // Currently used for stats reporting purpose
func (node *Node) countNumTransactionsInBlockchain() int { func (node *Node) countNumTransactionsInBlockchain() int {
count := 0 count := 0
for _, block := range node.blockchain.Blocks { for block := node.blockchain.CurrentBlock(); block != nil; block = node.blockchain.GetBlockByHash(block.Header().ParentHash) {
count += len(block.Transactions) count += len(block.Transactions())
}
return count
}
// Count the total number of transactions in the blockchain
// Currently used for stats reporting purpose
func (node *Node) countNumTransactionsInBlockchainAccount() int {
count := 0
for curBlock := node.Chain.CurrentBlock(); curBlock != nil; {
count += len(curBlock.Transactions())
curBlock = node.Chain.GetBlockByHash(curBlock.ParentHash())
} }
return count return count
} }
@ -277,46 +233,17 @@ func New(host host.Host, consensus *bft.Consensus, db *hdb.LDBDatabase) *Node {
if host != nil && consensus != nil { if host != nil && consensus != nil {
// Consensus and associated channel to communicate blocks // Consensus and associated channel to communicate blocks
node.Consensus = consensus node.Consensus = consensus
node.BlockChannel = make(chan blockchain.Block)
// Genesis Block
// TODO(minh): Use or implement new function in blockchain package for this.
genesisBlock := &blockchain.Blockchain{}
genesisBlock.Blocks = make([]*blockchain.Block, 0)
// TODO(RJ): use miner's address as coinbase address
coinbaseTx := blockchain.NewCoinbaseTX(pki.GetAddressFromInt(1), "0", node.Consensus.ShardID)
genesisBlock.Blocks = append(genesisBlock.Blocks, blockchain.NewGenesisBlock(coinbaseTx, node.Consensus.ShardID))
node.blockchain = genesisBlock
// UTXO pool from Genesis block
node.UtxoPool = blockchain.CreateUTXOPoolFromGenesisBlock(node.blockchain.Blocks[0])
// Initialize level db. // Initialize level db.
node.db = db node.db = db
// (account model) // Initialize genesis block and blockchain
rand.Seed(0) genesisAlloc := node.CreateGenesisAllocWithTestingAddresses(100)
len := 1000000 contractKey, _ := ecdsa.GenerateKey(crypto.S256(), strings.NewReader("Test contract key string blablablablablablablablablablablablablablablablabl"))
bytes := make([]byte, len)
for i := 0; i < len; i++ {
bytes[i] = byte(rand.Intn(100))
}
reader := strings.NewReader(string(bytes))
genesisAloc := make(core.GenesisAlloc)
for i := 0; i < 100; i++ {
testBankKey, _ := ecdsa.GenerateKey(crypto.S256(), reader)
testBankAddress := crypto.PubkeyToAddress(testBankKey.PublicKey)
testBankFunds := big.NewInt(1000)
testBankFunds = testBankFunds.Mul(testBankFunds, big.NewInt(params.Ether))
genesisAloc[testBankAddress] = core.GenesisAccount{Balance: testBankFunds}
node.TestBankKeys = append(node.TestBankKeys, testBankKey)
}
contractKey, _ := ecdsa.GenerateKey(crypto.S256(), reader)
contractAddress := crypto.PubkeyToAddress(contractKey.PublicKey) contractAddress := crypto.PubkeyToAddress(contractKey.PublicKey)
contractFunds := big.NewInt(9000000) contractFunds := big.NewInt(9000000)
contractFunds = contractFunds.Mul(contractFunds, big.NewInt(params.Ether)) contractFunds = contractFunds.Mul(contractFunds, big.NewInt(params.Ether))
genesisAloc[contractAddress] = core.GenesisAccount{Balance: contractFunds} genesisAlloc[contractAddress] = core.GenesisAccount{Balance: contractFunds}
node.ContractKeys = append(node.ContractKeys, contractKey) node.ContractKeys = append(node.ContractKeys, contractKey)
database := hdb.NewMemDatabase() database := hdb.NewMemDatabase()
@ -324,16 +251,15 @@ func New(host host.Host, consensus *bft.Consensus, db *hdb.LDBDatabase) *Node {
chainConfig.ChainID = big.NewInt(int64(node.Consensus.ShardID)) // Use ChainID as piggybacked ShardID chainConfig.ChainID = big.NewInt(int64(node.Consensus.ShardID)) // Use ChainID as piggybacked ShardID
gspec := core.Genesis{ gspec := core.Genesis{
Config: chainConfig, Config: chainConfig,
Alloc: genesisAloc, Alloc: genesisAlloc,
ShardID: uint32(node.Consensus.ShardID), ShardID: uint32(node.Consensus.ShardID),
} }
_ = gspec.MustCommit(database) _ = gspec.MustCommit(database)
chain, _ := core.NewBlockChain(database, nil, gspec.Config, node.Consensus, vm.Config{}, nil) chain, _ := core.NewBlockChain(database, nil, gspec.Config, node.Consensus, vm.Config{}, nil)
node.Chain = chain node.blockchain = chain
//This one is not used --- RJ. node.BlockChannel = make(chan *types.Block)
node.TxPool = core.NewTxPool(core.DefaultTxPoolConfig, params.TestChainConfig, chain) node.TxPool = core.NewTxPool(core.DefaultTxPoolConfig, params.TestChainConfig, chain)
node.BlockChannelAccount = make(chan *types.Block)
node.Worker = worker.New(params.TestChainConfig, chain, node.Consensus, pki.GetAddressFromPublicKey(node.SelfPeer.PubKey), node.Consensus.ShardID) node.Worker = worker.New(params.TestChainConfig, chain, node.Consensus, pki.GetAddressFromPublicKey(node.SelfPeer.PubKey), node.Consensus.ShardID)
node.AddSmartContractsToPendingTransactions() node.AddSmartContractsToPendingTransactions()
@ -439,12 +365,12 @@ func (node *Node) AddSmartContractsToPendingTransactions() {
mycontracttx, _ := types.SignTx(types.NewContractCreation(uint64(0), node.Consensus.ShardID, contractFunds, params.TxGasContractCreation*10, nil, dataEnc), types.HomesteadSigner{}, priKey) mycontracttx, _ := types.SignTx(types.NewContractCreation(uint64(0), node.Consensus.ShardID, contractFunds, params.TxGasContractCreation*10, nil, dataEnc), types.HomesteadSigner{}, priKey)
node.ContractAddresses = append(node.ContractAddresses, crypto.CreateAddress(crypto.PubkeyToAddress(priKey.PublicKey), uint64(0))) node.ContractAddresses = append(node.ContractAddresses, crypto.CreateAddress(crypto.PubkeyToAddress(priKey.PublicKey), uint64(0)))
node.addPendingTransactionsAccount(types.Transactions{mycontracttx}) node.addPendingTransactions(types.Transactions{mycontracttx})
} }
//CallFaucetContract invokes the faucet contract to give the walletAddress initial money //CallFaucetContract invokes the faucet contract to give the walletAddress initial money
func (node *Node) CallFaucetContract(walletAddress common.Address) common.Hash { func (node *Node) CallFaucetContract(walletAddress common.Address) common.Hash {
state, err := node.Chain.State() state, err := node.blockchain.State()
if err != nil { if err != nil {
log.Error("Failed to get chain state", "Error", err) log.Error("Failed to get chain state", "Error", err)
} }
@ -454,7 +380,7 @@ func (node *Node) CallFaucetContract(walletAddress common.Address) common.Hash {
dataEnc := common.FromHex(contractData) dataEnc := common.FromHex(contractData)
tx, _ := types.SignTx(types.NewTransaction(nonce, node.ContractAddresses[0], node.Consensus.ShardID, big.NewInt(0), params.TxGasContractCreation*10, nil, dataEnc), types.HomesteadSigner{}, node.ContractKeys[0]) tx, _ := types.SignTx(types.NewTransaction(nonce, node.ContractAddresses[0], node.Consensus.ShardID, big.NewInt(0), params.TxGasContractCreation*10, nil, dataEnc), types.HomesteadSigner{}, node.ContractKeys[0])
node.addPendingTransactionsAccount(types.Transactions{tx}) node.addPendingTransactions(types.Transactions{tx})
return tx.Hash() return tx.Hash()
} }
@ -505,7 +431,7 @@ func (node *Node) SupportExplorer() {
// InitClientServer initializes client server. // InitClientServer initializes client server.
func (node *Node) InitClientServer() { func (node *Node) InitClientServer() {
node.clientServer = clientService.NewServer(node.Chain.State, node.CallFaucetContract) node.clientServer = clientService.NewServer(node.blockchain.State, node.CallFaucetContract)
} }
// StartClientServer starts client server. // StartClientServer starts client server.
@ -537,13 +463,20 @@ func (node *Node) StartSyncingServer() {
func (node *Node) CalculateResponse(request *downloader_pb.DownloaderRequest) (*downloader_pb.DownloaderResponse, error) { func (node *Node) CalculateResponse(request *downloader_pb.DownloaderRequest) (*downloader_pb.DownloaderResponse, error) {
response := &downloader_pb.DownloaderResponse{} response := &downloader_pb.DownloaderResponse{}
if request.Type == downloader_pb.DownloaderRequest_HEADER { if request.Type == downloader_pb.DownloaderRequest_HEADER {
for _, block := range node.blockchain.Blocks {
response.Payload = append(response.Payload, block.Hash[:]) for block := node.blockchain.CurrentBlock(); block != nil; block = node.blockchain.GetBlockByHash(block.Header().ParentHash) {
blockHash := block.Hash()
response.Payload = append(response.Payload, blockHash[:])
} }
} else { } else {
for i := range request.Hashes { for _, bytes := range request.Hashes {
block := node.blockchain.FindBlock(request.Hashes[i]) var hash common.Hash
response.Payload = append(response.Payload, block.Serialize()) hash.SetBytes(bytes)
block := node.blockchain.GetBlockByHash(hash)
encodedBlock, err := rlp.EncodeToBytes(block)
if err != nil {
response.Payload = append(response.Payload, encodedBlock)
}
} }
} }
return response, nil return response, nil

319
node/node_handler.go
Unescape View File

@ -2,22 +2,19 @@ package node
import ( import (
"bytes" "bytes"
"encoding/gob"
"fmt" "fmt"
"github.com/ethereum/go-ethereum/common"
"os" "os"
"strconv"
"time" "time"
"github.com/dedis/kyber" "github.com/dedis/kyber"
"github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rlp"
"github.com/harmony-one/harmony/blockchain"
"github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/core/types"
hmy_crypto "github.com/harmony-one/harmony/crypto" hmy_crypto "github.com/harmony-one/harmony/crypto"
"github.com/harmony-one/harmony/crypto/pki" "github.com/harmony-one/harmony/crypto/pki"
"github.com/harmony-one/harmony/p2p" "github.com/harmony-one/harmony/p2p"
"github.com/harmony-one/harmony/p2p/host" "github.com/harmony-one/harmony/p2p/host"
"github.com/harmony-one/harmony/proto" "github.com/harmony-one/harmony/proto"
"github.com/harmony-one/harmony/proto/client"
"github.com/harmony-one/harmony/proto/consensus" "github.com/harmony-one/harmony/proto/consensus"
proto_identity "github.com/harmony-one/harmony/proto/identity" proto_identity "github.com/harmony-one/harmony/proto/identity"
proto_node "github.com/harmony-one/harmony/proto/node" proto_node "github.com/harmony-one/harmony/proto/node"
@ -118,75 +115,42 @@ func (node *Node) StreamHandler(s p2p.Stream) {
blockMsgType := proto_node.BlockMessageType(msgPayload[0]) blockMsgType := proto_node.BlockMessageType(msgPayload[0])
switch blockMsgType { switch blockMsgType {
case proto_node.Sync: case proto_node.Sync:
decoder := gob.NewDecoder(bytes.NewReader(msgPayload[1:])) // skip the Sync messge type var blocks []*types.Block
blocks := new([]*blockchain.Block) err := rlp.DecodeBytes(msgPayload[1:], &blocks) // skip the Sync messge type
decoder.Decode(blocks) if err != nil {
if node.Client != nil && node.Client.UpdateBlocks != nil && blocks != nil { node.log.Info("NET: received message: Node/Block", "error", err)
node.Client.UpdateBlocks(*blocks) } else {
if node.Client != nil && node.Client.UpdateBlocks != nil && blocks != nil {
node.Client.UpdateBlocks(blocks)
}
} }
} }
case proto_node.Client:
node.log.Info("NET: received message: Node/Client")
clientMsgType := proto_node.ClientMessageType(msgPayload[0])
switch clientMsgType {
case proto_node.LookupUtxo:
decoder := gob.NewDecoder(bytes.NewReader(msgPayload[1:])) // skip the LookupUtxo messge type
fetchUtxoMessage := new(proto_node.FetchUtxoMessage)
decoder.Decode(fetchUtxoMessage)
utxoMap := node.UtxoPool.GetUtxoMapByAddresses(fetchUtxoMessage.Addresses)
node.SendMessage(fetchUtxoMessage.Sender, client.ConstructFetchUtxoResponseMessage(&utxoMap, node.UtxoPool.ShardID))
}
case proto_node.Control: case proto_node.Control:
node.log.Info("NET: received message: Node/Control") node.log.Info("NET: received message: Node/Control")
controlType := msgPayload[0] controlType := msgPayload[0]
if proto_node.ControlMessageType(controlType) == proto_node.STOP { if proto_node.ControlMessageType(controlType) == proto_node.STOP {
if node.Chain == nil { node.log.Debug("Stopping Node", "node", node, "numBlocks", node.blockchain.CurrentBlock().NumberU64(), "numTxsProcessed", node.countNumTransactionsInBlockchain())
node.log.Debug("Stopping Node", "node", node, "numBlocks", len(node.blockchain.Blocks), "numTxsProcessed", node.countNumTransactionsInBlockchain())
var avgBlockSizeInBytes common.StorageSize
sizeInBytes := node.UtxoPool.GetSizeInByteOfUtxoMap() txCount := 0
node.log.Debug("UtxoPool Report", "numEntries", len(node.UtxoPool.UtxoMap), "sizeInBytes", sizeInBytes) blockCount := 0
avgTxSize := 0
avgBlockSizeInBytes := 0
txCount := 0 for block := node.blockchain.CurrentBlock(); block != nil; block = node.blockchain.GetBlockByHash(block.Header().ParentHash) {
blockCount := 0 avgBlockSizeInBytes += block.Size()
totalTxCount := 0 txCount += len(block.Transactions())
totalBlockCount := 0 bytes, _ := rlp.EncodeToBytes(block.Transactions())
avgTxSize := 0 avgTxSize += len(bytes)
blockCount++
for _, block := range node.blockchain.Blocks { }
if block.IsStateBlock() {
totalTxCount += int(block.State.NumTransactions)
totalBlockCount += int(block.State.NumBlocks)
} else {
byteBuffer := bytes.NewBuffer([]byte{})
encoder := gob.NewEncoder(byteBuffer)
encoder.Encode(block)
avgBlockSizeInBytes += len(byteBuffer.Bytes())
txCount += len(block.Transactions)
blockCount++
totalTxCount += len(block.TransactionIds)
totalBlockCount++
byteBuffer = bytes.NewBuffer([]byte{})
encoder = gob.NewEncoder(byteBuffer)
encoder.Encode(block.Transactions)
avgTxSize += len(byteBuffer.Bytes())
}
}
if blockCount != 0 {
avgBlockSizeInBytes = avgBlockSizeInBytes / blockCount
avgTxSize = avgTxSize / txCount
}
node.log.Debug("Blockchain Report", "totalNumBlocks", totalBlockCount, "avgBlockSizeInCurrentEpoch", avgBlockSizeInBytes, "totalNumTxs", totalTxCount, "avgTxSzieInCurrentEpoch", avgTxSize) if blockCount != 0 {
} else { avgBlockSizeInBytes = avgBlockSizeInBytes / common.StorageSize(blockCount)
node.log.Debug("Stopping Node (Account Model)", "node", node, "CurBlockNum", node.Chain.CurrentHeader().Number, "numTxsProcessed", node.countNumTransactionsInBlockchainAccount()) avgTxSize = avgTxSize / txCount
} }
node.log.Debug("Blockchain Report", "totalNumBlocks", blockCount, "avgBlockSizeInCurrentEpoch", avgBlockSizeInBytes, "totalNumTxs", txCount, "avgTxSzieInCurrentEpoch", avgTxSize)
os.Exit(0) os.Exit(0)
} }
case proto_node.PING: case proto_node.PING:
@ -194,15 +158,6 @@ func (node *Node) StreamHandler(s p2p.Stream) {
case proto_node.PONG: case proto_node.PONG:
node.pongMessageHandler(msgPayload) node.pongMessageHandler(msgPayload)
} }
case proto.Client:
actionType := client.MessageType(msgType)
node.log.Info("NET: received message: Client/Transaction")
switch actionType {
case client.Transaction:
if node.Client != nil {
node.Client.TransactionMessageHandler(msgPayload)
}
}
default: default:
node.log.Error("Unknown", "MsgCategory", msgCategory) node.log.Error("Unknown", "MsgCategory", msgCategory)
} }
@ -218,22 +173,12 @@ func (node *Node) transactionMessageHandler(msgPayload []byte) {
switch txMessageType { switch txMessageType {
case proto_node.Send: case proto_node.Send:
if node.Chain != nil { txs := types.Transactions{}
txs := types.Transactions{} err := rlp.Decode(bytes.NewReader(msgPayload[1:]), &txs) // skip the Send messge type
err := rlp.Decode(bytes.NewReader(msgPayload[1:]), &txs) // skip the Send messge type if err != nil {
if err != nil { node.log.Error("Failed to deserialize transaction list", "error", err)
node.log.Error("Failed to deserialize transaction list", "error", err)
}
node.addPendingTransactionsAccount(txs)
} else {
txDecoder := gob.NewDecoder(bytes.NewReader(msgPayload[1:])) // skip the Send messge type
txList := new([]*blockchain.Transaction)
err := txDecoder.Decode(&txList)
if err != nil {
node.log.Error("Failed to deserialize transaction list", "error", err)
}
node.addPendingTransactions(*txList)
} }
node.addPendingTransactions(txs)
case proto_node.Request: case proto_node.Request:
reader := bytes.NewBuffer(msgPayload[1:]) reader := bytes.NewBuffer(msgPayload[1:])
@ -250,87 +195,18 @@ func (node *Node) transactionMessageHandler(msgPayload []byte) {
txIDs[txID] = true txIDs[txID] = true
} }
var txToReturn []*blockchain.Transaction var txToReturn []*types.Transaction
for _, tx := range node.pendingTransactions { for _, tx := range node.pendingTransactions {
if txIDs[tx.ID] { if txIDs[tx.Hash()] {
txToReturn = append(txToReturn, tx) txToReturn = append(txToReturn, tx)
} }
} }
// TODO: return the transaction list to requester
case proto_node.Unlock:
txAndProofDecoder := gob.NewDecoder(bytes.NewReader(msgPayload[1:])) // skip the Unlock messge type
txAndProofs := new([]*blockchain.Transaction)
err := txAndProofDecoder.Decode(&txAndProofs)
if err != nil {
node.log.Error("Failed deserializing transaction and proofs list", "node", node)
}
node.log.Debug("RECEIVED Unlock MESSAGE", "num", len(*txAndProofs))
node.addPendingTransactions(*txAndProofs)
} }
} }
// WaitForConsensusReady ... // WaitForConsensusReady listen for the readiness signal from consensus and generate new block for consensus.
func (node *Node) WaitForConsensusReady(readySignal chan struct{}) { func (node *Node) WaitForConsensusReady(readySignal chan struct{}) {
node.log.Debug("Waiting for Consensus ready", "node", node) node.log.Debug("Waiting for Consensus ready", "node", node)
var newBlock *blockchain.Block
timeoutCount := 0
for { // keep waiting for Consensus ready
retry := false
// TODO(minhdoan, rj): Refactor by sending signal in channel instead of waiting for 10 seconds.
select {
case <-readySignal:
time.Sleep(100 * time.Millisecond) // Delay a bit so validator is catched up.
case <-time.After(200 * time.Second):
retry = true
node.Consensus.ResetState()
timeoutCount++
node.log.Debug("Consensus timeout, retry!", "count", timeoutCount, "node", node)
}
//node.log.Debug("Adding new block", "currentChainSize", len(node.blockchain.Blocks), "numTxs", len(node.blockchain.GetLatestBlock().Transactions), "PrevHash", node.blockchain.GetLatestBlock().PrevBlockHash, "Hash", node.blockchain.GetLatestBlock().Hash)
if !retry {
if len(node.blockchain.Blocks) > NumBlocksBeforeStateBlock {
// Generate state block and run consensus on it
newBlock = node.blockchain.CreateStateBlock(node.UtxoPool)
} else {
// Normal tx block consensus
for {
// Once we have pending transactions we will try creating a new block
if len(node.pendingTransactions) >= MaxNumberOfTransactionsPerBlock {
node.log.Debug("Start selecting transactions")
selectedTxs, crossShardTxAndProofs := node.getTransactionsForNewBlock(MaxNumberOfTransactionsPerBlock)
if len(selectedTxs) < MinNumberOfTransactionsPerBlock {
node.log.Debug("No valid transactions exist", "pendingTx", len(node.pendingTransactions))
} else {
node.log.Debug("Creating new block", "numAllTxs", len(selectedTxs), "numCrossTxs", len(crossShardTxAndProofs), "pendingTxs", len(node.pendingTransactions), "currentChainSize", len(node.blockchain.Blocks))
node.transactionInConsensus = selectedTxs
node.CrossTxsInConsensus = crossShardTxAndProofs
newBlock = blockchain.NewBlock(selectedTxs, node.blockchain.GetLatestBlock().Hash, node.Consensus.ShardID, false)
break
}
}
// If not enough transactions to run Consensus,
// periodically check whether we have enough transactions to package into block.
time.Sleep(1 * time.Second)
}
}
}
// Send the new block to Consensus so it can be confirmed.
if newBlock != nil {
node.BlockChannel <- *newBlock
}
}
}
// WaitForConsensusReadyAccount ...
func (node *Node) WaitForConsensusReadyAccount(readySignal chan struct{}) {
node.log.Debug("Waiting for Consensus ready", "node", node)
time.Sleep(15 * time.Second) time.Sleep(15 * time.Second)
firstTime := true firstTime := true
@ -347,14 +223,15 @@ func (node *Node) WaitForConsensusReadyAccount(readySignal chan struct{}) {
} }
for { for {
node.log.Debug("STARTING BLOCK")
threshold := 1 threshold := 1
if firstTime { if firstTime {
threshold = 2 threshold = 2
firstTime = false firstTime = false
} }
if len(node.pendingTransactionsAccount) >= threshold { if len(node.pendingTransactions) >= threshold {
// Normal tx block consensus // Normal tx block consensus
selectedTxs, _ := node.getTransactionsForNewBlockAccount(MaxNumberOfTransactionsPerBlock) selectedTxs := node.getTransactionsForNewBlock(MaxNumberOfTransactionsPerBlock)
node.Worker.CommitTransactions(selectedTxs) node.Worker.CommitTransactions(selectedTxs)
block, err := node.Worker.Commit() block, err := node.Worker.Commit()
if err != nil { if err != nil {
@ -370,63 +247,23 @@ func (node *Node) WaitForConsensusReadyAccount(readySignal chan struct{}) {
} }
// Send the new block to Consensus so it can be confirmed. // Send the new block to Consensus so it can be confirmed.
if newBlock != nil { if newBlock != nil {
node.BlockChannelAccount <- newBlock node.BlockChannel <- newBlock
}
}
}
// SendBackProofOfAcceptOrReject is called by consensus participants to verify the block they are running consensus on
func (node *Node) SendBackProofOfAcceptOrReject() {
if node.ClientPeer != nil && len(node.CrossTxsToReturn) != 0 {
node.crossTxToReturnMutex.Lock()
proofs := []blockchain.CrossShardTxProof{}
for _, txAndProof := range node.CrossTxsToReturn {
proofs = append(proofs, *txAndProof.Proof)
} }
node.CrossTxsToReturn = nil
node.crossTxToReturnMutex.Unlock()
node.log.Debug("SENDING PROOF TO CLIENT", "proofs", len(proofs))
node.SendMessage(*node.ClientPeer, client.ConstructProofOfAcceptOrRejectMessage(proofs))
} }
} }
// BroadcastNewBlock is called by consensus leader to sync new blocks with other clients/nodes. // BroadcastNewBlock is called by consensus leader to sync new blocks with other clients/nodes.
// NOTE: For now, just send to the client (basically not broadcasting) // NOTE: For now, just send to the client (basically not broadcasting)
func (node *Node) BroadcastNewBlock(newBlock *blockchain.Block) { func (node *Node) BroadcastNewBlock(newBlock *types.Block) {
if node.ClientPeer != nil { if node.ClientPeer != nil {
node.log.Debug("NET: SENDING NEW BLOCK TO CLIENT") node.log.Debug("NET: SENDING NEW BLOCK TO CLIENT", "client", node.ClientPeer)
node.SendMessage(*node.ClientPeer, proto_node.ConstructBlocksSyncMessage([]blockchain.Block{*newBlock})) node.SendMessage(*node.ClientPeer, proto_node.ConstructBlocksSyncMessage([]*types.Block{newBlock}))
} }
} }
// VerifyNewBlock is called by consensus participants to verify the block they are running consensus on // VerifyNewBlock is called by consensus participants to verify the block (account model) they are running consensus on
func (node *Node) VerifyNewBlock(newBlock *blockchain.Block) bool { func (node *Node) VerifyNewBlock(newBlock *types.Block) bool {
// TODO: just a reminder for syncing. we need to check if the new block is fit with the current blockchain. err := node.blockchain.ValidateNewBlock(newBlock, pki.GetAddressFromPublicKey(node.SelfPeer.PubKey))
// The current blockchain can be in the progress of being synced.
var verified bool
if newBlock.AccountBlock != nil {
accountBlock := new(types.Block)
err := rlp.DecodeBytes(newBlock.AccountBlock, accountBlock)
if err != nil {
node.log.Error("Failed decoding the block with RLP")
}
verified = node.VerifyNewBlockAccount(accountBlock)
} else if newBlock.IsStateBlock() {
verified = node.UtxoPool.VerifyStateBlock(newBlock)
} else {
verified = node.UtxoPool.VerifyTransactions(newBlock.Transactions)
}
if verified {
// Change the syncing state.
node.State = NodeDoingConsensus
}
return verified
}
// VerifyNewBlockAccount is called by consensus participants to verify the block (account model) they are running consensus on
func (node *Node) VerifyNewBlockAccount(newBlock *types.Block) bool {
err := node.Chain.ValidateNewBlock(newBlock, pki.GetAddressFromPublicKey(node.SelfPeer.PubKey))
if err != nil { if err != nil {
node.log.Debug("Failed verifying new block", "Error", err, "tx", newBlock.Transactions()[0]) node.log.Debug("Failed verifying new block", "Error", err, "tx", newBlock.Transactions()[0])
return false return false
@ -437,78 +274,22 @@ func (node *Node) VerifyNewBlockAccount(newBlock *types.Block) bool {
// PostConsensusProcessing is called by consensus participants, after consensus is done, to: // PostConsensusProcessing is called by consensus participants, after consensus is done, to:
// 1. add the new block to blockchain // 1. add the new block to blockchain
// 2. [leader] move cross shard tx and proof to the list where they wait to be sent to the client // 2. [leader] move cross shard tx and proof to the list where they wait to be sent to the client
func (node *Node) PostConsensusProcessing(newBlock *blockchain.Block) { func (node *Node) PostConsensusProcessing(newBlock *types.Block) {
//if newBlock.IsStateBlock() {
// // Clear out old tx blocks and put state block as genesis
// if node.db != nil {
// node.log.Info("Deleting old blocks.")
// for i := 1; i <= len(node.blockchain.Blocks); i++ {
// blockchain.Delete(node.db, strconv.Itoa(i))
// }
// }
// node.blockchain.Blocks = []*blockchain.Block{}
//}
if node.Consensus.IsLeader { if node.Consensus.IsLeader {
// Move crossTx-in-consensus into the list to be returned to client
//for _, crossTxAndProof := range node.CrossTxsInConsensus {
// crossTxAndProof.Proof.BlockHash = newBlock.Hash
// // TODO: fill in the signature proofs
//}
//if len(node.CrossTxsInConsensus) != 0 {
// node.addCrossTxsToReturn(node.CrossTxsInConsensus)
// node.CrossTxsInConsensus = []*blockchain.CrossShardTxAndProof{}
//}
//
//node.SendBackProofOfAcceptOrReject()
node.BroadcastNewBlock(newBlock) node.BroadcastNewBlock(newBlock)
} }
node.AddNewBlock(newBlock) node.AddNewBlock(newBlock)
node.UpdateUtxoAndState(newBlock)
} }
// AddNewBlockAccount is usedd to add new block into the blockchain. // AddNewBlock is usedd to add new block into the blockchain.
func (node *Node) AddNewBlockAccount(newBlock *types.Block) { func (node *Node) AddNewBlock(newBlock *types.Block) {
num, err := node.Chain.InsertChain([]*types.Block{newBlock}) num, err := node.blockchain.InsertChain([]*types.Block{newBlock})
if err != nil { if err != nil {
node.log.Debug("Error adding to chain", "numBlocks", num, "Error", err) node.log.Debug("Error adding to chain", "numBlocks", num, "Error", err)
} }
} }
// AddNewBlock is usedd to add new block into the utxo-based blockchain.
func (node *Node) AddNewBlock(newBlock *blockchain.Block) {
// Add it to blockchain
// node.blockchain.Blocks = append(node.blockchain.Blocks, newBlock)
// Store it into leveldb.
if node.db != nil {
node.log.Info("Writing new block into disk.")
newBlock.Write(node.db, strconv.Itoa(len(node.blockchain.Blocks)))
}
// Account model
accountBlock := new(types.Block)
err := rlp.DecodeBytes(newBlock.AccountBlock, accountBlock)
if err != nil {
node.log.Error("Failed decoding the block with RLP")
}
node.AddNewBlockAccount(accountBlock)
}
// UpdateUtxoAndState updates Utxo and state.
func (node *Node) UpdateUtxoAndState(newBlock *blockchain.Block) {
// Update UTXO pool
if newBlock.IsStateBlock() {
newUtxoPool := blockchain.CreateUTXOPoolFromGenesisBlock(newBlock)
node.UtxoPool.UtxoMap = newUtxoPool.UtxoMap
} else {
node.UtxoPool.Update(newBlock.Transactions)
}
// Clear transaction-in-Consensus list
node.transactionInConsensus = []*blockchain.Transaction{}
}
func (node *Node) pingMessageHandler(msgPayload []byte) int { func (node *Node) pingMessageHandler(msgPayload []byte) int {
ping, err := proto_node.GetPingMessage(msgPayload) ping, err := proto_node.GetPingMessage(msgPayload)
if err != nil { if err != nil {

24
node/node_test.go
Unescape View File

@ -30,31 +30,9 @@ func TestNewNewNode(t *testing.T) {
t.Error("Blockchain is not initialized for the node") t.Error("Blockchain is not initialized for the node")
} }
if len(node.blockchain.Blocks) != 1 { if node.blockchain.CurrentBlock() == nil {
t.Error("Genesis block is not initialized for the node") t.Error("Genesis block is not initialized for the node")
} }
if len(node.blockchain.Blocks[0].Transactions) != 1 {
t.Error("Coinbase TX is not initialized for the node")
}
if node.UtxoPool == nil {
t.Error("Utxo pool is not initialized for the node")
}
}
func TestCountNumTransactionsInBlockchain(t *testing.T) {
_, pubKey := utils.GenKey("1", "2")
leader := p2p.Peer{IP: "1", Port: "2", PubKey: pubKey}
validator := p2p.Peer{IP: "3", Port: "5"}
host := p2pimpl.NewHost(leader)
consensus := consensus.New(host, "0", []p2p.Peer{leader, validator}, leader)
node := New(host, consensus, nil)
node.AddTestingAddresses(1000)
if node.countNumTransactionsInBlockchain() != 1001 {
t.Error("Count of transactions in the blockchain is incorrect")
}
} }
func TestGetSyncingPeers(t *testing.T) { func TestGetSyncingPeers(t *testing.T) {

40
node/node_utils.go
Unescape View File

@ -1,20 +1,34 @@
package node package node
import ( import (
"github.com/harmony-one/harmony/blockchain" "crypto/ecdsa"
"github.com/harmony-one/harmony/crypto/pki" "github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/params"
"github.com/harmony-one/harmony/core"
"math/big"
"math/rand"
"strings"
) )
// AddTestingAddresses creates in genesis block numAddress transactions which assign k token to each address in [0 - numAddress) // CreateGenesisAllocWithTestingAddresses create the genesis block allocation that contains deterministically
// k = DefaultCoinbaseValue * DefaultNumUtxos // generated testing addressess with tokens.
// Assume we have S shards, then each account possesses k*S tokens // TODO: Consider to remove it later when moving to production.a
// This is used by client code. func (node *Node) CreateGenesisAllocWithTestingAddresses(numAddress int) core.GenesisAlloc {
// TODO: Consider to remove it later when moving to production. rand.Seed(0)
func (node *Node) AddTestingAddresses(numAddress int) { len := 1000000
txs := make([]*blockchain.Transaction, numAddress) bytes := make([]byte, len)
for i := range txs { for i := 0; i < len; i++ {
txs[i] = blockchain.NewCoinbaseTX(pki.GetAddressFromInt(i+1), "", node.Consensus.ShardID) bytes[i] = byte(rand.Intn(100))
} }
node.blockchain.Blocks[0].Transactions = append(node.blockchain.Blocks[0].Transactions, txs...) reader := strings.NewReader(string(bytes))
node.UtxoPool.Update(txs) genesisAloc := make(core.GenesisAlloc)
for i := 0; i < numAddress; i++ {
testBankKey, _ := ecdsa.GenerateKey(crypto.S256(), reader)
testBankAddress := crypto.PubkeyToAddress(testBankKey.PublicKey)
testBankFunds := big.NewInt(1000)
testBankFunds = testBankFunds.Mul(testBankFunds, big.NewInt(params.Ether))
genesisAloc[testBankAddress] = core.GenesisAccount{Balance: testBankFunds}
node.TestBankKeys = append(node.TestBankKeys, testBankKey)
}
return genesisAloc
} }

55
proto/client/client.go
Unescape View File

@ -1,55 +0,0 @@
package client
import (
"bytes"
"encoding/gob"
"github.com/harmony-one/harmony/blockchain"
"github.com/harmony-one/harmony/proto"
)
// MessageType is the specific types of message under Client category
type MessageType byte
// Message type supported by client
const (
Transaction MessageType = iota
// TODO: add more types
)
// TransactionMessageType defines the types of messages used for Client/Transaction
type TransactionMessageType int
// The proof of accept or reject returned by the leader to the client tnat issued cross shard transactions
const (
ProofOfLock TransactionMessageType = iota
UtxoResponse
)
// FetchUtxoResponseMessage is the data structure of UTXO map
type FetchUtxoResponseMessage struct {
UtxoMap blockchain.UtxoMap
ShardID uint32
}
// ConstructProofOfAcceptOrRejectMessage constructs the proof of accept or reject message that will be sent to client
func ConstructProofOfAcceptOrRejectMessage(proofs []blockchain.CrossShardTxProof) []byte {
byteBuffer := bytes.NewBuffer([]byte{byte(proto.Client)})
byteBuffer.WriteByte(byte(Transaction))
byteBuffer.WriteByte(byte(ProofOfLock))
encoder := gob.NewEncoder(byteBuffer)
encoder.Encode(proofs)
return byteBuffer.Bytes()
}
// ConstructFetchUtxoResponseMessage constructs the response message to fetch utxo message
func ConstructFetchUtxoResponseMessage(utxoMap *blockchain.UtxoMap, shardID uint32) []byte {
byteBuffer := bytes.NewBuffer([]byte{byte(proto.Client)})
byteBuffer.WriteByte(byte(Transaction))
byteBuffer.WriteByte(byte(UtxoResponse))
encoder := gob.NewEncoder(byteBuffer)
encoder.Encode(FetchUtxoResponseMessage{*utxoMap, shardID})
return byteBuffer.Bytes()
}

4
proto/consensus/consensus.go
Unescape View File

@ -94,7 +94,6 @@ const (
FinalCommit FinalCommit
FinalChallenge FinalChallenge
FinalResponse FinalResponse
StartConsensus
) )
// Returns string name for the MessageType enum // Returns string name for the MessageType enum
@ -108,10 +107,9 @@ func (msgType MessageType) String() string {
"FinalCommit", "FinalCommit",
"FinalChallenge", "FinalChallenge",
"FinalResponse", "FinalResponse",
"StartConsensus",
} }
if msgType < Announce || msgType > StartConsensus { if msgType < Announce || msgType > FinalResponse {
return "Unknown" return "Unknown"
} }
return names[msgType] return names[msgType]

57
proto/node/node.go
Unescape View File

@ -8,7 +8,6 @@ import (
"github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rlp"
"github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/core/types"
"github.com/harmony-one/harmony/blockchain"
"github.com/harmony-one/harmony/p2p" "github.com/harmony-one/harmony/p2p"
"github.com/harmony-one/harmony/proto" "github.com/harmony-one/harmony/proto"
) )
@ -66,14 +65,6 @@ const (
Sync BlockMessageType = iota Sync BlockMessageType = iota
) )
// ClientMessageType defines the type of messages used for Node/Block
type ClientMessageType int
// Constant of the client message subtype
const (
LookupUtxo ClientMessageType = iota
)
// ControlMessageType is the type of messages used for Node/Control // ControlMessageType is the type of messages used for Node/Control
type ControlMessageType int type ControlMessageType int
@ -110,48 +101,6 @@ func DeserializeBlockchainSyncMessage(d []byte) (*BlockchainSyncMessage, error)
return &blockchainSyncMessage, err return &blockchainSyncMessage, err
} }
// ConstructUnlockToCommitOrAbortMessage constructs the unlock to commit or abort message that will be sent to leaders.
// This is for client.
func ConstructUnlockToCommitOrAbortMessage(txsAndProofs []*blockchain.Transaction) []byte {
byteBuffer := bytes.NewBuffer([]byte{byte(proto.Node)})
byteBuffer.WriteByte(byte(Transaction))
byteBuffer.WriteByte(byte(Unlock))
encoder := gob.NewEncoder(byteBuffer)
encoder.Encode(txsAndProofs)
return byteBuffer.Bytes()
}
// ConstructFetchUtxoMessage constructs the fetch utxo message that will be sent to Harmony network.
// this is for client.
func ConstructFetchUtxoMessage(sender p2p.Peer, addresses [][20]byte) []byte {
byteBuffer := bytes.NewBuffer([]byte{byte(proto.Node)})
byteBuffer.WriteByte(byte(Client))
byteBuffer.WriteByte(byte(LookupUtxo))
encoder := gob.NewEncoder(byteBuffer)
encoder.Encode(FetchUtxoMessage{Addresses: addresses, Sender: sender})
return byteBuffer.Bytes()
}
// ConstructTransactionListMessage constructs serialized transactions
func ConstructTransactionListMessage(transactions []*blockchain.Transaction) []byte {
byteBuffer := bytes.NewBuffer([]byte{byte(proto.Node)})
byteBuffer.WriteByte(byte(Transaction))
byteBuffer.WriteByte(byte(Send))
encoder := gob.NewEncoder(byteBuffer)
// Copy over the tx data
txs := make([]blockchain.Transaction, len(transactions))
for i := range txs {
txs[i] = *transactions[i]
}
err := encoder.Encode(txs)
if err != nil {
return []byte{} // TODO(RJ): better handle of the error
}
return byteBuffer.Bytes()
}
// ConstructTransactionListMessageAccount constructs serialized transactions in account model // ConstructTransactionListMessageAccount constructs serialized transactions in account model
func ConstructTransactionListMessageAccount(transactions types.Transactions) []byte { func ConstructTransactionListMessageAccount(transactions types.Transactions) []byte {
byteBuffer := bytes.NewBuffer([]byte{byte(proto.Node)}) byteBuffer := bytes.NewBuffer([]byte{byte(proto.Node)})
@ -187,12 +136,12 @@ func ConstructStopMessage() []byte {
} }
// ConstructBlocksSyncMessage constructs blocks sync message to send blocks to other nodes // ConstructBlocksSyncMessage constructs blocks sync message to send blocks to other nodes
func ConstructBlocksSyncMessage(blocks []blockchain.Block) []byte { func ConstructBlocksSyncMessage(blocks []*types.Block) []byte {
byteBuffer := bytes.NewBuffer([]byte{byte(proto.Node)}) byteBuffer := bytes.NewBuffer([]byte{byte(proto.Node)})
byteBuffer.WriteByte(byte(Block)) byteBuffer.WriteByte(byte(Block))
byteBuffer.WriteByte(byte(Sync)) byteBuffer.WriteByte(byte(Sync))
encoder := gob.NewEncoder(byteBuffer)
encoder.Encode(blocks) blocksData, _ := rlp.EncodeToBytes(blocks)
byteBuffer.Write(blocksData)
return byteBuffer.Bytes() return byteBuffer.Bytes()
} }

15
services/explorer/storage.go
Unescape View File

@ -82,18 +82,21 @@ func (storage *Storage) GetDB() *db.LDBDatabase {
} }
// Dump extracts information from block and index them into lvdb for explorer. // Dump extracts information from block and index them into lvdb for explorer.
func (storage *Storage) Dump(accountBlock []byte, height uint32) { func (storage *Storage) Dump(block *types.Block, height uint32) {
fmt.Println("Dumping block ", height) Log.Info("Dumping block ", "block height", height)
if accountBlock == nil { if block == nil {
return return
} }
// Update block height. // Update block height.
storage.db.Put([]byte(BlockHeightKey), []byte(strconv.Itoa(int(height)))) storage.db.Put([]byte(BlockHeightKey), []byte(strconv.Itoa(int(height))))
// Store block. // Store block.
block := new(types.Block) blockData, err := rlp.EncodeToBytes(block)
rlp.DecodeBytes(accountBlock, block) if err == nil {
storage.db.Put([]byte(GetBlockKey(int(height))), accountBlock) storage.db.Put([]byte(GetBlockKey(int(height))), blockData)
} else {
Log.Debug("Failed to serialize block ", "error", err)
}
// Store block info. // Store block info.
blockInfo := BlockInfo{ blockInfo := BlockInfo{

111
services/syncing/downloader/server_test.go
Unescape View File

@ -1,111 +0,0 @@
package downloader_test
import (
"reflect"
"testing"
bc "github.com/harmony-one/harmony/blockchain"
"github.com/harmony-one/harmony/crypto/pki"
"github.com/harmony-one/harmony/services/syncing/downloader"
pb "github.com/harmony-one/harmony/services/syncing/downloader/proto"
)
const (
serverPort = "9997"
serverIP = "127.0.0.1"
clientPort = "9999"
)
var (
PriIntOne = 111
PriIntTwo = 222
TestAddressOne = pki.GetAddressFromInt(PriIntOne)
TestAddressTwo = pki.GetAddressFromInt(PriIntTwo)
ShardID = uint32(0)
)
type FakeNode struct {
bc *bc.Blockchain
}
// GetBlockHashes used for state download.
func (node *FakeNode) GetBlockHashes() [][]byte {
res := [][]byte{}
for _, block := range node.bc.Blocks {
res = append(res, block.Hash[:])
}
return res
}
// GetBlocks used for state download.
func (node *FakeNode) GetBlocks() [][]byte {
res := [][]byte{}
for _, block := range node.bc.Blocks {
res = append(res, block.Serialize())
}
return res
}
// SetBlockchain is used for testing
func (node *FakeNode) Init() {
addresses := [][20]byte{TestAddressOne, TestAddressTwo}
node.bc = bc.CreateBlockchainWithMoreBlocks(addresses, ShardID)
}
// CalculateResponse is the implementation for DownloadInterface.
func (node *FakeNode) CalculateResponse(request *pb.DownloaderRequest) (*pb.DownloaderResponse, error) {
response := &pb.DownloaderResponse{}
if request.Type == pb.DownloaderRequest_HEADER {
for _, block := range node.bc.Blocks {
response.Payload = append(response.Payload, block.Hash[:])
}
} else {
for i := range request.Hashes {
block := node.bc.FindBlock(request.Hashes[i])
response.Payload = append(response.Payload, block.Serialize())
}
}
return response, nil
}
// TestGetBlockHashes tests GetBlockHashes function.
func TestGetBlockHashes(t *testing.T) {
fakeNode := &FakeNode{}
fakeNode.Init()
s := downloader.NewServer(fakeNode)
grcpServer, err := s.Start(serverIP, serverPort)
if err != nil {
t.Error(err)
}
defer grcpServer.Stop()
client := downloader.ClientSetup(serverIP, serverPort)
defer client.Close()
response := client.GetBlockHashes()
if !reflect.DeepEqual(response.Payload, fakeNode.GetBlockHashes()) {
t.Error("not equal")
}
}
// TestGetBlocks tests GetBlocks function.
func TestGetBlocks(t *testing.T) {
fakeNode := &FakeNode{}
fakeNode.Init()
s := downloader.NewServer(fakeNode)
grcpServer, err := s.Start(serverIP, serverPort)
if err != nil {
t.Error(err)
}
defer grcpServer.Stop()
client := downloader.ClientSetup(serverIP, serverPort)
defer client.Close()
response := client.GetBlockHashes()
if !reflect.DeepEqual(response.Payload, fakeNode.GetBlockHashes()) {
t.Error("not equal")
}
response = client.GetBlocks([][]byte{response.Payload[0], response.Payload[1]})
if !reflect.DeepEqual(response.Payload, fakeNode.GetBlocks()) {
t.Error("not equal")
}
}

4
services/syncing/interface.go
Unescape View File

@ -1,7 +1,7 @@
package syncing package syncing
import ( import (
"github.com/harmony-one/harmony/blockchain" "github.com/harmony-one/harmony/core"
"github.com/harmony-one/harmony/p2p" "github.com/harmony-one/harmony/p2p"
) )
@ -9,5 +9,5 @@ import (
type StateSyncInterface interface { type StateSyncInterface interface {
// Syncing blockchain from other peers. // Syncing blockchain from other peers.
// The returned channel is the signal of syncing finish. // The returned channel is the signal of syncing finish.
ProcessStateSyncFromPeers(peers []p2p.Peer, bc *blockchain.Blockchain) (chan struct{}, error) ProcessStateSyncFromPeers(peers []p2p.Peer, bc *core.BlockChain) (chan struct{}, error)
} }

34
services/syncing/syncing.go
Unescape View File

@ -2,13 +2,14 @@ package syncing
import ( import (
"bytes" "bytes"
"github.com/harmony-one/harmony/core"
"github.com/simple-rules/harmony-benchmark/blockchain"
"reflect" "reflect"
"sort" "sort"
"sync" "sync"
"time" "time"
"github.com/Workiva/go-datastructures/queue" "github.com/Workiva/go-datastructures/queue"
"github.com/harmony-one/harmony/blockchain"
"github.com/harmony-one/harmony/log" "github.com/harmony-one/harmony/log"
"github.com/harmony-one/harmony/p2p" "github.com/harmony-one/harmony/p2p"
"github.com/harmony-one/harmony/services/syncing/downloader" "github.com/harmony-one/harmony/services/syncing/downloader"
@ -105,7 +106,7 @@ func (peerConfig *SyncPeerConfig) GetBlocks(hashes [][]byte) ([][]byte, error) {
} }
// ProcessStateSyncFromPeers used to do state sync. // ProcessStateSyncFromPeers used to do state sync.
func (ss *StateSync) ProcessStateSyncFromPeers(peers []p2p.Peer, bc *blockchain.Blockchain) (chan struct{}, error) { func (ss *StateSync) ProcessStateSyncFromPeers(peers []p2p.Peer, bc *core.BlockChain) (chan struct{}, error) {
// TODO: Validate peers. // TODO: Validate peers.
done := make(chan struct{}) done := make(chan struct{})
go func() { go func() {
@ -255,19 +256,20 @@ func (ss *StateSync) GetConsensusHashes() bool {
} }
// getConsensusHashes gets all hashes needed to download. // getConsensusHashes gets all hashes needed to download.
func (ss *StateSync) generateStateSyncTaskQueue(bc *blockchain.Blockchain) { func (ss *StateSync) generateStateSyncTaskQueue(bc *core.BlockChain) {
ss.stateSyncTaskQueue = queue.New(0) ss.stateSyncTaskQueue = queue.New(0)
for _, configPeer := range ss.syncConfig.peers { for _, configPeer := range ss.syncConfig.peers {
if configPeer.client != nil { if configPeer.client != nil {
ss.blockHeight = len(configPeer.blockHashes) ss.blockHeight = len(configPeer.blockHashes)
bc.Blocks = append(bc.Blocks, make([]*blockchain.Block, ss.blockHeight-len(bc.Blocks))...) // TODO (minh) rework the syncing for account model.
for id, blockHash := range configPeer.blockHashes { //bc.Blocks = append(bc.Blocks, make([]*blockchain.Block, ss.blockHeight-len(bc.Blocks))...)
if bc.Blocks[id] == nil || !reflect.DeepEqual(bc.Blocks[id].Hash[:], blockHash) { //for id, blockHash := range configPeer.blockHashes {
ss.stateSyncTaskQueue.Put(SyncBlockTask{index: id, blockHash: blockHash}) // if bc.Blocks[id] == nil || !reflect.DeepEqual(bc.Blocks[id].Hash[:], blockHash) {
// TODO(minhdoan): Check error // ss.stateSyncTaskQueue.Put(SyncBlockTask{index: id, blockHash: blockHash})
} // // TODO(minhdoan): Check error
} // }
//}
break break
} }
} }
@ -275,7 +277,7 @@ func (ss *StateSync) generateStateSyncTaskQueue(bc *blockchain.Blockchain) {
} }
// downloadBlocks downloads blocks from state sync task queue. // downloadBlocks downloads blocks from state sync task queue.
func (ss *StateSync) downloadBlocks(bc *blockchain.Blockchain) { func (ss *StateSync) downloadBlocks(bc *core.BlockChain) {
// Initialize blockchain // Initialize blockchain
var wg sync.WaitGroup var wg sync.WaitGroup
wg.Add(ss.activePeerNumber) wg.Add(ss.activePeerNumber)
@ -283,7 +285,7 @@ func (ss *StateSync) downloadBlocks(bc *blockchain.Blockchain) {
if ss.syncConfig.peers[i].client == nil { if ss.syncConfig.peers[i].client == nil {
continue continue
} }
go func(peerConfig *SyncPeerConfig, stateSyncTaskQueue *queue.Queue, bc *blockchain.Blockchain) { go func(peerConfig *SyncPeerConfig, stateSyncTaskQueue *queue.Queue, bc *core.BlockChain) {
defer wg.Done() defer wg.Done()
for !stateSyncTaskQueue.Empty() { for !stateSyncTaskQueue.Empty() {
task, err := stateSyncTaskQueue.Poll(1, time.Millisecond) task, err := stateSyncTaskQueue.Poll(1, time.Millisecond)
@ -292,11 +294,13 @@ func (ss *StateSync) downloadBlocks(bc *blockchain.Blockchain) {
} }
syncTask := task[0].(SyncBlockTask) syncTask := task[0].(SyncBlockTask)
for { for {
id := syncTask.index //id := syncTask.index
payload, err := peerConfig.GetBlocks([][]byte{syncTask.blockHash}) payload, err := peerConfig.GetBlocks([][]byte{syncTask.blockHash})
if err == nil { if err == nil {
// As of now, only send and ask for one block. // As of now, only send and ask for one block.
bc.Blocks[id], err = blockchain.DeserializeBlock(payload[0]) // TODO (minh) rework the syncing for account model.
//bc.Blocks[id], err = blockchain.DeserializeBlock(payload[0])
_, err = blockchain.DeserializeBlock(payload[0])
if err == nil { if err == nil {
break break
} }
@ -310,7 +314,7 @@ func (ss *StateSync) downloadBlocks(bc *blockchain.Blockchain) {
} }
// StartStateSync starts state sync. // StartStateSync starts state sync.
func (ss *StateSync) StartStateSync(peers []p2p.Peer, bc *blockchain.Blockchain) bool { func (ss *StateSync) StartStateSync(peers []p2p.Peer, bc *core.BlockChain) bool {
// Creates sync config. // Creates sync config.
ss.CreateSyncConfig(peers) ss.CreateSyncConfig(peers)
// Makes connections to peers. // Makes connections to peers.

191
services/syncing/syncing_test.go
Unescape View File

@ -1,191 +0,0 @@
package syncing_test
import (
"reflect"
"testing"
bc "github.com/harmony-one/harmony/blockchain"
"github.com/harmony-one/harmony/crypto/pki"
"github.com/harmony-one/harmony/p2p"
"github.com/harmony-one/harmony/services/syncing"
"github.com/harmony-one/harmony/services/syncing/downloader"
pb "github.com/harmony-one/harmony/services/syncing/downloader/proto"
"github.com/stretchr/testify/assert"
"google.golang.org/grpc"
)
const (
serverPort1 = "9996"
serverPort2 = "9997"
serverPort3 = "9998"
serverIP = "127.0.0.1"
clientPort = "9999"
)
var (
PriIntOne = 111
PriIntTwo = 222
PriIntThree = 222
TestAddressOne = pki.GetAddressFromInt(PriIntOne)
TestAddressTwo = pki.GetAddressFromInt(PriIntTwo)
TestAddressThree = pki.GetAddressFromInt(PriIntThree)
ShardID = uint32(0)
ServerPorts = []string{serverPort1, serverPort2, serverPort3}
)
type FakeNode struct {
bc *bc.Blockchain
server *downloader.Server
ip string
port string
grpcServer *grpc.Server
doneFirstTime bool
}
// GetBlockHashes used for state download.
func (node *FakeNode) GetBlockHashes() [][]byte {
res := [][]byte{}
for _, block := range node.bc.Blocks {
res = append(res, block.Hash[:])
}
return res
}
// GetBlocks used for state download.
func (node *FakeNode) GetBlocks() [][]byte {
res := [][]byte{}
for _, block := range node.bc.Blocks {
res = append(res, block.Serialize())
}
return res
}
// SetBlockchain is used for testing
func (node *FakeNode) Init(ip, port string) {
addresses := [][20]byte{TestAddressOne, TestAddressTwo}
node.bc = bc.CreateBlockchainWithMoreBlocks(addresses, ShardID)
node.ip = ip
node.port = port
node.server = downloader.NewServer(node)
}
// SetBlockchain is used for testing
func (node *FakeNode) Init2(ip, port string) {
addresses := [][20]byte{TestAddressOne}
node.bc = bc.CreateBlockchainWithMoreBlocks(addresses, ShardID)
node.ip = ip
node.port = port
node.server = downloader.NewServer(node)
}
// Start ...
func (node *FakeNode) Start() error {
var err error
node.grpcServer, err = node.server.Start(node.ip, node.port)
return err
}
func (node *FakeNode) addOneMoreBlock() {
addresses := [][20]byte{TestAddressThree}
node.bc.Blocks = append(node.bc.Blocks, bc.CreateMoreBlocks(addresses, ShardID)...)
}
func (node *FakeNode) CalculateResponse(request *pb.DownloaderRequest) (*pb.DownloaderResponse, error) {
response := &pb.DownloaderResponse{}
if request.Type == pb.DownloaderRequest_HEADER {
for _, block := range node.bc.Blocks {
response.Payload = append(response.Payload, block.Hash[:])
}
if !node.doneFirstTime {
node.addOneMoreBlock()
}
node.doneFirstTime = true
} else {
for i := range request.Hashes {
block := node.bc.FindBlock(request.Hashes[i])
response.Payload = append(response.Payload, block.Serialize())
}
}
return response, nil
}
func TestCompareSyncPeerConfigByBlockHashes(t *testing.T) {
a := syncing.CreateTestSyncPeerConfig(nil, [][]byte{{1, 2, 3, 4, 5, 6}, {1, 2, 3, 4, 5, 6}})
b := syncing.CreateTestSyncPeerConfig(nil, [][]byte{{1, 2, 3, 4, 5, 6}, {1, 2, 3, 4, 5, 6}})
assert.Equal(t, syncing.CompareSyncPeerConfigByblockHashes(a, b), 0, "they should be equal")
c := syncing.CreateTestSyncPeerConfig(nil, [][]byte{{1, 2, 3, 4, 5, 7}, {1, 2, 3, 4, 5, 6}})
assert.Equal(t, syncing.CompareSyncPeerConfigByblockHashes(a, c), -1, "a should be less than c")
d := syncing.CreateTestSyncPeerConfig(nil, [][]byte{{1, 2, 3, 4, 5, 4}, {1, 2, 3, 4, 5, 6}})
assert.Equal(t, syncing.CompareSyncPeerConfigByblockHashes(a, d), 1, "a should be greater than c")
}
func TestSyncing(t *testing.T) {
fakeNodes := []*FakeNode{&FakeNode{}, &FakeNode{}, &FakeNode{}}
for i := range fakeNodes {
fakeNodes[i].Init(serverIP, ServerPorts[i])
if err := fakeNodes[i].Start(); err != nil {
t.Error(err)
}
}
defer func() {
for _, fakeNode := range fakeNodes {
fakeNode.grpcServer.Stop()
}
}()
stateSync := &syncing.StateSync{}
bc := &bc.Blockchain{}
peers := make([]p2p.Peer, len(fakeNodes))
for i := range peers {
peers[i].IP = fakeNodes[i].ip
peers[i].Port = fakeNodes[i].port
}
stateSync.StartStateSync(peers, bc)
for i := range bc.Blocks {
if !reflect.DeepEqual(bc.Blocks[i], fakeNodes[0].bc.Blocks[i]) {
t.Error("not equal")
}
}
}
func TestSyncingIncludingBadNode(t *testing.T) {
fakeNodes := []*FakeNode{&FakeNode{}, &FakeNode{}, &FakeNode{}}
for i := range fakeNodes {
if i == 2 {
// Bad node.
fakeNodes[i].Init2(serverIP, ServerPorts[i])
} else {
// Good node.
fakeNodes[i].Init(serverIP, ServerPorts[i])
}
if err := fakeNodes[i].Start(); err != nil {
t.Error(err)
}
}
defer func() {
for _, fakeNode := range fakeNodes {
fakeNode.grpcServer.Stop()
}
}()
stateSync := &syncing.StateSync{}
bc := &bc.Blockchain{}
peers := make([]p2p.Peer, len(fakeNodes))
for i := range peers {
peers[i].IP = fakeNodes[i].ip
peers[i].Port = fakeNodes[i].port
}
assert.True(t, stateSync.StartStateSync(peers, bc), "should return true")
for i := range bc.Blocks {
if !reflect.DeepEqual(bc.Blocks[i], fakeNodes[0].bc.Blocks[i]) {
t.Error("not equal")
}
}
}
Write Preview
Loading…
Cancel
Save