Merge branch 'master' of https://github.com/simple-rules/harmony-benchmark

blockchain/block.go

@@ -28,6 +28,10 @@ type Block struct {
 	Signature [66]byte // Schnorr collective signature
 }
 
+func (b *Block) IsStateBlock() bool {
+	return bytes.Equal(b.PrevBlockHash[:], (&[32]byte{})[:]) // TODO: think of a better indicator to check
+}
+
 // Serialize serializes the block
 func (b *Block) Serialize() []byte {
 	var result bytes.Buffer
@@ -75,6 +79,10 @@ func (b *Block) Write(db db.Database, key string) error {
 	return db.Put([]byte(key), b.Serialize())
 }
 
+func Delete(db db.Database, key string) error {
+	return db.Delete([]byte(key))
+}
+
 // CalculateBlockHash returns a hash of the block
 func (b *Block) CalculateBlockHash() []byte {
 	var hashes [][]byte
@@ -112,3 +120,21 @@ func NewBlock(transactions []*Transaction, prevBlockHash [32]byte, shardId uint3
 func NewGenesisBlock(coinbase *Transaction, shardId uint32) *Block {
 	return NewBlock([]*Transaction{coinbase}, [32]byte{}, shardId)
 }
+
+// NewStateBlock creates and returns a state Block based on utxo pool.
+func NewStateBlock(utxoPool *UTXOPool) *Block {
+	stateTransactions := []*Transaction{}
+	for address, txHash2Vout2AmountMap := range utxoPool.UtxoMap {
+		stateTransaction := Transaction{}
+		for _, vout2AmountMap := range txHash2Vout2AmountMap {
+			for _, amount := range vout2AmountMap {
+				stateTransaction.TxOutput = append(stateTransaction.TxOutput, TXOutput{Amount: amount, Address: address, ShardID: utxoPool.ShardID})
+			}
+		}
+		if len(stateTransaction.TxOutput) != 0 {
+			stateTransaction.SetID()
+			stateTransactions = append(stateTransactions, &stateTransaction)
+		}
+	}
+	return NewBlock(stateTransactions, [32]byte{}, utxoPool.ShardID)
+}

blockchain/utxopool.go

@@ -85,6 +85,12 @@ func (utxoPool *UTXOPool) VerifyTransactions(transactions []*Transaction) bool {
 	return true
 }
 
+// VerifyStateBlock verifies if the given state block matches the current utxo pool.
+func (utxoPool *UTXOPool) VerifyStateBlock(stateBlock *Block) bool {
+	// TODO: implement this
+	return true
+}
+
 // VerifyOneTransaction verifies if a list of transactions valid.
 func (utxoPool *UTXOPool) VerifyOneTransaction(tx *Transaction, spentTXOs *map[[20]byte]map[string]map[uint32]bool) (valid, crossShard bool) {
 	if len(tx.Proofs) != 0 {
@@ -456,3 +462,8 @@ func (utxoPool *UTXOPool) GetSizeInByteOfUtxoMap() int {
 	encoder.Encode(utxoPool.UtxoMap)
 	return len(byteBuffer.Bytes())
 }
+
+// Create state block based on the utxos.
+func (utxoPool *UTXOPool) CreateStateBlock() *Block {
+	return NewStateBlock(utxoPool)
+}

client/wallet/main.go

@@ -28,42 +28,27 @@ import (
 func main() {
 	// Account subcommands
 	accountImportCommand := flag.NewFlagSet("import", flag.ExitOnError)
-	transferCommand := flag.NewFlagSet("transfer", flag.ExitOnError)
-	//accountListCommand := flag.NewFlagSet("list", flag.ExitOnError)
-	//
-	//// Transaction subcommands
-	//transactionNewCommand := flag.NewFlagSet("new", flag.ExitOnError)
-	//
-	//// Account subcommand flag pointers
-	//// Adding a new choice for --metric of 'substring' and a new --substring flag
 	accountImportPtr := accountImportCommand.String("privateKey", "", "Specify the private key to import")
 
+	// Transfer subcommands
+	transferCommand := flag.NewFlagSet("transfer", flag.ExitOnError)
 	transferSenderPtr := transferCommand.String("sender", "0", "Specify the sender account address or index")
 	transferReceiverPtr := transferCommand.String("receiver", "", "Specify the receiver account")
 	transferAmountPtr := transferCommand.Int("amount", 0, "Specify the amount to transfer")
-	//accountListPtr := accountNewCommand.Bool("new", false, "N/A")
-	//
-	//// Transaction subcommand flag pointers
-	//transactionNewPtr := transactionNewCommand.String("text", "", "Text to parse. (Required)")
 
 	// Verify that a subcommand has been provided
 	// os.Arg[0] is the main command
 	// os.Arg[1] will be the subcommand
 	if len(os.Args) < 2 {
-		fmt.Println("account or transaction subcommand is required")
+		fmt.Println("account or transfer subcommand is required")
 		os.Exit(1)
 	}
 
 	// Switch on the subcommand
-	// Parse the flags for appropriate FlagSet
-	// FlagSet.Parse() requires a set of arguments to parse as input
-	// os.Args[2:] will be all arguments starting after the subcommand at os.Args[1]
 	switch os.Args[1] {
 	case "account":
 		switch os.Args[2] {
 		case "new":
-			fmt.Println("Creating new account...")
-
 			randomBytes := [32]byte{}
 			_, err := io.ReadFull(rand.Reader, randomBytes[:])
 
@@ -74,7 +59,7 @@ func main() {
 			priKey := crypto.Ed25519Curve.Scalar().SetBytes(randomBytes[:])
 			priKeyBytes, err := priKey.MarshalBinary()
 			if err != nil {
-				panic("Failed to generate private key")
+				panic("Failed to serialize the private key")
 			}
 			pubKey := pki.GetPublicKeyFromScalar(priKey)
 			address := pki.GetAddressFromPublicKey(pubKey)
@@ -82,13 +67,12 @@ func main() {
 			fmt.Printf("New account created:\nAddress: {%x}\n", address)
 		case "list":
 			for i, address := range ReadAddresses() {
-				fmt.Printf("Account %d:\n {%x}\n", i+1, address)
+				fmt.Printf("Account %d:\n  {%x}\n", i+1, address)
 			}
 		case "clearAll":
-			fmt.Println("Deleting existing accounts...")
-			DeletePrivateKey()
+			ClearKeystore()
+			fmt.Println("All existing accounts deleted...")
 		case "import":
-			fmt.Println("Importing private key...")
 			accountImportCommand.Parse(os.Args[3:])
 			priKey := *accountImportPtr
 			if !accountImportCommand.Parsed() {
@@ -99,15 +83,10 @@ func main() {
 				panic("Failed to parse the private key into bytes")
 			}
 			StorePrivateKey(priKeyBytes)
+			fmt.Println("Private key imported...")
 		case "showBalance":
-			configr := client_config.NewConfig()
-			configr.ReadConfigFile("local_config_shards.txt")
-			leaders, _ := configr.GetLeadersAndShardIds()
-			clientPeer := configr.GetClientPeer()
-			walletNode := node.New(nil, nil)
-			walletNode.Client = client.NewClient(&leaders)
-			walletNode.ClientPeer = clientPeer
-			go walletNode.StartServer(clientPeer.Port)
+			walletNode := CreateWalletServerNode()
+			go walletNode.StartServer(walletNode.ClientPeer.Port)
 
 			shardUtxoMap, err := FetchUtxos(ReadAddresses(), walletNode)
 			if err != nil {
@@ -115,6 +94,7 @@ func main() {
 			}
 			PrintUtxoBalance(shardUtxoMap)
 		case "test":
+			// Testing code
 			priKey := pki.GetPrivateKeyScalarFromInt(444)
 			address := pki.GetAddressFromPrivateKey(priKey)
 			priKeyBytes, err := priKey.MarshalBinary()
@@ -138,7 +118,7 @@ func main() {
 		}
 		priKeys := ReadPrivateKeys()
 		if len(priKeys) == 0 {
-			fmt.Println("No existing account to send money from.")
+			fmt.Println("No imported account to use.")
 			return
 		}
 		senderIndex, err := strconv.Atoi(sender)
@@ -153,7 +133,7 @@ func main() {
 				}
 			}
 			if senderIndex == -1 {
-				fmt.Println("Specified sender account is not imported yet.")
+				fmt.Println("The specified sender account is not imported yet.")
 				break
 			}
 		}
@@ -163,7 +143,7 @@ func main() {
 		}
 		receiverAddress, err := hex.DecodeString(receiver)
 		if err != nil || len(receiverAddress) != 20 {
-			fmt.Println("The receiver address is not a valid address.")
+			fmt.Println("The receiver address is not a valid.")
 			return
 		}
 
@@ -175,14 +155,8 @@ func main() {
 		senderAddressBytes := pki.GetAddressFromPrivateKey(senderPriKey)
 
 		// Start client server
-		configr := client_config.NewConfig()
-		configr.ReadConfigFile("local_config_shards.txt")
-		leaders, _ := configr.GetLeadersAndShardIds()
-		clientPeer := configr.GetClientPeer()
-		walletNode := node.New(nil, nil)
-		walletNode.Client = client.NewClient(&leaders)
-		walletNode.ClientPeer = clientPeer
-		go walletNode.StartServer(clientPeer.Port)
+		walletNode := CreateWalletServerNode()
+		go walletNode.StartServer(walletNode.ClientPeer.Port)
 
 		shardUtxoMap, err := FetchUtxos([][20]byte{senderAddressBytes}, walletNode)
 		if err != nil {
@@ -227,21 +201,37 @@ func main() {
 			fmt.Println("Failed to deserialize public key", "error", err)
 		}
 
-		ExecuteTransaction(tx, walletNode)
+		err = ExecuteTransaction(tx, walletNode)
+
+		if err != nil {
+			fmt.Println(err)
+		} else {
+			fmt.Println("Transaction submitted successfully")
+		}
 	default:
 		flag.PrintDefaults()
 		os.Exit(1)
 	}
 }
 
+func CreateWalletServerNode() *node.Node {
+	configr := client_config.NewConfig()
+	configr.ReadConfigFile("local_config_shards.txt")
+	leaders, _ := configr.GetLeadersAndShardIds()
+	clientPeer := configr.GetClientPeer()
+	walletNode := node.New(nil, nil)
+	walletNode.Client = client.NewClient(&leaders)
+	walletNode.ClientPeer = clientPeer
+	return walletNode
+}
+
+// Issue the transaction to the Harmony network
 func ExecuteTransaction(tx blockchain.Transaction, walletNode *node.Node) error {
 	if tx.IsCrossShard() {
 		walletNode.Client.PendingCrossTxsMutex.Lock()
 		walletNode.Client.PendingCrossTxs[tx.ID] = &tx
 		walletNode.Client.PendingCrossTxsMutex.Unlock()
-
 	}
-	fmt.Println("Sending transaction...")
 
 	msg := proto_node.ConstructTransactionListMessage([]*blockchain.Transaction{&tx})
 	p2p.BroadcastMessage(*walletNode.Client.Leaders, msg)
@@ -261,10 +251,11 @@ func ExecuteTransaction(tx blockchain.Transaction, walletNode *node.Node) error
 		time.Sleep(100 * time.Millisecond)
 		return nil
 	case <-time.After(5 * time.Second):
-		return errors.New("Cross-shard tx timed out")
+		return errors.New("Cross-shard Transaction processing timed out")
 	}
 }
 
+// Fetch utxos of specified address from the Harmony network
 func FetchUtxos(addresses [][20]byte, walletNode *node.Node) (map[uint32]blockchain.UtxoMap, error) {
 	fmt.Println("Fetching account balance...")
 	walletNode.Client.ShardUtxoMap = make(map[uint32]blockchain.UtxoMap)
@@ -310,6 +301,7 @@ func PrintUtxoBalance(shardUtxoMap map[uint32]blockchain.UtxoMap) {
 	}
 }
 
+// Read the addresses stored in local keystore
 func ReadAddresses() [][20]byte {
 	priKeys := ReadPrivateKeys()
 	addresses := [][20]byte{}
@@ -319,6 +311,7 @@ func ReadAddresses() [][20]byte {
 	return addresses
 }
 
+// Store the specified private key in local keystore
 func StorePrivateKey(priKey []byte) {
 	for _, address := range ReadAddresses() {
 		if address == pki.GetAddressFromPrivateKey(crypto.Ed25519Curve.Scalar().SetBytes(priKey)) {
@@ -339,10 +332,12 @@ func StorePrivateKey(priKey []byte) {
 	f.Close()
 }
 
-func DeletePrivateKey() {
+// Delete all data in the local keystore
+func ClearKeystore() {
 	ioutil.WriteFile("keystore", []byte{}, 0644)
 }
 
+// Read all the private key stored in local keystore
 func ReadPrivateKeys() []kyber.Scalar {
 	keys, err := ioutil.ReadFile("keystore")
 	if err != nil {

consensus/consensus_leader.go

@@ -367,7 +367,6 @@ func (consensus *Consensus) processResponseMessage(payload []byte, targetState C
 				}
 
 				// Sign the block
-				// TODO(RJ): populate bitmap
 				copy(blockHeaderObj.Signature[:], collectiveSig[:])
 				copy(blockHeaderObj.Bitmap[:], bitmap)
 				consensus.OnConsensusDone(&blockHeaderObj)

node/node_handler.go

@@ -20,6 +20,8 @@ import (
 const (
 	// The max number of transaction per a block.
 	MaxNumberOfTransactionsPerBlock = 3000
+	// The number of blocks allowed before generating state block
+	NumBlocksBeforeStateBlock = 10
 )
 
 // NodeHandler handles a new incoming connection.
@@ -206,26 +208,32 @@ func (node *Node) WaitForConsensusReady(readySignal chan int) {
 
 		//node.log.Debug("Adding new block", "currentChainSize", len(node.blockchain.Blocks), "numTxs", len(node.blockchain.GetLatestBlock().Transactions), "PrevHash", node.blockchain.GetLatestBlock().PrevBlockHash, "Hash", node.blockchain.GetLatestBlock().Hash)
 		if !retry {
-			for {
-				// Once we have pending transactions we will try creating a new block
-				if len(node.pendingTransactions) >= 1 {
-					selectedTxs, crossShardTxAndProofs := node.getTransactionsForNewBlock(MaxNumberOfTransactionsPerBlock)
-
-					if len(selectedTxs) == 0 {
-						node.log.Debug("No valid transactions exist", "pendingTx", len(node.pendingTransactions))
-					} else {
-						node.log.Debug("Creating new block", "numTxs", len(selectedTxs), "pendingTxs", len(node.pendingTransactions), "currentChainSize", len(node.blockchain.Blocks))
-
-						node.transactionInConsensus = selectedTxs
-						node.log.Debug("CROSS SHARD TX", "num", len(crossShardTxAndProofs))
-						node.CrossTxsInConsensus = crossShardTxAndProofs
-						newBlock = blockchain.NewBlock(selectedTxs, node.blockchain.GetLatestBlock().Hash, node.Consensus.ShardID)
-						break
+			if len(node.blockchain.Blocks) > NumBlocksBeforeStateBlock {
+				// Generate state block and run consensus on it
+				newBlock = node.UtxoPool.CreateStateBlock()
+			} else {
+				// Normal tx block consensus
+				for {
+					// Once we have pending transactions we will try creating a new block
+					if len(node.pendingTransactions) >= 1 {
+						selectedTxs, crossShardTxAndProofs := node.getTransactionsForNewBlock(MaxNumberOfTransactionsPerBlock)
+
+						if len(selectedTxs) == 0 {
+							node.log.Debug("No valid transactions exist", "pendingTx", len(node.pendingTransactions))
+						} else {
+							node.log.Debug("Creating new block", "numTxs", len(selectedTxs), "pendingTxs", len(node.pendingTransactions), "currentChainSize", len(node.blockchain.Blocks))
+
+							node.transactionInConsensus = selectedTxs
+							node.log.Debug("CROSS SHARD TX", "num", len(crossShardTxAndProofs))
+							node.CrossTxsInConsensus = crossShardTxAndProofs
+							newBlock = blockchain.NewBlock(selectedTxs, node.blockchain.GetLatestBlock().Hash, node.Consensus.ShardID)
+							break
+						}
 					}
+					// If not enough transactions to run Consensus,
+					// periodically check whether we have enough transactions to package into block.
+					time.Sleep(1 * time.Second)
 				}
-				// If not enough transactions to run Consensus,
-				// periodically check whether we have enough transactions to package into block.
-				time.Sleep(1 * time.Second)
 			}
 		}
 
@@ -263,29 +271,47 @@ func (node *Node) BroadcastNewBlock(newBlock *blockchain.Block) {
 
 // This is called by consensus participants to verify the block they are running consensus on
 func (node *Node) VerifyNewBlock(newBlock *blockchain.Block) bool {
-	return node.UtxoPool.VerifyTransactions(newBlock.Transactions)
+	if newBlock.IsStateBlock() {
+		return node.UtxoPool.VerifyStateBlock(newBlock)
+	} else {
+		return node.UtxoPool.VerifyTransactions(newBlock.Transactions)
+	}
 }
 
 // This is called by consensus participants, after consensus is done, to:
 // 1. add the new block to blockchain
 // 2. [leader] move cross shard tx and proof to the list where they wait to be sent to the client
 func (node *Node) PostConsensusProcessing(newBlock *blockchain.Block) {
-	node.AddNewBlock(newBlock)
-
-	if node.Consensus.IsLeader {
-		// Move crossTx-in-consensus into the list to be returned to client
-		for _, crossTxAndProof := range node.CrossTxsInConsensus {
-			crossTxAndProof.Proof.BlockHash = newBlock.Hash
-			// TODO: fill in the signature proofs
-		}
-		if len(node.CrossTxsInConsensus) != 0 {
-			node.addCrossTxsToReturn(node.CrossTxsInConsensus)
-			node.CrossTxsInConsensus = []*blockchain.CrossShardTxAndProof{}
+	if newBlock.IsStateBlock() {
+		// Clear out old tx blocks and put state block as genesis
+		if node.db != nil {
+			node.log.Info("Deleting old blocks.")
+			for i := 1; i <= len(node.blockchain.Blocks); i++ {
+				blockchain.Delete(node.db, strconv.Itoa(i))
+			}
 		}
+		node.blockchain.Blocks = []*blockchain.Block{}
+		node.AddNewBlock(newBlock)
+	} else {
+		node.AddNewBlock(newBlock)
+		node.UpdateUtxoAndState(newBlock)
+
+		if node.Consensus.IsLeader {
+			// Move crossTx-in-consensus into the list to be returned to client
+			for _, crossTxAndProof := range node.CrossTxsInConsensus {
+				crossTxAndProof.Proof.BlockHash = newBlock.Hash
+				// TODO: fill in the signature proofs
+			}
+			if len(node.CrossTxsInConsensus) != 0 {
+				node.addCrossTxsToReturn(node.CrossTxsInConsensus)
+				node.CrossTxsInConsensus = []*blockchain.CrossShardTxAndProof{}
+			}
 
-		node.SendBackProofOfAcceptOrReject()
-		node.BroadcastNewBlock(newBlock)
+			node.SendBackProofOfAcceptOrReject()
+			node.BroadcastNewBlock(newBlock)
+		}
 	}
+
 }
 
 func (node *Node) AddNewBlock(newBlock *blockchain.Block) {
@@ -296,6 +322,9 @@ func (node *Node) AddNewBlock(newBlock *blockchain.Block) {
 		node.log.Info("Writing new block into disk.")
 		newBlock.Write(node.db, strconv.Itoa(len(node.blockchain.Blocks)))
 	}
+}
+
+func (node *Node) UpdateUtxoAndState(newBlock *blockchain.Block) {
 	// Update UTXO pool
 	node.UtxoPool.Update(newBlock.Transactions)
 	// Clear transaction-in-Consensus list