Merge branch 'master' of github.com:simple-rules/harmony-benchmark

client/btctxgen/main.go

@@ -13,13 +13,13 @@ import (
 	"harmony-benchmark/node"
 	"harmony-benchmark/p2p"
 	proto_node "harmony-benchmark/proto/node"
-	"math/rand"
 	"sync"
 	"time"
+
+	"github.com/piotrnar/gocoin/lib/btc"
 )
 
 type txGenSettings struct {
-	numOfAddress      int
 	crossShard        bool
 	maxNumTxsPerBatch int
 }
@@ -30,21 +30,6 @@ var (
 	btcTXIter     btctxiter.BTCTXIterator
 )
 
-type TxInfo struct {
-	// Global Input
-	shardID   int
-	dataNodes []*node.Node
-	// Temp Input
-	id      [32]byte
-	index   int
-	value   int
-	address string
-	// Output
-	txs      []*blockchain.Transaction
-	crossTxs []*blockchain.Transaction
-	txCount  int
-}
-
 // Generates at most "maxNumTxs" number of simulated transactions based on the current UtxoPools of all shards.
 // The transactions are generated by going through the existing utxos and
 // randomly select a subset of them as the input for each new transaction. The output
@@ -64,90 +49,75 @@ type TxInfo struct {
 //     all cross-shard txs
 func generateSimulatedTransactions(shardID int, dataNodes []*node.Node) ([]*blockchain.Transaction, []*blockchain.Transaction) {
 	/*
-	  UTXO map structure:
-	     address - [
-	                txId1 - [
-	                        outputIndex1 - value1
-	                        outputIndex2 - value2
-	                       ]
-	                txId2 - [
-	                        outputIndex1 - value1
-	                        outputIndex2 - value2
-	                       ]
-	               ]
+		  UTXO map structure:
+		  {
+			  address: {
+				  txID: {
+					  outputIndex: value
+				  }
+			  }
+		  }
 	*/
 
 	utxoPoolMutex.Lock()
-	txInfo := TxInfo{}
-	txInfo.shardID = shardID
-	txInfo.dataNodes = dataNodes
-	txInfo.txCount = 0
-
-UTXOLOOP:
-	// Loop over all addresses
-	for address, txMap := range dataNodes[shardID].UtxoPool.UtxoMap {
-		txInfo.address = address
-		// Loop over all txIds for the address
-		for txIdStr, utxoMap := range txMap {
-			// Parse TxId
-			id, err := hex.DecodeString(txIdStr)
-			if err != nil {
-				continue
-			}
-			copy(txInfo.id[:], id[:])
+	txs := []*blockchain.Transaction{}
+	crossTxs := []*blockchain.Transaction{}
 
-			// Loop over all utxos for the txId
-			for index, value := range utxoMap {
-				txInfo.index = index
-				txInfo.value = value
+	nodeShardID := dataNodes[shardID].Consensus.ShardID
+	cnt := 0
 
-				randNum := rand.Intn(100)
-				// 30% sample rate to select UTXO to use for new transactions
-				if randNum >= 30 {
-					continue
-				}
-				generateSingleShardTx(&txInfo)
-				if txInfo.txCount >= setting.maxNumTxsPerBatch {
-					break UTXOLOOP
-				}
+LOOP:
+	for true {
+		btcTx := btcTXIter.NextTx()
+		tx := blockchain.Transaction{}
+		// tx.ID = tx.Hash.String()
+		if btcTx.IsCoinBase() {
+			// TxIn coinbase, newly generated coins
+			prevTxID := [32]byte{}
+			// TODO: merge txID with txIndex in TxInput
+			tx.TxInput = []blockchain.TXInput{blockchain.TXInput{prevTxID, -1, "", nodeShardID}}
+		} else {
+			for _, txi := range btcTx.TxIn {
+				tx.TxInput = append(tx.TxInput, blockchain.TXInput{txi.Input.Hash, int(txi.Input.Vout), "", nodeShardID})
 			}
 		}
-	}
-	utxoPoolMutex.Unlock()
-
-	log.Debug("[Generator] generated transations", "single-shard", len(txInfo.txs), "cross-shard", len(txInfo.crossTxs))
-	return txInfo.txs, txInfo.crossTxs
-}
-
-func generateSingleShardTx(txInfo *TxInfo) {
-	// nodeShardID := txInfo.dataNodes[txInfo.shardID].Consensus.ShardID
-	// blk := btcTXIter.IterateBTCTX()
 
-	// Add the utxo as new tx input
-	// txin := blockchain.TXInput{txInfo.id, txInfo.index, txInfo.address, nodeShardID}
+		for _, txo := range btcTx.TxOut {
+			txoAddr := btc.NewAddrFromPkScript(txo.Pk_script, false)
+			if txoAddr == nil {
+				log.Warn("TxOut: can't decode address")
+			}
+			txout := blockchain.TXOutput{int(txo.Value), txoAddr.String(), nodeShardID}
+			tx.TxOutput = append(tx.TxOutput, txout)
+		}
+		tx.SetID()
+		txs = append(txs, &tx)
+		// log.Debug("[Generator] transformed btc tx", "block height", btcTXIter.GetBlockIndex(), "block tx count", btcTXIter.GetBlock().TxCount, "block tx cnt", len(btcTXIter.GetBlock().Txs), "txi", len(tx.TxInput), "txo", len(tx.TxOutput), "txCount", cnt)
+		cnt++
+		if cnt >= setting.maxNumTxsPerBatch {
+			break LOOP
+		}
+	}
 
-	// // Spend the utxo to a random address in [0 - N)
-	// txout := blockchain.TXOutput{txInfo.value, strconv.Itoa(rand.Intn(setting.numOfAddress)), nodeShardID}
-	// tx := blockchain.Transaction{[32]byte{}, []blockchain.TXInput{txin}, []blockchain.TXOutput{txout}, nil}
-	// tx.SetID()
+	utxoPoolMutex.Unlock()
 
-	// txInfo.txs = append(txInfo.txs, &tx)
-	txInfo.txCount++
+	log.Debug("[Generator] generated transations", "single-shard", len(txs), "cross-shard", len(crossTxs))
+	return txs, crossTxs
 }
 
 // A utility func that counts the total number of utxos in a pool.
 func countNumOfUtxos(utxoPool *blockchain.UTXOPool) int {
 	countAll := 0
 	for _, utxoMap := range utxoPool.UtxoMap {
-		for txIdStr, val := range utxoMap {
+		for txIDStr, val := range utxoMap {
 			_ = val
-			id, err := hex.DecodeString(txIdStr)
+			id, err := hex.DecodeString(txIDStr)
 			if err != nil {
 				continue
 			}
 
-			txId := [32]byte{}
-			copy(txId[:], id[:])
+			txID := [32]byte{}
+			copy(txID[:], id[:])
 			for _, utxo := range val {
 				_ = utxo
 				countAll++
@@ -191,7 +161,7 @@ func initClient(clientNode *node.Node, clientPort string, leaders *[]p2p.Peer, n
 
 func main() {
 	configFile := flag.String("config_file", "local_config.txt", "file containing all ip addresses and config")
-	maxNumTxsPerBatch := flag.Int("max_num_txs_per_batch", 100000, "number of transactions to send per message")
+	maxNumTxsPerBatch := flag.Int("max_num_txs_per_batch", 100, "number of transactions to send per message")
 	logFolder := flag.String("log_folder", "latest", "the folder collecting the logs of this execution")
 	flag.Parse()
 
@@ -199,7 +169,6 @@ func main() {
 	config, _ := configr.ReadConfigFile(*configFile)
 	leaders, shardIDs := configr.GetLeadersAndShardIds(&config)
 
-	setting.numOfAddress = 10000
 	// Do cross shard tx if there are more than one shard
 	setting.crossShard = len(shardIDs) > 1
 	setting.maxNumTxsPerBatch = *maxNumTxsPerBatch

client/btctxiter/btctxiter.go

@@ -8,7 +8,10 @@ import (
 )
 
 type BTCTXIterator struct {
-	index         int
+	blockIndex    int
+	block         *btc.Block
+	txIndex       int
+	tx            *btc.Tx
 	blockDatabase *blockdb.BlockDB
 }
 
@@ -18,21 +21,63 @@ func (iter *BTCTXIterator) Init() {
 
 	// Specify blocks directory
 	iter.blockDatabase = blockdb.NewBlockDB("/Users/ricl/Library/Application Support/Bitcoin/blocks", Magic)
-	iter.index = -1
+	iter.blockIndex = -1
+	iter.block = nil
+	iter.nextBlock()
 }
 
-func (iter *BTCTXIterator) IterateBTCTX() *btc.Block {
-	iter.index++
+// Move to the next transaction
+func (iter *BTCTXIterator) NextTx() *btc.Tx {
+	iter.txIndex++
+	if iter.txIndex >= iter.block.TxCount {
+		iter.nextBlock()
+		iter.txIndex++
+	}
+	iter.tx = iter.block.Txs[iter.txIndex]
+	return iter.tx
+}
+
+// Gets the index/height of the current block
+func (iter *BTCTXIterator) GetBlockIndex() int {
+	return iter.blockIndex
+}
+
+// Gets the current block
+func (iter *BTCTXIterator) GetBlock() *btc.Block {
+	return iter.block
+}
+
+// Gets the index of the current transaction
+func (iter *BTCTXIterator) GetTxIndex() int {
+	return iter.txIndex
+}
+
+// Gets the current transaction
+func (iter *BTCTXIterator) GetTx() *btc.Tx {
+	return iter.tx
+}
+
+func (iter *BTCTXIterator) resetTx() {
+	iter.txIndex = -1
+	iter.tx = nil
+}
+
+// Move to the next block
+func (iter *BTCTXIterator) nextBlock() *btc.Block {
+	iter.blockIndex++
 	dat, err := iter.blockDatabase.FetchNextBlock()
 	if dat == nil || err != nil {
 		log.Println("END of DB file")
 	}
-	blk, err := btc.NewBlock(dat[:])
+	iter.block, err = btc.NewBlock(dat[:])
 
 	if err != nil {
 		println("Block inconsistent:", err.Error())
 	}
 
-	blk.BuildTxList()
-	return blk
+	iter.block.BuildTxList()
+
+	iter.resetTx()
+
+	return iter.block
 }

deploy_btctx.sh

@@ -0,0 +1,29 @@
+# Kill nodes if any
+./kill_node.sh
+
+# Since `go run` will generate a temporary exe every time,
+# On windows, your system will pop up a network security dialog for each instance
+# and you won't be able to turn it off. With `go build` generating one
+# exe, the dialog will only pop up once at the very first time.
+# Also it's recommended to use `go build` for testing the whole exe. 
+go build -o bin/benchmark
+go build -o bin/btctxgen client/btctxgen/main.go
+
+# Create a tmp folder for logs
+t=`date +"%Y%m%d-%H%M%S"`
+log_folder="tmp_log/log-$t"
+
+mkdir -p $log_folder
+
+# Start nodes
+config=$1
+while IFS='' read -r line || [[ -n "$line" ]]; do
+  IFS=' ' read ip port mode shardId <<< $line
+	#echo $ip $port $mode
+  if [ "$mode" != "client" ]; then
+    ./bin/benchmark -ip $ip -port $port -config_file $config -log_folder $log_folder&
+  fi
+done < $config
+
+# Generate transactions
+./bin/btctxgen -config_file $config -log_folder $log_folder

p2p/peer.go

@@ -3,12 +3,13 @@ package p2p
 import (
 	"bytes"
 	"encoding/binary"
-	"github.com/dedis/kyber"
 	"harmony-benchmark/attack"
 	"log"
 	"net"
 	"strings"
 	"sync"
+
+	"github.com/dedis/kyber"
 )
 
 // Peer is the object for a p2p peer (node)