update btctxgen

client/btctxgen/main.go

@@ -12,13 +12,13 @@ import (
 	"harmony-benchmark/log"
 	"harmony-benchmark/node"
 	"harmony-benchmark/p2p"
-	"math/rand"
 	"sync"
 	"time"
+
+	"github.com/piotrnar/gocoin/lib/btc"
 )
 
 type txGenSettings struct {
-	numOfAddress      int
 	crossShard        bool
 	maxNumTxsPerBatch int
 }
@@ -29,21 +29,6 @@ var (
 	btcTXIter     btctxiter.BTCTXIterator
 )
 
-type TxInfo struct {
-	// Global Input
-	shardID   int
-	dataNodes []*node.Node
-	// Temp Input
-	id      [32]byte
-	index   int
-	value   int
-	address string
-	// Output
-	txs      []*blockchain.Transaction
-	crossTxs []*blockchain.Transaction
-	txCount  int
-}
-
 // Generates at most "maxNumTxs" number of simulated transactions based on the current UtxoPools of all shards.
 // The transactions are generated by going through the existing utxos and
 // randomly select a subset of them as the input for each new transaction. The output
@@ -64,89 +49,77 @@ type TxInfo struct {
 func generateSimulatedTransactions(shardID int, dataNodes []*node.Node) ([]*blockchain.Transaction, []*blockchain.Transaction) {
 	/*
 		  UTXO map structure:
-	     address - [
-	                txId1 - [
-	                        outputIndex1 - value1
-	                        outputIndex2 - value2
-	                       ]
-	                txId2 - [
-	                        outputIndex1 - value1
-	                        outputIndex2 - value2
-	                       ]
-	               ]
+		  {
+			  address: {
+				  txID: {
+					  outputIndex: value
+				  }
+			  }
+		  }
 	*/
 
 	utxoPoolMutex.Lock()
-	txInfo := TxInfo{}
-	txInfo.shardID = shardID
-	txInfo.dataNodes = dataNodes
-	txInfo.txCount = 0
-
-UTXOLOOP:
-	// Loop over all addresses
-	for address, txMap := range dataNodes[shardID].UtxoPool.UtxoMap {
-		txInfo.address = address
-		// Loop over all txIds for the address
-		for txIdStr, utxoMap := range txMap {
-			// Parse TxId
-			id, err := hex.DecodeString(txIdStr)
-			if err != nil {
-				continue
-			}
-			copy(txInfo.id[:], id[:])
+	txs := []*blockchain.Transaction{}
+	crossTxs := []*blockchain.Transaction{}
 
-			// Loop over all utxos for the txId
-			for index, value := range utxoMap {
-				txInfo.index = index
-				txInfo.value = value
+	nodeShardID := dataNodes[shardID].Consensus.ShardID
+	cnt := 0
 
-				randNum := rand.Intn(100)
-				// 30% sample rate to select UTXO to use for new transactions
-				if randNum >= 30 {
-					continue
+LOOP:
+	for true {
+		blk := btcTXIter.IterateBTCTX()
+		blk.BuildTxList()
+		for _, btcTx := range blk.Txs {
+			tx := blockchain.Transaction{}
+			// tx.ID = tx.Hash.String()
+			if btcTx.IsCoinBase() {
+				// TxIn coinbase, newly generated coins
+				prevTxID := [32]byte{}
+				// TODO: merge txID with txIndex
+				tx.TxInput = []blockchain.TXInput{blockchain.TXInput{prevTxID, -1, "", nodeShardID}}
+			} else {
+				for _, txi := range btcTx.TxIn {
+					tx.TxInput = append(tx.TxInput, blockchain.TXInput{txi.Input.Hash, int(txi.Input.Vout), "", nodeShardID})
 				}
-				generateSingleShardTx(&txInfo)
-				if txInfo.txCount >= setting.maxNumTxsPerBatch {
-					break UTXOLOOP
 			}
+
+			for _, txo := range btcTx.TxOut {
+				txoAddr := btc.NewAddrFromPkScript(txo.Pk_script, false)
+				if txoAddr == nil {
+					log.Warn("TxOut: can't decode address")
+				}
+				txout := blockchain.TXOutput{int(txo.Value), txoAddr.String(), nodeShardID}
+				tx.TxOutput = append(tx.TxOutput, txout)
+			}
+			tx.SetID()
+			txs = append(txs, &tx)
+			log.Debug("[Generator] transformed btc tx", "block height", btcTXIter.GetIndex(), "txi", len(tx.TxInput), "txo", len(tx.TxOutput), "txCount", cnt)
+			cnt++
+			if cnt >= setting.maxNumTxsPerBatch {
+				break LOOP
 			}
 		}
 	}
-	utxoPoolMutex.Unlock()
-
-	log.Debug("[Generator] generated transations", "single-shard", len(txInfo.txs), "cross-shard", len(txInfo.crossTxs))
-	return txInfo.txs, txInfo.crossTxs
-}
-
-func generateSingleShardTx(txInfo *TxInfo) {
-	// nodeShardID := txInfo.dataNodes[txInfo.shardID].Consensus.ShardID
-	// blk := btcTXIter.IterateBTCTX()
-
-	// Add the utxo as new tx input
-	// txin := blockchain.TXInput{txInfo.id, txInfo.index, txInfo.address, nodeShardID}
 
-	// // Spend the utxo to a random address in [0 - N)
-	// txout := blockchain.TXOutput{txInfo.value, strconv.Itoa(rand.Intn(setting.numOfAddress)), nodeShardID}
-	// tx := blockchain.Transaction{[32]byte{}, []blockchain.TXInput{txin}, []blockchain.TXOutput{txout}, nil}
-	// tx.SetID()
+	utxoPoolMutex.Unlock()
 
-	// txInfo.txs = append(txInfo.txs, &tx)
-	txInfo.txCount++
+	log.Debug("[Generator] generated transations", "single-shard", len(txs), "cross-shard", len(crossTxs))
+	return txs, crossTxs
 }
 
 // A utility func that counts the total number of utxos in a pool.
 func countNumOfUtxos(utxoPool *blockchain.UTXOPool) int {
 	countAll := 0
 	for _, utxoMap := range utxoPool.UtxoMap {
-		for txIdStr, val := range utxoMap {
+		for txIDStr, val := range utxoMap {
 			_ = val
-			id, err := hex.DecodeString(txIdStr)
+			id, err := hex.DecodeString(txIDStr)
 			if err != nil {
 				continue
 			}
 
-			txId := [32]byte{}
-			copy(txId[:], id[:])
+			txID := [32]byte{}
+			copy(txID[:], id[:])
 			for _, utxo := range val {
 				_ = utxo
 				countAll++
@@ -190,7 +163,7 @@ func initClient(clientNode *node.Node, clientPort string, leaders *[]p2p.Peer, n
 
 func main() {
 	configFile := flag.String("config_file", "local_config.txt", "file containing all ip addresses and config")
-	maxNumTxsPerBatch := flag.Int("max_num_txs_per_batch", 100000, "number of transactions to send per message")
+	maxNumTxsPerBatch := flag.Int("max_num_txs_per_batch", 100, "number of transactions to send per message")
 	logFolder := flag.String("log_folder", "latest", "the folder collecting the logs of this execution")
 	flag.Parse()
 
@@ -198,7 +171,6 @@ func main() {
 	config, _ := configr.ReadConfigFile(*configFile)
 	leaders, shardIDs := configr.GetLeadersAndShardIds(&config)
 
-	setting.numOfAddress = 10000
 	// Do cross shard tx if there are more than one shard
 	setting.crossShard = len(shardIDs) > 1
 	setting.maxNumTxsPerBatch = *maxNumTxsPerBatch

client/btctxiter/btctxiter.go

@@ -36,3 +36,7 @@ func (iter *BTCTXIterator) IterateBTCTX() *btc.Block {
 	blk.BuildTxList()
 	return blk
 }
+
+func (iter *BTCTXIterator) GetIndex() int {
+	return iter.index
+}

deploy_btctx.sh

@@ -0,0 +1,29 @@
+# Kill nodes if any
+./kill_node.sh
+
+# Since `go run` will generate a temporary exe every time,
+# On windows, your system will pop up a network security dialog for each instance
+# and you won't be able to turn it off. With `go build` generating one
+# exe, the dialog will only pop up once at the very first time.
+# Also it's recommended to use `go build` for testing the whole exe. 
+go build -o bin/benchmark
+go build -o bin/btctxgen client/btctxgen/main.go
+
+# Create a tmp folder for logs
+t=`date +"%Y%m%d-%H%M%S"`
+log_folder="tmp_log/log-$t"
+
+mkdir -p $log_folder
+
+# Start nodes
+config=$1
+while IFS='' read -r line || [[ -n "$line" ]]; do
+  IFS=' ' read ip port mode shardId <<< $line
+	#echo $ip $port $mode
+  if [ "$mode" != "client" ]; then
+    ./bin/benchmark -ip $ip -port $port -config_file $config -log_folder $log_folder&
+  fi
+done < $config
+
+# Generate transactions
+./bin/btctxgen -config_file $config -log_folder $log_folder