update btctxgen

client/btctxgen/main.go

@@ -24,9 +24,6 @@ import (
 	"sync"
 	"time"
 
-	btcblockchain "github.com/btcsuite/btcd/blockchain"
-	"github.com/btcsuite/btcd/chaincfg"
-	"github.com/btcsuite/btcd/txscript"
 	"github.com/simple-rules/harmony-benchmark/blockchain"
 	"github.com/simple-rules/harmony-benchmark/client"
 	"github.com/simple-rules/harmony-benchmark/client/btctxiter"
@@ -53,7 +50,7 @@ type TXRef struct {
 var (
 	utxoPoolMutex sync.Mutex
 	setting       txGenSettings
-	btcTXIter     btctxiter.BTCTXIterator
+	iter          btctxiter.BTCTXIterator
 	utxoMapping   map[string]TXRef // btcTXID to { txID, shardID }
 	// map from bitcoin address to a int value (the privKey in hmy)
 	addressMapping map[[20]byte]int
@@ -108,49 +105,42 @@ func generateSimulatedTransactions(shardID int, dataNodes []*node.Node) ([]*bloc
 
 LOOP:
 	for true {
-		btcTx := btcTXIter.NextTx()
+		btcTx := iter.NextTx()
+		if btcTx == nil {
+			log.Error("Failed to parse tx", "height", iter.GetBlockIndex())
+		}
 		tx := blockchain.Transaction{}
 		isCrossShardTx := false
 
-		if btcblockchain.IsCoinBaseTx(btcTx.MsgTx()) {
+		if iter.IsCoinBaseTx(btcTx) {
 			// ricl: coinbase tx should just have one txo
-			btcTXO := btcTx.MsgTx().TxOut[0]
+			btcTXO := btcTx.Vout[0]
-			_, addresses, _, _ := txscript.ExtractPkScriptAddrs(
+			btcTXOAddr := btcTXO.ScriptPubKey.Addresses[0]
-				btcTXO.PkScript, &chaincfg.MainNetParams)
-			btcTXOAddr := addresses[0]
-			if btcTXOAddr == nil {
-				log.Warn("TxOut: can't decode address")
-			}
 			var toAddress [20]byte
-			copy(toAddress[:], btcTXOAddr.ScriptAddress())
+			copy(toAddress[:], btcTXOAddr) // TODO(ricl): string to [20]byte
 			hmyInt := getHmyInt(toAddress)
 			tx = *blockchain.NewCoinbaseTX(pki.GetAddressFromInt(hmyInt), "", nodeShardID)
 
-			utxoMapping[btcTx.Hash().String()] = TXRef{tx.ID, nodeShardID, toAddress}
+			utxoMapping[btcTx.Hash] = TXRef{tx.ID, nodeShardID, toAddress}
 		} else {
 			var btcFromAddresses [][20]byte
-			for _, btcTXI := range btcTx.MsgTx().TxIn {
+			for _, btcTXI := range btcTx.Vin {
-				btcTXIDStr := btcTXI.PreviousOutPoint.Hash.String()
+				btcTXIDStr := btcTXI.Txid
 				txRef := utxoMapping[btcTXIDStr] // find the corresponding harmony tx info
 				if txRef.shardID != nodeShardID {
 					isCrossShardTx = true
 				}
-				tx.TxInput = append(tx.TxInput, *blockchain.NewTXInput(blockchain.NewOutPoint(&txRef.txID, btcTXI.PreviousOutPoint.Index), [20]byte{}, txRef.shardID))
+				tx.TxInput = append(tx.TxInput, *blockchain.NewTXInput(blockchain.NewOutPoint(&txRef.txID, btcTXI.Vout), [20]byte{}, txRef.shardID))
 				// Add the from address to array, so that we can later use it to sign the tx.
 				btcFromAddresses = append(btcFromAddresses, txRef.toAddress)
 			}
-			for _, btcTXO := range btcTx.MsgTx().TxOut {
+			for _, btcTXO := range btcTx.Vout {
-				_, addresses, _, _ := txscript.ExtractPkScriptAddrs(
+				for _, btcTXOAddr := range btcTXO.ScriptPubKey.Addresses {
-					btcTXO.PkScript, &chaincfg.MainNetParams)
-				for _, btcTXOAddr := range addresses {
-					if btcTXOAddr == nil {
-						log.Warn("TxOut: can't decode address")
-					}
 					var toAddress [20]byte
-					copy(toAddress[:], btcTXOAddr.ScriptAddress())
+					copy(toAddress[:], btcTXOAddr) //TODO(ricl): string to [20]byte
 					txo := blockchain.TXOutput{Amount: int(btcTXO.Value), Address: toAddress, ShardID: nodeShardID}
 					tx.TxOutput = append(tx.TxOutput, txo)
-					utxoMapping[btcTx.Hash().String()] = TXRef{tx.ID, nodeShardID, toAddress}
+					utxoMapping[btcTx.Txid] = TXRef{tx.ID, nodeShardID, toAddress}
 				}
 			}
 			// get private key and sign the tx
@@ -166,7 +156,7 @@ LOOP:
 		} else {
 			txs = append(txs, &tx)
 		}
-		// log.Debug("[Generator] transformed btc tx", "block height", btcTXIter.GetBlockIndex(), "block tx count", btcTXIter.GetBlock().TxCount, "block tx cnt", len(btcTXIter.GetBlock().Txs), "txi", len(tx.TxInput), "txo", len(tx.TxOutput), "txCount", cnt)
+		// log.Debug("[Generator] transformed btc tx", "block height", iter.GetBlockIndex(), "block tx count", iter.GetBlock().TxCount, "block tx cnt", len(iter.GetBlock().Txs), "txi", len(tx.TxInput), "txo", len(tx.TxOutput), "txCount", cnt)
 		cnt++
 		if cnt >= setting.maxNumTxsPerBatch {
 			break LOOP
@@ -236,7 +226,7 @@ func main() {
 	)
 	log.Root().SetHandler(h)
 
-	btcTXIter.Init()
+	iter.Init()
 	utxoMapping = make(map[string]TXRef)
 	addressMapping = make(map[[20]byte]int)
 
@@ -272,7 +262,7 @@ func main() {
 			txs, crossTxs := generateSimulatedTransactions(int(shardId), nodes)
 			allCrossTxs = append(allCrossTxs, crossTxs...)
 
-			log.Debug("[Generator] Sending single-shard txs ...", "leader", leader, "numTxs", len(txs), "numCrossTxs", len(crossTxs), "block height", btcTXIter.GetBlockIndex())
+			log.Debug("[Generator] Sending single-shard txs ...", "leader", leader, "numTxs", len(txs), "numCrossTxs", len(crossTxs), "block height", iter.GetBlockIndex())
 			msg := proto_node.ConstructTransactionListMessage(txs)
 			p2p.SendMessage(leader, msg)
 			// Note cross shard txs are later sent in batch

client/btctxiter/btctxiter.go

@@ -1,34 +1,36 @@
 package btctxiter
 
 import (
+	"io/ioutil"
 	"log"
+	"path/filepath"
 
-	"github.com/btcsuite/btcutil"
+	"github.com/btcsuite/btcd/btcjson"
-
 	"github.com/btcsuite/btcd/rpcclient"
-	"github.com/btcsuite/btcd/wire"
+	"github.com/btcsuite/btcutil"
 )
 
 type BTCTXIterator struct {
 	blockIndex int64
-	block      *wire.MsgBlock
+	block      *btcjson.GetBlockVerboseResult
 	txIndex    int
-	tx         *btcutil.Tx
+	tx         *btcjson.TxRawResult
 	client     *rpcclient.Client
 }
 
 func (iter *BTCTXIterator) Init() {
-	// Connect to local bitcoin core RPC server using HTTP POST mode.
+	btcdHomeDir := btcutil.AppDataDir("btcd", false)
+	certs, err := ioutil.ReadFile(filepath.Join(btcdHomeDir, "rpc.cert"))
+	if err != nil {
+		log.Fatal(err)
+	}
 	connCfg := &rpcclient.ConnConfig{
-		Host:         "localhost:8332",
+		Host:         "localhost:8334",
-		User:         "ricl",
+		Endpoint:     "ws",
-		Pass:         "123",
+		User:         "yourusername",
-		HTTPPostMode: true, // Bitcoin core only supports HTTP POST mode
+		Pass:         "yourpassword",
-		DisableTLS:   true, // Bitcoin core does not provide TLS by default
+		Certificates: certs,
-	}
+	}
-	// Notice the notification parameter is nil since notifications are
-	// not supported in HTTP POST mode.
-	var err error
 	iter.client, err = rpcclient.New(connCfg, nil)
 	if err != nil {
 		log.Fatal(err)
@@ -40,23 +42,14 @@ func (iter *BTCTXIterator) Init() {
 }
 
 // Move to the next transaction
-func (iter *BTCTXIterator) NextTx() *btcutil.Tx {
+func (iter *BTCTXIterator) NextTx() *btcjson.TxRawResult {
 	iter.txIndex++
-	hashes, err := iter.block.TxHashes()
+	if iter.txIndex >= len(iter.block.RawTx) {
-	if err != nil {
-		log.Println("Failed to get tx hashes", iter.blockIndex, iter.txIndex, err)
-		return nil
-	}
-	if iter.txIndex >= len(hashes) {
 		iter.nextBlock()
 		iter.txIndex++
 	}
-	iter.tx, err = iter.client.GetRawTransaction(&hashes[iter.txIndex])
+	iter.tx = &iter.block.RawTx[iter.txIndex]
-	if err != nil {
+	// log.Println(iter.blockIndex, iter.txIndex, hashes[iter.txIndex])
-		log.Println("Failed to get raw tx", iter.blockIndex, iter.txIndex, hashes[iter.txIndex], err)
-		return nil
-	}
-	log.Println(iter.blockIndex, iter.txIndex, hashes[iter.txIndex])
 	return iter.tx
 }
 
@@ -66,7 +59,7 @@ func (iter *BTCTXIterator) GetBlockIndex() int64 {
 }
 
 // Gets the current block
-func (iter *BTCTXIterator) GetBlock() *wire.MsgBlock {
+func (iter *BTCTXIterator) GetBlock() *btcjson.GetBlockVerboseResult {
 	return iter.block
 }
 
@@ -76,25 +69,34 @@ func (iter *BTCTXIterator) GetTxIndex() int {
 }
 
 // Gets the current transaction
-func (iter *BTCTXIterator) GetTx() *btcutil.Tx {
+func (iter *BTCTXIterator) GetTx() *btcjson.TxRawResult {
 	return iter.tx
 }
 
+func (iter *BTCTXIterator) IsCoinBaseTx(tx *btcjson.TxRawResult) bool {
+	// A coin base must only have one transaction input.
+	if len(tx.Vin) != 1 {
+		return false
+	}
+
+	return tx.Vin[0].IsCoinBase()
+}
+
 func (iter *BTCTXIterator) resetTx() {
 	iter.txIndex = -1
 	iter.tx = nil
 }
 
 // Move to the next block
-func (iter *BTCTXIterator) nextBlock() *wire.MsgBlock {
+func (iter *BTCTXIterator) nextBlock() *btcjson.GetBlockVerboseResult {
 	iter.blockIndex++
 	hash, err := iter.client.GetBlockHash(iter.blockIndex)
 	if err != nil {
-		log.Println("Failed to get block hash at", iter.blockIndex)
+		log.Panic("Failed to get block hash at", iter.blockIndex, err)
 	}
-	iter.block, err = iter.client.GetBlock(hash)
+	iter.block, err = iter.client.GetBlockVerboseTx(hash)
 	if err != nil {
-		log.Println("Failed to get block", iter.blockIndex, iter.block)
+		log.Panic("Failed to get block", iter.blockIndex, err)
 	}
 	iter.resetTx()