From 416cef7a2bff1c42f55ccf524ac48e3be2601d7a Mon Sep 17 00:00:00 2001 From: alok Date: Mon, 18 Jun 2018 21:14:14 -0700 Subject: [PATCH 1/9] branch to consutruct 1 node per instance --- aws-scripts/preprocess_peerlist.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 aws-scripts/preprocess_peerlist.py diff --git a/aws-scripts/preprocess_peerlist.py b/aws-scripts/preprocess_peerlist.py new file mode 100644 index 000000000..faecebf07 --- /dev/null +++ b/aws-scripts/preprocess_peerlist.py @@ -0,0 +1,19 @@ +from collections import namedtuple +import requests +amazon_ipv4_url = "http://169.254.169.254/latest/meta-data/public-ipv4" +if __name__ == "__main__": + current_ip = requests.get(amazon_ipv4_url).text + f = open("all_peerlist.txt",r) + for myline in f: + mylist = myline.strip().split(" ") + ip = mylist[0] + if ip != current_ip: + peerList.append(myline) + f.close() + g = open("global_peerlist.txt") + for myline peerList: + g.write(myline + "\n") + g.close() + + + \ No newline at end of file From 1748662dfbbbc2c89d02c19c9ad3e65c80db18c2 Mon Sep 17 00:00:00 2001 From: alok Date: Mon, 18 Jun 2018 23:15:53 -0700 Subject: [PATCH 2/9] transaction generator on seperate node --- aws-code/transaction_generator.go | 5 ++--- aws-scripts/preprocess_peerlist.py | 20 +++++++++++--------- aws-scripts/run_instances.sh | 2 +- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/aws-code/transaction_generator.go b/aws-code/transaction_generator.go index 786d47b9b..748c29989 100644 --- a/aws-code/transaction_generator.go +++ b/aws-code/transaction_generator.go @@ -63,10 +63,10 @@ func readConfigFile(configFile string) [][]string { } func main() { - configFile := flag.String("config_file", "local_config.txt", "file containing all ip addresses and config") + configFile := flag.String("config_file", "global_nodes.txt", "file containing all ip addresses and config") flag.Parse() config := readConfigFile(*configFile) - + time.Sleep(totalTime) //Sleep Time to let all instances come up start := time.Now() totalTime := 60.0 txs := make([]blockchain.Transaction, 10) @@ -79,7 +79,6 @@ func main() { } for i := range txs { txs[i] = newRandTransaction() - } msg := node.ConstructTransactionListMessage(txs) log.Printf("[Generator] Sending txs to %d leader[s]\n", len(leaders)) diff --git a/aws-scripts/preprocess_peerlist.py b/aws-scripts/preprocess_peerlist.py index faecebf07..3c2db0d3f 100644 --- a/aws-scripts/preprocess_peerlist.py +++ b/aws-scripts/preprocess_peerlist.py @@ -1,19 +1,21 @@ -from collections import namedtuple import requests amazon_ipv4_url = "http://169.254.169.254/latest/meta-data/public-ipv4" if __name__ == "__main__": current_ip = requests.get(amazon_ipv4_url).text - f = open("all_peerlist.txt",r) + f = open("global_nodes.txt",r) for myline in f: - mylist = myline.strip().split(" ") + mylist = myline.split(" ") ip = mylist[0] - if ip != current_ip: + node = mylist[2] + if ip != current_ip or node != "transaction": peerList.append(myline) + else: + h = open("isTransaction.txt",w) + h.write("I am just a transaction generator node") + h.close() f.close() - g = open("global_peerlist.txt") + g = open("global_peerlist.txt",w) for myline peerList: - g.write(myline + "\n") - g.close() - - + g.write(myline) + g.close() \ No newline at end of file diff --git a/aws-scripts/run_instances.sh b/aws-scripts/run_instances.sh index bd4b7f59f..dedf1d741 100755 --- a/aws-scripts/run_instances.sh +++ b/aws-scripts/run_instances.sh @@ -1,3 +1,3 @@ #!/bin/bash -x cd /home/ec2-user/projects/src/harmony-benchmark -./deploy_linux.sh local_config2.txt \ No newline at end of file +./deploy_one_instance.sh global_peerlist.txt \ No newline at end of file From ba9a9bda5137f0acf16f950568879bcb966ad35e Mon Sep 17 00:00:00 2001 From: alok Date: Mon, 18 Jun 2018 23:22:59 -0700 Subject: [PATCH 3/9] moving deploy to the right place --- deploy_one_instance.sh | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 deploy_one_instance.sh diff --git a/deploy_one_instance.sh b/deploy_one_instance.sh new file mode 100644 index 000000000..d05205bea --- /dev/null +++ b/deploy_one_instance.sh @@ -0,0 +1,22 @@ +#!/bin/bash -x +##The commented suffix is for linux +##Reference: https://github.com/Zilliqa/Zilliqa/blob/master/tests/Node/test_node_simple.sh +sudo sysctl net.core.somaxconn=1024 +sudo sysctl net.core.netdev_max_backlog=65536; +sudo sysctl net.ipv4.tcp_tw_reuse=1; +sudo sysctl -w net.ipv4.tcp_rmem='65536 873800 1534217728'; +sudo sysctl -w net.ipv4.tcp_wmem='65536 873800 1534217728'; +sudo sysctl -w net.ipv4.tcp_mem='65536 873800 1534217728'; + +./kill_node.sh +source ~/.bash_profile +python aws-scripts/preprocess_peerlist.py +FILE='isTransaction.txt' +config=$1 +if [ -f $FILE ]; then + go run ./aws-code/transaction_generator.go -config_file $config +else: + go run ./benchmark_main.go -config_file $config& + + + From 2228801e10a7aed18d1af27f2e0694d158254bdc Mon Sep 17 00:00:00 2001 From: alok Date: Mon, 18 Jun 2018 23:25:06 -0700 Subject: [PATCH 4/9] Setting right permissions for deploy --- deploy_one_instance.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 deploy_one_instance.sh diff --git a/deploy_one_instance.sh b/deploy_one_instance.sh old mode 100644 new mode 100755 From 5185e0aff1716e0cf8abba12846ed90ec2835784 Mon Sep 17 00:00:00 2001 From: alok Date: Mon, 18 Jun 2018 23:47:17 -0700 Subject: [PATCH 5/9] typos/bad code removed --- aws-code/transaction_generator.go | 2 +- aws-scripts/preprocess_peerlist.py | 5 +++-- deploy_one_instance.sh | 6 ++---- global_nodes.txt | 3 +++ 4 files changed, 9 insertions(+), 7 deletions(-) create mode 100644 global_nodes.txt diff --git a/aws-code/transaction_generator.go b/aws-code/transaction_generator.go index 748c29989..172d3fb56 100644 --- a/aws-code/transaction_generator.go +++ b/aws-code/transaction_generator.go @@ -66,9 +66,9 @@ func main() { configFile := flag.String("config_file", "global_nodes.txt", "file containing all ip addresses and config") flag.Parse() config := readConfigFile(*configFile) + totalTime := 60.0 time.Sleep(totalTime) //Sleep Time to let all instances come up start := time.Now() - totalTime := 60.0 txs := make([]blockchain.Transaction, 10) leaders := getLeaders(&config) for true { diff --git a/aws-scripts/preprocess_peerlist.py b/aws-scripts/preprocess_peerlist.py index 3c2db0d3f..9dd8ed70b 100644 --- a/aws-scripts/preprocess_peerlist.py +++ b/aws-scripts/preprocess_peerlist.py @@ -3,11 +3,12 @@ amazon_ipv4_url = "http://169.254.169.254/latest/meta-data/public-ipv4" if __name__ == "__main__": current_ip = requests.get(amazon_ipv4_url).text f = open("global_nodes.txt",r) + peerList = [] for myline in f: mylist = myline.split(" ") ip = mylist[0] node = mylist[2] - if ip != current_ip or node != "transaction": + if node != "transaction": peerList.append(myline) else: h = open("isTransaction.txt",w) @@ -15,7 +16,7 @@ if __name__ == "__main__": h.close() f.close() g = open("global_peerlist.txt",w) - for myline peerList: + for myline in peerList: g.write(myline) g.close() \ No newline at end of file diff --git a/deploy_one_instance.sh b/deploy_one_instance.sh index d05205bea..fd5131045 100755 --- a/deploy_one_instance.sh +++ b/deploy_one_instance.sh @@ -15,8 +15,6 @@ FILE='isTransaction.txt' config=$1 if [ -f $FILE ]; then go run ./aws-code/transaction_generator.go -config_file $config -else: +else go run ./benchmark_main.go -config_file $config& - - - +fi \ No newline at end of file diff --git a/global_nodes.txt b/global_nodes.txt new file mode 100644 index 000000000..9826ae250 --- /dev/null +++ b/global_nodes.txt @@ -0,0 +1,3 @@ +35.160.197.62 9000 validator 0 +54.68.213.133 9000 leader 0 +34.216.209.90 9000 transaction 0 \ No newline at end of file From aadf3f9b34b9b4b84841dce554814f05f120aed1 Mon Sep 17 00:00:00 2001 From: alok Date: Mon, 18 Jun 2018 23:54:20 -0700 Subject: [PATCH 6/9] more typos --- aws-scripts/preprocess_peerlist.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/aws-scripts/preprocess_peerlist.py b/aws-scripts/preprocess_peerlist.py index 9dd8ed70b..eda1e7fe3 100644 --- a/aws-scripts/preprocess_peerlist.py +++ b/aws-scripts/preprocess_peerlist.py @@ -2,7 +2,7 @@ import requests amazon_ipv4_url = "http://169.254.169.254/latest/meta-data/public-ipv4" if __name__ == "__main__": current_ip = requests.get(amazon_ipv4_url).text - f = open("global_nodes.txt",r) + f = open("global_nodes.txt","r") peerList = [] for myline in f: mylist = myline.split(" ") @@ -11,12 +11,11 @@ if __name__ == "__main__": if node != "transaction": peerList.append(myline) else: - h = open("isTransaction.txt",w) + h = open("isTransaction.txt","w") h.write("I am just a transaction generator node") h.close() f.close() - g = open("global_peerlist.txt",w) + g = open("global_peerlist.txt","w") for myline in peerList: g.write(myline) g.close() - \ No newline at end of file From bafa2c6ce2320418c2557b63dcea455b73b0c0b8 Mon Sep 17 00:00:00 2001 From: alok Date: Wed, 20 Jun 2018 21:35:19 -0700 Subject: [PATCH 7/9] transaction generator --- aws-code/transaction_generator.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/aws-code/transaction_generator.go b/aws-code/transaction_generator.go index 786d47b9b..66113bfd5 100644 --- a/aws-code/transaction_generator.go +++ b/aws-code/transaction_generator.go @@ -66,7 +66,6 @@ func main() { configFile := flag.String("config_file", "local_config.txt", "file containing all ip addresses and config") flag.Parse() config := readConfigFile(*configFile) - start := time.Now() totalTime := 60.0 txs := make([]blockchain.Transaction, 10) @@ -79,7 +78,6 @@ func main() { } for i := range txs { txs[i] = newRandTransaction() - } msg := node.ConstructTransactionListMessage(txs) log.Printf("[Generator] Sending txs to %d leader[s]\n", len(leaders)) From 4d2d0f1f870e50051033f861ef05a266a7613e93 Mon Sep 17 00:00:00 2001 From: alok Date: Wed, 20 Jun 2018 22:24:55 -0700 Subject: [PATCH 8/9] vscode changes not pushed --- aws-code/transaction_generator.go | 24 +++++------------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/aws-code/transaction_generator.go b/aws-code/transaction_generator.go index 991b5e99c..fabf60788 100644 --- a/aws-code/transaction_generator.go +++ b/aws-code/transaction_generator.go @@ -2,19 +2,19 @@ package main import ( "bufio" + "encoding/hex" "flag" "fmt" "harmony-benchmark/blockchain" + "harmony-benchmark/consensus" "harmony-benchmark/log" "harmony-benchmark/node" "harmony-benchmark/p2p" "math/rand" "os" + "strconv" "strings" "time" - "harmony-benchmark/consensus" - "encoding/hex" - "strconv" ) // Get numTxs number of Fake transactions based on the existing UtxoPool. @@ -50,7 +50,7 @@ func getNewFakeTransactions(dataNode *node.Node, numTxs int) []*blockchain.Trans for index, value := range utxoMap { countAll++ if rand.Intn(100) <= 20 { // 20% sample rate to select UTXO to use for new transactions - // Spend the money of current UTXO to a random address in [1 - 1000] + // Spend the money of current UTXO to a random address in [1 - 1000] txin := blockchain.TXInput{txId, index, address} txout := blockchain.TXOutput{value, strconv.Itoa(rand.Intn(1000))} tx := blockchain.Transaction{[32]byte{}, []blockchain.TXInput{txin}, []blockchain.TXOutput{txout}} @@ -110,7 +110,6 @@ func readConfigFile(configFile string) [][]string { } func main() { -<<<<<<< HEAD // Setup a stdout logger h := log.CallerFileHandler(log.StdoutHandler) log.Root().SetHandler(h) @@ -119,28 +118,15 @@ func main() { numTxsPerBatch := flag.Int("num_txs_per_batch", 1000, "number of transactions to send per message") flag.Parse() config := readConfigFile(*configFile) -<<<<<<< HEAD -======= leaders := getLeaders(&config) // Testing node to mirror the node data in consensus dataNode := node.NewNode(&consensus.Consensus{}) dataNode.AddMoreFakeTransactions() ->>>>>>> 31f111ed5d49c01a9c3a20e92e053016c05ebafc start := time.Now() - totalTime := 60.0 + totalTime := 600.0 time.Sleep(3 * time.Second) // wait for nodes to be ready -======= - configFile := flag.String("config_file", "global_nodes.txt", "file containing all ip addresses and config") - flag.Parse() - config := readConfigFile(*configFile) - totalTime := 60.0 - time.Sleep(totalTime) //Sleep Time to let all instances come up - start := time.Now() - txs := make([]blockchain.Transaction, 10) - leaders := getLeaders(&config) ->>>>>>> aadf3f9b34b9b4b84841dce554814f05f120aed1 for true { t := time.Now() if t.Sub(start).Seconds() >= totalTime { From 50ca9f2a8ceba86e128d5ae5e4820ca08bb96c5b Mon Sep 17 00:00:00 2001 From: alok Date: Wed, 20 Jun 2018 22:28:00 -0700 Subject: [PATCH 9/9] move to merge into master --- aws-scripts/run_instances.sh | 2 +- global_nodes.txt | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/aws-scripts/run_instances.sh b/aws-scripts/run_instances.sh index dedf1d741..fd79e6d26 100755 --- a/aws-scripts/run_instances.sh +++ b/aws-scripts/run_instances.sh @@ -1,3 +1,3 @@ #!/bin/bash -x cd /home/ec2-user/projects/src/harmony-benchmark -./deploy_one_instance.sh global_peerlist.txt \ No newline at end of file +./deploy_one_instance.sh global_nodes.txt \ No newline at end of file diff --git a/global_nodes.txt b/global_nodes.txt index 9826ae250..4a6a3c34e 100644 --- a/global_nodes.txt +++ b/global_nodes.txt @@ -1,3 +1,3 @@ -35.160.197.62 9000 validator 0 -54.68.213.133 9000 leader 0 -34.216.209.90 9000 transaction 0 \ No newline at end of file +34.219.57.67 9000 validator 0 +52.43.81.62 9000 leader 0 +54.201.72.153 9000 transaction 0 \ No newline at end of file