Merge pull request #234 from harmony-one/leochen

add go.mod for go v1.11

go.mod

@@ -0,0 +1 @@
+module github.com/harmony-one/harmony

test/cal_tps.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+
+function usage
+{
+   ME=$(basename $0)
+   cat<<EOT
+Usage: $ME list_of_leaders list_of_validators
+
+This script calculate the number of consensus and TPS based on leader log and validator log files.
+
+list_of_leaders contains a list of leader log files
+list_of_validators contains a list of validator log files
+
+EOT
+   exit 0
+}
+
+if [ "$#" -ne 2 ]; then
+   usage
+fi
+
+LEADERS=$1
+VALIDATORS=$2
+
+FILES=( $(cat $LEADERS) )
+
+if [ -n "$VALIDATORS" ]; then
+   NUM_VALIDATORS=$(wc -l $VALIDATORS | awk ' { print $1 } ')
+fi
+
+NUM_SHARDS=${#FILES[@]}
+SUM=0
+NUM_CONSENSUS=0
+
+declare -A TPS
+
+for f in "${FILES[@]}"; do
+   leader=$( echo $(basename $f) | cut -f 2 -d\- )
+   num_consensus=$(grep HOORAY $f | wc -l)
+   if [ $num_consensus -gt 0 ]; then
+      avg_tps=$(grep TPS $f | cut -f 2 -d: | cut -f 1 -d , | awk '{s+=$1} END {print s/NR}')
+      printf -v avg_tps_int %.0f $avg_tps
+   else
+      avg_tps=0
+   fi
+   TPS[$leader]="$num_consensus, $avg_tps"
+   NUM_CONSENSUS=$(expr $NUM_CONSENSUS + $num_consensus )
+   SUM=$( expr $SUM + $avg_tps_int )
+done
+
+echo $NUM_SHARDS shards, $NUM_CONSENSUS consensus, $SUM total TPS, $NUM_VALIDATORS nodes
+for t in "${!TPS[@]}"; do
+   echo $t, ${TPS[$t]}
+done

test/configs/local_config.txt

@@ -99,4 +99,4 @@
 127.0.0.1 9099 validator 0
 127.0.0.1 9100 validator 0
 127.0.0.1 9000 leader 0
-127.0.0.1 9999 client 0
+127.0.0.1 19999 client 0

test/configs/local_config1.txt

@@ -9,4 +9,4 @@
 127.0.0.1 9008 validator 0
 127.0.0.1 9009 validator 0
 127.0.0.1 9010 validator 0
-127.0.0.1 9999 client 0
+127.0.0.1 19999 client 0

test/configs/local_config2.txt

@@ -9,4 +9,4 @@
 127.0.0.1 9009 validator 0
 127.0.0.1 9010 validator 0
 127.0.0.1 9000 leader 0
-127.0.0.1 9999 client 0
+127.0.0.1 19999 client 0

test/configs/local_config3.txt

@@ -299,4 +299,4 @@
 127.0.0.1 9299 validator 0
 127.0.0.1 9300 validator 0
 127.0.0.1 9000 leader 0
-127.0.0.1 9999 client 0
+127.0.0.1 19999 client 0

test/configs/local_config4.txt

@@ -10,4 +10,4 @@
 127.0.0.1 9010 validator 1
 127.0.0.1 9001 leader 1
 127.0.0.1 9000 leader 0
-127.0.0.1 9999 client 0
+127.0.0.1 19999 client 0

test/configs/local_config5.txt

@@ -1,3 +1,3 @@
 127.0.0.1 9000 leader 0
 127.0.0.1 9001 validator 0
-127.0.0.1 9999 client 0
+127.0.0.1 19999 client 0

test/configs/local_config_8shards.txt

@@ -96,4 +96,4 @@
 127.0.0.1 9006 leader 6
 127.0.0.1 9007 leader 7
 127.0.0.1 9008 leader 8
-127.0.0.1 9999 client 0
+127.0.0.1 19999 client 0

test/configs/local_config_shards.txt

@@ -20,4 +20,4 @@
 127.0.0.1 9029 validator 1
 127.0.0.1 9000 leader 0
 127.0.0.1 9001 leader 1
-127.0.0.1 9999 client 0
+127.0.0.1 19999 client 0

test/deploy.sh

@@ -1,14 +1,26 @@
 #!/bin/bash
 
 ROOT=$(dirname $0)/..
+USER=$(whoami)
 
+set -x
 set -eo pipefail
 
+function check_result() {
+   find $log_folder -name leader-*.log > $log_folder/all-leaders.txt
+   find $log_folder -name validator-*.log > $log_folder/all-validators.txt
+
+   echo ====== RESULTS ======
+   results=$($ROOT/test/cal_tps.sh $log_folder/all-leaders.txt $log_folder/all-validators.txt)
+   echo $results | tee -a $LOG_FILE
+   echo $results > $log_folder/tps.log
+}
+
 function cleanup() {
    for pid in `/bin/ps -fu $USER| grep "harmony\|txgen\|soldier\|commander\|profiler\|beacon" | grep -v "grep" | grep -v "vi" | awk '{print $2}'`;
    do
        echo 'Killed process: '$pid
-       kill -9 $pid 2> /dev/null
+       $DRYRUN kill -9 $pid 2> /dev/null
    done
 }
 
@@ -18,7 +30,7 @@ function killnode() {
    if [ -n "port" ]; then
       pid=$(/bin/ps -fu $USER | grep "harmony" | grep "$port" | awk '{print $2}')
       echo "killing node with port: $port"
-      kill -9 $pid 2> /dev/null
+      $DRYRUN kill -9 $pid 2> /dev/null
       echo "node with port: $port is killed"
    fi
 }
@@ -38,6 +50,7 @@ USAGE: $ME [OPTIONS] config_file_name
    -m min_peers   minimal number of peers to start consensus (default: $MIN)
    -s shards      number of shards (default: $SHARDS)
    -k nodeport    kill the node with specified port number (default: $KILLPORT)
+   -n             dryrun mode (default: $DRYRUN)
 
 This script will build all the binaries and start harmony and txgen based on the configuration file.
 
@@ -56,8 +69,9 @@ DURATION=90
 MIN=5
 SHARDS=2
 KILLPORT=9004
+DRYRUN=
 
-while getopts "hdtD:m:s:k:" option; do
+while getopts "hdtD:m:s:k:n" option; do
    case $option in
       h) usage ;;
       d) DB='-db_supported' ;;
@@ -66,6 +80,7 @@ while getopts "hdtD:m:s:k:" option; do
       m) MIN=$OPTARG ;;
       s) SHARDS=$OPTARG ;;
       k) KILLPORT=$OPTARG ;;
+      n) DRYRUN=echo ;;
    esac
 done
 
@@ -97,9 +112,10 @@ t=`date +"%Y%m%d-%H%M%S"`
 log_folder="tmp_log/log-$t"
 
 mkdir -p $log_folder
+LOG_FILE=$log_folder/r.log
 
 echo "launching beacon chain ..."
-$ROOT/bin/beacon -numShards $SHARDS > $log_folder/beacon.log 2>&1 &
+$DRYRUN $ROOT/bin/beacon -numShards $SHARDS > $log_folder/beacon.log 2>&1 | tee -a $LOG_FILE &
 sleep 1 #waiting for beaconchain
 
 # Start nodes
@@ -107,7 +123,7 @@ while IFS='' read -r line || [[ -n "$line" ]]; do
   IFS=' ' read ip port mode shardID <<< $line
 	#echo $ip $port $mode
   if [ "$mode" != "client" ]; then
-      $ROOT/bin/harmony -ip $ip -port $port -log_folder $log_folder $DB -min_peers $MIN &
+      $DRYRUN $ROOT/bin/harmony -ip $ip -port $port -log_folder $log_folder $DB -min_peers $MIN 2>&1 | tee -a $LOG_FILE &
       sleep 0.5
   fi
 done < $config
@@ -115,10 +131,14 @@ done < $config
 # Emulate node offline
 (sleep 45; killnode $KILLPORT) &
 
-echo "launching txgen ..."
 if [ "$TXGEN" == "true" ]; then
    echo "launching txgen ..."
-   $ROOT/bin/txgen -log_folder $log_folder -duration $DURATION
+   line=$(grep client $config)
+   IFS=' ' read ip port mode shardID <<< $line
+   if [ "$mode" == "client" ]; then
+      $DRYRUN $ROOT/bin/txgen -log_folder $log_folder -duration $DURATION -ip $ip -port $port 2>&1 | tee -a $LOG_FILE
+   fi
 fi
 
 cleanup
+check_result