From bb9bb922733d1f5552f8709f03b1aed325e1d11b Mon Sep 17 00:00:00 2001 From: Daniel Van Der Maden Date: Thu, 9 Jul 2020 23:53:07 -0700 Subject: [PATCH] [test] Update localnet deploy script * Remove unused staking test function * Make config more flexible for specifying BLS keys * Refactor script to functional components for legibility --- test/configs/local-resharding.txt | 47 +++--- test/deploy.sh | 231 ++++++++++++++---------------- 2 files changed, 132 insertions(+), 146 deletions(-) diff --git a/test/configs/local-resharding.txt b/test/configs/local-resharding.txt index 718332f51..bd21bad45 100644 --- a/test/configs/local-resharding.txt +++ b/test/configs/local-resharding.txt @@ -1,25 +1,24 @@ -127.0.0.1 9000 validator one1pdv9lrdwl0rg5vglh4xtyrv3wjk3wsqket7zxy 65f55eb3052f9e9f632b2923be594ba77c55543f5c58ee1454b9cfd658d25e06373b0f7d42a19c84768139ea294f6204 -127.0.0.1 9001 validator one1m6m0ll3q7ljdqgmth2t5j7dfe6stykucpj2nr5 40379eed79ed82bebfb4310894fd33b6a3f8413a78dc4d43b98d0adc9ef69f3285df05eaab9f2ce5f7227f8cb920e809 -127.0.0.1 9002 validator one12fuf7x9rgtdgqg7vgq0962c556m3p7afsxgvll 02c8ff0b88f313717bc3a627d2f8bb172ba3ad3bb9ba3ecb8eed4b7c878653d3d4faf769876c528b73f343967f74a917 -127.0.0.1 9003 validator one16qsd5ant9v94jrs89mruzx62h7ekcfxmduh2rx ee2474f93cba9241562efc7475ac2721ab0899edf8f7f115a656c0c1f9ef8203add678064878d174bb478fa2e6630502 -127.0.0.1 9004 validator one1pf75h0t4am90z8uv3y0dgunfqp4lj8wr3t5rsp e751ec995defe4931273aaebcb2cd14bf37e629c554a57d3f334c37881a34a6188a93e76113c55ef3481da23b7d7ab09 -127.0.0.1 9005 validator one1est2gxcvavmtnzc7mhd73gzadm3xxcv5zczdtw 776f3b8704f4e1092a302a60e84f81e476c212d6f458092b696df420ea19ff84a6179e8e23d090b9297dc041600bc100 -127.0.0.1 9006 validator one1spshr72utf6rwxseaz339j09ed8p6f8ke370zj 2d61379e44a772e5757e27ee2b3874254f56073e6bd226eb8b160371cc3c18b8c4977bd3dcb71fd57dc62bf0e143fd08 -127.0.0.1 9007 validator one1a0x3d6xpmr6f8wsyaxd9v36pytvp48zckswvv9 c4e4708b6cf2a2ceeb59981677e9821eebafc5cf483fb5364a28fa604cc0ce69beeed40f3f03815c9e196fdaec5f1097 -127.0.0.1 9008 validator one1d2rngmem4x2c6zxsjjz29dlah0jzkr0k2n88wc 86dc2fdc2ceec18f6923b99fd86a68405c132e1005cf1df72dca75db0adfaeb53d201d66af37916d61f079f34f21fb96 -127.0.0.1 9009 validator one1658znfwf40epvy7e46cqrmzyy54h4n0qa73nep 49d15743b36334399f9985feb0753430a2b287b2d68b84495bbb15381854cbf01bca9d1d9f4c9c8f18509b2bfa6bd40f -127.0.0.1 9010 validator one1z05g55zamqzfw9qs432n33gycdmyvs38xjemyl 95117937cd8c09acd2dfae847d74041a67834ea88662a7cbed1e170350bc329e53db151e5a0ef3e712e35287ae954818 -127.0.0.1 9011 validator one1ljznytjyn269azvszjlcqvpcj6hjm822yrcp2e 68ae289d73332872ec8d04ac256ca0f5453c88ad392730c5741b6055bc3ec3d086ab03637713a29f459177aaa8340615 - - -127.0.0.1 9100 validator one1a50tun737ulcvwy0yvve0pvu5skq0kjargvhwe 52ecce5f64db21cbe374c9268188f5d2cdd5bec1a3112276a350349860e35fb81f8cfe447a311e0550d961cf25cb988d -127.0.0.1 9101 validator one1uyshu2jgv8w465yc8kkny36thlt2wvel89tcmg a547a9bf6fdde4f4934cde21473748861a3cc0fe8bbb5e57225a29f483b05b72531f002f8187675743d819c955a86100 -127.0.0.1 9102 validator one103q7qe5t2505lypvltkqtddaef5tzfxwsse4z7 678ec9670899bf6af85b877058bea4fc1301a5a3a376987e826e3ca150b80e3eaadffedad0fedfa111576fa76ded980c -127.0.0.1 9103 validator one129r9pj3sk0re76f7zs3qz92rggmdgjhtwge62k 63f479f249c59f0486fda8caa2ffb247209489dae009dfde6144ff38c370230963d360dffd318cfb26c213320e89a512 -127.0.0.1 9104 validator one1d2rngmem4x2c6zxsjjz29dlah0jzkr0k2n88wc 16513c487a6bb76f37219f3c2927a4f281f9dd3fd6ed2e3a64e500de6545cf391dd973cc228d24f9bd01efe94912e714 -127.0.0.1 9105 validator one1658znfwf40epvy7e46cqrmzyy54h4n0qa73nep 576d3c48294e00d6be4a22b07b66a870ddee03052fe48a5abbd180222e5d5a1f8946a78d55b025de21635fd743bbad90 -127.0.0.1 9106 validator one1ghkz3frhske7emk79p7v2afmj4a5t0kmjyt4s5 eca09c1808b729ca56f1b5a6a287c6e1c3ae09e29ccf7efa35453471fcab07d9f73cee249e2b91f5ee44eb9618be3904 -127.0.0.1 9107 validator one1d7jfnr6yraxnrycgaemyktkmhmajhp8kl0yahv f47238daef97d60deedbde5302d05dea5de67608f11f406576e363661f7dcbc4a1385948549b31a6c70f6fde8a391486 -127.0.0.1 9108 validator one1r4zyyjqrulf935a479sgqlpa78kz7zlcg2jfen fc4b9c535ee91f015efff3f32fbb9d32cdd9bfc8a837bb3eee89b8fff653c7af2050a4e147ebe5c7233dc2d5df06ee0a -127.0.0.1 9109 validator one1p7ht2d4kl8ve7a8jxw746yfnx4wnfxtp8jqxwe ca86e551ee42adaaa6477322d7db869d3e203c00d7b86c82ebee629ad79cb6d57b8f3db28336778ec2180e56a8e07296 +127.0.0.1 9000 validator .hmy/65f55eb3052f9e9f632b2923be594ba77c55543f5c58ee1454b9cfd658d25e06373b0f7d42a19c84768139ea294f6204.key one1pdv9lrdwl0rg5vglh4xtyrv3wjk3wsqket7zxy +127.0.0.1 9001 validator .hmy/40379eed79ed82bebfb4310894fd33b6a3f8413a78dc4d43b98d0adc9ef69f3285df05eaab9f2ce5f7227f8cb920e809.key one1m6m0ll3q7ljdqgmth2t5j7dfe6stykucpj2nr5 +127.0.0.1 9002 validator .hmy/02c8ff0b88f313717bc3a627d2f8bb172ba3ad3bb9ba3ecb8eed4b7c878653d3d4faf769876c528b73f343967f74a917.key one12fuf7x9rgtdgqg7vgq0962c556m3p7afsxgvll +127.0.0.1 9003 validator .hmy/ee2474f93cba9241562efc7475ac2721ab0899edf8f7f115a656c0c1f9ef8203add678064878d174bb478fa2e6630502.key one16qsd5ant9v94jrs89mruzx62h7ekcfxmduh2rx +127.0.0.1 9004 validator .hmy/e751ec995defe4931273aaebcb2cd14bf37e629c554a57d3f334c37881a34a6188a93e76113c55ef3481da23b7d7ab09.key one1pf75h0t4am90z8uv3y0dgunfqp4lj8wr3t5rsp +127.0.0.1 9005 validator .hmy/776f3b8704f4e1092a302a60e84f81e476c212d6f458092b696df420ea19ff84a6179e8e23d090b9297dc041600bc100.key one1est2gxcvavmtnzc7mhd73gzadm3xxcv5zczdtw +127.0.0.1 9006 validator .hmy/2d61379e44a772e5757e27ee2b3874254f56073e6bd226eb8b160371cc3c18b8c4977bd3dcb71fd57dc62bf0e143fd08.key one1spshr72utf6rwxseaz339j09ed8p6f8ke370zj +127.0.0.1 9007 validator .hmy/c4e4708b6cf2a2ceeb59981677e9821eebafc5cf483fb5364a28fa604cc0ce69beeed40f3f03815c9e196fdaec5f1097.key one1a0x3d6xpmr6f8wsyaxd9v36pytvp48zckswvv9 +127.0.0.1 9008 validator .hmy/86dc2fdc2ceec18f6923b99fd86a68405c132e1005cf1df72dca75db0adfaeb53d201d66af37916d61f079f34f21fb96.key one1d2rngmem4x2c6zxsjjz29dlah0jzkr0k2n88wc +127.0.0.1 9009 validator .hmy/49d15743b36334399f9985feb0753430a2b287b2d68b84495bbb15381854cbf01bca9d1d9f4c9c8f18509b2bfa6bd40f.key one1658znfwf40epvy7e46cqrmzyy54h4n0qa73nep +127.0.0.1 9010 validator .hmy/95117937cd8c09acd2dfae847d74041a67834ea88662a7cbed1e170350bc329e53db151e5a0ef3e712e35287ae954818.key one1z05g55zamqzfw9qs432n33gycdmyvs38xjemyl +127.0.0.1 9011 validator .hmy/68ae289d73332872ec8d04ac256ca0f5453c88ad392730c5741b6055bc3ec3d086ab03637713a29f459177aaa8340615.key one1ljznytjyn269azvszjlcqvpcj6hjm822yrcp2e 127.0.0.1 9099 explorer + +127.0.0.1 9100 validator .hmy/52ecce5f64db21cbe374c9268188f5d2cdd5bec1a3112276a350349860e35fb81f8cfe447a311e0550d961cf25cb988d.key one1a50tun737ulcvwy0yvve0pvu5skq0kjargvhwe +127.0.0.1 9101 validator .hmy/a547a9bf6fdde4f4934cde21473748861a3cc0fe8bbb5e57225a29f483b05b72531f002f8187675743d819c955a86100.key one1uyshu2jgv8w465yc8kkny36thlt2wvel89tcmg +127.0.0.1 9102 validator .hmy/678ec9670899bf6af85b877058bea4fc1301a5a3a376987e826e3ca150b80e3eaadffedad0fedfa111576fa76ded980c.key one103q7qe5t2505lypvltkqtddaef5tzfxwsse4z7 +127.0.0.1 9103 validator .hmy/63f479f249c59f0486fda8caa2ffb247209489dae009dfde6144ff38c370230963d360dffd318cfb26c213320e89a512.key one129r9pj3sk0re76f7zs3qz92rggmdgjhtwge62k +127.0.0.1 9104 validator .hmy/16513c487a6bb76f37219f3c2927a4f281f9dd3fd6ed2e3a64e500de6545cf391dd973cc228d24f9bd01efe94912e714.key one1d2rngmem4x2c6zxsjjz29dlah0jzkr0k2n88wc +127.0.0.1 9105 validator .hmy/576d3c48294e00d6be4a22b07b66a870ddee03052fe48a5abbd180222e5d5a1f8946a78d55b025de21635fd743bbad90.key one1658znfwf40epvy7e46cqrmzyy54h4n0qa73nep +127.0.0.1 9106 validator .hmy/eca09c1808b729ca56f1b5a6a287c6e1c3ae09e29ccf7efa35453471fcab07d9f73cee249e2b91f5ee44eb9618be3904.key one1ghkz3frhske7emk79p7v2afmj4a5t0kmjyt4s5 +127.0.0.1 9107 validator .hmy/f47238daef97d60deedbde5302d05dea5de67608f11f406576e363661f7dcbc4a1385948549b31a6c70f6fde8a391486.key one1d7jfnr6yraxnrycgaemyktkmhmajhp8kl0yahv +127.0.0.1 9108 validator .hmy/fc4b9c535ee91f015efff3f32fbb9d32cdd9bfc8a837bb3eee89b8fff653c7af2050a4e147ebe5c7233dc2d5df06ee0a.key one1r4zyyjqrulf935a479sgqlpa78kz7zlcg2jfen +127.0.0.1 9109 validator .hmy/ca86e551ee42adaaa6477322d7db869d3e203c00d7b86c82ebee629ad79cb6d57b8f3db28336778ec2180e56a8e07296.key one1p7ht2d4kl8ve7a8jxw746yfnx4wnfxtp8jqxwe \ No newline at end of file diff --git a/test/deploy.sh b/test/deploy.sh index 3aee75aa4..093d5ca7d 100755 --- a/test/deploy.sh +++ b/test/deploy.sh @@ -1,4 +1,5 @@ -#!/bin/bash +#!/usr/bin/env bash +set -eo pipefail unset -v progdir case "${0}" in @@ -8,59 +9,116 @@ esac ROOT="${progdir}/.." USER=$(whoami) - -. "${ROOT}/scripts/setup_bls_build_flags.sh" - -# set -x -set -eo pipefail - -export GO111MODULE=on OS=$(uname -s) -mkdir -p .hmy -if [ -f ".hmy/blspass.txt" ] -then - echo ".hmy/blspass.txt already in local." -else - touch .hmy/blspass.txt -fi +. "${ROOT}/scripts/setup_bls_build_flags.sh" function cleanup() { "${progdir}/kill_node.sh" } -function cleanup_and_result() { - "${ROOT}/test/kill_node.sh" 2> /dev/null || true +function build() { + pushd ${ROOT} + export GO111MODULE=on + if [[ "$OS" = "Darwin" ]]; then + # MacOS doesn't support static build + scripts/go_executable_build.sh -S + else + # Static build on Linux platform + scripts/go_executable_build.sh -s + fi + popd +} + +function setup() { + # Setup blspass file + mkdir -p ${ROOT}/.hmy + if [[ ! -f "${ROOT}/.hmy/blspass.txt" ]]; then + touch "${ROOT}/.hmy/blspass.txt" + fi + + # Kill nodes if any + cleanup + + # Note that the binarys only works on MacOS & Linux + if [[ "${NOBUILD}" != "true" ]]; then + build + fi + + # Create a tmp folder for logs + t=`date +"%Y%m%d-%H%M%S"` + log_folder="${ROOT}/tmp_log/log-$t" + mkdir -p ${log_folder} + LOG_FILE=${log_folder}/r.log } -function debug_staking() { - source "$(go env GOPATH)/src/github.com/harmony-one/harmony/scripts/setup_bls_build_flags.sh" - hmy_gosdk="$(go env GOPATH)/src/github.com/harmony-one/go-sdk" - hmy_bin="${hmy_gosdk}/hmy" - hmy_ops="/tmp/harmony-ops" - keystore="${hmy_ops}/test-automation/api-tests/LocalnetValidatorKeys" - - rm -rf $hmy_ops - git clone https://github.com/harmony-one/harmony-ops.git $hmy_ops - - if [ ! -d "${hmy_gosdk}" ]; then - git clone https://github.com/harmony-one/go-sdk.git $hmy_gosdk - fi - if [ ! -f "${hmy_bin}" ]; then - make -C $hmy_gosdk - fi - - hmy_version=$($hmy_bin version 2>&1 >/dev/null) - if [[ "${hmy_version:32:3}" -lt "135" ]]; then - echo "Aborting staking tests since CLI version is out of date." - return - fi - - python3 -m pip install pyhmy - python3 -m pip install requests - python3 "${hmy_ops}/test-automation/api-tests/test.py" --keystore $keystore \ - --cli_path $hmy_bin --test_dir "${hmy_ops}/test-automation/api-tests/tests/" \ - --rpc_endpoint_src="http://localhost:9500/" --rpc_endpoint_dst="http://localhost:9501/" --ignore_regression_test +function launch_bootnode(){ + echo "launching boot node ..." + ${DRYRUN} ${ROOT}/bin/bootnode -port 19876 > ${log_folder}/bootnode.log 2>&1 | tee -a ${LOG_FILE} & + sleep 1 + BN_MA=$(grep "BN_MA" ${log_folder}/bootnode.log | awk -F\= ' { print $2 } ') + echo "bootnode launched." + " $BN_MA" +} + +function launch_localnet(){ + launch_bootnode + + unset -v base_args + declare -a base_args args + + if ${VERBOSE}; then + verbosity=5 + else + verbosity=3 + fi + + base_args=(-log_folder "${log_folder}" -min_peers "${MIN}" -bootnodes "${BN_MA}" -network_type="$NETWORK" -blspass file:"${ROOT}/.hmy/blspass.txt" -dns=false -verbosity="${verbosity}") + sleep 2 + + # Start nodes + i=-1 + while IFS='' read -r line || [[ -n "$line" ]]; do + i=$((i+1)) + + # Read config for i-th node form config file + IFS=' ' read ip port mode bls_key account <<< $line + args=("${base_args[@]}" -ip "${ip}" -port "${port}" -key "/tmp/${ip}-${port}.key" -db_dir "${ROOT}/db-${ip}-${port}" -broadcast_invalid_tx=true) + if [[ -z "$ip" || -z "$port" ]]; then + echo "skip empty node" + continue + fi + + # Setup BLS key for i-th localnet node + if [[ ! -e "$bls_key" ]]; then + args=("${args[@]}" -blskey_file "BLSKEY") + elif [[ -f "$bls_key" ]]; then + args=("${args[@]}" -blskey_file "${ROOT}/${bls_key}") + elif [[ -d "$bls_key" ]]; then + args=("${args[@]}" -blsfolder "${ROOT}/${bls_key}") + else + echo "skipping unknown node" + continue + fi + + # Setup flags for i-th node based on config + case "${mode}" in + explorer*) + args=("${args[@]}" -node_type=explorer -shard_id=0) + ;; + *archival|archival) + args=("${args[@]}" -is_archival) + ;; + leader*) + args=("${args[@]}" -is_leader) + ;; + client) + continue + ;; + esac + + # Start the node + ${DRYRUN} "${ROOT}/bin/harmony" "${args[@]}" "${extra_args[@]}" 2>&1 | tee -a "${LOG_FILE}" & + done < ${config} } trap cleanup_and_result SIGINT SIGTERM @@ -68,7 +126,7 @@ trap cleanup_and_result SIGINT SIGTERM function usage { local ME=$(basename $0) - cat< $log_folder/bootnode.log 2>&1 | tee -a $LOG_FILE & -sleep 1 -BN_MA=$(grep "BN_MA" $log_folder/bootnode.log | awk -F\= ' { print $2 } ') -echo "bootnode launched." + " $BN_MA" - -unset -v base_args -declare -a base_args args - -if $VERBOSE; then - verbosity=5 -else - verbosity=3 -fi - -base_args=(-log_folder "${log_folder}" -min_peers "${MIN}" -bootnodes "${BN_MA}" -network_type="$NETWORK" -blspass file:.hmy/blspass.txt -dns=false -verbosity="${verbosity}") -sleep 2 - -# Start nodes -i=0 -while IFS='' read -r line || [[ -n "$line" ]]; do - IFS=' ' read ip port mode account blspub <<< $line - args=("${base_args[@]}" -ip "${ip}" -port "${port}" -key "/tmp/${ip}-${port}.key" -db_dir "db-${ip}-${port}") - if [[ -z "$ip" || -z "$port" ]]; then - echo "skip empty node" - continue - fi - - if [ ! -e .hmy/${blspub}.key ]; then - args=("${args[@]}" -blskey_file "BLSKEY") - else - args=("${args[@]}" -blskey_file ".hmy/${blspub}.key") - fi - - case "${mode}" in leader*) args=("${args[@]}" -is_leader);; esac - case "${mode}" in *archival|archival) args=("${args[@]}" -is_archival);; esac - case "${mode}" in explorer*) args=("${args[@]}" -node_type=explorer -shard_id=0);; esac - case "${mode}" in - client) ;; - *) $DRYRUN "${ROOT}/bin/harmony" "${args[@]}" "${extra_args[@]}" 2>&1 | tee -a "${LOG_FILE}" &;; - esac - i=$((i+1)) -done < $config - -sleep $DURATION - -cleanup_and_result +setup +launch_localnet +sleep ${DURATION} +"${ROOT}/test/kill_node.sh" 2> /dev/null || true \ No newline at end of file