Merge pull request #1046 from harmony-ek/node_sh_auto_update

Make node.sh auto-update/restart node

scripts/node.sh

@@ -21,17 +21,6 @@ err() {
    exit "${code}"
 }
 
-function killnode() {
-   local port=$1
-
-   if [ -n "port" ]; then
-      pid=$(/bin/ps -fu $USER | grep "harmony" | grep "$port" | awk '{print $2}')
-      echo "killing node with port: $port"
-      $DRYRUN kill -9 $pid 2> /dev/null
-      echo "node with port: $port is killed"
-   fi
-}
-
 # https://www.linuxjournal.com/content/validating-ip-address-bash-script
 function valid_ip()
 {
@@ -54,10 +43,9 @@ function myip() {
 # get ipv4 address only, right now only support ipv4 addresses
    PUB_IP=$(dig -4 @resolver1.opendns.com ANY myip.opendns.com +short)
    if valid_ip $PUB_IP; then
-      echo MYIP = $PUB_IP
+      msg "public IP address autodetected: $PUB_IP"
    else
-      echo NO valid public IP found: $PUB_IP
-      exit 1
+      err 1 "NO valid public IP found: $PUB_IP"
    fi
 }
 
@@ -89,19 +77,10 @@ function setup_env
    add_env /etc/pam.d/common-session "session required pam_limits.so"
 }
 
-function find_harmony_process
-{
-   unset -v pidfile pid
-   pidfile="harmony-${PUB_IP}.pid"
-   pid=$!
-   echo "${pid}" > "${pidfile}"
-   ps -f -p "${pid}"
-}
-
 ######## main #########
 if [[ $EUID -ne 0 ]]; then
-   echo "This script must be run as root"
-   echo Please use \"sudo $0\"
+   msg "this script must be run as root"
+   msg please use \"sudo $0\"
    exit 1
 fi
 
@@ -114,17 +93,19 @@ usage() {
    exit 64  # EX_USAGE
 }
 
-unset start_clean
+unset start_clean loop
 start_clean=false
+loop=true
 
 unset OPTIND OPTARG opt
 OPTIND=1
-while getopts :c opt
+while getopts :c1 opt
 do
    case "${opt}" in
    '?') usage "unrecognized option -${OPTARG}";;
    ':') usage "missing argument for -${OPTARG}";;
    c) start_clean=true;;
+   1) loop=false;;
    *) err 70 "unhandled option -${OPTARG}";;  # EX_SOFTWARE
    esac
 done
@@ -147,8 +128,6 @@ case $# in
    ;;
 esac
 
-killnode
-
 BUCKET=pub.harmony.one
 OS=$(uname -s)
 REL=drum
@@ -167,11 +146,18 @@ for bin in "${BIN[@]}"; do
    rm -f ${bin}
 done
 
-# download all the binaries
-for bin in "${BIN[@]}"; do
-   curl http://${BUCKET}.s3.amazonaws.com/${FOLDER}${bin} -o ${bin}
-done
-chmod +x harmony
+download_binaries() {
+   local outdir
+   outdir="${1:-.}"
+   mkdir -p "${outdir}"
+   for bin in "${BIN[@]}"; do
+      curl http://${BUCKET}.s3.amazonaws.com/${FOLDER}${bin} -o "${outdir}/${bin}" || return $?
+   done
+   chmod +x "${outdir}/harmony"
+   (cd "${outdir}" && exec openssl sha256 "${BIN[@]}") > "${outdir}/harmony-checksums.txt"
+}
+
+download_binaries || err 69 "initial node software update failed"
 
 NODE_PORT=9000
 PUB_IP=
@@ -200,22 +186,119 @@ then
 fi
 mkdir -p latest
 
-echo "############### Running Harmony Process ###############"
-if [ "$OS" == "Linux" ]; then
-# Run Harmony Node
-   LD_LIBRARY_PATH=$(pwd) ./harmony -bootnodes $BN_MA -ip $PUB_IP -port $NODE_PORT -is_genesis -is_archival -accounts $IDX
-else
-   DYLD_FALLBACK_LIBRARY_PATH=$(pwd) ./harmony -bootnodes $BN_MA -ip $PUB_IP -port $NODE_PORT -is_genesis -is_archival -accounts $IDX
-fi
+unset -v check_update_pid
+
+cleanup() {
+   local trap_sig kill_sig
+
+   trap_sig="${1:-EXIT}"
+
+   kill_sig="${trap_sig}"
+   case "${kill_sig}" in
+   0|EXIT|2|INT) kill_sig=TERM;;
+   esac
+
+   case "${check_update_pid+set}" in
+   set)
+      msg "terminating update checker (pid ${check_update_pid})"
+      kill -${kill_sig} "${check_update_pid}"
+      ;;
+   esac
+}
+
+unset -v trap_sigs trap_sig
+trap_sigs="EXIT HUP INT TERM"
+
+trap_func() {
+   local trap_sig="${1-EXIT}"
+   case "${trap_sig}" in
+   0|EXIT) msg "exiting";;
+   *) msg "received SIG${trap_sig}";;
+   esac
 
-find_harmony_process
-echo
-echo
+   trap - ${trap_sigs}
 
-# echo Please run the following command to inspect the log
-# echo "tail -f harmony-${PUB_IP}.log"
+   cleanup "${trap_sig}"
 
-echo
-echo You may use \"sudo pkill harmony\" to terminate running harmony node program.
+   case "${trap_sig}" in
+   ""|0|EXIT) ;;
+   *) kill -"${trap_sig}" "$$";;
+   esac
+}
+
+for trap_sig in ${trap_sigs}
+do
+   trap "trap_func ${trap_sig}" ${trap_sig}
+done
 
-trap killnode SIGINT SIGTERM
+# Kill the given PID, ensuring that it is a child of this script ($$).
+kill_child() {
+   local pid
+   pid="${1}"
+   case $(($(ps -oppid= -p"${pid}" || :) + 0)) in
+   $$) ;;
+   *) return 1;;
+   esac
+   msg "killing pid ${pid}"
+   kill "${pid}"
+}
+
+# Kill nodes that are direct child of this script (pid $$),
+# i.e. run directly from main loop.
+kill_node() {
+   local pids pid delay
+
+   msg "finding node processes that are our children"
+   pids=$(
+      ps axcwwo "pid=,ppid=,command=" |
+      awk -v me=$$ '$2 == me && $3 == "harmony" { print $1; }'
+   )
+   msg "found node processes: ${pids:-"<none>"}"
+   for pid in ${pids}
+   do
+      delay=0
+      while kill_child ${pid}
+      do
+         sleep ${delay}
+         delay=1
+      done
+      msg "pid ${pid} no longer running"
+   done
+}
+
+{
+   while :
+   do
+      msg "re-downloading binaries in 5m"
+      sleep 300
+      while ! download_binaries staging
+      do
+         msg "staging download failed; retrying in 30s"
+         sleep 30
+      done
+      if diff staging/harmony-checksums.txt harmony-checksums.txt
+      then
+         msg "binaries did not change"
+         continue
+      fi
+      msg "binaries changed; moving from staging into main"
+      (cd staging; exec mv harmony-checksums.txt "${BIN[@]}" ..) || continue
+      msg "binaries updated, killing node to restart"
+      kill_node
+   done
+} > harmony-update.out 2>&1 &
+check_update_pid=$!
+
+while :
+do
+   msg "############### Running Harmony Process ###############"
+   if [ "$OS" == "Linux" ]; then
+   # Run Harmony Node
+      LD_LIBRARY_PATH=$(pwd) ./harmony -bootnodes $BN_MA -ip $PUB_IP -port $NODE_PORT -is_genesis -is_archival -accounts $IDX
+   else
+      DYLD_FALLBACK_LIBRARY_PATH=$(pwd) ./harmony -bootnodes $BN_MA -ip $PUB_IP -port $NODE_PORT -is_genesis -is_archival -accounts $IDX
+   fi || msg "node process finished with status $?"
+   ${loop} || break
+   msg "restarting in 10s..."
+   sleep 10
+done