diff --git a/.travis.yml b/.travis.yml index e3d9a2066..8751e5009 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,14 @@ +sudo: required +services: + - docker language: go go: - 1.14 go_import_path: github.com/harmony-one/harmony +env: + - TEST="go build -v ./..." + - TEST="./scripts/travis_checker.sh" + - TEST="./scripts/travis_node_checker.sh" install: - export GO111MODULE=on - export GOPATH=$HOME/gopath @@ -21,11 +28,10 @@ install: - cd ../harmony - go get -v ./... - scripts/install_build_tools.sh + - chmod +x scripts/travis_checker.sh + - chmod +x scripts/travis_node_checker.sh script: - - go build -v ./... - - ./scripts/travis_checker.sh -notifications: - slack: harmonyone:gggCd1QQopsQAW8JYgBWiH7M + - ${TEST} after_success: - wget https://raw.githubusercontent.com/DiscordHooks/travis-ci-discord-webhook/master/send.sh - chmod +x send.sh diff --git a/Dockerfile b/Dockerfile index f84d401d9..1038df164 100644 --- a/Dockerfile +++ b/Dockerfile @@ -74,7 +74,7 @@ WORKDIR ${HMY_PATH}/harmony RUN eval "$(~/bin/gimme ${GIMME_GO_VERSION})" ; scripts/install_build_tools.sh -RUN eval "$(~/bin/gimme ${GIMME_GO_VERSION})" ; scripts/go_executable_build.sh +RUN eval "$(~/bin/gimme ${GIMME_GO_VERSION})" ; scripts/go_executable_build.sh -S RUN cd ${HMY_PATH}/go-sdk && make -j8 && cp hmy /root/bin diff --git a/Makefile b/Makefile index 450911a03..0f62c7238 100644 --- a/Makefile +++ b/Makefile @@ -6,28 +6,64 @@ export LIBRARY_PATH:=$(LD_LIBRARY_PATH) export DYLD_FALLBACK_LIBRARY_PATH:=$(LD_LIBRARY_PATH) export GO111MODULE:=on -.PHONY: all libs exe test +.PHONY: all help libs exe race trace-pointer debug debug-kill test test-go test-api test-api-attach linux_static all: libs - ./scripts/go_executable_build.sh -S + bash ./scripts/go_executable_build.sh -S + +help: + @echo "all - build the harmony binary & bootnode along with the MCL & BLS libs (if necessary)" + @echo "libs - build only the MCL & BLS libs (if necessary) " + @echo "exe - build the harmony binary & bootnode" + @echo "race - build the harmony binary & bootnode with race condition checks" + @echo "trace-pointer - build the harmony binary & bootnode with pointer analysis" + @echo "debug - start a localnet with 2 shards (s0 rpc endpoint = localhost:9599; s1 rpc endpoint = localhost:9598)" + @echo "debug-kill - force kill the localnet" + @echo "clean - remove node files & logs created by localnet" + @echo "test - run the entire test suite (go test & Node API test)" + @echo "test-go - run the go test (with go lint, fmt, imports, mod, and generate checks)" + @echo "test-api - run the Node API test" + @echo "test-api-attach - attach onto the Node API testing docker container for inspection" + @echo "linux_static - static build the harmony binary & bootnode along with the MCL & BLS libs (for linux)" libs: make -C $(TOP)/mcl -j8 make -C $(TOP)/bls BLS_SWAP_G=1 -j8 exe: - ./scripts/go_executable_build.sh -S + bash ./scripts/go_executable_build.sh -S race: - ./scripts/go_executable_build.sh -r + bash ./scripts/go_executable_build.sh -r trace-pointer: - ./scripts/go_executable_build.sh -t + bash ./scripts/go_executable_build.sh -t + +debug: + bash ./test/debug.sh + +debug-kill: + bash ./test/kill_node.sh + +clean: + rm -rf ./tmp_log* + rm -rf ./.dht* + rm -rf ./db-* + rm -f ./*.rlp test: - ./test/debug.sh + bash ./test/all.sh + +test-go: + bash ./test/go.sh + +test-api: + bash ./test/api.sh run + +test-api-attach: + bash ./test/api.sh attach linux_static: make -C $(TOP)/mcl -j8 make -C $(TOP)/bls minimised_static BLS_SWAP_G=1 -j8 - ./scripts/go_executable_build.sh -s + bash ./scripts/go_executable_build.sh -s diff --git a/README.md b/README.md index fa45c6951..64d8940c0 100644 --- a/README.md +++ b/README.md @@ -1,166 +1,109 @@ # Harmony [![Build Status](https://travis-ci.com/harmony-one/harmony.svg?token=DnoYvYiTAk7pqTo9XsTi&branch=master)](https://travis-ci.com/harmony-one/harmony) -![gopherbadger-tag-do-not-edit](https://img.shields.io/badge/Go%20Coverage-45%25-brightgreen.svg?longCache=true&style=flat) -![Discord](https://img.shields.io/discord/532383335348043777.svg) -[![Coverage Status](https://coveralls.io/repos/github/harmony-one/harmony/badge.svg?branch=master)](https://coveralls.io/github/harmony-one/harmony?branch=master) +![gopherbadger-tag-do-not-edit](https://img.shields.io/badge/Go%20Coverage-45%25-brightgreen.svg?longCache=true&style=flat) +![Discord](https://img.shields.io/discord/532383335348043777.svg) +[![Coverage Status](https://coveralls.io/repos/github/harmony-one/harmony/badge.svg?branch=main)](https://coveralls.io/github/harmony-one/harmony?branch=master) [![Go Report Card](https://goreportcard.com/badge/github.com/harmony-one/harmony)](https://goreportcard.com/report/github.com/harmony-one/harmony) -# Play with the blockchain +## General Documentation -## Running the harmony blockchain with docker - -The easiest way to run our blockchain locally is to use docker. -If you are on `OS X`, then install docker via `brew`. - -``` -$ brew cask install docker -$ open /Applications/Docker.app -``` +https://docs.harmony.one -then you can run +## API Guide -``` -$ docker run -it harmonyone/main:stable /bin/bash -``` +http://api.hmny.io/ -You'll be in this current repository when the shell opens up, now let's catch up on any missing -commits (all the following commands assume you are in the running docker container) +## Requirements -### Bleeding edge master +1. **GMP and OpenSSL** +On macOS: +```bash +brew install gmp +brew install openssl ``` -$ git fetch -$ git reset --hard origin/master -``` - -### Mainnet release intended branch - +On Linux (Ubuntu) +```bash +sudo apt install glibc-static gmp-devel gmp-static openssl-libs openssl-static gcc-c++ ``` -$ git fetch -$ git checkout t3 -$ git reset --hard origin/t3 +On Linux (Cent OS / Amazon Linux 2) +```bash +sudo yum install glibc-static gmp-devel gmp-static openssl-libs openssl-static gcc-c++ ``` +2. **Docker** (for testing) -And now run the local blockchain - -``` -$ test/debug.sh +On macOS: +```bash +brew cask install docker +open /Applications/Docker.app ``` +On Linux, reference official documentation [here](https://docs.docker.com/engine/install/). +3. **Go 1.14.1** +4. **Bash 4+** -## Using our hmy command line tool +For macOS, you can reference this [guide](http://tldrdevnotes.com/bash-upgrade-3-4-macos). For Linux, you can reference this [guide](https://fossbytes.com/installing-gnu-bash-4-4-linux-distros/). -Assuming that you got `test/debug.sh` running in your docker container, we can interact with -the running blockchain using our `hmy` command line tool. Its part of our -[go-sdk](https://github.com/harmony-one/go-sdk) repo, but our docker container already has it as -well and built. +## Dev Environment -Find your running container's ID, here's an example +**Most repos from [harmony-one](https://github.com/harmony-one) assumes the GOPATH convention. More information [here](https://github.com/golang/go/wiki/GOPATH).** -``` -$ docker container ls -CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES -62572a199bac harmonyone/main:stable "/bin/bash" 2 minutes ago Up 2 minutes awesome_cohen -``` - -now lets get our second shell in the running container: - -``` -$ docker exec -it 62572a199bac /bin/bash -``` - -The container already comes with a prebuilt `hmy` that you can invoke anywhere, but lets go ahead -and build the latest version. +### First Install +Clone and set up all of the repos with the following set of commands: +1. Create the appropriate directories: +```bash +mkdir -p $(go env GOPATH)/src/github.com/harmony-one +cd $(go env GOPATH)/src/github.com/harmony-one ``` -$ cd ../go-sdk -$ git fetch -$ git reset --hard origin/master -$ make -$ ./hmy version -Harmony (C) 2020. hmy, version v211-698b282 (@harmony.one 2020-01-17T00:58:51+0000) -``` - -Then checkout `./hmy cookbook` for example usages. The majority of commands output legal `JSON`, -here is one example: +> If you get 'unknown command' or something along those lines, make sure to install [golang](https://golang.org/doc/install) first. +2. Clone this repo & dependent repos. +```bash +git clone git@github.com:harmony-one/mcl.git +git clone git@github.com:harmony-one/bls.git +git clone git@github.com:harmony-one/harmony.git +cd harmony ``` -root@62572a199bac:~/go/src/github.com/harmony-one/go-sdk# ./hmy blockchain latest-header -{ - "id": "0", - "jsonrpc": "2.0", - "result": { - "blockHash": "0x34a8b155f90b8fc22342fc8b5d1c969ed836a2f666c506e4017b570dc337e88c", - "blockNumber": 0, - "epoch": 0, - "lastCommitBitmap": "", - "lastCommitSig": "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "leader": "one1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqquzw7vz", - "shardID": 0, - "timestamp": "2019-06-28 15:00:00 +0000 UTC", - "unixtime": 1561734000, - "viewID": 0 - } -} -``` - -## Developing in the container - -The docker container is a full harmony development environment and comes with emacs, vim, ag, tig -and other creature comforts. Most importantly, it already has the go environment with our C/C++ -based library dependencies (`libbls` and `mcl`) setup correctly for you. You can also change go -versions easily, an example: +3. Build the harmony binary & dependent libs ``` -$ eval $(gimme 1.14.1) +make ``` +> Run `bash scripts/install_build_tools.sh` to ensure build tools are of correct versions. -Note that changing the go version might mean that dependencies won't work out right when trying to -run `test/debug.sh` again, get the correct environment again easily by `exec bash`. +## Dev Docker Container -## Installation Requirements for directly on your MacOS X - -GMP and OpenSSL +Included in this repo is a Dockerfile that has a full harmony development environment and +comes with emacs, vim, ag, tig and other creature comforts. Most importantly, it already has the go environment +with our C/C++ based library dependencies (`libbls` and `mcl`) set up correctly for you. +You can build the docker container for yourself with the following commands: ```bash -brew install gmp -brew install openssl - +cd $(go env GOPATH)/src/github.com/harmony-one/harmony +make clean +docker build -t harmony . ``` -## On Linux Box (assuming Amazon Linux 2 or Cent OS) - +Then you can start your docker container with the following command: ```bash -sudo yum install glibc-static gmp-devel gmp-static openssl-libs openssl-static gcc-c++ +docker rm harmony # Remove old docker container +docker run --name harmony -it -v "$(go env GOPATH)/src/github.com/harmony-one/harmony:/root/go/src/github.com/harmony-one/harmony" harmony /bin/bash ``` +> Note that the harmony repo will be shared between your docker container and your host machine. However, everything else in the docker container will be ephemeral. -## Dev Environment Setup - -The required go version is: **go1.14.1** - +If you need to open another shell, just do: ```bash -export GOPATH=$HOME/ - -mkdir -p $HOME//src/github.com/harmony-one - -cd $HOME//src/github.com/harmony-one - -git clone git@github.com:harmony-one/mcl.git -git clone git@github.com:harmony-one/bls.git -git clone git@github.com:harmony-one/harmony.git - -cd harmony - -make - +docker exec -it harmony /bin/bash ``` -Note: make sure to run `scripts/install_build_tools.sh`to make sure build tools are of correct versions. +Learn more about docker [here](https://docker-curriculum.com/). ## Build -If you want to bypass the Makefile: +The `make` command should automatically build the Harmony binary & all dependent libs. +However, if you wish to bypass the Makefile, first export the build flags: ```bash export CGO_CFLAGS="-I$GOPATH/src/github.com/harmony-one/bls/include -I$GOPATH/src/github.com/harmony-one/mcl/include -I/usr/local/opt/openssl/include" export CGO_LDFLAGS="-L$GOPATH/src/github.com/harmony-one/bls/lib -L/usr/local/opt/openssl/lib" @@ -170,59 +113,67 @@ export DYLD_FALLBACK_LIBRARY_PATH=$LD_LIBRARY_PATH export GO111MODULE=on ``` -Note : Some of our scripts require bash 4.x support, please [install bash 4.x](http://tldrdevnotes.com/bash-upgrade-3-4-macos) on MacOS X. - -### Build all executables - -You can run the script `./scripts/go_executable_build.sh` to build all the executables. - -### Build individual executables - -Harmony server / main node: - +Then you can build all executables with the following command: ```bash -./scripts/go_executable_build.sh harmony - +bash ./scripts/go_executable_build.sh -S ``` +> Reference `bash ./scripts/go_executable_build.sh -h` for more build options -### Harmony docs and guides - -https://docs.harmony.one - -### API guides - -https://docs.harmony.one/home/developers/api - -### Running locally - -The debug.sh script calls test/deploy.sh script to create a local environment of Harmony blockchain devnet based on the configuration file. -The configuration file configures number of nodes and their IP/Port. -The script starts 2 shards and 7 nodes in each shard. +## Debugging +One can start a local network (a.k.a localnet) with your current code using the following command: ```bash -./test/debug.sh +make debug ``` +> This localnet has 2 shards, with 11 nodes on shard 0 (+1 explorer node) and 10 nodes on shard 0 (+1 explorer node). +> +> The shard 0 endpoint will be on the explorer at `http://localhost:9599`. The shard 1 endpoint will be on the explorer at `http://localhost:9598`. +> +> You can view the localnet configuration at `/test/configs/local-resharding.txt`. The fields for the config are (space-delimited & in order) `ip`, `port`, `mode`, `bls_pub_key`, and `shard` (optional). -### Terminate the local blockchain - +One can force kill the local network with the following command: ```bash -./test/kill_nodes.sh +make debug-kill ``` +> You can view all make commands with `make help` ## Testing -Make sure you use the following command and make sure everything passed before submitting your code. +To keep things consistent, we have a docker image to run all tests. **These are the same tests ran on the pull request checks**. + +### Go tests +To run this test do: +```bash +make test-go +``` +This test runs the go tests along with go lint, go fmt, go imports, go mod, and go generate checks. + +### API tests +To run this test do: +```bash +make test-api +``` +This test starts a localnet (within the Docker container), **ensures it reaches consensus**, and runs a series of tests to ensure correct Node API behavior. +This test also acts as a preliminary integration test (more through tests are done on the testnets). +> The tests ran by this command can be found [here](https://github.com/harmony-one/harmony-test/tree/master/localnet). +If you wish to debug further with the localnet after the tests are done, open a new shell and run: ```bash -./test/test_before_submit.sh +make test-api-attach ``` +> This will open a shell in the docker container that is running the Node API tests. +> +> Note that the docker container has the [Harmony CLI](https://docs.harmony.one/home/wallets/harmony-cli) on path, +> therefore you can use that to debug if needed. For example, one could do `hmy blockchain latest-headers` to check +> the current block height of localnet. Reference the documentation for the CLI [here](https://docs.harmony.one/home/wallets/harmony-cli) +> for more details & commands. ## License Harmony is licensed under the MIT License. See [`LICENSE`](LICENSE) file for the terms and conditions. -Harmony includes third-party open source code. In general, a source subtree +Harmony includes third-party open-source code. In general, a source subtree with a `LICENSE` or `COPYRIGHT` file is from a third party, and our modifications thereto are licensed under the same third-party open source license. @@ -237,7 +188,7 @@ See [`CONTRIBUTING`](CONTRIBUTING.md) for details. ## Development Status -### Features Done +### Finished Features - Fully sharded network with beacon chain and shard chains - Sharded P2P network and P2P gossiping @@ -250,9 +201,9 @@ See [`CONTRIBUTING`](CONTRIBUTING.md) for details. - EPoS staking mechanism - Kademlia routing -### Features Planned after Mainnet Open Staking +### Planned Features - Resharding - Integration with WASM - Fast state synchronization -- Auditable privacy asset using ZK proof +- Auditable privacy asset using ZK proof \ No newline at end of file diff --git a/scripts/travis_checker.sh b/scripts/travis_checker.sh index 16e819e6d..85e771f78 100755 --- a/scripts/travis_checker.sh +++ b/scripts/travis_checker.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash unset -v ok tmpdir gomod_diff_output goimports_output golint_output progdir ok=true diff --git a/scripts/travis_node_checker.sh b/scripts/travis_node_checker.sh new file mode 100644 index 000000000..2713f10bf --- /dev/null +++ b/scripts/travis_node_checker.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -e +OS=$(uname -s) +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +echo "$(pwd)" +docker pull harmonyone/localnet-test +docker run -v "$DIR/../:/go/src/github.com/harmony-one/harmony" harmonyone/localnet-test -n \ No newline at end of file diff --git a/test/all.sh b/test/all.sh new file mode 100644 index 000000000..cda933e3a --- /dev/null +++ b/test/all.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +bash "$DIR/kill_node.sh" +docker pull harmonyone/localnet-test +docker run -it --expose 9000-9999 -v "$DIR/../:/go/src/github.com/harmony-one/harmony" harmonyone/localnet-test \ No newline at end of file diff --git a/test/api.sh b/test/api.sh new file mode 100755 index 000000000..1dd47dc20 --- /dev/null +++ b/test/api.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +bash "$DIR/kill_node.sh" + +docker_name="harmony-localnet-test" + +case ${1} in +run) + docker pull harmonyone/localnet-test + docker rm "$docker_name" + docker run -it --name "$docker_name" --expose 9000-9999 -v "$DIR/../:/go/src/github.com/harmony-one/harmony" harmonyone/localnet-test -n -k + ;; +attach) + docker exec -it "$docker_name" /bin/bash + ;; +*) + echo " +Node API tests + +Param: Help: +run Run the Node API tests +attach Attach onto the Node API testing docker image for inspection +" + exit 0 + ;; +esac + diff --git a/test/configs/local-resharding.txt b/test/configs/local-resharding.txt index 718332f51..ba9fa717a 100644 --- a/test/configs/local-resharding.txt +++ b/test/configs/local-resharding.txt @@ -1,25 +1,25 @@ -127.0.0.1 9000 validator one1pdv9lrdwl0rg5vglh4xtyrv3wjk3wsqket7zxy 65f55eb3052f9e9f632b2923be594ba77c55543f5c58ee1454b9cfd658d25e06373b0f7d42a19c84768139ea294f6204 -127.0.0.1 9001 validator one1m6m0ll3q7ljdqgmth2t5j7dfe6stykucpj2nr5 40379eed79ed82bebfb4310894fd33b6a3f8413a78dc4d43b98d0adc9ef69f3285df05eaab9f2ce5f7227f8cb920e809 -127.0.0.1 9002 validator one12fuf7x9rgtdgqg7vgq0962c556m3p7afsxgvll 02c8ff0b88f313717bc3a627d2f8bb172ba3ad3bb9ba3ecb8eed4b7c878653d3d4faf769876c528b73f343967f74a917 -127.0.0.1 9003 validator one16qsd5ant9v94jrs89mruzx62h7ekcfxmduh2rx ee2474f93cba9241562efc7475ac2721ab0899edf8f7f115a656c0c1f9ef8203add678064878d174bb478fa2e6630502 -127.0.0.1 9004 validator one1pf75h0t4am90z8uv3y0dgunfqp4lj8wr3t5rsp e751ec995defe4931273aaebcb2cd14bf37e629c554a57d3f334c37881a34a6188a93e76113c55ef3481da23b7d7ab09 -127.0.0.1 9005 validator one1est2gxcvavmtnzc7mhd73gzadm3xxcv5zczdtw 776f3b8704f4e1092a302a60e84f81e476c212d6f458092b696df420ea19ff84a6179e8e23d090b9297dc041600bc100 -127.0.0.1 9006 validator one1spshr72utf6rwxseaz339j09ed8p6f8ke370zj 2d61379e44a772e5757e27ee2b3874254f56073e6bd226eb8b160371cc3c18b8c4977bd3dcb71fd57dc62bf0e143fd08 -127.0.0.1 9007 validator one1a0x3d6xpmr6f8wsyaxd9v36pytvp48zckswvv9 c4e4708b6cf2a2ceeb59981677e9821eebafc5cf483fb5364a28fa604cc0ce69beeed40f3f03815c9e196fdaec5f1097 -127.0.0.1 9008 validator one1d2rngmem4x2c6zxsjjz29dlah0jzkr0k2n88wc 86dc2fdc2ceec18f6923b99fd86a68405c132e1005cf1df72dca75db0adfaeb53d201d66af37916d61f079f34f21fb96 -127.0.0.1 9009 validator one1658znfwf40epvy7e46cqrmzyy54h4n0qa73nep 49d15743b36334399f9985feb0753430a2b287b2d68b84495bbb15381854cbf01bca9d1d9f4c9c8f18509b2bfa6bd40f -127.0.0.1 9010 validator one1z05g55zamqzfw9qs432n33gycdmyvs38xjemyl 95117937cd8c09acd2dfae847d74041a67834ea88662a7cbed1e170350bc329e53db151e5a0ef3e712e35287ae954818 -127.0.0.1 9011 validator one1ljznytjyn269azvszjlcqvpcj6hjm822yrcp2e 68ae289d73332872ec8d04ac256ca0f5453c88ad392730c5741b6055bc3ec3d086ab03637713a29f459177aaa8340615 +127.0.0.1 9000 validator .hmy/65f55eb3052f9e9f632b2923be594ba77c55543f5c58ee1454b9cfd658d25e06373b0f7d42a19c84768139ea294f6204.key +127.0.0.1 9001 validator .hmy/40379eed79ed82bebfb4310894fd33b6a3f8413a78dc4d43b98d0adc9ef69f3285df05eaab9f2ce5f7227f8cb920e809.key +127.0.0.1 9002 validator .hmy/02c8ff0b88f313717bc3a627d2f8bb172ba3ad3bb9ba3ecb8eed4b7c878653d3d4faf769876c528b73f343967f74a917.key +127.0.0.1 9003 validator .hmy/ee2474f93cba9241562efc7475ac2721ab0899edf8f7f115a656c0c1f9ef8203add678064878d174bb478fa2e6630502.key +127.0.0.1 9004 validator .hmy/e751ec995defe4931273aaebcb2cd14bf37e629c554a57d3f334c37881a34a6188a93e76113c55ef3481da23b7d7ab09.key +127.0.0.1 9005 validator .hmy/776f3b8704f4e1092a302a60e84f81e476c212d6f458092b696df420ea19ff84a6179e8e23d090b9297dc041600bc100.key +127.0.0.1 9006 validator .hmy/2d61379e44a772e5757e27ee2b3874254f56073e6bd226eb8b160371cc3c18b8c4977bd3dcb71fd57dc62bf0e143fd08.key +127.0.0.1 9007 validator .hmy/c4e4708b6cf2a2ceeb59981677e9821eebafc5cf483fb5364a28fa604cc0ce69beeed40f3f03815c9e196fdaec5f1097.key +127.0.0.1 9008 validator .hmy/86dc2fdc2ceec18f6923b99fd86a68405c132e1005cf1df72dca75db0adfaeb53d201d66af37916d61f079f34f21fb96.key +127.0.0.1 9009 validator .hmy/49d15743b36334399f9985feb0753430a2b287b2d68b84495bbb15381854cbf01bca9d1d9f4c9c8f18509b2bfa6bd40f.key +127.0.0.1 9010 validator .hmy/95117937cd8c09acd2dfae847d74041a67834ea88662a7cbed1e170350bc329e53db151e5a0ef3e712e35287ae954818.key +127.0.0.1 9011 validator .hmy/68ae289d73332872ec8d04ac256ca0f5453c88ad392730c5741b6055bc3ec3d086ab03637713a29f459177aaa8340615.key +127.0.0.1 9099 explorer null 0 - -127.0.0.1 9100 validator one1a50tun737ulcvwy0yvve0pvu5skq0kjargvhwe 52ecce5f64db21cbe374c9268188f5d2cdd5bec1a3112276a350349860e35fb81f8cfe447a311e0550d961cf25cb988d -127.0.0.1 9101 validator one1uyshu2jgv8w465yc8kkny36thlt2wvel89tcmg a547a9bf6fdde4f4934cde21473748861a3cc0fe8bbb5e57225a29f483b05b72531f002f8187675743d819c955a86100 -127.0.0.1 9102 validator one103q7qe5t2505lypvltkqtddaef5tzfxwsse4z7 678ec9670899bf6af85b877058bea4fc1301a5a3a376987e826e3ca150b80e3eaadffedad0fedfa111576fa76ded980c -127.0.0.1 9103 validator one129r9pj3sk0re76f7zs3qz92rggmdgjhtwge62k 63f479f249c59f0486fda8caa2ffb247209489dae009dfde6144ff38c370230963d360dffd318cfb26c213320e89a512 -127.0.0.1 9104 validator one1d2rngmem4x2c6zxsjjz29dlah0jzkr0k2n88wc 16513c487a6bb76f37219f3c2927a4f281f9dd3fd6ed2e3a64e500de6545cf391dd973cc228d24f9bd01efe94912e714 -127.0.0.1 9105 validator one1658znfwf40epvy7e46cqrmzyy54h4n0qa73nep 576d3c48294e00d6be4a22b07b66a870ddee03052fe48a5abbd180222e5d5a1f8946a78d55b025de21635fd743bbad90 -127.0.0.1 9106 validator one1ghkz3frhske7emk79p7v2afmj4a5t0kmjyt4s5 eca09c1808b729ca56f1b5a6a287c6e1c3ae09e29ccf7efa35453471fcab07d9f73cee249e2b91f5ee44eb9618be3904 -127.0.0.1 9107 validator one1d7jfnr6yraxnrycgaemyktkmhmajhp8kl0yahv f47238daef97d60deedbde5302d05dea5de67608f11f406576e363661f7dcbc4a1385948549b31a6c70f6fde8a391486 -127.0.0.1 9108 validator one1r4zyyjqrulf935a479sgqlpa78kz7zlcg2jfen fc4b9c535ee91f015efff3f32fbb9d32cdd9bfc8a837bb3eee89b8fff653c7af2050a4e147ebe5c7233dc2d5df06ee0a -127.0.0.1 9109 validator one1p7ht2d4kl8ve7a8jxw746yfnx4wnfxtp8jqxwe ca86e551ee42adaaa6477322d7db869d3e203c00d7b86c82ebee629ad79cb6d57b8f3db28336778ec2180e56a8e07296 -127.0.0.1 9099 explorer +127.0.0.1 9100 validator .hmy/52ecce5f64db21cbe374c9268188f5d2cdd5bec1a3112276a350349860e35fb81f8cfe447a311e0550d961cf25cb988d.key +127.0.0.1 9101 validator .hmy/a547a9bf6fdde4f4934cde21473748861a3cc0fe8bbb5e57225a29f483b05b72531f002f8187675743d819c955a86100.key +127.0.0.1 9102 validator .hmy/678ec9670899bf6af85b877058bea4fc1301a5a3a376987e826e3ca150b80e3eaadffedad0fedfa111576fa76ded980c.key +127.0.0.1 9103 validator .hmy/63f479f249c59f0486fda8caa2ffb247209489dae009dfde6144ff38c370230963d360dffd318cfb26c213320e89a512.key +127.0.0.1 9104 validator .hmy/16513c487a6bb76f37219f3c2927a4f281f9dd3fd6ed2e3a64e500de6545cf391dd973cc228d24f9bd01efe94912e714.key +127.0.0.1 9105 validator .hmy/576d3c48294e00d6be4a22b07b66a870ddee03052fe48a5abbd180222e5d5a1f8946a78d55b025de21635fd743bbad90.key +127.0.0.1 9106 validator .hmy/eca09c1808b729ca56f1b5a6a287c6e1c3ae09e29ccf7efa35453471fcab07d9f73cee249e2b91f5ee44eb9618be3904.key +127.0.0.1 9107 validator .hmy/f47238daef97d60deedbde5302d05dea5de67608f11f406576e363661f7dcbc4a1385948549b31a6c70f6fde8a391486.key +127.0.0.1 9108 validator .hmy/fc4b9c535ee91f015efff3f32fbb9d32cdd9bfc8a837bb3eee89b8fff653c7af2050a4e147ebe5c7233dc2d5df06ee0a.key +127.0.0.1 9109 validator .hmy/ca86e551ee42adaaa6477322d7db869d3e203c00d7b86c82ebee629ad79cb6d57b8f3db28336778ec2180e56a8e07296.key +127.0.0.1 9098 explorer null 1 \ No newline at end of file diff --git a/test/debug.sh b/test/debug.sh index 0962f43b3..b8b267c4b 100755 --- a/test/debug.sh +++ b/test/debug.sh @@ -1,4 +1,8 @@ +#!/usr/bin/env bash + ./test/kill_node.sh rm -rf tmp_log* rm *.rlp -./test/deploy.sh -D 60000 ./test/configs/local-resharding.txt +rm -rf .dht* +scripts/go_executable_build.sh -S || exit 1 # dynamic builds are faster for debug iteration... +./test/deploy.sh -B -D 600000 ./test/configs/local-resharding.txt diff --git a/test/deploy.sh b/test/deploy.sh index 3aee75aa4..2e7f1c13d 100755 --- a/test/deploy.sh +++ b/test/deploy.sh @@ -1,74 +1,135 @@ -#!/bin/bash +#!/usr/bin/env bash +set -eo pipefail unset -v progdir case "${0}" in */*) progdir="${0%/*}" ;; -*) progdir=. ;; +*) progdir=. ;; esac ROOT="${progdir}/.." USER=$(whoami) +OS=$(uname -s) . "${ROOT}/scripts/setup_bls_build_flags.sh" -# set -x -set -eo pipefail +function cleanup() { + "${progdir}/kill_node.sh" +} -export GO111MODULE=on -OS=$(uname -s) +function build() { + if [[ "${NOBUILD}" != "true" ]]; then + pushd ${ROOT} + export GO111MODULE=on + if [[ "$OS" == "Darwin" ]]; then + # MacOS doesn't support static build + scripts/go_executable_build.sh -S + else + # Static build on Linux platform + scripts/go_executable_build.sh -s + fi + popd + fi +} -mkdir -p .hmy -if [ -f ".hmy/blspass.txt" ] -then - echo ".hmy/blspass.txt already in local." -else - touch .hmy/blspass.txt -fi +function setup() { + # Setup blspass file + mkdir -p ${ROOT}/.hmy + if [[ ! -f "${ROOT}/.hmy/blspass.txt" ]]; then + touch "${ROOT}/.hmy/blspass.txt" + fi -function cleanup() { - "${progdir}/kill_node.sh" + # Kill nodes if any + cleanup + + # Note that the binarys only works on MacOS & Linux + build + + # Create a tmp folder for logs + t=$(date +"%Y%m%d-%H%M%S") + log_folder="${ROOT}/tmp_log/log-$t" + mkdir -p "${log_folder}" + LOG_FILE=${log_folder}/r.log } -function cleanup_and_result() { - "${ROOT}/test/kill_node.sh" 2> /dev/null || true +function launch_bootnode() { + echo "launching boot node ..." + ${DRYRUN} ${ROOT}/bin/bootnode -port 19876 >"${log_folder}"/bootnode.log 2>&1 | tee -a "${LOG_FILE}" & + sleep 1 + BN_MA=$(grep "BN_MA" "${log_folder}"/bootnode.log | awk -F\= ' { print $2 } ') + echo "bootnode launched." + " $BN_MA" } -function debug_staking() { - source "$(go env GOPATH)/src/github.com/harmony-one/harmony/scripts/setup_bls_build_flags.sh" - hmy_gosdk="$(go env GOPATH)/src/github.com/harmony-one/go-sdk" - hmy_bin="${hmy_gosdk}/hmy" - hmy_ops="/tmp/harmony-ops" - keystore="${hmy_ops}/test-automation/api-tests/LocalnetValidatorKeys" - - rm -rf $hmy_ops - git clone https://github.com/harmony-one/harmony-ops.git $hmy_ops - - if [ ! -d "${hmy_gosdk}" ]; then - git clone https://github.com/harmony-one/go-sdk.git $hmy_gosdk - fi - if [ ! -f "${hmy_bin}" ]; then - make -C $hmy_gosdk - fi - - hmy_version=$($hmy_bin version 2>&1 >/dev/null) - if [[ "${hmy_version:32:3}" -lt "135" ]]; then - echo "Aborting staking tests since CLI version is out of date." - return - fi - - python3 -m pip install pyhmy - python3 -m pip install requests - python3 "${hmy_ops}/test-automation/api-tests/test.py" --keystore $keystore \ - --cli_path $hmy_bin --test_dir "${hmy_ops}/test-automation/api-tests/tests/" \ - --rpc_endpoint_src="http://localhost:9500/" --rpc_endpoint_dst="http://localhost:9501/" --ignore_regression_test +function launch_localnet() { + launch_bootnode + + unset -v base_args + declare -a base_args args + + if ${VERBOSE}; then + verbosity=5 + else + verbosity=3 + fi + + base_args=(-log_folder "${log_folder}" -min_peers "${MIN}" -bootnodes "${BN_MA}" "-network_type=$NETWORK" -blspass file:"${ROOT}/.hmy/blspass.txt" "-dns=false" "-verbosity=${verbosity}") + sleep 2 + + # Start nodes + i=-1 + while IFS='' read -r line || [[ -n "$line" ]]; do + i=$((i + 1)) + + # Read config for i-th node form config file + IFS=' ' read -r ip port mode bls_key shard <<<"${line}" + args=("${base_args[@]}" -ip "${ip}" -port "${port}" -key "/tmp/${ip}-${port}.key" -db_dir "${ROOT}/db-${ip}-${port}" "-broadcast_invalid_tx=true") + if [[ -z "$ip" || -z "$port" ]]; then + echo "skip empty node" + continue + fi + + # Setup BLS key for i-th localnet node + if [[ ! -e "$bls_key" ]]; then + args=("${args[@]}" -blskey_file "BLSKEY") + elif [[ -f "$bls_key" ]]; then + args=("${args[@]}" -blskey_file "${ROOT}/${bls_key}") + elif [[ -d "$bls_key" ]]; then + args=("${args[@]}" -blsfolder "${ROOT}/${bls_key}") + else + echo "skipping unknown node" + continue + fi + + # Setup flags for i-th node based on config + case "${mode}" in + explorer) + args=("${args[@]}" "-node_type=explorer" "-shard_id=${shard}") + ;; + archival) + args=("${args[@]}" -is_archival) + ;; + leader) + args=("${args[@]}" -is_leader) + ;; + external) + args=("${args[@]}" "-staking=true") + ;; + client) + continue + ;; + esac + + # Start the node + ${DRYRUN} "${ROOT}/bin/harmony" "${args[@]}" "${extra_args[@]}" 2>&1 | tee -a "${LOG_FILE}" & + done <"${config}" } -trap cleanup_and_result SIGINT SIGTERM +trap cleanup SIGINT SIGTERM -function usage { - local ME=$(basename $0) +function usage() { + local ME=$(basename $0) - cat< $log_folder/bootnode.log 2>&1 | tee -a $LOG_FILE & -sleep 1 -BN_MA=$(grep "BN_MA" $log_folder/bootnode.log | awk -F\= ' { print $2 } ') -echo "bootnode launched." + " $BN_MA" - -unset -v base_args -declare -a base_args args - -if $VERBOSE; then - verbosity=5 -else - verbosity=3 -fi - -base_args=(-log_folder "${log_folder}" -min_peers "${MIN}" -bootnodes "${BN_MA}" -network_type="$NETWORK" -blspass file:.hmy/blspass.txt -dns=false -verbosity="${verbosity}") -sleep 2 - -# Start nodes -i=0 -while IFS='' read -r line || [[ -n "$line" ]]; do - IFS=' ' read ip port mode account blspub <<< $line - args=("${base_args[@]}" -ip "${ip}" -port "${port}" -key "/tmp/${ip}-${port}.key" -db_dir "db-${ip}-${port}") - if [[ -z "$ip" || -z "$port" ]]; then - echo "skip empty node" - continue - fi - - if [ ! -e .hmy/${blspub}.key ]; then - args=("${args[@]}" -blskey_file "BLSKEY") - else - args=("${args[@]}" -blskey_file ".hmy/${blspub}.key") - fi - - case "${mode}" in leader*) args=("${args[@]}" -is_leader);; esac - case "${mode}" in *archival|archival) args=("${args[@]}" -is_archival);; esac - case "${mode}" in explorer*) args=("${args[@]}" -node_type=explorer -shard_id=0);; esac - case "${mode}" in - client) ;; - *) $DRYRUN "${ROOT}/bin/harmony" "${args[@]}" "${extra_args[@]}" 2>&1 | tee -a "${LOG_FILE}" &;; - esac - i=$((i+1)) -done < $config - -sleep $DURATION - -cleanup_and_result +setup +launch_localnet +sleep "${DURATION}" +cleanup || true diff --git a/test/go.sh b/test/go.sh new file mode 100644 index 000000000..a37662308 --- /dev/null +++ b/test/go.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +docker pull harmonyone/localnet-test +docker run -it -v "$DIR/../:/go/src/github.com/harmony-one/harmony" harmonyone/localnet-test -g