stable localnet with external validator (#4388)

* stable localnet with external validator

* ignore deploy config file comments

* reduce node launched in localnet

* update makefile

* localnet configuration - add more fn

* fix validator information command typo

.hmy/extbls/4f41a37a3a8d0695dd6edcc58142c6b7d98e74da5c90e79b587b3b960b6a4f5e048e6d8b8a000d77a478d44cd640270c.key

@@ -0,0 +1 @@
+a010bc1550956a3a4935cdc6f8633ea770bd0321094cfaccd25d3ea3b5382417cb5c150de0357ee8b6eeeec135ae0ec379701c7f35ae33da9030e93a630dc3a988577d1a1b0a9e60976f79d5cefa68123ac77c636f2ccab149fc8fc7

.hmy/extbls/5a18d4aa3e6aff4835f07588ae66be19684476d38799f63e54c6b5732fad1e86dce7458b1c295404fb54a0d61e50bb97.key

@@ -0,0 +1 @@
+e176a1a8d9c533ad7efbaa9caca1d839146495683f669be35dddfebe2f39497c73fc2a6d7da116acd3698f582fcfe6b7b9fc18fd286a42bcdf007dc7a618bab958eb5c97f6082104ff723705d36744289b9885a692c01731cb45c698

.hmy/extbls/7dcc035a943e29e17959dabe636efad7303d2c6f273ace457ba9dcc2fd19d3f37e70ba1cd8d082cf8ff7be2f861db48c.key

@@ -0,0 +1 @@
+eb388b59c8a4ed1d6a1713551a3404e775fdf27b5f92d302b01874e2ced1465ff9baa93eba1d3e24620d71e0557231087bbfea0c89cdb829c3868e990abaaa7595a61a13ba40d61262c5395066bccf2681f65b9f53a621a37e1b8123

.hmy/extbls/81296eedba05047594385e3086e1dab52c9eb9e56f46d86f58447cccc20535d646120171961d74968d27a2ec0f8af285.key

@@ -0,0 +1 @@
+b64f0601353691a6bbe68658103c044ec0e075ebe2cdb8328a07269c1cf4e5574ac81fb48fb49c90cf1e0fbf0e5ed97a5806a30505104717a68d7ae341c08a7ef98bf5d4c607c236ef80d9dbfc0d3212191e0c0436b4d78890b7da68

.hmy/extbls/89eab762e7364d6cf89f7a6c54da794f74eba2e29147992ac66adcef0f0654ef8a727710ee55ad8b532da0dd87811915.key

@@ -0,0 +1 @@
+2ea158c2fd1d4cefcfe4dc4d987099ef93731decee88db87d007d6a5da2b87e4528abe3013814fc7183651ac1e301b2f3caa8f03071d2bd110746be8c1004dd88f22495449ae6d8c5a7cce5783a6964a4663c9319570433a68ef2f31

.hmy/extbls/b0917378b179a519a5055259c4f8980cce37d58af300b00dd98b07076d3d9a3b16c4a55f84522f553872225a7b1efc0c.key

@@ -0,0 +1 @@
+81e67ac67dc10c4d89baa7ae800d6afffbc4105766a2202ee30998669e118ce2842ae64cb8a20282d10974c1bba6ce3cdda6e2bb00124bf4ab154aeebe081a7c44ce5b010b0b069bfa37d35beccffe44209ac376b4b14885f5b3625a

.hmy/extkeystore/one17ughrllgnzx9sfa46p568k8rdmtz7qj85slc6t.key

@@ -0,0 +1 @@
+{"address":"f71171ffe8988c5827b5d069a3d8e36ed62f0247","crypto":{"cipher":"aes-128-ctr","ciphertext":"5c7590efba91015b67234ffa063dc56e0460af0b68fa0265ac46d46a3c28448e","cipherparams":{"iv":"e8927574296604528514d7dabc6895ad"},"kdf":"scrypt","kdfparams":{"dklen":32,"n":262144,"p":1,"r":8,"salt":"d2573eadafda1a530300683d610ab67465fded0e33d0a9dff101481dd4e9c043"},"mac":"0666235e715fc0c52b0cc480891b2c7514e4731d99fe93bee3d2678679f0de19"},"id":"94a50e8b-5973-4989-aaac-420cdb243863","version":3}

.hmy/extkeystore/one19aw2wcr5y4lxeuwt0ajgt5aw3a3qkjdgg67ygj.key

@@ -0,0 +1 @@
+{"address":"2f5ca76074257e6cf1cb7f6485d3ae8f620b49a8","crypto":{"cipher":"aes-128-ctr","ciphertext":"0f4905db05ba9cf8d99e0f090dc36bc3e15b2fe6e0247b4800c0d092bcd61a6d","cipherparams":{"iv":"fa9e222a88e5954274195fce7d2e3cc2"},"kdf":"scrypt","kdfparams":{"dklen":32,"n":262144,"p":1,"r":8,"salt":"f6b32095d86b01d85764f099f6f82e2a9ae141eaf058025047e55f9ba90d0ac5"},"mac":"05f3de89292e8b16ee63181e762feb0b9fd46856b8370ebbe2c9eb806b3d26b7"},"id":"5342b382-696e-46d3-9894-272a6da3f5b3","version":3}

.hmy/extkeystore/one19zzwsxr0uf2fe34y8qkadek2v0eh6h5mg2deg6.key

@@ -0,0 +1 @@
+{"address":"2884e8186fe2549cc6a4382dd6e6ca63f37d5e9b","crypto":{"cipher":"aes-128-ctr","ciphertext":"7308d5b4904e912d0f312208b9e8a95eee04d9cacf80f2c89dd802bdcd6b9cb5","cipherparams":{"iv":"ff07874ea030a9c3c9a0c8ce4d87f06a"},"kdf":"scrypt","kdfparams":{"dklen":32,"n":262144,"p":1,"r":8,"salt":"972333a39a53e1dab69d50d2cd9a97e19595896e2b79e88c9ffd81b66fe93bea"},"mac":"828fba7c39d73996a5b37d399013b7659c4a9cebc40d6e1837e6253a01b1c4f1"},"id":"76c4c1ba-cd6d-4613-a89e-f9faaa7d130d","version":3}

.hmy/extkeystore/one1auqndgthqu5lznsn7tuma8s5333cq0y07cwc6x.key

@@ -0,0 +1 @@
+{"address":"ef0136a1770729f14e13f2f9be9e148c63803c8f","crypto":{"cipher":"aes-128-ctr","ciphertext":"367d95f415cf795aa7498025efb7b44ef31a698f3820bb77e3cd447d57dcfc80","cipherparams":{"iv":"206f60b938aad4f2a6763ba2199ce737"},"kdf":"scrypt","kdfparams":{"dklen":32,"n":262144,"p":1,"r":8,"salt":"02c7e01ef989794d695d7ee87a5413fbea79751a8e725f529ac39e6b611da0ab"},"mac":"65e93d97f3a89bc45da4b1bcf8c38ef3b0dbee457c91207bf2c9ff787aa12ebe"},"id":"9b6fb83f-90b2-4ae6-a24a-5b13436db30f","version":3}

.hmy/extkeystore/one1eenp9ujcrmyaq22ef6jrpry2k97tjz4xs6ppcf.key

@@ -0,0 +1 @@
+{"address":"ce6612f2581ec9d029594ea4308c8ab17cb90aa6","crypto":{"cipher":"aes-128-ctr","ciphertext":"7c4ba34f62d62734df18c383bafc03c6a8483f99ebbb1103517a7d6060c68c10","cipherparams":{"iv":"a300902fe8cc9bb4bfc686eb77b6bf26"},"kdf":"scrypt","kdfparams":{"dklen":32,"n":262144,"p":1,"r":8,"salt":"693073fcc4cdb230aae22796094a1f8b17c298f78b3cabff3f8c78b81da8a222"},"mac":"624ed38319f3b46405a7d5e831fdb60dbe4d7f66f16ff9cc67bb0919ab3862e9"},"id":"19af9349-6c8d-4dd3-8053-68b03a646d59","version":3}

.hmy/extkeystore/one1lctumupg2y009pjmnhnmn4nqjk0zf0dspjanf7.key

@@ -0,0 +1 @@
+{"address":"fe17cdf028511ef2865b9de7b9d660959e24bdb0","crypto":{"cipher":"aes-128-ctr","ciphertext":"5cb69fbe4f2aaa0500d4b197e8411fa4d9fa75ff4b4bf067760c1ff79e57a0b6","cipherparams":{"iv":"5d1e9bda47aae3385d31bf036ba6db67"},"kdf":"scrypt","kdfparams":{"dklen":32,"n":262144,"p":1,"r":8,"salt":"779fba35725c4711bc5772102234f3b718c46d751bf106153c06e61984e8e827"},"mac":"c1de2e2e1e5aebd5a083aec65e18eec21c67d257799a8d6402c3878045678ea8"},"id":"c2e234e2-bc52-4c38-8f82-6a6cfb65f342","version":3}

Makefile

@@ -12,7 +12,7 @@ RPMBUILD=$(HOME)/rpmbuild
 DEBBUILD=$(HOME)/debbuild
 SHELL := bash
 
-.PHONY: all help libs exe race trace-pointer debug debug-kill test test-go test-api test-api-attach linux_static deb_init deb_build deb debpub_dev debpub_prod rpm_init rpm_build rpm rpmpub_dev rpmpub_prod clean distclean docker
+.PHONY: all help libs exe race trace-pointer debug debug-ext debug-kill test test-go test-api test-api-attach linux_static deb_init deb_build deb debpub_dev debpub_prod rpm_init rpm_build rpm rpmpub_dev rpmpub_prod clean distclean docker
 
 all: libs
 	bash ./scripts/go_executable_build.sh -S
@@ -23,8 +23,9 @@ help:
 	@echo "exe - build the harmony binary & bootnode"
 	@echo "race - build the harmony binary & bootnode with race condition checks"
 	@echo "trace-pointer - build the harmony binary & bootnode with pointer analysis"
-	@echo "debug - start a localnet with 2 shards (s0 rpc endpoint = localhost:9599; s1 rpc endpoint = localhost:9598)"
+	@echo "debug - start a localnet with 2 shards (s0 rpc endpoint = localhost:9700; s1 rpc endpoint = localhost:9800)"
 	@echo "debug-kill - force kill the localnet"
+	@echo "debug-ext - start a localnet with 2 shards and external (s0 rpc endpoint = localhost:9598; s1 rpc endpoint = localhost:9596)"
 	@echo "clean - remove node files & logs created by localnet"
 	@echo "distclean - remove node files & logs created by localnet, and all libs"
 	@echo "test - run the entire test suite (go test & Node API test)"
@@ -60,6 +61,9 @@ debug:
 debug-kill:
 	bash ./test/kill_node.sh
 
+debug-ext:
+	bash ./test/debug-external.sh
+
 clean:
 	rm -rf ./tmp_log*
 	rm -rf ./.dht*

internal/configs/sharding/localnet.go

@@ -23,8 +23,8 @@ const (
 	localnetV1Epoch = 1
 
 	localnetEpochBlock1      = 5
-	localnetBlocksPerEpoch   = 5
-	localnetBlocksPerEpochV2 = 10
+	localnetBlocksPerEpoch   = 64
+	localnetBlocksPerEpochV2 = 64
 
 	localnetVdfDifficulty = 5000 // This takes about 10s to finish the vdf
 )

test/build-localnet-validator.sh

@@ -0,0 +1,70 @@
+#! /bin/bash
+
+echo "Make sure the validator account are imported"
+hmy keys import-ks .hmy/extkeystore/one17ughrllgnzx9sfa46p568k8rdmtz7qj85slc6t.key 2> /dev/null
+hmy keys import-ks .hmy/extkeystore/one1auqndgthqu5lznsn7tuma8s5333cq0y07cwc6x.key 2> /dev/null
+hmy keys import-ks .hmy/extkeystore/one19aw2wcr5y4lxeuwt0ajgt5aw3a3qkjdgg67ygj.key 2> /dev/null
+hmy keys import-ks .hmy/extkeystore/one1eenp9ujcrmyaq22ef6jrpry2k97tjz4xs6ppcf.key 2> /dev/null
+hmy keys import-ks .hmy/extkeystore/one19zzwsxr0uf2fe34y8qkadek2v0eh6h5mg2deg6.key 2> /dev/null
+hmy keys import-ks .hmy/extkeystore/one1lctumupg2y009pjmnhnmn4nqjk0zf0dspjanf7.key 2> /dev/null
+
+echo "Let's fund all the validator account"
+hmy --node=http://127.0.0.1:9500 transfer     --from one1zksj3evekayy90xt4psrz8h6j2v3hla4qwz4ur --to one17ughrllgnzx9sfa46p568k8rdmtz7qj85slc6t     --from-shard 0 --to-shard 0 --amount 110000 
+hmy --node=http://127.0.0.1:9500 transfer     --from one1zksj3evekayy90xt4psrz8h6j2v3hla4qwz4ur --to one1auqndgthqu5lznsn7tuma8s5333cq0y07cwc6x     --from-shard 0 --to-shard 0 --amount 110000 
+hmy --node=http://127.0.0.1:9500 transfer     --from one1zksj3evekayy90xt4psrz8h6j2v3hla4qwz4ur --to one19aw2wcr5y4lxeuwt0ajgt5aw3a3qkjdgg67ygj     --from-shard 0 --to-shard 0 --amount 110000 
+hmy --node=http://127.0.0.1:9500 transfer     --from one1zksj3evekayy90xt4psrz8h6j2v3hla4qwz4ur --to one1eenp9ujcrmyaq22ef6jrpry2k97tjz4xs6ppcf     --from-shard 0 --to-shard 0 --amount 110000 
+hmy --node=http://127.0.0.1:9500 transfer     --from one1zksj3evekayy90xt4psrz8h6j2v3hla4qwz4ur --to one19zzwsxr0uf2fe34y8qkadek2v0eh6h5mg2deg6     --from-shard 0 --to-shard 0 --amount 110000 
+hmy --node=http://127.0.0.1:9500 transfer     --from one1zksj3evekayy90xt4psrz8h6j2v3hla4qwz4ur --to one1lctumupg2y009pjmnhnmn4nqjk0zf0dspjanf7     --from-shard 0 --to-shard 0 --amount 110000 
+
+
+#wait for epoch 2
+epoch=$(hmy blockchain latest-headers --node="http://localhost:9500" | jq -r '.["result"]["beacon-chain-header"]["epoch"]')
+while (( epoch < 2 )); do
+	echo "Not yet on epoch 2 .. waiting 30s"
+	epoch=$(hmy blockchain latest-headers --node="http://localhost:9500" | jq -r '.["result"]["beacon-chain-header"]["epoch"]')
+	sleep 30
+done
+
+echo "Now in epoch 2, we'll create the external validators"
+
+hmy --node="http://localhost:9500" staking create-validator \
+    --validator-addr one17ughrllgnzx9sfa46p568k8rdmtz7qj85slc6t --amount 10000 \
+    --bls-pubkeys 4f41a37a3a8d0695dd6edcc58142c6b7d98e74da5c90e79b587b3b960b6a4f5e048e6d8b8a000d77a478d44cd640270c,7dcc035a943e29e17959dabe636efad7303d2c6f273ace457ba9dcc2fd19d3f37e70ba1cd8d082cf8ff7be2f861db48c \
+    --name "s0-localnet-validator1" --identity "validator1" --details "validator1" \
+    --security-contact "localnet" --website "localnet.one" \
+    --max-change-rate 0.1 --max-rate 0.1 --rate 0.1 \
+    --max-total-delegation 100000000 --min-self-delegation 10000 --bls-pubkeys-dir .hmy/extbls/
+	
+hmy --node="http://localhost:9500" staking create-validator \
+    --validator-addr one1auqndgthqu5lznsn7tuma8s5333cq0y07cwc6x --amount 10000 \
+    --bls-pubkeys b0917378b179a519a5055259c4f8980cce37d58af300b00dd98b07076d3d9a3b16c4a55f84522f553872225a7b1efc0c \
+    --name "s0-localnet-validator2" --identity "validator2" --details "validator2" \
+    --security-contact "localnet" --website "localnet.one" \
+    --max-change-rate 0.1 --max-rate 0.1 --rate 0.1 \
+    --max-total-delegation 100000000 --min-self-delegation 10000 --bls-pubkeys-dir .hmy/extbls/
+	
+hmy --node="http://localhost:9500" staking create-validator \
+    --validator-addr one19aw2wcr5y4lxeuwt0ajgt5aw3a3qkjdgg67ygj --amount 10000 \
+    --bls-pubkeys 5a18d4aa3e6aff4835f07588ae66be19684476d38799f63e54c6b5732fad1e86dce7458b1c295404fb54a0d61e50bb97,81296eedba05047594385e3086e1dab52c9eb9e56f46d86f58447cccc20535d646120171961d74968d27a2ec0f8af285 \
+    --name "s1-localnet-validator3" --identity "validator3" --details "validator3" \
+    --security-contact "localnet" --website "localnet.one" \
+    --max-change-rate 0.1 --max-rate 0.1 --rate 0.1 \
+    --max-total-delegation 100000000 --min-self-delegation 10000 --bls-pubkeys-dir .hmy/extbls/
+	
+hmy --node="http://localhost:9500" staking create-validator \
+    --validator-addr one1eenp9ujcrmyaq22ef6jrpry2k97tjz4xs6ppcf --amount 10000 \
+    --bls-pubkeys 89eab762e7364d6cf89f7a6c54da794f74eba2e29147992ac66adcef0f0654ef8a727710ee55ad8b532da0dd87811915 \
+    --name "s1-localnet-validator4" --identity "validator4" --details "validator4" \
+    --security-contact "localnet" --website "localnet.one" \
+    --max-change-rate 0.1 --max-rate 0.1 --rate 0.1 \
+    --max-total-delegation 100000000 --min-self-delegation 10000 --bls-pubkeys-dir .hmy/extbls/
+
+
+echo "validator created"
+echo '''check their information
+hmy blockchain validator information one17ughrllgnzx9sfa46p568k8rdmtz7qj85slc6t --node="http://localhost:9500"
+hmy blockchain validator information one1auqndgthqu5lznsn7tuma8s5333cq0y07cwc6x --node="http://localhost:9500"
+hmy blockchain validator information one19aw2wcr5y4lxeuwt0ajgt5aw3a3qkjdgg67ygj --node="http://localhost:9500"
+hmy blockchain validator information one1eenp9ujcrmyaq22ef6jrpry2k97tjz4xs6ppcf --node="http://localhost:9500"
+'''
+

test/configs/local-resharding-with-external.txt

@@ -0,0 +1,37 @@
+# shard 0
+# internal node
+127.0.0.1 9000 validator .hmy/65f55eb3052f9e9f632b2923be594ba77c55543f5c58ee1454b9cfd658d25e06373b0f7d42a19c84768139ea294f6204.key
+127.0.0.1 9002 validator .hmy/02c8ff0b88f313717bc3a627d2f8bb172ba3ad3bb9ba3ecb8eed4b7c878653d3d4faf769876c528b73f343967f74a917.key
+127.0.0.1 9004 validator .hmy/e751ec995defe4931273aaebcb2cd14bf37e629c554a57d3f334c37881a34a6188a93e76113c55ef3481da23b7d7ab09.key
+127.0.0.1 9006 validator .hmy/2d61379e44a772e5757e27ee2b3874254f56073e6bd226eb8b160371cc3c18b8c4977bd3dcb71fd57dc62bf0e143fd08.key
+127.0.0.1 9008 validator .hmy/86dc2fdc2ceec18f6923b99fd86a68405c132e1005cf1df72dca75db0adfaeb53d201d66af37916d61f079f34f21fb96.key
+127.0.0.1 9010 validator .hmy/95117937cd8c09acd2dfae847d74041a67834ea88662a7cbed1e170350bc329e53db151e5a0ef3e712e35287ae954818.key
+# external node
+127.0.0.1 9014 external  .hmy/extbls/4f41a37a3a8d0695dd6edcc58142c6b7d98e74da5c90e79b587b3b960b6a4f5e048e6d8b8a000d77a478d44cd640270c.key
+127.0.0.1 9016 external  .hmy/extbls/7dcc035a943e29e17959dabe636efad7303d2c6f273ace457ba9dcc2fd19d3f37e70ba1cd8d082cf8ff7be2f861db48c.key
+127.0.0.1 9018 external  .hmy/extbls/b0917378b179a519a5055259c4f8980cce37d58af300b00dd98b07076d3d9a3b16c4a55f84522f553872225a7b1efc0c.key
+# fn node
+127.0.0.1 9050 validator .hmy/52ecce5f64db21cbe374c9268188f5d2cdd5bec1a3112276a350349860e35fb81f8cfe447a311e0550d961cf25cb988d.key
+127.0.0.1 9052 validator .hmy/678ec9670899bf6af85b877058bea4fc1301a5a3a376987e826e3ca150b80e3eaadffedad0fedfa111576fa76ded980c.key
+#127.0.0.1 9054 validator .hmy/16513c487a6bb76f37219f3c2927a4f281f9dd3fd6ed2e3a64e500de6545cf391dd973cc228d24f9bd01efe94912e714.key
+# explorer node
+127.0.0.1 9098 explorer null 0
+
+# shard 1
+# internal node
+127.0.0.1 9100 validator .hmy/40379eed79ed82bebfb4310894fd33b6a3f8413a78dc4d43b98d0adc9ef69f3285df05eaab9f2ce5f7227f8cb920e809.key
+127.0.0.1 9102 validator .hmy/ee2474f93cba9241562efc7475ac2721ab0899edf8f7f115a656c0c1f9ef8203add678064878d174bb478fa2e6630502.key
+127.0.0.1 9104 validator .hmy/776f3b8704f4e1092a302a60e84f81e476c212d6f458092b696df420ea19ff84a6179e8e23d090b9297dc041600bc100.key
+127.0.0.1 9106 validator .hmy/c4e4708b6cf2a2ceeb59981677e9821eebafc5cf483fb5364a28fa604cc0ce69beeed40f3f03815c9e196fdaec5f1097.key
+127.0.0.1 9108 validator .hmy/49d15743b36334399f9985feb0753430a2b287b2d68b84495bbb15381854cbf01bca9d1d9f4c9c8f18509b2bfa6bd40f.key
+127.0.0.1 9110 validator .hmy/68ae289d73332872ec8d04ac256ca0f5453c88ad392730c5741b6055bc3ec3d086ab03637713a29f459177aaa8340615.key
+# external node
+127.0.0.1 9114 external  .hmy/extbls/5a18d4aa3e6aff4835f07588ae66be19684476d38799f63e54c6b5732fad1e86dce7458b1c295404fb54a0d61e50bb97.key
+127.0.0.1 9116 external  .hmy/extbls/81296eedba05047594385e3086e1dab52c9eb9e56f46d86f58447cccc20535d646120171961d74968d27a2ec0f8af285.key
+127.0.0.1 9118 external  .hmy/extbls/89eab762e7364d6cf89f7a6c54da794f74eba2e29147992ac66adcef0f0654ef8a727710ee55ad8b532da0dd87811915.key
+# fn node
+127.0.0.1 9150 validator .hmy/a547a9bf6fdde4f4934cde21473748861a3cc0fe8bbb5e57225a29f483b05b72531f002f8187675743d819c955a86100.key
+127.0.0.1 9152 validator .hmy/63f479f249c59f0486fda8caa2ffb247209489dae009dfde6144ff38c370230963d360dffd318cfb26c213320e89a512.key
+#127.0.0.1 9154 validator .hmy/576d3c48294e00d6be4a22b07b66a870ddee03052fe48a5abbd180222e5d5a1f8946a78d55b025de21635fd743bbad90.key
+# explorer node
+127.0.0.1 9096 explorer null 1

test/debug-external.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+./test/kill_node.sh
+rm -rf tmp_log*
+rm *.rlp
+rm -rf .dht*
+scripts/go_executable_build.sh -S || exit 1  # dynamic builds are faster for debug iteration...
+./test/deploy.sh -B -D 600000 ./test/configs/local-resharding-with-external.txt

test/deploy.sh

@@ -83,8 +83,8 @@ function launch_localnet() {
     # Read config for i-th node form config file
     IFS=' ' read -r ip port mode bls_key shard node_config <<<"${line}"
     args=("${base_args[@]}" --ip "${ip}" --port "${port}" --key "/tmp/${ip}-${port}.key" --db_dir "${ROOT}/db-${ip}-${port}" "--broadcast_invalid_tx=false")
-    if [[ -z "$ip" || -z "$port" ]]; then
-      echo "skip empty node"
+    if [[ -z "$ip" || -z "$port" || "$ip" == "#" ]]; then
+      echo "skip empty line or node or comment"
       continue
     fi
     if [[ $EXPOSEAPIS == "true" ]]; then