From 41d72b4c4ee93dc29bfc259adf9cc39ecea65875 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?=
 <36589218+GheisMohammadi@users.noreply.github.com>
Date: Fri, 20 Jan 2023 10:50:03 +0800
Subject: [PATCH] add new config option ( ForceReachabilityPublic ) to fix
 local-net consensus issue

---
 cmd/bootnode/main.go   | 13 ++++++----
 cmd/harmony/default.go | 10 ++++----
 cmd/harmony/main.go    |  8 ++++++
 p2p/host.go            | 57 ++++++++++++++++++++++++++++++------------
 test/deploy.sh         |  2 +-
 5 files changed, 63 insertions(+), 27 deletions(-)

diff --git a/cmd/bootnode/main.go b/cmd/bootnode/main.go
index 6e3a8bbbc..92d6674b9 100644
--- a/cmd/bootnode/main.go
+++ b/cmd/bootnode/main.go
@@ -102,6 +102,7 @@ func main() {
 	verbosity := flag.Int("verbosity", 5, "Logging verbosity: 0=silent, 1=error, 2=warn, 3=info, 4=debug, 5=detail (default: 5)")
 	logConn := flag.Bool("log_conn", false, "log incoming/outgoing connections")
 	maxConnPerIP := flag.Int("max_conn_per_ip", 10, "max connections number for same ip")
+	forceReachabilityPublic := flag.Bool("force_public", false, "forcing the local node to believe it is reachable externally")
 
 	flag.Parse()
 
@@ -124,12 +125,14 @@ func main() {
 	// For bootstrap nodes, we shall keep .dht file.
 	dataStorePath := fmt.Sprintf(".dht-%s-%s", *ip, *port)
 	selfPeer := p2p.Peer{IP: *ip, Port: *port}
+
 	host, err := p2p.NewHost(p2p.HostConfig{
-		Self:          &selfPeer,
-		BLSKey:        privKey,
-		BootNodes:     nil, // Boot nodes have no boot nodes :) Will be connected when other nodes joined
-		DataStoreFile: &dataStorePath,
-		MaxConnPerIP:  *maxConnPerIP,
+		Self:                    &selfPeer,
+		BLSKey:                  privKey,
+		BootNodes:               nil, // Boot nodes have no boot nodes :) Will be connected when other nodes joined
+		DataStoreFile:           &dataStorePath,
+		MaxConnPerIP:            *maxConnPerIP,
+		ForceReachabilityPublic: *forceReachabilityPublic,
 	})
 	if err != nil {
 		utils.FatalErrMsg(err, "cannot initialize network")
diff --git a/cmd/harmony/default.go b/cmd/harmony/default.go
index 7bdcc02d0..2a6c7a75c 100644
--- a/cmd/harmony/default.go
+++ b/cmd/harmony/default.go
@@ -191,14 +191,14 @@ var (
 	defaultLocalNetSyncConfig = harmonyconfig.SyncConfig{
 		Enabled:              true,
 		Downloader:           true,
-		StagedSync:           false,
+		StagedSync:           true,
 		StagedSyncCfg:        defaultStagedSyncConfig,
 		Concurrency:          4,
-		MinPeers:             5,
-		InitStreams:          5,
+		MinPeers:             4,
+		InitStreams:          4,
 		MaxAdvertiseWaitTime: 5, //minutes
-		DiscSoftLowCap:       5,
-		DiscHardLowCap:       5,
+		DiscSoftLowCap:       4,
+		DiscHardLowCap:       4,
 		DiscHighCap:          1024,
 		DiscBatch:            8,
 	}
diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go
index dbb18227d..8543158b3 100644
--- a/cmd/harmony/main.go
+++ b/cmd/harmony/main.go
@@ -629,6 +629,13 @@ func createGlobalConfig(hc harmonyconfig.HarmonyConfig) (*nodeconfig.ConfigType,
 		Port:            strconv.Itoa(hc.P2P.Port),
 		ConsensusPubKey: nodeConfig.ConsensusPriKey[0].Pub.Object,
 	}
+
+	// for local-net the node has to be forced to assume it is public reachable
+	forceReachabilityPublic := false
+	if hc.Network.NetworkType == nodeconfig.Localnet {
+		forceReachabilityPublic = true
+	}
+
 	myHost, err = p2p.NewHost(p2p.HostConfig{
 		Self:                     &selfPeer,
 		BLSKey:                   nodeConfig.P2PPriKey,
@@ -639,6 +646,7 @@ func createGlobalConfig(hc harmonyconfig.HarmonyConfig) (*nodeconfig.ConfigType,
 		DisablePrivateIPScan:     hc.P2P.DisablePrivateIPScan,
 		MaxPeers:                 hc.P2P.MaxPeers,
 		WaitForEachPeerToConnect: hc.P2P.WaitForEachPeerToConnect,
+		ForceReachabilityPublic:  forceReachabilityPublic,
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "cannot create P2P network host")
diff --git a/p2p/host.go b/p2p/host.go
index fcd2ea3b4..bab1da691 100644
--- a/p2p/host.go
+++ b/p2p/host.go
@@ -16,6 +16,7 @@ import (
 	"github.com/libp2p/go-libp2p-core/routing"
 	dht "github.com/libp2p/go-libp2p-kad-dht"
 	libp2p_pubsub "github.com/libp2p/go-libp2p-pubsub"
+	libp2p_config "github.com/libp2p/go-libp2p/config"
 	libp2p_crypto "github.com/libp2p/go-libp2p/core/crypto"
 	libp2p_host "github.com/libp2p/go-libp2p/core/host"
 	libp2p_network "github.com/libp2p/go-libp2p/core/network"
@@ -96,6 +97,7 @@ type HostConfig struct {
 	DisablePrivateIPScan     bool
 	MaxPeers                 int64
 	WaitForEachPeerToConnect bool
+	ForceReachabilityPublic  bool
 }
 
 func init() {
@@ -112,6 +114,19 @@ func init() {
 	libp2p_pubsub.GossipSubMaxIHaveLength = 1000
 }
 
+func forceReachabilityPublic(f bool) libp2p_config.Option {
+	if f {
+		return func(cfg *libp2p_config.Config) error {
+			public := libp2p_network.Reachability(libp2p_network.ReachabilityPublic)
+			cfg.AutoNATConfig.ForceReachability = &public
+			return nil
+		}
+	}
+	return func(p2pConfig *libp2p_config.Config) error {
+		return nil
+	}
+}
+
 // NewHost ..
 func NewHost(cfg HostConfig) (Host, error) {
 	var (
@@ -128,8 +143,8 @@ func NewHost(cfg HostConfig) (Host, error) {
 	ctx, cancel := context.WithCancel(context.Background())
 	// TODO: move low and high to configs
 	connmgr, err := connmgr.NewConnManager(
-		int(10),                     // LowWater
-		int(10000)*cfg.MaxConnPerIP, // HighWater,
+		int(cfg.MaxConnPerIP),      // LowWater
+		int(1024)*cfg.MaxConnPerIP, // HighWater,
 		connmgr.WithGracePeriod(time.Minute),
 	)
 	if err != nil {
@@ -138,16 +153,16 @@ func NewHost(cfg HostConfig) (Host, error) {
 	}
 	var idht *dht.IpfsDHT
 	var opt discovery.DHTConfig
-	p2pHost, err := libp2p.New(
+	p2pHostConfig := []libp2p.Option{
 		listenAddr,
 		libp2p.Identity(key),
-		// support TLS connections
+		// Support TLS connections
 		libp2p.Security(libp2ptls.ID, libp2ptls.New),
-		// support noise connections
+		// Support noise connections
 		libp2p.Security(noise.ID, noise.New),
-		// support any other default transports (TCP)
+		// Support any other default transports (TCP)
 		libp2p.DefaultTransports,
-		// Let's prevent our peer from having too many
+		// Prevent the peer from having too many
 		// connections by attaching a connection manager.
 		libp2p.ConnectionManager(connmgr),
 		// Attempt to open ports using uPNP for NATed hosts.
@@ -165,8 +180,7 @@ func NewHost(cfg HostConfig) (Host, error) {
 			idht, err = dht.New(ctx, h, opts...)
 			return idht, err
 		}),
-
-		// to help other peers to figure out if they are behind
+		// To help other peers to figure out if they are behind
 		// NATs, launch the server-side of AutoNAT too (AutoRelay
 		// already runs the client)
 		// This service is highly rate-limited and should not cause any
@@ -174,14 +188,23 @@ func NewHost(cfg HostConfig) (Host, error) {
 		libp2p.EnableNATService(),
 		// Bandwidth Reporter
 		libp2p.BandwidthReporter(newCounter()),
-		// ForceReachabilityPublic overrides automatic reachability detection in the AutoNAT subsystem,
-		// forcing the local node to believe it is reachable externally.
-		// libp2p.ForceReachabilityPublic(),
-		// libp2p.DisableRelay(),
+		// Enable relay service, to disable relay we can use libp2p.DisableRelay()
 		libp2p.EnableRelayService(),
-		// prevent dialing of public addresses
-		// libp2p.ConnectionGater(NewGater(cfg.DisablePrivateIPScan)),
-	)
+	}
+
+	if cfg.ForceReachabilityPublic {
+		// ForceReachabilityPublic overrides automatic reachability detection in the AutoNAT subsystem,
+		// forcing the local node to believe it is reachable externally
+		p2pHostConfig = append(p2pHostConfig, libp2p.ForceReachabilityPublic())
+	}
+
+	if cfg.DisablePrivateIPScan {
+		// Prevent dialing of public addresses
+		p2pHostConfig = append(p2pHostConfig, libp2p.ConnectionGater(NewGater(cfg.DisablePrivateIPScan)))
+	}
+
+	// create p2p host
+	p2pHost, err := libp2p.New(p2pHostConfig...)
 	if err != nil {
 		cancel()
 		return nil, errors.Wrapf(err, "cannot initialize libp2p host")
@@ -190,6 +213,7 @@ func NewHost(cfg HostConfig) (Host, error) {
 	disc, err := discovery.NewDHTDiscovery(ctx, cancel, p2pHost, idht, opt)
 	if err != nil {
 		cancel()
+		p2pHost.Close()
 		return nil, errors.Wrap(err, "cannot create DHT discovery")
 	}
 
@@ -230,6 +254,7 @@ func NewHost(cfg HostConfig) (Host, error) {
 	pubsub, err := libp2p_pubsub.NewGossipSub(ctx, p2pHost, options...)
 	if err != nil {
 		cancel()
+		p2pHost.Close()
 		return nil, errors.Wrapf(err, "cannot initialize libp2p pub-sub")
 	}
 
diff --git a/test/deploy.sh b/test/deploy.sh
index 6bbb12eb9..9ab694311 100755
--- a/test/deploy.sh
+++ b/test/deploy.sh
@@ -54,7 +54,7 @@ function setup() {
 
 function launch_bootnode() {
   echo "launching boot node ..."
-  ${DRYRUN} ${ROOT}/bin/bootnode -port 19876 -max_conn_per_ip 100 >"${log_folder}"/bootnode.log 2>&1 | tee -a "${LOG_FILE}" &
+  ${DRYRUN} ${ROOT}/bin/bootnode -port 19876 -max_conn_per_ip 100 -force_public true >"${log_folder}"/bootnode.log 2>&1 | tee -a "${LOG_FILE}" &
   sleep 1
   BN_MA=$(grep "BN_MA" "${log_folder}"/bootnode.log | awk -F\= ' { print $2 } ')
   echo "bootnode launched." + " $BN_MA"