Offline Node Support (#3383)

* [cmd] Add IsOffline flag

* Add IsOffline to node config

Signed-off-by: Daniel Van Der Maden <dvandermaden0@berkeley.edu>

* [node] Disable syncing and p2p msgs if offline

Signed-off-by: Daniel Van Der Maden <dvandermaden0@berkeley.edu>

* [rosetta] Clarify syncing status msg

Signed-off-by: Daniel Van Der Maden <dvandermaden0@berkeley.edu>

* [internal/config] Add DefaultLocalListenIP & DefaultPublicListenIP

Signed-off-by: Daniel Van Der Maden <dvandermaden0@berkeley.edu>

* [cmd] Add IP to p2p config

* Use default local listening ip for p2p hors if node is in offline

Signed-off-by: Daniel Van Der Maden <dvandermaden0@berkeley.edu>

* [p2p] Use peer ip when creating listenAddr

Signed-off-by: Daniel Van Der Maden <dvandermaden0@berkeley.edu>

* [cmd] Fix unit tests & bump config version

* Fix TestAddPeer & TestConnectionToInvalidPeer p2p test

Signed-off-by: Daniel Van Der Maden <dvandermaden0@berkeley.edu>

* [cmd] Fix lint

Signed-off-by: Daniel Van Der Maden <dvandermaden0@berkeley.edu>

* [cmd] Address PR comments

Signed-off-by: Daniel Van Der Maden <dvandermaden0@berkeley.edu>

cmd/harmony/config.go

@@ -44,6 +44,7 @@ type networkConfig struct {
 
 type p2pConfig struct {
 	Port    int
+	IP      string
 	KeyFile string
 }
 
@@ -52,6 +53,7 @@ type generalConfig struct {
 	NoStaking  bool
 	ShardID    int
 	IsArchival bool
+	IsOffline  bool
 	DataDir    string
 }
 
@@ -160,6 +162,10 @@ func validateHarmonyConfig(config harmonyConfig) error {
 		return errors.New("flag --run.shard must be specified for explorer node")
 	}
 
+	if config.General.IsOffline && config.P2P.IP != nodeconfig.DefaultLocalListenIP {
+		return fmt.Errorf("flag --run.offline must have p2p IP be %v", nodeconfig.DefaultLocalListenIP)
+	}
+
 	return nil
 }
 
@@ -242,6 +248,9 @@ func loadHarmonyConfig(file string) (harmonyConfig, error) {
 	if config.HTTP.RosettaPort == 0 {
 		config.HTTP.RosettaPort = defaultConfig.HTTP.RosettaPort
 	}
+	if config.P2P.IP == "" {
+		config.P2P.IP = defaultConfig.P2P.IP
+	}
 	return config, nil
 }
 

cmd/harmony/config_test.go

@@ -99,6 +99,12 @@ func TestV1_0_0Config(t *testing.T) {
 	if config.HTTP.RosettaEnabled {
 		t.Errorf("Expected rosetta http server to be disabled when loading old config")
 	}
+	if config.General.IsOffline {
+		t.Errorf("Expect node to de online when loading old config")
+	}
+	if config.P2P.IP != defaultConfig.P2P.IP {
+		t.Errorf("Expect default p2p IP if old config is provided")
+	}
 	if config.Version != "1.0.0" {
 		t.Errorf("Expected config version: 1.0.0, not %v", config.Version)
 	}

cmd/harmony/default.go

@@ -2,7 +2,7 @@ package main
 
 import nodeconfig "github.com/harmony-one/harmony/internal/configs/node"
 
-const tomlConfigVersion = "1.0.1"
+const tomlConfigVersion = "1.0.2"
 
 const (
 	defNetworkType = nodeconfig.Mainnet
@@ -15,11 +15,13 @@ var defaultConfig = harmonyConfig{
 		NoStaking:  false,
 		ShardID:    -1,
 		IsArchival: false,
+		IsOffline:  false,
 		DataDir:    "./",
 	},
 	Network: getDefaultNetworkConfig(defNetworkType),
 	P2P: p2pConfig{
 		Port:    nodeconfig.DefaultP2PPort,
+		IP:      nodeconfig.DefaultPublicListenIP,
 		KeyFile: "./.hmykey",
 	},
 	HTTP: httpConfig{
@@ -144,8 +146,3 @@ const (
 	legacyBLSKmsTypeFile    = "file"
 	legacyBLSKmsTypeNone    = "none"
 )
-
-const (
-	localListenIP  = "127.0.0.1"
-	publicListenIP = "0.0.0.0"
-)

cmd/harmony/flags.go

@@ -15,6 +15,7 @@ var (
 		noStakingFlag,
 		shardIDFlag,
 		isArchiveFlag,
+		isOfflineFlag,
 		dataDirFlag,
 
 		legacyNodeTypeFlag,
@@ -38,6 +39,7 @@ var (
 
 	p2pFlags = []cli.Flag{
 		p2pPortFlag,
+		p2pIPFlag,
 		p2pKeyFileFlag,
 
 		legacyKeyFileFlag,
@@ -178,6 +180,11 @@ var (
 		Usage:    "run node in archive mode",
 		DefValue: defaultConfig.General.IsArchival,
 	}
+	isOfflineFlag = cli.BoolFlag{
+		Name:     "run.offline",
+		Usage:    "run node in offline mode",
+		DefValue: defaultConfig.General.IsOffline,
+	}
 	dataDirFlag = cli.StringFlag{
 		Name:     "datadir",
 		Usage:    "directory of chain database",
@@ -271,6 +278,10 @@ func applyGeneralFlags(cmd *cobra.Command, config *harmonyConfig) {
 	} else if cli.IsFlagChanged(cmd, legacyDataDirFlag) {
 		config.General.DataDir = cli.GetStringFlagValue(cmd, legacyDataDirFlag)
 	}
+
+	if cli.IsFlagChanged(cmd, isOfflineFlag) {
+		config.General.IsOffline = cli.GetBoolFlagValue(cmd, isOfflineFlag)
+	}
 }
 
 // network flags
@@ -360,6 +371,11 @@ var (
 		Usage:    "port to listen for p2p protocols",
 		DefValue: defaultConfig.P2P.Port,
 	}
+	p2pIPFlag = cli.StringFlag{
+		Name:     "p2p.ip",
+		Usage:    "ip to listen for p2p protocols",
+		DefValue: defaultConfig.P2P.IP,
+	}
 	p2pKeyFileFlag = cli.StringFlag{
 		Name:     "p2p.keyfile",
 		Usage:    "the p2p key file of the harmony node",
@@ -377,6 +393,12 @@ func applyP2PFlags(cmd *cobra.Command, config *harmonyConfig) {
 	if cli.IsFlagChanged(cmd, p2pPortFlag) {
 		config.P2P.Port = cli.GetIntFlagValue(cmd, p2pPortFlag)
 	}
+	if cli.IsFlagChanged(cmd, p2pIPFlag) {
+		config.P2P.IP = cli.GetStringFlagValue(cmd, p2pIPFlag)
+	}
+	if config.General.IsOffline {
+		config.P2P.IP = nodeconfig.DefaultLocalListenIP
+	}
 
 	if cli.IsFlagChanged(cmd, p2pKeyFileFlag) {
 		config.P2P.KeyFile = cli.GetStringFlagValue(cmd, p2pKeyFileFlag)
@@ -1068,11 +1090,11 @@ func applyLegacyMiscFlags(cmd *cobra.Command, config *harmonyConfig) {
 
 	if cli.IsFlagChanged(cmd, legacyPublicRPCFlag) {
 		if !cli.GetBoolFlagValue(cmd, legacyPublicRPCFlag) {
-			config.HTTP.IP = localListenIP
+			config.HTTP.IP = nodeconfig.DefaultLocalListenIP
-			config.WS.IP = localListenIP
+			config.WS.IP = nodeconfig.DefaultLocalListenIP
 		} else {
-			config.HTTP.IP = publicListenIP
+			config.HTTP.IP = nodeconfig.DefaultPublicListenIP
-			config.WS.IP = publicListenIP
+			config.WS.IP = nodeconfig.DefaultPublicListenIP
 		}
 	}
 

cmd/harmony/flags_test.go

@@ -51,6 +51,7 @@ func TestHarmonyFlags(t *testing.T) {
 				},
 				P2P: p2pConfig{
 					Port:    9000,
+					IP:      defaultConfig.P2P.IP,
 					KeyFile: defaultConfig.P2P.KeyFile,
 				},
 				HTTP: httpConfig{
@@ -299,6 +300,7 @@ func TestP2PFlags(t *testing.T) {
 			args: []string{"--p2p.port", "9001", "--p2p.keyfile", "./key.file"},
 			expConfig: p2pConfig{
 				Port:    9001,
+				IP:      nodeconfig.DefaultPublicListenIP,
 				KeyFile: "./key.file",
 			},
 		},
@@ -306,6 +308,7 @@ func TestP2PFlags(t *testing.T) {
 			args: []string{"--port", "9001", "--key", "./key.file"},
 			expConfig: p2pConfig{
 				Port:    9001,
+				IP:      nodeconfig.DefaultPublicListenIP,
 				KeyFile: "./key.file",
 			},
 		},
@@ -388,7 +391,7 @@ func TestRPCFlags(t *testing.T) {
 			expConfig: httpConfig{
 				Enabled:        true,
 				RosettaEnabled: false,
-				IP:             publicListenIP,
+				IP:             nodeconfig.DefaultPublicListenIP,
 				Port:           9501,
 				RosettaPort:    9701,
 			},
@@ -448,7 +451,7 @@ func TestWSFlags(t *testing.T) {
 			args: []string{"--ip", "8.8.8.8", "--port", "9001", "--public_rpc"},
 			expConfig: wsConfig{
 				Enabled: true,
-				IP:      publicListenIP,
+				IP:      nodeconfig.DefaultPublicListenIP,
 				Port:    9801,
 			},
 		},

cmd/harmony/main.go

@@ -271,6 +271,7 @@ func setupNodeAndRun(hc harmonyConfig) {
 	}
 	currentNode := setupConsensusAndNode(hc, nodeConfig)
 	nodeconfig.GetDefaultConfig().ShardID = nodeConfig.ShardID
+	nodeconfig.GetDefaultConfig().IsOffline = nodeConfig.IsOffline
 
 	// Prepare for graceful shutdown from os signals
 	osSignal := make(chan os.Signal)
@@ -345,7 +346,7 @@ func setupNodeAndRun(hc harmonyConfig) {
 		Str("Role", currentNode.NodeConfig.Role().String()).
 		Str("Version", getHarmonyVersion()).
 		Str("multiaddress",
-			fmt.Sprintf("/ip4/%s/tcp/%d/p2p/%s", publicListenIP, hc.P2P.Port, myHost.GetID().Pretty()),
+			fmt.Sprintf("/ip4/%s/tcp/%d/p2p/%s", hc.P2P.IP, hc.P2P.Port, myHost.GetID().Pretty()),
 		).
 		Msg(startMsg)
 
@@ -369,7 +370,9 @@ func setupNodeAndRun(hc harmonyConfig) {
 
 	if err := currentNode.BootstrapConsensus(); err != nil {
 		fmt.Println("could not bootstrap consensus", err.Error())
-		os.Exit(-1)
+		if !currentNode.NodeConfig.IsOffline {
+			os.Exit(-1)
+		}
 	}
 
 	if err := currentNode.Start(); err != nil {
@@ -494,6 +497,7 @@ func createGlobalConfig(hc harmonyConfig) (*nodeconfig.ConfigType, error) {
 	netType := nodeconfig.NetworkType(hc.Network.NetworkType)
 	nodeconfig.SetNetworkType(netType) // sets for both global and shard configs
 	nodeConfig.SetArchival(hc.General.IsArchival)
+	nodeConfig.IsOffline = hc.General.IsOffline
 
 	// P2P private key is used for secure message transfer between p2p nodes.
 	nodeConfig.P2PPriKey, _, err = utils.LoadKeyFromFile(hc.P2P.KeyFile)
@@ -503,7 +507,7 @@ func createGlobalConfig(hc harmonyConfig) (*nodeconfig.ConfigType, error) {
 	}
 
 	selfPeer := p2p.Peer{
-		IP:              publicListenIP,
+		IP:              hc.P2P.IP,
 		Port:            strconv.Itoa(hc.P2P.Port),
 		ConsensusPubKey: nodeConfig.ConsensusPriKey[0].Pub.Object,
 	}
@@ -575,11 +579,11 @@ func setupConsensusAndNode(hc harmonyConfig, nodeConfig *nodeconfig.ConfigType)
 	}
 
 	// Syncing provider is provided by following rules:
-	//   1. If starting with a localnet, use local sync peers.
+	//   1. If starting with a localnet or offline, use local sync peers.
 	//   2. If specified with --dns=false, use legacy syncing which is syncing through self-
 	//      discover peers.
 	//   3. Else, use the dns for syncing.
-	if hc.Network.NetworkType == nodeconfig.Localnet {
+	if hc.Network.NetworkType == nodeconfig.Localnet || hc.General.IsOffline {
 		epochConfig := shard.Schedule.InstanceForEpoch(ethCommon.Big0)
 		selfPort := hc.P2P.Port
 		currentNode.SyncingPeerProvider = node.NewLocalSyncingPeerProvider(

internal/configs/node/config.go

@@ -78,6 +78,7 @@ type ConfigType struct {
 	IP              string              // IP of the node.
 	RPCServer       RPCServerConfig     // RPC server port and ip
 	RosettaServer   RosettaServerConfig // rosetta server port and ip
+	IsOffline       bool
 	StringRole      string
 	P2PPriKey       p2p_crypto.PrivKey
 	ConsensusPriKey multibls.PrivateKeys

internal/configs/node/network.go

@@ -39,6 +39,10 @@ const (
 )
 
 const (
+	// DefaultLocalListenIP is the IP used for local hosting
+	DefaultLocalListenIP = "127.0.0.1"
+	// DefaultPublicListenIP is the IP used for public hosting
+	DefaultPublicListenIP = "0.0.0.0"
 	// DefaultP2PPort is the key to be used for p2p communication
 	DefaultP2PPort = 9000
 	// DefaultDNSPort is the default DNS port. The actual port used is DNSPort - 3000. This is a

node/node.go

@@ -556,17 +556,19 @@ func (node *Node) Start() error {
 		Uint32("shard-id", node.Consensus.ShardID).
 		Msg("starting with these topics")
 
-	for key, isCon := range groups {
+	if !node.NodeConfig.IsOffline {
-		topicHandle, err := node.host.GetOrJoin(string(key))
+		for key, isCon := range groups {
-		if err != nil {
+			topicHandle, err := node.host.GetOrJoin(string(key))
-			return err
+			if err != nil {
+				return err
+			}
+			allTopics = append(
+				allTopics, u{
+					NamedTopic:     p2p.NamedTopic{string(key), topicHandle},
+					consensusBound: isCon,
+				},
+			)
 		}
-		allTopics = append(
-			allTopics, u{
-				NamedTopic:     p2p.NamedTopic{string(key), topicHandle},
-				consensusBound: isCon,
-			},
-		)
 	}
 	pubsub := node.host.PubSub()
 	ownID := node.host.GetID()

node/node_syncing.go

@@ -173,6 +173,10 @@ func (p *LocalSyncingPeerProvider) SyncingPeers(shardID uint32) (peers []p2p.Pee
 
 // DoBeaconSyncing update received beaconchain blocks and downloads missing beacon chain blocks
 func (node *Node) DoBeaconSyncing() {
+	if node.NodeConfig.IsOffline {
+		return
+	}
+
 	go func(node *Node) {
 		// TODO ek – infinite loop; add shutdown/cleanup logic
 		for beaconBlock := range node.BeaconBlockChannel {
@@ -218,6 +222,10 @@ func (node *Node) DoBeaconSyncing() {
 
 // DoSyncing keep the node in sync with other peers, willJoinConsensus means the node will try to join consensus after catch up
 func (node *Node) DoSyncing(bc *core.BlockChain, worker *worker.Worker, willJoinConsensus bool) {
+	if node.NodeConfig.IsOffline {
+		return
+	}
+
 	ticker := time.NewTicker(time.Duration(SyncFrequency) * time.Second)
 	// TODO ek – infinite loop; add shutdown/cleanup logic
 	for {
@@ -349,6 +357,10 @@ func (node *Node) SendNewBlockToUnsync() {
 // CalculateResponse implements DownloadInterface on Node object.
 func (node *Node) CalculateResponse(request *downloader_pb.DownloaderRequest, incomingPeer string) (*downloader_pb.DownloaderResponse, error) {
 	response := &downloader_pb.DownloaderResponse{}
+	if node.NodeConfig.IsOffline {
+		return response, nil
+	}
+
 	switch request.Type {
 	case downloader_pb.DownloaderRequest_BLOCKHASH:
 		if request.BlockHash == nil {

p2p/host.go

@@ -13,7 +13,7 @@ import (
 	"github.com/harmony-one/bls/ffi/go/bls"
 	nodeconfig "github.com/harmony-one/harmony/internal/configs/node"
 	"github.com/harmony-one/harmony/internal/utils"
-	libp2p "github.com/libp2p/go-libp2p"
+	"github.com/libp2p/go-libp2p"
 	libp2p_crypto "github.com/libp2p/go-libp2p-core/crypto"
 	libp2p_host "github.com/libp2p/go-libp2p-core/host"
 	libp2p_network "github.com/libp2p/go-libp2p-core/network"
@@ -65,7 +65,7 @@ const (
 
 // NewHost ..
 func NewHost(self *Peer, key libp2p_crypto.PrivKey) (Host, error) {
-	listenAddr, err := ma.NewMultiaddr(fmt.Sprintf("/ip4/0.0.0.0/tcp/%s", self.Port))
+	listenAddr, err := ma.NewMultiaddr(fmt.Sprintf("/ip4/%s/tcp/%s", self.IP, self.Port))
 	if err != nil {
 		return nil, errors.Wrapf(err,
 			"cannot create listen multiaddr from port %#v", self.Port)

p2p/tests/host_test.go

@@ -77,6 +77,7 @@ func TestConnectionToInvalidPeer(t *testing.T) {
 	assert.NotEmpty(t, discoveredHost.GetID())
 
 	discoveredPeer := discoveredHost.GetSelfPeer()
+	discoveredPeer.IP = "8.8.8.8" // force invalid peer
 
 	err = host.ConnectHostPeer(discoveredPeer)
 	assert.Error(t, err)

rosetta/common/config.go

@@ -55,14 +55,14 @@ type SyncStatus int
 
 // Sync status enum
 const (
-	SyncingStartup SyncStatus = iota
+	SyncingUnknown SyncStatus = iota
 	SyncingNewBlock
 	SyncingFinish
 )
 
 // String ..
 func (s SyncStatus) String() string {
-	return [...]string{"booting syncing service", "syncing new block(s)", "fully synced"}[s]
+	return [...]string{"unknown", "syncing new block(s)", "fully synced"}[s]
 }
 
 // SubNetworkMetadata for the sub network identifier of a shard

rosetta/services/network.go

@@ -87,7 +87,7 @@ func (s *NetworkAPI) NetworkStatus(
 	if s.hmy.NodeAPI.IsOutOfSync(s.hmy.BlockChain) {
 		syncStatus = common.SyncingNewBlock
 	} else if targetHeight == 0 {
-		syncStatus = common.SyncingStartup
+		syncStatus = common.SyncingUnknown
 	}
 	stage := syncStatus.String()
 

test/helpers/p2p.go

@@ -3,6 +3,7 @@ package helpers
 import (
 	"github.com/harmony-one/bls/ffi/go/bls"
 	harmony_bls "github.com/harmony-one/harmony/crypto/bls"
+	nodeconfig "github.com/harmony-one/harmony/internal/configs/node"
 	"github.com/harmony-one/harmony/p2p"
 	libp2p_crypto "github.com/libp2p/go-libp2p-crypto"
 	"github.com/pkg/errors"
@@ -25,8 +26,8 @@ var (
 
 func init() {
 	Hosts = []Host{
-		{IP: "127.0.0.1", Port: "9000"},
+		{IP: nodeconfig.DefaultLocalListenIP, Port: "9000"},
-		{IP: "8.8.8.8", Port: "9001"},
+		{IP: nodeconfig.DefaultLocalListenIP, Port: "9001"},
 	}
 
 	Topics = []string{