From 6540ffa70b91b2c1924c72bca06653fa5fa8c602 Mon Sep 17 00:00:00 2001 From: Rongjian Lan Date: Sun, 23 Jun 2019 13:58:41 -0700 Subject: [PATCH 1/9] Fine tune maxprocs and minpeer interval --- api/service/networkinfo/service.go | 2 +- cmd/harmony/main.go | 6 +++++- consensus/consensus_v2.go | 16 ++++++++-------- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/api/service/networkinfo/service.go b/api/service/networkinfo/service.go index 8162dae68..834e3a499 100644 --- a/api/service/networkinfo/service.go +++ b/api/service/networkinfo/service.go @@ -46,7 +46,7 @@ var ( const ( waitInRetry = 2 * time.Second connectionTimeout = 3 * time.Minute - findPeerInterval = 30 * time.Second + findPeerInterval = 60 * time.Second // register to bootnode every ticker dhtTicker = 6 * time.Hour diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index 59ef0ba13..9aa839c09 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -7,6 +7,7 @@ import ( "math/rand" "os" "path" + "runtime" "time" "github.com/ethereum/go-ethereum/ethdb" @@ -140,6 +141,9 @@ var ( ) func initSetup() { + // Add GOMAXPROCS to achieve max performance. + runtime.GOMAXPROCS(runtime.NumCPU() * 4) + // Set port and ip to global config. nodeconfig.GetDefaultConfig().Port = *port nodeconfig.GetDefaultConfig().IP = *ip @@ -281,7 +285,7 @@ func createGlobalConfig() *nodeconfig.ConfigType { } nodeConfig.Host, err = p2pimpl.NewHost(&nodeConfig.SelfPeer, nodeConfig.P2pPriKey) - if *logConn { + if *logConn && nodeConfig.GetNetworkType() != nodeconfig.Mainnet { nodeConfig.Host.GetP2PHost().Network().Notify(utils.NewConnLogger(utils.GetLogInstance())) } if err != nil { diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index d571431b6..29850fb56 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -116,7 +116,7 @@ func (consensus *Consensus) announce(block *types.Block) { if err := consensus.host.SendMessageToGroups([]p2p.GroupID{p2p.NewGroupIDByShardID(p2p.ShardID(consensus.ShardID))}, host.ConstructP2pMessage(byte(17), msgToSend)); err != nil { consensus.getLogger().Warn("[Announce] Cannot send announce message", "groupID", p2p.NewGroupIDByShardID(p2p.ShardID(consensus.ShardID))) } else { - consensus.getLogger().Debug("[Announce] Sent Announce Message!!", "BlockHash", block.Hash(), "BlockNum", block.NumberU64()) + consensus.getLogger().Info("[Announce] Sent Announce Message!!", "BlockHash", block.Hash(), "BlockNum", block.NumberU64()) } consensus.getLogger().Debug("[Announce] Switching phase", "From", consensus.phase, "To", Prepare) @@ -260,7 +260,7 @@ func (consensus *Consensus) onPrepare(msg *msg_pb.Message) { defer consensus.mutex.Unlock() if len(prepareSigs) >= consensus.Quorum() { // already have enough signatures - consensus.getLogger().Info("[OnPrepare] Received Additional Prepare Message", "ValidatorPubKey", validatorPubKey) + consensus.getLogger().Debug("[OnPrepare] Received Additional Prepare Message", "ValidatorPubKey", validatorPubKey) return } // proceed only when the message is not received before @@ -282,7 +282,7 @@ func (consensus *Consensus) onPrepare(msg *msg_pb.Message) { return } - consensus.getLogger().Debug("[OnPrepare] Received New Prepare Signature", "NumReceivedSoFar", len(prepareSigs), "validatorPubKey", validatorPubKey, "PublicKeys", len(consensus.PublicKeys)) + consensus.getLogger().Info("[OnPrepare] Received New Prepare Signature", "NumReceivedSoFar", len(prepareSigs), "validatorPubKey", validatorPubKey, "PublicKeys", len(consensus.PublicKeys)) prepareSigs[validatorPubKey] = &sign // Set the bitmap indicating that this validator signed. if err := prepareBitmap.SetKey(recvMsg.SenderPubkey, true); err != nil { @@ -463,7 +463,7 @@ func (consensus *Consensus) onPrepared(msg *msg_pb.Message) { if err := consensus.host.SendMessageToGroups([]p2p.GroupID{p2p.NewGroupIDByShardID(p2p.ShardID(consensus.ShardID))}, host.ConstructP2pMessage(byte(17), msgToSend)); err != nil { consensus.getLogger().Warn("[OnPrepared] Cannot send commit message!!") } else { - consensus.getLogger().Debug("[OnPrepared] Sent Commit Message!!", "BlockHash", consensus.blockHash, "BlockNum", consensus.blockNum) + consensus.getLogger().Info("[OnPrepared] Sent Commit Message!!", "BlockHash", consensus.blockHash, "BlockNum", consensus.blockNum) } consensus.getLogger().Debug("[OnPrepared] Switching phase", "From", consensus.phase, "To", Commit) @@ -527,7 +527,7 @@ func (consensus *Consensus) onCommit(msg *msg_pb.Message) { // proceed only when the message is not received before _, ok := commitSigs[validatorPubKey] if ok { - consensus.getLogger().Info("[OnCommit] Already received commit message from the validator", "validatorPubKey", validatorPubKey) + consensus.getLogger().Debug("[OnCommit] Already received commit message from the validator", "validatorPubKey", validatorPubKey) return } @@ -548,7 +548,7 @@ func (consensus *Consensus) onCommit(msg *msg_pb.Message) { return } - consensus.getLogger().Debug("[OnCommit] Received new commit message", "numReceivedSoFar", len(commitSigs), "MsgViewID", recvMsg.ViewID, "MsgBlockNum", recvMsg.BlockNum, "validatorPubKey", validatorPubKey) + consensus.getLogger().Info("[OnCommit] Received new commit message", "numReceivedSoFar", len(commitSigs), "MsgViewID", recvMsg.ViewID, "MsgBlockNum", recvMsg.BlockNum, "validatorPubKey", validatorPubKey) commitSigs[validatorPubKey] = &sign // Set the bitmap indicating that this validator signed. if err := commitBitmap.SetKey(recvMsg.SenderPubkey, true); err != nil { @@ -571,7 +571,7 @@ func (consensus *Consensus) onCommit(msg *msg_pb.Message) { if rewardThresholdIsMet { go func(viewID uint64) { consensus.commitFinishChan <- viewID - consensus.getLogger().Debug("[OnCommit] 90% Enough commits received", "NumCommits", len(commitSigs)) + consensus.getLogger().Info("[OnCommit] 90% Enough commits received", "NumCommits", len(commitSigs)) }(consensus.viewID) } } @@ -612,7 +612,7 @@ func (consensus *Consensus) finalizeCommits() { if err := consensus.host.SendMessageToGroups([]p2p.GroupID{p2p.NewGroupIDByShardID(p2p.ShardID(consensus.ShardID))}, host.ConstructP2pMessage(byte(17), msgToSend)); err != nil { consensus.getLogger().Warn("[Finalizing] Cannot send committed message", "error", err) } else { - consensus.getLogger().Debug("[Finalizing] Sent Committed Message", "BlockHash", consensus.blockHash, "BlockNum", consensus.blockNum) + consensus.getLogger().Info("[Finalizing] Sent Committed Message", "BlockHash", consensus.blockHash, "BlockNum", consensus.blockNum) } consensus.reportMetrics(*block) From b0f010912785057a0eeb6ac6a007391039629909 Mon Sep 17 00:00:00 2001 From: Rongjian Lan Date: Sun, 23 Jun 2019 14:34:53 -0700 Subject: [PATCH 2/9] Add genesis string --- node/node_genesis.go | 1 + 1 file changed, 1 insertion(+) diff --git a/node/node_genesis.go b/node/node_genesis.go index cc4fb3c4c..96350bb31 100644 --- a/node/node_genesis.go +++ b/node/node_genesis.go @@ -102,6 +102,7 @@ func (node *Node) SetupGenesisBlock(db ethdb.Database, shardID uint32, myShardSt GasLimit: params.GenesisGasLimit * 1000, ShardStateHash: myShardState.Hash(), ShardState: myShardState.DeepCopy(), + ExtraData: []byte("Harmony for One and All, Open Consensus for 10B"), } // Store genesis block into db. From 5a5730506fd4b20f4a982dd0385f14e662456cbe Mon Sep 17 00:00:00 2001 From: Rongjian Lan Date: Sun, 23 Jun 2019 15:00:07 -0700 Subject: [PATCH 3/9] Add more log --- consensus/consensus_v2.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 29850fb56..e3c029789 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -103,6 +103,7 @@ func (consensus *Consensus) announce(block *types.Block) { } consensus.PbftLog.AddMessage(pbftMsg) + consensus.getLogger().Debug("[Announce] Added Announce message in pbftLog", "MsgblockHash", pbftMsg.BlockHash, "MsgViewID", pbftMsg.ViewID, "MsgBlockNum", pbftMsg.BlockNum) consensus.PbftLog.AddBlock(block) // Leader sign the block hash itself @@ -247,7 +248,7 @@ func (consensus *Consensus) onPrepare(msg *msg_pb.Message) { if !consensus.PbftLog.HasMatchingViewAnnounce(consensus.blockNum, consensus.viewID, recvMsg.BlockHash) { consensus.getLogger().Debug("[OnPrepare] No Matching Announce message", "MsgblockHash", recvMsg.BlockHash, "MsgBlockNum", recvMsg.BlockNum) - return + //return } validatorPubKey := recvMsg.SenderPubkey.SerializeToHexStr() From 0fba928ed78c8efaeea33b7d5f952904a420eff3 Mon Sep 17 00:00:00 2001 From: Rongjian Lan Date: Sun, 23 Jun 2019 15:01:47 -0700 Subject: [PATCH 4/9] Add delete log --- consensus/pbft_log.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/consensus/pbft_log.go b/consensus/pbft_log.go index 5a571063b..c1ff61c4f 100644 --- a/consensus/pbft_log.go +++ b/consensus/pbft_log.go @@ -90,6 +90,7 @@ func (log *PbftLog) GetBlocksByNumber(number uint64) []*types.Block { // DeleteBlocksLessThan deletes blocks less than given block number func (log *PbftLog) DeleteBlocksLessThan(number uint64) { + utils.GetLogInstance().Info("Deleted Block Less Than", "num", number) found := mapset.NewSet() it := log.Blocks().Iterator() for block := range it.C { @@ -102,6 +103,7 @@ func (log *PbftLog) DeleteBlocksLessThan(number uint64) { // DeleteBlockByNumber deletes block of specific number func (log *PbftLog) DeleteBlockByNumber(number uint64) { + utils.GetLogInstance().Info("Deleted Block Equal", "num", number) found := mapset.NewSet() it := log.Blocks().Iterator() for block := range it.C { @@ -114,6 +116,7 @@ func (log *PbftLog) DeleteBlockByNumber(number uint64) { // DeleteMessagesLessThan deletes messages less than given block number func (log *PbftLog) DeleteMessagesLessThan(number uint64) { + utils.GetLogInstance().Info("Deleted PbftLog Less Than", "num", number) found := mapset.NewSet() it := log.Messages().Iterator() for msg := range it.C { From 62100d0c2129ce709414bd3379a2937be5e3b833 Mon Sep 17 00:00:00 2001 From: Rongjian Lan Date: Sun, 23 Jun 2019 15:02:30 -0700 Subject: [PATCH 5/9] disable reset state in retry --- node/node_newblock.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/node_newblock.go b/node/node_newblock.go index e11e8a5c7..fd19ac031 100644 --- a/node/node_newblock.go +++ b/node/node_newblock.go @@ -45,7 +45,7 @@ func (node *Node) WaitForConsensusReadyv2(readySignal chan struct{}, stopChan ch case <-time.After(ConsensusTimeOut * time.Second): if node.Consensus.PubKey.IsEqual(node.Consensus.LeaderPubKey) { utils.GetLogInstance().Debug("Leader consensus timeout, retry!", "count", timeoutCount) - node.Consensus.ResetState() + //node.Consensus.ResetState() timeoutCount++ if newBlock != nil { // Send the new block to Consensus so it can be confirmed. From 06f883cf583fc673cabc313d38b0d66b6516cdd0 Mon Sep 17 00:00:00 2001 From: Rongjian Lan Date: Sun, 23 Jun 2019 15:27:09 -0700 Subject: [PATCH 6/9] Fix mainnet --- consensus/consensus_v2.go | 3 ++- consensus/pbft_log.go | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index e3c029789..dc763d29d 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -176,7 +176,8 @@ func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) { consensus.getLogger().Debug("[OnAnnounce] Leader is malicious", "leaderKey", consensus.LeaderPubKey.SerializeToHexStr()) consensus.startViewChange(consensus.viewID + 1) } - return + consensus.getLogger().Debug("[OnAnnounce] Announce message received again", "leaderKey", consensus.LeaderPubKey.SerializeToHexStr()) + //return } consensus.getLogger().Debug("[OnAnnounce] Announce message Added", "MsgViewID", recvMsg.ViewID, "MsgBlockNum", recvMsg.BlockNum) diff --git a/consensus/pbft_log.go b/consensus/pbft_log.go index c1ff61c4f..b4e5121e0 100644 --- a/consensus/pbft_log.go +++ b/consensus/pbft_log.go @@ -171,13 +171,13 @@ func (log *PbftLog) GetMessagesByTypeSeqHash(typ msg_pb.MessageType, blockNum ui // HasMatchingAnnounce returns whether the log contains announce type message with given blockNum, blockHash func (log *PbftLog) HasMatchingAnnounce(blockNum uint64, blockHash common.Hash) bool { found := log.GetMessagesByTypeSeqHash(msg_pb.MessageType_ANNOUNCE, blockNum, blockHash) - return len(found) == 1 + return len(found) >= 1 } // HasMatchingViewAnnounce returns whether the log contains announce type message with given blockNum, viewID and blockHash func (log *PbftLog) HasMatchingViewAnnounce(blockNum uint64, viewID uint64, blockHash common.Hash) bool { found := log.GetMessagesByTypeSeqViewHash(msg_pb.MessageType_ANNOUNCE, blockNum, viewID, blockHash) - return len(found) == 1 + return len(found) >= 1 } // HasMatchingPrepared returns whether the log contains prepared message with given blockNum, viewID and blockHash @@ -189,7 +189,7 @@ func (log *PbftLog) HasMatchingPrepared(blockNum uint64, blockHash common.Hash) // HasMatchingViewPrepared returns whether the log contains prepared message with given blockNum, viewID and blockHash func (log *PbftLog) HasMatchingViewPrepared(blockNum uint64, viewID uint64, blockHash common.Hash) bool { found := log.GetMessagesByTypeSeqViewHash(msg_pb.MessageType_PREPARED, blockNum, viewID, blockHash) - return len(found) == 1 + return len(found) >= 1 } // GetMessagesByTypeSeqView returns pbft messages with matching type, blockNum and viewID From 622f0d8f0b59fae7896fa257c65fc41e8c86a415 Mon Sep 17 00:00:00 2001 From: Rongjian Lan Date: Sun, 23 Jun 2019 15:48:05 -0700 Subject: [PATCH 7/9] mainnet! --- consensus/consensus.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consensus/consensus.go b/consensus/consensus.go index f943e0da7..eadfa03b6 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -28,7 +28,7 @@ import ( ) // BlockReward is the block reward, to be split evenly among block signers. -var BlockReward = new(big.Int).Mul(big.NewInt(30), big.NewInt(denominations.One)) +var BlockReward = new(big.Int).Mul(big.NewInt(24), big.NewInt(denominations.One)) // Consensus is the main struct with all states and data related to consensus process. type Consensus struct { From c7f2fed3e8eb27b4f0a50eb9f387ab8fbfac7abb Mon Sep 17 00:00:00 2001 From: Rongjian Lan Date: Sun, 23 Jun 2019 15:50:47 -0700 Subject: [PATCH 8/9] Remove unncesssary logs --- api/service/networkinfo/service.go | 1 - consensus/pbft_log.go | 3 --- 2 files changed, 4 deletions(-) diff --git a/api/service/networkinfo/service.go b/api/service/networkinfo/service.go index 834e3a499..a85fcd6d2 100644 --- a/api/service/networkinfo/service.go +++ b/api/service/networkinfo/service.go @@ -179,7 +179,6 @@ func (s *Service) findPeers() { return } for peer := range s.peerInfo { - utils.GetLogInstance().Info("Got peers", "peer", peer) if peer.ID != s.Host.GetP2PHost().ID() && len(peer.ID) > 0 { // utils.GetLogInstance().Info("Found Peer", "peer", peer.ID, "addr", peer.Addrs, "my ID", s.Host.GetP2PHost().ID()) if err := s.Host.GetP2PHost().Connect(ctx, peer); err != nil { diff --git a/consensus/pbft_log.go b/consensus/pbft_log.go index b4e5121e0..d84606a90 100644 --- a/consensus/pbft_log.go +++ b/consensus/pbft_log.go @@ -90,7 +90,6 @@ func (log *PbftLog) GetBlocksByNumber(number uint64) []*types.Block { // DeleteBlocksLessThan deletes blocks less than given block number func (log *PbftLog) DeleteBlocksLessThan(number uint64) { - utils.GetLogInstance().Info("Deleted Block Less Than", "num", number) found := mapset.NewSet() it := log.Blocks().Iterator() for block := range it.C { @@ -103,7 +102,6 @@ func (log *PbftLog) DeleteBlocksLessThan(number uint64) { // DeleteBlockByNumber deletes block of specific number func (log *PbftLog) DeleteBlockByNumber(number uint64) { - utils.GetLogInstance().Info("Deleted Block Equal", "num", number) found := mapset.NewSet() it := log.Blocks().Iterator() for block := range it.C { @@ -116,7 +114,6 @@ func (log *PbftLog) DeleteBlockByNumber(number uint64) { // DeleteMessagesLessThan deletes messages less than given block number func (log *PbftLog) DeleteMessagesLessThan(number uint64) { - utils.GetLogInstance().Info("Deleted PbftLog Less Than", "num", number) found := mapset.NewSet() it := log.Messages().Iterator() for msg := range it.C { From 191ed3ee98fc7d9b17bf54260fb399d1e107da14 Mon Sep 17 00:00:00 2001 From: Rongjian Lan Date: Sun, 23 Jun 2019 15:55:27 -0700 Subject: [PATCH 9/9] Correct genesis string --- node/node_genesis.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/node_genesis.go b/node/node_genesis.go index 96350bb31..b6adb47da 100644 --- a/node/node_genesis.go +++ b/node/node_genesis.go @@ -102,7 +102,7 @@ func (node *Node) SetupGenesisBlock(db ethdb.Database, shardID uint32, myShardSt GasLimit: params.GenesisGasLimit * 1000, ShardStateHash: myShardState.Hash(), ShardState: myShardState.DeepCopy(), - ExtraData: []byte("Harmony for One and All, Open Consensus for 10B"), + ExtraData: []byte("Harmony for One and All. Open Consensus for 10B."), } // Store genesis block into db.