Add consensus retry so the consensus process won't get stuck forever

benchmark_main.go

@@ -99,11 +99,6 @@ func main() {
 		go func() {
 			node.WaitForConsensusReady(consensus.ReadySignal)
 		}()
-	} else {
-		// Node waiting to add new block to the blockchain
-		go func() {
-			node.WaitForConsensusReady(consensus.ReadySignal)
-		}()
 	}
 
 	node.StartServer(*port)

consensus/consensus_leader.go

@@ -276,16 +276,19 @@ func (consensus *Consensus) processResponseMessage(payload []byte) {
 	_ = response
 	_ = signature
 
+
+	// proceed only when the message is not received before and this consensus phase is not done.
+
+	shouldProcess := true
+	consensus.mutex.Lock()
 	// check consensus Id
 	if consensusId != consensus.consensusId {
+		shouldProcess = false
 		consensus.Log.Debug("[ERROR] Received RESPONSE with wrong consensus Id", "myConsensusId", consensus.consensusId, "theirConsensusId", consensusId, "consensus", consensus)
-		return
 	}
 
-	// proceed only when the message is not received before and this consensus phase is not done.
-	consensus.mutex.Lock()
 	_, ok := consensus.responses[validatorId]
-	shouldProcess := !ok
+	shouldProcess = shouldProcess && !ok
 	if shouldProcess {
 		consensus.responses[validatorId] = validatorId
 		//consensus.Log.Debug("Number of responses received", "count", len(consensus.responses), "consensudId", consensusId)

node/node_handler.go

@@ -113,9 +113,19 @@ func (node *Node) WaitForConsensusReady(readySignal chan int) {
 	node.log.Debug("Waiting for consensus ready", "node", node)
 
 	var newBlock *blockchain.Block
+	timeoutCount := 0
 	for { // keep waiting for consensus ready
-		<-readySignal
+		retry := false
+		select {
+		case <-readySignal:
+		case <-time.After(8 * time.Second):
+			retry = true
+			timeoutCount++
+			node.log.Debug("Consensus timeout, retry!", "count", timeoutCount, "node", node)
+		}
+
 		//node.log.Debug("Adding new block", "currentChainSize", len(node.blockchain.Blocks), "numTxs", len(node.blockchain.GetLatestBlock().Transactions), "PrevHash", node.blockchain.GetLatestBlock().PrevBlockHash, "Hash", node.blockchain.GetLatestBlock().Hash)
+		if !retry {
 			for {
 				// Once we have more than 10 transactions pending we will try creating a new block
 				if len(node.pendingTransactions) >= 10 {
@@ -133,11 +143,14 @@ func (node *Node) WaitForConsensusReady(readySignal chan int) {
 				}
 				// If not enough transactions to run consensus,
 				// periodically check whether we have enough transactions to package into block.
+				time.Sleep(1 * time.Second)
+			}
 		}
 
 		// Send the new block to consensus so it can be confirmed.
+		if newBlock != nil {
 			node.BlockChannel <- *newBlock
-		time.Sleep(2 * time.Second)
+		}
 	}
 }