From 3ebcccc132de4bb31ed03920b701f4ea087caaa9 Mon Sep 17 00:00:00 2001 From: Rongjian Lan Date: Fri, 25 Jan 2019 12:20:41 -0800 Subject: [PATCH 01/14] Remove init on bls test --- test/crypto/bls/main.go | 1 - 1 file changed, 1 deletion(-) diff --git a/test/crypto/bls/main.go b/test/crypto/bls/main.go index bdd7181a2..372e2f858 100644 --- a/test/crypto/bls/main.go +++ b/test/crypto/bls/main.go @@ -13,7 +13,6 @@ func main() { startTime := time.Now() for i := 0; i < 1000; i++ { - bls.Init(bls.BLS12_381) var sec bls.SecretKey sec.SetByCSPRNG() if i == 0 { From e9732bc3ff833a7965c1f0ada3494d8622c29ca7 Mon Sep 17 00:00:00 2001 From: Rongjian Lan Date: Wed, 13 Feb 2019 11:47:59 -0800 Subject: [PATCH 02/14] address comments from last PR --- cmd/harmony.go | 1 + drand/drand_leader.go | 4 ++++ node/node_handler.go | 1 + test/p2pchat/chat.go | 4 ++-- 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/cmd/harmony.go b/cmd/harmony.go index 5d463553c..9922527c4 100644 --- a/cmd/harmony.go +++ b/cmd/harmony.go @@ -264,6 +264,7 @@ func main() { // Add randomness protocol // TODO: enable drand only for beacon chain + // TODO: put this in a better place other than main. dRand := drand.New(host, shardID, peers, leader, currentNode.ConfirmedBlockChannel) currentNode.DRand = dRand diff --git a/drand/drand_leader.go b/drand/drand_leader.go index 31ce31bbf..9f9954737 100644 --- a/drand/drand_leader.go +++ b/drand/drand_leader.go @@ -3,6 +3,7 @@ package drand import ( protobuf "github.com/golang/protobuf/proto" drand_proto "github.com/harmony-one/harmony/api/drand" + "github.com/harmony-one/harmony/core" "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/p2p/host" @@ -17,6 +18,9 @@ func (dRand *DRand) WaitForEpochBlock(blockChannel chan *types.Block, stopChan c default: // keep waiting for epoch block newBlock := <-blockChannel + if newBlock.NumberU64()%core.BlocksPerEpoch == 0 { + + } dRand.init(newBlock) case <-stopChan: return diff --git a/node/node_handler.go b/node/node_handler.go index 3c8da7610..aed0b511c 100644 --- a/node/node_handler.go +++ b/node/node_handler.go @@ -261,6 +261,7 @@ func (node *Node) PostConsensusProcessing(newBlock *types.Block) { node.AddNewBlock(newBlock) // TODO: enable drand only for beacon chain + // ConfirmedBlockChannel which is listened by drand leader who will initiate DRG if its a epoch block (first block of a epoch) if node.DRand != nil { go func() { node.ConfirmedBlockChannel <- newBlock diff --git a/test/p2pchat/chat.go b/test/p2pchat/chat.go index 02440bab0..3bfac7c9f 100644 --- a/test/p2pchat/chat.go +++ b/test/p2pchat/chat.go @@ -8,8 +8,8 @@ import ( "os" "sync" - "github.com/ipfs/go-log" - "github.com/libp2p/go-libp2p" + log "github.com/ipfs/go-log" + libp2p "github.com/libp2p/go-libp2p" discovery "github.com/libp2p/go-libp2p-discovery" libp2pdht "github.com/libp2p/go-libp2p-kad-dht" peer "github.com/libp2p/go-libp2p-peer" From 8fae634aba5b524e532fd407bea581c98e753c46 Mon Sep 17 00:00:00 2001 From: Rongjian Lan Date: Wed, 13 Feb 2019 15:38:05 -0800 Subject: [PATCH 03/14] Break tie of core-consensus-core import cycle; add pRnd channel for consensus --- cmd/harmony.go | 1 + consensus/consensus.go | 33 ++++++++++++++-------- consensus/consensus_leader.go | 15 ++++++++++ consensus/consensus_validator.go | 3 +- consensus/{ => engine}/consensus_engine.go | 2 +- consensus/{ => engine}/errors.go | 2 +- core/block_validator.go | 14 ++++----- core/blockchain.go | 32 +++++++++++++++------ core/chain_makers.go | 12 ++++---- core/evm.go | 4 +-- core/headerchain.go | 10 +++---- core/state_processor.go | 10 +++---- drand/drand.go | 5 +++- drand/drand_leader.go | 14 +++++++-- node/worker/worker.go | 6 ++-- 15 files changed, 108 insertions(+), 55 deletions(-) rename consensus/{ => engine}/consensus_engine.go (99%) rename consensus/{ => engine}/errors.go (99%) diff --git a/cmd/harmony.go b/cmd/harmony.go index 9922527c4..3fa3ea517 100644 --- a/cmd/harmony.go +++ b/cmd/harmony.go @@ -266,6 +266,7 @@ func main() { // TODO: enable drand only for beacon chain // TODO: put this in a better place other than main. dRand := drand.New(host, shardID, peers, leader, currentNode.ConfirmedBlockChannel) + currentNode.Consensus.RegisterPRndChannel(dRand.PRndChannel) currentNode.DRand = dRand // If there is a client configured in the node list. diff --git a/consensus/consensus.go b/consensus/consensus.go index 60d067955..c1252363e 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -18,6 +18,7 @@ import ( protobuf "github.com/golang/protobuf/proto" "github.com/harmony-one/bls/ffi/go/bls" consensus_proto "github.com/harmony-one/harmony/api/consensus" + consensus_engine "github.com/harmony-one/harmony/consensus/engine" "github.com/harmony-one/harmony/core/state" "github.com/harmony-one/harmony/core/types" bls_cosi "github.com/harmony-one/harmony/crypto/bls" @@ -96,6 +97,9 @@ type Consensus struct { // verified block to state sync broadcast VerifiedNewBlock chan *types.Block + // Channel for DRG protocol to send pRnd (preimage of randomness resulting from combined vrf randomnesses) to consensus. The first 32 bytes are randomness, the rest is for bitmap. + PRndChannel chan []byte + uniqueIDInstance *utils.UniqueValidatorID // The p2p host used to send/receive p2p messages @@ -213,6 +217,11 @@ func New(host p2p.Host, ShardID string, peers []p2p.Peer, leader p2p.Peer) *Cons return &consensus } +// RegisterPRndChannel registers the channel for receiving randomness preimage from DRG protocol +func (consensus *Consensus) RegisterPRndChannel(pRndChannel chan []byte) { + consensus.PRndChannel = pRndChannel +} + // Checks the basic meta of a consensus message, including the signature. func (consensus *Consensus) checkConsensusMessage(message consensus_proto.Message, publicKey *bls.PublicKey) error { consensusID := message.ConsensusId @@ -222,18 +231,18 @@ func (consensus *Consensus) checkConsensusMessage(message consensus_proto.Messag err := verifyMessageSig(publicKey, message) if err != nil { utils.GetLogInstance().Warn("Failed to verify the message signature", "Error", err) - return ErrInvalidConsensusMessage + return consensus_engine.ErrInvalidConsensusMessage } // check consensus Id if consensusID != consensus.consensusID { utils.GetLogInstance().Warn("Wrong consensus Id", "myConsensusId", consensus.consensusID, "theirConsensusId", consensusID, "consensus", consensus) - return ErrConsensusIDNotMatch + return consensus_engine.ErrConsensusIDNotMatch } if !bytes.Equal(blockHash, consensus.blockHash[:]) { utils.GetLogInstance().Warn("Wrong blockHash", "consensus", consensus) - return ErrInvalidConsensusMessage + return consensus_engine.ErrInvalidConsensusMessage } return nil } @@ -479,7 +488,7 @@ func NewFaker() *Consensus { // VerifyHeader checks whether a header conforms to the consensus rules of the // stock bft engine. -func (consensus *Consensus) VerifyHeader(chain ChainReader, header *types.Header, seal bool) error { +func (consensus *Consensus) VerifyHeader(chain consensus_engine.ChainReader, header *types.Header, seal bool) error { // TODO: implement this return nil } @@ -487,7 +496,7 @@ func (consensus *Consensus) VerifyHeader(chain ChainReader, header *types.Header // VerifyHeaders is similar to VerifyHeader, but verifies a batch of headers // concurrently. The method returns a quit channel to abort the operations and // a results channel to retrieve the async verifications. -func (consensus *Consensus) VerifyHeaders(chain ChainReader, headers []*types.Header, seals []bool) (chan<- struct{}, <-chan error) { +func (consensus *Consensus) VerifyHeaders(chain consensus_engine.ChainReader, headers []*types.Header, seals []bool) (chan<- struct{}, <-chan error) { abort, results := make(chan struct{}), make(chan error, len(headers)) for i := 0; i < len(headers); i++ { results <- nil @@ -495,7 +504,7 @@ func (consensus *Consensus) VerifyHeaders(chain ChainReader, headers []*types.He return abort, results } -func (consensus *Consensus) verifyHeaderWorker(chain ChainReader, headers []*types.Header, seals []bool, index int) error { +func (consensus *Consensus) verifyHeaderWorker(chain consensus_engine.ChainReader, headers []*types.Header, seals []bool, index int) error { var parent *types.Header if index == 0 { parent = chain.GetHeader(headers[0].ParentHash, headers[0].Number.Uint64()-1) @@ -503,7 +512,7 @@ func (consensus *Consensus) verifyHeaderWorker(chain ChainReader, headers []*typ parent = headers[index-1] } if parent == nil { - return ErrUnknownAncestor + return consensus_engine.ErrUnknownAncestor } if chain.GetHeader(headers[index].Hash(), headers[index].Number.Uint64()) != nil { return nil // known block @@ -513,19 +522,19 @@ func (consensus *Consensus) verifyHeaderWorker(chain ChainReader, headers []*typ // verifyHeader checks whether a header conforms to the consensus rules of the // stock bft engine. -func (consensus *Consensus) verifyHeader(chain ChainReader, header, parent *types.Header, uncle bool, seal bool) error { +func (consensus *Consensus) verifyHeader(chain consensus_engine.ChainReader, header, parent *types.Header, uncle bool, seal bool) error { return nil } // VerifySeal implements consensus.Engine, checking whether the given block satisfies // the PoW difficulty requirements. -func (consensus *Consensus) VerifySeal(chain ChainReader, header *types.Header) error { +func (consensus *Consensus) VerifySeal(chain consensus_engine.ChainReader, header *types.Header) error { return nil } // Finalize implements consensus.Engine, accumulating the block and uncle rewards, // setting the final state and assembling the block. -func (consensus *Consensus) Finalize(chain ChainReader, header *types.Header, state *state.DB, txs []*types.Transaction, receipts []*types.Receipt) (*types.Block, error) { +func (consensus *Consensus) Finalize(chain consensus_engine.ChainReader, header *types.Header, state *state.DB, txs []*types.Transaction, receipts []*types.Receipt) (*types.Block, error) { // Accumulate any block and uncle rewards and commit the final state root // Header seems complete, assemble into a block and return accumulateRewards(chain.Config(), state, header) @@ -556,14 +565,14 @@ func (consensus *Consensus) SealHash(header *types.Header) (hash common.Hash) { } // Seal is to seal final block. -func (consensus *Consensus) Seal(chain ChainReader, block *types.Block, results chan<- *types.Block, stop <-chan struct{}) error { +func (consensus *Consensus) Seal(chain consensus_engine.ChainReader, block *types.Block, results chan<- *types.Block, stop <-chan struct{}) error { // TODO: implement final block sealing return nil } // Prepare is to prepare ... // TODO(RJ): fix it. -func (consensus *Consensus) Prepare(chain ChainReader, header *types.Header) error { +func (consensus *Consensus) Prepare(chain consensus_engine.ChainReader, header *types.Header) error { // TODO: implement prepare method return nil } diff --git a/consensus/consensus_leader.go b/consensus/consensus_leader.go index e1a70a82d..2b1759001 100644 --- a/consensus/consensus_leader.go +++ b/consensus/consensus_leader.go @@ -5,6 +5,8 @@ import ( "strconv" "time" + "github.com/harmony-one/harmony/core" + "github.com/ethereum/go-ethereum/rlp" protobuf "github.com/golang/protobuf/proto" "github.com/harmony-one/bls/ffi/go/bls" @@ -47,6 +49,19 @@ func (consensus *Consensus) WaitForNewBlock(blockChannel chan *types.Block, stop time.Sleep(waitForEnoughValidators * time.Millisecond) } + if core.IsEpochBlock(newBlock) { + // Receive pRnd from DRG protocol + utils.GetLogInstance().Debug("[DRG] Waiting for pRnd") + pRndAndBitmap := <-consensus.PRndChannel + utils.GetLogInstance().Debug("[DRG] GOT pRnd", "pRnd", pRndAndBitmap) + pRnd := pRndAndBitmap[:32] + bitmap := pRndAndBitmap[32:] + vrfBitmap, _ := bls_cosi.NewMask(consensus.PublicKeys, consensus.leader.PubKey) + vrfBitmap.SetMask(bitmap) + + // TODO: check validity of pRnd + _ = pRnd + } startTime = time.Now() utils.GetLogInstance().Debug("STARTING CONSENSUS", "numTxs", len(newBlock.Transactions()), "consensus", consensus, "startTime", startTime, "publicKeys", len(consensus.PublicKeys)) for { // Wait until last consensus is finished diff --git a/consensus/consensus_validator.go b/consensus/consensus_validator.go index 783095173..fcfab1f44 100644 --- a/consensus/consensus_validator.go +++ b/consensus/consensus_validator.go @@ -7,6 +7,7 @@ import ( "github.com/ethereum/go-ethereum/rlp" protobuf "github.com/golang/protobuf/proto" consensus_proto "github.com/harmony-one/harmony/api/consensus" + consensus_engine "github.com/harmony-one/harmony/consensus/engine" "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/internal/attack" "github.com/harmony-one/harmony/internal/utils" @@ -74,7 +75,7 @@ func (consensus *Consensus) processAnnounceMessage(message consensus_proto.Messa if err := consensus.checkConsensusMessage(message, consensus.leader.PubKey); err != nil { utils.GetLogInstance().Debug("Failed to check the leader message") - if err == ErrConsensusIDNotMatch { + if err == consensus_engine.ErrConsensusIDNotMatch { utils.GetLogInstance().Debug("sending bft block to state syncing") consensus.sendBFTBlockToStateSyncing(consensusID) } diff --git a/consensus/consensus_engine.go b/consensus/engine/consensus_engine.go similarity index 99% rename from consensus/consensus_engine.go rename to consensus/engine/consensus_engine.go index bd0cf6653..98b26cf0d 100644 --- a/consensus/consensus_engine.go +++ b/consensus/engine/consensus_engine.go @@ -1,4 +1,4 @@ -package consensus +package engine import ( "github.com/ethereum/go-ethereum/common" diff --git a/consensus/errors.go b/consensus/engine/errors.go similarity index 99% rename from consensus/errors.go rename to consensus/engine/errors.go index 6156c4aed..c9581c55b 100644 --- a/consensus/errors.go +++ b/consensus/engine/errors.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package consensus +package engine import "errors" diff --git a/core/block_validator.go b/core/block_validator.go index 4a2b583bd..866732b59 100644 --- a/core/block_validator.go +++ b/core/block_validator.go @@ -20,7 +20,7 @@ import ( "fmt" "github.com/ethereum/go-ethereum/params" - "github.com/harmony-one/harmony/consensus" + consensus_engine "github.com/harmony-one/harmony/consensus/engine" "github.com/harmony-one/harmony/core/state" "github.com/harmony-one/harmony/core/types" ) @@ -30,13 +30,13 @@ import ( // // BlockValidator implements Validator. type BlockValidator struct { - config *params.ChainConfig // Chain configuration options - bc *BlockChain // Canonical block chain - engine consensus.Engine // Consensus engine used for validating + config *params.ChainConfig // Chain configuration options + bc *BlockChain // Canonical block chain + engine consensus_engine.Engine // Consensus engine used for validating } // NewBlockValidator returns a new block validator which is safe for re-use -func NewBlockValidator(config *params.ChainConfig, blockchain *BlockChain, engine consensus.Engine) *BlockValidator { +func NewBlockValidator(config *params.ChainConfig, blockchain *BlockChain, engine consensus_engine.Engine) *BlockValidator { validator := &BlockValidator{ config: config, engine: engine, @@ -55,9 +55,9 @@ func (v *BlockValidator) ValidateBody(block *types.Block) error { } if !v.bc.HasBlockAndState(block.ParentHash(), block.NumberU64()-1) { if !v.bc.HasBlock(block.ParentHash(), block.NumberU64()-1) { - return consensus.ErrUnknownAncestor + return consensus_engine.ErrUnknownAncestor } - return consensus.ErrPrunedAncestor + return consensus_engine.ErrPrunedAncestor } // Header validity is known at this point, check the uncles and transactions header := block.Header() diff --git a/core/blockchain.go b/core/blockchain.go index 90decdb11..43319bfde 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -38,7 +38,7 @@ import ( "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" - "github.com/harmony-one/harmony/consensus" + consensus_engine "github.com/harmony-one/harmony/consensus/engine" "github.com/harmony-one/harmony/core/rawdb" "github.com/harmony-one/harmony/core/state" "github.com/harmony-one/harmony/core/types" @@ -135,7 +135,7 @@ type BlockChain struct { procInterrupt int32 // interrupt signaler for block processing wg sync.WaitGroup // chain processing wait group for shutting down - engine consensus.Engine + engine consensus_engine.Engine processor Processor // block processor interface validator Validator // block and state validator interface vmConfig vm.Config @@ -147,7 +147,7 @@ type BlockChain struct { // NewBlockChain returns a fully initialised block chain using information // available in the database. It initialises the default Ethereum Validator and // Processor. -func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *params.ChainConfig, engine consensus.Engine, vmConfig vm.Config, shouldPreserve func(block *types.Block) bool) (*BlockChain, error) { +func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *params.ChainConfig, engine consensus_engine.Engine, vmConfig vm.Config, shouldPreserve func(block *types.Block) bool) (*BlockChain, error) { if cacheConfig == nil { cacheConfig = &CacheConfig{ TrieNodeLimit: 256 * 1024 * 1024, @@ -223,6 +223,22 @@ func (bc *BlockChain) ValidateNewBlock(block *types.Block, address common.Addres return nil } +// IsEpochBlock returns whether this block is the first block of an epoch. +func IsEpochBlock(block *types.Block) bool { + if block.NumberU64()%BlocksPerEpoch == 0 { + return true + } + return false +} + +// IsEpochLastBlock returns whether this block is the last block of an epoch. +func IsEpochLastBlock(block *types.Block) bool { + if block.NumberU64()%BlocksPerEpoch == BlocksPerEpoch-1 { + return true + } + return false +} + func (bc *BlockChain) getProcInterrupt() bool { return atomic.LoadInt32(&bc.procInterrupt) == 1 } @@ -931,7 +947,7 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types. // Calculate the total difficulty of the block ptd := bc.GetTd(block.ParentHash(), block.NumberU64()-1) if ptd == nil { - return NonStatTy, consensus.ErrUnknownAncestor + return NonStatTy, consensus_engine.ErrUnknownAncestor } // Make sure no inconsistent state is leaked during insertion bc.mu.Lock() @@ -1135,7 +1151,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty continue } - case err == consensus.ErrFutureBlock: + case err == consensus_engine.ErrFutureBlock: // Allow up to MaxFuture second in the future blocks. If this limit is exceeded // the chain is discarded and processed at a later time if given. max := big.NewInt(time.Now().Unix() + maxTimeFutureBlocks) @@ -1146,12 +1162,12 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty stats.queued++ continue - case err == consensus.ErrUnknownAncestor && bc.futureBlocks.Contains(block.ParentHash()): + case err == consensus_engine.ErrUnknownAncestor && bc.futureBlocks.Contains(block.ParentHash()): bc.futureBlocks.Add(block.Hash(), block) stats.queued++ continue - case err == consensus.ErrPrunedAncestor: + case err == consensus_engine.ErrPrunedAncestor: // Block competing with the canonical chain, store in the db, but don't process // until the competitor TD goes above the canonical TD currentBlock := bc.CurrentBlock() @@ -1611,7 +1627,7 @@ func (bc *BlockChain) GetHeaderByNumber(number uint64) *types.Header { func (bc *BlockChain) Config() *params.ChainConfig { return bc.chainConfig } // Engine retrieves the blockchain's consensus engine. -func (bc *BlockChain) Engine() consensus.Engine { return bc.engine } +func (bc *BlockChain) Engine() consensus_engine.Engine { return bc.engine } // SubscribeRemovedLogsEvent registers a subscription of RemovedLogsEvent. func (bc *BlockChain) SubscribeRemovedLogsEvent(ch chan<- RemovedLogsEvent) event.Subscription { diff --git a/core/chain_makers.go b/core/chain_makers.go index f9dbafb44..8c412b41c 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -23,7 +23,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/params" - "github.com/harmony-one/harmony/consensus" + consensus_engine "github.com/harmony-one/harmony/consensus/engine" "github.com/harmony-one/harmony/core/state" "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/core/vm" @@ -44,7 +44,7 @@ type BlockGen struct { uncles []*types.Header config *params.ChainConfig - engine consensus.Engine + engine consensus_engine.Engine } // SetCoinbase sets the coinbase of the generated block. @@ -161,7 +161,7 @@ func (b *BlockGen) PrevBlock(index int) *types.Block { // Blocks created by GenerateChain do not contain valid proof of work // values. Inserting them into BlockChain requires use of FakePow or // a similar non-validating proof of work implementation. -func GenerateChain(config *params.ChainConfig, parent *types.Block, engine consensus.Engine, db ethdb.Database, n int, gen func(int, *BlockGen)) ([]*types.Block, []types.Receipts) { +func GenerateChain(config *params.ChainConfig, parent *types.Block, engine consensus_engine.Engine, db ethdb.Database, n int, gen func(int, *BlockGen)) ([]*types.Block, []types.Receipts) { if config == nil { config = params.TestChainConfig } @@ -216,7 +216,7 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse return blocks, receipts } -func makeHeader(chain consensus.ChainReader, parent *types.Block, state *state.DB, engine consensus.Engine) *types.Header { +func makeHeader(chain consensus_engine.ChainReader, parent *types.Block, state *state.DB, engine consensus_engine.Engine) *types.Header { var time *big.Int if parent.Time() == nil { time = big.NewInt(10) @@ -241,7 +241,7 @@ func makeHeader(chain consensus.ChainReader, parent *types.Block, state *state.D } // makeHeaderChain creates a deterministic chain of headers rooted at parent. -func makeHeaderChain(parent *types.Header, n int, engine consensus.Engine, db ethdb.Database, seed int) []*types.Header { +func makeHeaderChain(parent *types.Header, n int, engine consensus_engine.Engine, db ethdb.Database, seed int) []*types.Header { blocks := makeBlockChain(types.NewBlockWithHeader(parent), n, engine, db, seed) headers := make([]*types.Header, len(blocks)) for i, block := range blocks { @@ -251,7 +251,7 @@ func makeHeaderChain(parent *types.Header, n int, engine consensus.Engine, db et } // makeBlockChain creates a deterministic chain of blocks rooted at parent. -func makeBlockChain(parent *types.Block, n int, engine consensus.Engine, db ethdb.Database, seed int) []*types.Block { +func makeBlockChain(parent *types.Block, n int, engine consensus_engine.Engine, db ethdb.Database, seed int) []*types.Block { blocks, _ := GenerateChain(params.TestChainConfig, parent, engine, db, n, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{0: byte(seed), 19: byte(i)}) }) diff --git a/core/evm.go b/core/evm.go index 0f5799e96..f94a7d522 100644 --- a/core/evm.go +++ b/core/evm.go @@ -20,7 +20,7 @@ import ( "math/big" "github.com/ethereum/go-ethereum/common" - "github.com/harmony-one/harmony/consensus" + consensus_engine "github.com/harmony-one/harmony/consensus/engine" "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/core/vm" ) @@ -29,7 +29,7 @@ import ( // current blockchain to be used during transaction processing. type ChainContext interface { // Engine retrieves the chain's consensus engine. - Engine() consensus.Engine + Engine() consensus_engine.Engine // GetHeader returns the hash corresponding to their hash. GetHeader(common.Hash, uint64) *types.Header diff --git a/core/headerchain.go b/core/headerchain.go index 38ac08413..55699902b 100644 --- a/core/headerchain.go +++ b/core/headerchain.go @@ -30,7 +30,7 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" - "github.com/harmony-one/harmony/consensus" + consensus_engine "github.com/harmony-one/harmony/consensus/engine" "github.com/harmony-one/harmony/core/rawdb" "github.com/harmony-one/harmony/core/types" lru "github.com/hashicorp/golang-lru" @@ -63,14 +63,14 @@ type HeaderChain struct { procInterrupt func() bool rand *mrand.Rand - engine consensus.Engine + engine consensus_engine.Engine } // NewHeaderChain creates a new HeaderChain structure. // getValidator should return the parent's validator // procInterrupt points to the parent's interrupt semaphore // wg points to the parent's shutdown wait group -func NewHeaderChain(chainDb ethdb.Database, config *params.ChainConfig, engine consensus.Engine, procInterrupt func() bool) (*HeaderChain, error) { +func NewHeaderChain(chainDb ethdb.Database, config *params.ChainConfig, engine consensus_engine.Engine, procInterrupt func() bool) (*HeaderChain, error) { headerCache, _ := lru.New(headerCacheLimit) tdCache, _ := lru.New(tdCacheLimit) numberCache, _ := lru.New(numberCacheLimit) @@ -140,7 +140,7 @@ func (hc *HeaderChain) WriteHeader(header *types.Header) (status WriteStatus, er // Calculate the total difficulty of the header ptd := hc.GetTd(header.ParentHash, number-1) if ptd == nil { - return NonStatTy, consensus.ErrUnknownAncestor + return NonStatTy, consensus_engine.ErrUnknownAncestor } localTd := hc.GetTd(hc.currentHeaderHash, hc.CurrentHeader().Number.Uint64()) externTd := new(big.Int).Add(header.Difficulty, ptd) @@ -498,7 +498,7 @@ func (hc *HeaderChain) SetGenesis(head *types.Header) { func (hc *HeaderChain) Config() *params.ChainConfig { return hc.config } // Engine retrieves the header chain's consensus engine. -func (hc *HeaderChain) Engine() consensus.Engine { return hc.engine } +func (hc *HeaderChain) Engine() consensus_engine.Engine { return hc.engine } // GetBlock implements consensus.ChainReader, and returns nil for every input as // a header chain does not have blocks available for retrieval. diff --git a/core/state_processor.go b/core/state_processor.go index 862961bdd..f009f53f4 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -20,7 +20,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/params" - "github.com/harmony-one/harmony/consensus" + consensus_engine "github.com/harmony-one/harmony/consensus/engine" "github.com/harmony-one/harmony/core/state" "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/core/vm" @@ -31,13 +31,13 @@ import ( // // StateProcessor implements Processor. type StateProcessor struct { - config *params.ChainConfig // Chain configuration options - bc *BlockChain // Canonical block chain - engine consensus.Engine // Consensus engine used for block rewards + config *params.ChainConfig // Chain configuration options + bc *BlockChain // Canonical block chain + engine consensus_engine.Engine // Consensus engine used for block rewards } // NewStateProcessor initialises a new StateProcessor. -func NewStateProcessor(config *params.ChainConfig, bc *BlockChain, engine consensus.Engine) *StateProcessor { +func NewStateProcessor(config *params.ChainConfig, bc *BlockChain, engine consensus_engine.Engine) *StateProcessor { return &StateProcessor{ config: config, bc: bc, diff --git a/drand/drand.go b/drand/drand.go index 85c4fbd88..2409daa7e 100644 --- a/drand/drand.go +++ b/drand/drand.go @@ -23,7 +23,8 @@ type DRand struct { bitmap *bls_cosi.Mask pRand *[32]byte rand *[32]byte - ConfirmedBlockChannel chan *types.Block // Channel for confirmed blocks + ConfirmedBlockChannel chan *types.Block // Channel to receive confirmed blocks + PRndChannel chan []byte // Channel to send pRnd (preimage of randomness resulting from combined vrf randomnesses) to consensus. The first 32 bytes are randomness, the rest is for bitmap. // map of nodeID to validator Peer object // FIXME: should use PubKey of p2p.Peer as the hashkey @@ -65,6 +66,8 @@ func New(host p2p.Host, ShardID string, peers []p2p.Peer, leader p2p.Peer, confi dRand.ConfirmedBlockChannel = confirmedBlockChannel } + dRand.PRndChannel = make(chan []byte) + selfPeer := host.GetSelfPeer() if leader.Port == selfPeer.Port && leader.IP == selfPeer.IP { dRand.IsLeader = true diff --git a/drand/drand_leader.go b/drand/drand_leader.go index 9f9954737..6a37f97be 100644 --- a/drand/drand_leader.go +++ b/drand/drand_leader.go @@ -18,10 +18,9 @@ func (dRand *DRand) WaitForEpochBlock(blockChannel chan *types.Block, stopChan c default: // keep waiting for epoch block newBlock := <-blockChannel - if newBlock.NumberU64()%core.BlocksPerEpoch == 0 { - + if core.IsEpochLastBlock(newBlock) { + dRand.init(newBlock) } - dRand.init(newBlock) case <-stopChan: return } @@ -100,5 +99,14 @@ func (dRand *DRand) processCommitMessage(message drand_proto.Message) { // Construct pRand and initiate consensus on it utils.GetLogInstance().Debug("Received enough randomness commit", "numReceivedSoFar", len((*vrfs)), "validatorID", validatorID, "PublicKeys", len(dRand.PublicKeys)) // TODO: communicate the pRand to consensus + + pRnd := [32]byte{} + // Bitwise XOR on all the submitted vrfs + for _, vrf := range *vrfs { + for i := 0; i < len(pRnd); i++ { + pRnd[i] = pRnd[i] ^ vrf[i] + } + } + dRand.PRndChannel <- append(pRnd[:], dRand.bitmap.Bitmap...) } } diff --git a/node/worker/worker.go b/node/worker/worker.go index 993da4dfa..3651ed200 100644 --- a/node/worker/worker.go +++ b/node/worker/worker.go @@ -8,7 +8,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/params" - "github.com/harmony-one/harmony/consensus" + consensus_engine "github.com/harmony-one/harmony/consensus/engine" "github.com/harmony-one/harmony/core" "github.com/harmony-one/harmony/core/state" "github.com/harmony-one/harmony/core/types" @@ -33,7 +33,7 @@ type Worker struct { current *environment // An environment for current running cycle. coinbase common.Address - engine consensus.Engine + engine consensus_engine.Engine gasFloor uint64 gasCeil uint64 @@ -156,7 +156,7 @@ func (w *Worker) Commit() (*types.Block, error) { } // New create a new worker object. -func New(config *params.ChainConfig, chain *core.BlockChain, engine consensus.Engine, coinbase common.Address, shardID uint32) *Worker { +func New(config *params.ChainConfig, chain *core.BlockChain, engine consensus_engine.Engine, coinbase common.Address, shardID uint32) *Worker { worker := &Worker{ config: config, chain: chain, From 9ba931abb9c27c5c942641ec7db0bbf7f0c2dfe9 Mon Sep 17 00:00:00 2001 From: Leo Chen Date: Tue, 12 Feb 2019 00:26:36 -0800 Subject: [PATCH 04/14] simplify networkinfo/discovery services Signed-off-by: Leo Chen --- api/service/discovery/service.go | 29 ++++++++------ api/service/networkinfo/service.go | 63 ++++++++++++------------------ 2 files changed, 44 insertions(+), 48 deletions(-) diff --git a/api/service/discovery/service.go b/api/service/discovery/service.go index 93f2169c9..5c28a3a99 100644 --- a/api/service/discovery/service.go +++ b/api/service/discovery/service.go @@ -1,6 +1,8 @@ package discovery import ( + "time" + "github.com/ethereum/go-ethereum/log" proto_discovery "github.com/harmony-one/harmony/api/proto/discovery" "github.com/harmony-one/harmony/p2p" @@ -46,6 +48,7 @@ func (s *Service) StopService() { // Run is the main function of the service func (s *Service) Run() { go s.contactP2pPeers() + go s.pingPeer() } func (s *Service) contactP2pPeers() { @@ -62,7 +65,9 @@ func (s *Service) contactP2pPeers() { log.Debug("[DISCOVERY]", "add outgoing peer", peer) // TODO: stop ping if pinged before // TODO: call staking servcie here if it is a new node - s.pingPeer(peer) + if s.stakingChan != nil { + s.stakingChan <- peer + } case <-s.stopChan: return } @@ -74,19 +79,21 @@ func (s *Service) Init() { log.Info("Init discovery service") } -func (s *Service) pingPeer(peer p2p.Peer) { +func (s *Service) pingPeer() { + tick := time.NewTicker(5 * time.Second) ping := proto_discovery.NewPingMessage(s.host.GetSelfPeer()) buffer := ping.ConstructPingMessage() content := host.ConstructP2pMessage(byte(0), buffer) + + for { + select { + case <-tick.C: + err := s.host.SendMessageToGroups([]p2p.GroupID{p2p.GroupIDBeacon}, content) + if err != nil { + log.Error("Failed to send ping message", "group", p2p.GroupIDBeacon) + } + } + } // s.host.SendMessage(peer, content) // log.Debug("Sent Ping Message via unicast to", "peer", peer) - err := s.host.SendMessageToGroups([]p2p.GroupID{p2p.GroupIDBeacon}, content) - if err != nil { - log.Error("Failed to send ping message", "group", p2p.GroupIDBeacon) - } else { - log.Debug("[PING] sent Ping Message via group send to", "peer", peer) - } - if s.stakingChan != nil { - s.stakingChan <- peer - } } diff --git a/api/service/networkinfo/service.go b/api/service/networkinfo/service.go index 92a48ad5f..e7f434aad 100644 --- a/api/service/networkinfo/service.go +++ b/api/service/networkinfo/service.go @@ -49,7 +49,6 @@ func New(h p2p.Host, rendezvous string, peerChan chan p2p.Peer) *Service { stopChan: make(chan struct{}), stoppedChan: make(chan struct{}), peerChan: peerChan, - peerInfo: make(<-chan peerstore.PeerInfo), } } @@ -91,8 +90,6 @@ func (s *Service) Init() error { libp2pdis.Advertise(s.ctx, s.discovery, s.Rendezvous) utils.GetLogInstance().Info("Successfully announced!") - go s.DoService() - return nil } @@ -104,50 +101,42 @@ func (s *Service) Run() { if err != nil { utils.GetLogInstance().Error("FindPeers", "error", err) } + + go s.DoService() } // DoService does network info. func (s *Service) DoService() { - for { - select { - case peer, ok := <-s.peerInfo: - if !ok { - utils.GetLogInstance().Debug("no more peer info", "peer", peer.ID) - return + for peer := range s.peerInfo { + if peer.ID != s.Host.GetP2PHost().ID() && len(peer.ID) > 0 { + utils.GetLogInstance().Info("Found Peer", "peer", peer.ID, "addr", peer.Addrs, "my ID", s.Host.GetP2PHost().ID()) + s.lock.Lock() + if err := s.Host.GetP2PHost().Connect(s.ctx, peer); err != nil { + utils.GetLogInstance().Warn("can't connect to peer node", "error", err) + } else { + utils.GetLogInstance().Info("connected to peer node", "peer", peer) } - if peer.ID != s.Host.GetP2PHost().ID() && len(peer.ID) > 0 { - utils.GetLogInstance().Info("Found Peer", "peer", peer.ID, "addr", peer.Addrs, "my ID", s.Host.GetP2PHost().ID()) - s.lock.Lock() - if err := s.Host.GetP2PHost().Connect(s.ctx, peer); err != nil { - utils.GetLogInstance().Warn("can't connect to peer node", "error", err) - } else { - utils.GetLogInstance().Info("connected to peer node", "peer", peer) + s.lock.Unlock() + // figure out the public ip/port + ip := "127.0.0.1" + var port string + for _, addr := range peer.Addrs { + netaddr, err := manet.ToNetAddr(addr) + if err != nil { + continue } - s.lock.Unlock() - // figure out the public ip/port - ip := "127.0.0.1" - var port string - for _, addr := range peer.Addrs { - netaddr, err := manet.ToNetAddr(addr) - if err != nil { - continue - } - nip := netaddr.(*net.TCPAddr).IP.String() - if strings.Compare(nip, "127.0.0.1") != 0 { - ip = nip - port = fmt.Sprintf("%d", netaddr.(*net.TCPAddr).Port) - break - } + nip := netaddr.(*net.TCPAddr).IP.String() + if strings.Compare(nip, "127.0.0.1") != 0 { + ip = nip + port = fmt.Sprintf("%d", netaddr.(*net.TCPAddr).Port) + break } - p := p2p.Peer{IP: ip, Port: port, PeerID: peer.ID, Addrs: peer.Addrs} - utils.GetLogInstance().Info("Notify peerChan", "peer", p) - s.peerChan <- p } - case <-s.ctx.Done(): - return + p := p2p.Peer{IP: ip, Port: port, PeerID: peer.ID, Addrs: peer.Addrs} + utils.GetLogInstance().Info("Notify peerChan", "peer", p) + s.peerChan <- p } } - } // StopService stops network info service. From 3ffb7bdea9354579afa11f8f7aa3cf60309d53f0 Mon Sep 17 00:00:00 2001 From: Leo Chen Date: Tue, 12 Feb 2019 10:08:40 -0800 Subject: [PATCH 05/14] do not handle duplicated ping message Since I changed the ping message to every 5 sec, there is a lot of duplicated ping messages flooded in the network. So, we have to ignore the duplicated ping message. Right now, I just ignore the ping message from the same sender identified by peer.ID. Signed-off-by: Leo Chen --- node/node.go | 5 +++++ node/node_handler.go | 20 +++++++++++++++----- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/node/node.go b/node/node.go index b9f95b409..1d6ccdbb9 100644 --- a/node/node.go +++ b/node/node.go @@ -185,6 +185,9 @@ type Node struct { // fully integrate with libp2p for networking // FIXME: this is temporary hack until we can fully replace the old one UseLibP2P bool + + // Duplicated Ping Message Received + duplicatedPing map[string]bool } // Blockchain returns the blockchain from node @@ -303,6 +306,8 @@ func New(host p2p.Host, consensus *bft.Consensus, db ethdb.Database) *Node { // start the goroutine to receive group message go node.ReceiveGroupMessage() + node.duplicatedPing = make(map[string]bool) + return &node } diff --git a/node/node_handler.go b/node/node_handler.go index 833aa21c5..c97c119d0 100644 --- a/node/node_handler.go +++ b/node/node_handler.go @@ -46,7 +46,7 @@ func (node *Node) StreamHandler(s p2p.Stream) { return } - node.messageHandler(content) + node.messageHandler(content, "") } // ReceiveGroupMessage use libp2p pubsub mechanism to receive broadcast messages @@ -62,14 +62,14 @@ func (node *Node) ReceiveGroupMessage() { utils.GetLogInstance().Info("[PUBSUB]", "received group msg", len(msg), "sender", sender) if err == nil { // skip the first 5 bytes, 1 byte is p2p type, 4 bytes are message size - node.messageHandler(msg[5:]) + node.messageHandler(msg[5:], string(sender)) } } } } // messageHandler parses the message and dispatch the actions -func (node *Node) messageHandler(content []byte) { +func (node *Node) messageHandler(content []byte, sender string) { node.MaybeBroadcastAsValidator(content) consensusObj := node.Consensus @@ -178,7 +178,7 @@ func (node *Node) messageHandler(content []byte) { os.Exit(0) } case proto_node.PING: - node.pingMessageHandler(msgPayload) + node.pingMessageHandler(msgPayload, sender) case proto_node.PONG: node.pongMessageHandler(msgPayload) } @@ -279,7 +279,17 @@ func (node *Node) AddNewBlock(newBlock *types.Block) { } } -func (node *Node) pingMessageHandler(msgPayload []byte) int { +func (node *Node) pingMessageHandler(msgPayload []byte, sender string) int { + if sender != "" { + _, ok := node.duplicatedPing[sender] + if !ok { + node.duplicatedPing[sender] = true + } else { + // duplicated ping message return + return 0 + } + } + ping, err := proto_discovery.GetPingMessage(msgPayload) if err != nil { utils.GetLogInstance().Error("Can't get Ping Message") From a410f8d588f15f37a79e877ea6fa6cc9738cb700 Mon Sep 17 00:00:00 2001 From: Leo Chen Date: Tue, 12 Feb 2019 10:24:55 -0800 Subject: [PATCH 06/14] add a config file to test just one shard Signed-off-by: Leo Chen --- test/configs/local_config2.txt | 12 ------------ test/configs/oneshard.txt | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 12 deletions(-) delete mode 100644 test/configs/local_config2.txt create mode 100644 test/configs/oneshard.txt diff --git a/test/configs/local_config2.txt b/test/configs/local_config2.txt deleted file mode 100644 index 2cec850a8..000000000 --- a/test/configs/local_config2.txt +++ /dev/null @@ -1,12 +0,0 @@ -127.0.0.1 9000 leader 0 -127.0.0.1 9001 validator 0 -127.0.0.1 9002 validator 0 -127.0.0.1 9003 validator 0 -127.0.0.1 9004 validator 0 -127.0.0.1 9006 newnode 0 -127.0.0.1 9007 newnode 0 -127.0.0.1 9008 newnode 0 -127.0.0.1 9009 newnode 0 -127.0.0.1 9010 newnode 0 -127.0.0.1 9011 newnode 0 -127.0.0.1 19999 client 0 diff --git a/test/configs/oneshard.txt b/test/configs/oneshard.txt new file mode 100644 index 000000000..6d0f6f2b2 --- /dev/null +++ b/test/configs/oneshard.txt @@ -0,0 +1,32 @@ +127.0.0.1 9000 leader 0 +127.0.0.1 9001 validator 0 +127.0.0.1 9002 validator 0 +127.0.0.1 9003 validator 0 +127.0.0.1 9004 validator 0 +127.0.0.1 9005 validator 0 +127.0.0.1 9006 validator 0 +127.0.0.1 9007 validator 0 +127.0.0.1 9008 validator 0 +127.0.0.1 9009 validator 0 +127.0.0.1 9010 validator 0 +127.0.0.1 9011 validator 0 +127.0.0.1 9012 validator 0 +127.0.0.1 9013 validator 0 +127.0.0.1 9014 validator 0 +127.0.0.1 9015 validator 0 +127.0.0.1 9016 validator 0 +127.0.0.1 9017 validator 0 +127.0.0.1 9018 validator 0 +127.0.0.1 9019 validator 0 +127.0.0.1 9020 validator 0 +127.0.0.1 9021 validator 0 +127.0.0.1 9022 validator 0 +127.0.0.1 9023 validator 0 +127.0.0.1 9024 validator 0 +127.0.0.1 9025 validator 0 +127.0.0.1 9026 validator 0 +127.0.0.1 9027 validator 0 +127.0.0.1 9028 validator 0 +127.0.0.1 9029 validator 0 +127.0.0.1 9030 validator 0 +127.0.0.1 19999 client 0 From 175e43fed7ffa4cf36c6b2d00919be95fcb579e1 Mon Sep 17 00:00:00 2001 From: Leo Chen Date: Tue, 12 Feb 2019 14:10:04 -0800 Subject: [PATCH 07/14] add a goroutine to send pong message periodically only send pong message when there is a stable number of peers Signed-off-by: Leo Chen --- cmd/harmony.go | 3 +++ node/node_handler.go | 59 +++++++++++++++++++++++++++++++++++--------- 2 files changed, 50 insertions(+), 12 deletions(-) diff --git a/cmd/harmony.go b/cmd/harmony.go index 5d463553c..ecf23ba68 100644 --- a/cmd/harmony.go +++ b/cmd/harmony.go @@ -284,6 +284,9 @@ func main() { go currentNode.JoinShard(leader) } } else { + if consensus.IsLeader { + go currentNode.SendPongMessage() + } currentNode.UseLibP2P = true } diff --git a/node/node_handler.go b/node/node_handler.go index c97c119d0..cb2108e2e 100644 --- a/node/node_handler.go +++ b/node/node_handler.go @@ -324,7 +324,8 @@ func (node *Node) pingMessageHandler(msgPayload []byte, sender string) int { // Add to Node's peer list anyway node.AddPeers([]*p2p.Peer{peer}) - if node.Consensus.IsLeader { + // This is the old way of broadcasting pong message + if node.Consensus.IsLeader && !node.UseLibP2P { peers := node.Consensus.GetValidatorPeers() pong := proto_discovery.NewPongMessage(peers, node.Consensus.PublicKeys) buffer := pong.ConstructPongMessage() @@ -342,21 +343,55 @@ func (node *Node) pingMessageHandler(msgPayload []byte, sender string) int { // Broadcast the message to all validators, as publicKeys is updated // FIXME: HAR-89 use a separate nodefind/neighbor message - if node.UseLibP2P { - content := host.ConstructP2pMessage(byte(0), buffer) - err := node.host.SendMessageToGroups([]p2p.GroupID{p2p.GroupIDBeacon}, content) - if err != nil { - utils.GetLogInstance().Error("[PONG] failed to send pong message", "group", p2p.GroupIDBeacon) + host.BroadcastMessageFromLeader(node.GetHost(), peers, buffer, node.Consensus.OfflinePeers) + utils.GetLogInstance().Info("PingMsgHandler send pong message") + } + + return 1 +} + +// SendPongMessage is the a goroutine to periodcally send pong message to all peers +func (node *Node) SendPongMessage() { + tick := time.NewTicker(10 * time.Second) + numPeers := len(node.Consensus.GetValidatorPeers()) + numPubKeys := len(node.Consensus.PublicKeys) + sentMessage := false + + // Send Pong Message only when there is change on the number of peers + for { + select { + case <-tick.C: + peers := node.Consensus.GetValidatorPeers() + numPeersNow := len(peers) + numPubKeysNow := len(node.Consensus.PublicKeys) + + // no peers, wait for another tick + if numPeersNow == 0 || numPubKeysNow == 0 { + continue + } + // new peers added + if numPubKeysNow != numPubKeys || numPeersNow != numPeers { + sentMessage = false } else { - utils.GetLogInstance().Debug("[PONG] sent Pong Message via group send", "group", p2p.GroupIDBeacon) + // stable number of peers/pubkeys, sent the pong message + if !sentMessage { + pong := proto_discovery.NewPongMessage(peers, node.Consensus.PublicKeys) + buffer := pong.ConstructPongMessage() + content := host.ConstructP2pMessage(byte(0), buffer) + err := node.host.SendMessageToGroups([]p2p.GroupID{p2p.GroupIDBeacon}, content) + if err != nil { + utils.GetLogInstance().Error("[PONG] failed to send pong message", "group", p2p.GroupIDBeacon) + continue + } else { + utils.GetLogInstance().Info("[PONG] sent pong message to", "group", p2p.GroupIDBeacon) + } + sentMessage = true + } } - } else { - host.BroadcastMessageFromLeader(node.GetHost(), peers, buffer, node.Consensus.OfflinePeers) - utils.GetLogInstance().Info("PingMsgHandler send pong message") + numPeers = numPeersNow + numPubKeys = numPubKeysNow } } - - return 1 } func (node *Node) pongMessageHandler(msgPayload []byte) int { From f2a5e5c5c9e5a732bb8bfda0b08a15cc450d21d4 Mon Sep 17 00:00:00 2001 From: Leo Chen Date: Tue, 12 Feb 2019 15:29:00 -0800 Subject: [PATCH 08/14] 1st attempt to replace broadcast with pubsub Signed-off-by: Leo Chen --- node/node.go | 6 +----- node/node_handler.go | 10 +++++++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/node/node.go b/node/node.go index 1d6ccdbb9..673d80016 100644 --- a/node/node.go +++ b/node/node.go @@ -182,10 +182,6 @@ type Node struct { // Group Message Receiver groupReceiver p2p.GroupReceiver - // fully integrate with libp2p for networking - // FIXME: this is temporary hack until we can fully replace the old one - UseLibP2P bool - // Duplicated Ping Message Received duplicatedPing map[string]bool } @@ -219,7 +215,7 @@ func (node *Node) getTransactionsForNewBlock(maxNumTxs int) types.Transactions { // StartServer starts a server and process the requests by a handler. func (node *Node) StartServer() { - if node.UseLibP2P { + if utils.UseLibP2P { select {} } else { node.host.BindHandlerAndServe(node.StreamHandler) diff --git a/node/node_handler.go b/node/node_handler.go index cb2108e2e..c111430cc 100644 --- a/node/node_handler.go +++ b/node/node_handler.go @@ -59,7 +59,7 @@ func (node *Node) ReceiveGroupMessage() { } msg, sender, err := node.groupReceiver.Receive(ctx) if sender != node.host.GetID() { - utils.GetLogInstance().Info("[PUBSUB]", "received group msg", len(msg), "sender", sender) + // utils.GetLogInstance().Info("[PUBSUB]", "received group msg", len(msg), "sender", sender) if err == nil { // skip the first 5 bytes, 1 byte is p2p type, 4 bytes are message size node.messageHandler(msg[5:], string(sender)) @@ -229,7 +229,11 @@ func (node *Node) transactionMessageHandler(msgPayload []byte) { func (node *Node) BroadcastNewBlock(newBlock *types.Block) { if node.ClientPeer != nil { utils.GetLogInstance().Debug("Sending new block to client", "client", node.ClientPeer) - node.SendMessage(*node.ClientPeer, proto_node.ConstructBlocksSyncMessage([]*types.Block{newBlock})) + if utils.UseLibP2P { + node.host.SendMessageToGroups([]p2p.GroupID{p2p.GroupIDBeacon}, proto_node.ConstructBlocksSyncMessage([]*types.Block{newBlock})) + } else { + node.SendMessage(*node.ClientPeer, proto_node.ConstructBlocksSyncMessage([]*types.Block{newBlock})) + } } } @@ -325,7 +329,7 @@ func (node *Node) pingMessageHandler(msgPayload []byte, sender string) int { node.AddPeers([]*p2p.Peer{peer}) // This is the old way of broadcasting pong message - if node.Consensus.IsLeader && !node.UseLibP2P { + if node.Consensus.IsLeader && !utils.UseLibP2P { peers := node.Consensus.GetValidatorPeers() pong := proto_discovery.NewPongMessage(peers, node.Consensus.PublicKeys) buffer := pong.ConstructPongMessage() From a55b4514dda9201d77ecd270c3b837fc0ed39c09 Mon Sep 17 00:00:00 2001 From: Leo Chen Date: Tue, 12 Feb 2019 16:37:51 -0800 Subject: [PATCH 09/14] use gossip for all consensus messages Signed-off-by: Leo Chen --- cmd/harmony.go | 3 ++- consensus/consensus.go | 6 +++++- consensus/consensus_leader.go | 21 ++++++++++++++++++--- consensus/consensus_validator.go | 13 +++++++++++-- internal/utils/singleton.go | 4 ++++ p2p/host/hostv2/hostv2.go | 4 ++-- 6 files changed, 42 insertions(+), 9 deletions(-) diff --git a/cmd/harmony.go b/cmd/harmony.go index ecf23ba68..0f4eb070d 100644 --- a/cmd/harmony.go +++ b/cmd/harmony.go @@ -199,6 +199,7 @@ func main() { // Attack determination. attack.GetInstance().SetAttackEnabled(attackDetermination(*attackedMode)) } + utils.UseLibP2P = false } else { if *isLeader { role = "leader" @@ -206,6 +207,7 @@ func main() { } else { role = "validator" } + utils.UseLibP2P = true } // Init logging. loggingInit(*logFolder, role, *ip, *port, *onlyLogTps) @@ -287,7 +289,6 @@ func main() { if consensus.IsLeader { go currentNode.SendPongMessage() } - currentNode.UseLibP2P = true } go currentNode.SupportSyncing() diff --git a/consensus/consensus.go b/consensus/consensus.go index 60d067955..b3ef03d1d 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -432,7 +432,11 @@ func (consensus *Consensus) RemovePeers(peers []p2p.Peer) int { pong := proto_discovery.NewPongMessage(validators, consensus.PublicKeys) buffer := pong.ConstructPongMessage() - host.BroadcastMessageFromLeader(consensus.host, validators, buffer, consensus.OfflinePeers) + if utils.UseLibP2P { + consensus.host.SendMessageToGroups([]p2p.GroupID{p2p.GroupIDBeacon}, buffer) + } else { + host.BroadcastMessageFromLeader(consensus.host, validators, buffer, consensus.OfflinePeers) + } } return count2 diff --git a/consensus/consensus_leader.go b/consensus/consensus_leader.go index e1a70a82d..48afe4a01 100644 --- a/consensus/consensus_leader.go +++ b/consensus/consensus_leader.go @@ -14,6 +14,7 @@ import ( bls_cosi "github.com/harmony-one/harmony/crypto/bls" "github.com/harmony-one/harmony/internal/profiler" "github.com/harmony-one/harmony/internal/utils" + "github.com/harmony-one/harmony/p2p" "github.com/harmony-one/harmony/p2p/host" ) @@ -107,7 +108,11 @@ func (consensus *Consensus) startConsensus(newBlock *types.Block) { // Leader sign the block hash itself consensus.prepareSigs[consensus.nodeID] = consensus.priKey.SignHash(consensus.blockHash[:]) - host.BroadcastMessageFromLeader(consensus.host, consensus.GetValidatorPeers(), msgToSend, consensus.OfflinePeers) + if utils.UseLibP2P { + consensus.host.SendMessageToGroups([]p2p.GroupID{p2p.GroupIDBeacon}, msgToSend) + } else { + host.BroadcastMessageFromLeader(consensus.host, consensus.GetValidatorPeers(), msgToSend, consensus.OfflinePeers) + } } // processPrepareMessage processes the prepare message sent from validators @@ -164,7 +169,12 @@ func (consensus *Consensus) processPrepareMessage(message consensus_proto.Messag // Construct and broadcast prepared message msgToSend, aggSig := consensus.constructPreparedMessage() consensus.aggregatedPrepareSig = aggSig - host.BroadcastMessageFromLeader(consensus.host, consensus.GetValidatorPeers(), msgToSend, consensus.OfflinePeers) + + if utils.UseLibP2P { + consensus.host.SendMessageToGroups([]p2p.GroupID{p2p.GroupIDBeacon}, msgToSend) + } else { + host.BroadcastMessageFromLeader(consensus.host, consensus.GetValidatorPeers(), msgToSend, consensus.OfflinePeers) + } // Set state to targetState consensus.state = targetState @@ -230,7 +240,12 @@ func (consensus *Consensus) processCommitMessage(message consensus_proto.Message // Construct and broadcast committed message msgToSend, aggSig := consensus.constructCommittedMessage() consensus.aggregatedCommitSig = aggSig - host.BroadcastMessageFromLeader(consensus.host, consensus.GetValidatorPeers(), msgToSend, consensus.OfflinePeers) + + if utils.UseLibP2P { + consensus.host.SendMessageToGroups([]p2p.GroupID{p2p.GroupIDBeacon}, msgToSend) + } else { + host.BroadcastMessageFromLeader(consensus.host, consensus.GetValidatorPeers(), msgToSend, consensus.OfflinePeers) + } var blockObj types.Block err := rlp.DecodeBytes(consensus.block, &blockObj) diff --git a/consensus/consensus_validator.go b/consensus/consensus_validator.go index 783095173..381bb8df1 100644 --- a/consensus/consensus_validator.go +++ b/consensus/consensus_validator.go @@ -3,6 +3,7 @@ package consensus import ( "github.com/harmony-one/bls/ffi/go/bls" bls_cosi "github.com/harmony-one/harmony/crypto/bls" + "github.com/harmony-one/harmony/p2p" "github.com/ethereum/go-ethereum/rlp" protobuf "github.com/golang/protobuf/proto" @@ -103,7 +104,11 @@ func (consensus *Consensus) processAnnounceMessage(message consensus_proto.Messa // Construct and send prepare message msgToSend := consensus.constructPrepareMessage() - consensus.SendMessage(consensus.leader, msgToSend) + if utils.UseLibP2P { + consensus.host.SendMessageToGroups([]p2p.GroupID{p2p.GroupIDBeacon}, msgToSend) + } else { + consensus.SendMessage(consensus.leader, msgToSend) + } consensus.state = PrepareDone } @@ -163,7 +168,11 @@ func (consensus *Consensus) processPreparedMessage(message consensus_proto.Messa // Construct and send the commit message multiSigAndBitmap := append(multiSig, bitmap...) msgToSend := consensus.constructCommitMessage(multiSigAndBitmap) - consensus.SendMessage(consensus.leader, msgToSend) + if utils.UseLibP2P { + consensus.host.SendMessageToGroups([]p2p.GroupID{p2p.GroupIDBeacon}, msgToSend) + } else { + consensus.SendMessage(consensus.leader, msgToSend) + } consensus.state = CommitDone } diff --git a/internal/utils/singleton.go b/internal/utils/singleton.go index 5735e7a7a..48a93a2fd 100644 --- a/internal/utils/singleton.go +++ b/internal/utils/singleton.go @@ -13,6 +13,10 @@ import ( var ( Port string IP string + // Global Variable to use libp2p for networking + // FIXME: this is a temporary hack, once we totally switch to libp2p + // this variable shouldn't be used + UseLibP2P bool ) // SetPortAndIP used to print out loggings of node with Port and IP. diff --git a/p2p/host/hostv2/hostv2.go b/p2p/host/hostv2/hostv2.go index d37519c9e..6eb097039 100644 --- a/p2p/host/hostv2/hostv2.go +++ b/p2p/host/hostv2/hostv2.go @@ -162,8 +162,8 @@ func New(self *p2p.Peer, priKey p2p_crypto.PrivKey, opts ...p2p_config.Option) * append(opts, libp2p.ListenAddrs(listenAddr), libp2p.Identity(priKey))..., ) catchError(err) - // pubsub, err := pubsub.NewGossipSub(ctx, p2pHost) - pubsub, err := pubsub.NewFloodSub(ctx, p2pHost) + pubsub, err := pubsub.NewGossipSub(ctx, p2pHost) + // pubsub, err := pubsub.NewFloodSub(ctx, p2pHost) catchError(err) self.PeerID = p2pHost.ID() From 25485c655c3c6a15ebb7b847b35b9d3333646063 Mon Sep 17 00:00:00 2001 From: Leo Chen Date: Wed, 13 Feb 2019 00:05:02 -0800 Subject: [PATCH 10/14] add stop servie in discovery service Signed-off-by: Leo Chen --- api/service/discovery/service.go | 15 +++++++- api/service/networkinfo/service.go | 55 ++++++++++++++++-------------- node/node_handler.go | 3 ++ 3 files changed, 47 insertions(+), 26 deletions(-) diff --git a/api/service/discovery/service.go b/api/service/discovery/service.go index 5c28a3a99..24b1e8fd1 100644 --- a/api/service/discovery/service.go +++ b/api/service/discovery/service.go @@ -48,10 +48,14 @@ func (s *Service) StopService() { // Run is the main function of the service func (s *Service) Run() { go s.contactP2pPeers() - go s.pingPeer() + // go s.pingPeer() } func (s *Service) contactP2pPeers() { + tick := time.NewTicker(5 * time.Second) + ping := proto_discovery.NewPingMessage(s.host.GetSelfPeer()) + buffer := ping.ConstructPingMessage() + content := host.ConstructP2pMessage(byte(0), buffer) for { select { case peer, ok := <-s.peerChan: @@ -69,7 +73,13 @@ func (s *Service) contactP2pPeers() { s.stakingChan <- peer } case <-s.stopChan: + log.Debug("[DISCOVERY] stop") return + case <-tick.C: + err := s.host.SendMessageToGroups([]p2p.GroupID{p2p.GroupIDBeacon}, content) + if err != nil { + log.Error("Failed to send ping message", "group", p2p.GroupIDBeacon) + } } } } @@ -92,6 +102,9 @@ func (s *Service) pingPeer() { if err != nil { log.Error("Failed to send ping message", "group", p2p.GroupIDBeacon) } + case <-s.stopChan: + log.Info("Stop sending ping message") + return } } // s.host.SendMessage(peer, content) diff --git a/api/service/networkinfo/service.go b/api/service/networkinfo/service.go index e7f434aad..738270cb1 100644 --- a/api/service/networkinfo/service.go +++ b/api/service/networkinfo/service.go @@ -107,34 +107,39 @@ func (s *Service) Run() { // DoService does network info. func (s *Service) DoService() { - for peer := range s.peerInfo { - if peer.ID != s.Host.GetP2PHost().ID() && len(peer.ID) > 0 { - utils.GetLogInstance().Info("Found Peer", "peer", peer.ID, "addr", peer.Addrs, "my ID", s.Host.GetP2PHost().ID()) - s.lock.Lock() - if err := s.Host.GetP2PHost().Connect(s.ctx, peer); err != nil { - utils.GetLogInstance().Warn("can't connect to peer node", "error", err) - } else { - utils.GetLogInstance().Info("connected to peer node", "peer", peer) - } - s.lock.Unlock() - // figure out the public ip/port - ip := "127.0.0.1" - var port string - for _, addr := range peer.Addrs { - netaddr, err := manet.ToNetAddr(addr) - if err != nil { - continue + for { + select { + case peer := <-s.peerInfo: + if peer.ID != s.Host.GetP2PHost().ID() && len(peer.ID) > 0 { + utils.GetLogInstance().Info("Found Peer", "peer", peer.ID, "addr", peer.Addrs, "my ID", s.Host.GetP2PHost().ID()) + s.lock.Lock() + if err := s.Host.GetP2PHost().Connect(s.ctx, peer); err != nil { + utils.GetLogInstance().Warn("can't connect to peer node", "error", err) + } else { + utils.GetLogInstance().Info("connected to peer node", "peer", peer) } - nip := netaddr.(*net.TCPAddr).IP.String() - if strings.Compare(nip, "127.0.0.1") != 0 { - ip = nip - port = fmt.Sprintf("%d", netaddr.(*net.TCPAddr).Port) - break + s.lock.Unlock() + // figure out the public ip/port + ip := "127.0.0.1" + var port string + for _, addr := range peer.Addrs { + netaddr, err := manet.ToNetAddr(addr) + if err != nil { + continue + } + nip := netaddr.(*net.TCPAddr).IP.String() + if strings.Compare(nip, "127.0.0.1") != 0 { + ip = nip + port = fmt.Sprintf("%d", netaddr.(*net.TCPAddr).Port) + break + } } + p := p2p.Peer{IP: ip, Port: port, PeerID: peer.ID, Addrs: peer.Addrs} + utils.GetLogInstance().Info("Notify peerChan", "peer", p) + s.peerChan <- p } - p := p2p.Peer{IP: ip, Port: port, PeerID: peer.ID, Addrs: peer.Addrs} - utils.GetLogInstance().Info("Notify peerChan", "peer", p) - s.peerChan <- p + case <-s.stopChan: + return } } } diff --git a/node/node_handler.go b/node/node_handler.go index c111430cc..0c251a9b1 100644 --- a/node/node_handler.go +++ b/node/node_handler.go @@ -14,6 +14,7 @@ import ( proto_discovery "github.com/harmony-one/harmony/api/proto/discovery" proto_identity "github.com/harmony-one/harmony/api/proto/identity" proto_node "github.com/harmony-one/harmony/api/proto/node" + "github.com/harmony-one/harmony/api/service" "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/crypto/pki" "github.com/harmony-one/harmony/internal/utils" @@ -455,5 +456,7 @@ func (node *Node) pongMessageHandler(msgPayload []byte) int { } } + // Stop discovery service after received pong message + node.serviceManager.TakeAction(&service.Action{Action: service.Stop, ServiceType: service.PeerDiscovery}) return node.Consensus.UpdatePublicKeys(publicKeys) } From 7b0769c6513792e2a10c86258b56332e3f2206ed Mon Sep 17 00:00:00 2001 From: Leo Chen Date: Wed, 13 Feb 2019 14:26:36 -0800 Subject: [PATCH 11/14] gossip all consensus messages clean up some debug logs Signed-off-by: Leo Chen --- api/service/networkinfo/service.go | 3 --- consensus/consensus.go | 2 +- consensus/consensus_leader.go | 7 ++++--- consensus/consensus_validator.go | 5 +++-- node/node_handler.go | 16 +++++++++------- p2p/host/hostv2/hostv2.go | 2 -- test/configs/oneshard1.txt | 5 +++++ test/deploy.sh | 2 +- 8 files changed, 23 insertions(+), 19 deletions(-) create mode 100644 test/configs/oneshard1.txt diff --git a/api/service/networkinfo/service.go b/api/service/networkinfo/service.go index 738270cb1..a8b493e24 100644 --- a/api/service/networkinfo/service.go +++ b/api/service/networkinfo/service.go @@ -28,7 +28,6 @@ type Service struct { peerChan chan p2p.Peer peerInfo <-chan peerstore.PeerInfo discovery *libp2pdis.RoutingDiscovery - lock sync.Mutex } // New returns role conversion service. @@ -112,13 +111,11 @@ func (s *Service) DoService() { case peer := <-s.peerInfo: if peer.ID != s.Host.GetP2PHost().ID() && len(peer.ID) > 0 { utils.GetLogInstance().Info("Found Peer", "peer", peer.ID, "addr", peer.Addrs, "my ID", s.Host.GetP2PHost().ID()) - s.lock.Lock() if err := s.Host.GetP2PHost().Connect(s.ctx, peer); err != nil { utils.GetLogInstance().Warn("can't connect to peer node", "error", err) } else { utils.GetLogInstance().Info("connected to peer node", "peer", peer) } - s.lock.Unlock() // figure out the public ip/port ip := "127.0.0.1" var port string diff --git a/consensus/consensus.go b/consensus/consensus.go index b3ef03d1d..72a433bd4 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -379,7 +379,7 @@ func (consensus *Consensus) AddPeers(peers []*p2p.Peer) int { consensus.pubKeyLock.Lock() consensus.PublicKeys = append(consensus.PublicKeys, peer.PubKey) consensus.pubKeyLock.Unlock() - utils.GetLogInstance().Debug("[SYNC]", "new peer added", peer) + // utils.GetLogInstance().Debug("[SYNC]", "new peer added", peer) } count++ } diff --git a/consensus/consensus_leader.go b/consensus/consensus_leader.go index 48afe4a01..50bffa0f7 100644 --- a/consensus/consensus_leader.go +++ b/consensus/consensus_leader.go @@ -109,7 +109,8 @@ func (consensus *Consensus) startConsensus(newBlock *types.Block) { consensus.prepareSigs[consensus.nodeID] = consensus.priKey.SignHash(consensus.blockHash[:]) if utils.UseLibP2P { - consensus.host.SendMessageToGroups([]p2p.GroupID{p2p.GroupIDBeacon}, msgToSend) + // Construct broadcast p2p message + consensus.host.SendMessageToGroups([]p2p.GroupID{p2p.GroupIDBeacon}, host.ConstructP2pMessage(byte(17), msgToSend)) } else { host.BroadcastMessageFromLeader(consensus.host, consensus.GetValidatorPeers(), msgToSend, consensus.OfflinePeers) } @@ -171,7 +172,7 @@ func (consensus *Consensus) processPrepareMessage(message consensus_proto.Messag consensus.aggregatedPrepareSig = aggSig if utils.UseLibP2P { - consensus.host.SendMessageToGroups([]p2p.GroupID{p2p.GroupIDBeacon}, msgToSend) + consensus.host.SendMessageToGroups([]p2p.GroupID{p2p.GroupIDBeacon}, host.ConstructP2pMessage(byte(17), msgToSend)) } else { host.BroadcastMessageFromLeader(consensus.host, consensus.GetValidatorPeers(), msgToSend, consensus.OfflinePeers) } @@ -242,7 +243,7 @@ func (consensus *Consensus) processCommitMessage(message consensus_proto.Message consensus.aggregatedCommitSig = aggSig if utils.UseLibP2P { - consensus.host.SendMessageToGroups([]p2p.GroupID{p2p.GroupIDBeacon}, msgToSend) + consensus.host.SendMessageToGroups([]p2p.GroupID{p2p.GroupIDBeacon}, host.ConstructP2pMessage(byte(17), msgToSend)) } else { host.BroadcastMessageFromLeader(consensus.host, consensus.GetValidatorPeers(), msgToSend, consensus.OfflinePeers) } diff --git a/consensus/consensus_validator.go b/consensus/consensus_validator.go index 381bb8df1..3a89b3a2a 100644 --- a/consensus/consensus_validator.go +++ b/consensus/consensus_validator.go @@ -4,6 +4,7 @@ import ( "github.com/harmony-one/bls/ffi/go/bls" bls_cosi "github.com/harmony-one/harmony/crypto/bls" "github.com/harmony-one/harmony/p2p" + "github.com/harmony-one/harmony/p2p/host" "github.com/ethereum/go-ethereum/rlp" protobuf "github.com/golang/protobuf/proto" @@ -105,7 +106,7 @@ func (consensus *Consensus) processAnnounceMessage(message consensus_proto.Messa // Construct and send prepare message msgToSend := consensus.constructPrepareMessage() if utils.UseLibP2P { - consensus.host.SendMessageToGroups([]p2p.GroupID{p2p.GroupIDBeacon}, msgToSend) + consensus.host.SendMessageToGroups([]p2p.GroupID{p2p.GroupIDBeacon}, host.ConstructP2pMessage(byte(17), msgToSend)) } else { consensus.SendMessage(consensus.leader, msgToSend) } @@ -169,7 +170,7 @@ func (consensus *Consensus) processPreparedMessage(message consensus_proto.Messa multiSigAndBitmap := append(multiSig, bitmap...) msgToSend := consensus.constructCommitMessage(multiSigAndBitmap) if utils.UseLibP2P { - consensus.host.SendMessageToGroups([]p2p.GroupID{p2p.GroupIDBeacon}, msgToSend) + consensus.host.SendMessageToGroups([]p2p.GroupID{p2p.GroupIDBeacon}, host.ConstructP2pMessage(byte(17), msgToSend)) } else { consensus.SendMessage(consensus.leader, msgToSend) } diff --git a/node/node_handler.go b/node/node_handler.go index 0c251a9b1..70addde56 100644 --- a/node/node_handler.go +++ b/node/node_handler.go @@ -71,7 +71,7 @@ func (node *Node) ReceiveGroupMessage() { // messageHandler parses the message and dispatch the actions func (node *Node) messageHandler(content []byte, sender string) { - node.MaybeBroadcastAsValidator(content) + // node.MaybeBroadcastAsValidator(content) consensusObj := node.Consensus @@ -314,11 +314,13 @@ func (node *Node) pingMessageHandler(msgPayload []byte, sender string) int { return -1 } - utils.GetLogInstance().Debug("[pingMessageHandler]", "incoming peer", peer) + // utils.GetLogInstance().Debug("[pingMessageHandler]", "incoming peer", peer) // add to incoming peer list node.host.AddIncomingPeer(*peer) - node.host.ConnectHostPeer(*peer) + if utils.UseLibP2P { + node.host.ConnectHostPeer(*peer) + } if ping.Node.Role == proto_node.ClientRole { utils.GetLogInstance().Info("Add Client Peer to Node", "Node", node.Consensus.GetNodeID(), "Client", peer) @@ -349,7 +351,7 @@ func (node *Node) pingMessageHandler(msgPayload []byte, sender string) int { // FIXME: HAR-89 use a separate nodefind/neighbor message host.BroadcastMessageFromLeader(node.GetHost(), peers, buffer, node.Consensus.OfflinePeers) - utils.GetLogInstance().Info("PingMsgHandler send pong message") + // utils.GetLogInstance().Info("PingMsgHandler send pong message") } return 1 @@ -391,6 +393,8 @@ func (node *Node) SendPongMessage() { utils.GetLogInstance().Info("[PONG] sent pong message to", "group", p2p.GroupIDBeacon) } sentMessage = true + // stop sending ping message + node.serviceManager.TakeAction(&service.Action{Action: service.Stop, ServiceType: service.PeerDiscovery}) } } numPeers = numPeersNow @@ -424,8 +428,6 @@ func (node *Node) pongMessageHandler(msgPayload []byte) int { peers = append(peers, peer) } - utils.GetLogInstance().Debug("[pongMessageHandler]", "received msg #peers", len(peers)) - if len(peers) > 0 { node.AddPeers(peers) } @@ -446,7 +448,7 @@ func (node *Node) pongMessageHandler(msgPayload []byte) int { publicKeys = append(publicKeys, &key) } - utils.GetLogInstance().Debug("[pongMessageHandler]", "received msg #keys", len(publicKeys)) + utils.GetLogInstance().Debug("[pongMessageHandler]", "#keys", len(publicKeys), "#peers", len(peers)) if node.State == NodeWaitToJoin { node.State = NodeReadyForConsensus diff --git a/p2p/host/hostv2/hostv2.go b/p2p/host/hostv2/hostv2.go index 6eb097039..87caa2cec 100644 --- a/p2p/host/hostv2/hostv2.go +++ b/p2p/host/hostv2/hostv2.go @@ -246,8 +246,6 @@ func (host *HostV2) ConnectHostPeer(peer p2p.Peer) { utils.GetLogInstance().Error("ConnectHostPeer", "new peerinfo error", err, "peer", peer) return } - host.lock.Lock() - defer host.lock.Unlock() if err := host.h.Connect(ctx, *peerInfo); err != nil { utils.GetLogInstance().Warn("can't connect to peer", "error", err, "peer", peer) } else { diff --git a/test/configs/oneshard1.txt b/test/configs/oneshard1.txt new file mode 100644 index 000000000..995c9ec97 --- /dev/null +++ b/test/configs/oneshard1.txt @@ -0,0 +1,5 @@ +127.0.0.1 9000 leader 0 +127.0.0.1 9001 validator 0 +127.0.0.1 9002 validator 0 +127.0.0.1 9003 validator 0 +127.0.0.1 9004 validator 0 diff --git a/test/deploy.sh b/test/deploy.sh index 3e3f67e9a..f2de4d8c3 100755 --- a/test/deploy.sh +++ b/test/deploy.sh @@ -73,7 +73,7 @@ EOU DB= TXGEN=true DURATION=90 -MIN=5 +MIN=2 SHARDS=2 KILLPORT=9004 SYNC=false From 0f087c195b4219a33d45d17ca843d9a74b2e017d Mon Sep 17 00:00:00 2001 From: chaosma Date: Wed, 13 Feb 2019 16:11:22 -0800 Subject: [PATCH 12/14] add log for state syncing test; will be used for jenkins server later (#443) --- test/cal_tps.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/cal_tps.sh b/test/cal_tps.sh index 3a0985262..aa6069037 100755 --- a/test/cal_tps.sh +++ b/test/cal_tps.sh @@ -31,6 +31,8 @@ fi NUM_SHARDS=${#FILES[@]} SUM=0 NUM_CONSENSUS=0 +NUM_TOTAL_NODES=$( expr $NUM_SHARDS + $NUM_VALIDATORS ) +NUM_SIGS=0 declare -A TPS @@ -43,12 +45,14 @@ for f in "${FILES[@]}"; do else avg_tps=0 fi + num_sigs=$(grep numOfSignatures $f | tail -1 | cut -f 5 -d , | cut -f 2 -d :) TPS[$leader]="$num_consensus, $avg_tps" NUM_CONSENSUS=$(expr $NUM_CONSENSUS + $num_consensus ) SUM=$( expr $SUM + $avg_tps_int ) + NUM_SIGS=$( expr $NUM_SIGS + $num_sigs) done -echo $NUM_SHARDS shards, $NUM_CONSENSUS consensus, $SUM total TPS, $NUM_VALIDATORS nodes +echo $NUM_SHARDS shards, $NUM_CONSENSUS consensus, $SUM total TPS, $NUM_VALIDATORS nodes, $NUM_TOTAL_NODES total nodes, $NUM_SIGS total signatures for t in "${!TPS[@]}"; do echo $t, ${TPS[$t]} done From e4cf39b6c37b3533d0a5e7c2fd36dcf746ee561d Mon Sep 17 00:00:00 2001 From: Rongjian Lan Date: Wed, 13 Feb 2019 23:25:03 -0800 Subject: [PATCH 13/14] remove CheckEpochBlock --- core/blockchain.go | 14 ++++---------- core/resharding.go | 5 ----- node/node_newblock.go | 4 ++-- 3 files changed, 6 insertions(+), 17 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index 43319bfde..f23bb50d2 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -225,18 +225,12 @@ func (bc *BlockChain) ValidateNewBlock(block *types.Block, address common.Addres // IsEpochBlock returns whether this block is the first block of an epoch. func IsEpochBlock(block *types.Block) bool { - if block.NumberU64()%BlocksPerEpoch == 0 { - return true - } - return false + return block.NumberU64()%BlocksPerEpoch == 0 } // IsEpochLastBlock returns whether this block is the last block of an epoch. func IsEpochLastBlock(block *types.Block) bool { - if block.NumberU64()%BlocksPerEpoch == BlocksPerEpoch-1 { - return true - } - return false + return block.NumberU64()%BlocksPerEpoch == BlocksPerEpoch-1 } func (bc *BlockChain) getProcInterrupt() bool { @@ -1699,11 +1693,11 @@ func (bc *BlockChain) GetRandSeedByNumber(number uint64) int64 { // epoch block is where the new shard state stored func (bc *BlockChain) GetNewShardState(block *types.Block) types.ShardState { hash := block.Hash() - number := block.NumberU64() // just ignore non-epoch block - if !CheckEpochBlock(number) { + if !IsEpochBlock(block) { return nil } + number := block.NumberU64() shardState := bc.GetShardState(hash, number) if shardState == nil { epoch := GetEpochFromBlockNumber(number) diff --git a/core/resharding.go b/core/resharding.go index ab928cd53..9c7c45a0f 100644 --- a/core/resharding.go +++ b/core/resharding.go @@ -94,11 +94,6 @@ func GetEpochFromBlockNumber(blockNumber uint64) uint64 { return blockNumber / uint64(BlocksPerEpoch) } -// CheckEpochBlock check whethere a given block number is the one to store epoch information -func CheckEpochBlock(blockNumber uint64) bool { - return blockNumber%uint64(BlocksPerEpoch) == 0 -} - // GetPreviousEpochBlockNumber gets the epoch block number of previous epoch func GetPreviousEpochBlockNumber(blockNumber uint64) uint64 { epoch := GetEpochFromBlockNumber(blockNumber) diff --git a/node/node_newblock.go b/node/node_newblock.go index 0785d85d9..2d5941850 100644 --- a/node/node_newblock.go +++ b/node/node_newblock.go @@ -82,12 +82,12 @@ func (node *Node) addNewShardState(block *types.Block) { } func (node *Node) addNewRandSeed(block *types.Block) { - blockNumber := block.NumberU64() - if !core.CheckEpochBlock(blockNumber) { + if !core.IsEpochBlock(block) { return } var rnd int64 + blockNumber := block.NumberU64() epoch := core.GetEpochFromBlockNumber(blockNumber) if epoch == 1 { rnd = core.InitialSeed From b274cef42084eee99f31d60010ef4357d521aa21 Mon Sep 17 00:00:00 2001 From: Rongjian Lan Date: Thu, 14 Feb 2019 16:32:41 -0800 Subject: [PATCH 14/14] Fix test on p256 --- crypto/vrf/p256/p256_test.go | 4 +- test/testdata/directory.json | 4 +- test/testdata/getentryresponse.json | 782 +++++++++++++++++++++++++--- 3 files changed, 712 insertions(+), 78 deletions(-) diff --git a/crypto/vrf/p256/p256_test.go b/crypto/vrf/p256/p256_test.go index bf16a80b1..534701cc7 100644 --- a/crypto/vrf/p256/p256_test.go +++ b/crypto/vrf/p256/p256_test.go @@ -207,13 +207,13 @@ func TestProofToHash(t *testing.T) { if err != nil { t.Fatalf("ReadFile(%v): %v", respFile, err) } - var getUserResponses []testdata.GetUserResponseVector + var getUserResponses []testdata.ResponseVector if err := json.Unmarshal(b, &getUserResponses); err != nil { t.Fatalf("Unmarshal(): %v", err) } for _, tc := range getUserResponses { t.Run(tc.Desc, func(t *testing.T) { - _, err := pk.ProofToHash([]byte(tc.UserID), tc.Resp.GetLeaf().GetVrfProof()) + _, err := pk.ProofToHash([]byte(tc.UserIDs[0]), tc.GetUserResp.GetLeaf().GetVrfProof()) if err != nil { t.Errorf("ProofToHash(%v): %v)", tc.Desc, err) } diff --git a/test/testdata/directory.json b/test/testdata/directory.json index 945fad4ff..217b7dd4e 100644 --- a/test/testdata/directory.json +++ b/test/testdata/directory.json @@ -1,7 +1,7 @@ { "directoryId": "integration", "log": { - "treeId": "8541686838476068721", + "treeId": "6511398593182094144", "treeType": "PREORDERED_LOG", "hashStrategy": "RFC6962_SHA256", "hashAlgorithm": "SHA256", @@ -11,7 +11,7 @@ } }, "map": { - "treeId": "6598072539431303895", + "treeId": "7627063266021945174", "treeType": "MAP", "hashStrategy": "CONIKS_SHA256", "hashAlgorithm": "SHA256", diff --git a/test/testdata/getentryresponse.json b/test/testdata/getentryresponse.json index 7cf5de274..a5c5357ca 100644 --- a/test/testdata/getentryresponse.json +++ b/test/testdata/getentryresponse.json @@ -1,29 +1,31 @@ [ { "Desc": "empty_alice", - "UserID": "alice", - "Resp": { + "UserIDs": [ + "alice" + ], + "GetUserResp": { "revision": { "map_root": { "map_root": { - "map_root": "AAEgT5c5rf7RjwprNaZTxAls9fuKTJ0h1PFrBF5VyUgfglMVfD9mQONvCAAAAAAAAAAAAAA=", - "signature": "MEUCIQCNSY1U94y28kwMU6JEzxpVT0NfxgSlBqW+Q+9gss6JqAIgDVQvDTPEC80qML4i0g0d/ZBRU/mRyW6Zf4IcWmFu4Fc=" + "map_root": "AAEgoMiXEtKsVlKmOMkHhZ8DIdUpVxBXevDQqyz8HlVTYpwVg0ku8ZcHVAAAAAAAAAAAAAA=", + "signature": "MEYCIQDhwON1ZH6j7qw5/kQTtGgO4a3zI028ajCIOQ6BdtMB3AIhAJE+NqncdDfJmYGklOFxyD9vfEWm5RlqRXvkRxV/jgEl" } }, "latest_log_root": { "log_root": { - "timestamp_nanos": 1548182080757002000, - "root_hash": "oFazZRJgxpnfHPgM+Ki1hu+w5plbmP2mk8OC7pNtYUM=", + "timestamp_nanos": 1550163163063238915, + "root_hash": "zLL4sTnEciIm2uaPl/n7XPbiawbtx+63RyxDJz6fVJI=", "tree_size": 1, "tree_revision": 1, - "key_hint": "dooq7H3+13E=", - "log_root": "AAEAAAAAAAAAASCgVrNlEmDGmd8c+Az4qLWG77DmmVuY/aaTw4Luk21hQxV8P2ZbJW8QAAAAAAAAAAEAAA==", - "log_root_signature": "MEUCIQC9f1bDtioj1ToCJE7pFpDXoR4WDW81MZq5LeNkgiNt3wIgfiMGSLgRp1yIAC2L7bFz7qzkBpSCDJ8ECGZ/628aZms=" + "key_hint": "Wl0ihYmY00A=", + "log_root": "AAEAAAAAAAAAASDMsvixOcRyIiba5o+X+ftc9uJrBu3H7rdHLEMnPp9UkhWDSS8GR8EDAAAAAAAAAAEAAA==", + "log_root_signature": "MEYCIQDV9wVMePn/6JEOnmvr56evMDPWAs9ocmx9BZryPbumJgIhAM/RuOeXp+9eyqnuX0Xb07efPjvDX/ugRFy/ILp3CuZo" } } }, "leaf": { - "vrf_proof": "rxm+eUnt0zCGVeHtrn0SF9dUZ6ZPgRcWYqroKpjNonQ8y+scOc+/3QuS9K9c6otPQK2yuHbSySnCDjsVGFbwawRTJUYW21qW/fCBqGRPufN2+S2CHVyVSP1EWLTIn1M0zJgw+OQcAgHUdSP1DkqoE5X9Le+VWO+nuJtc4eJODJCo", + "vrf_proof": "CsrlVdKdfH2wcfVeaNtlVpVgUyLFmuRtLr9Q6sZa1m+Y+GgE1x2VXo+mfpKU0Txz7OPUv3JH0fMotV+NRWj33ARTJUYW21qW/fCBqGRPufN2+S2CHVyVSP1EWLTIn1M0zJgw+OQcAgHUdSP1DkqoE5X9Le+VWO+nuJtc4eJODJCo", "map_inclusion": { "leaf": { "index": "A9/B/HF0DP6pap5CSp8/Jo05FhoGWECfsTAnRlVdzvM=" @@ -289,39 +291,42 @@ } } }, + "BatchListUserRevisionsResp": null, "TrustNewLog": true }, { "Desc": "bob0_set", - "UserID": "bob", - "Resp": { + "UserIDs": [ + "bob" + ], + "GetUserResp": { "revision": { "map_root": { "map_root": { - "map_root": "AAEgvZoGMrg0Mz2GPvObuOxCbt0iwm3F4j6oFPssBYof/hgVfD9mqlPR0AAAAAAAAAABABISAhgBEgwQsf2F/Onsj74VGAI=", - "signature": "MEQCIEpt5DMWdTCRsQA9fnzA2lO5XqD5p/fuJBNkO5sJhaX3AiBwPXdC16D69a7/YZY6pICK3GK7n2oYQxSWOVbBHJhqLg==" + "map_root": "AAEg+O7XQigTSTtnjWvypQfnoTvYtsZMirOiOG6wsJEZk+UVg0kvHAH/3QAAAAAAAAABABISAhgBEgwQmv2Yq/Gl0sEVGAI=", + "signature": "MEUCIGHAhD1602oqYuCCV/SK4Y+isQzqGWvPCRGUAMf5CeMYAiEAxKMu4h5+elzr+/S5c4aOlPN9hS2EZSZK0QMxVVa3S34=" }, "log_inclusion": [ - "oFazZRJgxpnfHPgM+Ki1hu+w5plbmP2mk8OC7pNtYUM=" + "zLL4sTnEciIm2uaPl/n7XPbiawbtx+63RyxDJz6fVJI=" ] }, "latest_log_root": { "log_root": { - "timestamp_nanos": 1548182082188587000, - "root_hash": "iLzyO7OaPvSPpetDbGd9eNpo/NfUcEYeBta8z/hYG3M=", + "timestamp_nanos": 1550163163560158238, + "root_hash": "VigWWNMYkkC/tjfy+5vtTukZXZnppaYja73iLZoyD7c=", "tree_size": 2, "tree_revision": 2, - "key_hint": "dooq7H3+13E=", - "log_root": "AAEAAAAAAAAAAiCIvPI7s5o+9I+l60NsZ3142mj819RwRh4G1rzP+FgbcxV8P2awea/4AAAAAAAAAAIAAA==", - "log_root_signature": "MEUCIC8BasnxCIVXyPFjgdRYy64Pnu4ln/fTGHERN+5CqRpKAiEAoYwgFeRYZZoRaKdBgUcu2noiCr4LDoyohzSltYIOX58=" + "key_hint": "Wl0ihYmY00A=", + "log_root": "AAEAAAAAAAAAAiBWKBZY0xiSQL+2N/L7m+1O6RldmemlpiNrveItmjIPtxWDSS8j5iQeAAAAAAAAAAIAAA==", + "log_root_signature": "MEUCIHhitMsUiIXsnyQjLImwpMxZRt6OjQ9N5RVqtNqxxPqaAiEA5xHhyJctJsoiWNhysxmeHNrkqkNcJctY8Z77va6prkA=" }, "log_consistency": [ - "74BJt1hw9+CNRkcAlNhl0+nTymtLDwUBRsf4xRZP+Wk=" + "bg5yr6Foqkjs6NdFXL3HqQqTBzy8hIsJ/a4JdgCH2jg=" ] } }, "leaf": { - "vrf_proof": "5QGNxndM4zopl1q2G3ry8eGSW5tB9nqP44e6LYpb8duYiNlm42VkdehwD6IDQus3CJOssIz62JxgwJpk/yESggSHy1HLhWFLT+nQFEzzYq4x2psj6PyUNlaPWfnwVizyOB31qTBMPiNmlf7Qgp/yArNoYDSuvrjR9Jmlku+iA5MU", + "vrf_proof": "eKDyJl+cnnPH5/6ssSxdSiizLlATcMNwgYAFaynyuImzKHJqvl6cQ3TCnArWDnImX6OOGBGrsK4ZOf9qilihEwSHy1HLhWFLT+nQFEzzYq4x2psj6PyUNlaPWfnwVizyOB31qTBMPiNmlf7Qgp/yArNoYDSuvrjR9Jmlku+iA5MU", "map_inclusion": { "leaf": { "index": "Umn2fclcSdomcb9UlHcDY1SLm1A/wILzh8NdoYffbcQ=" @@ -581,46 +586,49 @@ "", "", "", - "KQiyHtT5vfRpQnj3OgxqeTUJjwgqKRh8YrhyDumC9EE=", + "3ZKTcseQ0R0OSbWOic6vzi/yxHE5c0lpJCz3pjhLlKc=", "" ] } } }, + "BatchListUserRevisionsResp": null, "TrustNewLog": false }, { "Desc": "set_carol", - "UserID": "carol", - "Resp": { + "UserIDs": [ + "carol" + ], + "GetUserResp": { "revision": { "map_root": { "map_root": { - "map_root": "AAEgq+MeLyWm6X86AnvprF02hvXcohWdvDQ80iI3oSmegPwVfD9mx3OVmAAAAAAAAAACABwSAhgBEhYIsf2F/Onsj74VEIGz9Nvr7I++FRgC", - "signature": "MEYCIQD07j/mrSgmwpxSrJ1Y+Wphn8OQUefOAnVf1uhk8ayylAIhAMcZ3Whap5q86/DoB3PPc559er22dhswOzc2f8Fju+N0" + "map_root": "AAEgwS/id1lfTZjhbfKdlZN0T6SNj+lsTgrvgC3tLQYJqakVg0kvNAxqPwAAAAAAAAACABwSAhgBEhYImv2Yq/Gl0sEVEMrXsu/ypdLBFRgC", + "signature": "MEYCIQDdMG9YU9AsqeGRbBcB3TPoBmpXXROtqDti1eWNzKRV9wIhANPkgGRQ7AZPxMIL3phdEwoxom+T8nuNTCHxMMC0VGYV" }, "log_inclusion": [ - "iLzyO7OaPvSPpetDbGd9eNpo/NfUcEYeBta8z/hYG3M=" + "VigWWNMYkkC/tjfy+5vtTukZXZnppaYja73iLZoyD7c=" ] }, "latest_log_root": { "log_root": { - "timestamp_nanos": 1548182082690022000, - "root_hash": "e+oc/pnJysibFt+ukJjutna0OsAuYxSjpTnSQ/mfYtg=", + "timestamp_nanos": 1550163164059865137, + "root_hash": "nC756jJgJOBGCYGS/xtyZ7eTNNkwKZ95MSHz+d766bM=", "tree_size": 3, "tree_revision": 3, - "key_hint": "dooq7H3+13E=", - "log_root": "AAEAAAAAAAAAAyB76hz+mcnKyJsW366QmO62drQ6wC5jFKOlOdJD+Z9i2BV8P2bOXPpwAAAAAAAAAAMAAA==", - "log_root_signature": "MEUCIQDEMBnuDL0+CdMf7tLf53eD0+oA5uaBaZ1nRwWZYf5shAIgZJvUuSH2x3H5KrL+tmPYALaTXCfG7RggbxHnai2DYqA=" + "key_hint": "Wl0ihYmY00A=", + "log_root": "AAEAAAAAAAAAAyCcLvnqMmAk4EYJgZL/G3Jnt5M02TApn3kxIfP53vrpsxWDSS9BrxAxAAAAAAAAAAMAAA==", + "log_root_signature": "MEQCIClfsNvnG/4xpxbTs4jzwsE4Vikpe0Fro0OfdT3efOM4AiADIW6Z63SOUxg+lYUl2MX8rYdlAfKp2n+NxgalObtZuA==" }, "log_consistency": [ - "74BJt1hw9+CNRkcAlNhl0+nTymtLDwUBRsf4xRZP+Wk=", - "JwjPnsR+/v/pG79eO4J9a+7huD2EB4ulIYwrrWhH7V0=" + "bg5yr6Foqkjs6NdFXL3HqQqTBzy8hIsJ/a4JdgCH2jg=", + "jfNr9jOWGPAwb9oanLSYBabZXp5ShHREKQkDXxJ58Ts=" ] } }, "leaf": { - "vrf_proof": "2oz80eq1m7ogFje1YDDFi82+kApvcQ3/wSMyegOlE5gNZEzguYGUGWdcG09XWokfGMJwd3NEWR2TLf/uxZkENwSmXjmxq3oAt/q89fwRZiF4eoAGepK4YcyzKtBD4mfS6gB7/AQ7PNRUocMVfUQnZCienTNXyrdtaOTCtzwaDIjc", + "vrf_proof": "jUkQi8HRJjH94aNdAxbVNcIP+Yq8wh937Odr8hZCgBkQfNa5pbQrNbyrc3Kn7DXa/djUY9pqrhiAb6VfnKxzNwSmXjmxq3oAt/q89fwRZiF4eoAGepK4YcyzKtBD4mfS6gB7/AQ7PNRUocMVfUQnZCienTNXyrdtaOTCtzwaDIjc", "map_inclusion": { "leaf": { "index": "JYx5mwUZM4rLnxSas5/NC9GiuqMYVWThqNqJdDz6bvw=" @@ -879,53 +887,56 @@ "", "", "", - "PhzerX1zlg+7TcTDd4dqhaEoCHIBEx/gkOHP4HwMrEE=", - "+Yz47fGxCilRS8cEhVPZNxVJDbwWMtb8kc7ckdny0Yw=", + "CS0fqBO+DMoGNyIO4OtmyMS99zrCjYAsidzOoIuujVU=", + "5QCCepR0tYVczBRnCvxlBMim8tO00KBKfX6dgKpDDQ8=", "" ] } } }, + "BatchListUserRevisionsResp": null, "TrustNewLog": false }, { "Desc": "bob1_get", - "UserID": "bob", - "Resp": { + "UserIDs": [ + "bob" + ], + "GetUserResp": { "revision": { "map_root": { "map_root": { - "map_root": "AAEgPJMMcS3zYA+lydtkvfNk8bgbQzLfemGSX7FhLr1kM4QVfD9m4Ul9wAAAAAAAAAADABwSAhgBEhYIgbP02+vsj74VEPm2kLrt7I++FRgC", - "signature": "MEUCIBtnr8CGfXlQvwD5zlGHu8YkTLktLv5QWbajrB1/kKQQAiEAzTdT19rnuMNtOROFryClaVo2f2KSRwuxTYFxOA/qNk4=" + "map_root": "AAEgr8xAKP6EGtoSi+ZFNI6ZwPimd9+KywJtad/yEJNevXgVg0kvUUZDTwAAAAAAAAADACYSDBDchvfv9KXSwRUYARIWCMrXsu/ypdLBFRDK17Lv8qXSwRUYAg==", + "signature": "MEUCIQDOirn/pstnBvgxdGn1kjNh5+fU5r2nrQvixjrboEHWigIgUFhzwB9b2+wEtd7hEuhTOOVx51/mO2CL+7tRrdFQ4D8=" }, "log_inclusion": [ - "JwjPnsR+/v/pG79eO4J9a+7huD2EB4ulIYwrrWhH7V0=", - "iLzyO7OaPvSPpetDbGd9eNpo/NfUcEYeBta8z/hYG3M=" + "jfNr9jOWGPAwb9oanLSYBabZXp5ShHREKQkDXxJ58Ts=", + "VigWWNMYkkC/tjfy+5vtTukZXZnppaYja73iLZoyD7c=" ] }, "latest_log_root": { "log_root": { - "timestamp_nanos": 1548182083190127000, - "root_hash": "lE/bH+OqOzXs2rFNvTfgu7gAyqYdx8s9kdUPLjHhS1M=", + "timestamp_nanos": 1550163164559404835, + "root_hash": "rPSTqha5qIxXu6quztMmbIFdA6JNSzcUt+SvWyfwoEw=", "tree_size": 4, "tree_revision": 4, - "key_hint": "dooq7H3+13E=", - "log_root": "AAEAAAAAAAAABCCUT9sf46o7NezasU29N+C7uADKph3Hyz2R1Q8uMeFLUxV8P2bsK/mYAAAAAAAAAAQAAA==", - "log_root_signature": "MEUCIBMfUpMx8mu2yHupR8QZJrTGZL2HSHQW89iDo60LGETzAiEAjmthkwb3+Tzkz5QKSOU93XEmVKA7qzZgwqRuVClWXZc=" + "key_hint": "Wl0ihYmY00A=", + "log_root": "AAEAAAAAAAAABCCs9JOqFrmojFe7qq7O0yZsgV0Dok1LNxS35K9bJ/CgTBWDSS9fdW8jAAAAAAAAAAQAAA==", + "log_root_signature": "MEQCIDM6z/nW99h5K/uRJXb26Mhby7CIfXOWJjw8Uw57HLuhAiA0zeJKHaA0PKpi2gI1xVYVm2GFI3U0I1nmIuWqhLSbsw==" }, "log_consistency": [ - "74BJt1hw9+CNRkcAlNhl0+nTymtLDwUBRsf4xRZP+Wk=", - "KpnnGZxuBOimg2bVX+gxdBjrJEAWe/6Fr71HavKgH3M=" + "bg5yr6Foqkjs6NdFXL3HqQqTBzy8hIsJ/a4JdgCH2jg=", + "vhBdOlNe4DDPlfxT3Bd1xabPmwVsu5CKflo21y6C8Ms=" ] } }, "leaf": { - "vrf_proof": "BGmKJMizUD1YVQuhuDdCGQkYuwo7fyJVx5naAXFgT8gySm+KCJHXHGQT4B6HVUoFSrDG5Hw3JRdKRn3GgLKyzgSHy1HLhWFLT+nQFEzzYq4x2psj6PyUNlaPWfnwVizyOB31qTBMPiNmlf7Qgp/yArNoYDSuvrjR9Jmlku+iA5MU", + "vrf_proof": "X4rdMUgJvFL9XtkGlXq8e4D5nxsI13ZGSlcGYlGhtn0RM0u8sBtETbUIxDkZqYAIIHSNKzzeNwELrmNES9gr5ASHy1HLhWFLT+nQFEzzYq4x2psj6PyUNlaPWfnwVizyOB31qTBMPiNmlf7Qgp/yArNoYDSuvrjR9Jmlku+iA5MU", "map_inclusion": { "leaf": { "index": "Umn2fclcSdomcb9UlHcDY1SLm1A/wILzh8NdoYffbcQ=", - "leaf_hash": "dHeM44usHkJb0qjAYu8JX02MCq8VHolTyVXLDGJbWdI=", - "leaf_value": "Cv4BGiBSafZ9yVxJ2iZxv1SUdwNjVIubUD/AgvOHw12hh99txDIgHo4zXOSWjFcn5ACKbEwbEI9J4dXDq/gCobR85mMaO3E6lQEIARKQAQqHAQo1dHlwZS5nb29nbGVhcGlzLmNvbS9nb29nbGUuY3J5cHRvLnRpbmsuRWNkc2FQdWJsaWNLZXkSTBIGCAMQAhgCGiD7FU52mGR+kS2Xs4XygLK9bDfV1XiGcZtcM9t0WUvWeSIgmsoZ6shH0XVzZaQU9lPYV3EsZYiiNax6wCRQ8d53LbQYAxABGAEgAUIg47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFUSTAEAAAABMEUCIFaVVPyv3/XXtr4t9EaFRN7L0aqd6MvqNf5TuriTLAbKAiEA5aSSC4erRALn7WKjs+eXaXG2wa8WYxEjAaHC4a2QLlQ=" + "leaf_hash": "SQW0xHDpdSOozSCd4yT9rzz1fKmsJECsgcucG7F2ax0=", + "leaf_value": "Cv4BGiBSafZ9yVxJ2iZxv1SUdwNjVIubUD/AgvOHw12hh99txDIgUnwYF+DKuiCzn5AtkpYfMrJLLhmHjQFOs5PN/b2TTDQ6lQEIARKQAQqHAQo1dHlwZS5nb29nbGVhcGlzLmNvbS9nb29nbGUuY3J5cHRvLnRpbmsuRWNkc2FQdWJsaWNLZXkSTBIGCAMQAhgCGiD7FU52mGR+kS2Xs4XygLK9bDfV1XiGcZtcM9t0WUvWeSIgmsoZ6shH0XVzZaQU9lPYV3EsZYiiNax6wCRQ8d53LbQYAxABGAEgA0Ig47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFUSRjBEAiBlnjoyOans9beKGWEzPPslAcXzzI2f2c6iMNbSvcxwbwIgKaJFc0fkIYLqmGXTcoifGz/SpWsxxG0VWmmcJuqm03k=" }, "inclusion": [ "", @@ -1182,56 +1193,59 @@ "", "", "", - "H8rbxUJbcMZtCaW49J4nTPywn99uazGOTxYgJwV0XJI=", + "Rkta1z50aDLsEglMAtxm5hm1eskPnOzZaIoDVWuYRTo=", "" ] }, "committed": { - "key": "9yOpkO/50Op5NJJKdx1etw==", + "key": "lOyD/kDFCHddGQaufT0A9Q==", "data": "Ym9iLWtleTE=" } } }, + "BatchListUserRevisionsResp": null, "TrustNewLog": false }, { "Desc": "bob1_set", - "UserID": "bob", - "Resp": { + "UserIDs": [ + "bob" + ], + "GetUserResp": { "revision": { "map_root": { "map_root": { - "map_root": "AAEgPJMMcS3zYA+lydtkvfNk8bgbQzLfemGSX7FhLr1kM4QVfD9m4Ul9wAAAAAAAAAADABwSAhgBEhYIgbP02+vsj74VEPm2kLrt7I++FRgC", - "signature": "MEUCIBtnr8CGfXlQvwD5zlGHu8YkTLktLv5QWbajrB1/kKQQAiEAzTdT19rnuMNtOROFryClaVo2f2KSRwuxTYFxOA/qNk4=" + "map_root": "AAEgr8xAKP6EGtoSi+ZFNI6ZwPimd9+KywJtad/yEJNevXgVg0kvUUZDTwAAAAAAAAADACYSDBDchvfv9KXSwRUYARIWCMrXsu/ypdLBFRDK17Lv8qXSwRUYAg==", + "signature": "MEUCIQDOirn/pstnBvgxdGn1kjNh5+fU5r2nrQvixjrboEHWigIgUFhzwB9b2+wEtd7hEuhTOOVx51/mO2CL+7tRrdFQ4D8=" }, "log_inclusion": [ - "JwjPnsR+/v/pG79eO4J9a+7huD2EB4ulIYwrrWhH7V0=", - "iLzyO7OaPvSPpetDbGd9eNpo/NfUcEYeBta8z/hYG3M=" + "jfNr9jOWGPAwb9oanLSYBabZXp5ShHREKQkDXxJ58Ts=", + "VigWWNMYkkC/tjfy+5vtTukZXZnppaYja73iLZoyD7c=" ] }, "latest_log_root": { "log_root": { - "timestamp_nanos": 1548182083190127000, - "root_hash": "lE/bH+OqOzXs2rFNvTfgu7gAyqYdx8s9kdUPLjHhS1M=", + "timestamp_nanos": 1550163164559404835, + "root_hash": "rPSTqha5qIxXu6quztMmbIFdA6JNSzcUt+SvWyfwoEw=", "tree_size": 4, "tree_revision": 4, - "key_hint": "dooq7H3+13E=", - "log_root": "AAEAAAAAAAAABCCUT9sf46o7NezasU29N+C7uADKph3Hyz2R1Q8uMeFLUxV8P2bsK/mYAAAAAAAAAAQAAA==", - "log_root_signature": "MEUCIBMfUpMx8mu2yHupR8QZJrTGZL2HSHQW89iDo60LGETzAiEAjmthkwb3+Tzkz5QKSOU93XEmVKA7qzZgwqRuVClWXZc=" + "key_hint": "Wl0ihYmY00A=", + "log_root": "AAEAAAAAAAAABCCs9JOqFrmojFe7qq7O0yZsgV0Dok1LNxS35K9bJ/CgTBWDSS9fdW8jAAAAAAAAAAQAAA==", + "log_root_signature": "MEQCIDM6z/nW99h5K/uRJXb26Mhby7CIfXOWJjw8Uw57HLuhAiA0zeJKHaA0PKpi2gI1xVYVm2GFI3U0I1nmIuWqhLSbsw==" }, "log_consistency": [ - "74BJt1hw9+CNRkcAlNhl0+nTymtLDwUBRsf4xRZP+Wk=", - "KpnnGZxuBOimg2bVX+gxdBjrJEAWe/6Fr71HavKgH3M=" + "bg5yr6Foqkjs6NdFXL3HqQqTBzy8hIsJ/a4JdgCH2jg=", + "vhBdOlNe4DDPlfxT3Bd1xabPmwVsu5CKflo21y6C8Ms=" ] } }, "leaf": { - "vrf_proof": "GUOB6Ea7R/98Hdbx8iAcupj7RwWYDucLOTR5h2I4VhfTRhIYL5gdSxCeLHpeM4ryKt+OMjbXMbFbPYLpLf56JQSHy1HLhWFLT+nQFEzzYq4x2psj6PyUNlaPWfnwVizyOB31qTBMPiNmlf7Qgp/yArNoYDSuvrjR9Jmlku+iA5MU", + "vrf_proof": "1AP1ak1iddu5JsgtPI+bH0DOnqV40H/10G5oa92euFHvrxacC0E3bf4Sx4c9l01939a+ggMbJW+bQtaVVfoW2gSHy1HLhWFLT+nQFEzzYq4x2psj6PyUNlaPWfnwVizyOB31qTBMPiNmlf7Qgp/yArNoYDSuvrjR9Jmlku+iA5MU", "map_inclusion": { "leaf": { "index": "Umn2fclcSdomcb9UlHcDY1SLm1A/wILzh8NdoYffbcQ=", - "leaf_hash": "dHeM44usHkJb0qjAYu8JX02MCq8VHolTyVXLDGJbWdI=", - "leaf_value": "Cv4BGiBSafZ9yVxJ2iZxv1SUdwNjVIubUD/AgvOHw12hh99txDIgHo4zXOSWjFcn5ACKbEwbEI9J4dXDq/gCobR85mMaO3E6lQEIARKQAQqHAQo1dHlwZS5nb29nbGVhcGlzLmNvbS9nb29nbGUuY3J5cHRvLnRpbmsuRWNkc2FQdWJsaWNLZXkSTBIGCAMQAhgCGiD7FU52mGR+kS2Xs4XygLK9bDfV1XiGcZtcM9t0WUvWeSIgmsoZ6shH0XVzZaQU9lPYV3EsZYiiNax6wCRQ8d53LbQYAxABGAEgAUIg47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFUSTAEAAAABMEUCIFaVVPyv3/XXtr4t9EaFRN7L0aqd6MvqNf5TuriTLAbKAiEA5aSSC4erRALn7WKjs+eXaXG2wa8WYxEjAaHC4a2QLlQ=" + "leaf_hash": "SQW0xHDpdSOozSCd4yT9rzz1fKmsJECsgcucG7F2ax0=", + "leaf_value": "Cv4BGiBSafZ9yVxJ2iZxv1SUdwNjVIubUD/AgvOHw12hh99txDIgUnwYF+DKuiCzn5AtkpYfMrJLLhmHjQFOs5PN/b2TTDQ6lQEIARKQAQqHAQo1dHlwZS5nb29nbGVhcGlzLmNvbS9nb29nbGUuY3J5cHRvLnRpbmsuRWNkc2FQdWJsaWNLZXkSTBIGCAMQAhgCGiD7FU52mGR+kS2Xs4XygLK9bDfV1XiGcZtcM9t0WUvWeSIgmsoZ6shH0XVzZaQU9lPYV3EsZYiiNax6wCRQ8d53LbQYAxABGAEgA0Ig47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFUSRjBEAiBlnjoyOans9beKGWEzPPslAcXzzI2f2c6iMNbSvcxwbwIgKaJFc0fkIYLqmGXTcoifGz/SpWsxxG0VWmmcJuqm03k=" }, "inclusion": [ "", @@ -1488,16 +1502,636 @@ "", "", "", - "H8rbxUJbcMZtCaW49J4nTPywn99uazGOTxYgJwV0XJI=", + "Rkta1z50aDLsEglMAtxm5hm1eskPnOzZaIoDVWuYRTo=", "" ] }, "committed": { - "key": "9yOpkO/50Op5NJJKdx1etw==", + "key": "lOyD/kDFCHddGQaufT0A9Q==", "data": "Ym9iLWtleTE=" } } }, + "BatchListUserRevisionsResp": null, "TrustNewLog": false + }, + { + "Desc": "bob2_setkeys", + "UserIDs": [ + "bob" + ], + "GetUserResp": { + "revision": { + "map_root": { + "map_root": { + "map_root": "AAEg5FcjGwHeNaom4LLIVsDoAq78pdUAmPh4aO73uTsFxgkVg0kvddsCAwAAAAAAAAAEADASFgjchvfv9KXSwRUQ3Ib37/Sl0sEVGAESFgjK17Lv8qXSwRUQ1Lm98fal0sEVGAI=", + "signature": "MEUCIHWmtQTKyy50UJSRn5LwvDTbaCUxu0p81Ni9C3w9/wj8AiEAhab0XFv1oRLDYY6glFksjySMozOKs96TjQOv6lf/UX4=" + }, + "log_inclusion": [ + "rPSTqha5qIxXu6quztMmbIFdA6JNSzcUt+SvWyfwoEw=" + ] + }, + "latest_log_root": { + "log_root": { + "timestamp_nanos": 1550163165060111318, + "root_hash": "LAc8Z7Ikc8riPIslJt9F5znG1UQ++Dkn17hCGMzDqfc=", + "tree_size": 5, + "tree_revision": 5, + "key_hint": "Wl0ihYmY00A=", + "log_root": "AAEAAAAAAAAABSAsBzxnsiRzyuI8iyUm30XnOcbVRD74OSfXuEIYzMOp9xWDSS99TZvWAAAAAAAAAAUAAA==", + "log_root_signature": "MEYCIQC4mAyVpIIvN0RY6qnIKP1JzpWZ3Pu6KW7YyZvcJOjuVwIhAKfT+h+rqHo/89BcbDlr1WifsSJTqPK2Oyn3DuT5am7A" + }, + "log_consistency": [ + "bg5yr6Foqkjs6NdFXL3HqQqTBzy8hIsJ/a4JdgCH2jg=", + "vhBdOlNe4DDPlfxT3Bd1xabPmwVsu5CKflo21y6C8Ms=", + "M8kPsR06rZIcu6/1lIROQQF0CMbJyu9lax9B9R6YK8c=" + ] + } + }, + "leaf": { + "vrf_proof": "GLa+Gy7Oacf0fOcldzfdn8u22z30nk79AOodoUPGSo2xW3ch7yU415QSuJq81bicrnUOI0SHyE6DAJMeSoqWZgSHy1HLhWFLT+nQFEzzYq4x2psj6PyUNlaPWfnwVizyOB31qTBMPiNmlf7Qgp/yArNoYDSuvrjR9Jmlku+iA5MU", + "map_inclusion": { + "leaf": { + "index": "Umn2fclcSdomcb9UlHcDY1SLm1A/wILzh8NdoYffbcQ=", + "leaf_hash": "N6v5xDU33hGz222cGxckUks1jgMs/v5CQl9wUEhIVQU=", + "leaf_value": "Cv4BGiBSafZ9yVxJ2iZxv1SUdwNjVIubUD/AgvOHw12hh99txDIgS3wBoYbDyPPn1y3yT4dSrxcsUDwCTY0nsXHyPptnH3I6lQEIARKQAQqHAQo1dHlwZS5nb29nbGVhcGlzLmNvbS9nb29nbGUuY3J5cHRvLnRpbmsuRWNkc2FQdWJsaWNLZXkSTBIGCAMQAhgCGiD7FU52mGR+kS2Xs4XygLK9bDfV1XiGcZtcM9t0WUvWeSIgmsoZ6shH0XVzZaQU9lPYV3EsZYiiNax6wCRQ8d53LbQYAxABGAEgA0IgjKqSavCq1amOKbJKnkFkGDb6+bovPPwVkOU6pEwvIGQSRzBFAiEA8EyAUwNQjico7PZJHAJj9bENb1BFu++5FYwm2AFnkQUCIGCLB6s9C3t+O8hzmF9qhOCRKkUCdxIr79ZitbSdrHHn" + }, + "inclusion": [ + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "Rkta1z50aDLsEglMAtxm5hm1eskPnOzZaIoDVWuYRTo=", + "" + ] + }, + "committed": { + "key": "TIb45cgNVG99ZcMZZwKITg==", + "data": "Ym9iLWtleTI=" + } + } + }, + "BatchListUserRevisionsResp": null, + "TrustNewLog": false + }, + { + "Desc": "bob3_setnewkeys", + "UserIDs": [ + "bob" + ], + "GetUserResp": { + "revision": { + "map_root": { + "map_root": { + "map_root": "AAEgDEDkZ5Tp6QdJEtRV64qx1/BDWrHSacJRpuqjquFRwHgVg0kvjXzLqgAAAAAAAAAFADASFgjchvfv9KXSwRUQ3Ib37/Sl0sEVGAESFgjUub3x9qXSwRUQ/ISMyfil0sEVGAI=", + "signature": "MEUCIF/M2SrwldR0o6165z08Z+9qO2VZbai3VNPu/bFFQYJHAiEA9xFhOEKVvCEbDIIgP8n1G21IZwXRDzPsQnj3zvMs1LM=" + }, + "log_inclusion": [ + "M8kPsR06rZIcu6/1lIROQQF0CMbJyu9lax9B9R6YK8c=", + "rPSTqha5qIxXu6quztMmbIFdA6JNSzcUt+SvWyfwoEw=" + ] + }, + "latest_log_root": { + "log_root": { + "timestamp_nanos": 1550163165559842130, + "root_hash": "yqUFUtrPFww3aKN+UqSpmY02Gjp0Dyq8PzXHynF6tEE=", + "tree_size": 6, + "tree_revision": 6, + "key_hint": "Wl0ihYmY00A=", + "log_root": "AAEAAAAAAAAABiDKpQVS2s8XDDdoo35SpKmZjTYaOnQPKrw/NcfKcXq0QRWDSS+bFuVSAAAAAAAAAAYAAA==", + "log_root_signature": "MEQCIAZ9y3+bWwHTN232J6i52pBpdCGNdvmTEgOVVkztLq7WAiAarGKYogkwbmvSSvLUoLLcnB2pGI3CayBfC0TkWCGtAw==" + }, + "log_consistency": [ + "bg5yr6Foqkjs6NdFXL3HqQqTBzy8hIsJ/a4JdgCH2jg=", + "vhBdOlNe4DDPlfxT3Bd1xabPmwVsu5CKflo21y6C8Ms=", + "a/zGHQv9tBdH+doHigIOUzgLqJlKMamdrgsOC59ah2o=" + ] + } + }, + "leaf": { + "vrf_proof": "4SwdCKyFF9PinfDMpOdaxQYU2RPOxGNBT61RkTZ/81aaozLD9UIt1zfau/0IJy+wQ5A9QSdAuNKE2MV2J4G7cwSHy1HLhWFLT+nQFEzzYq4x2psj6PyUNlaPWfnwVizyOB31qTBMPiNmlf7Qgp/yArNoYDSuvrjR9Jmlku+iA5MU", + "map_inclusion": { + "leaf": { + "index": "Umn2fclcSdomcb9UlHcDY1SLm1A/wILzh8NdoYffbcQ=", + "leaf_hash": "Z8QYCCHOq9Wu/AGoO/nievbwKK1LDO+Ob3qcjprsbqE=", + "leaf_value": "CpEDGiBSafZ9yVxJ2iZxv1SUdwNjVIubUD/AgvOHw12hh99txDIgvpXp3sPS32ohOJNBIQi0hEPCxpaxv3ONUe26VKz0Tk06qAIIAhKQAQqHAQo1dHlwZS5nb29nbGVhcGlzLmNvbS9nb29nbGUuY3J5cHRvLnRpbmsuRWNkc2FQdWJsaWNLZXkSTBIGCAMQAhgCGiD7FU52mGR+kS2Xs4XygLK9bDfV1XiGcZtcM9t0WUvWeSIgmsoZ6shH0XVzZaQU9lPYV3EsZYiiNax6wCRQ8d53LbQYAxABGAEgAxKQAQqHAQo1dHlwZS5nb29nbGVhcGlzLmNvbS9nb29nbGUuY3J5cHRvLnRpbmsuRWNkc2FQdWJsaWNLZXkSTBIGCAMQAhgCGiAkoNtHi7KFIxdbzTHTY21hFQWUxBss4D60t78xvBnr1CIgkXRukcfdn/bWwunc+1FbMiO7yI0vvetbBCwOhXkfdZQYAxABGAIgA0IgSQjPfrKaUfNwT7SzUAlctcI2sRG/9MOqWc9FwP+ZSrISRjBEAiBFlOdB2SS19frUv7lXcwaMJmooaRP7l7JfRku+tSjVTQIgO7Ctz7gBqv7FEqrM04WfzmBUQU4bUUHNTzaxhZPgd9oSRzBFAiEAmdsQwMPZxuUUoYPgCAP+yWgUR8/5sR5Rfby+3JFjpcoCIB5shoI3OKMpO97q5VdCSuUOkWbra7tRehiFmQ/JAT12" + }, + "inclusion": [ + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "Rkta1z50aDLsEglMAtxm5hm1eskPnOzZaIoDVWuYRTo=", + "" + ] + }, + "committed": { + "key": "ijmEOG0QwH9tYl3DhXBT5g==", + "data": "Ym9iLWtleTM=" + } + } + }, + "BatchListUserRevisionsResp": null, + "TrustNewLog": true } ] \ No newline at end of file