Merge pull request #1639 from rlan35/mainnet_release_0921_merge_s3

Mainnet release 0921 merge s3

.hmy/wallet.ini

@@ -1,4 +1,5 @@
 [main]
+chain_id = 1
 bootnode = /ip4/100.26.90.187/tcp/9874/p2p/Qmdfjtk6hPoyrH1zVD9PEH4zfWLo38dP2mDvvKXfh3tnEv
 bootnode = /ip4/54.213.43.194/tcp/9874/p2p/QmZJJx6AdaoEkGLrYG4JeLCKeCKDjnFz2wfHNHxAqFSGA9
 bootnode = /ip4/13.113.101.219/tcp/12019/p2p/QmQayinFSgMMw5cSpDUiD9pQ2WeP6WNmGxpZ6ou3mdVFJX
@@ -22,6 +23,7 @@ rpc = l3.t.hmny.io:14555
 rpc = s3.t.hmny.io:14555
 
 [local]
+chain_id = 2
 bootnode = /ip4/127.0.0.1/tcp/19876/p2p/Qmc1V6W7BwX8Ugb42Ti8RnXF1rY5PF7nnZ6bKBryCgi6cv
 shards = 2
 
@@ -36,6 +38,7 @@ rpc = 127.0.0.1:14558
 rpc = 127.0.0.1:14560
 
 [beta]
+chain_id = 2
 bootnode = /ip4/54.213.43.194/tcp/9868/p2p/QmZJJx6AdaoEkGLrYG4JeLCKeCKDjnFz2wfHNHxAqFSGA9
 bootnode = /ip4/100.26.90.187/tcp/9868/p2p/Qmdfjtk6hPoyrH1zVD9PEH4zfWLo38dP2mDvvKXfh3tnEv
 bootnode = /ip4/13.113.101.219/tcp/12018/p2p/QmQayinFSgMMw5cSpDUiD9pQ2WeP6WNmGxpZ6ou3mdVFJX
@@ -50,6 +53,7 @@ rpc = l1.b.hmny.io:14555
 rpc = s1.b.hmny.io:14555
 
 [pangaea]
+chain_id = 3
 bootnode = /ip4/54.86.126.90/tcp/9889/p2p/Qmdfjtk6hPoyrH1zVD9PEH4zfWLo38dP2mDvvKXfh3tnEv
 bootnode = /ip4/52.40.84.2/tcp/9889/p2p/QmZJJx6AdaoEkGLrYG4JeLCKeCKDjnFz2wfHNHxAqFSGA9
 shards = 4

Makefile

@@ -6,7 +6,7 @@ export LIBRARY_PATH:=$(LD_LIBRARY_PATH)
 export DYLD_FALLBACK_LIBRARY_PATH:=$(LD_LIBRARY_PATH)
 export GO111MODULE:=on
 
-.PHONY: all libs exe
+.PHONY: all libs exe test
 
 all: libs
 	./scripts/go_executable_build.sh
@@ -17,3 +17,6 @@ libs:
 
 exe:
 	./scripts/go_executable_build.sh
+
+test:
+	./test/debug.sh

README.md

@@ -36,6 +36,8 @@ make
 
 ```
 
+Note: make sure to run ``` scripts/install_build_tools.sh ```to make sure build tools are of correct versions.
+
 ## Build
 
 If you want to bypass the Makefile:

api/service/explorer/service.go

@@ -50,7 +50,7 @@ type Service struct {
 	Port              string
 	GetNodeIDs        func() []libp2p_peer.ID
 	ShardID           uint32
-	storage           *Storage
+	Storage           *Storage
 	server            *http.Server
 	messageChan       chan *msg_pb.Message
 	GetAccountBalance func(common.Address) (*big.Int, error)
@@ -67,6 +67,16 @@ func New(selfPeer *p2p.Peer, shardID uint32, GetNodeIDs func() []libp2p_peer.ID,
 	}
 }
 
+// ServiceAPI is rpc api.
+type ServiceAPI struct {
+	Service *Service
+}
+
+// NewServiceAPI returns explorer service api.
+func NewServiceAPI(explorerService *Service) *ServiceAPI {
+	return &ServiceAPI{Service: explorerService}
+}
+
 // StartService starts explorer service.
 func (s *Service) StartService() {
 	utils.Logger().Info().Msg("Starting explorer service.")
@@ -95,7 +105,7 @@ func GetExplorerPort(nodePort string) string {
 
 // Init is to initialize for ExplorerService.
 func (s *Service) Init(remove bool) {
-	s.storage = GetStorageInstance(s.IP, s.Port, remove)
+	s.Storage = GetStorageInstance(s.IP, s.Port, remove)
 }
 
 // Run is to run serving explorer.
@@ -150,7 +160,7 @@ func (s *Service) ReadBlocksFromDB(from, to int) []*types.Block {
 			continue
 		}
 		key := GetBlockKey(i)
-		data, err := storage.db.Get([]byte(key))
+		data, err := s.Storage.db.Get([]byte(key))
 		if err != nil {
 			blocks = append(blocks, nil)
 			continue
@@ -172,6 +182,12 @@ func (s *Service) GetExplorerBlocks(w http.ResponseWriter, r *http.Request) {
 	to := r.FormValue("to")
 	pageParam := r.FormValue("page")
 	offsetParam := r.FormValue("offset")
+	withSignersParam := r.FormValue("with_signers")
+	withSigners := false
+	if withSignersParam == "true" {
+		withSigners = true
+	}
+
 	data := &Data{
 		Blocks: []*Block{},
 	}
@@ -186,7 +202,7 @@ func (s *Service) GetExplorerBlocks(w http.ResponseWriter, r *http.Request) {
 		w.WriteHeader(http.StatusBadRequest)
 		return
 	}
-	db := s.storage.GetDB()
+	db := s.Storage.GetDB()
 	fromInt, err := strconv.Atoi(from)
 	if err != nil {
 		utils.Logger().Warn().Err(err).Msg("invalid from parameter")
@@ -236,19 +252,21 @@ func (s *Service) GetExplorerBlocks(w http.ResponseWriter, r *http.Request) {
 	accountBlocks := s.ReadBlocksFromDB(fromInt, toInt)
 	curEpoch := int64(-1)
 	committee := &shard.Committee{}
+	if withSigners {
+		if bytes, err := db.Get([]byte(GetCommitteeKey(uint32(s.ShardID), 0))); err == nil {
+			if err = rlp.DecodeBytes(bytes, committee); err != nil {
+				utils.Logger().Warn().Err(err).Msg("cannot read committee for new epoch")
+			}
+		}
+	}
 	for id, accountBlock := range accountBlocks {
 		if id == 0 || id == len(accountBlocks)-1 || accountBlock == nil {
 			continue
 		}
 		block := NewBlock(accountBlock, id+fromInt-1)
-		if int64(block.Epoch) > curEpoch {
+		if withSigners && int64(block.Epoch) > curEpoch {
-			if bytes, err := db.Get([]byte(GetCommitteeKey(uint32(s.ShardID), block.Epoch))); err == nil {
+			if accountBlocks[id-1] != nil {
-				committee = &shard.Committee{}
+				state, err := accountBlocks[id-1].Header().GetShardState()
-				if err = rlp.DecodeBytes(bytes, committee); err != nil {
-					utils.Logger().Warn().Err(err).Msg("cannot read committee for new epoch")
-				}
-			} else {
-				state, err := accountBlock.Header().GetShardState()
 				if err == nil {
 					for _, shardCommittee := range state {
 						if shardCommittee.ShardID == accountBlock.ShardID() {
@@ -256,28 +274,32 @@ func (s *Service) GetExplorerBlocks(w http.ResponseWriter, r *http.Request) {
 							break
 						}
 					}
+				} else {
+					utils.Logger().Warn().Err(err).Msg("error parsing shard state")
 				}
 			}
 			curEpoch = int64(block.Epoch)
 		}
-		pubkeys := make([]*bls.PublicKey, len(committee.NodeList))
+		if withSigners {
-		for i, validator := range committee.NodeList {
+			pubkeys := make([]*bls.PublicKey, len(committee.NodeList))
-			pubkeys[i] = new(bls.PublicKey)
+			for i, validator := range committee.NodeList {
-			validator.BlsPublicKey.ToLibBLSPublicKey(pubkeys[i])
+				pubkeys[i] = new(bls.PublicKey)
-		}
+				validator.BlsPublicKey.ToLibBLSPublicKey(pubkeys[i])
-		mask, err := bls2.NewMask(pubkeys, nil)
+			}
-		if err == nil && accountBlocks[id+1] != nil {
+			mask, err := bls2.NewMask(pubkeys, nil)
-			err = mask.SetMask(accountBlocks[id+1].Header().LastCommitBitmap())
+			if err == nil && accountBlocks[id+1] != nil {
-			if err == nil {
+				err = mask.SetMask(accountBlocks[id+1].Header().LastCommitBitmap())
-				for _, validator := range committee.NodeList {
+				if err == nil {
-					oneAddress, err := common2.AddressToBech32(validator.EcdsaAddress)
+					for _, validator := range committee.NodeList {
-					if err != nil {
+						oneAddress, err := common2.AddressToBech32(validator.EcdsaAddress)
-						continue
+						if err != nil {
-					}
+							continue
-					blsPublicKey := new(bls.PublicKey)
+						}
-					validator.BlsPublicKey.ToLibBLSPublicKey(blsPublicKey)
+						blsPublicKey := new(bls.PublicKey)
-					if ok, err := mask.KeyEnabled(blsPublicKey); err == nil && ok {
+						validator.BlsPublicKey.ToLibBLSPublicKey(blsPublicKey)
-						block.Signers = append(block.Signers, oneAddress)
+						if ok, err := mask.KeyEnabled(blsPublicKey); err == nil && ok {
+							block.Signers = append(block.Signers, oneAddress)
+						}
 					}
 				}
 			}
@@ -315,12 +337,122 @@ func (s *Service) GetExplorerBlocks(w http.ResponseWriter, r *http.Request) {
 		}
 		data.Blocks = append(data.Blocks, block)
 	}
+	if offset*page+offset > len(data.Blocks) {
+		data.Blocks = data.Blocks[offset*page:]
+	} else {
+		data.Blocks = data.Blocks[offset*page : offset*page+offset]
+	}
+}
 
-	paginatedBlocks := make([]*Block, 0)
+// GetExplorerBlocks rpc end-point.
-	for i := 0; i < offset && i+offset*page < len(data.Blocks); i++ {
+func (s *ServiceAPI) GetExplorerBlocks(ctx context.Context, from, to, page, offset int, withSigners bool) ([]*Block, error) {
-		paginatedBlocks = append(paginatedBlocks, data.Blocks[i+offset*page])
+	if offset == 0 {
+		offset = paginationOffset
+	}
+	db := s.Service.Storage.GetDB()
+	if to == 0 {
+		bytes, err := db.Get([]byte(BlockHeightKey))
+		if err == nil {
+			to, err = strconv.Atoi(string(bytes))
+			if err != nil {
+				utils.Logger().Info().Msg("failed to fetch block height")
+				return nil, err
+			}
+		}
 	}
-	data.Blocks = paginatedBlocks
+	blocks := make([]*Block, 0)
+	accountBlocks := s.Service.ReadBlocksFromDB(from, to)
+	curEpoch := int64(-1)
+	committee := &shard.Committee{}
+	if withSigners {
+		if bytes, err := db.Get([]byte(GetCommitteeKey(uint32(s.Service.ShardID), 0))); err == nil {
+			if err = rlp.DecodeBytes(bytes, committee); err != nil {
+				utils.Logger().Warn().Err(err).Msg("cannot read committee for new epoch")
+			}
+		}
+	}
+	for id, accountBlock := range accountBlocks {
+		if id == 0 || id == len(accountBlocks)-1 || accountBlock == nil {
+			continue
+		}
+		block := NewBlock(accountBlock, id+from-1)
+		if withSigners && int64(block.Epoch) > curEpoch {
+			if accountBlocks[id-1] != nil {
+				state, err := accountBlocks[id-1].Header().GetShardState()
+				if err == nil {
+					for _, shardCommittee := range state {
+						if shardCommittee.ShardID == accountBlock.ShardID() {
+							committee = &shardCommittee
+							break
+						}
+					}
+				}
+			}
+			curEpoch = int64(block.Epoch)
+		}
+		if withSigners {
+			pubkeys := make([]*bls.PublicKey, len(committee.NodeList))
+			for i, validator := range committee.NodeList {
+				pubkeys[i] = new(bls.PublicKey)
+				validator.BlsPublicKey.ToLibBLSPublicKey(pubkeys[i])
+			}
+			mask, err := bls2.NewMask(pubkeys, nil)
+			if err == nil && accountBlocks[id+1] != nil {
+				err = mask.SetMask(accountBlocks[id+1].Header().LastCommitBitmap())
+				if err == nil {
+					for _, validator := range committee.NodeList {
+						oneAddress, err := common2.AddressToBech32(validator.EcdsaAddress)
+						if err != nil {
+							continue
+						}
+						blsPublicKey := new(bls.PublicKey)
+						validator.BlsPublicKey.ToLibBLSPublicKey(blsPublicKey)
+						if ok, err := mask.KeyEnabled(blsPublicKey); err == nil && ok {
+							block.Signers = append(block.Signers, oneAddress)
+						}
+					}
+				}
+			}
+		}
+		// Populate transactions
+		for _, tx := range accountBlock.Transactions() {
+			transaction := GetTransaction(tx, accountBlock)
+			if transaction != nil {
+				block.TXs = append(block.TXs, transaction)
+			}
+		}
+		if accountBlocks[id-1] == nil {
+			block.BlockTime = int64(0)
+			block.PrevBlock = RefBlock{
+				ID:     "",
+				Height: "",
+			}
+		} else {
+			block.BlockTime = accountBlock.Time().Int64() - accountBlocks[id-1].Time().Int64()
+			block.PrevBlock = RefBlock{
+				ID:     accountBlocks[id-1].Hash().Hex(),
+				Height: strconv.Itoa(id + from - 2),
+			}
+		}
+		if accountBlocks[id+1] == nil {
+			block.NextBlock = RefBlock{
+				ID:     "",
+				Height: "",
+			}
+		} else {
+			block.NextBlock = RefBlock{
+				ID:     accountBlocks[id+1].Hash().Hex(),
+				Height: strconv.Itoa(id + from),
+			}
+		}
+		blocks = append(blocks, block)
+	}
+	if offset*page+offset > len(blocks) {
+		blocks = blocks[offset*page:]
+	} else {
+		blocks = blocks[offset*page : offset*page+offset]
+	}
+	return blocks, nil
 }
 
 // GetExplorerTransaction servers /tx end-point.
@@ -339,7 +471,7 @@ func (s *Service) GetExplorerTransaction(w http.ResponseWriter, r *http.Request)
 		w.WriteHeader(http.StatusBadRequest)
 		return
 	}
-	db := s.storage.GetDB()
+	db := s.Storage.GetDB()
 	bytes, err := db.Get([]byte(GetTXKey(id)))
 	if err != nil {
 		utils.Logger().Warn().Err(err).Str("id", id).Msg("cannot read TX")
@@ -355,6 +487,26 @@ func (s *Service) GetExplorerTransaction(w http.ResponseWriter, r *http.Request)
 	data.TX = *tx
 }
 
+// GetExplorerTransaction rpc end-point.
+func (s *ServiceAPI) GetExplorerTransaction(ctx context.Context, id string) (*Transaction, error) {
+	if id == "" {
+		utils.Logger().Warn().Msg("invalid id parameter")
+		return nil, nil
+	}
+	db := s.Service.Storage.GetDB()
+	bytes, err := db.Get([]byte(GetTXKey(id)))
+	if err != nil {
+		utils.Logger().Warn().Err(err).Str("id", id).Msg("cannot read TX")
+		return nil, err
+	}
+	tx := new(Transaction)
+	if rlp.DecodeBytes(bytes, tx) != nil {
+		utils.Logger().Warn().Str("id", id).Msg("cannot convert data from DB")
+		return nil, err
+	}
+	return tx, nil
+}
+
 // GetExplorerCommittee servers /comittee end-point.
 func (s *Service) GetExplorerCommittee(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Content-Type", "application/json")
@@ -385,7 +537,7 @@ func (s *Service) GetExplorerCommittee(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	// fetch current epoch if epoch is 0
-	db := s.storage.GetDB()
+	db := s.Storage.GetDB()
 	if epoch == 0 {
 		bytes, err := db.Get([]byte(BlockHeightKey))
 		blockHeight, err := strconv.Atoi(string(bytes))
@@ -435,6 +587,56 @@ func (s *Service) GetExplorerCommittee(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+// GetExplorerCommittee rpc end-point.
+func (s *ServiceAPI) GetExplorerCommittee(ctx context.Context, shardID uint32, epoch uint64) (*Committee, error) {
+	if s.Service.ShardID != uint32(shardID) {
+		utils.Logger().Warn().Msg("incorrect shard id")
+		return nil, nil
+	}
+	// fetch current epoch if epoch is 0
+	db := s.Service.Storage.GetDB()
+	if epoch == 0 {
+		bytes, err := db.Get([]byte(BlockHeightKey))
+		blockHeight, err := strconv.Atoi(string(bytes))
+		if err != nil {
+			utils.Logger().Warn().Err(err).Msg("cannot decode block height from DB")
+			return nil, err
+		}
+		key := GetBlockKey(blockHeight)
+		data, err := db.Get([]byte(key))
+		block := new(types.Block)
+		if rlp.DecodeBytes(data, block) != nil {
+			utils.Logger().Warn().Err(err).Msg("cannot get block from db")
+			return nil, err
+		}
+		epoch = block.Epoch().Uint64()
+	}
+	bytes, err := db.Get([]byte(GetCommitteeKey(uint32(shardID), epoch)))
+	if err != nil {
+		utils.Logger().Warn().Err(err).Msg("cannot read committee")
+		return nil, err
+	}
+	committee := &shard.Committee{}
+	if err := rlp.DecodeBytes(bytes, committee); err != nil {
+		utils.Logger().Warn().Err(err).Msg("cannot decode committee data from DB")
+		return nil, err
+	}
+	validators := &Committee{}
+	for _, validator := range committee.NodeList {
+		validatorBalance := big.NewInt(0)
+		validatorBalance, err := s.Service.GetAccountBalance(validator.EcdsaAddress)
+		if err != nil {
+			continue
+		}
+		oneAddress, err := common2.AddressToBech32(validator.EcdsaAddress)
+		if err != nil {
+			continue
+		}
+		validators.Validators = append(validators.Validators, &Validator{Address: oneAddress, Balance: validatorBalance})
+	}
+	return validators, nil
+}
+
 // GetExplorerAddress serves /address end-point.
 func (s *Service) GetExplorerAddress(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Content-Type", "application/json")
@@ -493,24 +695,20 @@ func (s *Service) GetExplorerAddress(w http.ResponseWriter, r *http.Request) {
 		balance, err := s.GetAccountBalance(address)
 		if err == nil {
 			balanceAddr = balance
-			data.Address.Balance = balance
 		}
 	}
 
-	db := s.storage.GetDB()
+	db := s.Storage.GetDB()
 	bytes, err := db.Get([]byte(key))
-	if err != nil {
+
-		utils.Logger().Warn().Err(err).Str("id", id).Msg("cannot read address from db")
-		return
-	}
 	if err = rlp.DecodeBytes(bytes, &data.Address); err != nil {
 		utils.Logger().Warn().Str("id", id).Msg("cannot convert data from DB")
 		w.WriteHeader(http.StatusInternalServerError)
 		return
 	}
-	if balanceAddr.Cmp(big.NewInt(0)) != 0 {
+
-		data.Address.Balance = balanceAddr
+	data.Address.Balance = balanceAddr
-	}
+
 	switch txView {
 	case txViewNone:
 		data.Address.TXs = nil
@@ -531,11 +729,80 @@ func (s *Service) GetExplorerAddress(w http.ResponseWriter, r *http.Request) {
 		}
 		data.Address.TXs = sentTXs
 	}
-	paginatedTXs := make([]*Transaction, 0)
+	if offset*page+offset > len(data.Address.TXs) {
-	for i := 0; i < offset && i+offset*page < len(data.Address.TXs); i++ {
+		data.Address.TXs = data.Address.TXs[offset*page:]
-		paginatedTXs = append(paginatedTXs, data.Address.TXs[i+offset*page])
+	} else {
+		data.Address.TXs = data.Address.TXs[offset*page : offset*page+offset]
+	}
+}
+
+// GetExplorerAddress rpc end-point.
+func (s *ServiceAPI) GetExplorerAddress(ctx context.Context, id, txView string, page, offset int) (*Address, error) {
+	if offset == 0 {
+		offset = paginationOffset
+	}
+	if txView == "" {
+		txView = txViewNone
+	}
+	utils.Logger().Info().Str("Address", id).Msg("Querying address")
+	if id == "" {
+		utils.Logger().Warn().Msg("missing address id param")
+		return nil, nil
+	}
+	address := &Address{}
+	address.ID = id
+	// Try to populate the banace by directly calling get balance.
+	// Check the balance from blockchain rather than local DB dump
+	balanceAddr := big.NewInt(0)
+	if s.Service.GetAccountBalance != nil {
+		addr := common2.ParseAddr(id)
+		balance, err := s.Service.GetAccountBalance(addr)
+		if err == nil {
+			balanceAddr = balance
+			address.Balance = balance
+		}
+	}
+
+	key := GetAddressKey(id)
+	db := s.Service.Storage.GetDB()
+	bytes, err := db.Get([]byte(key))
+	if err != nil {
+		utils.Logger().Warn().Err(err).Str("id", id).Msg("cannot read address from db")
+		return address, nil
+	}
+	if err = rlp.DecodeBytes(bytes, &address); err != nil {
+		utils.Logger().Warn().Str("id", id).Msg("cannot convert data from DB")
+		return nil, err
+	}
+
+	address.Balance = balanceAddr
+
+	switch txView {
+	case txViewNone:
+		address.TXs = nil
+	case Received:
+		receivedTXs := make([]*Transaction, 0)
+		for _, tx := range address.TXs {
+			if tx.Type == Received {
+				receivedTXs = append(receivedTXs, tx)
+			}
+		}
+		address.TXs = receivedTXs
+	case Sent:
+		sentTXs := make([]*Transaction, 0)
+		for _, tx := range address.TXs {
+			if tx.Type == Sent {
+				sentTXs = append(sentTXs, tx)
+			}
+		}
+		address.TXs = sentTXs
+	}
+	if offset*page+offset > len(address.TXs) {
+		address.TXs = address.TXs[offset*page:]
+	} else {
+		address.TXs = address.TXs[offset*page : offset*page+offset]
 	}
-	data.Address.TXs = paginatedTXs
+	return address, nil
 }
 
 // GetExplorerNodeCount serves /nodes end-point.
@@ -547,7 +814,12 @@ func (s *Service) GetExplorerNodeCount(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
-// GetExplorerShard serves /shard end-point
+// GetExplorerNodeCount rpc end-point.
+func (s *ServiceAPI) GetExplorerNodeCount(ctx context.Context) int {
+	return len(s.Service.GetNodeIDs())
+}
+
+// GetExplorerShard serves /shard end-point.
 func (s *Service) GetExplorerShard(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Content-Type", "application/json")
 	var nodes []Node
@@ -562,7 +834,18 @@ func (s *Service) GetExplorerShard(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
-// NotifyService notify service
+// GetExplorerShard rpc end-point.
+func (s *ServiceAPI) GetExplorerShard(ctx context.Context) *Shard {
+	var nodes []Node
+	for _, nodeID := range s.Service.GetNodeIDs() {
+		nodes = append(nodes, Node{
+			ID: libp2p_peer.IDB58Encode(nodeID),
+		})
+	}
+	return &Shard{Nodes: nodes}
+}
+
+// NotifyService notify service.
 func (s *Service) NotifyService(params map[string]interface{}) {
 	return
 }
@@ -574,5 +857,12 @@ func (s *Service) SetMessageChan(messageChan chan *msg_pb.Message) {
 
 // APIs for the services.
 func (s *Service) APIs() []rpc.API {
-	return nil
+	return []rpc.API{
+		{
+			Namespace: "explorer",
+			Version:   "1.0",
+			Service:   NewServiceAPI(s),
+			Public:    true,
+		},
+	}
 }

api/service/explorer/storage.go

@@ -155,6 +155,7 @@ func (storage *Storage) UpdateTXStorage(batch ethdb.Batch, explorerTransaction *
 }
 
 // UpdateAddress ...
+// TODO: deprecate this logic
 func (storage *Storage) UpdateAddress(batch ethdb.Batch, explorerTransaction *Transaction, tx *types.Transaction) {
 	explorerTransaction.Type = Received
 	storage.UpdateAddressStorage(batch, explorerTransaction.To, explorerTransaction, tx)
@@ -163,6 +164,7 @@ func (storage *Storage) UpdateAddress(batch ethdb.Batch, explorerTransaction *Tr
 }
 
 // UpdateAddressStorage updates specific addr Address.
+// TODO: deprecate this logic
 func (storage *Storage) UpdateAddressStorage(batch ethdb.Batch, addr string, explorerTransaction *Transaction, tx *types.Transaction) {
 	key := GetAddressKey(addr)
 
@@ -170,7 +172,11 @@ func (storage *Storage) UpdateAddressStorage(batch ethdb.Batch, addr string, exp
 	if data, err := storage.db.Get([]byte(key)); err == nil {
 		err = rlp.DecodeBytes(data, &address)
 		if err == nil {
-			address.Balance.Add(address.Balance, tx.Value())
+			if explorerTransaction.Type == Received {
+				address.Balance.Add(address.Balance, tx.Value())
+			} else {
+				address.Balance.Sub(address.Balance, tx.Value())
+			}
 		} else {
 			utils.Logger().Error().Err(err).Msg("Failed to error")
 		}

api/service/explorer/structs.go

@@ -57,6 +57,9 @@ type Transaction struct {
 	Value     *big.Int `json:"value"`
 	Bytes     string   `json:"bytes"`
 	Data      string   `json:"data"`
+	GasFee    *big.Int `json:"gasFee"`
+	FromShard uint32   `json:"fromShard"`
+	ToShard   uint32   `json:"toShard"`
 	Type      string   `json:"type"`
 }
 
@@ -118,6 +121,8 @@ func GetTransaction(tx *types.Transaction, addressBlock *types.Block) *Transacti
 	if err != nil {
 		utils.Logger().Error().Err(err).Msg("Error when parsing tx into message")
 	}
+	gasFee := big.NewInt(0)
+	gasFee = gasFee.Mul(tx.GasPrice(), new(big.Int).SetUint64(tx.Gas()))
 	return &Transaction{
 		ID:        tx.Hash().Hex(),
 		Timestamp: strconv.Itoa(int(addressBlock.Time().Int64() * 1000)),
@@ -126,6 +131,9 @@ func GetTransaction(tx *types.Transaction, addressBlock *types.Block) *Transacti
 		Value:     msg.Value(),
 		Bytes:     strconv.Itoa(int(tx.Size())),
 		Data:      hex.EncodeToString(tx.Data()),
+		GasFee:    gasFee,
+		FromShard: tx.ShardID(),
+		ToShard:   tx.ToShardID(),
 		Type:      "",
 	}
 }

api/service/networkinfo/service.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	manet "github.com/multiformats/go-multiaddr-net"
+	"github.com/pkg/errors"
 
 	"github.com/ethereum/go-ethereum/rpc"
 	msg_pb "github.com/harmony-one/harmony/api/proto/message"
@@ -17,6 +18,7 @@ import (
 	coredis "github.com/libp2p/go-libp2p-core/discovery"
 	libp2pdis "github.com/libp2p/go-libp2p-discovery"
 	libp2pdht "github.com/libp2p/go-libp2p-kad-dht"
+	libp2pdhtopts "github.com/libp2p/go-libp2p-kad-dht/opts"
 	peerstore "github.com/libp2p/go-libp2p-peerstore"
 )
 
@@ -56,16 +58,31 @@ const (
 	discoveryLimit = 32
 )
 
-// New returns role conversion service.
+// New returns role conversion service.  If dataStorePath is not empty, it
-func New(h p2p.Host, rendezvous p2p.GroupID, peerChan chan p2p.Peer, bootnodes utils.AddrList) *Service {
+// points to a persistent database directory to use.
+func New(
+	h p2p.Host, rendezvous p2p.GroupID, peerChan chan p2p.Peer,
+	bootnodes utils.AddrList, dataStorePath string,
+) (*Service, error) {
 	var cancel context.CancelFunc
 	ctx, cancel = context.WithTimeout(context.Background(), connectionTimeout)
-	dataStore, err := badger.NewDatastore(fmt.Sprintf(".dht-%s-%s", h.GetSelfPeer().IP, h.GetSelfPeer().Port), nil)
+	var dhtOpts []libp2pdhtopts.Option
-	if err != nil {
+	if dataStorePath != "" {
-		panic(err)
+		dataStore, err := badger.NewDatastore(dataStorePath, nil)
+		if err != nil {
+			return nil, errors.Wrapf(err,
+				"cannot open Badger datastore at %s", dataStorePath)
+		}
+		utils.Logger().Info().
+			Str("dataStorePath", dataStorePath).
+			Msg("backing DHT with Badger datastore")
+		dhtOpts = append(dhtOpts, libp2pdhtopts.Datastore(dataStore))
 	}
 
-	dht := libp2pdht.NewDHT(ctx, h.GetP2PHost(), dataStore)
+	dht, err := libp2pdht.New(ctx, h.GetP2PHost(), dhtOpts...)
+	if err != nil {
+		return nil, errors.Wrapf(err, "cannot create DHT")
+	}
 
 	return &Service{
 		Host:        h,
@@ -78,7 +95,19 @@ func New(h p2p.Host, rendezvous p2p.GroupID, peerChan chan p2p.Peer, bootnodes u
 		bootnodes:   bootnodes,
 		discovery:   nil,
 		started:     false,
+	}, nil
+}
+
+// MustNew is a panic-on-error version of New.
+func MustNew(
+	h p2p.Host, rendezvous p2p.GroupID, peerChan chan p2p.Peer,
+	bootnodes utils.AddrList, dataStorePath string,
+) *Service {
+	service, err := New(h, rendezvous, peerChan, bootnodes, dataStorePath)
+	if err != nil {
+		panic(err)
 	}
+	return service
 }
 
 // StartService starts network info service.

api/service/networkinfo/service_test.go

@@ -28,7 +28,10 @@ func TestService(t *testing.T) {
 		t.Fatal("unable to new host in harmony")
 	}
 
-	s := New(host, p2p.GroupIDBeaconClient, nil, nil)
+	s, err := New(host, p2p.GroupIDBeaconClient, nil, nil, "")
+	if err != nil {
+		t.Fatalf("New() failed: %s", err)
+	}
 
 	s.StartService()
 

api/service/syncing/downloader/client.go

@@ -44,7 +44,7 @@ func (client *Client) Close() {
 func (client *Client) GetBlockHashes(startHash []byte, size uint32, ip, port string) *pb.DownloaderResponse {
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
-	request := &pb.DownloaderRequest{Type: pb.DownloaderRequest_HEADER, BlockHash: startHash, Size: size}
+	request := &pb.DownloaderRequest{Type: pb.DownloaderRequest_BLOCKHASH, BlockHash: startHash, Size: size}
 	request.Ip = ip
 	request.Port = port
 	response, err := client.dlClient.Query(ctx, request)
@@ -54,6 +54,23 @@ func (client *Client) GetBlockHashes(startHash []byte, size uint32, ip, port str
 	return response
 }
 
+// GetBlockHeaders gets block headers in serialization byte array by calling a grpc request.
+func (client *Client) GetBlockHeaders(hashes [][]byte) *pb.DownloaderResponse {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	request := &pb.DownloaderRequest{Type: pb.DownloaderRequest_BLOCKHEADER}
+	request.Hashes = make([][]byte, len(hashes))
+	for i := range hashes {
+		request.Hashes[i] = make([]byte, len(hashes[i]))
+		copy(request.Hashes[i], hashes[i])
+	}
+	response, err := client.dlClient.Query(ctx, request)
+	if err != nil {
+		utils.Logger().Error().Err(err).Str("target", client.conn.Target()).Msg("[SYNC] downloader/client.go:GetBlockHeaders query failed")
+	}
+	return response
+}
+
 // GetBlocks gets blocks in serialization byte array by calling a grpc request.
 func (client *Client) GetBlocks(hashes [][]byte) *pb.DownloaderResponse {
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)

api/service/syncing/downloader/gen.sh

@@ -1 +1,2 @@
-protoc -I proto/ proto/downloader.proto --go_out=plugins=grpc:proto
+SRC_DIR=$(dirname $0)
+protoc -I ${SRC_DIR}/proto/ ${SRC_DIR}/proto/downloader.proto --go_out=plugins=grpc:${SRC_DIR}/proto

api/service/syncing/downloader/proto/downloader.pb.go

@@ -25,33 +25,36 @@ const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
 type DownloaderRequest_RequestType int32
 
 const (
-	DownloaderRequest_HEADER          DownloaderRequest_RequestType = 0
+	DownloaderRequest_BLOCKHASH       DownloaderRequest_RequestType = 0
 	DownloaderRequest_BLOCK           DownloaderRequest_RequestType = 1
 	DownloaderRequest_NEWBLOCK        DownloaderRequest_RequestType = 2
 	DownloaderRequest_BLOCKHEIGHT     DownloaderRequest_RequestType = 3
 	DownloaderRequest_REGISTER        DownloaderRequest_RequestType = 4
 	DownloaderRequest_REGISTERTIMEOUT DownloaderRequest_RequestType = 5
 	DownloaderRequest_UNKNOWN         DownloaderRequest_RequestType = 6
+	DownloaderRequest_BLOCKHEADER     DownloaderRequest_RequestType = 7
 )
 
 var DownloaderRequest_RequestType_name = map[int32]string{
-	0: "HEADER",
+	0: "BLOCKHASH",
 	1: "BLOCK",
 	2: "NEWBLOCK",
 	3: "BLOCKHEIGHT",
 	4: "REGISTER",
 	5: "REGISTERTIMEOUT",
 	6: "UNKNOWN",
+	7: "BLOCKHEADER",
 }
 
 var DownloaderRequest_RequestType_value = map[string]int32{
-	"HEADER":          0,
+	"BLOCKHASH":       0,
 	"BLOCK":           1,
 	"NEWBLOCK":        2,
 	"BLOCKHEIGHT":     3,
 	"REGISTER":        4,
 	"REGISTERTIMEOUT": 5,
 	"UNKNOWN":         6,
+	"BLOCKHEADER":     7,
 }
 
 func (x DownloaderRequest_RequestType) String() string {
@@ -135,7 +138,7 @@ func (m *DownloaderRequest) GetType() DownloaderRequest_RequestType {
 	if m != nil {
 		return m.Type
 	}
-	return DownloaderRequest_HEADER
+	return DownloaderRequest_BLOCKHASH
 }
 
 func (m *DownloaderRequest) GetHashes() [][]byte {
@@ -248,32 +251,33 @@ func init() {
 func init() { proto.RegisterFile("downloader.proto", fileDescriptor_6a99ec95c7ab1ff1) }
 
 var fileDescriptor_6a99ec95c7ab1ff1 = []byte{
-	// 399 bytes of a gzipped FileDescriptorProto
+	// 410 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x92, 0xc1, 0x6e, 0xd3, 0x40,
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x52, 0xcd, 0x6e, 0xd3, 0x40,
-	0x10, 0x86, 0xb3, 0x8e, 0xe3, 0x24, 0x93, 0xd0, 0x2e, 0x03, 0x42, 0xab, 0x0a, 0x90, 0xe5, 0x93,
+	0x10, 0xce, 0x3a, 0x8e, 0x93, 0x4c, 0xd2, 0x76, 0x19, 0x10, 0x5a, 0x55, 0x80, 0x2c, 0x9f, 0xcc,
-	0xb9, 0xf8, 0xd0, 0x9e, 0x38, 0x70, 0x28, 0xee, 0x12, 0x5b, 0x2d, 0x8e, 0x58, 0x3b, 0x54, 0x1c,
+	0xc5, 0x87, 0xf6, 0xc4, 0x81, 0x43, 0x70, 0x4d, 0x6c, 0xb5, 0x38, 0x62, 0xed, 0x50, 0x71, 0x74,
-	0x5d, 0xba, 0xaa, 0x2d, 0xaa, 0x7a, 0xf1, 0xba, 0x42, 0xe6, 0x2d, 0x91, 0x78, 0x20, 0xe4, 0x4d,
+	0xe9, 0xaa, 0xb6, 0xa8, 0xea, 0xc5, 0xeb, 0x0a, 0x99, 0x17, 0xe0, 0x05, 0x79, 0x06, 0x9e, 0x03,
-	0x5a, 0x5b, 0x02, 0x72, 0xda, 0xf9, 0xff, 0xd9, 0x19, 0xcd, 0x7c, 0x1a, 0xa0, 0xd7, 0xd5, 0x8f,
+	0x79, 0x9d, 0xd6, 0x96, 0x80, 0x9c, 0x76, 0xbf, 0x6f, 0x7e, 0x34, 0xf3, 0x7d, 0x03, 0xf4, 0xba,
-	0xbb, 0xdb, 0x2a, 0xbf, 0x96, 0x75, 0xa0, 0xea, 0xaa, 0xa9, 0x10, 0x7a, 0xc7, 0xfb, 0x65, 0xc1,
+	0xfc, 0x7e, 0x77, 0x5b, 0x66, 0xd7, 0xa2, 0xf2, 0x64, 0x55, 0xd6, 0x25, 0x42, 0xcf, 0x38, 0xbf,
-	0xd3, 0xb3, 0x47, 0x29, 0xe4, 0xf7, 0x7b, 0xa9, 0x1b, 0x7c, 0x07, 0x76, 0xd3, 0x2a, 0xc9, 0x88,
+	0x0d, 0x78, 0x72, 0xf6, 0x08, 0xb9, 0xf8, 0x76, 0x2f, 0x54, 0x8d, 0x6f, 0xc1, 0xac, 0x1b, 0x29,
-	0x4b, 0xfc, 0x83, 0xe3, 0x37, 0xc1, 0xa0, 0xc5, 0x5f, 0x9f, 0x83, 0xdd, 0x9b, 0xb5, 0x4a, 0x0a,
+	0x18, 0xb1, 0x89, 0x7b, 0x78, 0xf2, 0xda, 0x1b, 0xb4, 0xf8, 0x2b, 0xd9, 0xdb, 0xbd, 0x69, 0x23,
-	0x53, 0x86, 0x2f, 0xc0, 0x29, 0x72, 0x5d, 0x48, 0xcd, 0x2c, 0x77, 0xec, 0x2f, 0xc5, 0x4e, 0xe1,
+	0x05, 0xd7, 0x65, 0xf8, 0x1c, 0xac, 0x3c, 0x53, 0xb9, 0x50, 0xcc, 0xb0, 0xc7, 0xee, 0x92, 0xef,
-	0x11, 0xcc, 0x94, 0x94, 0x75, 0x94, 0xeb, 0x82, 0x8d, 0x5d, 0xe2, 0x2f, 0xc5, 0xa3, 0xc6, 0x97,
+	0x10, 0x1e, 0xc3, 0x4c, 0x0a, 0x51, 0x85, 0x99, 0xca, 0xd9, 0xd8, 0x26, 0xee, 0x92, 0x3f, 0x62,
-	0x30, 0xbf, 0xba, 0xad, 0xbe, 0x7e, 0x33, 0x49, 0xdb, 0x24, 0x7b, 0x03, 0x0f, 0xc0, 0x2a, 0x15,
+	0x7c, 0x01, 0xf3, 0xab, 0xdb, 0xf2, 0xcb, 0x57, 0x1d, 0x34, 0x75, 0xb0, 0x27, 0xf0, 0x10, 0x8c,
-	0x9b, 0xb8, 0xc4, 0x9f, 0x0b, 0xab, 0x54, 0x88, 0x60, 0xab, 0xaa, 0x6e, 0x98, 0x63, 0x1c, 0x13,
+	0x42, 0xb2, 0x89, 0x4d, 0xdc, 0x39, 0x37, 0x0a, 0x89, 0x08, 0xa6, 0x2c, 0xab, 0x9a, 0x59, 0x9a,
-	0x77, 0x9e, 0x2e, 0x7f, 0x4a, 0x36, 0x75, 0x89, 0xff, 0x44, 0x98, 0xd8, 0xd3, 0xb0, 0x18, 0x8c,
+	0xd1, 0xff, 0x96, 0x53, 0xc5, 0x0f, 0xc1, 0xa6, 0x36, 0x71, 0x0f, 0xb8, 0xfe, 0x3b, 0x3f, 0x09,
-	0x87, 0x00, 0x4e, 0xc4, 0x4f, 0xcf, 0xb8, 0xa0, 0x23, 0x9c, 0xc3, 0xe4, 0xfd, 0xc5, 0x3a, 0x3c,
+	0x2c, 0x06, 0xf3, 0xe1, 0x01, 0xcc, 0xdf, 0x5d, 0x6c, 0xfc, 0xf3, 0x70, 0x95, 0x84, 0x74, 0x84,
-	0xa7, 0x04, 0x97, 0x30, 0x4b, 0xf8, 0xe5, 0x56, 0x59, 0x78, 0x08, 0x0b, 0x13, 0x46, 0x3c, 0x5e,
+	0x73, 0x98, 0x68, 0x48, 0x09, 0x2e, 0x61, 0x16, 0x07, 0x97, 0x1d, 0x32, 0xf0, 0x08, 0x16, 0x5d,
-	0x45, 0x19, 0x1d, 0x77, 0x69, 0xc1, 0x57, 0x71, 0x9a, 0x71, 0x41, 0x6d, 0x7c, 0x06, 0x87, 0x0f,
+	0x5e, 0x10, 0xad, 0xc3, 0x94, 0x8e, 0xdb, 0x30, 0x0f, 0xd6, 0x51, 0x92, 0x06, 0x9c, 0x9a, 0xf8,
-	0x2a, 0x8b, 0x3f, 0xf2, 0xf5, 0x26, 0xa3, 0x13, 0x5c, 0xc0, 0x74, 0x93, 0x9c, 0x27, 0xeb, 0xcb,
+	0x14, 0x8e, 0x1e, 0x50, 0x1a, 0x7d, 0x08, 0x36, 0xdb, 0x94, 0x4e, 0x70, 0x01, 0xd3, 0x6d, 0x7c,
-	0x84, 0x3a, 0xde, 0x6f, 0x02, 0x38, 0xc4, 0xa4, 0x55, 0x75, 0xa7, 0x25, 0x32, 0x98, 0xaa, 0xbc,
+	0x1e, 0x6f, 0x2e, 0x63, 0x6a, 0x0d, 0x1a, 0xac, 0xce, 0x02, 0x4e, 0xa7, 0xce, 0x2f, 0x02, 0x38,
-	0xed, 0x4c, 0x46, 0x0c, 0x96, 0x07, 0x89, 0xab, 0x1d, 0x6e, 0xcb, 0xe0, 0x3e, 0xf9, 0x1f, 0xee,
+	0xd4, 0x4e, 0xc9, 0xf2, 0x4e, 0x09, 0x64, 0x30, 0x95, 0x59, 0xd3, 0x92, 0x8c, 0x68, 0xad, 0x1e,
-	0x6d, 0x9f, 0x40, 0xc8, 0x9b, 0x52, 0x37, 0xbd, 0x31, 0x00, 0xef, 0xc2, 0x62, 0xcb, 0x4c, 0x96,
+	0x20, 0xae, 0x77, 0x1e, 0x18, 0xda, 0x83, 0xd3, 0xff, 0x79, 0xd0, 0xf5, 0xf1, 0xb8, 0xb8, 0x29,
-	0x37, 0x45, 0x63, 0x18, 0xdb, 0x62, 0x68, 0x79, 0x6f, 0xe1, 0xf9, 0xbf, 0xea, 0xbb, 0x05, 0xd2,
+	0x54, 0xdd, 0x13, 0x03, 0x37, 0x6c, 0x58, 0x74, 0x42, 0x8a, 0xe2, 0x26, 0xaf, 0xb5, 0xf0, 0x26,
-	0x4d, 0x18, 0xf2, 0x34, 0xa5, 0x23, 0x9c, 0x81, 0xfd, 0xe1, 0x34, 0xbe, 0xa0, 0xa4, 0x03, 0x16,
+	0x1f, 0x52, 0xce, 0x1b, 0x78, 0xf6, 0xaf, 0xfa, 0x76, 0xa3, 0x64, 0xeb, 0xfb, 0x41, 0x92, 0xd0,
-	0x27, 0xe9, 0x97, 0x24, 0xa4, 0xd6, 0xf1, 0x67, 0x80, 0x7e, 0x1a, 0x8c, 0x60, 0xf2, 0xe9, 0x5e,
+	0x11, 0xce, 0xc0, 0x7c, 0xbf, 0x8a, 0x2e, 0x28, 0x41, 0x00, 0x2b, 0x8a, 0x93, 0xcf, 0xb1, 0x4f,
-	0xd6, 0x2d, 0xbe, 0xda, 0x7b, 0x1d, 0x47, 0xaf, 0xf7, 0x6f, 0xe3, 0x8d, 0xae, 0x1c, 0x73, 0x95,
+	0x8d, 0x93, 0x4f, 0x00, 0xfd, 0x34, 0x18, 0xc2, 0xe4, 0xe3, 0xbd, 0xa8, 0x1a, 0x7c, 0xb9, 0xf7,
-	0x27, 0x7f, 0x02, 0x00, 0x00, 0xff, 0xff, 0x68, 0xc1, 0xb5, 0x25, 0xa9, 0x02, 0x00, 0x00,
+	0x64, 0x8e, 0x5f, 0xed, 0xdf, 0xc6, 0x19, 0x5d, 0x59, 0xfa, 0x54, 0x4f, 0xff, 0x04, 0x00, 0x00,
+	0xff, 0xff, 0x00, 0xaf, 0x33, 0x33, 0xbe, 0x02, 0x00, 0x00,
 }
 
 // Reference imports to suppress errors if they are not otherwise used.

api/service/syncing/downloader/proto/downloader.proto

@@ -10,15 +10,16 @@ service Downloader {
 // DownloaderRequest is the generic download request.
 message DownloaderRequest {
   enum RequestType {
-    HEADER = 0;
+    BLOCKHASH = 0;
     BLOCK = 1;
     NEWBLOCK = 2;
     BLOCKHEIGHT = 3;
     REGISTER = 4;
     REGISTERTIMEOUT = 5;
     UNKNOWN = 6;
+    BLOCKHEADER = 7;
   }
- 
+
   // Request type.
   RequestType type = 1;
 

api/service/syncing/syncing.go

@@ -346,8 +346,8 @@ func (ss *StateSync) GetConsensusHashes(startHash []byte, size uint32) bool {
 				response := peerConfig.client.GetBlockHashes(startHash, size, ss.selfip, ss.selfport)
 				if response == nil {
 					utils.Logger().Warn().
-						Str("peer IP", peerConfig.ip).
+						Str("peerIP", peerConfig.ip).
-						Str("peer Port", peerConfig.port).
+						Str("peerPort", peerConfig.port).
 						Msg("[SYNC] GetConsensusHashes Nil Response")
 					return
 				}
@@ -546,9 +546,15 @@ func (ss *StateSync) updateBlockAndStatus(block *types.Block, bc *core.BlockChai
 	// Verify block signatures
 	// TODO chao: only when block is verified against last commit sigs, we can update the block and status
 	if block.NumberU64() > 1 {
-		err := bc.Engine().VerifyHeader(bc, block.Header(), true)
+		// Verify signature every 100 blocks
+		verifySig := block.NumberU64()%100 == 0
+		err := bc.Engine().VerifyHeader(bc, block.Header(), verifySig)
 		if err != nil {
 			utils.Logger().Error().Err(err).Msgf("[SYNC] failed verifying signatures for new block %d", block.NumberU64())
+			utils.Logger().Debug().Interface("block", bc.CurrentBlock()).Msg("[SYNC] Rolling back last 99 blocks!")
+			for i := 0; i < 99; i++ {
+				bc.Rollback([]common.Hash{bc.CurrentBlock().Hash()})
+			}
 			return false
 		}
 	}
@@ -561,11 +567,6 @@ func (ss *StateSync) updateBlockAndStatus(block *types.Block, bc *core.BlockChai
 		bc.Rollback([]common.Hash{bc.CurrentBlock().Hash()})
 		return false
 	}
-	ss.syncMux.Lock()
-	if err := worker.UpdateCurrent(block.Header().Coinbase()); err != nil {
-		utils.Logger().Warn().Err(err).Msg("[SYNC] (*Worker).UpdateCurrent failed")
-	}
-	ss.syncMux.Unlock()
 	utils.Logger().Info().
 		Uint64("blockHeight", bc.CurrentBlock().NumberU64()).
 		Str("blockHex", bc.CurrentBlock().Hash().Hex()).
@@ -702,10 +703,10 @@ func (ss *StateSync) getMaxPeerHeight(isBeacon bool) uint64 {
 		go func() {
 			defer wg.Done()
 			//debug
-			// utils.Logger().Debug().Bool("isBeacon", isBeacon).Str("IP", peerConfig.ip).Str("Port", peerConfig.port).Msg("[Sync]getMaxPeerHeight")
+			// utils.Logger().Debug().Bool("isBeacon", isBeacon).Str("peerIP", peerConfig.ip).Str("peerPort", peerConfig.port).Msg("[Sync]getMaxPeerHeight")
 			response, err := peerConfig.client.GetBlockChainHeight()
 			if err != nil {
-				utils.Logger().Warn().Err(err).Str("IP", peerConfig.ip).Str("Port", peerConfig.port).Msg("[Sync]GetBlockChainHeight failed")
+				utils.Logger().Warn().Err(err).Str("peerIP", peerConfig.ip).Str("peerPort", peerConfig.port).Msg("[Sync]GetBlockChainHeight failed")
 				return
 			}
 			ss.syncMux.Lock()
@@ -740,7 +741,7 @@ func (ss *StateSync) IsOutOfSync(bc *core.BlockChain) bool {
 }
 
 // SyncLoop will keep syncing with peers until catches up
-func (ss *StateSync) SyncLoop(bc *core.BlockChain, worker *worker.Worker, willJoinConsensus bool, isBeacon bool) {
+func (ss *StateSync) SyncLoop(bc *core.BlockChain, worker *worker.Worker, isBeacon bool) {
 	if !isBeacon {
 		ss.RegisterNodeInfo()
 	}

block/factory/factory.go

@@ -45,6 +45,7 @@ var (
 	ForTest    = NewFactory(params.TestChainConfig)
 	ForTestnet = NewFactory(params.TestnetChainConfig)
 	ForMainnet = NewFactory(params.MainnetChainConfig)
+	ForPangaea = NewFactory(params.PangaeaChainConfig)
 )
 
 // NewTestHeader creates a new, empty header object for epoch 0 using the test

cmd/client/wallet/generated_wallet.ini.go

@@ -2,6 +2,7 @@ package main
 
 const (
 	defaultWalletIni = `[main]
+chain_id = 1
 bootnode = /ip4/100.26.90.187/tcp/9874/p2p/Qmdfjtk6hPoyrH1zVD9PEH4zfWLo38dP2mDvvKXfh3tnEv
 bootnode = /ip4/54.213.43.194/tcp/9874/p2p/QmZJJx6AdaoEkGLrYG4JeLCKeCKDjnFz2wfHNHxAqFSGA9
 bootnode = /ip4/13.113.101.219/tcp/12019/p2p/QmQayinFSgMMw5cSpDUiD9pQ2WeP6WNmGxpZ6ou3mdVFJX
@@ -25,6 +26,7 @@ rpc = l3.t.hmny.io:14555
 rpc = s3.t.hmny.io:14555
 
 [local]
+chain_id = 2
 bootnode = /ip4/127.0.0.1/tcp/19876/p2p/Qmc1V6W7BwX8Ugb42Ti8RnXF1rY5PF7nnZ6bKBryCgi6cv
 shards = 2
 
@@ -39,6 +41,7 @@ rpc = 127.0.0.1:14558
 rpc = 127.0.0.1:14560
 
 [beta]
+chain_id = 2
 bootnode = /ip4/54.213.43.194/tcp/9868/p2p/QmZJJx6AdaoEkGLrYG4JeLCKeCKDjnFz2wfHNHxAqFSGA9
 bootnode = /ip4/100.26.90.187/tcp/9868/p2p/Qmdfjtk6hPoyrH1zVD9PEH4zfWLo38dP2mDvvKXfh3tnEv
 bootnode = /ip4/13.113.101.219/tcp/12018/p2p/QmQayinFSgMMw5cSpDUiD9pQ2WeP6WNmGxpZ6ou3mdVFJX
@@ -53,6 +56,7 @@ rpc = l1.b.hmny.io:14555
 rpc = s1.b.hmny.io:14555
 
 [pangaea]
+chain_id = 3
 bootnode = /ip4/54.86.126.90/tcp/9889/p2p/Qmdfjtk6hPoyrH1zVD9PEH4zfWLo38dP2mDvvKXfh3tnEv
 bootnode = /ip4/52.40.84.2/tcp/9889/p2p/QmZJJx6AdaoEkGLrYG4JeLCKeCKDjnFz2wfHNHxAqFSGA9
 shards = 4

cmd/client/wallet/main.go

@@ -15,8 +15,9 @@ import (
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/log"
-	color "github.com/fatih/color"
+	"github.com/fatih/color"
 	ffi_bls "github.com/harmony-one/bls/ffi/go/bls"
+
 	"github.com/harmony-one/harmony/accounts"
 	"github.com/harmony-one/harmony/accounts/keystore"
 	"github.com/harmony-one/harmony/api/client"
@@ -29,7 +30,6 @@ import (
 	common2 "github.com/harmony-one/harmony/internal/common"
 	nodeconfig "github.com/harmony-one/harmony/internal/configs/node"
 	"github.com/harmony-one/harmony/internal/ctxerror"
-	"github.com/harmony-one/harmony/internal/params"
 	"github.com/harmony-one/harmony/internal/shardchain"
 	"github.com/harmony-one/harmony/internal/utils"
 	"github.com/harmony-one/harmony/node"
@@ -767,11 +767,12 @@ func processTransferCommand() {
 
 	fmt.Printf("Unlock account succeeded! '%v'\n", senderPass)
 
-	chainConfig := params.MainnetChainConfig
+	chainID, ok := new(big.Int).SetString(walletProfile.ChainID, 0)
-	if walletProfile.Profile != defaultProfile {
+	if !ok {
-		chainConfig = params.TestnetChainConfig
+		fmt.Printf("invalid chain ID %#v in config", walletProfile.ChainID)
+		return
 	}
-	tx, err = ks.SignTx(account, tx, chainConfig.ChainID)
+	tx, err = ks.SignTx(account, tx, chainID)
 	if err != nil {
 		fmt.Printf("SignTx Error: %v\n", err)
 		return

cmd/client/wallet_stress_test/generated_wallet.ini.go

@@ -2,6 +2,7 @@ package main
 
 const (
 	defaultWalletIni = `[main]
+chain_id = 1
 bootnode = /ip4/100.26.90.187/tcp/9874/p2p/Qmdfjtk6hPoyrH1zVD9PEH4zfWLo38dP2mDvvKXfh3tnEv
 bootnode = /ip4/54.213.43.194/tcp/9874/p2p/QmZJJx6AdaoEkGLrYG4JeLCKeCKDjnFz2wfHNHxAqFSGA9
 bootnode = /ip4/13.113.101.219/tcp/12019/p2p/QmQayinFSgMMw5cSpDUiD9pQ2WeP6WNmGxpZ6ou3mdVFJX
@@ -25,6 +26,7 @@ rpc = l3.t.hmny.io:14555
 rpc = s3.t.hmny.io:14555
 
 [local]
+chain_id = 2
 bootnode = /ip4/127.0.0.1/tcp/19876/p2p/Qmc1V6W7BwX8Ugb42Ti8RnXF1rY5PF7nnZ6bKBryCgi6cv
 shards = 2
 
@@ -39,6 +41,7 @@ rpc = 127.0.0.1:14558
 rpc = 127.0.0.1:14560
 
 [beta]
+chain_id = 2
 bootnode = /ip4/54.213.43.194/tcp/9868/p2p/QmZJJx6AdaoEkGLrYG4JeLCKeCKDjnFz2wfHNHxAqFSGA9
 bootnode = /ip4/100.26.90.187/tcp/9868/p2p/Qmdfjtk6hPoyrH1zVD9PEH4zfWLo38dP2mDvvKXfh3tnEv
 bootnode = /ip4/13.113.101.219/tcp/12018/p2p/QmQayinFSgMMw5cSpDUiD9pQ2WeP6WNmGxpZ6ou3mdVFJX
@@ -53,6 +56,7 @@ rpc = l1.b.hmny.io:14555
 rpc = s1.b.hmny.io:14555
 
 [pangaea]
+chain_id = 3
 bootnode = /ip4/54.86.126.90/tcp/9889/p2p/Qmdfjtk6hPoyrH1zVD9PEH4zfWLo38dP2mDvvKXfh3tnEv
 bootnode = /ip4/52.40.84.2/tcp/9889/p2p/QmZJJx6AdaoEkGLrYG4JeLCKeCKDjnFz2wfHNHxAqFSGA9
 shards = 4

cmd/client/wallet_stress_test/main.go

@@ -11,10 +11,9 @@ import (
 	"sync"
 	"time"
 
-	"github.com/harmony-one/harmony/internal/params"
-
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/log"
+
 	"github.com/harmony-one/harmony/accounts"
 	"github.com/harmony-one/harmony/accounts/keystore"
 	"github.com/harmony-one/harmony/api/client"
@@ -216,6 +215,12 @@ func processStressTestCommand() {
 
 	*/
 
+	chainID, ok := new(big.Int).SetString(walletProfile.ChainID, 0)
+	if !ok {
+		fmt.Printf("invalid chain ID %#v in config", walletProfile.ChainID)
+		return
+	}
+
 	fmt.Println("Creating wallet node")
 	walletNode := createWalletNode()
 
@@ -288,11 +293,7 @@ func processStressTestCommand() {
 
 		ks.Unlock(account, senderPass)
 
-		chainConfig := params.MainnetChainConfig
+		tx, _ = ks.SignTx(account, tx, chainID)
-		if walletProfile.Profile != defaultProfile {
-			chainConfig = params.TestnetChainConfig
-		}
-		tx, _ = ks.SignTx(account, tx, chainConfig.ChainID)
 
 		if err := submitTransaction(tx, walletNode, uint32(shardID)); err != nil {
 			fmt.Println(ctxerror.New("submitTransaction failed",

cmd/harmony/main.go

@@ -126,6 +126,8 @@ var (
 	metricsFlag     = flag.Bool("metrics", false, "Collect and upload node metrics")
 	pushgatewayIP   = flag.String("pushgateway_ip", "grafana.harmony.one", "Metrics view ip")
 	pushgatewayPort = flag.String("pushgateway_port", "9091", "Metrics view port")
+
+	publicRPC = flag.Bool("public_rpc", false, "Enable Public RPC Access (default: false)")
 )
 
 func initSetup() {
@@ -367,8 +369,8 @@ func setupConsensusAndNode(nodeConfig *nodeconfig.ConfigType) *node.Node {
 	// currentNode.DRand = dRand
 
 	// This needs to be executed after consensus and drand are setup
-	if err := currentNode.GetInitShardState(); err != nil {
+	if err := currentNode.CalculateInitShardState(); err != nil {
-		ctxerror.Crit(utils.GetLogger(), err, "GetInitShardState failed",
+		ctxerror.Crit(utils.GetLogger(), err, "CalculateInitShardState failed",
 			"shardID", *shardID)
 	}
 
@@ -408,6 +410,7 @@ func main() {
 		os.Exit(1)
 	}
 
+	nodeconfig.SetPublicRPC(*publicRPC)
 	nodeconfig.SetVersion(fmt.Sprintf("Harmony (C) 2019. %v, version %v-%v (%v %v)", path.Base(os.Args[0]), version, commit, builtBy, builtAt))
 	if *versionFlag {
 		printVersion()
@@ -459,8 +462,8 @@ func main() {
 	nodeConfig := createGlobalConfig()
 	currentNode := setupConsensusAndNode(nodeConfig)
 
-	if nodeConfig.ShardID != 0 {
+	if nodeConfig.ShardID != 0 && currentNode.NodeConfig.Role() != nodeconfig.ExplorerNode {
-		utils.GetLogInstance().Info("SupportBeaconSyncing", "shardID", currentNode.Blockchain().ShardID(), "shardID1", nodeConfig.ShardID)
+		utils.GetLogInstance().Info("SupportBeaconSyncing", "shardID", currentNode.Blockchain().ShardID(), "shardID", nodeConfig.ShardID)
 		go currentNode.SupportBeaconSyncing()
 	}
 
@@ -485,11 +488,11 @@ func main() {
 	go currentNode.SupportSyncing()
 	currentNode.ServiceManagerSetup()
 
+	currentNode.RunServices()
 	// RPC for SDK not supported for mainnet.
 	if err := currentNode.StartRPC(*port); err != nil {
 		ctxerror.Warn(utils.GetLogger(), err, "StartRPC failed")
 	}
-	currentNode.RunServices()
 
 	// Run additional node collectors
 	// Collect node metrics if metrics flag is set

consensus/consensus_service.go

@@ -505,7 +505,7 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode {
 	header := consensus.ChainReader.CurrentHeader()
 
 	epoch := header.Epoch()
-	curPubKeys := core.GetPublicKeys(epoch, header.ShardID())
+	curPubKeys := core.CalculatePublicKeys(epoch, header.ShardID())
 	consensus.numPrevPubKeys = len(curPubKeys)
 
 	consensus.getLogger().Info().Msg("[UpdateConsensusInformation] Updating.....")
@@ -515,7 +515,7 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode {
 		consensus.SetEpochNum(epoch.Uint64() + 1)
 		consensus.getLogger().Info().Uint64("headerNum", header.Number().Uint64()).Msg("[UpdateConsensusInformation] Epoch updated for next epoch")
 		nextEpoch := new(big.Int).Add(epoch, common.Big1)
-		pubKeys = core.GetPublicKeys(nextEpoch, header.ShardID())
+		pubKeys = core.CalculatePublicKeys(nextEpoch, header.ShardID())
 	} else {
 		consensus.SetEpochNum(epoch.Uint64())
 		pubKeys = curPubKeys

consensus/consensus_v2.go

@@ -107,6 +107,7 @@ func (consensus *Consensus) announce(block *types.Block) {
 
 	// save announce message to PbftLog
 	msgPayload, _ := proto.GetConsensusMessagePayload(msgToSend)
+	// TODO(chao): don't unmarshall the message here and direclty pass the original object.
 	msg := &msg_pb.Message{}
 	_ = protobuf.Unmarshal(msgPayload, msg)
 	pbftMsg, err := ParsePbftMessage(msg)
@@ -115,6 +116,7 @@ func (consensus *Consensus) announce(block *types.Block) {
 		return
 	}
 
+	// TODO(chao): review pbft log data structure
 	consensus.PbftLog.AddMessage(pbftMsg)
 	utils.Logger().Debug().
 		Str("MsgBlockHash", pbftMsg.BlockHash.Hex()).
@@ -183,6 +185,7 @@ func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) {
 	}
 
 	// verify validity of block header object
+	// TODO: think about just sending the block hash instead of the header.
 	encodedHeader := recvMsg.Payload
 	header := new(block.Header)
 	err = rlp.DecodeBytes(encodedHeader, header)
@@ -384,6 +387,7 @@ func (consensus *Consensus) onPrepare(msg *msg_pb.Message) {
 		consensus.aggregatedPrepareSig = aggSig
 
 		//leader adds prepared message to log
+		// TODO(chao): don't unmarshall the payload again
 		msgPayload, _ := proto.GetConsensusMessagePayload(msgToSend)
 		msg := &msg_pb.Message{}
 		_ = protobuf.Unmarshal(msgPayload, msg)
@@ -678,6 +682,7 @@ func (consensus *Consensus) onCommit(msg *msg_pb.Message) {
 		return
 	}
 
+	// has to be called before verifying signature
 	quorumWasMet := len(commitSigs) >= consensus.Quorum()
 
 	// Verify the signature on commitPayload is correct

consensus/engine/consensus_engine.go

@@ -56,7 +56,7 @@ type Engine interface {
 	// is used for verifying "incoming" block header against commit signature and bitmap sent from the other chain cross-shard via libp2p.
 	// i.e. this header verification api is more flexible since the caller specifies which commit signature and bitmap to use
 	// for verifying the block header, which is necessary for cross-shard block header verification. Example of such is cross-shard transaction.
-	VerifyHeaderWithSignature(header *block.Header, commitSig []byte, commitBitmap []byte) error
+	VerifyHeaderWithSignature(chain ChainReader, header *block.Header, commitSig []byte, commitBitmap []byte) error
 
 	// VerifyHeaders is similar to VerifyHeader, but verifies a batch of headers
 	// concurrently. The method returns a quit channel to abort the operations and

core/block_validator.go

@@ -225,5 +225,5 @@ func (v *BlockValidator) ValidateCXReceiptsProof(cxp *types.CXReceiptsProof) err
 	}
 
 	// (4) verify blockHeader with seal
-	return v.engine.VerifyHeaderWithSignature(cxp.Header, cxp.CommitSig, cxp.CommitBitmap)
+	return v.engine.VerifyHeaderWithSignature(v.bc, cxp.Header, cxp.CommitSig, cxp.CommitBitmap)
 }

core/blockchain.go

@@ -1109,6 +1109,9 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types.
 		rawdb.WriteTxLookupEntries(batch, block)
 		rawdb.WritePreimages(batch, block.NumberU64(), state.Preimages())
 
+		// write the positional metadata for CXReceipts lookups
+		rawdb.WriteCxLookupEntries(batch, block)
+
 		status = CanonStatTy
 	} else {
 		status = SideStatTy

core/rawdb/accessors_indexes.go

@@ -123,3 +123,76 @@ func WriteBloomBits(db DatabaseWriter, bit uint, section uint64, head common.Has
 		utils.Logger().Error().Err(err).Msg("Failed to store bloom bits")
 	}
 }
+
+// ReadCxLookupEntry retrieves the positional metadata associated with a transaction hash
+// to allow retrieving cross shard receipt by hash in destination shard
+// not the original transaction in source shard
+// return nil if not found
+func ReadCxLookupEntry(db DatabaseReader, hash common.Hash) (common.Hash, uint64, uint64) {
+	data, _ := db.Get(cxLookupKey(hash))
+	if len(data) == 0 {
+		return common.Hash{}, 0, 0
+	}
+	var entry TxLookupEntry
+	if err := rlp.DecodeBytes(data, &entry); err != nil {
+		utils.Logger().Error().Err(err).Str("hash", hash.Hex()).Msg("Invalid transaction lookup entry RLP")
+		return common.Hash{}, 0, 0
+	}
+	return entry.BlockHash, entry.BlockIndex, entry.Index
+}
+
+// WriteCxLookupEntries stores a positional metadata for every transaction from
+// a block, enabling hash based transaction and receipt lookups.
+func WriteCxLookupEntries(db DatabaseWriter, block *types.Block) {
+	previousSum := 0
+	for _, cxp := range block.IncomingReceipts() {
+		for j, cx := range cxp.Receipts {
+			entry := TxLookupEntry{
+				BlockHash:  block.Hash(),
+				BlockIndex: block.NumberU64(),
+				Index:      uint64(j + previousSum),
+			}
+			data, err := rlp.EncodeToBytes(entry)
+			if err != nil {
+				utils.Logger().Error().Err(err).Msg("Failed to encode transaction lookup entry")
+			}
+			if err := db.Put(cxLookupKey(cx.TxHash), data); err != nil {
+				utils.Logger().Error().Err(err).Msg("Failed to store transaction lookup entry")
+			}
+		}
+		previousSum += len(cxp.Receipts)
+	}
+}
+
+// DeleteCxLookupEntry removes all transaction data associated with a hash.
+func DeleteCxLookupEntry(db DatabaseDeleter, hash common.Hash) {
+	db.Delete(cxLookupKey(hash))
+}
+
+// ReadCXReceipt retrieves a specific transaction from the database, along with
+// its added positional metadata.
+func ReadCXReceipt(db DatabaseReader, hash common.Hash) (*types.CXReceipt, common.Hash, uint64, uint64) {
+	blockHash, blockNumber, cxIndex := ReadCxLookupEntry(db, hash)
+	if blockHash == (common.Hash{}) {
+		return nil, common.Hash{}, 0, 0
+	}
+	body := ReadBody(db, blockHash, blockNumber)
+	if body == nil {
+		utils.Logger().Error().
+			Uint64("number", blockNumber).
+			Str("hash", blockHash.Hex()).
+			Uint64("index", cxIndex).
+			Msg("block Body referenced missing")
+		return nil, common.Hash{}, 0, 0
+	}
+	cx := body.CXReceiptAt(int(cxIndex))
+	if cx == nil {
+		utils.Logger().Error().
+			Uint64("number", blockNumber).
+			Str("hash", blockHash.Hex()).
+			Uint64("index", cxIndex).
+			Msg("CXReceipt referenced missing")
+		return nil, common.Hash{}, 0, 0
+	}
+	return cx, blockHash, blockNumber, cxIndex
+}

core/rawdb/schema.go

@@ -51,8 +51,9 @@ var (
 	blockBodyPrefix     = []byte("b") // blockBodyPrefix + num (uint64 big endian) + hash -> block body
 	blockReceiptsPrefix = []byte("r") // blockReceiptsPrefix + num (uint64 big endian) + hash -> block receipts
 
-	txLookupPrefix  = []byte("l") // txLookupPrefix + hash -> transaction/receipt lookup metadata
+	txLookupPrefix  = []byte("l")  // txLookupPrefix + hash -> transaction/receipt lookup metadata
-	bloomBitsPrefix = []byte("B") // bloomBitsPrefix + bit (uint16 big endian) + section (uint64 big endian) + hash -> bloom bits
+	cxLookupPrefix  = []byte("cx") // cxLookupPrefix + hash -> cxReceipt lookup metadata
+	bloomBitsPrefix = []byte("B")  // bloomBitsPrefix + bit (uint16 big endian) + section (uint64 big endian) + hash -> bloom bits
 
 	shardStatePrefix = []byte("ss") // shardStatePrefix + num (uint64 big endian) + hash -> shardState
 	lastCommitsKey   = []byte("LastCommits")
@@ -137,6 +138,11 @@ func txLookupKey(hash common.Hash) []byte {
 	return append(txLookupPrefix, hash.Bytes()...)
 }
 
+// cxLookupKey = cxLookupPrefix + hash
+func cxLookupKey(hash common.Hash) []byte {
+	return append(cxLookupPrefix, hash.Bytes()...)
+}
+
 // bloomBitsKey = bloomBitsPrefix + bit (uint16 big endian) + section (uint64 big endian) + hash
 func bloomBitsKey(bit uint, section uint64, hash common.Hash) []byte {
 	key := append(append(bloomBitsPrefix, make([]byte, 10)...), hash.Bytes()...)

core/resharding.go

@@ -163,7 +163,7 @@ func CalculateNewShardState(
 	stakeInfo *map[common.Address]*structs.StakeInfo,
 ) (shard.State, error) {
 	if epoch.Cmp(big.NewInt(GenesisEpoch)) == 0 {
-		return GetInitShardState(), nil
+		return CalculateInitShardState(), nil
 	}
 	prevEpoch := new(big.Int).Sub(epoch, common.Big1)
 	ss, err := GetShardingStateFromBlockChain(bc, prevEpoch)
@@ -215,15 +215,15 @@ func (ss *ShardingState) UpdateShardingState(stakeInfo *map[common.Address]*stru
 // Depends on the type of the network.  Defaults to the mainnet schedule.
 var ShardingSchedule shardingconfig.Schedule = shardingconfig.MainnetSchedule
 
-// GetInitShardState returns the initial shard state at genesis.
+// CalculateInitShardState returns the initial shard state at genesis.
-func GetInitShardState() shard.State {
+func CalculateInitShardState() shard.State {
-	return GetShardState(big.NewInt(GenesisEpoch))
+	return CalculateShardState(big.NewInt(GenesisEpoch))
 }
 
-// GetShardState returns the shard state based on epoch number
+// CalculateShardState returns the shard state based on epoch number
 // This api for getting shard state is what should be used to get shard state regardless of
 // current chain dependency (ex. getting shard state from block header received during cross-shard transaction)
-func GetShardState(epoch *big.Int) shard.State {
+func CalculateShardState(epoch *big.Int) shard.State {
 	utils.Logger().Info().Int64("epoch", epoch.Int64()).Msg("Get Shard State of Epoch.")
 	shardingConfig := ShardingSchedule.InstanceForEpoch(epoch)
 	shardNum := int(shardingConfig.NumShards())
@@ -271,9 +271,9 @@ func GetShardState(epoch *big.Int) shard.State {
 	return shardState
 }
 
-// GetPublicKeys returns the publickeys given epoch and shardID
+// CalculatePublicKeys returns the publickeys given epoch and shardID
-func GetPublicKeys(epoch *big.Int, shardID uint32) []*bls.PublicKey {
+func CalculatePublicKeys(epoch *big.Int, shardID uint32) []*bls.PublicKey {
-	shardState := GetShardState(epoch)
+	shardState := CalculateShardState(epoch)
 
 	// Update validator public keys
 	committee := shardState.FindCommitteeByID(shardID)

core/types/block.go

@@ -86,6 +86,10 @@ type BodyInterface interface {
 	// It returns nil if index is out of bounds.
 	TransactionAt(index int) *Transaction
 
+	// CXReceiptAt returns the CXReceipt given index (calculated from IncomingReceipts)
+	// It returns nil if index is out of bounds
+	CXReceiptAt(index int) *CXReceipt
+
 	// SetTransactions sets the list of transactions with a deep copy of the
 	// given list.
 	SetTransactions(newTransactions []*Transaction)

core/types/bodyv0.go

@@ -39,6 +39,13 @@ func (b *BodyV0) TransactionAt(index int) *Transaction {
 	return b.f.Transactions[index].Copy()
 }
 
+// CXReceiptAt returns the CXReceipt at given index in this block
+// It returns nil if index is out of bounds
+// V0 will just return nil because we don't support CXReceipt
+func (b *BodyV0) CXReceiptAt(index int) *CXReceipt {
+	return nil
+}
+
 // SetTransactions sets the list of transactions with a deep copy of the given
 // list.
 func (b *BodyV0) SetTransactions(newTransactions []*Transaction) {

core/types/bodyv1.go

@@ -39,6 +39,22 @@ func (b *BodyV1) TransactionAt(index int) *Transaction {
 	return b.f.Transactions[index].Copy()
 }
 
+// CXReceiptAt returns the CXReceipt at given index in this block
+// It returns nil if index is out of bounds
+func (b *BodyV1) CXReceiptAt(index int) *CXReceipt {
+	if index < 0 {
+		return nil
+	}
+	for _, cxp := range b.f.IncomingReceipts {
+		cxs := cxp.Receipts
+		if index < len(cxs) {
+			return cxs[index].Copy()
+		}
+		index -= len(cxs)
+	}
+	return nil
+}
+
 // SetTransactions sets the list of transactions with a deep copy of the given
 // list.
 func (b *BodyV1) SetTransactions(newTransactions []*Transaction) {

hmy/api_backend.go

@@ -20,6 +20,7 @@ import (
 	"github.com/harmony-one/harmony/core/types"
 	"github.com/harmony-one/harmony/core/vm"
 	"github.com/harmony-one/harmony/internal/params"
+	"github.com/harmony-one/harmony/shard"
 )
 
 // APIBackend An implementation of internal/hmyapi/Backend. Full client.
@@ -233,6 +234,20 @@ func (b *APIBackend) GetShardID() uint32 {
 	return b.hmy.shardID
 }
 
+// GetCommittee returns committee for a particular epoch.
+func (b *APIBackend) GetCommittee(epoch *big.Int) (*shard.Committee, error) {
+	state, err := b.hmy.BlockChain().ReadShardState(epoch)
+	if err != nil {
+		return nil, err
+	}
+	for _, committee := range state {
+		if committee.ShardID == b.GetShardID() {
+			return &committee, nil
+		}
+	}
+	return nil, nil
+}
+
 // ResendCx retrieve blockHash from txID and add blockHash to CxPool for resending
 func (b *APIBackend) ResendCx(ctx context.Context, txID common.Hash) (uint64, bool) {
 	blockHash, blockNum, index := b.hmy.BlockChain().ReadTxLookupEntry(txID)

internal/chain/engine.go

@@ -11,7 +11,6 @@ import (
 
 	"github.com/harmony-one/harmony/block"
 	"github.com/harmony-one/harmony/consensus/engine"
-	"github.com/harmony-one/harmony/core"
 	"github.com/harmony-one/harmony/core/state"
 	"github.com/harmony-one/harmony/core/types"
 	"github.com/harmony-one/harmony/internal/ctxerror"
@@ -104,29 +103,7 @@ func (e *engineImpl) VerifyHeaders(chain engine.ChainReader, headers []*block.He
 // ReadPublicKeysFromLastBlock finds the public keys of last block's committee
 func ReadPublicKeysFromLastBlock(bc engine.ChainReader, header *block.Header) ([]*bls.PublicKey, error) {
 	parentHeader := bc.GetHeaderByHash(header.ParentHash())
-	if parentHeader == nil {
+	return GetPublicKeys(bc, parentHeader)
-		return nil, ctxerror.New("cannot find parent block header in DB",
-			"parentHash", header.ParentHash())
-	}
-	parentShardState := core.GetShardState(parentHeader.Epoch())
-	parentCommittee := parentShardState.FindCommitteeByID(parentHeader.ShardID())
-	if parentCommittee == nil {
-		return nil, ctxerror.New("cannot find shard in the shard state",
-			"parentBlockNumber", parentHeader.Number(),
-			"shardID", parentHeader.ShardID(),
-		)
-	}
-	var committerKeys []*bls.PublicKey
-	for _, member := range parentCommittee.NodeList {
-		committerKey := new(bls.PublicKey)
-		err := member.BlsPublicKey.ToLibBLSPublicKey(committerKey)
-		if err != nil {
-			return nil, ctxerror.New("cannot convert BLS public key",
-				"blsPublicKey", member.BlsPublicKey).WithCause(err)
-		}
-		committerKeys = append(committerKeys, committerKey)
-	}
-	return committerKeys, nil
 }
 
 // VerifySeal implements Engine, checking whether the given block's parent block satisfies
@@ -148,7 +125,7 @@ func (e *engineImpl) VerifySeal(chain engine.ChainReader, header *block.Header)
 	}
 	parentHash := header.ParentHash()
 	parentHeader := chain.GetHeader(parentHash, header.Number().Uint64()-1)
-	parentQuorum, err := QuorumForBlock(parentHeader)
+	parentQuorum, err := QuorumForBlock(chain, parentHeader)
 	if err != nil {
 		return errors.Wrapf(err,
 			"cannot calculate quorum for block %s", header.Number())
@@ -181,8 +158,12 @@ func (e *engineImpl) Finalize(chain engine.ChainReader, header *block.Header, st
 }
 
 // QuorumForBlock returns the quorum for the given block header.
-func QuorumForBlock(h *block.Header) (quorum int, err error) {
+func QuorumForBlock(chain engine.ChainReader, h *block.Header) (quorum int, err error) {
-	ss := core.GetShardState(h.Epoch())
+	ss, err := chain.ReadShardState(h.Epoch())
+	if err != nil {
+		return 0, ctxerror.New("failed to read shard state of epoch",
+			"epoch", h.Epoch().Uint64())
+	}
 	c := ss.FindCommitteeByID(h.ShardID())
 	if c == nil {
 		return 0, errors.Errorf(
@@ -195,8 +176,8 @@ func QuorumForBlock(h *block.Header) (quorum int, err error) {
 // is used for verifying "incoming" block header against commit signature and bitmap sent from the other chain cross-shard via libp2p.
 // i.e. this header verification api is more flexible since the caller specifies which commit signature and bitmap to use
 // for verifying the block header, which is necessary for cross-shard block header verification. Example of such is cross-shard transaction.
-func (e *engineImpl) VerifyHeaderWithSignature(header *block.Header, commitSig []byte, commitBitmap []byte) error {
+func (e *engineImpl) VerifyHeaderWithSignature(chain engine.ChainReader, header *block.Header, commitSig []byte, commitBitmap []byte) error {
-	publicKeys, err := GetPublicKeys(header)
+	publicKeys, err := GetPublicKeys(chain, header)
 	if err != nil {
 		return ctxerror.New("[VerifyHeaderWithSignature] Cannot get publickeys for block header").WithCause(err)
 	}
@@ -208,7 +189,7 @@ func (e *engineImpl) VerifyHeaderWithSignature(header *block.Header, commitSig [
 	}
 
 	hash := header.Hash()
-	quorum, err := QuorumForBlock(header)
+	quorum, err := QuorumForBlock(chain, header)
 	if err != nil {
 		return errors.Wrapf(err,
 			"cannot calculate quorum for block %s", header.Number())
@@ -229,8 +210,13 @@ func (e *engineImpl) VerifyHeaderWithSignature(header *block.Header, commitSig [
 }
 
 // GetPublicKeys finds the public keys of the committee that signed the block header
-func GetPublicKeys(header *block.Header) ([]*bls.PublicKey, error) {
+func GetPublicKeys(chain engine.ChainReader, header *block.Header) ([]*bls.PublicKey, error) {
-	shardState := core.GetShardState(header.Epoch())
+	shardState, err := chain.ReadShardState(header.Epoch())
+	if err != nil {
+		return nil, ctxerror.New("failed to read shard state of epoch",
+			"epoch", header.Epoch().Uint64())
+	}
+
 	committee := shardState.FindCommitteeByID(header.ShardID())
 	if committee == nil {
 		return nil, ctxerror.New("cannot find shard in the shard state",

internal/configs/node/config.go

@@ -62,6 +62,7 @@ const (
 )
 
 var version string
+var publicRPC bool // enable public RPC access
 
 // ConfigType is the structure of all node related configuration variables
 type ConfigType struct {
@@ -246,3 +247,13 @@ func SetVersion(ver string) {
 func GetVersion() string {
 	return version
 }
+
+// SetPublicRPC set the boolean value of public RPC access
+func SetPublicRPC(v bool) {
+	publicRPC = v
+}
+
+// GetPublicRPC get the boolean value of public RPC access
+func GetPublicRPC() bool {
+	return publicRPC
+}

internal/hmyapi/README.md

@@ -10,6 +10,7 @@
 * [x] hmy_protocolVersion - check protocol version
 * [ ] net_version - get network id
 * [ ] net_peerCount - peer count
+* [x] hmy_getNodeMetadata - get node's version, bls key
 
 ### BlockChain info related
 * [ ] hmy_gasPrice - return min-gas-price
@@ -87,4 +88,4 @@ The ``shh`` is for the whisper protocol to communicate p2p and broadcast
 * [ ] shh_newFilter
 * [ ] shh_uninstallFilter
 * [ ] shh_getFilterChanges
-* [ ] shh_getMessages
+* [ ] shh_getMessages

internal/hmyapi/backend.go

@@ -17,6 +17,7 @@ import (
 	"github.com/harmony-one/harmony/core/types"
 	"github.com/harmony-one/harmony/core/vm"
 	"github.com/harmony-one/harmony/internal/params"
+	"github.com/harmony-one/harmony/shard"
 )
 
 // Backend interface provides the common API services (that are provided by
@@ -63,6 +64,8 @@ type Backend interface {
 	CurrentBlock() *types.Block
 	// Get balance
 	GetBalance(address common.Address) (*hexutil.Big, error)
+	// Get committee for a particular epoch
+	GetCommittee(epoch *big.Int) (*shard.Committee, error)
 	GetShardID() uint32
 
 	// retrieve the blockHash using txID and add blockHash to CxPool for resending

internal/hmyapi/blockchain.go

@@ -63,6 +63,36 @@ func (s *PublicBlockChainAPI) GetBlockByHash(ctx context.Context, blockHash comm
 	return nil, err
 }
 
+// GetCommittee returns committee for a particular epoch.
+func (s *PublicBlockChainAPI) GetCommittee(ctx context.Context, epoch int64) (map[string]interface{}, error) {
+	committee, err := s.b.GetCommittee(big.NewInt(epoch))
+	if err != nil {
+		return nil, err
+	}
+	validators := make([]map[string]interface{}, 0)
+	for _, validator := range committee.NodeList {
+		validatorBalance := new(hexutil.Big)
+		validatorBalance, err = s.b.GetBalance(validator.EcdsaAddress)
+		if err != nil {
+			return nil, err
+		}
+		oneAddress, err := internal_common.AddressToBech32(validator.EcdsaAddress)
+		if err != nil {
+			return nil, err
+		}
+		validatorsFields := map[string]interface{}{
+			"address": oneAddress,
+			"balance": validatorBalance,
+		}
+		validators = append(validators, validatorsFields)
+	}
+	result := map[string]interface{}{
+		"shardID":    committee.ShardID,
+		"validators": validators,
+	}
+	return result, nil
+}
+
 // GetShardingStructure returns an array of sharding structures.
 func (s *PublicBlockChainAPI) GetShardingStructure(ctx context.Context) ([]map[string]interface{}, error) {
 	// Get header and number of shards.
@@ -73,6 +103,11 @@ func (s *PublicBlockChainAPI) GetShardingStructure(ctx context.Context) ([]map[s
 	return core.ShardingSchedule.GetShardingStructure(int(numShard), int(s.b.GetShardID())), nil
 }
 
+// GetShardID returns shard ID of the requested node.
+func (s *PublicBlockChainAPI) GetShardID(ctx context.Context) (int, error) {
+	return int(s.b.GetShardID()), nil
+}
+
 // GetCode returns the code stored at the given address in the state for the given block number.
 func (s *PublicBlockChainAPI) GetCode(ctx context.Context, addr string, blockNr rpc.BlockNumber) (hexutil.Bytes, error) {
 	address := internal_common.ParseAddr(addr)
@@ -225,3 +260,9 @@ func doCall(ctx context.Context, b Backend, args CallArgs, blockNr rpc.BlockNumb
 	}
 	return res, gas, failed, err
 }
+
+// LatestHeader returns the latest header information
+func (s *PublicBlockChainAPI) LatestHeader(ctx context.Context) *HeaderInformation {
+	header, _ := s.b.HeaderByNumber(context.Background(), rpc.LatestBlockNumber) // latest header should always be available
+	return newHeaderInformation(header)
+}

internal/hmyapi/error.go

@@ -0,0 +1,8 @@
+package hmyapi
+
+import "errors"
+
+var (
+	// ErrIncorrectChainID is an incorrect chain ID.
+	ErrIncorrectChainID = errors.New("Incorrect chain ID")
+)

internal/hmyapi/harmony.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/harmony-one/harmony/api/proto"
+	nodeconfig "github.com/harmony-one/harmony/internal/configs/node"
 )
 
 // PublicHarmonyAPI provides an API to access Harmony related information.
@@ -41,3 +42,22 @@ func (s *PublicHarmonyAPI) GasPrice(ctx context.Context) (*hexutil.Big, error) {
 	// TODO(ricl): add SuggestPrice API
 	return (*hexutil.Big)(big.NewInt(1)), nil
 }
+
+// NodeMetadata captures select metadata of the RPC answering node
+type NodeMetadata struct {
+	BLSPublicKey string `json:"blskey"`
+	Version      string `json:"version"`
+	NetworkType  string `json:"network"`
+	ChainID      string `json:"chainid"`
+}
+
+// GetNodeMetadata produces a NodeMetadata record. Note the data is from the answering RPC
+func (s *PublicHarmonyAPI) GetNodeMetadata() NodeMetadata {
+	cfg := nodeconfig.GetDefaultConfig()
+	return NodeMetadata{
+		cfg.ConsensusPubKey.SerializeToHexStr(),
+		nodeconfig.GetVersion(),
+		string(cfg.GetNetworkType()),
+		s.b.ChainConfig().ChainID.String(),
+	}
+}

internal/hmyapi/transactionpool.go

@@ -131,6 +131,9 @@ func (s *PublicTransactionPoolAPI) SendRawTransaction(ctx context.Context, encod
 	if err := rlp.DecodeBytes(encodedTx, tx); err != nil {
 		return common.Hash{}, err
 	}
+	if tx.ChainID() != s.b.ChainConfig().ChainID {
+		return common.Hash{}, ErrIncorrectChainID
+	}
 	return SubmitTransaction(ctx, s.b, tx)
 }
 
@@ -212,3 +215,11 @@ func (s *PublicTransactionPoolAPI) PendingTransactions() ([]*RPCTransaction, err
 	}
 	return transactions, nil
 }
+
+// GetCXReceiptByHash returns the transaction for the given hash
+func (s *PublicTransactionPoolAPI) GetCXReceiptByHash(ctx context.Context, hash common.Hash) *RPCCXReceipt {
+	if cx, blockHash, blockNumber, _ := rawdb.ReadCXReceipt(s.b.ChainDb(), hash); cx != nil {
+		return newRPCCXReceipt(cx, blockHash, blockNumber)
+	}
+	return nil
+}

internal/hmyapi/types.go

@@ -1,12 +1,16 @@
 package hmyapi
 
 import (
+	"encoding/hex"
 	"math/big"
+	"time"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	ethtypes "github.com/ethereum/go-ethereum/core/types"
+	"github.com/harmony-one/harmony/block"
 	"github.com/harmony-one/harmony/core/types"
+	common2 "github.com/harmony-one/harmony/internal/common"
 )
 
 // RPCTransaction represents a transaction that will serialize to the RPC representation of a transaction
@@ -22,11 +26,77 @@ type RPCTransaction struct {
 	To               *common.Address `json:"to"`
 	TransactionIndex hexutil.Uint    `json:"transactionIndex"`
 	Value            *hexutil.Big    `json:"value"`
+	ShardID          uint32          `json:"shardID"`
+	ToShardID        uint32          `json:"toShardID"`
 	V                *hexutil.Big    `json:"v"`
 	R                *hexutil.Big    `json:"r"`
 	S                *hexutil.Big    `json:"s"`
 }
 
+// RPCCXReceipt represents a CXReceipt that will serialize to the RPC representation of a CXReceipt
+type RPCCXReceipt struct {
+	BlockHash   common.Hash     `json:"blockHash"`
+	BlockNumber *hexutil.Big    `json:"blockNumber"`
+	TxHash      common.Hash     `json:"hash"`
+	From        common.Address  `json:"from"`
+	To          *common.Address `json:"to"`
+	ShardID     uint32          `json:"shardID"`
+	ToShardID   uint32          `json:"toShardID"`
+	Amount      *hexutil.Big    `json:"value"`
+}
+
+// HeaderInformation represents the latest consensus information
+type HeaderInformation struct {
+	BlockHash        common.Hash `json:"blockHash"`
+	BlockNumber      uint64      `json:"blockNumber"`
+	ShardID          uint32      `json:"shardID"`
+	Leader           string      `json:"leader"`
+	ViewID           uint64      `json:"viewID"`
+	Epoch            uint64      `json:"epoch"`
+	Timestamp        string      `json:"timestamp"`
+	UnixTime         uint64      `json:"unixtime"`
+	LastCommitSig    string      `json:"lastCommitSig"`
+	LastCommitBitmap string      `json:"lastCommitBitmap"`
+}
+
+func newHeaderInformation(header *block.Header) *HeaderInformation {
+	if header == nil {
+		return nil
+	}
+	result := &HeaderInformation{
+		BlockHash:        header.Hash(),
+		BlockNumber:      header.Number().Uint64(),
+		ShardID:          header.ShardID(),
+		Leader:           common2.MustAddressToBech32(header.Coinbase()),
+		ViewID:           header.ViewID().Uint64(),
+		Epoch:            header.Epoch().Uint64(),
+		UnixTime:         header.Time().Uint64(),
+		Timestamp:        time.Unix(header.Time().Int64(), 0).UTC().String(),
+		LastCommitBitmap: hex.EncodeToString(header.LastCommitBitmap()),
+	}
+	sig := header.LastCommitSignature()
+	result.LastCommitSig = hex.EncodeToString(sig[:])
+	return result
+}
+
+// newRPCCXReceipt returns a CXReceipt that will serialize to the RPC representation
+func newRPCCXReceipt(cx *types.CXReceipt, blockHash common.Hash, blockNumber uint64) *RPCCXReceipt {
+	result := &RPCCXReceipt{
+		BlockHash: blockHash,
+		TxHash:    cx.TxHash,
+		From:      cx.From,
+		To:        cx.To,
+		Amount:    (*hexutil.Big)(cx.Amount),
+		ShardID:   cx.ShardID,
+		ToShardID: cx.ToShardID,
+	}
+	if blockHash != (common.Hash{}) {
+		result.BlockHash = blockHash
+		result.BlockNumber = (*hexutil.Big)(new(big.Int).SetUint64(blockNumber))
+	}
+	return result
+}
+
 // newRPCTransaction returns a transaction that will serialize to the RPC
 // representation, with the given location metadata set (if available).
 func newRPCTransaction(tx *types.Transaction, blockHash common.Hash, blockNumber uint64, index uint64) *RPCTransaction {
@@ -38,17 +108,19 @@ func newRPCTransaction(tx *types.Transaction, blockHash common.Hash, blockNumber
 	v, r, s := tx.RawSignatureValues()
 
 	result := &RPCTransaction{
-		From:     from,
+		From:      from,
-		Gas:      hexutil.Uint64(tx.Gas()),
+		Gas:       hexutil.Uint64(tx.Gas()),
-		GasPrice: (*hexutil.Big)(tx.GasPrice()),
+		GasPrice:  (*hexutil.Big)(tx.GasPrice()),
-		Hash:     tx.Hash(),
+		Hash:      tx.Hash(),
-		Input:    hexutil.Bytes(tx.Data()),
+		Input:     hexutil.Bytes(tx.Data()),
-		Nonce:    hexutil.Uint64(tx.Nonce()),
+		Nonce:     hexutil.Uint64(tx.Nonce()),
-		To:       tx.To(),
+		To:        tx.To(),
-		Value:    (*hexutil.Big)(tx.Value()),
+		Value:     (*hexutil.Big)(tx.Value()),
-		V:        (*hexutil.Big)(v),
+		ShardID:   tx.ShardID(),
-		R:        (*hexutil.Big)(r),
+		ToShardID: tx.ToShardID(),
-		S:        (*hexutil.Big)(s),
+		V:         (*hexutil.Big)(v),
+		R:         (*hexutil.Big)(r),
+		S:         (*hexutil.Big)(s),
 	}
 	if blockHash != (common.Hash{}) {
 		result.BlockHash = blockHash

internal/params/config.go

@@ -7,45 +7,68 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 )
 
+// Well-known chain IDs.
+var (
+	MainnetChainID            = big.NewInt(1)
+	TestnetChainID            = big.NewInt(2)
+	PangaeaChainID            = big.NewInt(3)
+	TestChainID               = big.NewInt(99)  // not a real network
+	AllProtocolChangesChainID = big.NewInt(100) // not a real network
+)
+
+// EpochTBD is a large, “not anytime soon” epoch.  It used as a placeholder
+// until the exact epoch is decided.
+var EpochTBD = big.NewInt(10000000)
+
 var (
 	// MainnetChainConfig is the chain parameters to run a node on the main network.
 	MainnetChainConfig = &ChainConfig{
-		ChainID:        big.NewInt(1),
+		ChainID:        MainnetChainID,
 		CrossTxEpoch:   big.NewInt(28),
-		CrossLinkEpoch: big.NewInt(10000000), // Temporarily made very large until a exact number is decided.
+		CrossLinkEpoch: EpochTBD,
 		EIP155Epoch:    big.NewInt(28),
 		S3Epoch:        big.NewInt(28),
 	}
 
 	// TestnetChainConfig contains the chain parameters to run a node on the harmony test network.
 	TestnetChainConfig = &ChainConfig{
-		ChainID:        big.NewInt(2),
+		ChainID:        TestnetChainID,
 		CrossTxEpoch:   big.NewInt(1),
 		CrossLinkEpoch: big.NewInt(2),
 		EIP155Epoch:    big.NewInt(0),
 		S3Epoch:        big.NewInt(0),
 	}
 
+	// PangaeaChainConfig contains the chain parameters for the Pangaea network.
+	// All features except for CrossLink are enabled at launch.
+	PangaeaChainConfig = &ChainConfig{
+		ChainID:        PangaeaChainID,
+		CrossTxEpoch:   big.NewInt(0),
+		CrossLinkEpoch: EpochTBD,
+		EIP155Epoch:    big.NewInt(0),
+		S3Epoch:        big.NewInt(0),
+	}
+
 	// AllProtocolChanges ...
 	// This configuration is intentionally not using keyed fields to force anyone
 	// adding flags to the config to also have to set these fields.
 	AllProtocolChanges = &ChainConfig{
-		big.NewInt(100), // ChainID
+		AllProtocolChangesChainID, // ChainID
-		big.NewInt(0),   // CrossTxEpoch
+		big.NewInt(0),             // CrossTxEpoch
-		big.NewInt(0),   // CrossLinkEpoch
+		big.NewInt(0),             // CrossLinkEpoch
-		big.NewInt(0),   // EIP155Epoch
+		big.NewInt(0),             // EIP155Epoch
-		big.NewInt(0),   // S3Epoch
+		big.NewInt(0),             // S3Epoch
 	}
 
 	// TestChainConfig ...
 	// This configuration is intentionally not using keyed fields to force anyone
 	// adding flags to the config to also have to set these fields.
 	TestChainConfig = &ChainConfig{
-		big.NewInt(99), // ChainID
+		TestChainID,   // ChainID
-		big.NewInt(0),  // CrossTxEpoch
+		big.NewInt(0), // CrossTxEpoch
-		big.NewInt(0),  // CrossLinkEpoch
+		big.NewInt(0), // CrossLinkEpoch
-		big.NewInt(0),  // EIP155Epoch
+		big.NewInt(0), // EIP155Epoch
-		big.NewInt(0),  // S3Epoch
+		big.NewInt(0), // S3Epoch
 	}
 
 	// TestRules ...

internal/utils/configfile.go

@@ -3,15 +3,19 @@ package utils
 // this module in utils handles the ini file read/write
 import (
 	"fmt"
+	"os"
 	"strings"
 
-	"github.com/harmony-one/harmony/p2p"
 	ini "gopkg.in/ini.v1"
+
+	"github.com/harmony-one/harmony/internal/params"
+	"github.com/harmony-one/harmony/p2p"
 )
 
 // WalletProfile contains a section and key value pair map
 type WalletProfile struct {
 	Profile   string
+	ChainID   string
 	Bootnodes []string
 	Shards    int
 	RPCServer [][]p2p.Peer
@@ -31,12 +35,32 @@ func ReadWalletProfile(iniBytes []byte, profile string) (*WalletProfile, error)
 	if err != nil {
 		return nil, err
 	}
+	profile = sec.Name() // sanitized name
 
 	if sec.HasKey("bootnode") {
 		config.Bootnodes = sec.Key("bootnode").ValueWithShadows()
 	} else {
 		return nil, fmt.Errorf("can't find bootnode key")
 	}
+	if sec.HasKey("chain_id") {
+		config.ChainID = sec.Key("chain_id").String()
+	} else {
+		// backward compatibility; use profile name to determine
+		// (deprecated; require chain_id after 2010-01).
+		switch profile {
+		case "main", "default":
+			config.ChainID = params.MainnetChainID.String()
+		case "pangaea":
+			config.ChainID = params.PangaeaChainID.String()
+		default:
+			config.ChainID = params.TestnetChainID.String()
+		}
+		_, _ = fmt.Fprintf(os.Stderr,
+			"NOTICE: Chain ID not found in config profile, assuming %s; "+
+				"please add \"chain_id = %s\" to section [%s] of wallet.ini "+
+				"before 2020-01\n",
+			config.ChainID, config.ChainID, profile)
+	}
 
 	if sec.HasKey("shards") {
 		config.Shards = sec.Key("shards").MustInt()

internal/utils/configfile_test.go

@@ -5,6 +5,7 @@ import (
 	"reflect"
 	"testing"
 
+	"github.com/harmony-one/harmony/internal/params"
 	"github.com/harmony-one/harmony/p2p"
 )
 
@@ -12,6 +13,7 @@ func TestReadWalletProfile(t *testing.T) {
 	config := []*WalletProfile{
 		{
 			Profile:   "default",
+			ChainID:   params.MainnetChainID.String(),
 			Bootnodes: []string{"127.0.0.1:9000/abcd", "127.0.0.1:9999/daeg"},
 			Shards:    4,
 			RPCServer: [][]p2p.Peer{
@@ -59,6 +61,7 @@ func TestReadWalletProfile(t *testing.T) {
 		},
 		{
 			Profile:   "testnet",
+			ChainID:   params.TestnetChainID.String(),
 			Bootnodes: []string{"192.168.0.1:9990/abcd", "127.0.0.1:8888/daeg"},
 			Shards:    3,
 			RPCServer: [][]p2p.Peer{

node/contract.go

@@ -150,7 +150,9 @@ func (node *Node) GetBalanceOfAddress(address common.Address) (*big.Int, error)
 		utils.Logger().Error().Err(err).Msg("Failed to get chain state")
 		return nil, err
 	}
-	return state.GetBalance(address), nil
+	balance := big.NewInt(0)
+	balance.SetBytes(state.GetBalance(address).Bytes())
+	return balance, nil
 }
 
 // AddFaucetContractToPendingTransactions adds the faucet contract the genesis block.

node/node.go

@@ -192,6 +192,9 @@ type Node struct {
 	// node configuration, including group ID, shard ID, etc
 	NodeConfig *nodeconfig.ConfigType
 
+	// Chain configuration.
+	chainConfig params.ChainConfig
+
 	// map of service type to its message channel.
 	serviceMessageChan map[service.Type]chan *msg_pb.Message
 
@@ -367,9 +370,13 @@ func New(host p2p.Host, consensusObj *consensus.Consensus, chainDBFactory shardc
 	}
 
 	chainConfig := *params.TestnetChainConfig
-	if node.NodeConfig.GetNetworkType() == nodeconfig.Mainnet {
+	switch node.NodeConfig.GetNetworkType() {
+	case nodeconfig.Mainnet:
 		chainConfig = *params.MainnetChainConfig
+	case nodeconfig.Pangaea:
+		chainConfig = *params.PangaeaChainConfig
 	}
+	node.chainConfig = chainConfig
 
 	collection := shardchain.NewCollection(
 		chainDBFactory, &genesisInitializer{&node}, chain.Engine, &chainConfig)
@@ -392,9 +399,9 @@ func New(host p2p.Host, consensusObj *consensus.Consensus, chainDBFactory shardc
 		node.recentTxsStats = make(types.RecentTxsStats)
 		node.TxPool = core.NewTxPool(core.DefaultTxPoolConfig, node.Blockchain().Config(), blockchain)
 		node.CxPool = core.NewCxPool(core.CxPoolSize)
-		node.Worker = worker.New(node.Blockchain().Config(), blockchain, chain.Engine, node.Consensus.ShardID)
+		node.Worker = worker.New(node.Blockchain().Config(), blockchain, chain.Engine)
 		if node.Blockchain().ShardID() != 0 {
-			node.BeaconWorker = worker.New(node.Beaconchain().Config(), beaconChain, chain.Engine, node.Consensus.ShardID)
+			node.BeaconWorker = worker.New(node.Beaconchain().Config(), beaconChain, chain.Engine)
 		}
 
 		node.Consensus.VerifiedNewBlock = make(chan *types.Block)
@@ -458,10 +465,10 @@ func New(host p2p.Host, consensusObj *consensus.Consensus, chainDBFactory shardc
 	return &node
 }
 
-// GetInitShardState initialize shard state from latest epoch and update committee pub keys for consensus and drand
+// CalculateInitShardState initialize shard state from latest epoch and update committee pub keys for consensus and drand
-func (node *Node) GetInitShardState() (err error) {
+func (node *Node) CalculateInitShardState() (err error) {
 	if node.Consensus == nil {
-		return ctxerror.New("[GetInitShardState] consenus is nil; Cannot figure out shardID")
+		return ctxerror.New("[CalculateInitShardState] consenus is nil; Cannot figure out shardID")
 	}
 	shardID := node.Consensus.ShardID
 
@@ -473,11 +480,11 @@ func (node *Node) GetInitShardState() (err error) {
 		Uint64("blockNum", blockNum).
 		Uint32("shardID", shardID).
 		Uint64("epoch", epoch.Uint64()).
-		Msg("[GetInitShardState] Try To Get PublicKeys from database")
+		Msg("[CalculateInitShardState] Try To Get PublicKeys from database")
-	pubKeys := core.GetPublicKeys(epoch, shardID)
+	pubKeys := core.CalculatePublicKeys(epoch, shardID)
 	if len(pubKeys) == 0 {
 		return ctxerror.New(
-			"[GetInitShardState] PublicKeys is Empty, Cannot update public keys",
+			"[CalculateInitShardState] PublicKeys is Empty, Cannot update public keys",
 			"shardID", shardID,
 			"blockNum", blockNum)
 	}
@@ -487,7 +494,7 @@ func (node *Node) GetInitShardState() (err error) {
 			utils.Logger().Info().
 				Uint64("blockNum", blockNum).
 				Int("numPubKeys", len(pubKeys)).
-				Msg("[GetInitShardState] Successfully updated public keys")
+				Msg("[CalculateInitShardState] Successfully updated public keys")
 			node.Consensus.UpdatePublicKeys(pubKeys)
 			node.Consensus.SetMode(consensus.Normal)
 			return nil
@@ -567,3 +574,8 @@ func (node *Node) initNodeConfiguration() (service.NodeConfig, chan p2p.Peer) {
 func (node *Node) AccountManager() *accounts.Manager {
 	return node.accountManager
 }
+
+// ServiceManager ...
+func (node *Node) ServiceManager() *service.Manager {
+	return node.serviceManager
+}

node/node_cross_shard.go

@@ -58,7 +58,7 @@ func (node *Node) BroadcastCXReceiptsWithShardID(block *types.Block, commitSig [
 
 	cxReceipts, err := node.Blockchain().ReadCXReceipts(toShardID, block.NumberU64(), block.Hash(), false)
 	if err != nil || len(cxReceipts) == 0 {
-		utils.Logger().Warn().Err(err).Uint32("ToShardID", toShardID).Int("numCXReceipts", len(cxReceipts)).Msg("[BroadcastCXReceiptsWithShardID] No ReadCXReceipts found")
+		utils.Logger().Info().Err(err).Uint32("ToShardID", toShardID).Int("numCXReceipts", len(cxReceipts)).Msg("[BroadcastCXReceiptsWithShardID] No ReadCXReceipts found")
 		return
 	}
 	merkleProof, err := node.Blockchain().CXMerkleProof(toShardID, block)

node/node_explorer.go

@@ -72,7 +72,7 @@ func (node *Node) ExplorerMessageHandler(payload []byte) {
 			return
 		}
 
-		node.AddNewBlockForExplorer()
+		node.AddNewBlockForExplorer(block)
 		node.commitBlockForExplorer(block)
 	} else if msg.Type == msg_pb.MessageType_PREPARED {
 
@@ -91,7 +91,7 @@ func (node *Node) ExplorerMessageHandler(payload []byte) {
 		msgs := node.Consensus.PbftLog.GetMessagesByTypeSeqHash(msg_pb.MessageType_COMMITTED, blockObj.NumberU64(), blockObj.Hash())
 		// If found, then add the new block into blockchain db.
 		if len(msgs) > 0 {
-			node.AddNewBlockForExplorer()
+			node.AddNewBlockForExplorer(blockObj)
 			node.commitBlockForExplorer(blockObj)
 		}
 	}
@@ -99,41 +99,29 @@ func (node *Node) ExplorerMessageHandler(payload []byte) {
 }
 
 // AddNewBlockForExplorer add new block for explorer.
-func (node *Node) AddNewBlockForExplorer() {
+func (node *Node) AddNewBlockForExplorer(block *types.Block) {
-	utils.Logger().Debug().Msg("[Explorer] Add new block for explorer")
+	utils.Logger().Debug().Uint64("blockHeight", block.NumberU64()).Msg("[Explorer] Adding new block for explorer node")
-	// Search for the next block in PbftLog and commit the block into blockchain for explorer node.
+	if err := node.AddNewBlock(block); err == nil {
-	for {
+		if core.IsEpochLastBlock(block) {
-		blocks := node.Consensus.PbftLog.GetBlocksByNumber(node.Blockchain().CurrentBlock().NumberU64() + 1)
+			node.Consensus.UpdateConsensusInformation()
-		if len(blocks) == 0 {
-			break
-		} else {
-			if len(blocks) > 1 {
-				utils.Logger().Error().Msg("[Explorer] We should have not received more than one block with the same block height.")
-			}
-			utils.Logger().Debug().Uint64("blockHeight", blocks[0].NumberU64()).Msg("Adding new block for explorer node")
-			if err := node.AddNewBlock(blocks[0]); err == nil {
-				if core.IsEpochLastBlock(blocks[0]) {
-					node.Consensus.UpdateConsensusInformation()
-				}
-				// Clean up the blocks to avoid OOM.
-				node.Consensus.PbftLog.DeleteBlockByNumber(blocks[0].NumberU64())
-				// Do dump all blocks from state syncing for explorer one time
-				// TODO: some blocks can be dumped before state syncing finished.
-				// And they would be dumped again here. Please fix it.
-				once.Do(func() {
-					utils.Logger().Info().Int64("starting height", int64(blocks[0].NumberU64())-1).
-						Msg("[Explorer] Populating explorer data from state synced blocks")
-					go func() {
-						for blockHeight := int64(blocks[0].NumberU64()) - 1; blockHeight >= 0; blockHeight-- {
-							explorer.GetStorageInstance(node.SelfPeer.IP, node.SelfPeer.Port, true).Dump(
-								node.Blockchain().GetBlockByNumber(uint64(blockHeight)), uint64(blockHeight))
-						}
-					}()
-				})
-			} else {
-				utils.Logger().Error().Err(err).Msg("[Explorer] Error when adding new block for explorer node")
-			}
 		}
+		// Clean up the blocks to avoid OOM.
+		node.Consensus.PbftLog.DeleteBlockByNumber(block.NumberU64())
+		// Do dump all blocks from state syncing for explorer one time
+		// TODO: some blocks can be dumped before state syncing finished.
+		// And they would be dumped again here. Please fix it.
+		once.Do(func() {
+			utils.Logger().Info().Int64("starting height", int64(block.NumberU64())-1).
+				Msg("[Explorer] Populating explorer data from state synced blocks")
+			go func() {
+				for blockHeight := int64(block.NumberU64()) - 1; blockHeight >= 0; blockHeight-- {
+					explorer.GetStorageInstance(node.SelfPeer.IP, node.SelfPeer.Port, true).Dump(
+						node.Blockchain().GetBlockByNumber(uint64(blockHeight)), uint64(blockHeight))
+				}
+			}()
+		})
+	} else {
+		utils.Logger().Error().Err(err).Msg("[Explorer] Error when adding new block for explorer node")
 	}
 }
 

node/node_genesis.go

@@ -41,7 +41,7 @@ type genesisInitializer struct {
 
 // InitChainDB sets up a new genesis block in the database for the given shard.
 func (gi *genesisInitializer) InitChainDB(db ethdb.Database, shardID uint32) error {
-	shardState := core.GetInitShardState()
+	shardState := core.CalculateInitShardState()
 	if shardID != 0 {
 		// store only the local shard for shard chains
 		c := shardState.FindCommitteeByID(shardID)
@@ -64,7 +64,7 @@ func (node *Node) SetupGenesisBlock(db ethdb.Database, shardID uint32, myShardSt
 	// Initialize genesis block and blockchain
 
 	genesisAlloc := make(core.GenesisAlloc)
-	chainConfig := params.ChainConfig{}
+	chainConfig := *params.TestnetChainConfig
 
 	switch node.NodeConfig.GetNetworkType() {
 	case nodeconfig.Mainnet:
@@ -75,8 +75,10 @@ func (node *Node) SetupGenesisBlock(db ethdb.Database, shardID uint32, myShardSt
 			genesisFunds = genesisFunds.Mul(genesisFunds, big.NewInt(denominations.One))
 			genesisAlloc[foundationAddress] = core.GenesisAccount{Balance: genesisFunds}
 		}
+	case nodeconfig.Pangaea:
+		chainConfig = *params.PangaeaChainConfig
+		fallthrough // the rest is the same as testnet
 	default: // all other types share testnet config
-		chainConfig = *params.TestnetChainConfig
 		// Tests account for txgen to use
 		node.AddTestingAddresses(genesisAlloc, TestAccountNumber)
 

node/node_handler.go

@@ -366,6 +366,7 @@ func (node *Node) PostConsensusProcessing(newBlock *types.Block, commitSigAndBit
 
 	// Update last consensus time for metrics
 	// TODO: randomly selected a few validators to broadcast messages instead of only leader broadcast
+	// TODO: refactor the asynchronous calls to separate go routine.
 	node.lastConsensusTime = time.Now().Unix()
 	if node.Consensus.PubKey.IsEqual(node.Consensus.LeaderPubKey) {
 		if node.NodeConfig.ShardID == 0 {
@@ -440,7 +441,7 @@ func (node *Node) PostConsensusProcessing(newBlock *types.Block, commitSigAndBit
 		//			ctxerror.Log15(utils.Logger().Error, e)
 		//		}
 		//	}
-		//	shardState, err := newBlockHeader.GetShardState()
+		//	shardState, err := newBlockHeader.CalculateShardState()
 		//	if err != nil {
 		//		e := ctxerror.New("cannot get shard state from header").WithCause(err)
 		//		ctxerror.Log15(utils.Logger().Error, e)
@@ -483,7 +484,7 @@ var (
 )
 
 func initGenesisCatalog() {
-	genesisShardState := core.GetInitShardState()
+	genesisShardState := core.CalculateInitShardState()
 	for _, committee := range genesisShardState {
 		for i, nodeID := range committee.NodeList {
 			genesisNode := &genesisNode{

node/node_newblock.go

@@ -131,7 +131,7 @@ func (node *Node) proposeShardStateWithoutBeaconSync(block *types.Block) shard.S
 	}
 
 	nextEpoch := new(big.Int).Add(block.Header().Epoch(), common.Big1)
-	return core.GetShardState(nextEpoch)
+	return core.CalculateShardState(nextEpoch)
 }
 
 func (node *Node) proposeShardState(block *types.Block) error {

node/node_syncing.go

@@ -190,7 +190,7 @@ func (node *Node) DoBeaconSyncing() {
 				continue
 			}
 		}
-		node.beaconSync.SyncLoop(node.Beaconchain(), node.BeaconWorker, false, true)
+		node.beaconSync.SyncLoop(node.Beaconchain(), node.BeaconWorker, true)
 		time.Sleep(BeaconSyncFrequency * time.Second)
 	}
 }
@@ -230,7 +230,10 @@ SyncingLoop:
 			if willJoinConsensus {
 				node.Consensus.BlocksNotSynchronized()
 			}
-			node.stateSync.SyncLoop(bc, worker, willJoinConsensus, false)
+			node.stateSync.SyncLoop(bc, worker, false)
+			if node.NodeConfig.Role() == nodeconfig.ExplorerNode {
+				node.Consensus.UpdateConsensusInformation()
+			}
 			if willJoinConsensus {
 				node.stateMutex.Lock()
 				node.State = NodeReadyForConsensus
@@ -324,7 +327,7 @@ func (node *Node) SendNewBlockToUnsync() {
 func (node *Node) CalculateResponse(request *downloader_pb.DownloaderRequest, incomingPeer string) (*downloader_pb.DownloaderResponse, error) {
 	response := &downloader_pb.DownloaderResponse{}
 	switch request.Type {
-	case downloader_pb.DownloaderRequest_HEADER:
+	case downloader_pb.DownloaderRequest_BLOCKHASH:
 		if request.BlockHash == nil {
 			return response, fmt.Errorf("[SYNC] GetBlockHashes Request BlockHash is NIL")
 		}
@@ -361,9 +364,24 @@ func (node *Node) CalculateResponse(request *downloader_pb.DownloaderRequest, in
 			response.Payload = append(response.Payload, blockHash[:])
 		}
 
+	case downloader_pb.DownloaderRequest_BLOCKHEADER:
+		var hash common.Hash
+		for _, bytes := range request.Hashes {
+			hash.SetBytes(bytes)
+			blockHeader := node.Blockchain().GetHeaderByHash(hash)
+			if blockHeader == nil {
+				continue
+			}
+			encodedBlockHeader, err := rlp.EncodeToBytes(blockHeader)
+
+			if err == nil {
+				response.Payload = append(response.Payload, encodedBlockHeader)
+			}
+		}
+
 	case downloader_pb.DownloaderRequest_BLOCK:
+		var hash common.Hash
 		for _, bytes := range request.Hashes {
-			var hash common.Hash
 			hash.SetBytes(bytes)
 			block := node.Blockchain().GetBlockByHash(hash)
 			if block == nil {

node/rpc.go

@@ -10,6 +10,7 @@ import (
 	"github.com/ethereum/go-ethereum/rpc"
 
 	"github.com/harmony-one/harmony/hmy"
+	nodeconfig "github.com/harmony-one/harmony/internal/configs/node"
 	"github.com/harmony-one/harmony/internal/hmyapi"
 	"github.com/harmony-one/harmony/internal/hmyapi/filters"
 	"github.com/harmony-one/harmony/internal/utils"
@@ -32,7 +33,7 @@ var (
 	httpEndpoint = ""
 	wsEndpoint   = ""
 
-	httpModules      = []string{"hmy", "net"}
+	httpModules      = []string{"hmy", "net", "explorer"}
 	httpVirtualHosts = []string{"*"}
 	httpTimeouts     = rpc.DefaultHTTPTimeouts
 	httpOrigins      = []string{"*"}
@@ -56,12 +57,17 @@ func (node *Node) StartRPC(nodePort string) error {
 
 	port, _ := strconv.Atoi(nodePort)
 
-	httpEndpoint = fmt.Sprintf(":%v", port+rpcHTTPPortOffset)
+	ip := ""
+	if !nodeconfig.GetPublicRPC() {
+		ip = "127.0.0.1"
+	}
+	httpEndpoint = fmt.Sprintf("%v:%v", ip, port+rpcHTTPPortOffset)
+
 	if err := node.startHTTP(httpEndpoint, apis, httpModules, httpOrigins, httpVirtualHosts, httpTimeouts); err != nil {
 		return err
 	}
 
-	wsEndpoint = fmt.Sprintf(":%v", port+rpcWSPortOffset)
+	wsEndpoint = fmt.Sprintf("%v:%v", ip, port+rpcWSPortOffset)
 	if err := node.startWS(wsEndpoint, apis, wsModules, wsOrigins, true); err != nil {
 		node.stopHTTP()
 		return err

node/service_setup.go

@@ -1,6 +1,8 @@
 package node
 
 import (
+	"fmt"
+
 	msg_pb "github.com/harmony-one/harmony/api/proto/message"
 	"github.com/harmony-one/harmony/api/service"
 	"github.com/harmony-one/harmony/api/service/blockproposal"
@@ -22,7 +24,7 @@ func (node *Node) setupForValidator() {
 	// Register peer discovery service. No need to do staking for beacon chain node.
 	node.serviceManager.RegisterService(service.PeerDiscovery, discovery.New(node.host, nodeConfig, chanPeer, node.AddBeaconPeer))
 	// Register networkinfo service. "0" is the beacon shard ID
-	node.serviceManager.RegisterService(service.NetworkInfo, networkinfo.New(node.host, node.NodeConfig.GetShardGroupID(), chanPeer, nil))
+	node.serviceManager.RegisterService(service.NetworkInfo, networkinfo.MustNew(node.host, node.NodeConfig.GetShardGroupID(), chanPeer, nil, node.networkInfoDHTPath()))
 	// Register consensus service.
 	node.serviceManager.RegisterService(service.Consensus, consensus.New(node.BlockChannel, node.Consensus, node.startConsensus))
 	// Register new block service.
@@ -51,7 +53,7 @@ func (node *Node) setupForNewNode() {
 	// Register peer discovery service. "0" is the beacon shard ID
 	node.serviceManager.RegisterService(service.PeerDiscovery, discovery.New(node.host, nodeConfig, chanPeer, node.AddBeaconPeer))
 	// Register networkinfo service. "0" is the beacon shard ID
-	node.serviceManager.RegisterService(service.NetworkInfo, networkinfo.New(node.host, node.NodeConfig.GetBeaconGroupID(), chanPeer, nil))
+	node.serviceManager.RegisterService(service.NetworkInfo, networkinfo.MustNew(node.host, node.NodeConfig.GetBeaconGroupID(), chanPeer, nil, node.networkInfoDHTPath()))
 	// Register new metrics service
 	if node.NodeConfig.GetMetricsFlag() {
 		node.serviceManager.RegisterService(service.Metrics, metrics.New(&node.SelfPeer, node.NodeConfig.ConsensusPubKey.SerializeToHexStr(), node.NodeConfig.GetPushgatewayIP(), node.NodeConfig.GetPushgatewayPort()))
@@ -60,7 +62,7 @@ func (node *Node) setupForNewNode() {
 
 func (node *Node) setupForClientNode() {
 	// Register networkinfo service. "0" is the beacon shard ID
-	node.serviceManager.RegisterService(service.NetworkInfo, networkinfo.New(node.host, p2p.GroupIDBeacon, nil, nil))
+	node.serviceManager.RegisterService(service.NetworkInfo, networkinfo.MustNew(node.host, p2p.GroupIDBeacon, nil, nil, node.networkInfoDHTPath()))
 }
 
 func (node *Node) setupForExplorerNode() {
@@ -69,7 +71,7 @@ func (node *Node) setupForExplorerNode() {
 	// Register peer discovery service.
 	node.serviceManager.RegisterService(service.PeerDiscovery, discovery.New(node.host, nodeConfig, chanPeer, nil))
 	// Register networkinfo service.
-	node.serviceManager.RegisterService(service.NetworkInfo, networkinfo.New(node.host, node.NodeConfig.GetShardGroupID(), chanPeer, nil))
+	node.serviceManager.RegisterService(service.NetworkInfo, networkinfo.MustNew(node.host, node.NodeConfig.GetShardGroupID(), chanPeer, nil, node.networkInfoDHTPath()))
 	// Register explorer service.
 	node.serviceManager.RegisterService(service.SupportExplorer, explorer.New(&node.SelfPeer, node.NodeConfig.GetShardID(), node.Consensus.GetNodeIDs, node.GetBalanceOfAddress))
 	// Register explorer service.
@@ -107,3 +109,11 @@ func (node *Node) StopServices() {
 	}
 	node.serviceManager.StopServicesByRole([]service.Type{})
 }
+
+func (node *Node) networkInfoDHTPath() string {
+	return fmt.Sprintf(".dht-%s-%s-c%s",
+		node.NodeConfig.SelfPeer.IP,
+		node.NodeConfig.SelfPeer.Port,
+		node.chainConfig.ChainID,
+	)
+}

node/worker/worker.go

@@ -50,8 +50,6 @@ type Worker struct {
 
 	gasFloor uint64
 	gasCeil  uint64
-
-	shardID uint32
 }
 
 // Returns a tuple where the first value is the txs sender account address,
@@ -114,7 +112,7 @@ func (w *Worker) SelectTransactionsForNewBlock(newBlockNum uint64, txs types.Tra
 	unselected := types.Transactions{}
 	invalid := types.Transactions{}
 	for _, tx := range txs {
-		if tx.ShardID() != w.shardID {
+		if tx.ShardID() != w.chain.ShardID() {
 			invalid = append(invalid, tx)
 			continue
 		}
@@ -301,7 +299,7 @@ func (w *Worker) ProposeShardStateWithoutBeaconSync() shard.State {
 		return nil
 	}
 	nextEpoch := new(big.Int).Add(w.current.header.Epoch(), common.Big1)
-	return core.GetShardState(nextEpoch)
+	return core.CalculateShardState(nextEpoch)
 }
 
 // FinalizeNewBlock generate a new block for the next consensus round.
@@ -353,7 +351,7 @@ func (w *Worker) FinalizeNewBlock(sig []byte, signers []byte, viewID uint64, coi
 }
 
 // New create a new worker object.
-func New(config *params.ChainConfig, chain *core.BlockChain, engine consensus_engine.Engine, shardID uint32) *Worker {
+func New(config *params.ChainConfig, chain *core.BlockChain, engine consensus_engine.Engine) *Worker {
 	worker := &Worker{
 		config:  config,
 		factory: blockfactory.NewFactory(config),
@@ -362,7 +360,6 @@ func New(config *params.ChainConfig, chain *core.BlockChain, engine consensus_en
 	}
 	worker.gasFloor = 500000000000000000
 	worker.gasCeil = 1000000000000000000
-	worker.shardID = shardID
 
 	parent := worker.chain.CurrentBlock()
 	num := parent.Number()

node/worker/worker_test.go

@@ -44,7 +44,7 @@ func TestNewWorker(t *testing.T) {
 	chain, _ := core.NewBlockChain(database, nil, gspec.Config, chain2.Engine, vm.Config{}, nil)
 
 	// Create a new worker
-	worker := New(params.TestChainConfig, chain, chain2.Engine, 0)
+	worker := New(params.TestChainConfig, chain, chain2.Engine)
 
 	if worker.GetCurrentState().GetBalance(crypto.PubkeyToAddress(testBankKey.PublicKey)).Cmp(testBankFunds) != 0 {
 		t.Error("Worker state is not setup correctly")
@@ -67,7 +67,7 @@ func TestCommitTransactions(t *testing.T) {
 	chain, _ := core.NewBlockChain(database, nil, gspec.Config, chain2.Engine, vm.Config{}, nil)
 
 	// Create a new worker
-	worker := New(params.TestChainConfig, chain, chain2.Engine, 0)
+	worker := New(params.TestChainConfig, chain, chain2.Engine)
 
 	// Generate a test tx
 	baseNonce := worker.GetCurrentState().GetNonce(crypto.PubkeyToAddress(testBankKey.PublicKey))

scripts/go_executable_build.sh

@@ -94,6 +94,8 @@ function build_only
    BUILTBY=${USER}@
    local build=$1
 
+   set -e
+
    for bin in "${!SRC[@]}"; do
       if [[ -z "$build" || "$bin" == "$build" ]]; then
          rm -f $BINDIR/$bin

scripts/node.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 unset -v progname
 progname="${0##*/}"
@@ -108,11 +108,28 @@ usage: ${progname} [-1ch] [-k KEYFILE]
    -t             equivalent to -N pangaea (deprecated)
    -T nodetype    specify the node type (validator, explorer; default: validator)
    -i shardid     specify the shard id (valid only with explorer node; default: 1)
+   -b             download harmony_db files from shard specified by -i <shardid> (default: off)
+   -a dbfile      specify the db file to download (default:off)
+   -U FOLDER      specify the upgrade folder to download binaries
 
-example:
+examples:
 
+# start node program w/o root account
    ${progname} -S -k mybls.key
 
+# download beacon chain (shard0) db snapshot
+   ${progname} -i 0 -b
+
+# just re-download the harmony binaries
+   ${progname} -d
+
+# start a non-validating node in shard 1
+# you need to have a dummy BLSKEY/pass file using 'touch BLSKEY; touch blspass'
+   ${progname} -S -k BLSKEY -p blspass -T explorer -i 1
+
+# upgrade harmony binaries from specified repo
+   ${progname} -1 -U upgrade
+
 ENDEND
 }
 
@@ -126,7 +143,8 @@ usage() {
 BUCKET=pub.harmony.one
 OS=$(uname -s)
 
-unset start_clean loop run_as_root blspass do_not_download metrics network
+unset start_clean loop run_as_root blspass do_not_download download_only metrics network node_type shard_id download_harmony_db db_file_to_dl
+unset upgrade_rel
 start_clean=false
 loop=true
 run_as_root=true
@@ -136,15 +154,17 @@ metrics=false
 network=main
 node_type=validator
 shard_id=1
+download_harmony_db=false
 ${BLSKEYFILE=}
 
 unset OPTIND OPTARG opt
 OPTIND=1
-while getopts :1chk:sSp:dDmN:tT:i: opt
+while getopts :1chk:sSp:dDmN:tT:i:ba:U: opt
 do
    case "${opt}" in
    '?') usage "unrecognized option -${OPTARG}";;
    ':') usage "missing argument for -${OPTARG}";;
+   b) download_harmony_db=true;;
    c) start_clean=true;;
    1) loop=false;;
    h) print_usage; exit 0;;
@@ -159,6 +179,8 @@ do
    t) network=pangaea;;
    T) node_type="${OPTARG}";;
    i) shard_id="${OPTARG}";;
+   a) db_file_to_dl="${OPTARG}";;
+   U) upgrade_rel="${OPTARG}";;
    *) err 70 "unhandled option -${OPTARG}";;  # EX_SOFTWARE
    esac
 done
@@ -214,6 +236,11 @@ case $# in
    ;;
 esac
 
+# reset REL if upgrade_rel is set
+if [ -n "$upgrade_rel" ]; then
+   REL="${upgrade_rel}"
+fi
+
 if [ "$OS" == "Darwin" ]; then
    FOLDER=release/darwin-x86_64/$REL/
    BIN=( harmony libbls384_256.dylib libcrypto.1.0.0.dylib libgmp.10.dylib libgmpxx.4.dylib libmcl.dylib md5sum.txt )
@@ -248,7 +275,7 @@ verify_checksum() {
    checksum_file="${3}"
    [ -f "${dir}/${checksum_file}" ] || return 0
    checksum_for_file="${dir}/${checksum_file}::${file}"
-   extract_checksum "${file}" < "${dir}/${checksum_file}" > "${dir}/${checksum_for_file}"
+   extract_checksum "${file}" < "${dir}/${checksum_file}" > "${checksum_for_file}"
    [ -s "${dir}/${checksum_for_file}" ] || return 0
    if ! (cd "${dir}" && exec md5sum -c --status "${checksum_for_file}")
    then
@@ -272,8 +299,105 @@ download_binaries() {
    (cd "${outdir}" && exec openssl sha256 "${BIN[@]}") > "${outdir}/harmony-checksums.txt"
 }
 
+check_free_disk() {
+   local dir
+   dir="${1:-.}"
+   local free_disk=$(df -BG $dir | tail -n 1 | awk ' { print $4 } ' | tr -d G)
+   # need at least 50G free disk space
+   local need_disk=50
+
+   if [ $free_disk -gt $need_disk ]; then
+      return 0
+   else
+      return 1
+   fi
+}
+
+_curl_check_exist() {
+   local url=$1
+   local statuscode=$(curl -I --silent --output /dev/null --write-out "%{http_code}" $url)
+   if [ $statuscode -ne 200 ]; then
+      return 1
+   else
+      return 0
+   fi
+}
+
+_curl_download() {
+   local url=$1
+   local outdir=$2
+   local filename=$3
+
+   mkdir -p "${outdir}"
+   if _curl_check_exist $url; then
+      curl --progress-bar -Sf $url -o "${outdir}/$filename" || return $?
+      return 0
+   else
+      msg "failed to find/download $url"
+      return 1
+   fi
+}
+
+download_harmony_db_file() {
+   local shard_id
+   shard_id="${1}"
+   local file_to_dl="${2}"
+   local outdir=db
+   if ! check_free_disk; then
+      err 70 "do not have enough free disk space to download db tarball"
+   fi
+
+   url="http://${BUCKET}.s3.amazonaws.com/${FOLDER}db/md5sum.txt"
+   rm -f "${outdir}/md5sum.txt"
+   if ! _curl_download $url "${outdir}" md5sum.txt; then
+      err 70 "cannot download md5sum.txt"
+   fi
+
+   if [ -n "${file_to_dl}" ]; then
+      if grep -q "${file_to_dl}" "${outdir}/md5sum.txt"; then
+         url="http://${BUCKET}.s3.amazonaws.com/${FOLDER}db/${file_to_dl}"
+         if _curl_download $url "${outdir}" ${file_to_dl}; then
+            verify_checksum "${outdir}" "${file_to_dl}" md5sum.txt || return $?
+            msg "downloaded ${file_to_dl}, extracting ..."
+            tar -C "${outdir}" -xvf "${outdir}/${file_to_dl}"
+         else
+            msg "can't download ${file_to_dl}"
+         fi
+      fi
+      return
+   fi
+
+   files=$(awk '{ print $2 }' ${outdir}/md5sum.txt)
+   echo "[available harmony db files for shard ${shard_id}]"
+   grep -oE "harmony_db_${shard_id}"-.*.tar "${outdir}/md5sum.txt"
+   echo
+   for file in $files; do
+      if [[ $file =~ "harmony_db_${shard_id}" ]]; then
+         echo -n "Do you want to download ${file} (choose one only) [y/n]?"
+         read yesno
+         if [[ "$yesno" = "y" || "$yesno" = "Y" ]]; then
+            url="http://${BUCKET}.s3.amazonaws.com/${FOLDER}db/$file"
+            if _curl_download $url "${outdir}" $file; then
+               verify_checksum "${outdir}" "${file}" md5sum.txt || return $?
+               msg "downloaded $file, extracting ..."
+               tar -C "${outdir}" -xvf "${outdir}/${file}"
+            else
+               msg "can't download $file"
+            fi
+            break
+         fi
+      fi
+   done
+}
+
 if ${download_only}; then
-   download_binaries || err 69 "download node software failed"
+   download_binaries staging || err 69 "download node software failed"
+   msg "downloaded files are in staging direectory"
+   exit 0
+fi
+
+if ${download_harmony_db}; then
+   download_harmony_db_file "${shard_id}" "${db_file_to_dl}" || err 70 "download harmony_db file failed"
    exit 0
 fi
 
@@ -449,7 +573,7 @@ kill_node() {
 }
 
 {
-   while :
+   while ${loop}
    do
       msg "re-downloading binaries in 5m"
       sleep 300

test/chain/main.go

@@ -107,7 +107,7 @@ func fundFaucetContract(chain *core.BlockChain) {
 	fmt.Println("--------- Funding addresses for Faucet Contract Call ---------")
 	fmt.Println()
 
-	contractworker = pkgworker.New(params.TestChainConfig, chain, chain.Engine(), 0)
+	contractworker = pkgworker.New(params.TestChainConfig, chain, chain.Engine())
 	nonce = contractworker.GetCurrentState().GetNonce(crypto.PubkeyToAddress(FaucetPriKey.PublicKey))
 	dataEnc = common.FromHex(FaucetContractBinary)
 	ftx, _ := types.SignTx(types.NewContractCreation(nonce, 0, big.NewInt(7000000000000000000), params.TxGasContractCreation*10, nil, dataEnc), types.HomesteadSigner{}, FaucetPriKey)