Merge branch 'master' of github.com:harmony-one/harmony into beaconchain-mock

.gitignore

@@ -38,3 +38,6 @@ tmp_log
 # RLP encoding
 *.rlp
 **/*.rlp
+
+# Others
+*.png

README.md

@@ -5,7 +5,7 @@
 
 ## Coding Guidelines
 
-* In general, we should follow [effective_go](https://golang.org/doc/effective_go.html)
+* In general, we follow [effective_go](https://golang.org/doc/effective_go.html)
 * Code must adhere to the official [Go formatting guidelines](https://golang.org/doc/effective_go.html#formatting) (i.e. uses [gofmt](https://golang.org/cmd/gofmt/)).
 * Code must be documented adhering to the official Go [commentary](https://golang.org/doc/effective_go.html#commentary) guidelines.
 
@@ -15,26 +15,22 @@
 ```
 export GOPATH=$HOME/<path_of_your_choice>
 
-mkdir -p $HOME/<path_of_your_choice>/src
+mkdir -p $HOME/<path_of_your_choice>/src/github.com/harmony-one
 
-cd $HOME/<path_of_your_choice>/src
+cd $HOME/<path_of_your_choice>/src/github.com/harmony-one
 
 git clone git@github.com:harmony-one/harmony.git
 
-cd harmony-benchmark
+cd harmony
 
 go get ./...
 ```
-## Usage
 
-### Running local test without db
+## Usage
-```
-./deploy.sh local_config.txt
-```
 
-### Running local test with db
+### Running local test
 ```
-./deploy.sh local_config.txt 1
+./test/deploy.sh ./test/configs/local_config1.txt
 ```
 
 ## Testing

api/README.md

@@ -0,0 +1 @@
+The api folder contains all the Harmony protocol's interfaces definitions including protocol apis and gRpc protobuf definition.

api/proto/node/node.go

@@ -10,7 +10,6 @@ import (
 	"github.com/harmony-one/harmony/core/types"
 
 	"github.com/harmony-one/harmony/api/proto"
-	"github.com/harmony-one/harmony/p2p"
 )
 
 // MessageType is to indicate the specific type of message under Node category
@@ -74,12 +73,6 @@ const (
 	STOP ControlMessageType = iota
 )
 
-// FetchUtxoMessage is the wrapper struct FetchUtxoMessage sent from client wallet.
-type FetchUtxoMessage struct {
-	Addresses [][20]byte
-	Sender    p2p.Peer
-}
-
 // SerializeBlockchainSyncMessage serializes BlockchainSyncMessage.
 func SerializeBlockchainSyncMessage(blockchainSyncMessage *BlockchainSyncMessage) []byte {
 	var result bytes.Buffer

api/proto/node/node_test.go

@@ -0,0 +1,107 @@
+package node
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/params"
+	"github.com/harmony-one/harmony/core/state"
+	"github.com/harmony-one/harmony/core/types"
+	hdb "github.com/harmony-one/harmony/internal/db"
+
+	//	"fmt"
+	"math/big"
+	"reflect"
+	"testing"
+)
+
+var (
+	senderPriKey, _   = crypto.GenerateKey()
+	receiverPriKey, _ = crypto.GenerateKey()
+	receiverAddress   = crypto.PubkeyToAddress(receiverPriKey.PublicKey)
+
+	amountBigInt = big.NewInt(8000000000000000000)
+)
+
+func TestSerializeBlockchainSyncMessage(t *testing.T) {
+	h1 := common.HexToHash("123")
+	h2 := common.HexToHash("abc")
+
+	msg := BlockchainSyncMessage{
+		BlockHeight: 2,
+		BlockHashes: []common.Hash{
+			h1,
+			h2,
+		},
+	}
+
+	serializedByte := SerializeBlockchainSyncMessage(&msg)
+
+	dMsg, err := DeserializeBlockchainSyncMessage(serializedByte)
+
+	if err != nil || !reflect.DeepEqual(msg, *dMsg) {
+		t.Errorf("Failed to serialize/deserialize blockchain sync message\n")
+	}
+}
+
+func TestConstructTransactionListMessageAccount(t *testing.T) {
+	tx, _ := types.SignTx(types.NewTransaction(100, receiverAddress, uint32(0), amountBigInt, params.TxGas, nil, nil), types.HomesteadSigner{}, senderPriKey)
+	transactions := types.Transactions{tx}
+	buf := ConstructTransactionListMessageAccount(transactions)
+	if len(buf) == 0 {
+		t.Error("Failed to contruct transaction list message")
+	}
+}
+
+func TestConstructRequestTransactionsMessage(t *testing.T) {
+	txIDs := [][]byte{
+		[]byte{1, 2},
+		[]byte{3, 4},
+	}
+
+	buf := ConstructRequestTransactionsMessage(txIDs)
+
+	if len(buf) == 0 {
+		t.Error("Failed to contruct request transaction message")
+	}
+}
+
+func TestConstructStopMessage(t *testing.T) {
+	buf := ConstructStopMessage()
+
+	if len(buf) == 0 {
+		t.Error("Failed to contruct STOP message")
+	}
+}
+
+func TestConstructBlocksSyncMessage(t *testing.T) {
+
+	db := hdb.NewMemDatabase()
+	statedb, _ := state.New(common.Hash{}, state.NewDatabase(db))
+
+	root := statedb.IntermediateRoot(false)
+	head := &types.Header{
+		Number:  new(big.Int).SetUint64(uint64(10000)),
+		Nonce:   types.EncodeNonce(uint64(10000)),
+		ShardID: types.EncodeShardID(uint32(0)),
+		Time:    new(big.Int).SetUint64(uint64(100000)),
+		Root:    root,
+	}
+	head.GasLimit = 10000000000
+	head.Difficulty = params.GenesisDifficulty
+
+	statedb.Commit(false)
+	statedb.Database().TrieDB().Commit(root, true)
+
+	block1 := types.NewBlock(head, nil, nil)
+
+	blocks := []*types.Block{
+		block1,
+	}
+
+	buf := ConstructBlocksSyncMessage(blocks)
+
+	if len(buf) == 0 {
+		t.Error("Failed to contruct block sync message")
+	}
+
+}

api/services/explorer/storage_test.go

@@ -0,0 +1,95 @@
+package explorer
+
+import (
+	"bytes"
+	"math/big"
+	"strconv"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/harmony-one/harmony/core/types"
+	"github.com/stretchr/testify/assert"
+)
+
+// Test for GetBlockInfoKey
+func TestGetBlockInfoKey(t *testing.T) {
+	assert.Equal(t, GetBlockInfoKey(3), "bi_3", "error")
+}
+
+// Test for GetAddressKey
+func TestGetAddressKey(t *testing.T) {
+	assert.Equal(t, GetAddressKey("abcd"), "ad_abcd", "error")
+}
+
+// Test for GetBlockKey
+func TestGetBlockKey(t *testing.T) {
+	assert.Equal(t, GetBlockKey(3), "b_3", "error")
+}
+
+// Test for GetTXKey
+func TestGetTXKey(t *testing.T) {
+	assert.Equal(t, GetTXKey("abcd"), "tx_abcd", "error")
+}
+
+func TestInit(t *testing.T) {
+	ins := GetStorageInstance("1.1.1.1", "3333", true)
+	ins.GetDB().Put([]byte{1}, []byte{2})
+	value, err := ins.GetDB().Get([]byte{1})
+	assert.Equal(t, bytes.Compare(value, []byte{2}), 0, "value should be []byte{2}")
+	assert.Nil(t, err, "error should be nil")
+}
+
+func TestDump(t *testing.T) {
+	tx1 := types.NewTransaction(1, common.BytesToAddress([]byte{0x11}), 0, big.NewInt(111), 1111, big.NewInt(11111), []byte{0x11, 0x11, 0x11})
+	tx2 := types.NewTransaction(2, common.BytesToAddress([]byte{0x22}), 0, big.NewInt(222), 2222, big.NewInt(22222), []byte{0x22, 0x22, 0x22})
+	tx3 := types.NewTransaction(3, common.BytesToAddress([]byte{0x33}), 0, big.NewInt(333), 3333, big.NewInt(33333), []byte{0x33, 0x33, 0x33})
+	txs := []*types.Transaction{tx1, tx2, tx3}
+
+	block := types.NewBlock(&types.Header{Number: big.NewInt(314)}, txs, nil)
+	ins := GetStorageInstance("1.1.1.1", "3333", true)
+	ins.Dump(block, uint32(1))
+	db := ins.GetDB()
+
+	res, err := db.Get([]byte(BlockHeightKey))
+	if err == nil {
+		toInt, err := strconv.Atoi(string(res))
+		assert.Equal(t, toInt, 1, "error")
+		assert.Nil(t, err, "error")
+	} else {
+		t.Error("Error")
+	}
+
+	data, err := db.Get([]byte(GetBlockKey(1)))
+	assert.Nil(t, err, "should be nil")
+	blockData, err := rlp.EncodeToBytes(block)
+	assert.Nil(t, err, "should be nil")
+	assert.Equal(t, bytes.Compare(data, blockData), 0, "should be equal")
+}
+
+func TestUpdateAddressStorage(t *testing.T) {
+	tx1 := types.NewTransaction(1, common.BytesToAddress([]byte{0x11}), 0, big.NewInt(111), 1111, big.NewInt(11111), []byte{0x11, 0x11, 0x11})
+	tx2 := types.NewTransaction(2, common.BytesToAddress([]byte{0x22}), 0, big.NewInt(222), 2222, big.NewInt(22222), []byte{0x22, 0x22, 0x22})
+	tx3 := types.NewTransaction(3, common.BytesToAddress([]byte{0x33}), 0, big.NewInt(333), 3333, big.NewInt(33333), []byte{0x33, 0x33, 0x33})
+	txs := []*types.Transaction{tx1, tx2, tx3}
+
+	block := types.NewBlock(&types.Header{Number: big.NewInt(314)}, txs, nil)
+	ins := GetStorageInstance("1.1.1.1", "3333", true)
+	ins.Dump(block, uint32(1))
+	db := ins.GetDB()
+
+	res, err := db.Get([]byte(BlockHeightKey))
+	if err == nil {
+		toInt, err := strconv.Atoi(string(res))
+		assert.Equal(t, toInt, 1, "error")
+		assert.Nil(t, err, "error")
+	} else {
+		t.Error("Error")
+	}
+
+	data, err := db.Get([]byte(GetBlockKey(1)))
+	assert.Nil(t, err, "should be nil")
+	blockData, err := rlp.EncodeToBytes(block)
+	assert.Nil(t, err, "should be nil")
+	assert.Equal(t, bytes.Compare(data, blockData), 0, "should be equal")
+}

api/services/explorer/structs_test.go

@@ -0,0 +1,27 @@
+package explorer
+
+import (
+	"math/big"
+	"strconv"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/harmony-one/harmony/core/types"
+)
+
+// Test for GetBlockInfoKey
+func TestGetTransaction(t *testing.T) {
+	tx1 := types.NewTransaction(1, common.BytesToAddress([]byte{0x11}), 0, big.NewInt(111), 1111, big.NewInt(11111), []byte{0x11, 0x11, 0x11})
+	tx2 := types.NewTransaction(2, common.BytesToAddress([]byte{0x22}), 0, big.NewInt(222), 2222, big.NewInt(22222), []byte{0x22, 0x22, 0x22})
+	tx3 := types.NewTransaction(3, common.BytesToAddress([]byte{0x33}), 0, big.NewInt(333), 3333, big.NewInt(33333), []byte{0x33, 0x33, 0x33})
+	txs := []*types.Transaction{tx1, tx2, tx3}
+
+	block := types.NewBlock(&types.Header{Number: big.NewInt(314)}, txs, nil)
+
+	tx := GetTransaction(tx1, block)
+	assert.Equal(t, tx.ID, tx1.Hash().Hex(), "should be equal tx1.Hash()")
+	assert.Equal(t, tx.To, tx1.To().Hex(), "should be equal tx1.To()")
+	assert.Equal(t, tx.Bytes, strconv.Itoa(int(tx1.Size())), "should be equal tx1.Size()")
+}

cmd/client/txgen/README.md

@@ -0,0 +1 @@
+The txgen program is used to simulate transactions and hit the Harmony network to loadtest its performance and robustness.

cmd/client/txgen/main.go

@@ -27,7 +27,7 @@ var (
 	builtBy    string
 	builtAt    string
 	commit     string
-	utxoPoolMutex sync.Mutex
+	stateMutex sync.Mutex
 )
 
 func printVersion(me string) {
@@ -93,7 +93,7 @@ func main() {
 	)
 	log.Root().SetHandler(h)
 
-	// Nodes containing utxopools to mirror the shards' data in the network
+	// Nodes containing blockchain data to mirror the shards' data in the network
 	nodes := []*node.Node{}
 	for shardID := range shardIDLeaderMap {
 		_, pubKey := utils.GenKey(clientPeer.IP, clientPeer.Port)
@@ -110,15 +110,15 @@ func main() {
 	clientNode := node.New(host, consensusObj, nil)
 	clientNode.Client = client.NewClient(clientNode.GetHost(), &shardIDLeaderMap)
 
-	// This func is used to update the client's utxopool when new blocks are received from the leaders
 	readySignal := make(chan uint32)
 	go func() {
 		for i := range shardIDLeaderMap {
 			readySignal <- i
 		}
 	}()
+	// This func is used to update the client's blockchain when new blocks are received from the leaders
 	updateBlocksFunc := func(blocks []*types.Block) {
-		log.Info("RECEIVED BLOCK", "block", blocks)
+		log.Info("[Txgen] Received new block", "block", blocks)
 		for _, block := range blocks {
 			for _, node := range nodes {
 				shardID := block.ShardID()
@@ -128,9 +128,9 @@ func main() {
 					log.Info("Current Block", "hash", node.Blockchain().CurrentBlock().Hash().Hex())
 					log.Info("Adding block from leader", "txNum", len(block.Transactions()), "shardID", shardID, "preHash", block.ParentHash().Hex())
 					node.AddNewBlock(block)
-					utxoPoolMutex.Lock()
+					stateMutex.Lock()
 					node.Worker.UpdateCurrent()
-					utxoPoolMutex.Unlock()
+					stateMutex.Unlock()
 					readySignal <- shardID
 				} else {
 					continue
@@ -168,17 +168,15 @@ func main() {
 			shardIDTxsMap := make(map[uint32]types.Transactions)
 			lock := sync.Mutex{}
 
-			utxoPoolMutex.Lock()
+			stateMutex.Lock()
 			log.Warn("STARTING TX GEN", "gomaxprocs", runtime.GOMAXPROCS(0))
 			txs, _ := txgen.GenerateSimulatedTransactionsAccount(int(shardID), nodes, setting)
 
-			// TODO: Put cross shard tx into a pending list waiting for proofs from leaders
-
 			lock.Lock()
 			// Put txs into corresponding shards
 			shardIDTxsMap[shardID] = append(shardIDTxsMap[shardID], txs...)
 			lock.Unlock()
-			utxoPoolMutex.Unlock()
+			stateMutex.Unlock()
 
 			lock.Lock()
 			for shardID, txs := range shardIDTxsMap { // Send the txs to corresponding shards

cmd/client/wallet/README.md

@@ -0,0 +1 @@
+The wallet program is the demo wallet which talks to Harmony devnet for various kinds of operations. For detail, please compile and execute ./bin/wallet.

consensus/consensus_engine.go

@@ -9,6 +9,7 @@ import (
 
 // ChainReader defines a small collection of methods needed to access the local
 // blockchain during header and/or uncle verification.
+// Note this reader interface is still in process of being integrated with the BFT consensus.
 type ChainReader interface {
 	// Config retrieves the blockchain's chain configuration.
 	Config() *params.ChainConfig
@@ -30,6 +31,7 @@ type ChainReader interface {
 }
 
 // Engine is an algorithm agnostic consensus engine.
+// Note this engine interface is still in process of being integrated with the BFT consensus.
 type Engine interface {
 	// Author retrieves the Harmony address of the account that validated the given
 	// block.

coverage.sh

@@ -0,0 +1,3 @@
+go test ./... -coverprofile=/tmp/coverage.out;
+grep -v "harmony-one/harmony/core" /tmp/coverage.out | grep -v "harmony-one/harmony/internal/trie" | grep -v "harmony-one/harmony/internal/db" | grep -v "harmony-one/harmony/log"  > /tmp/coverage1.out
+go tool cover -func=/tmp/coverage1.out

internal/utils/bytes.go

@@ -2,18 +2,6 @@ package utils
 
 import "encoding/hex"
 
-// ToHex returns the hex representation of b, prefixed with '0x'.
-// For empty slices, the return value is "0x0".
-//
-// Deprecated: use hexutil.Encode instead.
-func ToHex(b []byte) string {
-	hex := Bytes2Hex(b)
-	if len(hex) == 0 {
-		hex = "0"
-	}
-	return "0x" + hex
-}
-
 // FromHex returns the bytes represented by the hexadecimal string s.
 // s may be prefixed with "0x".
 func FromHex(s string) []byte {
@@ -39,11 +27,6 @@ func CopyBytes(b []byte) (copiedBytes []byte) {
 	return
 }
 
-// hasHexPrefix validates str begins with '0x' or '0X'.
-func hasHexPrefix(str string) bool {
-	return len(str) >= 2 && str[0] == '0' && (str[1] == 'x' || str[1] == 'X')
-}
-
 // isHexCharacter returns bool of c being a valid hexadecimal.
 func isHexCharacter(c byte) bool {
 	return ('0' <= c && c <= '9') || ('a' <= c && c <= 'f') || ('A' <= c && c <= 'F')
@@ -62,31 +45,12 @@ func isHex(str string) bool {
 	return true
 }
 
-// Bytes2Hex returns the hexadecimal encoding of d.
-func Bytes2Hex(d []byte) string {
-	return hex.EncodeToString(d)
-}
-
 // Hex2Bytes returns the bytes represented by the hexadecimal string str.
 func Hex2Bytes(str string) []byte {
 	h, _ := hex.DecodeString(str)
 	return h
 }
 
-// Hex2BytesFixed returns bytes of a specified fixed length flen.
-func Hex2BytesFixed(str string, flen int) []byte {
-	h, _ := hex.DecodeString(str)
-	if len(h) == flen {
-		return h
-	}
-	if len(h) > flen {
-		return h[len(h)-flen:]
-	}
-	hh := make([]byte, flen)
-	copy(hh[flen-len(h):flen], h[:])
-	return hh
-}
-
 // RightPadBytes zero-pads slice to the right up to length l.
 func RightPadBytes(slice []byte, l int) []byte {
 	if l <= len(slice) {
@@ -110,14 +74,3 @@ func LeftPadBytes(slice []byte, l int) []byte {
 
 	return padded
 }
-
-// Get32BytesFromString parses the string representation of hex into 32 byte array
-func Get32BytesFromString(hashString string) ([32]byte, error) {
-	bytes, err := hex.DecodeString(hashString)
-	if err != nil {
-		return [32]byte{}, err
-	}
-	bytesArray := [32]byte{}
-	copy(bytesArray[:], bytes)
-	return bytesArray, nil
-}

internal/utils/bytes_test.go

@@ -87,3 +87,15 @@ func TestNoPrefixShortHexOddLength(t *testing.T) {
 		t.Errorf("Expected %x got %x", expected, result)
 	}
 }
+
+func TestCopyBytes(t *testing.T) {
+	expected := []byte{1, 2, 3}
+	result := CopyBytes(expected)
+	if !bytes.Equal(expected, result) {
+		t.Errorf("Expected %x got %x", expected, result)
+	}
+	expected[0] = 0
+	if result[0] == 0 {
+		t.Errorf("should not be 0")
+	}
+}

node/node_handler.go

@@ -27,7 +27,7 @@ const (
 
 // MaybeBroadcastAsValidator returns if the node is a validator node.
 func (node *Node) MaybeBroadcastAsValidator(content []byte) {
-	// TODO: this is tree broadcasting. this needs to be removed later. Actually the whole logic needs to be replaced by p2p.
+	// TODO: this is tree-based broadcasting. this needs to be replaced by p2p gossiping.
 	if node.SelfPeer.ValidatorID > 0 && node.SelfPeer.ValidatorID <= host.MaxBroadCast {
 		go host.BroadcastMessageFromValidator(node.host, node.SelfPeer, node.Consensus.GetValidatorPeers(), content)
 	}
@@ -75,8 +75,6 @@ func (node *Node) StreamHandler(s p2p.Stream) {
 			switch messageType {
 			case proto_identity.Register:
 				fmt.Println("received a identity message")
-				// TODO(ak): fix it.
-				// node.processPOWMessage(msgPayload)
 				node.log.Info("NET: received message: IDENTITY/REGISTER")
 			default:
 				node.log.Error("Announce message should be sent to IdentityChain")

p2p/helper.go

@@ -57,36 +57,16 @@ func ReadMessageContent(s Stream) ([]byte, error) {
 		log.Error("Failed reading the p2p message size field: only read", "Num of bytes", n)
 		return contentBuf.Bytes(), err
 	}
-	//log.Print(fourBytes)
+
-	// Number of bytes for the message content
+	contentLength := int(binary.BigEndian.Uint32(fourBytes))
-	bytesToRead := binary.BigEndian.Uint32(fourBytes)
+	tmpBuf := make([]byte, contentLength)
-	//log.Printf("The content size is %d bytes.", bytesToRead)
+	timeoutDuration = 20 * time.Second
-	//// Read the content in chunk of 16 * 1024 bytes
-	tmpBuf := make([]byte, BatchSizeInByte)
-ILOOP:
-	for {
-		timeoutDuration := 10 * time.Second
 	s.SetReadDeadline(time.Now().Add(timeoutDuration))
-		if bytesToRead < BatchSizeInByte {
+	m, err := io.ReadFull(r, tmpBuf)
-			// Read the last number of bytes less than 1024
+	if err != nil || m < contentLength {
-			tmpBuf = make([]byte, bytesToRead)
+		log.Error("Read %v bytes, we need %v bytes", m, contentLength)
-		}
-		n, err := r.Read(tmpBuf)
-		contentBuf.Write(tmpBuf[:n])
-		switch err {
-		case io.EOF:
-			// TODO: should we return error here, or just ignore it?
-			log.Error("EOF reached while reading p2p message")
-			break ILOOP
-		case nil:
-			bytesToRead -= uint32(n) // TODO: think about avoid the casting in every loop
-			if bytesToRead <= 0 {
-				break ILOOP
-			}
-		default:
-			log.Error("Error reading p2p message")
 		return []byte{}, err
 	}
-	}
+	contentBuf.Write(tmpBuf)
 	return contentBuf.Bytes(), nil
 }