diff --git a/client/txgen/main.go b/client/txgen/main.go index d7141678e..e77567a91 100644 --- a/client/txgen/main.go +++ b/client/txgen/main.go @@ -33,6 +33,7 @@ type txGenSettings struct { numOfAddress int crossShard bool maxNumTxsPerBatch int + crossShardRatio int } var ( @@ -115,15 +116,13 @@ UTXOLOOP: randNum := rand.Intn(100) - if randNum < 30 { - if setting.crossShard && randNum < 10 { // 1/3 cross shard transactions: add another txinput from another shard - generateCrossShardTx(&txInfo) - } else { - generateSingleShardTx(&txInfo) - } - if txInfo.txCount >= setting.maxNumTxsPerBatch { - break UTXOLOOP - } + if setting.crossShard && randNum < setting.crossShardRatio { // 30% cross shard transactions: add another txinput from another shard + generateCrossShardTx(&txInfo) + } else { + generateSingleShardTx(&txInfo) + } + if txInfo.txCount >= setting.maxNumTxsPerBatch { + break UTXOLOOP } } } @@ -244,8 +243,9 @@ func main() { maxNumTxsPerBatch := flag.Int("max_num_txs_per_batch", 10000, "number of transactions to send per message") logFolder := flag.String("log_folder", "latest", "the folder collecting the logs of this execution") numSubset := flag.Int("numSubset", 3, "the number of subsets of utxos to process separately") - duration := flag.Int("duration", 60, "duration of the tx generation in second") + duration := flag.Int("duration", 60, "duration of the tx generation in second. If it's negative, the experiment runs forever.") versionFlag := flag.Bool("version", false, "Output version info") + crossShardRatio := flag.Int("cross_shard_ratio", 30, "The percentage of cross shard transactions.") flag.Parse() if *versionFlag { @@ -261,6 +261,7 @@ func main() { // Do cross shard tx if there are more than one shard setting.crossShard = len(shardIds) > 1 setting.maxNumTxsPerBatch = *maxNumTxsPerBatch + setting.crossShardRatio = *crossShardRatio // TODO(Richard): refactor this chuck to a single method // Setup a logger to stdout and log file. @@ -322,7 +323,7 @@ func main() { batchCounter := 0 for true { t := time.Now() - if t.Sub(start).Seconds() >= totalTime { + if totalTime > 0 && t.Sub(start).Seconds() >= totalTime { log.Debug("Generator timer ended.", "duration", (int(t.Sub(start))), "startTime", start, "totalTime", totalTime) break } diff --git a/go_executable_build.sh b/go_executable_build.sh index f6fe973e4..22c9a3ae4 100755 --- a/go_executable_build.sh +++ b/go_executable_build.sh @@ -10,6 +10,7 @@ BINDIR=bin BUCKET=unique-bucket-bin GOOS=linux GOARCH=amd64 +FOLDER= function usage { @@ -19,11 +20,12 @@ function usage Usage: $ME [OPTIONS] ACTION OPTIONS: - -h print this help message - -p profile aws profile name - -a arch set build arch (default: $GOARCH) - -o os set build OS (default: $GOOS, windows is supported) - -b bucket set the upload bucket name (default: $BUCKET) + -h print this help message + -p profile aws profile name + -a arch set build arch (default: $GOARCH) + -o os set build OS (default: $GOOS, windows is supported) + -b bucket set the upload bucket name (default: $BUCKET) + -f folder set the upload folder name in the bucket (default: $FOLDER) ACTION: build build binaries only (default action) @@ -31,14 +33,14 @@ ACTION: EXAMPLES: -# build linux binaries by default +# build linux binaries only by default $ME # build windows binaries $ME -o windows -# upload binaries to my s3 bucket - $ME -b mybucket upload +# upload binaries to my s3 bucket, 0908 folder + $ME -b mybucket -f 0908 upload EOF exit 1 @@ -68,19 +70,20 @@ function upload fi for bin in "${!SRC[@]}"; do - [ -e $BINDIR/$bin ] && $AWSCLI s3 cp $BINDIR/$bin s3://$BUCKET/$bin --acl public-read + [ -e $BINDIR/$bin ] && $AWSCLI s3 cp $BINDIR/$bin s3://${BUCKET}$FOLDER/$bin --acl public-read done - [ -e $BINDIR/md5sum.txt ] && $AWSCLI s3 cp $BINDIR/md5sum.txt s3://$BUCKET/md5sum.txt --acl public-read + [ -e $BINDIR/md5sum.txt ] && $AWSCLI s3 cp $BINDIR/md5sum.txt s3://${BUCKET}$FOLDER/md5sum.txt --acl public-read } ################################ MAIN FUNCTION ############################## -while getopts "hp:a:o:b:" option; do +while getopts "hp:a:o:b:f:" option; do case $option in h) usage ;; p) PROFILE=$OPTARG ;; a) GOARCH=$OPTARG ;; o) GOOS=$OPTARG ;; - b) BUCKET=$OPTARG ;; + b) BUCKET=$OPTARG/ ;; + f) FOLDER=$OPTARG ;; esac done diff --git a/identitychain/identitychain.go b/identitychain/identitychain.go index c84cb412d..5f973ca5f 100644 --- a/identitychain/identitychain.go +++ b/identitychain/identitychain.go @@ -18,7 +18,8 @@ var identityPerBlock = 100000 // IdentityChain (Blockchain) keeps Identities per epoch, currently centralized! type IdentityChain struct { - Identities []*IdentityBlock + //Identities []*IdentityBlock //No need to have the identity block as of now + Identities []*node.Node PendingIdentities []*node.Node log log.Logger Peer p2p.Peer @@ -51,16 +52,21 @@ type GlobalBlockchainConfig struct { //Shard func (IDC *IdentityChain) Shard() { + + IDC.SelectIds() IDC.CreateShardAssignment() IDC.ElectLeaders() IDC.BroadCastNewConfiguration() } -// +//ElectLeaders func (IDC *IdentityChain) ElectLeaders() { + return } +//BroadCastNewConfiguration func (IDC *IdentityChain) BroadCastNewConfiguration() { + fmt.Println("Broadcasting leader and shard info to everyone!") // allPeers := make([]p2p.Peer, len(IDC.SelectedIdentitites)) // msgToSend := proto. // p2p.BroadCastMessage(allPeers, msgToSend) @@ -71,34 +77,38 @@ func (IDC *IdentityChain) BroadCastNewConfiguration() { func (IDC *IdentityChain) CreateShardAssignment() { num := seekRandomNumber(IDC.EpochNum, IDC.SelectedIdentitites) IDC.NumberOfShards = IDC.NumberOfShards + needNewShards() - IDC.SelectedIdentitites = generateRandomPermutations(num, IDC.SelectedIdentitites) + IDC.generateRandomPermutations(num) IDC.PeerToShardMap = make(map[*node.Node]int) numberInOneShard := len(IDC.SelectedIdentitites) / IDC.NumberOfShards - for peerNum := 1; peerNum <= len(IDC.SelectedIdentitites); peerNum++ { + fmt.Println(len(IDC.SelectedIdentitites)) + for peerNum := 0; peerNum < len(IDC.SelectedIdentitites); peerNum++ { IDC.PeerToShardMap[IDC.SelectedIdentitites[peerNum]] = peerNum / numberInOneShard } } -func generateRandomPermutations(num int, SelectedIdentitites []*node.Node) []*node.Node { +func (IDC *IdentityChain) generateRandomPermutations(num int) { src := rand.NewSource(int64(num)) rnd := rand.New(src) - perm := rnd.Perm(len(SelectedIdentitites)) - SelectedIdentititesCopy := make([]*node.Node, len(SelectedIdentitites)) + perm := rnd.Perm(len(IDC.SelectedIdentitites)) + SelectedIdentititesCopy := make([]*node.Node, len(IDC.SelectedIdentitites)) for j, i := range perm { - SelectedIdentititesCopy[j] = SelectedIdentitites[i] + SelectedIdentititesCopy[j] = IDC.SelectedIdentitites[i] } - return SelectedIdentititesCopy + IDC.SelectedIdentitites = SelectedIdentititesCopy } // SelectIds as func (IDC *IdentityChain) SelectIds() { selectNumber := IDC.NumberOfNodesInShard - len(IDC.Identities) - IB := IDC.GetLatestBlock() - currentIDS := IB.GetIdentities() + // Insert the lines below once you have a identity block + // IB := IDC.GetLatestBlock() + // currentIDS := IB.GetIdentities() + currentIDS := IDC.Identities selectNumber = int(math.Min(float64(len(IDC.PendingIdentities)), float64(selectNumber))) pending := IDC.PendingIdentities[:selectNumber] IDC.SelectedIdentitites = append(currentIDS, pending...) IDC.PendingIdentities = []*node.Node{} + } //Checks how many new shards we need. Currently we say 0. @@ -106,37 +116,6 @@ func needNewShards() int { return 0 } -// GetLatestBlock gests the latest block at the end of the chain -func (IDC *IdentityChain) GetLatestBlock() *IdentityBlock { - if len(IDC.Identities) == 0 { - return nil - } - return IDC.Identities[len(IDC.Identities)-1] -} - -//UpdateIdentityChain is to create the Blocks to be added to the chain -func (IDC *IdentityChain) UpdateIdentityChain() { - - //If there are no more Identities registring the blockchain is dead - if len(IDC.PendingIdentities) == 0 { - // This is abd, because previous block might not be alive - return - } - if len(IDC.Identities) == 0 { - block := NewGenesisBlock() - IDC.Identities = append(IDC.Identities, block) - } else { - prevBlock := IDC.GetLatestBlock() - prevBlockHash := prevBlock.CalculateBlockHash() - NewIdentities := IDC.PendingIdentities[:identityPerBlock] - IDC.PendingIdentities = []*node.Node{} - //All other blocks are dropped, we need to inform them that they are dropped? - IDBlock := NewBlock(NewIdentities, prevBlockHash) - IDC.Identities = append(IDC.Identities, IDBlock) - } - -} - //StartServer a server and process the request by a handler. func (IDC *IdentityChain) StartServer() { fmt.Println("Starting server...") @@ -171,6 +150,47 @@ func New(Peer p2p.Peer) *IdentityChain { IDC := IdentityChain{} IDC.Peer = Peer IDC.log = log.New() + IDC.NumberOfShards = 1 //to be filled via global config + IDC.NumberOfNodesInShard = 500 //to be filled via global config + IDC.Identities = make([]*node.Node, 0) + IDC.PendingIdentities = make([]*node.Node, 0) + IDC.SelectedIdentitites = make([]*node.Node, 0) IDC.PowMap = make(map[p2p.Peer]string) return &IDC } + +// ------------------------------------------------------------- + +// The code below is needed when we have a actual identity block +// GetLatestBlock gests the latest block at the end of the chain +// func (IDC *IdentityChain) GetLatestBlock() *IdentityBlock { +// if len(IDC.Identities) == 0 { +// return nil +// } +// return IDC.Identities[len(IDC.Identities)-1] +// } + +//UpdateIdentityChain is to create the Blocks to be added to the chain +// func (IDC *IdentityChain) UpdateIdentityChain() { + +// //If there are no more Identities registring the blockchain is dead +// if len(IDC.PendingIdentities) == 0 { +// // This is abd, because previous block might not be alive +// return +// } +// if len(IDC.Identities) == 0 { +// block := NewGenesisBlock() +// IDC.Identities = append(IDC.Identities, block) +// } else { +// prevBlock := IDC.GetLatestBlock() +// prevBlockHash := prevBlock.CalculateBlockHash() +// NewIdentities := IDC.PendingIdentities[:identityPerBlock] +// IDC.PendingIdentities = []*node.Node{} +// //All other blocks are dropped, we need to inform them that they are dropped? +// IDBlock := NewBlock(NewIdentities, prevBlockHash) +// IDC.Identities = append(IDC.Identities, IDBlock) +// } + +// } + +// ------------------------------------------------------------- diff --git a/identitychain/identitychain_handler.go b/identitychain/identitychain_handler.go index 7b004fd2e..48c4e937e 100644 --- a/identitychain/identitychain_handler.go +++ b/identitychain/identitychain_handler.go @@ -74,7 +74,6 @@ func (IDC *IdentityChain) registerIdentity(msgPayload []byte) { fmt.Println("identity payload read") } fmt.Println("we are now registering identities") - //reconstruct the challenge and check whether its correct offset := 0 proof := payload[offset : offset+32] offset = offset + 32 @@ -108,6 +107,7 @@ func (IDC *IdentityChain) acceptNewConnection(msgPayload []byte) { challengeNonce := int((rnd.Int31())) req := pow.NewRequest(5, []byte(strconv.Itoa(challengeNonce))) IDC.PowMap[Node.Self] = req + fmt.Println(Node.Self) fmt.Println(req) buffer.Write([]byte(req)) // 32 byte block hash diff --git a/identitychain/identitychain_test.go b/identitychain/identitychain_test.go deleted file mode 100644 index 10599f8ac..000000000 --- a/identitychain/identitychain_test.go +++ /dev/null @@ -1,20 +0,0 @@ -package identitychain - -import ( - "fmt" - "os" - "testing" - - "github.com/simple-rules/harmony-benchmark/p2p" -) - -func TestIDCFormed(test *testing.T) { - peer := p2p.Peer{Ip: "127.0.0.1", Port: "8080"} - IDC := New(peer) - if IDC == nil { - fmt.Println("IDC not formed.") - os.Exit(1) - } -} - -//TODO Mock netconnection to test whether identitychain is listening. diff --git a/node/node.go b/node/node.go index 35fc1f7a6..16b209a77 100644 --- a/node/node.go +++ b/node/node.go @@ -173,7 +173,6 @@ func (node *Node) processPOWMessage(message []byte) { IDCPeer := node.IDCPeer // 4 byte challengeNonce id req := string(payload) - fmt.Println(req) proof, _ := pow.Fulfil(req, []byte("")) //"This could be blockhasdata" buffer := bytes.NewBuffer([]byte{}) proofBytes := make([]byte, 32) //proof seems to be 32 byte here diff --git a/proto/identity/identity.go b/proto/identity/identity.go index a3ab381f9..c8e8a711a 100644 --- a/proto/identity/identity.go +++ b/proto/identity/identity.go @@ -22,6 +22,7 @@ type MessageType int const ( REGISTER MessageType = iota ANNOUNCE + CONFIG ) // Returns string name for the MessageType enum @@ -29,15 +30,16 @@ func (msgType MessageType) String() string { names := [...]string{ "REGISTER", "ANNOUNCE", + "CONFIG", } - if msgType < REGISTER || msgType > ANNOUNCE { + if msgType < REGISTER || msgType > CONFIG { return "Unknown" } return names[msgType] } -// GetIdentityMessageType Get the consensus message type from the identity message +// GetIdentityMessageType Get the identity message type from the identity message func GetIdentityMessageType(message []byte) (MessageType, error) { if len(message) < 1 { return 0, errors.New("Failed to get identity message type: no data available.") diff --git a/runid/run_identity.go b/runid/run_identity.go index 37de5082e..f8f0265dc 100644 --- a/runid/run_identity.go +++ b/runid/run_identity.go @@ -3,6 +3,7 @@ package main import ( "flag" "fmt" + "time" "github.com/simple-rules/harmony-benchmark/identitychain" "github.com/simple-rules/harmony-benchmark/p2p" @@ -15,5 +16,15 @@ func main() { peer := p2p.Peer{Ip: *ip, Port: *port} IDC := identitychain.New(peer) fmt.Println(IDC) + epochTimer := time.NewTicker(10 * time.Second) + go func() { + for t := range epochTimer.C { + + fmt.Println("Changing epoch at ", t) + IDC.Shard() + + } + }() IDC.StartServer() + } diff --git a/runwait/run_wait.go b/runwait/run_wait.go index 62431c679..254efe598 100644 --- a/runwait/run_wait.go +++ b/runwait/run_wait.go @@ -3,6 +3,7 @@ package main import ( "flag" "fmt" + "time" "github.com/simple-rules/harmony-benchmark/node" "github.com/simple-rules/harmony-benchmark/p2p" @@ -12,14 +13,18 @@ func main() { ip := flag.String("ip", "localhost", "IP of the node") port := flag.String("port", "8080", "port of the node") flag.Parse() + + i := 0 peer := p2p.Peer{Ip: *ip, Port: *port} + fmt.Println("Now onto node i", i) idcpeer := p2p.Peer{Ip: "localhost", Port: "9000"} //Hardcoded here. node := node.NewWaitNode(peer, idcpeer) go func() { node.ConnectIdentityChain() }() - fmt.Println("control is back with me") node.StartServer(*port) - fmt.Println("starting the server") + + time.Sleep(5 * time.Second) + //} } diff --git a/waitnode/pow.txt b/waitnode/pow.txt deleted file mode 100644 index e1ae54168..000000000 --- a/waitnode/pow.txt +++ /dev/null @@ -1,82 +0,0 @@ -package waitnode - -import ( - "bytes" - "crypto/sha256" - "fmt" - "math" - "math/big" -) - -var ( - maxNonce = math.MaxInt64 -) - -const targetBits = 24 - -// ProofOfWork represents a proof-of-work -type ProofOfWork struct { - Challenge int32 - target *big.Int -} - -// NewProofOfWork builds and returns a ProofOfWork -func NewProofOfWork(c int32) *ProofOfWork { - target := big.NewInt(1) - target.Lsh(target, uint(256-targetBits)) - - pow := &ProofOfWork{c, target} - - return pow -} - -func (pow *ProofOfWork) prepareData(nonce int) []byte { - data := bytes.Join( - [][]byte{ - pow.Challenge., - IntToHex(int64(targetBits)), - IntToHex(int64(nonce)), - }, - []byte{}, - ) - - return data -} - -// Run performs a proof-of-work -func (pow *ProofOfWork) Run() (int, []byte) { - var hashInt big.Int - var hash [32]byte - nonce := 0 - - fmt.Printf("Mining the block containing \"%s\"\n", pow.block.Data) - for nonce < maxNonce { - data := pow.prepareData(nonce) - - hash = sha256.Sum256(data) - fmt.Printf("\r%x", hash) - hashInt.SetBytes(hash[:]) - - if hashInt.Cmp(pow.target) == -1 { - break - } else { - nonce++ - } - } - fmt.Print("\n\n") - - return nonce, hash[:] -} - -// Validate validates block's PoW -func (pow *ProofOfWork) Validate() bool { - var hashInt big.Int - - data := pow.prepareData(pow.challenge.Nonce) - hash := sha256.Sum256(data) - hashInt.SetBytes(hash[:]) - - isValid := hashInt.Cmp(pow.target) == -1 - - return isValid -}