From 912b51ea6f7793e404dddc0bbdfc5ce6e5fcd5bc Mon Sep 17 00:00:00 2001 From: Minh Doan Date: Thu, 13 Dec 2018 23:24:28 -0800 Subject: [PATCH 1/2] remove breaking param set up leveldb to store explorer info add storage adding dumping new block logic into db add fake data and structs for fake data explorer backend service with fake data clean something up set up leveldb to store explorer info add logic for blockinfo & refactor code --- consensus/consensus_leader.go | 16 +-- services/explorer/block_update.go | 43 ++++++++ services/explorer/rest_test.go | 158 ------------------------------ services/explorer/service.go | 104 +++++++++++++++++--- services/explorer/storage.go | 118 ++++++++++++++++++++++ services/explorer/structs.go | 14 +-- services/explorer/test2/main.go | 5 +- 7 files changed, 273 insertions(+), 185 deletions(-) create mode 100644 services/explorer/block_update.go delete mode 100644 services/explorer/rest_test.go create mode 100644 services/explorer/storage.go diff --git a/consensus/consensus_leader.go b/consensus/consensus_leader.go index 5c28fb826..4af360e6f 100644 --- a/consensus/consensus_leader.go +++ b/consensus/consensus_leader.go @@ -12,6 +12,7 @@ import ( "github.com/ethereum/go-ethereum/rlp" "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/p2p/host" + "github.com/harmony-one/harmony/services/explorer" "github.com/harmony-one/harmony/profiler" @@ -432,12 +433,6 @@ func (consensus *Consensus) processResponseMessage(payload []byte, targetState S host.BroadcastMessageFromLeader(consensus.host, consensus.GetValidatorPeers(), msgToSend, consensus.OfflinePeers) consensus.commitByLeader(false) } else { - consensus.Log.Debug("Consensus reached with signatures.", "numOfSignatures", len(*responses)) - // Reset state to Finished, and clear other data. - consensus.ResetState() - consensus.consensusID++ - consensus.Log.Debug("HOORAY!!! CONSENSUS REACHED!!!", "consensusID", consensus.consensusID) - // TODO: reconstruct the whole block from header and transactions // For now, we used the stored whole block already stored in consensus.blockHeader txDecoder := gob.NewDecoder(bytes.NewReader(consensus.blockHeader)) @@ -453,6 +448,15 @@ func (consensus *Consensus) processResponseMessage(payload []byte, targetState S consensus.OnConsensusDone(&blockHeaderObj) consensus.reportMetrics(blockHeaderObj) + // Dump new block into level db. + explorer.GetStorageInstance(consensus.leader.IP, consensus.leader.Port, true).Dump(blockHeaderObj.AccountBlock, consensus.consensusID) + // Claim new consensus reached. + consensus.Log.Debug("Consensus reached with signatures.", "numOfSignatures", len(*responses)) + // Reset state to Finished, and clear other data. + consensus.ResetState() + consensus.consensusID++ + consensus.Log.Debug("HOORAY!!! CONSENSUS REACHED!!!", "consensusID", consensus.consensusID) + // Send signal to Node so the new block can be added and new round of consensus can be triggered consensus.ReadySignal <- struct{}{} } diff --git a/services/explorer/block_update.go b/services/explorer/block_update.go new file mode 100644 index 000000000..009edf895 --- /dev/null +++ b/services/explorer/block_update.go @@ -0,0 +1,43 @@ +package explorer + +import ( + "fmt" + "os" + "sync" + + "github.com/harmony-one/harmony/core/types" + "github.com/harmony-one/harmony/db" +) + +var blockUpdate *BlockUpdate +var once sync.Once + +// BlockUpdate dump the block info into leveldb. +type BlockUpdate struct { + db *db.LDBDatabase +} + +// GetInstance returns attack model by using singleton pattern. +func GetInstance() *BlockUpdate { + once.Do(func() { + blockUpdate = &BlockUpdate{} + }) + return blockUpdate +} + +// Init initializes the block update. +func (blockUpdate *BlockUpdate) Init(ip, port string) { + dbFileName := "/tmp/explorer_db_" + ip + "_" + port + var err = os.RemoveAll(dbFileName) + if err != nil { + fmt.Println(err.Error()) + } + if blockUpdate.db, err = db.NewLDBDatabase(dbFileName, 0, 0); err != nil { + fmt.Println(err.Error()) + os.Exit(1) + } +} + +// Dump extracts information from block and index them into lvdb for explorer. +func (blockUpdate *BlockUpdate) Dump(block *types.Block, height int) { +} diff --git a/services/explorer/rest_test.go b/services/explorer/rest_test.go deleted file mode 100644 index ad0cc70bc..000000000 --- a/services/explorer/rest_test.go +++ /dev/null @@ -1,158 +0,0 @@ -package explorer_test - -// http://www.golangprograms.com/golang-restful-api-using-grom-and-gorilla-mux.html -// https://dev.to/codehakase/building-a-restful-api-with-go -// https://thenewstack.io/make-a-restful-json-api-go/ -// https://medium.com/@kelvin_sp/building-and-testing-a-rest-api-in-golang-using-gorilla-mux-and-mysql-1f0518818ff6 -// var a App - -// func TestMain(m *testing.M) { -// a = App{} -// a.Initialize("root", "", "rest_api_example") - -// ensureTableExists() - -// code := m.Run() - -// clearTable() - -// os.Exit(code) -// } - -// func ensureTableExists() { -// if _, err := a.DB.Exec(tableCreationQuery); err != nil { -// log.Fatal(err) -// } -// } - -// func clearTable() { -// a.DB.Exec("DELETE FROM users") -// a.DB.Exec("ALTER TABLE users AUTO_INCREMENT = 1") -// } - -// func executeRequest(req *http.Request) *httptest.ResponseRecorder { -// rr := httptest.NewRecorder() -// a.Router.ServeHTTP(rr, req) - -// return rr -// } - -// func checkResponseCode(t *testing.T, expected, actual int) { -// if expected != actual { -// t.Errorf("Expected response code %d. Got %d\n", expected, actual) -// } -// } - -// func TestGetNonExistentUser(t *testing.T) { -// clearTable() - -// req, _ := http.NewRequest("GET", "/user/45", nil) -// response := executeRequest(req) - -// checkResponseCode(t, http.StatusNotFound, response.Code) - -// var m map[string]string -// json.Unmarshal(response.Body.Bytes(), &m) -// if m["error"] != "User not found" { -// t.Errorf("Expected the 'error' key of the response to be set to 'User not found'. Got '%s'", m["error"]) -// } -// } - -// func TestCreateUser(t *testing.T) { -// clearTable() - -// payload := []byte(`{"name":"test user","age":30}`) - -// req, _ := http.NewRequest("POST", "/user", bytes.NewBuffer(payload)) -// response := executeRequest(req) - -// checkResponseCode(t, http.StatusCreated, response.Code) - -// var m map[string]interface{} -// json.Unmarshal(response.Body.Bytes(), &m) - -// if m["name"] != "test user" { -// t.Errorf("Expected user name to be 'test user'. Got '%v'", m["name"]) -// } - -// if m["age"] != 30.0 { -// t.Errorf("Expected user age to be '30'. Got '%v'", m["age"]) -// } - -// // the id is compared to 1.0 because JSON unmarshaling converts numbers to -// // floats, when the target is a map[string]interface{} -// if m["id"] != 1.0 { -// t.Errorf("Expected product ID to be '1'. Got '%v'", m["id"]) -// } -// } - -// func addUsers(count int) { -// if count < 1 { -// count = 1 -// } - -// for i := 0; i < count; i++ { -// statement := fmt.Sprintf("INSERT INTO users(name, age) VALUES('%s', %d)", ("User " + strconv.Itoa(i+1)), ((i + 1) * 10)) -// a.DB.Exec(statement) -// } -// } - -// func TestGetUser(t *testing.T) { -// clearTable() -// addUsers(1) - -// req, _ := http.NewRequest("GET", "/user/1", nil) -// response := executeRequest(req) - -// checkResponseCode(t, http.StatusOK, response.Code) -// } - -// func TestUpdateUser(t *testing.T) { -// clearTable() -// addUsers(1) - -// req, _ := http.NewRequest("GET", "/user/1", nil) -// response := executeRequest(req) -// var originalUser map[string]interface{} -// json.Unmarshal(response.Body.Bytes(), &originalUser) - -// payload := []byte(`{"name":"test user - updated name","age":21}`) - -// req, _ = http.NewRequest("PUT", "/user/1", bytes.NewBuffer(payload)) -// response = executeRequest(req) - -// checkResponseCode(t, http.StatusOK, response.Code) - -// var m map[string]interface{} -// json.Unmarshal(response.Body.Bytes(), &m) - -// if m["id"] != originalUser["id"] { -// t.Errorf("Expected the id to remain the same (%v). Got %v", originalUser["id"], m["id"]) -// } - -// if m["name"] == originalUser["name"] { -// t.Errorf("Expected the name to change from '%v' to '%v'. Got '%v'", originalUser["name"], m["name"], m["name"]) -// } - -// if m["age"] == originalUser["age"] { -// t.Errorf("Expected the age to change from '%v' to '%v'. Got '%v'", originalUser["age"], m["age"], m["age"]) -// } -// } - -// func TestDeleteUser(t *testing.T) { -// clearTable() -// addUsers(1) - -// req, _ := http.NewRequest("GET", "/user/1", nil) -// response := executeRequest(req) -// checkResponseCode(t, http.StatusOK, response.Code) - -// req, _ = http.NewRequest("DELETE", "/user/1", nil) -// response = executeRequest(req) - -// checkResponseCode(t, http.StatusOK, response.Code) - -// req, _ = http.NewRequest("GET", "/user/1", nil) -// response = executeRequest(req) -// checkResponseCode(t, http.StatusNotFound, response.Code) -// } diff --git a/services/explorer/service.go b/services/explorer/service.go index 954ae0509..625ab0300 100644 --- a/services/explorer/service.go +++ b/services/explorer/service.go @@ -6,7 +6,10 @@ import ( "log" "net" "net/http" + "os" + "strconv" + "github.com/ethereum/go-ethereum/rlp" "github.com/gorilla/mux" ) @@ -17,13 +20,15 @@ const ( // Service is the struct for explorer service. type Service struct { - data Data - router *mux.Router + router *mux.Router + IP string + Port string + storage *Storage } // Init is to do init for ExplorerService. func (s *Service) Init() { - s.data = ReadFakeData() + s.storage = GetStorageInstance(s.IP, s.Port, false) } // Run is to run serving explorer. @@ -33,29 +38,102 @@ func (s *Service) Run() { // Set up router s.router = mux.NewRouter() - s.router.HandleFunc("/blocks", s.GetExplorerBlocks).Methods("GET") + s.router.Path("/block_info").Queries("from", "{[0-9]*?}", "to", "{[0-9]*?}").HandlerFunc(s.GetExplorerBlockInfo).Methods("GET") + s.router.Path("/block_info").HandlerFunc(s.GetExplorerBlockInfo) + s.router.HandleFunc("/block", s.GetExplorerBlock).Methods("GET") s.router.HandleFunc("/address", s.GetExplorerAddress).Methods("GET") - // s.router.HandleFunc("/people", s.GetPeopleEndpoint).Methods("GET") - // s.router.HandleFunc("/people/{id}", s.GetPersonEndpoint).Methods("GET") - // s.router.HandleFunc("/people/{id}", s.CreatePersonEndpoint).Methods("POST") - // s.router.HandleFunc("/people/{id}", s.DeletePersonEndpoint).Methods("DELETE") + s.router.HandleFunc("/tx", s.GetExplorerTransaction).Methods("GET") // Do serving now. fmt.Println("Listening to:", ExplorerServicePort) go log.Fatal(http.ListenAndServe(addr, s.router)) } -// GetExplorerBlocks ... -func (s *Service) GetExplorerBlocks(w http.ResponseWriter, r *http.Request) { - json.NewEncoder(w).Encode(s.data.Blocks) +// GetExplorerBlockInfo ... +func (s *Service) GetExplorerBlockInfo(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + from := r.FormValue("from") + to := r.FormValue("to") + + data := &Data{ + Blocks: []*BlockInfo{}, + } + if from == "" { + json.NewEncoder(w).Encode(data.Blocks) + return + } + db := s.storage.GetDB() + fromInt, err := strconv.Atoi(from) + var toInt int + if to == "" { + bytes, err := db.Get([]byte(BlockHeightKey)) + if err == nil { + toInt, err = strconv.Atoi(string(bytes)) + } + } else { + toInt, err = strconv.Atoi(to) + } + fmt.Println("from", fromInt, "to", toInt) + if err != nil { + json.NewEncoder(w).Encode(data.Blocks) + return + } + + data.Blocks = s.PopulateBlockInfo(fromInt, toInt) + json.NewEncoder(w).Encode(data.Blocks) +} + +// PopulateBlockInfo ... +func (s *Service) PopulateBlockInfo(from, to int) []*BlockInfo { + blocks := []*BlockInfo{} + for i := from; i <= to; i++ { + key := GetBlockInfoKey(i) + fmt.Println("getting blockinfo with key ", key) + data, err := storage.db.Get([]byte(key)) + if err != nil { + fmt.Println("Error on getting from db") + os.Exit(1) + } + block := new(BlockInfo) + if rlp.DecodeBytes(data, block) != nil { + fmt.Println("RLP Decoding error") + os.Exit(1) + } + blocks = append(blocks, block) + fmt.Println("rlp decode successfully ") + } + return blocks } // GetExplorerBlock ... func (s *Service) GetExplorerBlock(w http.ResponseWriter, r *http.Request) { - json.NewEncoder(w).Encode(s.data.Block) + // w.Header().Set("Content-Type", "application/json") + // id := r.FormValue("id") + + // data := &Data{} + // db := s.storage.GetDB() + + // if id == "" { + // json.NewEncoder(w).Encode(data.Block) + // return + // } + // idInt, err := strconv.Atoi(id) + // if err != nil { + // json.NewEncoder(w).Encode(data.Block) + // return + // } + // data, err := db.Get([]byte(GetBlockKey(idInt))) + + // data.Block = storage.GetDB() + // json.NewEncoder(w).Encode(data.Blocks) } // GetExplorerAddress ... func (s *Service) GetExplorerAddress(w http.ResponseWriter, r *http.Request) { - json.NewEncoder(w).Encode(s.data.Address) + // json.NewEncoder(w).Encode(s.data.Address) +} + +// GetExplorerTransaction ... +func (s *Service) GetExplorerTransaction(w http.ResponseWriter, r *http.Request) { + // json.NewEncoder(w).Encode(s.data.Address) } diff --git a/services/explorer/storage.go b/services/explorer/storage.go new file mode 100644 index 000000000..9feebf8a5 --- /dev/null +++ b/services/explorer/storage.go @@ -0,0 +1,118 @@ +package explorer + +import ( + "fmt" + "os" + "strconv" + "sync" + + "github.com/ethereum/go-ethereum/rlp" + "github.com/harmony-one/harmony/core/types" + "github.com/harmony-one/harmony/db" +) + +// Constants for storage. +const ( + BlockHeightKey = "bh" + BlockInfoPrefix = "bi" + BlockPrefix = "b" + TXPrefix = "tx" +) + +// GetBlockInfoKey ... +func GetBlockInfoKey(id int) string { + return fmt.Sprintf("%s_%d", BlockInfoPrefix, id) +} + +// GetBlockKey ... +func GetBlockKey(id int) string { + return fmt.Sprintf("%s_%d", BlockPrefix, id) +} + +// GetTXKey ... +func GetTXKey(hash string) string { + return fmt.Sprintf("%s_%s", TXPrefix, hash) +} + +var storage *Storage +var once sync.Once + +// Storage dump the block info into leveldb. +type Storage struct { + db *db.LDBDatabase +} + +// GetStorageInstance returns attack model by using singleton pattern. +func GetStorageInstance(ip, port string, remove bool) *Storage { + once.Do(func() { + storage = &Storage{} + storage.Init(ip, port, remove) + }) + return storage +} + +// Init initializes the block update. +func (storage *Storage) Init(ip, port string, remove bool) { + dbFileName := "/tmp/explorer_storage_" + ip + "_" + port + var err error + if remove { + var err = os.RemoveAll(dbFileName) + if err != nil { + fmt.Println(err.Error()) + } + } + if storage.db, err = db.NewLDBDatabase(dbFileName, 0, 0); err != nil { + fmt.Println(err.Error()) + os.Exit(1) + } +} + +// GetDB returns the LDBDatabase of the storage. +func (storage *Storage) GetDB() *db.LDBDatabase { + return storage.db +} + +// Dump extracts information from block and index them into lvdb for explorer. +func (storage *Storage) Dump(accountBlock []byte, height uint32) { + fmt.Println("Dumping block ", height) + if accountBlock == nil { + return + } + // Update block height. + storage.db.Put([]byte(BlockHeightKey), []byte(strconv.Itoa(int(height)))) + + // Store block. + block := new(types.Block) + rlp.DecodeBytes(accountBlock, block) + storage.db.Put([]byte(GetBlockKey(int(height))), accountBlock) + + // Store block info. + blockInfo := BlockInfo{ + ID: block.Hash().Hex(), + Height: string(height), + Timestamp: string(block.Time().Int64()), + TXCount: string(block.Transactions().Len()), + Size: block.Size().String(), + } + + if data, err := rlp.EncodeToBytes(blockInfo); err == nil { + key := GetBlockInfoKey(int(height)) + fmt.Println("store blockinfo with key ", key) + fmt.Println("data to store ", data) + storage.db.Put([]byte(key), data) + } else { + fmt.Println("EncodeRLP blockInfo error") + os.Exit(1) + } + + // Store txs + for _, tx := range block.Transactions() { + if data, err := rlp.EncodeToBytes(tx); err == nil { + key := GetTXKey(tx.Hash().Hex()) + storage.db.Put([]byte(key), data) + } else { + fmt.Println("EncodeRLP transaction error") + os.Exit(1) + } + } +} diff --git a/services/explorer/structs.go b/services/explorer/structs.go index 364bb360f..ee188e2d0 100644 --- a/services/explorer/structs.go +++ b/services/explorer/structs.go @@ -6,9 +6,9 @@ package explorer // Data ... type Data struct { - Blocks []Block `json:"blocks"` - Block Block2 `json:"block"` - Address Address `json:"address"` + Blocks []*BlockInfo `json:"blocks"` + Block Block `json:"block"` + Address Address `json:"address"` } // Address ... @@ -28,8 +28,8 @@ type Transaction struct { Value float64 `json:"value"` } -// Block ... -type Block struct { +// BlockInfo ... +type BlockInfo struct { ID string `json:"id"` Height string `json:"height"` Timestamp string `json:"timestamp"` @@ -37,8 +37,8 @@ type Block struct { Size string `json:"size"` } -// Block2 ... -type Block2 struct { +// Block ... +type Block struct { Height string `json:"height"` Hash string `json:"hash"` TXCount string `json:"txCount"` diff --git a/services/explorer/test2/main.go b/services/explorer/test2/main.go index c679c0a24..52982e8fd 100644 --- a/services/explorer/test2/main.go +++ b/services/explorer/test2/main.go @@ -3,7 +3,10 @@ package main import "github.com/harmony-one/harmony/services/explorer" func main() { - service := &explorer.Service{} + service := &explorer.Service{ + IP: "127.0.0.1", + Port: "9000", + } service.Init() service.Run() } From 007277ddc78fb2e4fc54012d40e6ed37fcd84896 Mon Sep 17 00:00:00 2001 From: Minh Doan Date: Fri, 14 Dec 2018 16:59:42 -0800 Subject: [PATCH 2/2] remove block_update --- services/explorer/block_update.go | 43 ------------------------------- 1 file changed, 43 deletions(-) delete mode 100644 services/explorer/block_update.go diff --git a/services/explorer/block_update.go b/services/explorer/block_update.go deleted file mode 100644 index 009edf895..000000000 --- a/services/explorer/block_update.go +++ /dev/null @@ -1,43 +0,0 @@ -package explorer - -import ( - "fmt" - "os" - "sync" - - "github.com/harmony-one/harmony/core/types" - "github.com/harmony-one/harmony/db" -) - -var blockUpdate *BlockUpdate -var once sync.Once - -// BlockUpdate dump the block info into leveldb. -type BlockUpdate struct { - db *db.LDBDatabase -} - -// GetInstance returns attack model by using singleton pattern. -func GetInstance() *BlockUpdate { - once.Do(func() { - blockUpdate = &BlockUpdate{} - }) - return blockUpdate -} - -// Init initializes the block update. -func (blockUpdate *BlockUpdate) Init(ip, port string) { - dbFileName := "/tmp/explorer_db_" + ip + "_" + port - var err = os.RemoveAll(dbFileName) - if err != nil { - fmt.Println(err.Error()) - } - if blockUpdate.db, err = db.NewLDBDatabase(dbFileName, 0, 0); err != nil { - fmt.Println(err.Error()) - os.Exit(1) - } -} - -// Dump extracts information from block and index them into lvdb for explorer. -func (blockUpdate *BlockUpdate) Dump(block *types.Block, height int) { -}