diff --git a/consensus/consensus_leader.go b/consensus/consensus_leader.go index 5c28fb826..4af360e6f 100644 --- a/consensus/consensus_leader.go +++ b/consensus/consensus_leader.go @@ -12,6 +12,7 @@ import ( "github.com/ethereum/go-ethereum/rlp" "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/p2p/host" + "github.com/harmony-one/harmony/services/explorer" "github.com/harmony-one/harmony/profiler" @@ -432,12 +433,6 @@ func (consensus *Consensus) processResponseMessage(payload []byte, targetState S host.BroadcastMessageFromLeader(consensus.host, consensus.GetValidatorPeers(), msgToSend, consensus.OfflinePeers) consensus.commitByLeader(false) } else { - consensus.Log.Debug("Consensus reached with signatures.", "numOfSignatures", len(*responses)) - // Reset state to Finished, and clear other data. - consensus.ResetState() - consensus.consensusID++ - consensus.Log.Debug("HOORAY!!! CONSENSUS REACHED!!!", "consensusID", consensus.consensusID) - // TODO: reconstruct the whole block from header and transactions // For now, we used the stored whole block already stored in consensus.blockHeader txDecoder := gob.NewDecoder(bytes.NewReader(consensus.blockHeader)) @@ -453,6 +448,15 @@ func (consensus *Consensus) processResponseMessage(payload []byte, targetState S consensus.OnConsensusDone(&blockHeaderObj) consensus.reportMetrics(blockHeaderObj) + // Dump new block into level db. + explorer.GetStorageInstance(consensus.leader.IP, consensus.leader.Port, true).Dump(blockHeaderObj.AccountBlock, consensus.consensusID) + // Claim new consensus reached. + consensus.Log.Debug("Consensus reached with signatures.", "numOfSignatures", len(*responses)) + // Reset state to Finished, and clear other data. + consensus.ResetState() + consensus.consensusID++ + consensus.Log.Debug("HOORAY!!! CONSENSUS REACHED!!!", "consensusID", consensus.consensusID) + // Send signal to Node so the new block can be added and new round of consensus can be triggered consensus.ReadySignal <- struct{}{} } diff --git a/services/explorer/block_update.go b/services/explorer/block_update.go new file mode 100644 index 000000000..009edf895 --- /dev/null +++ b/services/explorer/block_update.go @@ -0,0 +1,43 @@ +package explorer + +import ( + "fmt" + "os" + "sync" + + "github.com/harmony-one/harmony/core/types" + "github.com/harmony-one/harmony/db" +) + +var blockUpdate *BlockUpdate +var once sync.Once + +// BlockUpdate dump the block info into leveldb. +type BlockUpdate struct { + db *db.LDBDatabase +} + +// GetInstance returns attack model by using singleton pattern. +func GetInstance() *BlockUpdate { + once.Do(func() { + blockUpdate = &BlockUpdate{} + }) + return blockUpdate +} + +// Init initializes the block update. +func (blockUpdate *BlockUpdate) Init(ip, port string) { + dbFileName := "/tmp/explorer_db_" + ip + "_" + port + var err = os.RemoveAll(dbFileName) + if err != nil { + fmt.Println(err.Error()) + } + if blockUpdate.db, err = db.NewLDBDatabase(dbFileName, 0, 0); err != nil { + fmt.Println(err.Error()) + os.Exit(1) + } +} + +// Dump extracts information from block and index them into lvdb for explorer. +func (blockUpdate *BlockUpdate) Dump(block *types.Block, height int) { +} diff --git a/services/explorer/rest_test.go b/services/explorer/rest_test.go deleted file mode 100644 index ad0cc70bc..000000000 --- a/services/explorer/rest_test.go +++ /dev/null @@ -1,158 +0,0 @@ -package explorer_test - -// http://www.golangprograms.com/golang-restful-api-using-grom-and-gorilla-mux.html -// https://dev.to/codehakase/building-a-restful-api-with-go -// https://thenewstack.io/make-a-restful-json-api-go/ -// https://medium.com/@kelvin_sp/building-and-testing-a-rest-api-in-golang-using-gorilla-mux-and-mysql-1f0518818ff6 -// var a App - -// func TestMain(m *testing.M) { -// a = App{} -// a.Initialize("root", "", "rest_api_example") - -// ensureTableExists() - -// code := m.Run() - -// clearTable() - -// os.Exit(code) -// } - -// func ensureTableExists() { -// if _, err := a.DB.Exec(tableCreationQuery); err != nil { -// log.Fatal(err) -// } -// } - -// func clearTable() { -// a.DB.Exec("DELETE FROM users") -// a.DB.Exec("ALTER TABLE users AUTO_INCREMENT = 1") -// } - -// func executeRequest(req *http.Request) *httptest.ResponseRecorder { -// rr := httptest.NewRecorder() -// a.Router.ServeHTTP(rr, req) - -// return rr -// } - -// func checkResponseCode(t *testing.T, expected, actual int) { -// if expected != actual { -// t.Errorf("Expected response code %d. Got %d\n", expected, actual) -// } -// } - -// func TestGetNonExistentUser(t *testing.T) { -// clearTable() - -// req, _ := http.NewRequest("GET", "/user/45", nil) -// response := executeRequest(req) - -// checkResponseCode(t, http.StatusNotFound, response.Code) - -// var m map[string]string -// json.Unmarshal(response.Body.Bytes(), &m) -// if m["error"] != "User not found" { -// t.Errorf("Expected the 'error' key of the response to be set to 'User not found'. Got '%s'", m["error"]) -// } -// } - -// func TestCreateUser(t *testing.T) { -// clearTable() - -// payload := []byte(`{"name":"test user","age":30}`) - -// req, _ := http.NewRequest("POST", "/user", bytes.NewBuffer(payload)) -// response := executeRequest(req) - -// checkResponseCode(t, http.StatusCreated, response.Code) - -// var m map[string]interface{} -// json.Unmarshal(response.Body.Bytes(), &m) - -// if m["name"] != "test user" { -// t.Errorf("Expected user name to be 'test user'. Got '%v'", m["name"]) -// } - -// if m["age"] != 30.0 { -// t.Errorf("Expected user age to be '30'. Got '%v'", m["age"]) -// } - -// // the id is compared to 1.0 because JSON unmarshaling converts numbers to -// // floats, when the target is a map[string]interface{} -// if m["id"] != 1.0 { -// t.Errorf("Expected product ID to be '1'. Got '%v'", m["id"]) -// } -// } - -// func addUsers(count int) { -// if count < 1 { -// count = 1 -// } - -// for i := 0; i < count; i++ { -// statement := fmt.Sprintf("INSERT INTO users(name, age) VALUES('%s', %d)", ("User " + strconv.Itoa(i+1)), ((i + 1) * 10)) -// a.DB.Exec(statement) -// } -// } - -// func TestGetUser(t *testing.T) { -// clearTable() -// addUsers(1) - -// req, _ := http.NewRequest("GET", "/user/1", nil) -// response := executeRequest(req) - -// checkResponseCode(t, http.StatusOK, response.Code) -// } - -// func TestUpdateUser(t *testing.T) { -// clearTable() -// addUsers(1) - -// req, _ := http.NewRequest("GET", "/user/1", nil) -// response := executeRequest(req) -// var originalUser map[string]interface{} -// json.Unmarshal(response.Body.Bytes(), &originalUser) - -// payload := []byte(`{"name":"test user - updated name","age":21}`) - -// req, _ = http.NewRequest("PUT", "/user/1", bytes.NewBuffer(payload)) -// response = executeRequest(req) - -// checkResponseCode(t, http.StatusOK, response.Code) - -// var m map[string]interface{} -// json.Unmarshal(response.Body.Bytes(), &m) - -// if m["id"] != originalUser["id"] { -// t.Errorf("Expected the id to remain the same (%v). Got %v", originalUser["id"], m["id"]) -// } - -// if m["name"] == originalUser["name"] { -// t.Errorf("Expected the name to change from '%v' to '%v'. Got '%v'", originalUser["name"], m["name"], m["name"]) -// } - -// if m["age"] == originalUser["age"] { -// t.Errorf("Expected the age to change from '%v' to '%v'. Got '%v'", originalUser["age"], m["age"], m["age"]) -// } -// } - -// func TestDeleteUser(t *testing.T) { -// clearTable() -// addUsers(1) - -// req, _ := http.NewRequest("GET", "/user/1", nil) -// response := executeRequest(req) -// checkResponseCode(t, http.StatusOK, response.Code) - -// req, _ = http.NewRequest("DELETE", "/user/1", nil) -// response = executeRequest(req) - -// checkResponseCode(t, http.StatusOK, response.Code) - -// req, _ = http.NewRequest("GET", "/user/1", nil) -// response = executeRequest(req) -// checkResponseCode(t, http.StatusNotFound, response.Code) -// } diff --git a/services/explorer/service.go b/services/explorer/service.go index 954ae0509..625ab0300 100644 --- a/services/explorer/service.go +++ b/services/explorer/service.go @@ -6,7 +6,10 @@ import ( "log" "net" "net/http" + "os" + "strconv" + "github.com/ethereum/go-ethereum/rlp" "github.com/gorilla/mux" ) @@ -17,13 +20,15 @@ const ( // Service is the struct for explorer service. type Service struct { - data Data - router *mux.Router + router *mux.Router + IP string + Port string + storage *Storage } // Init is to do init for ExplorerService. func (s *Service) Init() { - s.data = ReadFakeData() + s.storage = GetStorageInstance(s.IP, s.Port, false) } // Run is to run serving explorer. @@ -33,29 +38,102 @@ func (s *Service) Run() { // Set up router s.router = mux.NewRouter() - s.router.HandleFunc("/blocks", s.GetExplorerBlocks).Methods("GET") + s.router.Path("/block_info").Queries("from", "{[0-9]*?}", "to", "{[0-9]*?}").HandlerFunc(s.GetExplorerBlockInfo).Methods("GET") + s.router.Path("/block_info").HandlerFunc(s.GetExplorerBlockInfo) + s.router.HandleFunc("/block", s.GetExplorerBlock).Methods("GET") s.router.HandleFunc("/address", s.GetExplorerAddress).Methods("GET") - // s.router.HandleFunc("/people", s.GetPeopleEndpoint).Methods("GET") - // s.router.HandleFunc("/people/{id}", s.GetPersonEndpoint).Methods("GET") - // s.router.HandleFunc("/people/{id}", s.CreatePersonEndpoint).Methods("POST") - // s.router.HandleFunc("/people/{id}", s.DeletePersonEndpoint).Methods("DELETE") + s.router.HandleFunc("/tx", s.GetExplorerTransaction).Methods("GET") // Do serving now. fmt.Println("Listening to:", ExplorerServicePort) go log.Fatal(http.ListenAndServe(addr, s.router)) } -// GetExplorerBlocks ... -func (s *Service) GetExplorerBlocks(w http.ResponseWriter, r *http.Request) { - json.NewEncoder(w).Encode(s.data.Blocks) +// GetExplorerBlockInfo ... +func (s *Service) GetExplorerBlockInfo(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + from := r.FormValue("from") + to := r.FormValue("to") + + data := &Data{ + Blocks: []*BlockInfo{}, + } + if from == "" { + json.NewEncoder(w).Encode(data.Blocks) + return + } + db := s.storage.GetDB() + fromInt, err := strconv.Atoi(from) + var toInt int + if to == "" { + bytes, err := db.Get([]byte(BlockHeightKey)) + if err == nil { + toInt, err = strconv.Atoi(string(bytes)) + } + } else { + toInt, err = strconv.Atoi(to) + } + fmt.Println("from", fromInt, "to", toInt) + if err != nil { + json.NewEncoder(w).Encode(data.Blocks) + return + } + + data.Blocks = s.PopulateBlockInfo(fromInt, toInt) + json.NewEncoder(w).Encode(data.Blocks) +} + +// PopulateBlockInfo ... +func (s *Service) PopulateBlockInfo(from, to int) []*BlockInfo { + blocks := []*BlockInfo{} + for i := from; i <= to; i++ { + key := GetBlockInfoKey(i) + fmt.Println("getting blockinfo with key ", key) + data, err := storage.db.Get([]byte(key)) + if err != nil { + fmt.Println("Error on getting from db") + os.Exit(1) + } + block := new(BlockInfo) + if rlp.DecodeBytes(data, block) != nil { + fmt.Println("RLP Decoding error") + os.Exit(1) + } + blocks = append(blocks, block) + fmt.Println("rlp decode successfully ") + } + return blocks } // GetExplorerBlock ... func (s *Service) GetExplorerBlock(w http.ResponseWriter, r *http.Request) { - json.NewEncoder(w).Encode(s.data.Block) + // w.Header().Set("Content-Type", "application/json") + // id := r.FormValue("id") + + // data := &Data{} + // db := s.storage.GetDB() + + // if id == "" { + // json.NewEncoder(w).Encode(data.Block) + // return + // } + // idInt, err := strconv.Atoi(id) + // if err != nil { + // json.NewEncoder(w).Encode(data.Block) + // return + // } + // data, err := db.Get([]byte(GetBlockKey(idInt))) + + // data.Block = storage.GetDB() + // json.NewEncoder(w).Encode(data.Blocks) } // GetExplorerAddress ... func (s *Service) GetExplorerAddress(w http.ResponseWriter, r *http.Request) { - json.NewEncoder(w).Encode(s.data.Address) + // json.NewEncoder(w).Encode(s.data.Address) +} + +// GetExplorerTransaction ... +func (s *Service) GetExplorerTransaction(w http.ResponseWriter, r *http.Request) { + // json.NewEncoder(w).Encode(s.data.Address) } diff --git a/services/explorer/storage.go b/services/explorer/storage.go new file mode 100644 index 000000000..9feebf8a5 --- /dev/null +++ b/services/explorer/storage.go @@ -0,0 +1,118 @@ +package explorer + +import ( + "fmt" + "os" + "strconv" + "sync" + + "github.com/ethereum/go-ethereum/rlp" + "github.com/harmony-one/harmony/core/types" + "github.com/harmony-one/harmony/db" +) + +// Constants for storage. +const ( + BlockHeightKey = "bh" + BlockInfoPrefix = "bi" + BlockPrefix = "b" + TXPrefix = "tx" +) + +// GetBlockInfoKey ... +func GetBlockInfoKey(id int) string { + return fmt.Sprintf("%s_%d", BlockInfoPrefix, id) +} + +// GetBlockKey ... +func GetBlockKey(id int) string { + return fmt.Sprintf("%s_%d", BlockPrefix, id) +} + +// GetTXKey ... +func GetTXKey(hash string) string { + return fmt.Sprintf("%s_%s", TXPrefix, hash) +} + +var storage *Storage +var once sync.Once + +// Storage dump the block info into leveldb. +type Storage struct { + db *db.LDBDatabase +} + +// GetStorageInstance returns attack model by using singleton pattern. +func GetStorageInstance(ip, port string, remove bool) *Storage { + once.Do(func() { + storage = &Storage{} + storage.Init(ip, port, remove) + }) + return storage +} + +// Init initializes the block update. +func (storage *Storage) Init(ip, port string, remove bool) { + dbFileName := "/tmp/explorer_storage_" + ip + "_" + port + var err error + if remove { + var err = os.RemoveAll(dbFileName) + if err != nil { + fmt.Println(err.Error()) + } + } + if storage.db, err = db.NewLDBDatabase(dbFileName, 0, 0); err != nil { + fmt.Println(err.Error()) + os.Exit(1) + } +} + +// GetDB returns the LDBDatabase of the storage. +func (storage *Storage) GetDB() *db.LDBDatabase { + return storage.db +} + +// Dump extracts information from block and index them into lvdb for explorer. +func (storage *Storage) Dump(accountBlock []byte, height uint32) { + fmt.Println("Dumping block ", height) + if accountBlock == nil { + return + } + // Update block height. + storage.db.Put([]byte(BlockHeightKey), []byte(strconv.Itoa(int(height)))) + + // Store block. + block := new(types.Block) + rlp.DecodeBytes(accountBlock, block) + storage.db.Put([]byte(GetBlockKey(int(height))), accountBlock) + + // Store block info. + blockInfo := BlockInfo{ + ID: block.Hash().Hex(), + Height: string(height), + Timestamp: string(block.Time().Int64()), + TXCount: string(block.Transactions().Len()), + Size: block.Size().String(), + } + + if data, err := rlp.EncodeToBytes(blockInfo); err == nil { + key := GetBlockInfoKey(int(height)) + fmt.Println("store blockinfo with key ", key) + fmt.Println("data to store ", data) + storage.db.Put([]byte(key), data) + } else { + fmt.Println("EncodeRLP blockInfo error") + os.Exit(1) + } + + // Store txs + for _, tx := range block.Transactions() { + if data, err := rlp.EncodeToBytes(tx); err == nil { + key := GetTXKey(tx.Hash().Hex()) + storage.db.Put([]byte(key), data) + } else { + fmt.Println("EncodeRLP transaction error") + os.Exit(1) + } + } +} diff --git a/services/explorer/structs.go b/services/explorer/structs.go index 364bb360f..ee188e2d0 100644 --- a/services/explorer/structs.go +++ b/services/explorer/structs.go @@ -6,9 +6,9 @@ package explorer // Data ... type Data struct { - Blocks []Block `json:"blocks"` - Block Block2 `json:"block"` - Address Address `json:"address"` + Blocks []*BlockInfo `json:"blocks"` + Block Block `json:"block"` + Address Address `json:"address"` } // Address ... @@ -28,8 +28,8 @@ type Transaction struct { Value float64 `json:"value"` } -// Block ... -type Block struct { +// BlockInfo ... +type BlockInfo struct { ID string `json:"id"` Height string `json:"height"` Timestamp string `json:"timestamp"` @@ -37,8 +37,8 @@ type Block struct { Size string `json:"size"` } -// Block2 ... -type Block2 struct { +// Block ... +type Block struct { Height string `json:"height"` Hash string `json:"hash"` TXCount string `json:"txCount"` diff --git a/services/explorer/test2/main.go b/services/explorer/test2/main.go index c679c0a24..52982e8fd 100644 --- a/services/explorer/test2/main.go +++ b/services/explorer/test2/main.go @@ -3,7 +3,10 @@ package main import "github.com/harmony-one/harmony/services/explorer" func main() { - service := &explorer.Service{} + service := &explorer.Service{ + IP: "127.0.0.1", + Port: "9000", + } service.Init() service.Run() }