Merge pull request #155 from harmony-one/new

Explorer backend
pull/156/head
Leo Chen 6 years ago committed by GitHub
commit 483175ace2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 16
      consensus/consensus_leader.go
  2. 158
      services/explorer/rest_test.go
  3. 104
      services/explorer/service.go
  4. 118
      services/explorer/storage.go
  5. 14
      services/explorer/structs.go
  6. 5
      services/explorer/test2/main.go

@ -12,6 +12,7 @@ import (
"github.com/ethereum/go-ethereum/rlp"
"github.com/harmony-one/harmony/core/types"
"github.com/harmony-one/harmony/p2p/host"
"github.com/harmony-one/harmony/services/explorer"
"github.com/harmony-one/harmony/profiler"
@ -430,12 +431,6 @@ func (consensus *Consensus) processResponseMessage(payload []byte, targetState S
host.BroadcastMessageFromLeader(consensus.host, consensus.GetValidatorPeers(), msgToSend, consensus.OfflinePeers)
consensus.commitByLeader(false)
} else {
consensus.Log.Debug("Consensus reached with signatures.", "numOfSignatures", len(*responses))
// Reset state to Finished, and clear other data.
consensus.ResetState()
consensus.consensusID++
consensus.Log.Debug("HOORAY!!! CONSENSUS REACHED!!!", "consensusID", consensus.consensusID)
// TODO: reconstruct the whole block from header and transactions
// For now, we used the stored whole block already stored in consensus.blockHeader
txDecoder := gob.NewDecoder(bytes.NewReader(consensus.blockHeader))
@ -451,6 +446,15 @@ func (consensus *Consensus) processResponseMessage(payload []byte, targetState S
consensus.OnConsensusDone(&blockHeaderObj)
consensus.reportMetrics(blockHeaderObj)
// Dump new block into level db.
explorer.GetStorageInstance(consensus.leader.IP, consensus.leader.Port, true).Dump(blockHeaderObj.AccountBlock, consensus.consensusID)
// Claim new consensus reached.
consensus.Log.Debug("Consensus reached with signatures.", "numOfSignatures", len(*responses))
// Reset state to Finished, and clear other data.
consensus.ResetState()
consensus.consensusID++
consensus.Log.Debug("HOORAY!!! CONSENSUS REACHED!!!", "consensusID", consensus.consensusID)
// Send signal to Node so the new block can be added and new round of consensus can be triggered
consensus.ReadySignal <- struct{}{}
}

@ -1,158 +0,0 @@
package explorer_test
// http://www.golangprograms.com/golang-restful-api-using-grom-and-gorilla-mux.html
// https://dev.to/codehakase/building-a-restful-api-with-go
// https://thenewstack.io/make-a-restful-json-api-go/
// https://medium.com/@kelvin_sp/building-and-testing-a-rest-api-in-golang-using-gorilla-mux-and-mysql-1f0518818ff6
// var a App
// func TestMain(m *testing.M) {
// a = App{}
// a.Initialize("root", "", "rest_api_example")
// ensureTableExists()
// code := m.Run()
// clearTable()
// os.Exit(code)
// }
// func ensureTableExists() {
// if _, err := a.DB.Exec(tableCreationQuery); err != nil {
// log.Fatal(err)
// }
// }
// func clearTable() {
// a.DB.Exec("DELETE FROM users")
// a.DB.Exec("ALTER TABLE users AUTO_INCREMENT = 1")
// }
// func executeRequest(req *http.Request) *httptest.ResponseRecorder {
// rr := httptest.NewRecorder()
// a.Router.ServeHTTP(rr, req)
// return rr
// }
// func checkResponseCode(t *testing.T, expected, actual int) {
// if expected != actual {
// t.Errorf("Expected response code %d. Got %d\n", expected, actual)
// }
// }
// func TestGetNonExistentUser(t *testing.T) {
// clearTable()
// req, _ := http.NewRequest("GET", "/user/45", nil)
// response := executeRequest(req)
// checkResponseCode(t, http.StatusNotFound, response.Code)
// var m map[string]string
// json.Unmarshal(response.Body.Bytes(), &m)
// if m["error"] != "User not found" {
// t.Errorf("Expected the 'error' key of the response to be set to 'User not found'. Got '%s'", m["error"])
// }
// }
// func TestCreateUser(t *testing.T) {
// clearTable()
// payload := []byte(`{"name":"test user","age":30}`)
// req, _ := http.NewRequest("POST", "/user", bytes.NewBuffer(payload))
// response := executeRequest(req)
// checkResponseCode(t, http.StatusCreated, response.Code)
// var m map[string]interface{}
// json.Unmarshal(response.Body.Bytes(), &m)
// if m["name"] != "test user" {
// t.Errorf("Expected user name to be 'test user'. Got '%v'", m["name"])
// }
// if m["age"] != 30.0 {
// t.Errorf("Expected user age to be '30'. Got '%v'", m["age"])
// }
// // the id is compared to 1.0 because JSON unmarshaling converts numbers to
// // floats, when the target is a map[string]interface{}
// if m["id"] != 1.0 {
// t.Errorf("Expected product ID to be '1'. Got '%v'", m["id"])
// }
// }
// func addUsers(count int) {
// if count < 1 {
// count = 1
// }
// for i := 0; i < count; i++ {
// statement := fmt.Sprintf("INSERT INTO users(name, age) VALUES('%s', %d)", ("User " + strconv.Itoa(i+1)), ((i + 1) * 10))
// a.DB.Exec(statement)
// }
// }
// func TestGetUser(t *testing.T) {
// clearTable()
// addUsers(1)
// req, _ := http.NewRequest("GET", "/user/1", nil)
// response := executeRequest(req)
// checkResponseCode(t, http.StatusOK, response.Code)
// }
// func TestUpdateUser(t *testing.T) {
// clearTable()
// addUsers(1)
// req, _ := http.NewRequest("GET", "/user/1", nil)
// response := executeRequest(req)
// var originalUser map[string]interface{}
// json.Unmarshal(response.Body.Bytes(), &originalUser)
// payload := []byte(`{"name":"test user - updated name","age":21}`)
// req, _ = http.NewRequest("PUT", "/user/1", bytes.NewBuffer(payload))
// response = executeRequest(req)
// checkResponseCode(t, http.StatusOK, response.Code)
// var m map[string]interface{}
// json.Unmarshal(response.Body.Bytes(), &m)
// if m["id"] != originalUser["id"] {
// t.Errorf("Expected the id to remain the same (%v). Got %v", originalUser["id"], m["id"])
// }
// if m["name"] == originalUser["name"] {
// t.Errorf("Expected the name to change from '%v' to '%v'. Got '%v'", originalUser["name"], m["name"], m["name"])
// }
// if m["age"] == originalUser["age"] {
// t.Errorf("Expected the age to change from '%v' to '%v'. Got '%v'", originalUser["age"], m["age"], m["age"])
// }
// }
// func TestDeleteUser(t *testing.T) {
// clearTable()
// addUsers(1)
// req, _ := http.NewRequest("GET", "/user/1", nil)
// response := executeRequest(req)
// checkResponseCode(t, http.StatusOK, response.Code)
// req, _ = http.NewRequest("DELETE", "/user/1", nil)
// response = executeRequest(req)
// checkResponseCode(t, http.StatusOK, response.Code)
// req, _ = http.NewRequest("GET", "/user/1", nil)
// response = executeRequest(req)
// checkResponseCode(t, http.StatusNotFound, response.Code)
// }

@ -6,7 +6,10 @@ import (
"log"
"net"
"net/http"
"os"
"strconv"
"github.com/ethereum/go-ethereum/rlp"
"github.com/gorilla/mux"
)
@ -17,13 +20,15 @@ const (
// Service is the struct for explorer service.
type Service struct {
data Data
router *mux.Router
router *mux.Router
IP string
Port string
storage *Storage
}
// Init is to do init for ExplorerService.
func (s *Service) Init() {
s.data = ReadFakeData()
s.storage = GetStorageInstance(s.IP, s.Port, false)
}
// Run is to run serving explorer.
@ -33,29 +38,102 @@ func (s *Service) Run() {
// Set up router
s.router = mux.NewRouter()
s.router.HandleFunc("/blocks", s.GetExplorerBlocks).Methods("GET")
s.router.Path("/block_info").Queries("from", "{[0-9]*?}", "to", "{[0-9]*?}").HandlerFunc(s.GetExplorerBlockInfo).Methods("GET")
s.router.Path("/block_info").HandlerFunc(s.GetExplorerBlockInfo)
s.router.HandleFunc("/block", s.GetExplorerBlock).Methods("GET")
s.router.HandleFunc("/address", s.GetExplorerAddress).Methods("GET")
// s.router.HandleFunc("/people", s.GetPeopleEndpoint).Methods("GET")
// s.router.HandleFunc("/people/{id}", s.GetPersonEndpoint).Methods("GET")
// s.router.HandleFunc("/people/{id}", s.CreatePersonEndpoint).Methods("POST")
// s.router.HandleFunc("/people/{id}", s.DeletePersonEndpoint).Methods("DELETE")
s.router.HandleFunc("/tx", s.GetExplorerTransaction).Methods("GET")
// Do serving now.
fmt.Println("Listening to:", ExplorerServicePort)
go log.Fatal(http.ListenAndServe(addr, s.router))
}
// GetExplorerBlocks ...
func (s *Service) GetExplorerBlocks(w http.ResponseWriter, r *http.Request) {
json.NewEncoder(w).Encode(s.data.Blocks)
// GetExplorerBlockInfo ...
func (s *Service) GetExplorerBlockInfo(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
from := r.FormValue("from")
to := r.FormValue("to")
data := &Data{
Blocks: []*BlockInfo{},
}
if from == "" {
json.NewEncoder(w).Encode(data.Blocks)
return
}
db := s.storage.GetDB()
fromInt, err := strconv.Atoi(from)
var toInt int
if to == "" {
bytes, err := db.Get([]byte(BlockHeightKey))
if err == nil {
toInt, err = strconv.Atoi(string(bytes))
}
} else {
toInt, err = strconv.Atoi(to)
}
fmt.Println("from", fromInt, "to", toInt)
if err != nil {
json.NewEncoder(w).Encode(data.Blocks)
return
}
data.Blocks = s.PopulateBlockInfo(fromInt, toInt)
json.NewEncoder(w).Encode(data.Blocks)
}
// PopulateBlockInfo ...
func (s *Service) PopulateBlockInfo(from, to int) []*BlockInfo {
blocks := []*BlockInfo{}
for i := from; i <= to; i++ {
key := GetBlockInfoKey(i)
fmt.Println("getting blockinfo with key ", key)
data, err := storage.db.Get([]byte(key))
if err != nil {
fmt.Println("Error on getting from db")
os.Exit(1)
}
block := new(BlockInfo)
if rlp.DecodeBytes(data, block) != nil {
fmt.Println("RLP Decoding error")
os.Exit(1)
}
blocks = append(blocks, block)
fmt.Println("rlp decode successfully ")
}
return blocks
}
// GetExplorerBlock ...
func (s *Service) GetExplorerBlock(w http.ResponseWriter, r *http.Request) {
json.NewEncoder(w).Encode(s.data.Block)
// w.Header().Set("Content-Type", "application/json")
// id := r.FormValue("id")
// data := &Data{}
// db := s.storage.GetDB()
// if id == "" {
// json.NewEncoder(w).Encode(data.Block)
// return
// }
// idInt, err := strconv.Atoi(id)
// if err != nil {
// json.NewEncoder(w).Encode(data.Block)
// return
// }
// data, err := db.Get([]byte(GetBlockKey(idInt)))
// data.Block = storage.GetDB()
// json.NewEncoder(w).Encode(data.Blocks)
}
// GetExplorerAddress ...
func (s *Service) GetExplorerAddress(w http.ResponseWriter, r *http.Request) {
json.NewEncoder(w).Encode(s.data.Address)
// json.NewEncoder(w).Encode(s.data.Address)
}
// GetExplorerTransaction ...
func (s *Service) GetExplorerTransaction(w http.ResponseWriter, r *http.Request) {
// json.NewEncoder(w).Encode(s.data.Address)
}

@ -0,0 +1,118 @@
package explorer
import (
"fmt"
"os"
"strconv"
"sync"
"github.com/ethereum/go-ethereum/rlp"
"github.com/harmony-one/harmony/core/types"
"github.com/harmony-one/harmony/db"
)
// Constants for storage.
const (
BlockHeightKey = "bh"
BlockInfoPrefix = "bi"
BlockPrefix = "b"
TXPrefix = "tx"
)
// GetBlockInfoKey ...
func GetBlockInfoKey(id int) string {
return fmt.Sprintf("%s_%d", BlockInfoPrefix, id)
}
// GetBlockKey ...
func GetBlockKey(id int) string {
return fmt.Sprintf("%s_%d", BlockPrefix, id)
}
// GetTXKey ...
func GetTXKey(hash string) string {
return fmt.Sprintf("%s_%s", TXPrefix, hash)
}
var storage *Storage
var once sync.Once
// Storage dump the block info into leveldb.
type Storage struct {
db *db.LDBDatabase
}
// GetStorageInstance returns attack model by using singleton pattern.
func GetStorageInstance(ip, port string, remove bool) *Storage {
once.Do(func() {
storage = &Storage{}
storage.Init(ip, port, remove)
})
return storage
}
// Init initializes the block update.
func (storage *Storage) Init(ip, port string, remove bool) {
dbFileName := "/tmp/explorer_storage_" + ip + "_" + port
var err error
if remove {
var err = os.RemoveAll(dbFileName)
if err != nil {
fmt.Println(err.Error())
}
}
if storage.db, err = db.NewLDBDatabase(dbFileName, 0, 0); err != nil {
fmt.Println(err.Error())
os.Exit(1)
}
}
// GetDB returns the LDBDatabase of the storage.
func (storage *Storage) GetDB() *db.LDBDatabase {
return storage.db
}
// Dump extracts information from block and index them into lvdb for explorer.
func (storage *Storage) Dump(accountBlock []byte, height uint32) {
fmt.Println("Dumping block ", height)
if accountBlock == nil {
return
}
// Update block height.
storage.db.Put([]byte(BlockHeightKey), []byte(strconv.Itoa(int(height))))
// Store block.
block := new(types.Block)
rlp.DecodeBytes(accountBlock, block)
storage.db.Put([]byte(GetBlockKey(int(height))), accountBlock)
// Store block info.
blockInfo := BlockInfo{
ID: block.Hash().Hex(),
Height: string(height),
Timestamp: string(block.Time().Int64()),
TXCount: string(block.Transactions().Len()),
Size: block.Size().String(),
}
if data, err := rlp.EncodeToBytes(blockInfo); err == nil {
key := GetBlockInfoKey(int(height))
fmt.Println("store blockinfo with key ", key)
fmt.Println("data to store ", data)
storage.db.Put([]byte(key), data)
} else {
fmt.Println("EncodeRLP blockInfo error")
os.Exit(1)
}
// Store txs
for _, tx := range block.Transactions() {
if data, err := rlp.EncodeToBytes(tx); err == nil {
key := GetTXKey(tx.Hash().Hex())
storage.db.Put([]byte(key), data)
} else {
fmt.Println("EncodeRLP transaction error")
os.Exit(1)
}
}
}

@ -6,9 +6,9 @@ package explorer
// Data ...
type Data struct {
Blocks []Block `json:"blocks"`
Block Block2 `json:"block"`
Address Address `json:"address"`
Blocks []*BlockInfo `json:"blocks"`
Block Block `json:"block"`
Address Address `json:"address"`
}
// Address ...
@ -28,8 +28,8 @@ type Transaction struct {
Value float64 `json:"value"`
}
// Block ...
type Block struct {
// BlockInfo ...
type BlockInfo struct {
ID string `json:"id"`
Height string `json:"height"`
Timestamp string `json:"timestamp"`
@ -37,8 +37,8 @@ type Block struct {
Size string `json:"size"`
}
// Block2 ...
type Block2 struct {
// Block ...
type Block struct {
Height string `json:"height"`
Hash string `json:"hash"`
TXCount string `json:"txCount"`

@ -3,7 +3,10 @@ package main
import "github.com/harmony-one/harmony/services/explorer"
func main() {
service := &explorer.Service{}
service := &explorer.Service{
IP: "127.0.0.1",
Port: "9000",
}
service.Init()
service.Run()
}

Loading…
Cancel
Save