From 60410760c57ad5fc22311eab072a8542fa1b8cbf Mon Sep 17 00:00:00 2001 From: Edgar Aroutiounian Date: Wed, 8 Apr 2020 12:33:30 -0700 Subject: [PATCH] [project] Use tracepointer (#2785) * [project] Use tracepointer * [project] Remove dead dependencies * [rpc] Add unix time at boot as value for node metadata rpc - need direct machine uptime --- Makefile | 3 ++ go.mod | 4 --- hmy/api_backend.go | 37 +++++++++++++++++++++++ hmy/backend.go | 1 + internal/hmyapi/apiv1/backend.go | 2 ++ internal/hmyapi/apiv1/harmony.go | 52 ++------------------------------ internal/hmyapi/apiv2/backend.go | 2 ++ internal/hmyapi/apiv2/harmony.go | 36 ++-------------------- internal/hmyapi/backend.go | 17 ++--------- internal/hmyapi/common/types.go | 19 ++++++++++++ node/node.go | 3 ++ node/rpc.go | 5 +++ scripts/go_executable_build.sh | 11 ++++--- 13 files changed, 88 insertions(+), 104 deletions(-) create mode 100644 internal/hmyapi/common/types.go diff --git a/Makefile b/Makefile index 28ebba790..db3dff4f2 100644 --- a/Makefile +++ b/Makefile @@ -21,6 +21,9 @@ exe: race: ./scripts/go_executable_build.sh -r +trace-pointer: + ./scripts/go_executable_build.sh -t + test: ./test/debug.sh diff --git a/go.mod b/go.mod index bd9e15ec1..5f646fd2b 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,6 @@ require ( github.com/deckarep/golang-set v1.7.1 github.com/edsrzf/mmap-go v1.0.0 // indirect github.com/ethereum/go-ethereum v1.8.27 - github.com/fatih/color v1.7.0 // indirect github.com/fjl/memsize v0.0.0-20180929194037-2a09253e352a // indirect github.com/garslo/gogen v0.0.0-20170307003452-d6ebae628c7c // indirect github.com/golang/mock v1.3.1 @@ -52,14 +51,12 @@ require ( github.com/natefinch/lumberjack v2.0.0+incompatible github.com/pborman/uuid v1.2.0 github.com/pkg/errors v0.9.1 - github.com/prometheus/client_golang v0.9.3 // indirect github.com/prometheus/common v0.4.1 // indirect github.com/prometheus/procfs v0.0.3 // indirect github.com/rjeczalik/notify v0.9.2 github.com/rs/cors v1.7.0 // indirect github.com/rs/zerolog v1.18.0 github.com/shirou/gopsutil v2.18.12+incompatible // indirect - github.com/spf13/cobra v0.0.5 // indirect github.com/spf13/viper v1.6.1 github.com/stretchr/testify v1.5.1 github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965 @@ -74,7 +71,6 @@ require ( golang.org/x/tools v0.0.0-20200408032209-46bd65c8538f google.golang.org/grpc v1.22.0 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 - gopkg.in/ini.v1 v1.51.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect gopkg.in/urfave/cli.v1 v1.20.0 // indirect diff --git a/hmy/api_backend.go b/hmy/api_backend.go index 928c96e0c..481ef18e9 100644 --- a/hmy/api_backend.go +++ b/hmy/api_backend.go @@ -20,6 +20,8 @@ import ( "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/core/vm" internal_common "github.com/harmony-one/harmony/internal/common" + nodeconfig "github.com/harmony-one/harmony/internal/configs/node" + commonRPC "github.com/harmony-one/harmony/internal/hmyapi/common" "github.com/harmony-one/harmony/internal/params" "github.com/harmony-one/harmony/numeric" "github.com/harmony-one/harmony/shard" @@ -597,3 +599,38 @@ func (b *APIBackend) GetLastCrossLinks() ([]*types.CrossLink, error) { return crossLinks, nil } + +// GetNodeMetadata .. +func (b *APIBackend) GetNodeMetadata() commonRPC.NodeMetadata { + cfg := nodeconfig.GetDefaultConfig() + header := b.CurrentBlock().Header() + var blockEpoch *uint64 + + if header.ShardID() == shard.BeaconChainShardID { + sched := shard.Schedule.InstanceForEpoch(header.Epoch()) + b := sched.BlocksPerEpoch() + blockEpoch = &b + } + + blsKeys := []string{} + if cfg.ConsensusPubKey != nil { + for _, key := range cfg.ConsensusPubKey.PublicKey { + blsKeys = append(blsKeys, key.SerializeToHexStr()) + } + } + + return commonRPC.NodeMetadata{ + blsKeys, + nodeconfig.GetVersion(), + string(cfg.GetNetworkType()), + *b.ChainConfig(), + b.IsLeader(), + b.GetShardID(), + header.Epoch().Uint64(), + blockEpoch, + cfg.Role().String(), + cfg.DNSZone, + cfg.GetArchival(), + b.hmy.nodeAPI.GetNodeBootTime(), + } +} diff --git a/hmy/backend.go b/hmy/backend.go index 5c091160c..a3f3b8d8d 100644 --- a/hmy/backend.go +++ b/hmy/backend.go @@ -50,6 +50,7 @@ type NodeAPI interface { ErroredStakingTransactionSink() []staking.RPCTransactionError ErroredTransactionSink() []types.RPCTransactionError PendingCXReceipts() []*types.CXReceiptsProof + GetNodeBootTime() int64 } // New creates a new Harmony object (including the diff --git a/internal/hmyapi/apiv1/backend.go b/internal/hmyapi/apiv1/backend.go index e00ffb617..8be8e43ec 100644 --- a/internal/hmyapi/apiv1/backend.go +++ b/internal/hmyapi/apiv1/backend.go @@ -14,6 +14,7 @@ import ( "github.com/harmony-one/harmony/core/state" "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/core/vm" + commonRPC "github.com/harmony-one/harmony/internal/hmyapi/common" "github.com/harmony-one/harmony/internal/params" "github.com/harmony-one/harmony/shard" "github.com/harmony-one/harmony/shard/committee" @@ -84,4 +85,5 @@ type Backend interface { GetCurrentBadBlocks() []core.BadBlock GetLastCrossLinks() ([]*types.CrossLink, error) GetLatestChainHeaders() *block.HeaderPair + GetNodeMetadata() commonRPC.NodeMetadata } diff --git a/internal/hmyapi/apiv1/harmony.go b/internal/hmyapi/apiv1/harmony.go index c48cb8f2d..141d373d7 100644 --- a/internal/hmyapi/apiv1/harmony.go +++ b/internal/hmyapi/apiv1/harmony.go @@ -6,9 +6,7 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" "github.com/harmony-one/harmony/api/proto" - nodeconfig "github.com/harmony-one/harmony/internal/configs/node" - "github.com/harmony-one/harmony/internal/params" - "github.com/harmony-one/harmony/shard" + commonRPC "github.com/harmony-one/harmony/internal/hmyapi/common" ) // PublicHarmonyAPI provides an API to access Harmony related information. @@ -45,51 +43,7 @@ func (s *PublicHarmonyAPI) GasPrice(ctx context.Context) (*hexutil.Big, error) { return (*hexutil.Big)(big.NewInt(1)), nil } -// NodeMetadata captures select metadata of the RPC answering node -type NodeMetadata struct { - BLSPublicKey []string `json:"blskey"` - Version string `json:"version"` - NetworkType string `json:"network"` - ChainConfig params.ChainConfig `json:"chain-config"` - IsLeader bool `json:"is-leader"` - ShardID uint32 `json:"shard-id"` - CurrentEpoch uint64 `json:"current-epoch"` - BlocksPerEpoch *uint64 `json:"blocks-per-epoch,omitempty"` - Role string `json:"role"` - DNSZone string `json:"dns-zone"` - Archival bool `json:"is-archival"` -} - // GetNodeMetadata produces a NodeMetadata record, data is from the answering RPC node -func (s *PublicHarmonyAPI) GetNodeMetadata() NodeMetadata { - cfg := nodeconfig.GetDefaultConfig() - header := s.b.CurrentBlock().Header() - var blockEpoch *uint64 - - if header.ShardID() == shard.BeaconChainShardID { - sched := shard.Schedule.InstanceForEpoch(header.Epoch()) - b := sched.BlocksPerEpoch() - blockEpoch = &b - } - - blsKeys := []string{} - if cfg.ConsensusPubKey != nil { - for _, key := range cfg.ConsensusPubKey.PublicKey { - blsKeys = append(blsKeys, key.SerializeToHexStr()) - } - } - - return NodeMetadata{ - blsKeys, - nodeconfig.GetVersion(), - string(cfg.GetNetworkType()), - *s.b.ChainConfig(), - s.b.IsLeader(), - s.b.GetShardID(), - header.Epoch().Uint64(), - blockEpoch, - cfg.Role().String(), - cfg.DNSZone, - cfg.GetArchival(), - } +func (s *PublicHarmonyAPI) GetNodeMetadata() commonRPC.NodeMetadata { + return s.b.GetNodeMetadata() } diff --git a/internal/hmyapi/apiv2/backend.go b/internal/hmyapi/apiv2/backend.go index 0e4f9dd65..a56cf064d 100644 --- a/internal/hmyapi/apiv2/backend.go +++ b/internal/hmyapi/apiv2/backend.go @@ -14,6 +14,7 @@ import ( "github.com/harmony-one/harmony/core/state" "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/core/vm" + commonRPC "github.com/harmony-one/harmony/internal/hmyapi/common" "github.com/harmony-one/harmony/internal/params" "github.com/harmony-one/harmony/shard" "github.com/harmony-one/harmony/shard/committee" @@ -78,4 +79,5 @@ type Backend interface { GetCurrentBadBlocks() []core.BadBlock GetLastCrossLinks() ([]*types.CrossLink, error) GetLatestChainHeaders() *block.HeaderPair + GetNodeMetadata() commonRPC.NodeMetadata } diff --git a/internal/hmyapi/apiv2/harmony.go b/internal/hmyapi/apiv2/harmony.go index 11c723841..0eccaf5e6 100644 --- a/internal/hmyapi/apiv2/harmony.go +++ b/internal/hmyapi/apiv2/harmony.go @@ -5,9 +5,8 @@ import ( "math/big" "github.com/harmony-one/harmony/api/proto" - nodeconfig "github.com/harmony-one/harmony/internal/configs/node" + commonRPC "github.com/harmony-one/harmony/internal/hmyapi/common" "github.com/harmony-one/harmony/internal/params" - "github.com/harmony-one/harmony/shard" ) // PublicHarmonyAPI provides an API to access Harmony related information. @@ -60,35 +59,6 @@ type NodeMetadata struct { } // GetNodeMetadata produces a NodeMetadata record, data is from the answering RPC node -func (s *PublicHarmonyAPI) GetNodeMetadata() NodeMetadata { - cfg := nodeconfig.GetDefaultConfig() - header := s.b.CurrentBlock().Header() - var blockEpoch *uint64 - - if header.ShardID() == shard.BeaconChainShardID { - sched := shard.Schedule.InstanceForEpoch(header.Epoch()) - b := sched.BlocksPerEpoch() - blockEpoch = &b - } - - blsKeys := []string{} - if cfg.ConsensusPubKey != nil { - for _, key := range cfg.ConsensusPubKey.PublicKey { - blsKeys = append(blsKeys, key.SerializeToHexStr()) - } - } - - return NodeMetadata{ - blsKeys, - nodeconfig.GetVersion(), - string(cfg.GetNetworkType()), - *s.b.ChainConfig(), - s.b.IsLeader(), - s.b.GetShardID(), - header.Epoch().Uint64(), - blockEpoch, - cfg.Role().String(), - cfg.DNSZone, - cfg.GetArchival(), - } +func (s *PublicHarmonyAPI) GetNodeMetadata() commonRPC.NodeMetadata { + return s.b.GetNodeMetadata() } diff --git a/internal/hmyapi/backend.go b/internal/hmyapi/backend.go index df43606bb..1a3cf5a6a 100644 --- a/internal/hmyapi/backend.go +++ b/internal/hmyapi/backend.go @@ -16,6 +16,7 @@ import ( "github.com/harmony-one/harmony/core/vm" "github.com/harmony-one/harmony/internal/hmyapi/apiv1" "github.com/harmony-one/harmony/internal/hmyapi/apiv2" + commonRPC "github.com/harmony-one/harmony/internal/hmyapi/common" "github.com/harmony-one/harmony/internal/params" "github.com/harmony-one/harmony/shard" "github.com/harmony-one/harmony/shard/committee" @@ -23,12 +24,8 @@ import ( staking "github.com/harmony-one/harmony/staking/types" ) -// Backend interface provides the common API services (that are provided by -// both full and light clients) with access to necessary functions. -// implementations: -// * hmy/api_backend.go +// Backend .. type Backend interface { - // NOTE(ricl): this is not in ETH Backend inteface. They put it directly in eth object. NetVersion() uint64 ProtocolVersion() int ChainDb() ethdb.Database @@ -39,32 +36,23 @@ type Backend interface { StateAndHeaderByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*state.DB, *block.Header, error) GetBlock(ctx context.Context, blockHash common.Hash) (*types.Block, error) GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error) - // GetTd(blockHash common.Hash) *big.Int GetEVM(ctx context.Context, msg core.Message, state *state.DB, header *block.Header) (*vm.EVM, func() error, error) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription SubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription SubscribeChainSideEvent(ch chan<- core.ChainSideEvent) event.Subscription // TxPool API SendTx(ctx context.Context, signedTx *types.Transaction) error - // GetTransaction(ctx context.Context, txHash common.Hash) (*types.Transaction, common.Hash, uint64, uint64, error) GetPoolTransactions() (types.PoolTransactions, error) GetPoolTransaction(txHash common.Hash) types.PoolTransaction GetPoolNonce(ctx context.Context, addr common.Address) (uint64, error) - // Stats() (pending int, queued int) - // TxPoolContent() (map[common.Address]types.Transactions, map[common.Address]types.Transactions) SubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription ChainConfig() *params.ChainConfig CurrentBlock() *types.Block - // Get balance GetBalance(ctx context.Context, address common.Address, blockNr rpc.BlockNumber) (*big.Int, error) - // Get validators for a particular epoch GetValidators(epoch *big.Int) (*shard.Committee, error) GetShardID() uint32 - // Get transactions history for an address GetTransactionsHistory(address, txType, order string) ([]common.Hash, error) - // Get staking transactions history for an address GetStakingTransactionsHistory(address, txType, order string) ([]common.Hash, error) - // retrieve the blockHash using txID and add blockHash to CxPool for resending ResendCx(ctx context.Context, txID common.Hash) (uint64, bool) IsLeader() bool SendStakingTx(ctx context.Context, newStakingTx *staking.StakingTransaction) error @@ -85,6 +73,7 @@ type Backend interface { GetCurrentBadBlocks() []core.BadBlock GetLastCrossLinks() ([]*types.CrossLink, error) GetLatestChainHeaders() *block.HeaderPair + GetNodeMetadata() commonRPC.NodeMetadata } // GetAPIs returns all the APIs. diff --git a/internal/hmyapi/common/types.go b/internal/hmyapi/common/types.go new file mode 100644 index 000000000..19fa7d2bb --- /dev/null +++ b/internal/hmyapi/common/types.go @@ -0,0 +1,19 @@ +package common + +import "github.com/harmony-one/harmony/internal/params" + +// NodeMetadata captures select metadata of the RPC answering node +type NodeMetadata struct { + BLSPublicKey []string `json:"blskey"` + Version string `json:"version"` + NetworkType string `json:"network"` + ChainConfig params.ChainConfig `json:"chain-config"` + IsLeader bool `json:"is-leader"` + ShardID uint32 `json:"shard-id"` + CurrentEpoch uint64 `json:"current-epoch"` + BlocksPerEpoch *uint64 `json:"blocks-per-epoch,omitempty"` + Role string `json:"role"` + DNSZone string `json:"dns-zone"` + Archival bool `json:"is-archival"` + NodeBootTime int64 `json:"node-unix-start-time"` +} diff --git a/node/node.go b/node/node.go index fda742719..81149ed66 100644 --- a/node/node.go +++ b/node/node.go @@ -7,6 +7,7 @@ import ( "os" "strings" "sync" + "time" "github.com/ethereum/go-ethereum/common" "github.com/harmony-one/harmony/api/client" @@ -179,6 +180,7 @@ type Node struct { failedStakingTxns *ring.Ring failedTxns *ring.Ring } + unixTimeAtNodeStart int64 } // Blockchain returns the blockchain for the node's current shard. @@ -408,6 +410,7 @@ func New( isArchival bool, ) *Node { node := Node{} + node.unixTimeAtNodeStart = time.Now().Unix() const sinkSize = 4096 node.errorSink = struct { sync.Mutex diff --git a/node/rpc.go b/node/rpc.go index 48870e33d..a037cfcc2 100644 --- a/node/rpc.go +++ b/node/rpc.go @@ -68,6 +68,11 @@ func (node *Node) ErroredStakingTransactionSink() []staking.RPCTransactionError return result } +// GetNodeBootTime .. +func (node *Node) GetNodeBootTime() int64 { + return node.unixTimeAtNodeStart +} + // ErroredTransactionSink is the inmemory failed transactions this node has func (node *Node) ErroredTransactionSink() []types.RPCTransactionError { node.errorSink.Lock() diff --git a/scripts/go_executable_build.sh b/scripts/go_executable_build.sh index c938f716d..5f670d592 100755 --- a/scripts/go_executable_build.sh +++ b/scripts/go_executable_build.sh @@ -14,6 +14,7 @@ GOOS=linux GOARCH=amd64 FOLDER=${WHOAMI:-$USER} RACE= +TRACEPTR= VERBOSE= GO_GCFLAGS="all=-c 2" DEBUG=false @@ -59,6 +60,7 @@ OPTIONS: -b bucket set the upload bucket name (default: $BUCKET) -f folder set the upload folder name in the bucket (default: $FOLDER) -r enable -race build option (default: $RACE) + -t full analysis on {pointer} build option (default: $TRACEPTR) -v verbose build process (default: $VERBOSE) -s build static linux executable (default: $STATIC) @@ -107,12 +109,12 @@ function build_only rm -f $BINDIR/$bin echo "building ${SRC[$bin]}" if [ "$DEBUG" == "true" ]; then - env GOOS=$GOOS GOARCH=$GOARCH go build $VERBOSE -gcflags="${GO_GCFLAGS}" -ldflags="-X main.version=v${VERSION} -X main.commit=${COMMIT} -X main.builtAt=${BUILTAT} -X main.builtBy=${BUILTBY}" -o $BINDIR/$bin $RACE ${SRC[$bin]} + env GOOS=$GOOS GOARCH=$GOARCH go build $VERBOSE -gcflags="${GO_GCFLAGS}" -ldflags="-X main.version=v${VERSION} -X main.commit=${COMMIT} -X main.builtAt=${BUILTAT} -X main.builtBy=${BUILTBY}" -o $BINDIR/$bin $RACE $TRACEPTR ${SRC[$bin]} else if [ "$STATIC" == "true" ]; then - env GOOS=$GOOS GOARCH=$GOARCH go build $VERBOSE -gcflags="${GO_GCFLAGS}" -ldflags="-X main.version=v${VERSION} -X main.commit=${COMMIT} -X main.builtAt=${BUILTAT} -X main.builtBy=${BUILTBY} -w -extldflags \"-static -lm\"" -o $BINDIR/$bin $RACE ${SRC[$bin]} + env GOOS=$GOOS GOARCH=$GOARCH go build $VERBOSE -gcflags="${GO_GCFLAGS}" -ldflags="-X main.version=v${VERSION} -X main.commit=${COMMIT} -X main.builtAt=${BUILTAT} -X main.builtBy=${BUILTBY} -w -extldflags \"-static -lm\"" -o $BINDIR/$bin $RACE $TRACEPTR ${SRC[$bin]} else - env GOOS=$GOOS GOARCH=$GOARCH go build $VERBOSE -gcflags="${GO_GCFLAGS}" -ldflags="-X main.version=v${VERSION} -X main.commit=${COMMIT} -X main.builtAt=${BUILTAT} -X main.builtBy=${BUILTBY}" -o $BINDIR/$bin $RACE ${SRC[$bin]} + env GOOS=$GOOS GOARCH=$GOARCH go build $VERBOSE -gcflags="${GO_GCFLAGS}" -ldflags="-X main.version=v${VERSION} -X main.commit=${COMMIT} -X main.builtAt=${BUILTAT} -X main.builtBy=${BUILTBY}" -o $BINDIR/$bin $RACE $TRACEPTR ${SRC[$bin]} fi fi if [ "$(uname -s)" == "Linux" ]; then @@ -231,7 +233,7 @@ function release ################################ MAIN FUNCTION ############################## -while getopts "hp:a:o:b:f:rvsdN:" option; do +while getopts "hp:a:o:b:f:rtvsdN:" option; do case $option in h) usage ;; p) PROFILE=$OPTARG ;; @@ -240,6 +242,7 @@ while getopts "hp:a:o:b:f:rvsdN:" option; do b) BUCKET=$OPTARG/ ;; f) FOLDER=$OPTARG ;; r) RACE=-race ;; + t) TRACEPTR='-gcflags=all=-d=checkptr' ;; v) VERBOSE='-v -x' ;; d) DEBUG=true ;; s) STATIC=true ;;