From 3815f3ca14c29e5a7b2583194fc9ae74940b9bd0 Mon Sep 17 00:00:00 2001 From: Richard Liu Date: Tue, 4 Sep 2018 19:07:55 -0700 Subject: [PATCH] update profiler to be singleton; add metricsReportURL argument. --- benchmark.go | 10 +++++--- consensus/consensus_leader.go | 18 ++++++------- go_executable_build.sh | 2 -- profiler/profiler.go | 48 +++++++++++++++++++++++++++-------- 4 files changed, 54 insertions(+), 24 deletions(-) diff --git a/benchmark.go b/benchmark.go index 45eb0f0eb..249944b26 100644 --- a/benchmark.go +++ b/benchmark.go @@ -50,6 +50,7 @@ func main() { attackedMode := flag.Int("attacked_mode", 0, "0 means not attacked, 1 means attacked, 2 means being open to be selected as attacked") dbSupported := flag.Bool("db_supported", false, "false means not db_supported, true means db_supported") profile := flag.Bool("profile", false, "Turn on profiling (CPU, Memory).") + metricsReportURL := flag.String("metrics_profile_url", "", "If set, reports metrics to this URL.") flag.Parse() // Set up randomization seed. @@ -90,9 +91,12 @@ func main() { consensus := consensus.NewConsensus(*ip, *port, shardID, peers, leader) // Start Profiler for leader if profile argument is on - if *profile && role == "leader" { - profiler := profiler.NewProfiler(consensus.Log, os.Getpid(), shardID) - profiler.Start() + if role == "leader" && (*profile || *metricsReportURL != "") { + prof := profiler.GetProfiler() + prof.Config(consensus.Log, shardID, *metricsReportURL) + if *profile { + prof.Start() + } } // Set logger to attack model. diff --git a/consensus/consensus_leader.go b/consensus/consensus_leader.go index 4ddc60f38..3a762e08a 100644 --- a/consensus/consensus_leader.go +++ b/consensus/consensus_leader.go @@ -6,11 +6,12 @@ import ( "encoding/gob" "encoding/hex" "errors" - "net/http" "net/url" "strconv" "time" + "github.com/simple-rules/harmony-benchmark/profiler" + "github.com/dedis/kyber" "github.com/dedis/kyber/sign/schnorr" "github.com/simple-rules/harmony-benchmark/blockchain" @@ -417,14 +418,18 @@ func (consensus *Consensus) reportMetrics(block blockchain.Block) { "consensus", consensus) // Post metrics - URL := "http://localhost:3000/report" + profiler := profiler.GetProfiler() + if profiler.MetricsReportURL == "" { + return + } + txHashes := []string{} for i := 1; i <= 3; i++ { if len(block.TransactionIds)-i >= 0 { txHashes = append(txHashes, hex.EncodeToString(block.TransactionIds[len(block.TransactionIds)-i][:])) } } - form := url.Values{ + metrics := url.Values{ "key": {consensus.pubKey.String()}, "tps": {strconv.FormatFloat(tps, 'f', 2, 64)}, "txCount": {strconv.Itoa(int(numOfTxs))}, @@ -433,10 +438,5 @@ func (consensus *Consensus) reportMetrics(block blockchain.Block) { "latestTxHashes": txHashes, "blockLatency": {strconv.Itoa(int(timeElapsed / time.Millisecond))}, } - - body := bytes.NewBufferString(form.Encode()) - rsp, err := http.Post(URL, "application/x-www-form-urlencoded", body) - if err == nil { - defer rsp.Body.Close() - } + profiler.LogMetrics(metrics) } diff --git a/go_executable_build.sh b/go_executable_build.sh index 49fdf186b..c90532e0d 100755 --- a/go_executable_build.sh +++ b/go_executable_build.sh @@ -4,7 +4,6 @@ GOOS=linux GOARCH=amd64 env GOOS=$GOOS GOARCH=$GOARCH go build -o bin/benchmark benchmark.go env GOOS=$GOOS GOARCH=$GOARCH go build -o bin/txgen client/txgen/main.go -env GOOS=$GOOS GOARCH=$GOARCH go build -o bin/profiler profiler/main.go AWSCLI=aws if [ "$1" != "" ]; then @@ -13,4 +12,3 @@ fi $AWSCLI s3 cp bin/benchmark s3://unique-bucket-bin/benchmark --acl public-read-write $AWSCLI s3 cp bin/txgen s3://unique-bucket-bin/txgen --acl public-read-write -$AWSCLI s3 cp bin/profiler s3://unique-bucket-bin/profiler --acl public-read-write diff --git a/profiler/profiler.go b/profiler/profiler.go index d5b1b00c3..aa0c72ec3 100644 --- a/profiler/profiler.go +++ b/profiler/profiler.go @@ -1,6 +1,11 @@ package profiler import ( + "bytes" + "net/http" + "net/url" + "os" + "sync" "time" "github.com/shirou/gopsutil/process" @@ -8,15 +13,30 @@ import ( ) type Profiler struct { - logger log.Logger - PID int32 - ShardID string - proc *process.Process + // parameters + logger log.Logger + pid int32 + shardID string + MetricsReportURL string + // Internal + proc *process.Process } -func NewProfiler(logger log.Logger, pid int, shardID string) *Profiler { - profiler := Profiler{logger, int32(pid), shardID, nil} - return &profiler +var singleton *Profiler +var once sync.Once + +func GetProfiler() *Profiler { + once.Do(func() { + singleton = &Profiler{} + }) + return singleton +} + +func (profiler *Profiler) Config(logger log.Logger, shardID string, metricsReportURL string) { + profiler.logger = logger + profiler.pid = int32(os.Getpid()) + profiler.shardID = shardID + profiler.MetricsReportURL = metricsReportURL } func (profiler *Profiler) LogMemory() { @@ -24,7 +44,7 @@ func (profiler *Profiler) LogMemory() { // log mem usage info, _ := profiler.proc.MemoryInfo() memMap, _ := profiler.proc.MemoryMaps(false) - profiler.logger.Info("Mem Report", "info", info, "map", memMap, "shardID", profiler.ShardID) + profiler.logger.Info("Mem Report", "info", info, "map", memMap, "shardID", profiler.shardID) time.Sleep(3 * time.Second) } @@ -35,14 +55,22 @@ func (profiler *Profiler) LogCPU() { // log cpu usage percent, _ := profiler.proc.CPUPercent() times, _ := profiler.proc.Times() - profiler.logger.Info("CPU Report", "percent", percent, "times", times, "shardID", profiler.ShardID) + profiler.logger.Info("CPU Report", "percent", percent, "times", times, "shardID", profiler.shardID) time.Sleep(3 * time.Second) } } +func (profiler *Profiler) LogMetrics(metrics url.Values) { + body := bytes.NewBufferString(metrics.Encode()) + rsp, err := http.Post(profiler.MetricsReportURL, "application/x-www-form-urlencoded", body) + if err == nil { + defer rsp.Body.Close() + } +} + func (profiler *Profiler) Start() { - profiler.proc, _ = process.NewProcess(profiler.PID) + profiler.proc, _ = process.NewProcess(profiler.pid) go profiler.LogCPU() go profiler.LogMemory() }