Minh Doan
6 years ago
commit
4ade99f476
@ -1,9 +0,0 @@ |
|||||||
if [ $# -eq 0 ]; then |
|
||||||
echo "Please the directory of the log" |
|
||||||
exit 1 |
|
||||||
fi |
|
||||||
DIR=$1 |
|
||||||
for file in $(ls $DIR/*leader*) |
|
||||||
do |
|
||||||
cat $file | egrep -o "TPS=[0-9]+" | cut -f2 -d "=" | awk '{ sum += $1; n++ } END { if (n > 0) print sum / n; }'; |
|
||||||
done |
|
||||||
@ -0,0 +1,96 @@ |
|||||||
|
import json |
||||||
|
import sys |
||||||
|
import os |
||||||
|
import argparse |
||||||
|
|
||||||
|
def formatFloat(v): |
||||||
|
return "%.2f" % v |
||||||
|
|
||||||
|
def formatPercent(v): |
||||||
|
return formatFloat(v) + "%" |
||||||
|
|
||||||
|
def formatMem(v): |
||||||
|
return formatFloat(float(v) / 10**6) + "MB" |
||||||
|
|
||||||
|
class Profiler: |
||||||
|
def __init__(self): |
||||||
|
self.tps = 0 |
||||||
|
self.tps_max = 0 |
||||||
|
self.tps_min = sys.maxsize |
||||||
|
self.tps_count = 0 |
||||||
|
|
||||||
|
self.cpu_percent = 0 |
||||||
|
self.cpu_usr = 0 |
||||||
|
self.cpu_sys = 0 |
||||||
|
self.cpu_count = 0 |
||||||
|
|
||||||
|
self.mem_rss = 0 |
||||||
|
self.mem_rss_max = 0 |
||||||
|
self.mem_count = 0 |
||||||
|
|
||||||
|
def handleTPS(self, obj): |
||||||
|
tps = obj["TPS"] |
||||||
|
self.tps += tps |
||||||
|
self.tps_max = max(self.tps_max, tps) |
||||||
|
self.tps_min = min(self.tps_min, tps) |
||||||
|
self.tps_count += 1 |
||||||
|
|
||||||
|
def handleCPU(self, obj): |
||||||
|
# http://psutil.readthedocs.io/en/latest/#psutil.Process.cpu_times |
||||||
|
# https://stackoverflow.com/questions/556405/what-do-real-user-and-sys-mean-in-the-output-of-time1 |
||||||
|
# http://psutil.readthedocs.io/en/latest/#psutil.Process.cpu_percent |
||||||
|
self.cpu_percent += obj["percent"] |
||||||
|
times = json.loads(obj["times"]) |
||||||
|
self.cpu_usr = times["user"] |
||||||
|
self.cpu_sys = times["system"] |
||||||
|
self.cpu_count += 1 |
||||||
|
|
||||||
|
def handleMem(self, obj): |
||||||
|
# http://psutil.readthedocs.io/en/latest/#psutil.Process.memory_info |
||||||
|
info = json.loads(obj["info"]) |
||||||
|
rss = info["rss"] |
||||||
|
self.mem_rss += rss |
||||||
|
self.mem_rss_max = max(self.mem_rss_max, rss) |
||||||
|
self.mem_count += 1 |
||||||
|
|
||||||
|
def report(self): |
||||||
|
print("TPS", |
||||||
|
"Avg", formatFloat(self.tps / self.tps_count), |
||||||
|
"Min", formatFloat(self.tps_min), |
||||||
|
"Max", formatFloat(self.tps_max)) |
||||||
|
print("CPU", |
||||||
|
"Percent (Avg)", formatPercent(self.cpu_percent / self.cpu_count), |
||||||
|
"Time (Usr)", str(self.cpu_usr) + "s", |
||||||
|
"Time (Sys)", str(self.cpu_sys) + "s") |
||||||
|
print("Mem", |
||||||
|
"RSS (Max)", formatMem(self.mem_rss_max), |
||||||
|
"RSS (Avg)", formatMem(self.mem_rss / self.mem_count)) |
||||||
|
|
||||||
|
def profileFile(path): |
||||||
|
print(path) |
||||||
|
profiler = Profiler() |
||||||
|
with open(path) as f: |
||||||
|
for line in f: |
||||||
|
obj = json.loads(line) |
||||||
|
if obj["lvl"] != "info": |
||||||
|
continue |
||||||
|
if obj["msg"] == "TPS Report": |
||||||
|
profiler.handleTPS(obj) |
||||||
|
elif obj["msg"] == "CPU Report": |
||||||
|
profiler.handleCPU(obj) |
||||||
|
elif obj["msg"] == "Mem Report": |
||||||
|
profiler.handleMem(obj) |
||||||
|
profiler.report() |
||||||
|
|
||||||
|
# Example: python report_extractor.py --folder ../tmp_log/log-20180713-205431 |
||||||
|
if __name__ == "__main__": |
||||||
|
parser = argparse.ArgumentParser( |
||||||
|
description="This script extracts reports from log files") |
||||||
|
parser.add_argument("--folder", type=str, dest="folder", |
||||||
|
default="", |
||||||
|
help="the path to the log folder") |
||||||
|
args = parser.parse_args() |
||||||
|
|
||||||
|
for filename in os.listdir(args.folder): |
||||||
|
if "leader" in filename: |
||||||
|
profileFile(os.path.join(args.folder, filename)) |
||||||
@ -1,324 +0,0 @@ |
|||||||
// +build go1.7
|
|
||||||
|
|
||||||
// Package stack implements utilities to capture, manipulate, and format call
|
|
||||||
// stacks. It provides a simpler API than package runtime.
|
|
||||||
//
|
|
||||||
// The implementation takes care of the minutia and special cases of
|
|
||||||
// interpreting the program counter (pc) values returned by runtime.Callers.
|
|
||||||
//
|
|
||||||
// Package stack's types implement fmt.Formatter, which provides a simple and
|
|
||||||
// flexible way to declaratively configure formatting when used with logging
|
|
||||||
// or error tracking packages.
|
|
||||||
package stack |
|
||||||
|
|
||||||
import ( |
|
||||||
"bytes" |
|
||||||
"errors" |
|
||||||
"fmt" |
|
||||||
"io" |
|
||||||
"runtime" |
|
||||||
"strconv" |
|
||||||
"strings" |
|
||||||
) |
|
||||||
|
|
||||||
// Call records a single function invocation from a goroutine stack.
|
|
||||||
type Call struct { |
|
||||||
frame runtime.Frame |
|
||||||
} |
|
||||||
|
|
||||||
// Caller returns a Call from the stack of the current goroutine. The argument
|
|
||||||
// skip is the number of stack frames to ascend, with 0 identifying the
|
|
||||||
// calling function.
|
|
||||||
func Caller(skip int) Call { |
|
||||||
// As of Go 1.9 we need room for up to three PC entries.
|
|
||||||
//
|
|
||||||
// 0. An entry for the stack frame prior to the target to check for
|
|
||||||
// special handling needed if that prior entry is runtime.sigpanic.
|
|
||||||
// 1. A possible second entry to hold metadata about skipped inlined
|
|
||||||
// functions. If inline functions were not skipped the target frame
|
|
||||||
// PC will be here.
|
|
||||||
// 2. A third entry for the target frame PC when the second entry
|
|
||||||
// is used for skipped inline functions.
|
|
||||||
var pcs [3]uintptr |
|
||||||
n := runtime.Callers(skip+1, pcs[:]) |
|
||||||
frames := runtime.CallersFrames(pcs[:n]) |
|
||||||
frame, _ := frames.Next() |
|
||||||
frame, _ = frames.Next() |
|
||||||
|
|
||||||
return Call{ |
|
||||||
frame: frame, |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
// String implements fmt.Stinger. It is equivalent to fmt.Sprintf("%v", c).
|
|
||||||
func (c Call) String() string { |
|
||||||
return fmt.Sprint(c) |
|
||||||
} |
|
||||||
|
|
||||||
// MarshalText implements encoding.TextMarshaler. It formats the Call the same
|
|
||||||
// as fmt.Sprintf("%v", c).
|
|
||||||
func (c Call) MarshalText() ([]byte, error) { |
|
||||||
if c.frame == (runtime.Frame{}) { |
|
||||||
return nil, ErrNoFunc |
|
||||||
} |
|
||||||
|
|
||||||
buf := bytes.Buffer{} |
|
||||||
fmt.Fprint(&buf, c) |
|
||||||
return buf.Bytes(), nil |
|
||||||
} |
|
||||||
|
|
||||||
// ErrNoFunc means that the Call has a nil *runtime.Func. The most likely
|
|
||||||
// cause is a Call with the zero value.
|
|
||||||
var ErrNoFunc = errors.New("no call stack information") |
|
||||||
|
|
||||||
// Format implements fmt.Formatter with support for the following verbs.
|
|
||||||
//
|
|
||||||
// %s source file
|
|
||||||
// %d line number
|
|
||||||
// %n function name
|
|
||||||
// %k last segment of the package path
|
|
||||||
// %v equivalent to %s:%d
|
|
||||||
//
|
|
||||||
// It accepts the '+' and '#' flags for most of the verbs as follows.
|
|
||||||
//
|
|
||||||
// %+s path of source file relative to the compile time GOPATH
|
|
||||||
// %#s full path of source file
|
|
||||||
// %+n import path qualified function name
|
|
||||||
// %+k full package path
|
|
||||||
// %+v equivalent to %+s:%d
|
|
||||||
// %#v equivalent to %#s:%d
|
|
||||||
func (c Call) Format(s fmt.State, verb rune) { |
|
||||||
if c.frame == (runtime.Frame{}) { |
|
||||||
fmt.Fprintf(s, "%%!%c(NOFUNC)", verb) |
|
||||||
return |
|
||||||
} |
|
||||||
|
|
||||||
switch verb { |
|
||||||
case 's', 'v': |
|
||||||
file := c.frame.File |
|
||||||
switch { |
|
||||||
case s.Flag('#'): |
|
||||||
// done
|
|
||||||
case s.Flag('+'): |
|
||||||
file = file[pkgIndex(file, c.frame.Function):] |
|
||||||
default: |
|
||||||
const sep = "/" |
|
||||||
if i := strings.LastIndex(file, sep); i != -1 { |
|
||||||
file = file[i+len(sep):] |
|
||||||
} |
|
||||||
} |
|
||||||
io.WriteString(s, file) |
|
||||||
if verb == 'v' { |
|
||||||
buf := [7]byte{':'} |
|
||||||
s.Write(strconv.AppendInt(buf[:1], int64(c.frame.Line), 10)) |
|
||||||
} |
|
||||||
|
|
||||||
case 'd': |
|
||||||
buf := [6]byte{} |
|
||||||
s.Write(strconv.AppendInt(buf[:0], int64(c.frame.Line), 10)) |
|
||||||
|
|
||||||
case 'k': |
|
||||||
name := c.frame.Function |
|
||||||
const pathSep = "/" |
|
||||||
start, end := 0, len(name) |
|
||||||
if i := strings.LastIndex(name, pathSep); i != -1 { |
|
||||||
start = i + len(pathSep) |
|
||||||
} |
|
||||||
const pkgSep = "." |
|
||||||
if i := strings.Index(name[start:], pkgSep); i != -1 { |
|
||||||
end = start + i |
|
||||||
} |
|
||||||
if s.Flag('+') { |
|
||||||
start = 0 |
|
||||||
} |
|
||||||
io.WriteString(s, name[start:end]) |
|
||||||
|
|
||||||
case 'n': |
|
||||||
name := c.frame.Function |
|
||||||
if !s.Flag('+') { |
|
||||||
const pathSep = "/" |
|
||||||
if i := strings.LastIndex(name, pathSep); i != -1 { |
|
||||||
name = name[i+len(pathSep):] |
|
||||||
} |
|
||||||
const pkgSep = "." |
|
||||||
if i := strings.Index(name, pkgSep); i != -1 { |
|
||||||
name = name[i+len(pkgSep):] |
|
||||||
} |
|
||||||
} |
|
||||||
io.WriteString(s, name) |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
// Frame returns the call frame infomation for the Call.
|
|
||||||
func (c Call) Frame() runtime.Frame { |
|
||||||
return c.frame |
|
||||||
} |
|
||||||
|
|
||||||
// PC returns the program counter for this call frame; multiple frames may
|
|
||||||
// have the same PC value.
|
|
||||||
//
|
|
||||||
// Deprecated: Use Call.Frame instead.
|
|
||||||
func (c Call) PC() uintptr { |
|
||||||
return c.frame.PC |
|
||||||
} |
|
||||||
|
|
||||||
// CallStack records a sequence of function invocations from a goroutine
|
|
||||||
// stack.
|
|
||||||
type CallStack []Call |
|
||||||
|
|
||||||
// String implements fmt.Stinger. It is equivalent to fmt.Sprintf("%v", cs).
|
|
||||||
func (cs CallStack) String() string { |
|
||||||
return fmt.Sprint(cs) |
|
||||||
} |
|
||||||
|
|
||||||
var ( |
|
||||||
openBracketBytes = []byte("[") |
|
||||||
closeBracketBytes = []byte("]") |
|
||||||
spaceBytes = []byte(" ") |
|
||||||
) |
|
||||||
|
|
||||||
// MarshalText implements encoding.TextMarshaler. It formats the CallStack the
|
|
||||||
// same as fmt.Sprintf("%v", cs).
|
|
||||||
func (cs CallStack) MarshalText() ([]byte, error) { |
|
||||||
buf := bytes.Buffer{} |
|
||||||
buf.Write(openBracketBytes) |
|
||||||
for i, pc := range cs { |
|
||||||
if i > 0 { |
|
||||||
buf.Write(spaceBytes) |
|
||||||
} |
|
||||||
fmt.Fprint(&buf, pc) |
|
||||||
} |
|
||||||
buf.Write(closeBracketBytes) |
|
||||||
return buf.Bytes(), nil |
|
||||||
} |
|
||||||
|
|
||||||
// Format implements fmt.Formatter by printing the CallStack as square brackets
|
|
||||||
// ([, ]) surrounding a space separated list of Calls each formatted with the
|
|
||||||
// supplied verb and options.
|
|
||||||
func (cs CallStack) Format(s fmt.State, verb rune) { |
|
||||||
s.Write(openBracketBytes) |
|
||||||
for i, pc := range cs { |
|
||||||
if i > 0 { |
|
||||||
s.Write(spaceBytes) |
|
||||||
} |
|
||||||
pc.Format(s, verb) |
|
||||||
} |
|
||||||
s.Write(closeBracketBytes) |
|
||||||
} |
|
||||||
|
|
||||||
// Trace returns a CallStack for the current goroutine with element 0
|
|
||||||
// identifying the calling function.
|
|
||||||
func Trace() CallStack { |
|
||||||
var pcs [512]uintptr |
|
||||||
n := runtime.Callers(1, pcs[:]) |
|
||||||
|
|
||||||
frames := runtime.CallersFrames(pcs[:n]) |
|
||||||
cs := make(CallStack, 0, n) |
|
||||||
|
|
||||||
// Skip extra frame retrieved just to make sure the runtime.sigpanic
|
|
||||||
// special case is handled.
|
|
||||||
frame, more := frames.Next() |
|
||||||
|
|
||||||
for more { |
|
||||||
frame, more = frames.Next() |
|
||||||
cs = append(cs, Call{frame: frame}) |
|
||||||
} |
|
||||||
|
|
||||||
return cs |
|
||||||
} |
|
||||||
|
|
||||||
// TrimBelow returns a slice of the CallStack with all entries below c
|
|
||||||
// removed.
|
|
||||||
func (cs CallStack) TrimBelow(c Call) CallStack { |
|
||||||
for len(cs) > 0 && cs[0] != c { |
|
||||||
cs = cs[1:] |
|
||||||
} |
|
||||||
return cs |
|
||||||
} |
|
||||||
|
|
||||||
// TrimAbove returns a slice of the CallStack with all entries above c
|
|
||||||
// removed.
|
|
||||||
func (cs CallStack) TrimAbove(c Call) CallStack { |
|
||||||
for len(cs) > 0 && cs[len(cs)-1] != c { |
|
||||||
cs = cs[:len(cs)-1] |
|
||||||
} |
|
||||||
return cs |
|
||||||
} |
|
||||||
|
|
||||||
// pkgIndex returns the index that results in file[index:] being the path of
|
|
||||||
// file relative to the compile time GOPATH, and file[:index] being the
|
|
||||||
// $GOPATH/src/ portion of file. funcName must be the name of a function in
|
|
||||||
// file as returned by runtime.Func.Name.
|
|
||||||
func pkgIndex(file, funcName string) int { |
|
||||||
// As of Go 1.6.2 there is no direct way to know the compile time GOPATH
|
|
||||||
// at runtime, but we can infer the number of path segments in the GOPATH.
|
|
||||||
// We note that runtime.Func.Name() returns the function name qualified by
|
|
||||||
// the import path, which does not include the GOPATH. Thus we can trim
|
|
||||||
// segments from the beginning of the file path until the number of path
|
|
||||||
// separators remaining is one more than the number of path separators in
|
|
||||||
// the function name. For example, given:
|
|
||||||
//
|
|
||||||
// GOPATH /home/user
|
|
||||||
// file /home/user/src/pkg/sub/file.go
|
|
||||||
// fn.Name() pkg/sub.Type.Method
|
|
||||||
//
|
|
||||||
// We want to produce:
|
|
||||||
//
|
|
||||||
// file[:idx] == /home/user/src/
|
|
||||||
// file[idx:] == pkg/sub/file.go
|
|
||||||
//
|
|
||||||
// From this we can easily see that fn.Name() has one less path separator
|
|
||||||
// than our desired result for file[idx:]. We count separators from the
|
|
||||||
// end of the file path until it finds two more than in the function name
|
|
||||||
// and then move one character forward to preserve the initial path
|
|
||||||
// segment without a leading separator.
|
|
||||||
const sep = "/" |
|
||||||
i := len(file) |
|
||||||
for n := strings.Count(funcName, sep) + 2; n > 0; n-- { |
|
||||||
i = strings.LastIndex(file[:i], sep) |
|
||||||
if i == -1 { |
|
||||||
i = -len(sep) |
|
||||||
break |
|
||||||
} |
|
||||||
} |
|
||||||
// get back to 0 or trim the leading separator
|
|
||||||
return i + len(sep) |
|
||||||
} |
|
||||||
|
|
||||||
var runtimePath string |
|
||||||
|
|
||||||
func init() { |
|
||||||
var pcs [3]uintptr |
|
||||||
runtime.Callers(0, pcs[:]) |
|
||||||
frames := runtime.CallersFrames(pcs[:]) |
|
||||||
frame, _ := frames.Next() |
|
||||||
file := frame.File |
|
||||||
|
|
||||||
idx := pkgIndex(frame.File, frame.Function) |
|
||||||
|
|
||||||
runtimePath = file[:idx] |
|
||||||
if runtime.GOOS == "windows" { |
|
||||||
runtimePath = strings.ToLower(runtimePath) |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
func inGoroot(c Call) bool { |
|
||||||
file := c.frame.File |
|
||||||
if len(file) == 0 || file[0] == '?' { |
|
||||||
return true |
|
||||||
} |
|
||||||
if runtime.GOOS == "windows" { |
|
||||||
file = strings.ToLower(file) |
|
||||||
} |
|
||||||
return strings.HasPrefix(file, runtimePath) || strings.HasSuffix(file, "/_testmain.go") |
|
||||||
} |
|
||||||
|
|
||||||
// TrimRuntime returns a slice of the CallStack with the topmost entries from
|
|
||||||
// the go runtime removed. It considers any calls originating from unknown
|
|
||||||
// files, files under GOROOT, or _testmain.go as part of the runtime.
|
|
||||||
func (cs CallStack) TrimRuntime() CallStack { |
|
||||||
for len(cs) > 0 && inGoroot(cs[len(cs)-1]) { |
|
||||||
cs = cs[:len(cs)-1] |
|
||||||
} |
|
||||||
return cs |
|
||||||
} |
|
||||||
Loading…
Reference in new issue