hmy: Gas Price Oracle improvements (#4436)

This change makes the gas price oracle options available as options in
the configuration file / command line. In addition, the gas price
oracle's suggestion mechanism has been modified to return the default
gas price when block utilization is low. In other words, the oracle can
be configured to return the 60th percentile gas price from the last 5
blocks with 3 transactions each, or return the default gas price if
those 5 blocks were utilized less than 50% of their capacity

Fixes #4357 and supersedes #4362
pull/4439/head
Max 2 years ago committed by GitHub
parent 81c2fb06c4
commit c4427231a6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 27
      cmd/harmony/config_migrations.go
  2. 12
      cmd/harmony/default.go
  3. 74
      cmd/harmony/flags.go
  4. 60
      cmd/harmony/flags_test.go
  5. 3
      cmd/harmony/main.go
  6. 125
      hmy/gasprice.go
  7. 8
      hmy/hmy.go
  8. 23
      internal/cli/flag.go
  9. 15
      internal/cli/parse.go
  10. 20
      internal/configs/harmony/harmony.go

@ -378,6 +378,33 @@ func init() {
return confTree
}
migrations["2.5.14"] = func(confTree *toml.Tree) *toml.Tree {
if confTree.Get("GPO.Blocks") == nil {
confTree.Set("GPO.Blocks", defaultConfig.GPO.Blocks)
}
if confTree.Get("GPO.Transactions") == nil {
confTree.Set("GPO.Transactions", defaultConfig.GPO.Transactions)
}
if confTree.Get("GPO.Percentile") == nil {
confTree.Set("GPO.Percentile", defaultConfig.GPO.Percentile)
}
if confTree.Get("GPO.DefaultPrice") == nil {
confTree.Set("GPO.DefaultPrice", defaultConfig.GPO.DefaultPrice)
}
if confTree.Get("GPO.MaxPrice") == nil {
confTree.Set("GPO.MaxPrice", defaultConfig.GPO.MaxPrice)
}
if confTree.Get("GPO.LowUsageThreshold") == nil {
confTree.Set("GPO.LowUsageThreshold", defaultConfig.GPO.LowUsageThreshold)
}
if confTree.Get("GPO.BlockGasLimit") == nil {
confTree.Set("GPO.BlockGasLimit", defaultConfig.GPO.BlockGasLimit)
}
// upgrade minor version because of `GPO` section introduction
confTree.Set("Version", "2.6.0")
return confTree
}
// check that the latest version here is the same as in default.go
largestKey := getNextVersion(migrations)
if largestKey != tomlConfigVersion {

@ -2,11 +2,12 @@ package main
import (
"github.com/harmony-one/harmony/core"
"github.com/harmony-one/harmony/hmy"
harmonyconfig "github.com/harmony-one/harmony/internal/configs/harmony"
nodeconfig "github.com/harmony-one/harmony/internal/configs/node"
)
const tomlConfigVersion = "2.5.14"
const tomlConfigVersion = "2.6.0"
const (
defNetworkType = nodeconfig.Mainnet
@ -120,6 +121,15 @@ var defaultConfig = harmonyconfig.HarmonyConfig{
CacheTime: 10,
CacheSize: 512,
},
GPO: harmonyconfig.GasPriceOracleConfig{
Blocks: hmy.DefaultGPOConfig.Blocks,
Transactions: hmy.DefaultGPOConfig.Transactions,
Percentile: hmy.DefaultGPOConfig.Percentile,
DefaultPrice: hmy.DefaultGPOConfig.DefaultPrice,
MaxPrice: hmy.DefaultGPOConfig.MaxPrice,
LowUsageThreshold: hmy.DefaultGPOConfig.LowUsageThreshold,
BlockGasLimit: hmy.DefaultGPOConfig.BlockGasLimit,
},
}
var defaultSysConfig = harmonyconfig.SysConfig{

@ -249,6 +249,16 @@ var (
cacheSizeFlag,
}
gpoFlags = []cli.Flag{
gpoBlocksFlag,
gpoTransactionsFlag,
gpoPercentileFlag,
gpoDefaultPriceFlag,
gpoMaxPriceFlag,
gpoLowUsageThresholdFlag,
gpoBlockGasLimitFlag,
}
metricsFlags = []cli.Flag{
metricsETHFlag,
metricsExpensiveETHFlag,
@ -364,6 +374,7 @@ func getRootFlags() []cli.Flag {
flags = append(flags, prometheusFlags...)
flags = append(flags, syncFlags...)
flags = append(flags, shardDataFlags...)
flags = append(flags, gpoFlags...)
flags = append(flags, metricsFlags...)
return flags
@ -1932,6 +1943,45 @@ var (
}
)
// gas price oracle flags
var (
gpoBlocksFlag = cli.IntFlag{
Name: "gpo.blocks",
Usage: "Number of recent blocks to check for gas prices",
DefValue: defaultConfig.GPO.Blocks,
}
gpoTransactionsFlag = cli.IntFlag{
Name: "gpo.transactions",
Usage: "Number of transactions to sample in a block",
DefValue: defaultConfig.GPO.Transactions,
}
gpoPercentileFlag = cli.IntFlag{
Name: "gpo.percentile",
Usage: "Suggested gas price is the given percentile of a set of recent transaction gas prices",
DefValue: defaultConfig.GPO.Percentile,
}
gpoDefaultPriceFlag = cli.Int64Flag{
Name: "gpo.defaultprice",
Usage: "The gas price to suggest before data is available, and the price to suggest when block utilization is low",
DefValue: defaultConfig.GPO.DefaultPrice,
}
gpoMaxPriceFlag = cli.Int64Flag{
Name: "gpo.maxprice",
Usage: "Maximum gasprice to be recommended by gpo",
DefValue: defaultConfig.GPO.MaxPrice,
}
gpoLowUsageThresholdFlag = cli.IntFlag{
Name: "gpo.low-usage-threshold",
Usage: "The block usage threshold below which the default gas price is suggested (0 to disable)",
DefValue: defaultConfig.GPO.LowUsageThreshold,
}
gpoBlockGasLimitFlag = cli.IntFlag{
Name: "gpo.block-gas-limit",
Usage: "The gas limit, per block. If set to 0, it is pulled from the block header",
DefValue: defaultConfig.GPO.BlockGasLimit,
}
)
// metrics flags required for the go-eth library
// https://github.com/ethereum/go-ethereum/blob/master/metrics/metrics.go#L35-L55
var (
@ -1965,3 +2015,27 @@ func applyShardDataFlags(cmd *cobra.Command, cfg *harmonyconfig.HarmonyConfig) {
cfg.ShardData.CacheSize = cli.GetIntFlagValue(cmd, cacheSizeFlag)
}
}
func applyGPOFlags(cmd *cobra.Command, cfg *harmonyconfig.HarmonyConfig) {
if cli.IsFlagChanged(cmd, gpoBlocksFlag) {
cfg.GPO.Blocks = cli.GetIntFlagValue(cmd, gpoBlocksFlag)
}
if cli.IsFlagChanged(cmd, gpoTransactionsFlag) {
cfg.GPO.Transactions = cli.GetIntFlagValue(cmd, gpoTransactionsFlag)
}
if cli.IsFlagChanged(cmd, gpoPercentileFlag) {
cfg.GPO.Percentile = cli.GetIntFlagValue(cmd, gpoPercentileFlag)
}
if cli.IsFlagChanged(cmd, gpoDefaultPriceFlag) {
cfg.GPO.DefaultPrice = cli.GetInt64FlagValue(cmd, gpoDefaultPriceFlag)
}
if cli.IsFlagChanged(cmd, gpoMaxPriceFlag) {
cfg.GPO.MaxPrice = cli.GetInt64FlagValue(cmd, gpoMaxPriceFlag)
}
if cli.IsFlagChanged(cmd, gpoLowUsageThresholdFlag) {
cfg.GPO.LowUsageThreshold = cli.GetIntFlagValue(cmd, gpoLowUsageThresholdFlag)
}
if cli.IsFlagChanged(cmd, gpoBlockGasLimitFlag) {
cfg.GPO.BlockGasLimit = cli.GetIntFlagValue(cmd, gpoBlockGasLimitFlag)
}
}

@ -7,6 +7,7 @@ import (
"testing"
"time"
"github.com/harmony-one/harmony/common/denominations"
harmonyconfig "github.com/harmony-one/harmony/internal/configs/harmony"
"github.com/spf13/cobra"
@ -172,6 +173,15 @@ func TestHarmonyFlags(t *testing.T) {
CacheTime: 10,
CacheSize: 512,
},
GPO: harmonyconfig.GasPriceOracleConfig{
Blocks: defaultConfig.GPO.Blocks,
Transactions: defaultConfig.GPO.Transactions,
Percentile: defaultConfig.GPO.Percentile,
DefaultPrice: defaultConfig.GPO.DefaultPrice,
MaxPrice: defaultConfig.GPO.MaxPrice,
LowUsageThreshold: defaultConfig.GPO.LowUsageThreshold,
BlockGasLimit: defaultConfig.GPO.BlockGasLimit,
},
},
},
}
@ -1350,6 +1360,56 @@ func TestSysFlags(t *testing.T) {
}
}
func TestGPOFlags(t *testing.T) {
tests := []struct {
args []string
expConfig harmonyconfig.GasPriceOracleConfig
expErr error
}{
{
args: []string{},
expConfig: harmonyconfig.GasPriceOracleConfig{
Blocks: defaultConfig.GPO.Blocks,
Transactions: defaultConfig.GPO.Transactions,
Percentile: defaultConfig.GPO.Percentile,
DefaultPrice: defaultConfig.GPO.DefaultPrice,
MaxPrice: defaultConfig.GPO.MaxPrice,
LowUsageThreshold: defaultConfig.GPO.LowUsageThreshold,
BlockGasLimit: defaultConfig.GPO.BlockGasLimit,
},
},
{
args: []string{"--gpo.blocks", "5", "--gpo.transactions", "1", "--gpo.percentile", "2", "--gpo.defaultprice", "101000000000", "--gpo.maxprice", "400000000000", "--gpo.low-usage-threshold", "60", "--gpo.block-gas-limit", "10000000"},
expConfig: harmonyconfig.GasPriceOracleConfig{
Blocks: 5,
Transactions: 1,
Percentile: 2,
DefaultPrice: 101 * denominations.Nano,
MaxPrice: 400 * denominations.Nano,
LowUsageThreshold: 60,
BlockGasLimit: 10_000_000,
},
},
}
for i, test := range tests {
ts := newFlagTestSuite(t, gpoFlags, applyGPOFlags)
hc, err := ts.run(test.args)
if assErr := assertError(err, test.expErr); assErr != nil {
t.Fatalf("Test %v: %v", i, assErr)
}
if err != nil || test.expErr != nil {
continue
}
if !reflect.DeepEqual(hc.GPO, test.expConfig) {
t.Errorf("Test %v:\n\t%+v\n\t%+v", i, hc.GPO, test.expConfig)
}
ts.tearDown()
}
}
func TestDevnetFlags(t *testing.T) {
tests := []struct {
args []string

@ -249,6 +249,7 @@ func applyRootFlags(cmd *cobra.Command, config *harmonyconfig.HarmonyConfig) {
applyPrometheusFlags(cmd, config)
applySyncFlags(cmd, config)
applyShardDataFlags(cmd, config)
applyGPOFlags(cmd, config)
}
func setupNodeLog(config harmonyconfig.HarmonyConfig) {
@ -262,7 +263,7 @@ func setupNodeLog(config harmonyconfig.HarmonyConfig) {
utils.SetLogContext(ip, strconv.Itoa(port))
}
if config.Log.Console != true {
if !config.Log.Console {
utils.AddLogFile(logPath, config.Log.RotateSize, config.Log.RotateCount, config.Log.RotateMaxAge)
}
}

@ -18,13 +18,14 @@ package hmy
import (
"context"
"fmt"
"math/big"
"sort"
"sync"
"github.com/harmony-one/harmony/block"
"github.com/harmony-one/harmony/common/denominations"
"github.com/harmony-one/harmony/eth/rpc"
"github.com/harmony-one/harmony/internal/configs/harmony"
"github.com/harmony-one/harmony/internal/utils"
"github.com/ethereum/go-ethereum/common"
@ -32,17 +33,6 @@ import (
"github.com/harmony-one/harmony/core/types"
)
const sampleNumber = 3 // Number of transactions sampled in a block
var DefaultMaxPrice = big.NewInt(1e12) // 1000 gwei is the max suggested limit
type GasPriceConfig struct {
Blocks int
Percentile int
Default *big.Int `toml:",omitempty"`
MaxPrice *big.Int `toml:",omitempty"`
}
// OracleBackend includes all necessary background APIs for oracle.
type OracleBackend interface {
HeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*block.Header, error)
@ -50,8 +40,7 @@ type OracleBackend interface {
ChainConfig() *params.ChainConfig
}
// Oracle recommends gas prices based on the content of recent
// blocks. Suitable for both light and full clients.
// Oracle recommends gas prices based on the content of recent blocks.
type Oracle struct {
backend *Harmony
lastHead common.Hash
@ -60,38 +49,74 @@ type Oracle struct {
cacheLock sync.RWMutex
fetchLock sync.Mutex
checkBlocks int
percentile int
checkBlocks int
percentile int
checkTxs int
lowUsageThreshold float64
blockGasLimit int
defaultPrice *big.Int
}
var DefaultGPOConfig = harmony.GasPriceOracleConfig{
Blocks: 20,
Transactions: 3,
Percentile: 60,
DefaultPrice: 100 * denominations.Nano, // 100 gwei
MaxPrice: 1000 * denominations.Nano, // 1000 gwei
LowUsageThreshold: 50,
BlockGasLimit: 0, // TODO should we set default to 30M?
}
// NewOracle returns a new gasprice oracle which can recommend suitable
// gasprice for newly created transaction.
func NewOracle(backend *Harmony, params GasPriceConfig) *Oracle {
func NewOracle(backend *Harmony, params *harmony.GasPriceOracleConfig) *Oracle {
blocks := params.Blocks
if blocks < 1 {
blocks = 1
utils.Logger().Warn().Msg(fmt.Sprint("Sanitizing invalid gasprice oracle sample blocks", "provided", params.Blocks, "updated", blocks))
blocks = DefaultGPOConfig.Blocks
utils.Logger().Warn().
Int("provided", params.Blocks).
Int("updated", blocks).
Msg("Sanitizing invalid gasprice oracle sample blocks")
}
percent := params.Percentile
if percent < 0 {
percent = 0
utils.Logger().Warn().Msg(fmt.Sprint("Sanitizing invalid gasprice oracle sample percentile", "provided", params.Percentile, "updated", percent))
txs := params.Transactions
if txs < 1 {
txs = DefaultGPOConfig.Transactions
utils.Logger().Warn().
Int("provided", params.Transactions).
Int("updated", txs).
Msg("Sanitizing invalid gasprice oracle sample transactions")
}
if percent > 100 {
percent = 100
utils.Logger().Warn().Msg(fmt.Sprint("Sanitizing invalid gasprice oracle sample percentile", "provided", params.Percentile, "updated", percent))
percentile := params.Percentile
if percentile < 0 || percentile > 100 {
percentile = DefaultGPOConfig.Percentile
utils.Logger().Warn().
Int("provided", params.Percentile).
Int("updated", percentile).
Msg("Sanitizing invalid gasprice oracle percentile")
}
maxPrice := params.MaxPrice
if maxPrice == nil || maxPrice.Int64() <= 0 {
maxPrice = DefaultMaxPrice
utils.Logger().Warn().Msg(fmt.Sprint("Sanitizing invalid gasprice oracle price cap", "provided", params.MaxPrice, "updated", maxPrice))
// no sanity check done, simply convert it
defaultPrice := big.NewInt(params.DefaultPrice)
maxPrice := big.NewInt(params.MaxPrice)
lowUsageThreshold := float64(params.LowUsageThreshold) / 100.0
if lowUsageThreshold < 0 || lowUsageThreshold > 1 {
lowUsageThreshold = float64(DefaultGPOConfig.LowUsageThreshold) / 100.0
utils.Logger().Warn().
Float64("provided", float64(params.LowUsageThreshold)/100.0).
Float64("updated", lowUsageThreshold).
Msg("Sanitizing invalid gasprice oracle lowUsageThreshold")
}
blockGasLimit := params.BlockGasLimit
return &Oracle{
backend: backend,
lastPrice: params.Default,
maxPrice: maxPrice,
checkBlocks: blocks,
percentile: percent,
backend: backend,
lastPrice: defaultPrice,
maxPrice: maxPrice,
checkBlocks: blocks,
percentile: percentile,
checkTxs: txs,
lowUsageThreshold: lowUsageThreshold,
blockGasLimit: blockGasLimit,
// do not reference lastPrice
defaultPrice: new(big.Int).Set(defaultPrice),
}
}
@ -124,9 +149,10 @@ func (gpo *Oracle) SuggestPrice(ctx context.Context) (*big.Int, error) {
result = make(chan getBlockPricesResult, gpo.checkBlocks)
quit = make(chan struct{})
txPrices []*big.Int
usageSum float64
)
for sent < gpo.checkBlocks && number > 0 {
go gpo.getBlockPrices(ctx, types.MakeSigner(gpo.backend.ChainConfig(), big.NewInt(int64(number))), number, sampleNumber, result, quit)
go gpo.getBlockPrices(ctx, types.MakeSigner(gpo.backend.ChainConfig(), big.NewInt(int64(number))), number, gpo.checkTxs, result, quit)
sent++
exp++
number--
@ -149,18 +175,26 @@ func (gpo *Oracle) SuggestPrice(ctx context.Context) (*big.Int, error) {
// meaningful returned, try to query more blocks. But the maximum
// is 2*checkBlocks.
if len(res.prices) == 1 && len(txPrices)+1+exp < gpo.checkBlocks*2 && number > 0 {
go gpo.getBlockPrices(ctx, types.MakeSigner(gpo.backend.ChainConfig(), big.NewInt(int64(number))), number, sampleNumber, result, quit)
go gpo.getBlockPrices(ctx, types.MakeSigner(gpo.backend.ChainConfig(), big.NewInt(int64(number))), number, gpo.checkTxs, result, quit)
sent++
exp++
number--
}
txPrices = append(txPrices, res.prices...)
usageSum += res.usage
}
price := lastPrice
if len(txPrices) > 0 {
sort.Sort(bigIntArray(txPrices))
price = txPrices[(len(txPrices)-1)*gpo.percentile/100]
}
// `sent` is the number of queries that are sent, while `exp` and `number` count down at query resolved, and sent respectively
// each query is per block, therefore `sent` is the number of blocks for which the usage was (successfully) determined
// approximation that only holds when the gas limits, of all blocks that are sampled, are equal
usage := usageSum / float64(sent)
if usage < gpo.lowUsageThreshold {
price = new(big.Int).Set(gpo.defaultPrice)
}
if price.Cmp(gpo.maxPrice) > 0 {
price = new(big.Int).Set(gpo.maxPrice)
}
@ -173,6 +207,7 @@ func (gpo *Oracle) SuggestPrice(ctx context.Context) (*big.Int, error) {
type getBlockPricesResult struct {
prices []*big.Int
usage float64
err error
}
@ -192,7 +227,9 @@ func (gpo *Oracle) getBlockPrices(ctx context.Context, signer types.Signer, bloc
block, err := gpo.backend.BlockByNumber(ctx, rpc.BlockNumber(blockNum))
if block == nil {
select {
case result <- getBlockPricesResult{nil, err}:
// when no block is found, `err` is returned which short-circuits the oracle entirely
// therefore the `usage` becomes irrelevant
case result <- getBlockPricesResult{nil, 2.0, err}:
case <-quit:
}
return
@ -212,8 +249,18 @@ func (gpo *Oracle) getBlockPrices(ctx context.Context, signer types.Signer, bloc
}
}
}
// HACK
var gasLimit float64
if gpo.blockGasLimit == 0 {
gasLimit = float64(block.GasLimit())
} else {
gasLimit = float64(gpo.blockGasLimit)
}
// if `gasLimit` is 0, no crash. +Inf is returned and percentile is applied
// this usage includes any transactions from the miner, which are excluded by the `prices` slice
usage := float64(block.GasUsed()) / gasLimit
select {
case result <- getBlockPricesResult{prices, nil}:
case result <- getBlockPricesResult{prices, usage, nil}:
case <-quit:
}
}

@ -157,12 +157,8 @@ func New(
}
// Setup gas price oracle
gpoParams := GasPriceConfig{
Blocks: 20, // take all eligible txs past 20 blocks and sort them
Percentile: 80, // get the 80th percentile when sorted in an ascending manner
Default: big.NewInt(100e9), // minimum of 100 gwei
}
gpo := NewOracle(backend, gpoParams)
config := nodeAPI.GetConfig().HarmonyConfig.GPO
gpo := NewOracle(backend, &config)
backend.gpo = gpo
return backend

@ -3,7 +3,9 @@
package cli
import "github.com/spf13/pflag"
import (
"github.com/spf13/pflag"
)
// Flag is the interface for cli flags.
// To get the value after cli parsing, use fs.GetString(flag.Name)
@ -62,6 +64,23 @@ func (f IntFlag) RegisterTo(fs *pflag.FlagSet) error {
return markHiddenOrDeprecated(fs, f.Name, f.Deprecated, f.Hidden)
}
// Int64Flag is the flag with int64 value, used for gwei configurations
type Int64Flag struct {
Name string
Shorthand string
Usage string
Deprecated string
Hidden bool
DefValue int64
}
// RegisterTo register the int flag to FlagSet
func (f Int64Flag) RegisterTo(fs *pflag.FlagSet) error {
fs.Int64P(f.Name, f.Shorthand, f.DefValue, f.Usage)
return markHiddenOrDeprecated(fs, f.Name, f.Deprecated, f.Hidden)
}
// StringSliceFlag is the flag with string slice value
type StringSliceFlag struct {
Name string
@ -122,6 +141,8 @@ func getFlagName(flag Flag) string {
return f.Name
case IntSliceFlag:
return f.Name
case Int64Flag:
return f.Name
}
return ""
}

@ -70,6 +70,12 @@ func GetIntFlagValue(cmd *cobra.Command, flag IntFlag) int {
return getIntFlagValue(cmd.Flags(), flag)
}
// GetInt64FlagValue get the int value for the given Int64Flag from the local flags of the
// cobra command.
func GetInt64FlagValue(cmd *cobra.Command, flag Int64Flag) int64 {
return getInt64FlagValue(cmd.Flags(), flag)
}
// GetIntPersistentFlagValue get the int value for the given IntFlag from the persistent
// flags of the cobra command.
func GetIntPersistentFlagValue(cmd *cobra.Command, flag IntFlag) int {
@ -85,6 +91,15 @@ func getIntFlagValue(fs *pflag.FlagSet, flag IntFlag) int {
return val
}
func getInt64FlagValue(fs *pflag.FlagSet, flag Int64Flag) int64 {
val, err := fs.GetInt64(flag.Name)
if err != nil {
handleParseError(err)
return 0
}
return val
}
// GetStringSliceFlagValue get the string slice value for the given StringSliceFlag from
// the local flags of the cobra command.
func GetStringSliceFlagValue(cmd *cobra.Command, flag StringSliceFlag) []string {

@ -35,6 +35,7 @@ type HarmonyConfig struct {
TiKV *TiKVConfig `toml:",omitempty"`
DNSSync DnsSync
ShardData ShardDataConfig
GPO GasPriceOracleConfig
}
func (hc HarmonyConfig) ToRPCServerConfig() nodeconfig.RPCServerConfig {
@ -157,6 +158,25 @@ type ShardDataConfig struct {
CacheSize int
}
type GasPriceOracleConfig struct {
// the number of blocks to sample
Blocks int
// the number of transactions to sample, per block
Transactions int
// the percentile to pick from there
Percentile int
// the default gas price, if the above data is not available
DefaultPrice int64
// the maximum suggested gas price
MaxPrice int64
// when block usage (gas) for last `Blocks` blocks is below `LowUsageThreshold`,
// we return the Default price
LowUsageThreshold int
// hack: our block header reports an 80m gas limit, but it is actually 30M.
// if set to non-zero, this is applied UNCHECKED
BlockGasLimit int
}
type ConsensusConfig struct {
MinPeers int
AggregateSig bool

Loading…
Cancel
Save