From ea96987816ecf26c4334b97ce3a65a724bfbee27 Mon Sep 17 00:00:00 2001 From: Gheis <36589218+GheisMohammadi@users.noreply.github.com> Date: Wed, 25 May 2022 07:06:47 +0800 Subject: [PATCH] Feature/rpc filter (#4173) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix unexpected chain id (https://github.com/harmony-one/harmony/issues/4129) * update epoch numbers for ChainId fix * updated ChainID Epoch for test net * align comment with other comments * fix the build using goimports * added flags to enable/disable rpc for staking and eth * add feature to be able to feed custom configuration for each single node in testnet * add method filter * add rpc method filters file and one new flag for legacy APIs * fixed tests and improved method filter and added more tests * fix format and removed extra comments * fix method_filter comments * remove extra debug print Co-authored-by: “GheisMohammadi” <“Gheis.Mohammadi@gmail.com”> --- cmd/harmony/config_migrations.go | 16 ++ cmd/harmony/config_migrations_test.go | 16 ++ cmd/harmony/default.go | 10 +- cmd/harmony/flags.go | 44 ++++++ cmd/harmony/flags_test.go | 102 +++++++++++-- cmd/harmony/main.go | 4 + eth/rpc/client.go | 2 +- eth/rpc/client_test.go | 2 +- eth/rpc/endpoints.go | 12 +- eth/rpc/method_filter.go | 175 ++++++++++++++++++++++ eth/rpc/method_filter_test.go | 205 ++++++++++++++++++++++++++ eth/rpc/server.go | 6 +- eth/rpc/server_test.go | 2 +- eth/rpc/service.go | 10 +- eth/rpc/subscription_test.go | 4 +- eth/rpc/testservice_test.go | 4 +- internal/configs/harmony/harmony.go | 10 +- internal/configs/node/config.go | 6 + node/api.go | 2 +- rosetta/infra/harmony-mainnet.conf | 4 + rosetta/infra/harmony-pstn.conf | 4 + rpc/rpc.go | 85 +++++++---- test/deploy.sh | 8 +- 23 files changed, 663 insertions(+), 70 deletions(-) create mode 100644 eth/rpc/method_filter.go create mode 100644 eth/rpc/method_filter_test.go diff --git a/cmd/harmony/config_migrations.go b/cmd/harmony/config_migrations.go index 815373705..df8b50893 100644 --- a/cmd/harmony/config_migrations.go +++ b/cmd/harmony/config_migrations.go @@ -92,6 +92,22 @@ func init() { confTree.Set("HTTP.RosettaPort", defaultConfig.HTTP.RosettaPort) } + if confTree.Get("RPCOpt.EthRPCsEnabled") == nil { + confTree.Set("RPCOpt.EthRPCsEnabled", defaultConfig.RPCOpt.EthRPCsEnabled) + } + + if confTree.Get("RPCOpt.StakingRPCsEnabled") == nil { + confTree.Set("RPCOpt.StakingRPCsEnabled", defaultConfig.RPCOpt.StakingRPCsEnabled) + } + + if confTree.Get("RPCOpt.LegacyRPCsEnabled") == nil { + confTree.Set("RPCOpt.LegacyRPCsEnabled", defaultConfig.RPCOpt.LegacyRPCsEnabled) + } + + if confTree.Get("RPCOpt.RpcFilterFile") == nil { + confTree.Set("RPCOpt.RpcFilterFile", defaultConfig.RPCOpt.RpcFilterFile) + } + if confTree.Get("RPCOpt.RateLimterEnabled") == nil { confTree.Set("RPCOpt.RateLimterEnabled", defaultConfig.RPCOpt.RateLimterEnabled) } diff --git a/cmd/harmony/config_migrations_test.go b/cmd/harmony/config_migrations_test.go index 573e41429..6f7047c7d 100644 --- a/cmd/harmony/config_migrations_test.go +++ b/cmd/harmony/config_migrations_test.go @@ -64,6 +64,10 @@ Version = "1.0.2" [RPCOpt] DebugEnabled = false + EthRPCsEnabled = true + StakingRPCsEnabled = true + LegacyRPCsEnabled = true + RpcFilterFile = "" [TxPool] BlacklistFile = "./.hmy/blacklist.txt" @@ -129,6 +133,10 @@ Version = "1.0.3" [RPCOpt] DebugEnabled = false + EthRPCsEnabled = true + StakingRPCsEnabled = true + LegacyRPCsEnabled = true + RpcFilterFile = "" [TxPool] BlacklistFile = "./.hmy/blacklist.txt" @@ -194,6 +202,10 @@ Version = "1.0.4" [RPCOpt] DebugEnabled = false + EthRPCsEnabled = true + StakingRPCsEnabled = true + LegacyRPCsEnabled = true + RpcFilterFile = "" [Sync] Concurrency = 6 @@ -271,6 +283,10 @@ Version = "1.0.4" [RPCOpt] DebugEnabled = false + EthRPCsEnabled = true + StakingRPCsEnabled = true + LegacyRPCsEnabled = true + RpcFilterFile = "" [Sync] Concurrency = 6 diff --git a/cmd/harmony/default.go b/cmd/harmony/default.go index 7b8a838d7..0440789e4 100644 --- a/cmd/harmony/default.go +++ b/cmd/harmony/default.go @@ -47,9 +47,13 @@ var defaultConfig = harmonyconfig.HarmonyConfig{ AuthPort: nodeconfig.DefaultAuthWSPort, }, RPCOpt: harmonyconfig.RpcOptConfig{ - DebugEnabled: false, - RateLimterEnabled: true, - RequestsPerSecond: nodeconfig.DefaultRPCRateLimit, + DebugEnabled: false, + EthRPCsEnabled: true, + StakingRPCsEnabled: true, + LegacyRPCsEnabled: true, + RpcFilterFile: "", + RateLimterEnabled: true, + RequestsPerSecond: nodeconfig.DefaultRPCRateLimit, }, BLSKeys: harmonyconfig.BlsConfig{ KeyDir: "./.hmy/blskeys", diff --git a/cmd/harmony/flags.go b/cmd/harmony/flags.go index 0151c1533..9ec6386d1 100644 --- a/cmd/harmony/flags.go +++ b/cmd/harmony/flags.go @@ -82,6 +82,10 @@ var ( rpcOptFlags = []cli.Flag{ rpcDebugEnabledFlag, + rpcEthRPCsEnabledFlag, + rpcStakingRPCsEnabledFlag, + rpcLegacyRPCsEnabledFlag, + rpcFilterFileFlag, rpcRateLimiterEnabledFlag, rpcRateLimitFlag, } @@ -718,6 +722,34 @@ var ( Hidden: true, } + rpcEthRPCsEnabledFlag = cli.BoolFlag{ + Name: "rpc.eth", + Usage: "enable eth apis", + DefValue: defaultConfig.RPCOpt.EthRPCsEnabled, + Hidden: true, + } + + rpcStakingRPCsEnabledFlag = cli.BoolFlag{ + Name: "rpc.staking", + Usage: "enable staking apis", + DefValue: defaultConfig.RPCOpt.StakingRPCsEnabled, + Hidden: true, + } + + rpcLegacyRPCsEnabledFlag = cli.BoolFlag{ + Name: "rpc.legacy", + Usage: "enable legacy apis", + DefValue: defaultConfig.RPCOpt.LegacyRPCsEnabled, + Hidden: true, + } + + rpcFilterFileFlag = cli.StringFlag{ + Name: "rpc.filterspath", + Usage: "toml file path for method exposure filters", + DefValue: defaultConfig.RPCOpt.RpcFilterFile, + Hidden: true, + } + rpcRateLimiterEnabledFlag = cli.BoolFlag{ Name: "rpc.ratelimiter", Usage: "enable rate limiter for RPCs", @@ -735,6 +767,18 @@ func applyRPCOptFlags(cmd *cobra.Command, config *harmonyconfig.HarmonyConfig) { if cli.IsFlagChanged(cmd, rpcDebugEnabledFlag) { config.RPCOpt.DebugEnabled = cli.GetBoolFlagValue(cmd, rpcDebugEnabledFlag) } + if cli.IsFlagChanged(cmd, rpcEthRPCsEnabledFlag) { + config.RPCOpt.EthRPCsEnabled = cli.GetBoolFlagValue(cmd, rpcEthRPCsEnabledFlag) + } + if cli.IsFlagChanged(cmd, rpcStakingRPCsEnabledFlag) { + config.RPCOpt.StakingRPCsEnabled = cli.GetBoolFlagValue(cmd, rpcStakingRPCsEnabledFlag) + } + if cli.IsFlagChanged(cmd, rpcLegacyRPCsEnabledFlag) { + config.RPCOpt.LegacyRPCsEnabled = cli.GetBoolFlagValue(cmd, rpcLegacyRPCsEnabledFlag) + } + if cli.IsFlagChanged(cmd, rpcFilterFileFlag) { + config.RPCOpt.RpcFilterFile = cli.GetStringFlagValue(cmd, rpcFilterFileFlag) + } if cli.IsFlagChanged(cmd, rpcRateLimiterEnabledFlag) { config.RPCOpt.RateLimterEnabled = cli.GetBoolFlagValue(cmd, rpcRateLimiterEnabledFlag) } diff --git a/cmd/harmony/flags_test.go b/cmd/harmony/flags_test.go index 2f755cee5..0b0f9efea 100644 --- a/cmd/harmony/flags_test.go +++ b/cmd/harmony/flags_test.go @@ -74,9 +74,13 @@ func TestHarmonyFlags(t *testing.T) { RosettaPort: 9700, }, RPCOpt: harmonyconfig.RpcOptConfig{ - DebugEnabled: false, - RateLimterEnabled: true, - RequestsPerSecond: 1000, + DebugEnabled: false, + EthRPCsEnabled: true, + StakingRPCsEnabled: true, + LegacyRPCsEnabled: true, + RpcFilterFile: "", + RateLimterEnabled: true, + RequestsPerSecond: 1000, }, WS: harmonyconfig.WsConfig{ Enabled: true, @@ -620,36 +624,104 @@ func TestRPCOptFlags(t *testing.T) { { args: []string{"--rpc.debug"}, expConfig: harmonyconfig.RpcOptConfig{ - DebugEnabled: true, - RateLimterEnabled: true, - RequestsPerSecond: 1000, + DebugEnabled: true, + EthRPCsEnabled: true, + StakingRPCsEnabled: true, + LegacyRPCsEnabled: true, + RpcFilterFile: "", + RateLimterEnabled: true, + RequestsPerSecond: 1000, + }, + }, + + { + args: []string{"--rpc.eth=false"}, + expConfig: harmonyconfig.RpcOptConfig{ + DebugEnabled: false, + EthRPCsEnabled: false, + StakingRPCsEnabled: true, + LegacyRPCsEnabled: true, + RpcFilterFile: "", + RateLimterEnabled: true, + RequestsPerSecond: 1000, + }, + }, + + { + args: []string{"--rpc.staking=false"}, + expConfig: harmonyconfig.RpcOptConfig{ + DebugEnabled: false, + EthRPCsEnabled: true, + StakingRPCsEnabled: false, + LegacyRPCsEnabled: true, + RpcFilterFile: "", + RateLimterEnabled: true, + RequestsPerSecond: 1000, + }, + }, + + { + args: []string{"--rpc.legacy=false"}, + expConfig: harmonyconfig.RpcOptConfig{ + DebugEnabled: false, + EthRPCsEnabled: true, + StakingRPCsEnabled: true, + LegacyRPCsEnabled: false, + RpcFilterFile: "", + RateLimterEnabled: true, + RequestsPerSecond: 1000, + }, + }, + + { + args: []string{"--rpc.filterspath=./rmf.toml"}, + expConfig: harmonyconfig.RpcOptConfig{ + DebugEnabled: false, + EthRPCsEnabled: true, + StakingRPCsEnabled: true, + LegacyRPCsEnabled: true, + RpcFilterFile: "./rmf.toml", + RateLimterEnabled: true, + RequestsPerSecond: 1000, }, }, { args: []string{}, expConfig: harmonyconfig.RpcOptConfig{ - DebugEnabled: false, - RateLimterEnabled: true, - RequestsPerSecond: 1000, + DebugEnabled: false, + EthRPCsEnabled: true, + StakingRPCsEnabled: true, + LegacyRPCsEnabled: true, + RpcFilterFile: "", + RateLimterEnabled: true, + RequestsPerSecond: 1000, }, }, { args: []string{"--rpc.ratelimiter", "--rpc.ratelimit", "2000"}, expConfig: harmonyconfig.RpcOptConfig{ - DebugEnabled: false, - RateLimterEnabled: true, - RequestsPerSecond: 2000, + DebugEnabled: false, + EthRPCsEnabled: true, + StakingRPCsEnabled: true, + LegacyRPCsEnabled: true, + RpcFilterFile: "", + RateLimterEnabled: true, + RequestsPerSecond: 2000, }, }, { args: []string{"--rpc.ratelimiter=false", "--rpc.ratelimit", "2000"}, expConfig: harmonyconfig.RpcOptConfig{ - DebugEnabled: false, - RateLimterEnabled: false, - RequestsPerSecond: 2000, + DebugEnabled: false, + EthRPCsEnabled: true, + StakingRPCsEnabled: true, + LegacyRPCsEnabled: true, + RpcFilterFile: "", + RateLimterEnabled: false, + RequestsPerSecond: 2000, }, }, } diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index 3b034e7f6..455db3277 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -329,6 +329,10 @@ func setupNodeAndRun(hc harmonyconfig.HarmonyConfig) { WSPort: hc.WS.Port, WSAuthPort: hc.WS.AuthPort, DebugEnabled: hc.RPCOpt.DebugEnabled, + EthRPCsEnabled: hc.RPCOpt.EthRPCsEnabled, + StakingRPCsEnabled: hc.RPCOpt.StakingRPCsEnabled, + LegacyRPCsEnabled: hc.RPCOpt.LegacyRPCsEnabled, + RpcFilterFile: hc.RPCOpt.RpcFilterFile, RateLimiterEnabled: hc.RPCOpt.RateLimterEnabled, RequestsPerSecond: hc.RPCOpt.RequestsPerSecond, } diff --git a/eth/rpc/client.go b/eth/rpc/client.go index 3fc80463a..bbf741a3d 100644 --- a/eth/rpc/client.go +++ b/eth/rpc/client.go @@ -230,7 +230,7 @@ func initClient(conn ServerCodec, idgen func() ID, services *serviceRegistry) *C // subscription an error is returned. Otherwise a new service is created and added to the // service collection this client provides to the server. func (c *Client) RegisterName(name string, receiver interface{}) error { - return c.services.registerName(name, receiver) + return c.services.registerName(name, receiver, nil) } func (c *Client) nextID() json.RawMessage { diff --git a/eth/rpc/client_test.go b/eth/rpc/client_test.go index acf6d1e20..7b6c01ae4 100644 --- a/eth/rpc/client_test.go +++ b/eth/rpc/client_test.go @@ -270,7 +270,7 @@ func TestClientSubscribeClose(t *testing.T) { gotHangSubscriptionReq: make(chan struct{}), unblockHangSubscription: make(chan struct{}), } - if err := server.RegisterName("nftest2", service); err != nil { + if err := server.RegisterName("nftest2", service, nil); err != nil { t.Fatal(err) } diff --git a/eth/rpc/endpoints.go b/eth/rpc/endpoints.go index 8ca6d4eb0..57de9fc49 100644 --- a/eth/rpc/endpoints.go +++ b/eth/rpc/endpoints.go @@ -23,7 +23,7 @@ import ( ) // StartHTTPEndpoint starts the HTTP RPC endpoint, configured with cors/vhosts/modules -func StartHTTPEndpoint(endpoint string, apis []API, modules []string, cors []string, vhosts []string, timeouts HTTPTimeouts) (net.Listener, *Server, error) { +func StartHTTPEndpoint(endpoint string, apis []API, modules []string, rmf *RpcMethodFilter, cors []string, vhosts []string, timeouts HTTPTimeouts) (net.Listener, *Server, error) { // Generate the whitelist based on the allowed modules whitelist := make(map[string]bool) for _, module := range modules { @@ -33,7 +33,7 @@ func StartHTTPEndpoint(endpoint string, apis []API, modules []string, cors []str handler := NewServer() for _, api := range apis { if whitelist[api.Namespace] || (len(whitelist) == 0 && api.Public) { - if err := handler.RegisterName(api.Namespace, api.Service); err != nil { + if err := handler.RegisterName(api.Namespace, api.Service, rmf); err != nil { return nil, nil, err } log.Debug("HTTP registered", "namespace", api.Namespace) @@ -52,7 +52,7 @@ func StartHTTPEndpoint(endpoint string, apis []API, modules []string, cors []str } // StartWSEndpoint starts a websocket endpoint -func StartWSEndpoint(endpoint string, apis []API, modules []string, wsOrigins []string, exposeAll bool) (net.Listener, *Server, error) { +func StartWSEndpoint(endpoint string, apis []API, modules []string, rmf *RpcMethodFilter, wsOrigins []string, exposeAll bool) (net.Listener, *Server, error) { // Generate the whitelist based on the allowed modules whitelist := make(map[string]bool) @@ -63,7 +63,7 @@ func StartWSEndpoint(endpoint string, apis []API, modules []string, wsOrigins [] handler := NewServer() for _, api := range apis { if exposeAll || whitelist[api.Namespace] || (len(whitelist) == 0 && api.Public) { - if err := handler.RegisterName(api.Namespace, api.Service); err != nil { + if err := handler.RegisterName(api.Namespace, api.Service, rmf); err != nil { return nil, nil, err } log.Debug("WebSocket registered", "service", api.Service, "namespace", api.Namespace) @@ -83,11 +83,11 @@ func StartWSEndpoint(endpoint string, apis []API, modules []string, wsOrigins [] } // StartIPCEndpoint starts an IPC endpoint. -func StartIPCEndpoint(ipcEndpoint string, apis []API) (net.Listener, *Server, error) { +func StartIPCEndpoint(ipcEndpoint string, apis []API, rmf *RpcMethodFilter) (net.Listener, *Server, error) { // Register all the APIs exposed by the services. handler := NewServer() for _, api := range apis { - if err := handler.RegisterName(api.Namespace, api.Service); err != nil { + if err := handler.RegisterName(api.Namespace, api.Service, rmf); err != nil { return nil, nil, err } log.Debug("IPC registered", "namespace", api.Namespace) diff --git a/eth/rpc/method_filter.go b/eth/rpc/method_filter.go new file mode 100644 index 000000000..f47c39785 --- /dev/null +++ b/eth/rpc/method_filter.go @@ -0,0 +1,175 @@ +package rpc + +import ( + "errors" + "fmt" + "io/ioutil" + "os" + "regexp" + "strings" + + "github.com/pelletier/go-toml" +) + +type RpcMethodFilter struct { + Allow []string + Deny []string +} + +// ExposeAll - init Allow and Deny array in a way to expose all APIs +func (rmf *RpcMethodFilter) ExposeAll() error { + rmf.Allow = rmf.Allow[:0] + rmf.Allow = rmf.Deny[:0] + rmf.Allow = append(rmf.Allow, "*") + return nil +} + +// LoadRpcMethodFilters - load RPC method filters from toml file +/* ex: filters.toml +Allow = [ ... ] +Deny = [ ... ] +*/ +func (rmf *RpcMethodFilter) LoadRpcMethodFiltersFromFile(file string) error { + // check if file exist + if _, err := os.Stat(file); err == nil { + b, errRead := ioutil.ReadFile(file) + if errRead != nil { + return fmt.Errorf("rpc filter file read error - %s", errRead.Error()) + } + return rmf.LoadRpcMethodFilters(b) + } else if errors.Is(err, os.ErrNotExist) { + // file path does not exist + return fmt.Errorf("rpc filter file doesn't exist") + } else { + // some other errors happened + return fmt.Errorf("rpc filter file stat error - %s", err.Error()) + } +} + +// LoadRpcMethodFilters - load RPC method filters from binary array (given from toml file) +func (rmf *RpcMethodFilter) LoadRpcMethodFilters(b []byte) error { + fTree, err := toml.LoadBytes(b) + if err != nil { + return fmt.Errorf("rpc filter file parse error - %s", err.Error()) + } + if err := fTree.Unmarshal(rmf); err != nil { + return fmt.Errorf("rpc filter parse error - %s", err.Error()) + } + if len(rmf.Allow) == 0 { + rmf.Allow = append(rmf.Allow, "*") + } + + return nil +} + +// Expose - checks whether specific method have to expose or not +func (rmf *RpcMethodFilter) Expose(name string) bool { + allow := checkFilters(rmf.Allow, name) + deny := checkFilters(rmf.Deny, name) + return allow && !deny +} + +// checkFilters - checks whether any of filters match with value +func checkFilters(filters []string, value string) bool { + if len(filters) == 0 { + return false + } + for _, filter := range filters { + if Match(filter, value) { + return true + } + } + return false +} + +// Match - finds whether the text matches/satisfies the pattern string. +// pattern can include match type (ex: regex:^[a-z]bc ) +func Match(pattern string, value string) bool { + parts := strings.SplitN(pattern, ":", 2) + + // check if pattern defines match type + if len(parts) > 1 { + matchType := strings.Trim(strings.ToLower(parts[0]), " ") + matchPattern := strings.Trim(parts[1], " ") + switch matchType { + case "exact": + return matchPattern == value + case "simple": + return MatchSimple(matchPattern, value) + case "wildcard": + return MatchWildCard(matchPattern, value) + case "regex": + isAllowed, _ := regexp.MatchString(matchPattern, value) + return isAllowed + default: + isAllowed, _ := regexp.MatchString(matchPattern, value) + return isAllowed + } + } + // auto detect simple checking or wildcard + if regexp.MustCompile(`^[a-zA-Z0-9_]+$`).MatchString(pattern) { + return strings.EqualFold(pattern, value) + } else if regexp.MustCompile(`^[a-zA-Z0-9_*?]+$`).MatchString(pattern) { + return MatchWildCard(pattern, value) + } + // by default we use regex matching + allowed, _ := regexp.MatchString(pattern, value) + return allowed +} + +// MatchSimple - finds whether the text matches/satisfies the pattern string. +// supports only '*' wildcard in the pattern. +func MatchSimple(pattern, name string) bool { + if pattern == "" { + return name == pattern + } + + if pattern == "*" { + return true + } + // Does only wildcard '*' match. + return deepMatchRune([]rune(name), []rune(pattern), true) +} + +// MatchWildCard - finds whether the text matches/satisfies the wildcard pattern string. +// supports '*' and '?' wildcards in the pattern string. +func MatchWildCard(pattern, name string) (matched bool) { + if pattern == "" { + return name == pattern + } + + if pattern == "*" { + return true + } + // Does extended wildcard '*' and '?' match. + return deepMatchRune([]rune(name), []rune(pattern), false) +} + +func deepMatchRune(str, pattern []rune, simple bool) bool { + for len(pattern) > 0 { + switch pattern[0] { + default: + if len(str) == 0 || str[0] != pattern[0] { + return false + } + case '?': + if len(str) == 0 && !simple { + return false + } + case '*': + return deepMatchRune(str, pattern[1:], simple) || + (len(str) > 0 && deepMatchRune(str[1:], pattern, simple)) + } + + str = str[1:] + pattern = pattern[1:] + } + + return len(str) == 0 && len(pattern) == 0 +} + +// MatchRegex - finds whether the text matches/satisfies the regex pattern string. +func MatchRegex(pattern string, value string) bool { + result, _ := regexp.MatchString(pattern, value) + return result +} diff --git a/eth/rpc/method_filter_test.go b/eth/rpc/method_filter_test.go new file mode 100644 index 000000000..dd0991303 --- /dev/null +++ b/eth/rpc/method_filter_test.go @@ -0,0 +1,205 @@ +package rpc + +import ( + "testing" +) + +func TestRpcMethodFilter(t *testing.T) { + method_filters_toml := ` + Allow = [ + "hmy_method1", + "wildcard:hmyv2_method?", + "eth*", + "hmy_getNetworkInfo", + "regex:^hmy_send[a-zA-Z]+" + ] + + Deny = [ + "*staking*", + "eth_get*", + "hmy_getNetworkInfo", + "exact:hmy_sendTx" + ] + ` + b := []byte(method_filters_toml) + + var rmf RpcMethodFilter + rmf.LoadRpcMethodFilters(b) + + tests := []struct { + name string + exposure bool + }{ + 0: {"hmy_method1", true}, // auto detected exact match which exists (case-insensitive) + 1: {"hmy_MeThoD1", true}, // check case-insensitive + 2: {"hmy_method2", false}, // not exist in allows + 3: {"hmyv2_method5", true}, // wildcard + 4: {"hmyv2_method", false}, // false case for wild card + 5: {"eth_chainID", true}, // auto detected wild card in allow filters + 6: {"eth_getValidator", false}, // auto detected wild card in deny filters + 7: {"hmy_getStakingInfo", false}, // deny wild card + 8: {"abc", false}, // not exist pattern + 9: {"hmy_getNetworkInfo", false}, // case-insensitive normal word match + 10: {"hmy_sendTx", false}, // exact match (case-sensitive) + 11: {"hmy_sendtx", true}, // exact match (case-sensitive) + } + + for i, test := range tests { + mustExpose := rmf.Expose(test.name) + + if mustExpose != test.exposure { + t.Errorf("Test %d got unexpected value, want %t, got %t", i, test.exposure, mustExpose) + } + } +} + +func TestRpcMethodAllowAllFilter(t *testing.T) { + method_filters_toml := ` + Allow = [ + "*" + ] + + Deny = [ + "mtd1", + "*staking*", + "eth_get*", + "^hmy_[a-z]+" + ] + ` + b := []byte(method_filters_toml) + + var rmf RpcMethodFilter + rmf.LoadRpcMethodFilters(b) + + tests := []struct { + name string + exposure bool + }{ + 0: {"mtd1", false}, + 1: {"hmy_method1", false}, + 2: {"hmyv2_method5", true}, + 3: {"hmyv2_method", true}, + 4: {"eth_chainID", true}, + 5: {"eth_getValidator", false}, + 6: {"hmy_getStakingInfo", false}, + 7: {"abc", true}, + 8: {"hmy_getStakingNetworkInfo", false}, + } + + for i, test := range tests { + mustExpose := rmf.Expose(test.name) + + if mustExpose != test.exposure { + t.Errorf("Test %d got unexpected value, want %t, got %t", i, test.exposure, mustExpose) + } + } +} + +func TestRpcMethodDenyAllFilter(t *testing.T) { + method_filters_toml := ` + Allow = [ + "mtd1", + "*staking*", + "eth_get*", + "regex:^hmy_[a-z]+" + ] + + Deny = [ + "*" + ] + ` + b := []byte(method_filters_toml) + + var rmf RpcMethodFilter + rmf.LoadRpcMethodFilters(b) + + tests := []struct { + name string + exposure bool + }{ + 0: {"mtd1", false}, + 1: {"hmy_method1", false}, + 2: {"hmyv2_method5", false}, + 3: {"hmyv2_method", false}, + 4: {"eth_chainID", false}, + 5: {"eth_getValidator", false}, + 6: {"hmy_getStakingInfo", false}, + 7: {"abc", false}, + 8: {"hmy_getStakingNetworkInfo", false}, + } + + for i, test := range tests { + mustExpose := rmf.Expose(test.name) + + if mustExpose != test.exposure { + t.Errorf("Test %d got unexpected value, want %t, got %t", i, test.exposure, mustExpose) + } + } +} + +func TestEmptyRpcMethodFilter(t *testing.T) { + + b := []byte("") + var rmf RpcMethodFilter + rmf.LoadRpcMethodFilters(b) + + tests := []struct { + name string + exposure bool + }{ + 0: {"hmy_method1", true}, + 1: {"hmy_method2", true}, + 2: {"hmyv2_method5", true}, + 3: {"hmyv2_method", true}, + 4: {"eth_chainID", true}, + 5: {"eth_getValidator", true}, + 6: {"hmy_getStakingInfo", true}, + 7: {"abc", true}, + 8: {"hmy_getNetworkInfo", true}, + } + + for i, test := range tests { + mustExpose := rmf.Expose(test.name) + + if mustExpose != test.exposure { + t.Errorf("Test %d got unexpected value, want %t, got %t", i, test.exposure, mustExpose) + } + } +} + +func TestFilter(t *testing.T) { + tests := []struct { + input string + pattern string + expectedAllowance bool + }{ + 0: {"abc", "abc", true}, + 1: {"AbC", "abc", true}, // case-insensitive + 2: {"AbC", "exact:AbC", true}, // case-insensitive + 3: {"AbC", "exact:abc", false}, // case-insensitive + 4: {"abcd", "*", true}, // check * to pass everything + 5: {"abc", "simple:abc", true}, // check simple matching + 6: {"abcd", "simple:abc", false}, + 7: {"abcd", "regex:^a([a-z]+)d$", true}, // check regex + 8: {"abcde", "regex:^a([a-z]+)d$", false}, + 9: {"abcd", "^a([a-z]+)d$", true}, // auto detected regex + 10: {"abc", "wildcard:abc*", true}, // check wild card + 11: {"abc", "abc*", true}, // auto detected wild card + 12: {"abcdef", "abc*", true}, + 13: {"dabcd", "?abc*", true}, // check * and ? for wild card + 14: {"abc", "*abc?", false}, // ? can't be empty + 15: {"abcdef", "*a?c*", true}, + 16: {"defabc", "*ab?*", true}, + 17: {"defabcghi", "*abc*", true}, + 18: {"ab", "*abc*", false}, + 19: {"defghabc", "*abc*", true}, + } + + for i, test := range tests { + isAllowed := Match(test.pattern, test.input) + + if isAllowed != test.expectedAllowance { + t.Errorf("Test %d got unexpected value, want %t, got %t", i, test.expectedAllowance, isAllowed) + } + } +} diff --git a/eth/rpc/server.go b/eth/rpc/server.go index ddd7c744c..d4fc1775a 100644 --- a/eth/rpc/server.go +++ b/eth/rpc/server.go @@ -54,7 +54,7 @@ func NewServer() *Server { // Register the default service providing meta information about the RPC service such // as the services and methods it offers. rpcService := &RPCService{server} - server.RegisterName(MetadataApi, rpcService) + server.RegisterName(MetadataApi, rpcService, nil) return server } @@ -62,8 +62,8 @@ func NewServer() *Server { // methods on the given receiver match the criteria to be either a RPC method or a // subscription an error is returned. Otherwise a new service is created and added to the // service collection this server provides to clients. -func (s *Server) RegisterName(name string, receiver interface{}) error { - return s.services.registerName(name, receiver) +func (s *Server) RegisterName(name string, receiver interface{}, rmf *RpcMethodFilter) error { + return s.services.registerName(name, receiver, rmf) } // ServeCodec reads incoming requests from codec, calls the appropriate callback and writes diff --git a/eth/rpc/server_test.go b/eth/rpc/server_test.go index a4ca1fde4..9002f6dc2 100644 --- a/eth/rpc/server_test.go +++ b/eth/rpc/server_test.go @@ -32,7 +32,7 @@ func TestServerRegisterName(t *testing.T) { server := NewServer() service := new(testService) - if err := server.RegisterName("test", service); err != nil { + if err := server.RegisterName("test", service, nil); err != nil { t.Fatalf("%v", err) } diff --git a/eth/rpc/service.go b/eth/rpc/service.go index bef891ea1..9470a3a32 100644 --- a/eth/rpc/service.go +++ b/eth/rpc/service.go @@ -58,7 +58,7 @@ type callback struct { isSubscribe bool // true if this is a subscription callback } -func (r *serviceRegistry) registerName(name string, rcvr interface{}) error { +func (r *serviceRegistry) registerName(name string, rcvr interface{}, rmf *RpcMethodFilter) error { rcvrVal := reflect.ValueOf(rcvr) if name == "" { return fmt.Errorf("no service name for type %s", rcvrVal.Type().String()) @@ -82,7 +82,15 @@ func (r *serviceRegistry) registerName(name string, rcvr interface{}) error { } r.services[name] = svc } + for name, cb := range callbacks { + //check if name is not blocked by method filters + if rmf != nil { + mustExpose := rmf.Expose(svc.name + "_" + name) + if !mustExpose { + continue + } + } if cb.isSubscribe { svc.subscriptions[name] = cb } else { diff --git a/eth/rpc/subscription_test.go b/eth/rpc/subscription_test.go index c3a918a83..2999a9ad0 100644 --- a/eth/rpc/subscription_test.go +++ b/eth/rpc/subscription_test.go @@ -64,7 +64,7 @@ func TestSubscriptions(t *testing.T) { // setup and start server for _, namespace := range namespaces { - if err := server.RegisterName(namespace, service); err != nil { + if err := server.RegisterName(namespace, service, nil); err != nil { t.Fatalf("unable to register test service %v", err) } } @@ -128,7 +128,7 @@ func TestServerUnsubscribe(t *testing.T) { // Start the server. server := newTestServer() service := ¬ificationTestService{unsubscribed: make(chan string)} - server.RegisterName("nftest2", service) + server.RegisterName("nftest2", service, nil) p1, p2 := net.Pipe() go server.ServeCodec(NewCodec(p1), 0) diff --git a/eth/rpc/testservice_test.go b/eth/rpc/testservice_test.go index 010fc4f0b..d1f002a63 100644 --- a/eth/rpc/testservice_test.go +++ b/eth/rpc/testservice_test.go @@ -27,10 +27,10 @@ import ( func newTestServer() *Server { server := NewServer() server.idgen = sequentialIDGenerator() - if err := server.RegisterName("test", new(testService)); err != nil { + if err := server.RegisterName("test", new(testService), nil); err != nil { panic(err) } - if err := server.RegisterName("nftest", new(notificationTestService)); err != nil { + if err := server.RegisterName("nftest", new(notificationTestService), nil); err != nil { panic(err) } return server diff --git a/internal/configs/harmony/harmony.go b/internal/configs/harmony/harmony.go index 55b70e787..75daef9a8 100644 --- a/internal/configs/harmony/harmony.go +++ b/internal/configs/harmony/harmony.go @@ -166,9 +166,13 @@ type WsConfig struct { } type RpcOptConfig struct { - DebugEnabled bool // Enables PrivateDebugService APIs, including the EVM tracer - RateLimterEnabled bool // Enable Rate limiter for RPC - RequestsPerSecond int // for RPC rate limiter + DebugEnabled bool // Enables PrivateDebugService APIs, including the EVM tracer + EthRPCsEnabled bool // Expose Eth RPCs + StakingRPCsEnabled bool // Expose Staking RPCs + LegacyRPCsEnabled bool // Expose Legacy RPCs + RpcFilterFile string // Define filters to enable/disable RPC exposure + RateLimterEnabled bool // Enable Rate limiter for RPC + RequestsPerSecond int // for RPC rate limiter } type DevnetConfig struct { diff --git a/internal/configs/node/config.go b/internal/configs/node/config.go index d8c9a2bbd..19798bef8 100644 --- a/internal/configs/node/config.go +++ b/internal/configs/node/config.go @@ -111,6 +111,12 @@ type RPCServerConfig struct { DebugEnabled bool + EthRPCsEnabled bool + StakingRPCsEnabled bool + LegacyRPCsEnabled bool + + RpcFilterFile string + RateLimiterEnabled bool RequestsPerSecond int } diff --git a/node/api.go b/node/api.go index 903858693..0388a3500 100644 --- a/node/api.go +++ b/node/api.go @@ -70,7 +70,7 @@ func (node *Node) StartRPC() error { // Gather all the possible APIs to surface apis := node.APIs(harmony) - return hmy_rpc.StartServers(harmony, apis, node.NodeConfig.RPCServer) + return hmy_rpc.StartServers(harmony, apis, node.NodeConfig.RPCServer, node.HarmonyConfig.RPCOpt) } // StopRPC stop RPC service diff --git a/rosetta/infra/harmony-mainnet.conf b/rosetta/infra/harmony-mainnet.conf index 535e3c65f..e1b96226e 100644 --- a/rosetta/infra/harmony-mainnet.conf +++ b/rosetta/infra/harmony-mainnet.conf @@ -74,6 +74,10 @@ Version = "2.5.2" [RPCOpt] DebugEnabled = false + EthRPCsEnabled = true + StakingRPCsEnabled = true + LegacyRPCsEnabled = true + RpcFilterFile = "" RateLimterEnabled = true RequestsPerSecond = 1000 diff --git a/rosetta/infra/harmony-pstn.conf b/rosetta/infra/harmony-pstn.conf index 8adf2d071..fc6c12c94 100644 --- a/rosetta/infra/harmony-pstn.conf +++ b/rosetta/infra/harmony-pstn.conf @@ -74,6 +74,10 @@ Version = "2.5.2" [RPCOpt] DebugEnabled = false + EthRPCsEnabled = true + StakingRPCsEnabled = true + LegacyRPCsEnabled = true + RpcFilterFile = "" RateLimterEnabled = true RequestsPerSecond = 1000 diff --git a/rpc/rpc.go b/rpc/rpc.go index 313ccd052..7c1267a75 100644 --- a/rpc/rpc.go +++ b/rpc/rpc.go @@ -8,6 +8,7 @@ import ( "github.com/harmony-one/harmony/eth/rpc" "github.com/harmony-one/harmony/hmy" + "github.com/harmony-one/harmony/internal/configs/harmony" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" "github.com/harmony-one/harmony/internal/utils" eth "github.com/harmony-one/harmony/rpc/eth" @@ -71,30 +72,39 @@ func (n Version) Namespace() string { } // StartServers starts the http & ws servers -func StartServers(hmy *hmy.Harmony, apis []rpc.API, config nodeconfig.RPCServerConfig) error { - apis = append(apis, getAPIs(hmy, config.DebugEnabled, config.RateLimiterEnabled, config.RequestsPerSecond)...) +func StartServers(hmy *hmy.Harmony, apis []rpc.API, config nodeconfig.RPCServerConfig, rpcOpt harmony.RpcOptConfig) error { + apis = append(apis, getAPIs(hmy, config)...) authApis := append(apis, getAuthAPIs(hmy, config.DebugEnabled, config.RateLimiterEnabled, config.RequestsPerSecond)...) - + // load method filter from file (if exist) + var rmf rpc.RpcMethodFilter + rpcFilterFilePath := strings.Trim(rpcOpt.RpcFilterFile, " ") + if len(rpcFilterFilePath) > 0 { + if err := rmf.LoadRpcMethodFiltersFromFile(rpcFilterFilePath); err != nil { + return err + } + } else { + rmf.ExposeAll() + } if config.HTTPEnabled { httpEndpoint = fmt.Sprintf("%v:%v", config.HTTPIp, config.HTTPPort) - if err := startHTTP(apis); err != nil { + if err := startHTTP(apis, &rmf); err != nil { return err } httpAuthEndpoint = fmt.Sprintf("%v:%v", config.HTTPIp, config.HTTPAuthPort) - if err := startAuthHTTP(authApis); err != nil { + if err := startAuthHTTP(authApis, &rmf); err != nil { return err } } if config.WSEnabled { wsEndpoint = fmt.Sprintf("%v:%v", config.WSIp, config.WSPort) - if err := startWS(apis); err != nil { + if err := startWS(apis, &rmf); err != nil { return err } wsAuthEndpoint = fmt.Sprintf("%v:%v", config.WSIp, config.WSAuthPort) - if err := startAuthWS(authApis); err != nil { + if err := startAuthWS(authApis, &rmf); err != nil { return err } } @@ -141,30 +151,45 @@ func getAuthAPIs(hmy *hmy.Harmony, debugEnable bool, rateLimiterEnable bool, rat } // getAPIs returns all the API methods for the RPC interface -func getAPIs(hmy *hmy.Harmony, debugEnable bool, rateLimiterEnable bool, ratelimit int) []rpc.API { +func getAPIs(hmy *hmy.Harmony, config nodeconfig.RPCServerConfig) []rpc.API { publicAPIs := []rpc.API{ // Public methods NewPublicHarmonyAPI(hmy, V1), NewPublicHarmonyAPI(hmy, V2), - NewPublicHarmonyAPI(hmy, Eth), - NewPublicBlockchainAPI(hmy, V1, rateLimiterEnable, ratelimit), - NewPublicBlockchainAPI(hmy, V2, rateLimiterEnable, ratelimit), - NewPublicBlockchainAPI(hmy, Eth, rateLimiterEnable, ratelimit), + NewPublicBlockchainAPI(hmy, V1, config.RateLimiterEnabled, config.RequestsPerSecond), + NewPublicBlockchainAPI(hmy, V2, config.RateLimiterEnabled, config.RequestsPerSecond), NewPublicContractAPI(hmy, V1), NewPublicContractAPI(hmy, V2), - NewPublicContractAPI(hmy, Eth), NewPublicTransactionAPI(hmy, V1), NewPublicTransactionAPI(hmy, V2), - NewPublicTransactionAPI(hmy, Eth), NewPublicPoolAPI(hmy, V1), NewPublicPoolAPI(hmy, V2), - NewPublicPoolAPI(hmy, Eth), - NewPublicStakingAPI(hmy, V1), - NewPublicStakingAPI(hmy, V2), - // Legacy methods (subject to removal) - v1.NewPublicLegacyAPI(hmy, "hmy"), - eth.NewPublicEthService(hmy, "eth"), - v2.NewPublicLegacyAPI(hmy, "hmyv2"), + } + + // Legacy methods (subject to removal) + if config.LegacyRPCsEnabled { + publicAPIs = append(publicAPIs, + v1.NewPublicLegacyAPI(hmy, "hmy"), + v2.NewPublicLegacyAPI(hmy, "hmyv2"), + ) + } + + if config.StakingRPCsEnabled { + publicAPIs = append(publicAPIs, + NewPublicStakingAPI(hmy, V1), + NewPublicStakingAPI(hmy, V2), + ) + } + + if config.EthRPCsEnabled { + publicAPIs = append(publicAPIs, + NewPublicHarmonyAPI(hmy, Eth), + NewPublicBlockchainAPI(hmy, Eth, config.RateLimiterEnabled, config.RequestsPerSecond), + NewPublicContractAPI(hmy, Eth), + NewPublicTransactionAPI(hmy, Eth), + NewPublicPoolAPI(hmy, Eth), + eth.NewPublicEthService(hmy, "eth"), + ) } publicDebugAPIs := []rpc.API{ @@ -178,16 +203,16 @@ func getAPIs(hmy *hmy.Harmony, debugEnable bool, rateLimiterEnable bool, ratelim NewPrivateDebugAPI(hmy, V2), } - if debugEnable { + if config.DebugEnabled { apis := append(publicAPIs, publicDebugAPIs...) return append(apis, privateAPIs...) } return publicAPIs } -func startHTTP(apis []rpc.API) (err error) { +func startHTTP(apis []rpc.API, rmf *rpc.RpcMethodFilter) (err error) { httpListener, httpHandler, err = rpc.StartHTTPEndpoint( - httpEndpoint, apis, HTTPModules, httpOrigins, httpVirtualHosts, httpTimeouts, + httpEndpoint, apis, HTTPModules, rmf, httpOrigins, httpVirtualHosts, httpTimeouts, ) if err != nil { return err @@ -202,9 +227,9 @@ func startHTTP(apis []rpc.API) (err error) { return nil } -func startAuthHTTP(apis []rpc.API) (err error) { +func startAuthHTTP(apis []rpc.API, rmf *rpc.RpcMethodFilter) (err error) { httpListener, httpHandler, err = rpc.StartHTTPEndpoint( - httpAuthEndpoint, apis, HTTPModules, httpOrigins, httpVirtualHosts, httpTimeouts, + httpAuthEndpoint, apis, HTTPModules, rmf, httpOrigins, httpVirtualHosts, httpTimeouts, ) if err != nil { return err @@ -219,8 +244,8 @@ func startAuthHTTP(apis []rpc.API) (err error) { return nil } -func startWS(apis []rpc.API) (err error) { - wsListener, wsHandler, err = rpc.StartWSEndpoint(wsEndpoint, apis, WSModules, wsOrigins, true) +func startWS(apis []rpc.API, rmf *rpc.RpcMethodFilter) (err error) { + wsListener, wsHandler, err = rpc.StartWSEndpoint(wsEndpoint, apis, WSModules, rmf, wsOrigins, true) if err != nil { return err } @@ -232,8 +257,8 @@ func startWS(apis []rpc.API) (err error) { return nil } -func startAuthWS(apis []rpc.API) (err error) { - wsListener, wsHandler, err = rpc.StartWSEndpoint(wsAuthEndpoint, apis, WSModules, wsOrigins, true) +func startAuthWS(apis []rpc.API, rmf *rpc.RpcMethodFilter) (err error) { + wsListener, wsHandler, err = rpc.StartWSEndpoint(wsAuthEndpoint, apis, WSModules, rmf, wsOrigins, true) if err != nil { return err } diff --git a/test/deploy.sh b/test/deploy.sh index e403b92f6..272675baa 100755 --- a/test/deploy.sh +++ b/test/deploy.sh @@ -81,7 +81,7 @@ function launch_localnet() { i=$((i + 1)) # Read config for i-th node form config file - IFS=' ' read -r ip port mode bls_key shard <<<"${line}" + IFS=' ' read -r ip port mode bls_key shard node_config <<<"${line}" args=("${base_args[@]}" --ip "${ip}" --port "${port}" --key "/tmp/${ip}-${port}.key" --db_dir "${ROOT}/db-${ip}-${port}" "--broadcast_invalid_tx=false") if [[ -z "$ip" || -z "$port" ]]; then echo "skip empty node" @@ -103,6 +103,12 @@ function launch_localnet() { continue fi + # Setup node config for i-th localnet node + if [[ -f "$node_config" ]]; then + echo "node ${i} configuration is loaded from: ${node_config}" + args=("${args[@]}" --config "${node_config}") + fi + # Setup flags for i-th node based on config case "${mode}" in explorer)