mirror of https://github.com/hyperledger/besu
Expose metrics to prometheus (#506)
A service like the JSON-RPC service is opened up, only serving /metrics requests in a file format for prometheus. New CLI flags are --metrics-enabled and --metrics-listen, just like the --rpc and --ws variants of the same. --host-whitelist is respected the same as the JSON-RPC endpoint.
Danno Ferrin
6 years ago
committed by
GitHub
parent
ae2e1eb520
commit
2c5f49dfef
@ -0,0 +1,104 @@ |
||||
/* |
||||
* Copyright 2019 ConsenSys AG. |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on |
||||
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the |
||||
* specific language governing permissions and limitations under the License. |
||||
*/ |
||||
package tech.pegasys.pantheon.metrics.prometheus; |
||||
|
||||
import java.util.Collection; |
||||
import java.util.Collections; |
||||
import java.util.Objects; |
||||
|
||||
import com.google.common.collect.Lists; |
||||
|
||||
public class MetricsConfiguration { |
||||
private static final String DEFAULT_METRICS_HOST = "127.0.0.1"; |
||||
public static final int DEFAULT_METRICS_PORT = 9545; |
||||
|
||||
private boolean enabled; |
||||
private int port; |
||||
private String host; |
||||
private Collection<String> hostsWhitelist = Collections.singletonList("localhost"); |
||||
|
||||
public static MetricsConfiguration createDefault() { |
||||
final MetricsConfiguration metricsConfiguration = new MetricsConfiguration(); |
||||
metricsConfiguration.setEnabled(false); |
||||
metricsConfiguration.setPort(DEFAULT_METRICS_PORT); |
||||
metricsConfiguration.setHost(DEFAULT_METRICS_HOST); |
||||
|
||||
return metricsConfiguration; |
||||
} |
||||
|
||||
private MetricsConfiguration() {} |
||||
|
||||
public boolean isEnabled() { |
||||
return enabled; |
||||
} |
||||
|
||||
public void setEnabled(final boolean enabled) { |
||||
this.enabled = enabled; |
||||
} |
||||
|
||||
public int getPort() { |
||||
return port; |
||||
} |
||||
|
||||
public void setPort(final int port) { |
||||
this.port = port; |
||||
} |
||||
|
||||
public String getHost() { |
||||
return host; |
||||
} |
||||
|
||||
public void setHost(final String host) { |
||||
this.host = host; |
||||
} |
||||
|
||||
Collection<String> getHostsWhitelist() { |
||||
return Collections.unmodifiableCollection(this.hostsWhitelist); |
||||
} |
||||
|
||||
public void setHostsWhitelist(final Collection<String> hostsWhitelist) { |
||||
this.hostsWhitelist = hostsWhitelist; |
||||
} |
||||
|
||||
@Override |
||||
public String toString() { |
||||
return "MetricsConfiguration{" |
||||
+ "enabled=" |
||||
+ enabled |
||||
+ ", port=" |
||||
+ port |
||||
+ ", host='" |
||||
+ host |
||||
+ '\'' |
||||
+ ", hostsWhitelist=" |
||||
+ hostsWhitelist |
||||
+ '}'; |
||||
} |
||||
|
||||
@Override |
||||
public boolean equals(final Object o) { |
||||
if (this == o) return true; |
||||
if (o == null || getClass() != o.getClass()) return false; |
||||
final MetricsConfiguration that = (MetricsConfiguration) o; |
||||
return enabled == that.enabled |
||||
&& port == that.port |
||||
&& Objects.equals(host, that.host) |
||||
&& com.google.common.base.Objects.equal( |
||||
Lists.newArrayList(hostsWhitelist), Lists.newArrayList(that.hostsWhitelist)); |
||||
} |
||||
|
||||
@Override |
||||
public int hashCode() { |
||||
return Objects.hash(enabled, port, host, hostsWhitelist); |
||||
} |
||||
} |
||||
@ -0,0 +1,231 @@ |
||||
/* |
||||
* Copyright 2019 ConsenSys AG. |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on |
||||
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the |
||||
* specific language governing permissions and limitations under the License. |
||||
*/ |
||||
package tech.pegasys.pantheon.metrics.prometheus; |
||||
|
||||
import static com.google.common.base.Preconditions.checkArgument; |
||||
import static com.google.common.collect.Streams.stream; |
||||
import static tech.pegasys.pantheon.util.NetworkUtility.urlForSocketAddress; |
||||
|
||||
import tech.pegasys.pantheon.metrics.MetricsSystem; |
||||
import tech.pegasys.pantheon.util.NetworkUtility; |
||||
|
||||
import java.io.ByteArrayOutputStream; |
||||
import java.io.IOException; |
||||
import java.io.OutputStreamWriter; |
||||
import java.net.InetSocketAddress; |
||||
import java.net.SocketException; |
||||
import java.nio.charset.StandardCharsets; |
||||
import java.util.Optional; |
||||
import java.util.Set; |
||||
import java.util.TreeSet; |
||||
import java.util.concurrent.CompletableFuture; |
||||
|
||||
import com.google.common.annotations.VisibleForTesting; |
||||
import com.google.common.base.Splitter; |
||||
import com.google.common.collect.Iterables; |
||||
import io.netty.handler.codec.http.HttpResponseStatus; |
||||
import io.prometheus.client.exporter.common.TextFormat; |
||||
import io.vertx.core.Handler; |
||||
import io.vertx.core.Vertx; |
||||
import io.vertx.core.http.HttpMethod; |
||||
import io.vertx.core.http.HttpServer; |
||||
import io.vertx.core.http.HttpServerOptions; |
||||
import io.vertx.core.http.HttpServerResponse; |
||||
import io.vertx.ext.web.Router; |
||||
import io.vertx.ext.web.RoutingContext; |
||||
import org.apache.logging.log4j.LogManager; |
||||
import org.apache.logging.log4j.Logger; |
||||
|
||||
public class MetricsHttpService { |
||||
private static final Logger LOG = LogManager.getLogger(); |
||||
|
||||
private static final InetSocketAddress EMPTY_SOCKET_ADDRESS = new InetSocketAddress("0.0.0.0", 0); |
||||
|
||||
private final Vertx vertx; |
||||
private final MetricsConfiguration config; |
||||
private final MetricsSystem metricsSystem; |
||||
|
||||
private HttpServer httpServer; |
||||
|
||||
public MetricsHttpService( |
||||
final Vertx vertx, |
||||
final MetricsConfiguration configuration, |
||||
final MetricsSystem metricsSystem) { |
||||
validateConfig(configuration); |
||||
this.vertx = vertx; |
||||
this.config = configuration; |
||||
this.metricsSystem = metricsSystem; |
||||
} |
||||
|
||||
private void validateConfig(final MetricsConfiguration config) { |
||||
checkArgument( |
||||
config.getPort() == 0 || NetworkUtility.isValidPort(config.getPort()), |
||||
"Invalid port configuration."); |
||||
checkArgument(config.getHost() != null, "Required host is not configured."); |
||||
} |
||||
|
||||
public CompletableFuture<?> start() { |
||||
LOG.info("Starting JsonRPC service on {}:{}", config.getHost(), config.getPort()); |
||||
// Create the HTTP server and a router object.
|
||||
httpServer = |
||||
vertx.createHttpServer( |
||||
new HttpServerOptions().setHost(config.getHost()).setPort(config.getPort())); |
||||
|
||||
// Handle json rpc requests
|
||||
final Router router = Router.router(vertx); |
||||
|
||||
// Verify Host header to avoid rebind attack.
|
||||
router.route().handler(checkWhitelistHostHeader()); |
||||
|
||||
router.route("/").method(HttpMethod.GET).handler(this::handleEmptyRequest); |
||||
router |
||||
.route("/metrics") |
||||
.method(HttpMethod.GET) |
||||
.produces(TextFormat.CONTENT_TYPE_004) |
||||
.handler(this::metricsRequest); |
||||
final CompletableFuture<?> resultFuture = new CompletableFuture<>(); |
||||
httpServer |
||||
.requestHandler(router::accept) |
||||
.listen( |
||||
res -> { |
||||
if (!res.failed()) { |
||||
resultFuture.complete(null); |
||||
LOG.info( |
||||
"Metrics service started and listening on {}:{}", |
||||
config.getHost(), |
||||
httpServer.actualPort()); |
||||
return; |
||||
} |
||||
httpServer = null; |
||||
final Throwable cause = res.cause(); |
||||
if (cause instanceof SocketException) { |
||||
resultFuture.completeExceptionally( |
||||
new RuntimeException( |
||||
String.format( |
||||
"Failed to bind metrics listener to %s:%s: %s", |
||||
config.getHost(), config.getPort(), cause.getMessage()))); |
||||
return; |
||||
} |
||||
resultFuture.completeExceptionally(cause); |
||||
}); |
||||
|
||||
return resultFuture; |
||||
} |
||||
|
||||
private Handler<RoutingContext> checkWhitelistHostHeader() { |
||||
return event -> { |
||||
final Optional<String> hostHeader = getAndValidateHostHeader(event); |
||||
if (config.getHostsWhitelist().contains("*") |
||||
|| (hostHeader.isPresent() && hostIsInWhitelist(hostHeader.get()))) { |
||||
event.next(); |
||||
} else { |
||||
event |
||||
.response() |
||||
.setStatusCode(403) |
||||
.putHeader("Content-Type", "text/plain; charset=utf-8") |
||||
.end("Host not authorized."); |
||||
} |
||||
}; |
||||
} |
||||
|
||||
private Optional<String> getAndValidateHostHeader(final RoutingContext event) { |
||||
final Iterable<String> splitHostHeader = Splitter.on(':').split(event.request().host()); |
||||
final long hostPieces = stream(splitHostHeader).count(); |
||||
if (hostPieces > 1) { |
||||
// If the host contains a colon, verify the host is correctly formed - host [ ":" port ]
|
||||
if (hostPieces > 2 || !Iterables.get(splitHostHeader, 1).matches("\\d{1,5}+")) { |
||||
return Optional.empty(); |
||||
} |
||||
} |
||||
return Optional.ofNullable(Iterables.get(splitHostHeader, 0)); |
||||
} |
||||
|
||||
private boolean hostIsInWhitelist(final String hostHeader) { |
||||
return config |
||||
.getHostsWhitelist() |
||||
.stream() |
||||
.anyMatch(whitelistEntry -> whitelistEntry.toLowerCase().equals(hostHeader.toLowerCase())); |
||||
} |
||||
|
||||
public CompletableFuture<?> stop() { |
||||
if (httpServer == null) { |
||||
return CompletableFuture.completedFuture(null); |
||||
} |
||||
|
||||
final CompletableFuture<?> resultFuture = new CompletableFuture<>(); |
||||
httpServer.close( |
||||
res -> { |
||||
if (res.failed()) { |
||||
resultFuture.completeExceptionally(res.cause()); |
||||
} else { |
||||
httpServer = null; |
||||
resultFuture.complete(null); |
||||
} |
||||
}); |
||||
return resultFuture; |
||||
} |
||||
|
||||
private void metricsRequest(final RoutingContext routingContext) { |
||||
final Set<String> names = new TreeSet<>(routingContext.queryParam("name[]")); |
||||
final HttpServerResponse response = routingContext.response(); |
||||
vertx.<String>executeBlocking( |
||||
future -> { |
||||
try { |
||||
final ByteArrayOutputStream metrics = new ByteArrayOutputStream(16 * 1024); |
||||
final OutputStreamWriter osw = new OutputStreamWriter(metrics, StandardCharsets.UTF_8); |
||||
TextFormat.write004( |
||||
osw, |
||||
((PrometheusMetricsSystem) (metricsSystem)) |
||||
.getRegistry() |
||||
.filteredMetricFamilySamples(names)); |
||||
osw.flush(); |
||||
osw.close(); |
||||
metrics.flush(); |
||||
metrics.close(); |
||||
future.complete(metrics.toString(StandardCharsets.UTF_8.name())); |
||||
} catch (final IOException ioe) { |
||||
future.fail(ioe); |
||||
} |
||||
}, |
||||
false, |
||||
(res) -> { |
||||
if (res.failed()) { |
||||
response.setStatusCode(HttpResponseStatus.INTERNAL_SERVER_ERROR.code()).end(); |
||||
} else { |
||||
response.setStatusCode(HttpResponseStatus.OK.code()); |
||||
response.putHeader("Content-Type", TextFormat.CONTENT_TYPE_004); |
||||
response.end(res.result()); |
||||
} |
||||
}); |
||||
} |
||||
|
||||
public InetSocketAddress socketAddress() { |
||||
if (httpServer == null) { |
||||
return EMPTY_SOCKET_ADDRESS; |
||||
} |
||||
return new InetSocketAddress(config.getHost(), httpServer.actualPort()); |
||||
} |
||||
|
||||
@VisibleForTesting |
||||
public String url() { |
||||
if (httpServer == null) { |
||||
return ""; |
||||
} |
||||
return urlForSocketAddress("http", socketAddress()); |
||||
} |
||||
|
||||
// Facilitate remote health-checks in AWS, inter alia.
|
||||
private void handleEmptyRequest(final RoutingContext routingContext) { |
||||
routingContext.response().setStatusCode(201).end(); |
||||
} |
||||
} |
||||
@ -0,0 +1,175 @@ |
||||
/* |
||||
* Copyright 2018 ConsenSys AG. |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on |
||||
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the |
||||
* specific language governing permissions and limitations under the License. |
||||
*/ |
||||
package tech.pegasys.pantheon.metrics.prometheus; |
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat; |
||||
|
||||
import java.net.InetSocketAddress; |
||||
import java.util.Collections; |
||||
import java.util.Properties; |
||||
|
||||
import io.vertx.core.Vertx; |
||||
import okhttp3.OkHttpClient; |
||||
import okhttp3.Request; |
||||
import okhttp3.Response; |
||||
import org.junit.AfterClass; |
||||
import org.junit.BeforeClass; |
||||
import org.junit.ClassRule; |
||||
import org.junit.Test; |
||||
import org.junit.rules.TemporaryFolder; |
||||
|
||||
public class MetricsHttpServiceTest { |
||||
|
||||
@ClassRule public static final TemporaryFolder folder = new TemporaryFolder(); |
||||
|
||||
private static final Vertx vertx = Vertx.vertx(); |
||||
|
||||
private static MetricsHttpService service; |
||||
private static OkHttpClient client; |
||||
private static String baseUrl; |
||||
|
||||
@BeforeClass |
||||
public static void initServerAndClient() { |
||||
service = createMetricsHttpService(); |
||||
service.start().join(); |
||||
|
||||
// Build an OkHttp client.
|
||||
client = new OkHttpClient(); |
||||
baseUrl = service.url(); |
||||
} |
||||
|
||||
private static MetricsHttpService createMetricsHttpService(final MetricsConfiguration config) { |
||||
return new MetricsHttpService(vertx, config, PrometheusMetricsSystem.init()); |
||||
} |
||||
|
||||
private static MetricsHttpService createMetricsHttpService() { |
||||
return new MetricsHttpService(vertx, createMetricsConfig(), PrometheusMetricsSystem.init()); |
||||
} |
||||
|
||||
private static MetricsConfiguration createMetricsConfig() { |
||||
final MetricsConfiguration config = MetricsConfiguration.createDefault(); |
||||
config.setPort(0); |
||||
config.setHostsWhitelist(Collections.singletonList("*")); |
||||
return config; |
||||
} |
||||
|
||||
/** Tears down the HTTP server. */ |
||||
@AfterClass |
||||
public static void shutdownServer() { |
||||
service.stop().join(); |
||||
} |
||||
|
||||
@Test |
||||
public void invalidCallToStart() { |
||||
service |
||||
.start() |
||||
.whenComplete( |
||||
(unused, exception) -> assertThat(exception).isInstanceOf(IllegalStateException.class)); |
||||
} |
||||
|
||||
@Test |
||||
public void http404() throws Exception { |
||||
try (final Response resp = client.newCall(buildGetRequest("/foo")).execute()) { |
||||
assertThat(resp.code()).isEqualTo(404); |
||||
} |
||||
} |
||||
|
||||
@Test |
||||
public void handleEmptyRequest() throws Exception { |
||||
try (final Response resp = client.newCall(buildGetRequest("")).execute()) { |
||||
assertThat(resp.code()).isEqualTo(201); |
||||
} |
||||
} |
||||
|
||||
@Test |
||||
public void getSocketAddressWhenActive() { |
||||
final InetSocketAddress socketAddress = service.socketAddress(); |
||||
assertThat("127.0.0.1").isEqualTo(socketAddress.getAddress().getHostAddress()); |
||||
assertThat(socketAddress.getPort() > 0).isTrue(); |
||||
} |
||||
|
||||
@Test |
||||
public void getSocketAddressWhenStoppedIsEmpty() { |
||||
final MetricsHttpService service = createMetricsHttpService(); |
||||
|
||||
final InetSocketAddress socketAddress = service.socketAddress(); |
||||
assertThat("0.0.0.0").isEqualTo(socketAddress.getAddress().getHostAddress()); |
||||
assertThat(0).isEqualTo(socketAddress.getPort()); |
||||
assertThat("").isEqualTo(service.url()); |
||||
} |
||||
|
||||
@Test |
||||
public void getSocketAddressWhenBindingToAllInterfaces() { |
||||
final MetricsConfiguration config = createMetricsConfig(); |
||||
config.setHost("0.0.0.0"); |
||||
final MetricsHttpService service = createMetricsHttpService(config); |
||||
service.start().join(); |
||||
|
||||
try { |
||||
final InetSocketAddress socketAddress = service.socketAddress(); |
||||
assertThat("0.0.0.0").isEqualTo(socketAddress.getAddress().getHostAddress()); |
||||
assertThat(socketAddress.getPort() > 0).isTrue(); |
||||
assertThat(!service.url().contains("0.0.0.0")).isTrue(); |
||||
} finally { |
||||
service.stop().join(); |
||||
} |
||||
} |
||||
|
||||
@Test |
||||
public void metricsArePresent() throws Exception { |
||||
final Request metricsRequest = new Request.Builder().url(baseUrl + "/metrics").build(); |
||||
try (final Response resp = client.newCall(metricsRequest).execute()) { |
||||
assertThat(resp.code()).isEqualTo(200); |
||||
// Check general format of result, it maps to java.util.Properties
|
||||
final Properties props = new Properties(); |
||||
props.load(resp.body().byteStream()); |
||||
|
||||
// We should have JVM metrics already loaded, verify a simple key.
|
||||
assertThat(props).containsKey("jvm_threads_deadlocked"); |
||||
} |
||||
} |
||||
|
||||
@Test |
||||
public void metricsArePresentWhenFiltered() throws Exception { |
||||
final Request metricsRequest = |
||||
new Request.Builder().url(baseUrl + "/metrics?name[]=jvm_threads_deadlocked").build(); |
||||
try (final Response resp = client.newCall(metricsRequest).execute()) { |
||||
assertThat(resp.code()).isEqualTo(200); |
||||
// Check general format of result, it maps to java.util.Properties
|
||||
final Properties props = new Properties(); |
||||
props.load(resp.body().byteStream()); |
||||
|
||||
// We should have JVM metrics already loaded, verify a simple key.
|
||||
assertThat(props).containsKey("jvm_threads_deadlocked"); |
||||
} |
||||
} |
||||
|
||||
@Test |
||||
public void metricsAreAbsentWhenFiltered() throws Exception { |
||||
final Request metricsRequest = |
||||
new Request.Builder().url(baseUrl + "/metrics?name[]=does_not_exist").build(); |
||||
try (final Response resp = client.newCall(metricsRequest).execute()) { |
||||
assertThat(resp.code()).isEqualTo(200); |
||||
// Check general format of result, it maps to java.util.Properties
|
||||
final Properties props = new Properties(); |
||||
props.load(resp.body().byteStream()); |
||||
|
||||
// We should have JVM metrics already loaded, verify a simple key.
|
||||
assertThat(props).isEmpty(); |
||||
} |
||||
} |
||||
|
||||
private Request buildGetRequest(final String path) { |
||||
return new Request.Builder().get().url(baseUrl + path).build(); |
||||
} |
||||
} |
||||
Loading…
Reference in new issue