Add prometheus support for tracing

Author: expenses

Cargo.lock

@@ -79,7 +79,7 @@ where
 			// Setup tracing.
 			if let Some(tracing_targets) = cli_args.import_params.tracing_targets.as_ref() {
 				let subscriber = sc_tracing::ProfilingSubscriber::new(
-					cli_args.import_params.tracing_receiver.into(), tracing_targets
+					cli_args.import_params.tracing_receiver.into(), tracing_targets, None
 				);
 				if let Err(e) = tracing::subscriber::set_global_default(subscriber) {
 					return Err(

bin/node/cli/src/command.rs

@@ -62,6 +62,7 @@ arg_enum! {
 	pub enum TracingReceiver {
 		Log,
 		Telemetry,
+		Prometheus
 	}
 }
 
@@ -70,6 +71,7 @@ impl Into<sc_tracing::TracingReceiver> for TracingReceiver {
 		match self {
 			TracingReceiver::Log => sc_tracing::TracingReceiver::Log,
 			TracingReceiver::Telemetry => sc_tracing::TracingReceiver::Telemetry,
+			TracingReceiver::Prometheus => sc_tracing::TracingReceiver::Prometheus,
 		}
 	}
 }

client/cli/src/arg_enums.rs

@@ -1032,24 +1032,25 @@ ServiceBuilder<
 		}
 
 		// Prometheus metrics
-		let metrics = if let Some((registry, port)) = prometheus_registry_and_port {
-			let metrics = ServiceMetrics::register(&registry)?;
+		let metrics = match prometheus_registry_and_port.clone() {
+			Some((registry, port)) => {
+				let metrics = ServiceMetrics::register(&registry)?;
 
-			metrics.node_roles.set(u64::from(config.roles.bits()));
+				metrics.node_roles.set(u64::from(config.roles.bits()));
 
-			let future = select(
-				prometheus_exporter::init_prometheus(port, registry).boxed(),
-				exit.clone()
-			).map(drop);
+				let future = select(
+					prometheus_exporter::init_prometheus(port, registry).boxed(),
+					exit.clone()
+				).map(drop);
 
-			let _ = to_spawn_tx.unbounded_send((
-				Box::pin(future),
-				From::from("prometheus-endpoint")
-			));
+				let _ = to_spawn_tx.unbounded_send((
+					Box::pin(future),
+					From::from("prometheus-endpoint")
+				));
 
-			Some(metrics)
-		} else {
-			None
+				Some(metrics)
+			}
+			None => None,
 		};
 		// Periodically notify the telemetry.
 		let transaction_pool_ = transaction_pool.clone();
@@ -1271,7 +1272,11 @@ ServiceBuilder<
 		// Instrumentation
 		if let Some(tracing_targets) = config.tracing_targets.as_ref() {
 			let subscriber = sc_tracing::ProfilingSubscriber::new(
-				config.tracing_receiver, tracing_targets
+				config.tracing_receiver, tracing_targets,
+				match prometheus_registry_and_port {
+					Some((reg, _)) => Some(sc_tracing::Metrics::register(&reg)?),
+					None => None
+				}
 			);
 			match tracing::subscriber::set_global_default(subscriber) {
 				Ok(_) => (),

client/service/src/builder.rs

@@ -17,6 +17,7 @@ slog = { version = "2.5.2", features = ["nested-values"] }
 tracing-core = "0.1.7"
 
 sc-telemetry = { version = "2.0.0-dev", path = "../telemetry" }
+prometheus-exporter = { version = "0.8.0-dev", path = "../../utils/prometheus" }
 
 [dev-dependencies]
 tracing = "0.1.10"

client/tracing/Cargo.toml

@@ -53,7 +53,8 @@ use tracing_core::{
 	subscriber::Subscriber
 };
 
-use sc_telemetry::{telemetry, SUBSTRATE_INFO};
+use sc_telemetry::{telemetry, SUBSTRATE_INFO}; 
+use prometheus_exporter::{register, U64, Registry, PrometheusError, Opts, GaugeVec};
 
 /// Used to configure how to receive the metrics
 #[derive(Debug, Clone)]
@@ -62,6 +63,8 @@ pub enum TracingReceiver {
 	Log,
 	/// Output to telemetry
 	Telemetry,
+	/// Output to prometheus
+	Prometheus
 }
 
 impl Default for TracingReceiver {
@@ -149,19 +152,21 @@ pub struct ProfilingSubscriber {
 	targets: Vec<(String, Level)>,
 	receiver: TracingReceiver,
 	span_data: Mutex<HashMap<u64, SpanDatum>>,
+	prometheus_metrics: Option<Metrics>
 }
 
 impl ProfilingSubscriber {
 	/// Takes a `Receiver` and a comma separated list of targets,
 	/// either with a level: "pallet=trace"
 	/// or without: "pallet".
-	pub fn new(receiver: TracingReceiver, targets: &str) -> Self {
+	pub fn new(receiver: TracingReceiver, targets: &str, prometheus_metrics: Option<Metrics>) -> Self {
 		let targets: Vec<_> = targets.split(',').map(|s| parse_target(s)).collect();
 		ProfilingSubscriber {
 			next_id: AtomicU64::new(1),
 			targets,
 			receiver,
 			span_data: Mutex::new(HashMap::new()),
+			prometheus_metrics,
 		}
 	}
 }
@@ -252,6 +257,9 @@ impl ProfilingSubscriber {
 		match self.receiver {
 			TracingReceiver::Log => print_log(span_datum),
 			TracingReceiver::Telemetry => send_telemetry(span_datum),
+			TracingReceiver::Prometheus => send_prometheus(
+				span_datum, self.prometheus_metrics.as_ref().expect("prometheus_metrics needs to be passed to trace to prometheus")
+			)
 		}
 	}
 }
@@ -287,3 +295,32 @@ fn send_telemetry(span_datum: SpanDatum) {
 	);
 }
 
+#[derive(Clone)]
+pub struct Metrics {
+	profiling: GaugeVec<U64>
+}
+
+impl fmt::Debug for Metrics {
+	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+		write!(f, "Metrics")
+	}
+}
+
+impl Metrics {
+	pub fn register(registry: &Registry) -> Result<Self, PrometheusError> {
+		Ok(Self {
+			profiling: register(GaugeVec::new(
+				Opts::new("profiling", "Profiling from tracing"),
+				&["name", "target", "line", "values"]
+			)?, registry)?,
+		})
+	}
+}
+
+fn send_prometheus(span_datum: SpanDatum, metrics: &Metrics) {
+	metrics.profiling
+		.with_label_values(&[
+			span_datum.name, span_datum.target, &span_datum.line.to_string(), &span_datum.values.to_string()
+		])
+		.set(span_datum.overall_time.as_nanos() as u64);
+}