track the number of used db connections with an histogram

The histogram should allow us to determine more accurately how many
connections are used, as the older approach of taking a sample whenever
metrics are scraped does not work well.

Author: pietroalbini

src/app.rs

@@ -128,6 +128,9 @@ impl App {
                 instance_metrics
                     .database_time_to_obtain_connection
                     .with_label_values(&["primary"]),
+                instance_metrics
+                    .database_used_conns_histogram
+                    .with_label_values(&["primary"]),
             )
             .unwrap()
         };
@@ -155,6 +158,9 @@ impl App {
                         instance_metrics
                             .database_time_to_obtain_connection
                             .with_label_values(&["follower"]),
+                        instance_metrics
+                            .database_used_conns_histogram
+                            .with_label_values(&["follower"]),
                     )
                     .unwrap(),
                 )

src/db.rs

@@ -13,6 +13,7 @@ use crate::middleware::app::RequestApp;
 pub enum DieselPool {
     Pool {
         pool: r2d2::Pool<ConnectionManager<PgConnection>>,
+        used_conns_metric: Histogram,
         time_to_obtain_connection_metric: Histogram,
     },
     Test(Arc<ReentrantMutex<PgConnection>>),
@@ -22,6 +23,7 @@ impl DieselPool {
     pub(crate) fn new(
         url: &str,
         config: r2d2::Builder<ConnectionManager<PgConnection>>,
+        used_conns_metric: Histogram,
         time_to_obtain_connection_metric: Histogram,
     ) -> Result<DieselPool, PoolError> {
         let manager = ConnectionManager::new(connection_url(url));
@@ -39,6 +41,7 @@ impl DieselPool {
         // automatically be marked as unhealthy and the rest of the application will adapt.
         let pool = DieselPool::Pool {
             pool: config.build_unchecked(manager),
+            used_conns_metric,
             time_to_obtain_connection_metric,
         };
         match pool.wait_until_healthy(Duration::from_secs(5)) {
@@ -62,8 +65,13 @@ impl DieselPool {
         match self {
             DieselPool::Pool {
                 pool,
+                used_conns_metric,
                 time_to_obtain_connection_metric,
             } => time_to_obtain_connection_metric.observe_closure_duration(|| {
+                // Record the number of used connections before obtaining the current one.
+                let state = pool.state();
+                used_conns_metric.observe((state.connections - state.idle_connections) as f64);
+
                 if let Some(conn) = pool.try_get() {
                     Ok(DieselPooledConn::Pool(conn))
                 } else if !self.is_healthy() {

src/metrics/histogram.rs

@@ -24,6 +24,13 @@ impl HistogramBuckets for TimingBuckets {
     ];
 }
 
+/// Buckets geared towards measuring how many database connections are used in the database pool.
+pub struct DatabasePoolBuckets;
+
+impl HistogramBuckets for DatabasePoolBuckets {
+    const BUCKETS: &'static [f64] = &[0.0, 1.0, 2.0, 5.0, 10.0, 15.0, 20.0, 30.0, 50.0, 100.0];
+}
+
 /// Wrapper type over [`prometheus::Histogram`] to support defining buckets.
 pub struct Histogram<B: HistogramBuckets> {
     inner: prometheus::Histogram,

src/metrics/instance.rs

@@ -17,7 +17,7 @@
 //! As a rule of thumb, if the metric requires a database query to be updated it's probably a
 //! service-level metric, and you should add it to `src/metrics/service.rs` instead.
 
-use crate::metrics::histogram::{Histogram, HistogramVec, TimingBuckets};
+use crate::metrics::histogram::{DatabasePoolBuckets, Histogram, HistogramVec, TimingBuckets};
 use crate::util::errors::AppResult;
 use crate::{app::App, db::DieselPool};
 use prometheus::{proto::MetricFamily, IntCounter, IntCounterVec, IntGauge, IntGaugeVec};
@@ -30,6 +30,8 @@ metrics! {
         database_used_conns: IntGaugeVec["pool"],
         /// Amount of time required to obtain a database connection
         pub database_time_to_obtain_connection: HistogramVec<TimingBuckets>["pool"],
+        /// Number of used database connections in the pool, as histogram
+        pub database_used_conns_histogram: HistogramVec<DatabasePoolBuckets>["pool"],
 
         /// Number of requests processed by this instance
         pub requests_total: IntCounter,