set TCP_USER_TIMEOUT to 15 seconds for database connections

In the December 22, 2021 crates.io outage, we experienced full packet
loss between the database server and the application server(s). While
the crates.io application supports running without a database during
outages, those mitigations kicked in only after a bit more than 15
minutes of the packet loss starting.

The default parameters of the Linux network stack result in a TCP
connection being marked as broken after roughly 15 minutes of no
acknowledgements being received, which is what I believe happened during
that outage.

Broken database mitigations kicking in after 15 minutes is too long for
crates.io, as we ideally want those mitigations to kick in as soon as
possible (ensuring crates.io users can continue downloading crates).

This commit tells libpq (the underlying PostgreSQL client used by
diesel) to set the Linux network stack timeout to a configurable value,
which we set as 15 seconds. With this change, if an outage like the one
on Dec 22 2021 one happens again, crates.io will be fully unavailable
only for 15 seconds rather than 15 minutes.

Author: pietroalbini

src/config/database_pools.rs

@@ -9,6 +9,7 @@
 //! - `DB_OFFLINE`: If set to `leader` then use the read-only follower as if it was the leader.
 //!   If set to `follower` then act as if `READ_ONLY_REPLICA_URL` was unset.
 //! - `READ_ONLY_MODE`: If defined (even as empty) then force all connections to be read-only.
+//! - `DB_TCP_TIMEOUT_MS`: TCP timeout in milliseconds. See the doc comment for more details.
 
 use crate::env;
 
@@ -18,6 +19,12 @@ pub struct DatabasePools {
     pub primary: DbPoolConfig,
     /// An optional follower database. Always read-only.
     pub replica: Option<DbPoolConfig>,
+    /// Number of seconds to wait for unacknowledged TCP packets before treating the connection as
+    /// broken. This value will determine how long crates.io stays unavailable in case of full
+    /// packet loss between the application and the database: setting it too high will result in an
+    /// unnecessarily long outage (before the unhealthy database logic kicks in), while setting it
+    /// too low might result in healthy connections being dropped.
+    pub tcp_timeout_ms: u64,
 }
 
 #[derive(Debug)]
@@ -67,6 +74,11 @@ impl DatabasePools {
             _ => None,
         };
 
+        let tcp_timeout_ms = match dotenv::var("DB_TCP_TIMEOUT_MS") {
+            Ok(num) => num.parse().expect("couldn't parse DB_TCP_TIMEOUT_MS"),
+            Err(_) => 15 * 1000, // 15 seconds
+        };
+
         match dotenv::var("DB_OFFLINE").as_deref() {
             // The actual leader is down, use the follower in read-only mode as the primary and
             // don't configure a replica.
@@ -79,6 +91,7 @@ impl DatabasePools {
                     min_idle: primary_min_idle,
                 },
                 replica: None,
+                tcp_timeout_ms,
             },
             // The follower is down, don't configure the replica.
             Ok("follower") => Self {
@@ -89,6 +102,7 @@ impl DatabasePools {
                     min_idle: primary_min_idle,
                 },
                 replica: None,
+                tcp_timeout_ms,
             },
             _ => Self {
                 primary: DbPoolConfig {
@@ -106,6 +120,7 @@ impl DatabasePools {
                     pool_size: replica_pool_size,
                     min_idle: replica_min_idle,
                 }),
+                tcp_timeout_ms,
             },
         }
     }
@@ -119,6 +134,7 @@ impl DatabasePools {
                 min_idle: None,
             },
             replica: None,
+            tcp_timeout_ms: 1000, // 1 second
         }
     }
 }

src/db.rs

@@ -54,8 +54,8 @@ impl DieselPool {
     }
 
     pub(crate) fn new_test(config: &config::Server, url: &str) -> DieselPool {
-        let conn =
-            PgConnection::establish(&connection_url(config, url)).expect("failed to establish connection");
+        let conn = PgConnection::establish(&connection_url(config, url))
+            .expect("failed to establish connection");
         conn.begin_test_transaction()
             .expect("failed to begin test transaction");
         DieselPool::Test(Arc::new(ReentrantMutex::new(conn)))
@@ -142,13 +142,27 @@ pub fn connection_url(config: &config::Server, url: &str) -> String {
     let mut url = Url::parse(url).expect("Invalid database URL");
 
     // Enforce secure connections in production.
-    if config.base.env == Env::Production && !url.query_pairs().any(|(k, _)| k == "sslmode") {
-        url.query_pairs_mut().append_pair("sslmode", "require");
+    if config.base.env == Env::Production {
+        maybe_append_url_param(&mut url, "sslmode", "require");
     }
 
+    // Configure the time it takes for diesel to return an error when there is full packet loss
+    // between the application and the database.
+    maybe_append_url_param(
+        &mut url,
+        "tcp_user_timeout",
+        &config.db.tcp_timeout_ms.to_string(),
+    );
+
     url.into()
 }
 
+fn maybe_append_url_param(url: &mut Url, key: &str, value: &str) {
+    if !url.query_pairs().any(|(k, _)| k == key) {
+        url.query_pairs_mut().append_pair(key, value);
+    }
+}
+
 pub trait RequestTransaction {
     /// Obtain a read/write database connection from the primary pool
     fn db_conn(&self) -> Result<DieselPooledConn<'_>, PoolError>;