Introduce CIDR filter list to block IP ranges

This is to reject requests from IP ranges that overload the API
endpoints by fetching information from multiple clients at the same
time.
A recent incident highlighted this issue where a number of `crates?page`
requests hit the ">0.5 failed requests threshold" alert. These requests
timed out due to a high number of expensive requests made. These calls
came in from about 100 different IPs in parallel.

Before this change individual IPs or specific User Agents could be
blocked. These existing approaches have some drawbacks, blocking single
IPs would not have done much here as this is quite ineffective for this
behaviour, while blocking a specific (common) User Agent may prevent
valid usage of the API.

Therefore a list of CIDR blocks should be used to check if IPs belong to
certain ranges that send too many requests to expensive API endpoints.

**Please note** this has to be done very carefully, it's generally a
tool that may block whole regions of the internet.

Author: justahero

Cargo.lock

@@ -35,6 +35,7 @@ cargo-registry-index = { path = "cargo-registry-index" }
 cargo-registry-markdown = { path = "cargo-registry-markdown" }
 cargo-registry-s3 = { path = "cargo-registry-s3" }
 chrono = { version = "=0.4.19", features = ["serde"] }
+cidr-utils = "=0.5.6"
 clap = { version = "=3.1.18", features = ["derive", "unicode"] }
 
 conduit = "=0.10.0"

Cargo.toml

@@ -1,3 +1,5 @@
+use cidr_utils::cidr::Ipv4Cidr;
+
 use crate::publish_rate_limit::PublishRateLimit;
 use crate::{env, env_optional, uploaders::Uploader, Env};
 
@@ -25,6 +27,7 @@ pub struct Server {
     pub blocked_traffic: Vec<(String, Vec<String>)>,
     pub max_allowed_page_offset: u32,
     pub page_offset_ua_blocklist: Vec<String>,
+    pub page_offset_cidr_blocklist: Vec<Ipv4Cidr>,
     pub excluded_crate_names: Vec<String>,
     pub domain_name: String,
     pub allowed_origins: Vec<String>,
@@ -63,6 +66,9 @@ impl Default for Server {
     ///   be blocked if `WEB_MAX_ALLOWED_PAGE_OFFSET` is exceeded. Including an empty string in the
     ///   list will block *all* user-agents exceeding the offset. If not set or empty, no blocking
     ///   will occur.
+    /// - `WEB_PAGE_OFFSET_CIDR_BLOCKLIST`: A comma separated list of CIDR blocks that will be used
+    ///   to block IP addresses given in the `X-Real-Ip` HTTP header, e.g. `192.168.1.0/24`.
+    ///   If not set or empty, no blocking will occur.
     /// - `INSTANCE_METRICS_LOG_EVERY_SECONDS`: How frequently should instance metrics be logged.
     ///   If the environment variable is not present instance metrics are not logged.
     /// - `FORCE_UNCONDITIONAL_REDIRECTS`: Whether to force unconditional redirects in the download
@@ -84,6 +90,13 @@ impl Default for Server {
             Some(s) if s.is_empty() => vec![],
             Some(s) => s.split(',').map(String::from).collect(),
         };
+        let page_offset_cidr_blocklist =
+            match env_optional::<String>("WEB_PAGE_OFFSET_CIDR_BLOCKLIST") {
+                None => vec![],
+                Some(s) if s.is_empty() => vec![],
+                Some(s) => s.split(',').map(String::from).collect(),
+            };
+
         let base = Base::from_environment();
         let excluded_crate_names = match env_optional::<String>("EXCLUDED_CRATE_NAMES") {
             None => vec![],
@@ -103,6 +116,7 @@ impl Default for Server {
             blocked_traffic: blocked_traffic(),
             max_allowed_page_offset: env_optional("WEB_MAX_ALLOWED_PAGE_OFFSET").unwrap_or(200),
             page_offset_ua_blocklist,
+            page_offset_cidr_blocklist: parse_cidr_blocks(&page_offset_cidr_blocklist),
             excluded_crate_names,
             domain_name: domain_name(),
             allowed_origins,
@@ -144,6 +158,33 @@ pub(crate) fn domain_name() -> String {
     dotenv::var("DOMAIN_NAME").unwrap_or_else(|_| "crates.io".into())
 }
 
+/// Parses list of CIDR block strings to valid `Ipv4Cidr` structs.
+///
+/// The purpose is to be able to block IP ranges that overload the API that contains pagination.
+/// A valid CIDR block has the following restriction:
+///
+/// * Only IPv4 blocks are currently supported.
+/// * The minimum number of host prefix bits must be at least 16.
+///
+fn parse_cidr_blocks(blocks: &[String]) -> Vec<Ipv4Cidr> {
+    blocks
+        .iter()
+        .map(|block| match Ipv4Cidr::from_str(block) {
+            Ok(cidr) => {
+                if cidr.get_bits() < 16 {
+                    panic!(
+                        "WEB_PAGE_OFFSET_CIDR_BLOCKLIST must only contain CIDR blocks with \
+                            a host prefix of at least 16 bits."
+                    )
+                } else {
+                    cidr
+                }
+            }
+            Err(_) => panic!("WEB_PAGE_OFFSET_CIDR_BLOCKLIST only allows IPv4 CIDR blocks"),
+        })
+        .collect::<Vec<_>>()
+}
+
 fn blocked_traffic() -> Vec<(String, Vec<String>)> {
     let pattern_list = dotenv::var("BLOCKED_TRAFFIC").unwrap_or_default();
     parse_traffic_patterns(&pattern_list)
@@ -183,3 +224,31 @@ fn parse_traffic_patterns_splits_on_comma_and_looks_for_equal_sign() {
 
     assert_none!(parse_traffic_patterns(pattern_string_3).next());
 }
+
+#[test]
+fn parse_cidr_block_list_successfully() {
+    let cidr_blocks = vec!["127.0.0.1/24".to_string(), "192.168.0.1/31".to_string()];
+
+    let blocks = parse_cidr_blocks(&cidr_blocks);
+    assert_eq!(
+        vec![
+            Ipv4Cidr::from_str("127.0.0.1/24").unwrap(),
+            Ipv4Cidr::from_str("192.168.0.1/31").unwrap(),
+        ],
+        blocks,
+    );
+}
+
+#[test]
+#[should_panic]
+fn parse_cidr_blocks_panics_when_host_prefix_is_too_low() {
+    let input = vec!["127.0.0.1/8".to_string()];
+    parse_cidr_blocks(&input);
+}
+
+#[test]
+#[should_panic]
+fn parse_cidr_blocks_panics_when_ipv6_is_given() {
+    let input = vec!["2002::1234:abcd:ffff:c0a8:101/64".to_string()];
+    parse_cidr_blocks(&input);
+}

src/config.rs

@@ -1,3 +1,4 @@
+use crate::config::Server;
 use crate::controllers::prelude::*;
 use crate::middleware::log_request::add_custom_metadata;
 use crate::models::helpers::with_count::*;
@@ -11,6 +12,8 @@ use diesel::query_dsl::LoadQuery;
 use diesel::sql_types::BigInt;
 use indexmap::IndexMap;
 use serde::{Deserialize, Serialize};
+use std::net::Ipv4Addr;
+use std::str::FromStr;
 use std::sync::Arc;
 
 const MAX_PAGE_BEFORE_SUSPECTED_BOT: u32 = 10;
@@ -96,14 +99,10 @@ impl PaginationOptionsBuilder {
                 }
 
                 // Block large offsets for known violators of the crawler policy
-                if let Some(app) = self.limit_page_numbers {
+                if let Some(ref app) = self.limit_page_numbers {
                     let config = &app.config;
-                    let user_agent = request_header(req, header::USER_AGENT);
                     if numeric_page > config.max_allowed_page_offset
-                        && config
-                            .page_offset_ua_blocklist
-                            .iter()
-                            .any(|blocked| user_agent.contains(blocked))
+                        && is_useragent_or_ip_blocked(config, req)
                     {
                         add_custom_metadata("cause", "large page offset");
                         return Err(bad_request("requested page offset is too large"));
@@ -257,6 +256,37 @@ impl RawSeekPayload {
     }
 }
 
+/// Function to check if the request is blocked.
+///
+/// A request can be blocked if either the User Agent is on the User Agent block list or if the client
+/// IP is on the CIDR block list.
+fn is_useragent_or_ip_blocked(config: &Server, req: &dyn RequestExt) -> bool {
+    let user_agent = request_header(req, header::USER_AGENT);
+    let client_ip = request_header(req, "x-real-ip");
+
+    // check if user agent is blocked
+    if config
+        .page_offset_ua_blocklist
+        .iter()
+        .any(|blocked| user_agent.contains(blocked))
+    {
+        return true;
+    }
+
+    // check if client ip is blocked, needs to be an IPv4 address
+    if let Ok(client_ip) = Ipv4Addr::from_str(client_ip) {
+        if config
+            .page_offset_cidr_blocklist
+            .iter()
+            .any(|blocked| blocked.contains(client_ip))
+        {
+            return true;
+        }
+    }
+
+    false
+}
+
 /// Encode a payload to be used as a seek key.
 ///
 /// The payload is base64-encoded to hint that it shouldn't be manually constructed. There is no

src/controllers/helpers/pagination.rs

@@ -3,6 +3,7 @@ use crate::util::{RequestHelper, TestApp};
 use crate::{new_category, new_user};
 use cargo_registry::models::Category;
 use cargo_registry::schema::crates;
+use cidr_utils::cidr::Ipv4Cidr;
 use diesel::{dsl::*, prelude::*, update};
 use http::StatusCode;
 
@@ -822,3 +823,27 @@ fn pagination_parameters_only_accept_integers() {
         json!({ "errors": [{ "detail": "invalid digit found in string" }] })
     );
 }
+
+#[test]
+fn pagination_blocks_ip_from_cidr_block_list() {
+    let (app, anon, user) = TestApp::init()
+        .with_config(|config| {
+            config.max_allowed_page_offset = 1;
+            config.page_offset_cidr_blocklist = vec![Ipv4Cidr::from_str("127.0.0.1/24").unwrap()];
+        })
+        .with_user();
+    let user = user.as_model();
+
+    app.db(|conn| {
+        CrateBuilder::new("pagination_links_1", user.id).expect_build(conn);
+        CrateBuilder::new("pagination_links_2", user.id).expect_build(conn);
+        CrateBuilder::new("pagination_links_3", user.id).expect_build(conn);
+    });
+
+    let response = anon.get_with_query::<()>("/api/v1/crates", "page=2&per_page=1");
+    assert_eq!(response.status(), StatusCode::BAD_REQUEST);
+    assert_eq!(
+        response.into_json(),
+        json!({ "errors": [{ "detail": "requested page offset is too large" }] })
+    );
+}

src/tests/krate/search.rs

@@ -202,6 +202,7 @@ pub trait RequestHelper {
 fn req(method: conduit::Method, path: &str) -> MockRequest {
     let mut request = MockRequest::new(method, path);
     request.header(header::USER_AGENT, "conduit-test");
+    request.header("x-real-ip", "127.0.0.1");
     request
 }
 

src/tests/util.rs

@@ -341,6 +341,7 @@ fn simple_config() -> config::Server {
         blocked_traffic: Default::default(),
         max_allowed_page_offset: 200,
         page_offset_ua_blocklist: vec![],
+        page_offset_cidr_blocklist: vec![],
         excluded_crate_names: vec![],
         domain_name: "crates.io".into(),
         allowed_origins: Vec::new(),