Merge pull request #4806 from ferrous-systems/feature/use-cidr-block-list

Use env var to support blocking IP ranges

Author: Turbo87

Cargo.lock

@@ -61,6 +61,7 @@ hex = "=0.4.3"
 http = "=0.2.7"
 hyper = { version = "=0.14.18", features = ["client", "http1"] }
 indexmap = { version = "=1.8.1", features = ["serde-1"] }
+ipnetwork = "=0.19.0"
 tikv-jemallocator = { version = "=0.4.3", features = ['unprefixed_malloc_on_supported_platforms', 'profiling'] }
 lettre = { version = "=0.10.0-rc.6", default-features = false, features = ["file-transport", "smtp-transport", "native-tls", "hostname", "builder"] }
 minijinja = "=0.15.0"

Cargo.toml

@@ -1,3 +1,5 @@
+use ipnetwork::IpNetwork;
+
 use crate::publish_rate_limit::PublishRateLimit;
 use crate::{env, env_optional, uploaders::Uploader, Env};
 
@@ -25,6 +27,7 @@ pub struct Server {
     pub blocked_traffic: Vec<(String, Vec<String>)>,
     pub max_allowed_page_offset: u32,
     pub page_offset_ua_blocklist: Vec<String>,
+    pub page_offset_cidr_blocklist: Vec<IpNetwork>,
     pub excluded_crate_names: Vec<String>,
     pub domain_name: String,
     pub allowed_origins: Vec<String>,
@@ -63,6 +66,9 @@ impl Default for Server {
     ///   be blocked if `WEB_MAX_ALLOWED_PAGE_OFFSET` is exceeded. Including an empty string in the
     ///   list will block *all* user-agents exceeding the offset. If not set or empty, no blocking
     ///   will occur.
+    /// - `WEB_PAGE_OFFSET_CIDR_BLOCKLIST`: A comma separated list of CIDR blocks that will be used
+    ///   to block IP addresses given in the `X-Real-Ip` HTTP header, e.g. `192.168.1.0/24`.
+    ///   If not set or empty, no blocking will occur.
     /// - `INSTANCE_METRICS_LOG_EVERY_SECONDS`: How frequently should instance metrics be logged.
     ///   If the environment variable is not present instance metrics are not logged.
     /// - `FORCE_UNCONDITIONAL_REDIRECTS`: Whether to force unconditional redirects in the download
@@ -84,6 +90,13 @@ impl Default for Server {
             Some(s) if s.is_empty() => vec![],
             Some(s) => s.split(',').map(String::from).collect(),
         };
+        let page_offset_cidr_blocklist =
+            match env_optional::<String>("WEB_PAGE_OFFSET_CIDR_BLOCKLIST") {
+                None => vec![],
+                Some(s) if s.is_empty() => vec![],
+                Some(s) => s.split(',').map(String::from).collect(),
+            };
+
         let base = Base::from_environment();
         let excluded_crate_names = match env_optional::<String>("EXCLUDED_CRATE_NAMES") {
             None => vec![],
@@ -103,6 +116,7 @@ impl Default for Server {
             blocked_traffic: blocked_traffic(),
             max_allowed_page_offset: env_optional("WEB_MAX_ALLOWED_PAGE_OFFSET").unwrap_or(200),
             page_offset_ua_blocklist,
+            page_offset_cidr_blocklist: parse_cidr_blocks(&page_offset_cidr_blocklist),
             excluded_crate_names,
             domain_name: domain_name(),
             allowed_origins,
@@ -144,6 +158,41 @@ pub(crate) fn domain_name() -> String {
     dotenv::var("DOMAIN_NAME").unwrap_or_else(|_| "crates.io".into())
 }
 
+/// Parses list of CIDR block strings to valid `IpNetwork` structs.
+///
+/// The purpose is to be able to block IP ranges that overload the API that uses pagination.
+///
+/// The minimum number of bits for a host prefix must be
+///
+/// * at least 16 for IPv4 based CIDRs.
+/// * at least 64 for IPv6 based CIDRs
+///
+fn parse_cidr_blocks(blocks: &[String]) -> Vec<IpNetwork> {
+    blocks
+        .iter()
+        .map(|block| {
+            let network = block.parse::<IpNetwork>();
+            match network {
+                Ok(cidr) => {
+                    let host_prefix = match cidr {
+                        IpNetwork::V4(_) => 16,
+                        IpNetwork::V6(_) => 64,
+                    };
+                    if cidr.prefix() < host_prefix {
+                        panic!(
+                            "WEB_PAGE_OFFSET_CIDR_BLOCKLIST only allows CIDR blocks with a host prefix \
+                                of at least 16 bits (IPv4) or 64 bits (IPv6)."
+                        );
+                    } else {
+                        cidr
+                    }
+                },
+                Err(_) => panic!("WEB_PAGE_OFFSET_CIDR_BLOCKLIST must contain IPv4 or IPv6 CIDR blocks."),
+            }
+        })
+        .collect::<Vec<_>>()
+}
+
 fn blocked_traffic() -> Vec<(String, Vec<String>)> {
     let pattern_list = dotenv::var("BLOCKED_TRAFFIC").unwrap_or_default();
     parse_traffic_patterns(&pattern_list)
@@ -183,3 +232,38 @@ fn parse_traffic_patterns_splits_on_comma_and_looks_for_equal_sign() {
 
     assert_none!(parse_traffic_patterns(pattern_string_3).next());
 }
+
+#[test]
+fn parse_cidr_block_list_successfully() {
+    let cidr_blocks = vec!["127.0.0.1/24".to_string(), "192.168.0.1/31".to_string()];
+
+    let blocks = parse_cidr_blocks(&cidr_blocks);
+    assert_eq!(
+        vec![
+            "127.0.0.1/24".parse::<IpNetwork>().unwrap(),
+            "192.168.0.1/31".parse::<IpNetwork>().unwrap(),
+        ],
+        blocks,
+    );
+}
+
+#[test]
+#[should_panic]
+fn parse_cidr_blocks_panics_when_host_ipv4_prefix_is_too_low() {
+    parse_cidr_blocks(&["127.0.0.1/8".to_string()]);
+}
+
+#[test]
+#[should_panic]
+fn parse_cidr_blocks_panics_when_host_ipv6_prefix_is_too_low() {
+    parse_cidr_blocks(&["2001:0db8:0123:4567:89ab:cdef:1234:5678/56".to_string()]);
+}
+
+#[test]
+fn parse_ipv6_based_cidr_blocks() {
+    let input = vec![
+        "2002::1234:abcd:ffff:c0a8:101/64".to_string(),
+        "2001:0db8:0123:4567:89ab:cdef:1234:5678/92".to_string(),
+    ];
+    assert_eq!(2, parse_cidr_blocks(&input).len());
+}

src/config.rs

@@ -1,3 +1,4 @@
+use crate::config::Server;
 use crate::controllers::prelude::*;
 use crate::middleware::log_request::add_custom_metadata;
 use crate::models::helpers::with_count::*;
@@ -11,6 +12,8 @@ use diesel::query_dsl::LoadQuery;
 use diesel::sql_types::BigInt;
 use indexmap::IndexMap;
 use serde::{Deserialize, Serialize};
+use std::net::IpAddr;
+use std::str::FromStr;
 use std::sync::Arc;
 
 const MAX_PAGE_BEFORE_SUSPECTED_BOT: u32 = 10;
@@ -96,14 +99,10 @@ impl PaginationOptionsBuilder {
                 }
 
                 // Block large offsets for known violators of the crawler policy
-                if let Some(app) = self.limit_page_numbers {
+                if let Some(ref app) = self.limit_page_numbers {
                     let config = &app.config;
-                    let user_agent = request_header(req, header::USER_AGENT);
                     if numeric_page > config.max_allowed_page_offset
-                        && config
-                            .page_offset_ua_blocklist
-                            .iter()
-                            .any(|blocked| user_agent.contains(blocked))
+                        && is_useragent_or_ip_blocked(config, req)
                     {
                         add_custom_metadata("cause", "large page offset");
                         return Err(bad_request("requested page offset is too large"));
@@ -257,6 +256,37 @@ impl RawSeekPayload {
     }
 }
 
+/// Function to check if the request is blocked.
+///
+/// A request can be blocked if either the User Agent is on the User Agent block list or if the client
+/// IP is on the CIDR block list.
+fn is_useragent_or_ip_blocked(config: &Server, req: &dyn RequestExt) -> bool {
+    let user_agent = request_header(req, header::USER_AGENT);
+    let client_ip = request_header(req, "x-real-ip");
+
+    // check if user agent is blocked
+    if config
+        .page_offset_ua_blocklist
+        .iter()
+        .any(|blocked| user_agent.contains(blocked))
+    {
+        return true;
+    }
+
+    // check if client ip is blocked, needs to be an IPv4 address
+    if let Ok(client_ip) = IpAddr::from_str(client_ip) {
+        if config
+            .page_offset_cidr_blocklist
+            .iter()
+            .any(|blocked| blocked.contains(client_ip))
+        {
+            return true;
+        }
+    }
+
+    false
+}
+
 /// Encode a payload to be used as a seek key.
 ///
 /// The payload is base64-encoded to hint that it shouldn't be manually constructed. There is no

src/controllers/helpers/pagination.rs

@@ -5,6 +5,7 @@ use cargo_registry::models::Category;
 use cargo_registry::schema::crates;
 use diesel::{dsl::*, prelude::*, update};
 use http::StatusCode;
+use ipnetwork::IpNetwork;
 
 #[test]
 fn index() {
@@ -822,3 +823,27 @@ fn pagination_parameters_only_accept_integers() {
         json!({ "errors": [{ "detail": "invalid digit found in string" }] })
     );
 }
+
+#[test]
+fn pagination_blocks_ip_from_cidr_block_list() {
+    let (app, anon, user) = TestApp::init()
+        .with_config(|config| {
+            config.max_allowed_page_offset = 1;
+            config.page_offset_cidr_blocklist = vec!["127.0.0.1/24".parse::<IpNetwork>().unwrap()];
+        })
+        .with_user();
+    let user = user.as_model();
+
+    app.db(|conn| {
+        CrateBuilder::new("pagination_links_1", user.id).expect_build(conn);
+        CrateBuilder::new("pagination_links_2", user.id).expect_build(conn);
+        CrateBuilder::new("pagination_links_3", user.id).expect_build(conn);
+    });
+
+    let response = anon.get_with_query::<()>("/api/v1/crates", "page=2&per_page=1");
+    assert_eq!(response.status(), StatusCode::BAD_REQUEST);
+    assert_eq!(
+        response.into_json(),
+        json!({ "errors": [{ "detail": "requested page offset is too large" }] })
+    );
+}

src/tests/krate/search.rs

@@ -202,6 +202,7 @@ pub trait RequestHelper {
 fn req(method: conduit::Method, path: &str) -> MockRequest {
     let mut request = MockRequest::new(method, path);
     request.header(header::USER_AGENT, "conduit-test");
+    request.header("x-real-ip", "127.0.0.1");
     request
 }
 

src/tests/util.rs

@@ -341,6 +341,7 @@ fn simple_config() -> config::Server {
         blocked_traffic: Default::default(),
         max_allowed_page_offset: 200,
         page_offset_ua_blocklist: vec![],
+        page_offset_cidr_blocklist: vec![],
         excluded_crate_names: vec![],
         domain_name: "crates.io".into(),
         allowed_origins: Vec::new(),