From 4334f94fb995762a53b342411a29bbc626df572c Mon Sep 17 00:00:00 2001
From: Nelson J Morais <njmorais@gmail.com>
Date: Mon, 21 Dec 2020 17:27:19 +0000
Subject: [PATCH 1/8] added github actions runner collector

---
 src/collectors/github_runners.rs | 180 +++++++++++++++++++++++++++++++
 src/collectors/mod.rs            |   9 ++
 src/config.rs                    |  11 ++
 src/http.rs                      |  37 +++++++
 src/lib.rs                       |   4 +-
 5 files changed, 240 insertions(+), 1 deletion(-)
 create mode 100644 src/collectors/github_runners.rs
 create mode 100644 src/http.rs

diff --git a/src/collectors/github_runners.rs b/src/collectors/github_runners.rs
new file mode 100644
index 0000000..a2cf842
--- /dev/null
+++ b/src/collectors/github_runners.rs
@@ -0,0 +1,180 @@
+use crate::{http, http::is_token_flagged, Config};
+use anyhow::Error;
+use log::{debug, error, warn};
+use prometheus::core::Desc;
+use prometheus::proto::MetricFamily;
+use prometheus::{core::Collector, IntGauge, Opts};
+use std::collections::HashMap;
+use std::sync::{Arc, RwLock};
+use tokio::time::Duration;
+
+const GH_RUNNERS_ENDPOINT: &str = "https://api.github.com/repos/{owner_repo}/actions/runners";
+
+#[derive(Debug, serde::Deserialize)]
+struct ApiResponse {
+    total_count: usize,
+    runners: Vec<Runner>,
+}
+
+#[derive(Debug, serde::Deserialize)]
+struct Runner {
+    id: usize,
+    name: String,
+    os: String,
+    status: String,
+    busy: bool,
+    labels: Vec<Label>,
+}
+
+#[derive(Debug, serde::Deserialize)]
+struct Label {
+    id: usize,
+    name: String,
+    #[serde(rename = "type")]
+    the_type: String,
+}
+
+#[derive(Clone)]
+pub struct GithubRunners {
+    //api token to use
+    token: String,
+    // repos to track gha runners
+    repos: Vec<String>,
+    // metric namespace
+    ns: String,
+    // actual metrics
+    metrics: Arc<RwLock<Vec<IntGauge>>>,
+    // default metric description
+    desc: Desc,
+}
+
+impl GithubRunners {
+    pub async fn new(config: &Config) -> Result<Self, Error> {
+        let token = config.rust_runners_token.to_string();
+        let repos: Vec<String> = config
+            .gha_runners_repos
+            .split(',')
+            .map(|v| v.trim().to_string())
+            .collect();
+
+        let ns = String::from("gha_runner");
+        let rv = Self {
+            token,
+            repos,
+            ns: ns.clone(),
+            metrics: Arc::new(RwLock::new(Vec::new())),
+            desc: Desc::new(
+                ns,
+                "GHA runner's status".to_string(),
+                Vec::new(),
+                HashMap::new(),
+            )
+                .unwrap(),
+        };
+
+        let refresh_rate = config.gha_runners_cache_refresh;
+        let mut rv2 = rv.clone();
+        tokio::spawn(async move {
+            loop {
+                if let Err(e) = rv2.update_stats().await {
+                    error!("{:#?}", e);
+                }
+
+                tokio::time::delay_for(Duration::from_secs(refresh_rate)).await;
+            }
+        });
+
+        Ok(rv)
+    }
+
+    async fn update_stats(&mut self) -> Result<(), Error> {
+        let mut gauges = Vec::new();
+        for repo in self.repos.iter() {
+            let url = String::from(GH_RUNNERS_ENDPOINT).replace("{owner_repo}", repo);
+
+            // does this token still able to query github api? (rate limit wise)
+            match is_token_flagged(&self.token).await {
+                Err(e) => {
+                    error!("checking if token is flagged: {:?}", e);
+                    continue;
+                }
+                Ok(true) => {
+                    warn!(
+                        "token: '{}' is currently flagged. skipping data gathering",
+                        &self.token
+                    );
+                    continue;
+                }
+                Ok(false) => {}
+            }
+
+            debug!("Querying gha runner's status at: {}", url);
+            let resp = http::get(&self.token, &url)
+                .send()
+                .await?
+                .json::<ApiResponse>()
+                .await?;
+
+            debug!("ApiResponse: {:#?}", resp);
+
+            // convert to metrics
+            for runner in resp.runners.iter() {
+                let status = &runner.status.clone();
+                let value_busy = if runner.busy { 1 } else { 0 };
+                let label_repo = repo.clone();
+                let label_runner = runner.name.clone();
+
+                // online
+                let online = IntGauge::with_opts(
+                    Opts::new("online", "runner is online.")
+                        .namespace(self.ns.clone())
+                        .const_label("repo", label_repo.clone())
+                        .const_label("runner", label_runner.clone()),
+                )
+                    .unwrap();
+
+                online.set(if status == "online" { 1 } else { 0 });
+                gauges.push(online);
+
+                // busy
+                let busy = IntGauge::with_opts(
+                    Opts::new("busy", "runner is busy.")
+                        .namespace(self.ns.clone())
+                        .const_label("repo", label_repo)
+                        .const_label("runner", label_runner),
+                )
+                    .unwrap();
+
+                busy.set(value_busy);
+                gauges.push(busy);
+            }
+        }
+
+        // lock and replace old data
+        let mut guard = self.metrics.write().unwrap();
+        *guard = gauges;
+
+        Ok(())
+    }
+}
+
+impl Collector for GithubRunners {
+    fn desc(&self) -> Vec<&Desc> {
+        vec![&self.desc]
+    }
+
+    fn collect(&self) -> Vec<MetricFamily> {
+        self.metrics.read().map_or_else(
+            |e| {
+                error!("Unable to collect: {:#?}", e);
+                Vec::new()
+            },
+            |guard| {
+                guard.iter().fold(Vec::new(), |mut acc, item| {
+                    acc.extend(item.collect());
+                    acc
+                })
+            },
+        )
+    }
+}
\ No newline at end of file
diff --git a/src/collectors/mod.rs b/src/collectors/mod.rs
index f6ba43c..1b45196 100644
--- a/src/collectors/mod.rs
+++ b/src/collectors/mod.rs
@@ -1,6 +1,8 @@
 mod github_rate_limit;
+mod github_runners;
 
 pub use crate::collectors::github_rate_limit::GitHubRateLimit;
+pub use crate::collectors::github_runners::GithubRunners;
 
 use crate::MetricProvider;
 use anyhow::{Error, Result};
@@ -14,5 +16,12 @@ pub async fn register_collectors(p: &MetricProvider) -> Result<(), Error> {
             info!("Registering GitHubRateLimit collector");
             p.register_collector(rl)
         })
+        .await?;
+
+    GithubRunners::new(&p.config)
+        .and_then(|gr| async {
+            info!("Registering GitHubActionsRunners collector");
+            p.register_collector(gr)
+        })
         .await
 }
diff --git a/src/config.rs b/src/config.rs
index fed220e..78a7213 100644
--- a/src/config.rs
+++ b/src/config.rs
@@ -14,6 +14,14 @@ pub struct Config {
     pub gh_rate_limit_tokens: String,
     // github rate limit stats data cache refresh rate frequency (in seconds)
     pub gh_rate_limit_stats_cache_refresh: u64,
+    // github api token to be used when querying for gha runner's status
+    // note: token must have (repo scope) authorization
+    pub rust_runners_token: String,
+    // gh runner's repos to track they status. multiple repos are allowed
+    // ex. "rust,cargo,docs.rs"
+    pub gha_runners_repos: String,
+    // gha runner's status refresh rate frequency (in seconds)
+    pub gha_runners_cache_refresh: u64,
 }
 
 impl Config {
@@ -23,6 +31,9 @@ impl Config {
             port: default_env("PORT", 3001)?,
             gh_rate_limit_tokens: require_env("RATE_LIMIT_TOKENS")?,
             gh_rate_limit_stats_cache_refresh: default_env("GH_RATE_LIMIT_STATS_REFRESH", 120)?,
+            rust_runners_token: require_env("RUST_ORG_TOKEN")?,
+            gha_runners_repos: require_env("RUNNERS_REPOS")?,
+            gha_runners_cache_refresh: default_env("GHA_RUNNERS_REFRESH", 120)?,
         })
     }
 }
diff --git a/src/http.rs b/src/http.rs
new file mode 100644
index 0000000..6611a27
--- /dev/null
+++ b/src/http.rs
@@ -0,0 +1,37 @@
+use anyhow::{Context, Error, Result};
+use reqwest::header::{ACCEPT, AUTHORIZATION, USER_AGENT};
+use reqwest::{Client, Method, RequestBuilder};
+use std::collections::HashMap;
+
+pub(crate) const GH_API_RATE_LIMIT_ENDPOINT: &str = "https://api.github.com/rate_limit";
+
+pub(crate) fn get(token: &str, url: &str) -> RequestBuilder {
+    Client::new()
+        .request(Method::GET, url)
+        .header(
+            USER_AGENT,
+            "https://github.com/rust-lang/monitorbot (infra@rust-lang.org)",
+        )
+        .header(AUTHORIZATION, format!("{} {}", "token", token))
+        .header(ACCEPT, "application/vnd.github.v3+json")
+}
+
+#[derive(Debug, serde::Deserialize)]
+pub(crate) struct GithubRateLimit {
+    pub rate: HashMap<String, usize>,
+}
+
+pub(crate) async fn is_token_flagged(token: &str) -> Result<bool, Error> {
+    get_token_rate_limit_stats(token)
+        .await
+        .map(|mut r| Ok(r.rate.remove("remaining").unwrap_or(0) == 0))?
+}
+
+pub(crate) async fn get_token_rate_limit_stats(token: &str) -> Result<GithubRateLimit, Error> {
+    get(token, GH_API_RATE_LIMIT_ENDPOINT)
+        .send()
+        .await?
+        .json::<GithubRateLimit>()
+        .await
+        .context("Unable to get rate limit stats")
+}
\ No newline at end of file
diff --git a/src/lib.rs b/src/lib.rs
index a0860be..c455354 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -1,6 +1,7 @@
 #![allow(clippy::new_without_default)]
 
 pub mod collectors;
+pub(crate) mod http;
 mod config;
 
 pub use config::Config;
@@ -25,7 +26,8 @@ pub struct MetricProvider {
 
 impl MetricProvider {
     pub fn new(config: Config) -> Self {
-        let register = Registry::new_custom(None, None).expect("Unable to build Registry");
+        let register = Registry::new_custom(Some("monitorbot".to_string()), None)
+            .expect("Unable to build Registry");
         Self { register, config }
     }
 

From 0454065856e9717a96f027a54ef9c1f7e6a73969 Mon Sep 17 00:00:00 2001
From: Nelson J Morais <njmorais@gmail.com>
Date: Mon, 21 Dec 2020 17:53:34 +0000
Subject: [PATCH 2/8] cargo fmt fixes

---
 src/collectors/github_runners.rs | 8 ++++----
 src/http.rs                      | 2 +-
 src/lib.rs                       | 2 +-
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/collectors/github_runners.rs b/src/collectors/github_runners.rs
index a2cf842..c3b7bd5 100644
--- a/src/collectors/github_runners.rs
+++ b/src/collectors/github_runners.rs
@@ -69,7 +69,7 @@ impl GithubRunners {
                 Vec::new(),
                 HashMap::new(),
             )
-                .unwrap(),
+            .unwrap(),
         };
 
         let refresh_rate = config.gha_runners_cache_refresh;
@@ -131,7 +131,7 @@ impl GithubRunners {
                         .const_label("repo", label_repo.clone())
                         .const_label("runner", label_runner.clone()),
                 )
-                    .unwrap();
+                .unwrap();
 
                 online.set(if status == "online" { 1 } else { 0 });
                 gauges.push(online);
@@ -143,7 +143,7 @@ impl GithubRunners {
                         .const_label("repo", label_repo)
                         .const_label("runner", label_runner),
                 )
-                    .unwrap();
+                .unwrap();
 
                 busy.set(value_busy);
                 gauges.push(busy);
@@ -177,4 +177,4 @@ impl Collector for GithubRunners {
             },
         )
     }
-}
\ No newline at end of file
+}
diff --git a/src/http.rs b/src/http.rs
index 6611a27..4f6b141 100644
--- a/src/http.rs
+++ b/src/http.rs
@@ -34,4 +34,4 @@ pub(crate) async fn get_token_rate_limit_stats(token: &str) -> Result<GithubRate
         .json::<GithubRateLimit>()
         .await
         .context("Unable to get rate limit stats")
-}
\ No newline at end of file
+}
diff --git a/src/lib.rs b/src/lib.rs
index c455354..fcc1c04 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -1,8 +1,8 @@
 #![allow(clippy::new_without_default)]
 
 pub mod collectors;
-pub(crate) mod http;
 mod config;
+pub(crate) mod http;
 
 pub use config::Config;
 

From c4c2def80c918661c63231296d2f637a4e7b2b85 Mon Sep 17 00:00:00 2001
From: Nelson J Morais <njmorais@gmail.com>
Date: Sun, 14 Mar 2021 00:52:38 +0000
Subject: [PATCH 3/8] rebase

---
 src/collectors/github_rate_limit.rs |   2 +-
 src/collectors/github_runners.rs    | 133 +++++++++++++---------------
 src/http.rs                         |  24 -----
 3 files changed, 62 insertions(+), 97 deletions(-)

diff --git a/src/collectors/github_rate_limit.rs b/src/collectors/github_rate_limit.rs
index 3aaafbe..703cca4 100644
--- a/src/collectors/github_rate_limit.rs
+++ b/src/collectors/github_rate_limit.rs
@@ -184,7 +184,7 @@ impl ProductMetrics {
         let gauge = |name, help| -> IntGauge {
             IntGauge::with_opts(
                 Opts::new(name, help)
-                    .namespace("monitorbot_github_rate_limit")
+                    .namespace("github_rate_limit")
                     .const_label("username", user)
                     .const_label("product", product),
             )
diff --git a/src/collectors/github_runners.rs b/src/collectors/github_runners.rs
index c3b7bd5..e0954a4 100644
--- a/src/collectors/github_runners.rs
+++ b/src/collectors/github_runners.rs
@@ -1,9 +1,11 @@
-use crate::{http, http::is_token_flagged, Config};
+use crate::{http, Config};
 use anyhow::Error;
 use log::{debug, error, warn};
-use prometheus::core::Desc;
+use prometheus::core::AtomicI64;
+use prometheus::core::{Desc, GenericGauge};
 use prometheus::proto::MetricFamily;
 use prometheus::{core::Collector, IntGauge, Opts};
+use reqwest::header::LINK;
 use std::collections::HashMap;
 use std::sync::{Arc, RwLock};
 use tokio::time::Duration;
@@ -23,15 +25,6 @@ struct Runner {
     os: String,
     status: String,
     busy: bool,
-    labels: Vec<Label>,
-}
-
-#[derive(Debug, serde::Deserialize)]
-struct Label {
-    id: usize,
-    name: String,
-    #[serde(rename = "type")]
-    the_type: String,
 }
 
 #[derive(Clone)]
@@ -40,8 +33,6 @@ pub struct GithubRunners {
     token: String,
     // repos to track gha runners
     repos: Vec<String>,
-    // metric namespace
-    ns: String,
     // actual metrics
     metrics: Arc<RwLock<Vec<IntGauge>>>,
     // default metric description
@@ -57,15 +48,13 @@ impl GithubRunners {
             .map(|v| v.trim().to_string())
             .collect();
 
-        let ns = String::from("gha_runner");
         let rv = Self {
             token,
             repos,
-            ns: ns.clone(),
             metrics: Arc::new(RwLock::new(Vec::new())),
             desc: Desc::new(
-                ns,
-                "GHA runner's status".to_string(),
+                String::from("gha_runner"),
+                String::from("GHA runner's status"),
                 Vec::new(),
                 HashMap::new(),
             )
@@ -90,63 +79,47 @@ impl GithubRunners {
     async fn update_stats(&mut self) -> Result<(), Error> {
         let mut gauges = Vec::new();
         for repo in self.repos.iter() {
-            let url = String::from(GH_RUNNERS_ENDPOINT).replace("{owner_repo}", repo);
+            let mut url: Option<String> = String::from(GH_RUNNERS_ENDPOINT)
+                .replace("{owner_repo}", repo)
+                .into();
+
+            while url.is_some() {
+                let response = http::get(&self.token, &url.unwrap()).send().await?;
+                url = match response.headers().get(LINK) {
+                    Some(data) => {
+                        //TODO: parse Link header for rel=next
+                        warn!("{:#?}", data);
+                        None
+                    }
+                    _ => None,
+                };
+
+                let resp = response.json::<ApiResponse>().await?;
+                debug!("{:#?}", resp);
+
+                for runner in resp.runners.iter() {
+                    let online = metric_factory(
+                        "online",
+                        "runner is online",
+                        &self.desc.fq_name,
+                        &repo,
+                        &runner.name,
+                    );
 
-            // does this token still able to query github api? (rate limit wise)
-            match is_token_flagged(&self.token).await {
-                Err(e) => {
-                    error!("checking if token is flagged: {:?}", e);
-                    continue;
-                }
-                Ok(true) => {
-                    warn!(
-                        "token: '{}' is currently flagged. skipping data gathering",
-                        &self.token
+                    online.set(if runner.status == "online" { 1 } else { 0 });
+                    gauges.push(online);
+
+                    // busy
+                    let busy = metric_factory(
+                        "busy",
+                        "runner is busy",
+                        &self.desc.fq_name,
+                        &repo,
+                        &runner.name,
                     );
-                    continue;
+                    busy.set(if runner.busy { 1 } else { 0 });
+                    gauges.push(busy);
                 }
-                Ok(false) => {}
-            }
-
-            debug!("Querying gha runner's status at: {}", url);
-            let resp = http::get(&self.token, &url)
-                .send()
-                .await?
-                .json::<ApiResponse>()
-                .await?;
-
-            debug!("ApiResponse: {:#?}", resp);
-
-            // convert to metrics
-            for runner in resp.runners.iter() {
-                let status = &runner.status.clone();
-                let value_busy = if runner.busy { 1 } else { 0 };
-                let label_repo = repo.clone();
-                let label_runner = runner.name.clone();
-
-                // online
-                let online = IntGauge::with_opts(
-                    Opts::new("online", "runner is online.")
-                        .namespace(self.ns.clone())
-                        .const_label("repo", label_repo.clone())
-                        .const_label("runner", label_runner.clone()),
-                )
-                .unwrap();
-
-                online.set(if status == "online" { 1 } else { 0 });
-                gauges.push(online);
-
-                // busy
-                let busy = IntGauge::with_opts(
-                    Opts::new("busy", "runner is busy.")
-                        .namespace(self.ns.clone())
-                        .const_label("repo", label_repo)
-                        .const_label("runner", label_runner),
-                )
-                .unwrap();
-
-                busy.set(value_busy);
-                gauges.push(busy);
             }
         }
 
@@ -167,7 +140,7 @@ impl Collector for GithubRunners {
         self.metrics.read().map_or_else(
             |e| {
                 error!("Unable to collect: {:#?}", e);
-                Vec::new()
+                Vec::with_capacity(0)
             },
             |guard| {
                 guard.iter().fold(Vec::new(), |mut acc, item| {
@@ -178,3 +151,19 @@ impl Collector for GithubRunners {
         )
     }
 }
+
+fn metric_factory<S: Into<String>>(
+    name: S,
+    help: S,
+    ns: S,
+    repo: S,
+    runner: S,
+) -> GenericGauge<AtomicI64> {
+    IntGauge::with_opts(
+        Opts::new(name, help)
+            .namespace(ns)
+            .const_label("repo", repo)
+            .const_label("runner", runner),
+    )
+    .unwrap()
+}
diff --git a/src/http.rs b/src/http.rs
index 4f6b141..0307001 100644
--- a/src/http.rs
+++ b/src/http.rs
@@ -1,9 +1,5 @@
-use anyhow::{Context, Error, Result};
 use reqwest::header::{ACCEPT, AUTHORIZATION, USER_AGENT};
 use reqwest::{Client, Method, RequestBuilder};
-use std::collections::HashMap;
-
-pub(crate) const GH_API_RATE_LIMIT_ENDPOINT: &str = "https://api.github.com/rate_limit";
 
 pub(crate) fn get(token: &str, url: &str) -> RequestBuilder {
     Client::new()
@@ -15,23 +11,3 @@ pub(crate) fn get(token: &str, url: &str) -> RequestBuilder {
         .header(AUTHORIZATION, format!("{} {}", "token", token))
         .header(ACCEPT, "application/vnd.github.v3+json")
 }
-
-#[derive(Debug, serde::Deserialize)]
-pub(crate) struct GithubRateLimit {
-    pub rate: HashMap<String, usize>,
-}
-
-pub(crate) async fn is_token_flagged(token: &str) -> Result<bool, Error> {
-    get_token_rate_limit_stats(token)
-        .await
-        .map(|mut r| Ok(r.rate.remove("remaining").unwrap_or(0) == 0))?
-}
-
-pub(crate) async fn get_token_rate_limit_stats(token: &str) -> Result<GithubRateLimit, Error> {
-    get(token, GH_API_RATE_LIMIT_ENDPOINT)
-        .send()
-        .await?
-        .json::<GithubRateLimit>()
-        .await
-        .context("Unable to get rate limit stats")
-}

From 08c655d00eacde60aee6010a3ccc69c58acbedf1 Mon Sep 17 00:00:00 2001
From: Nelson J Morais <njmorais@gmail.com>
Date: Sun, 14 Mar 2021 01:28:08 +0000
Subject: [PATCH 4/8] changed env var to GITHUB_TOKEN

---
 src/collectors/github_runners.rs | 2 +-
 src/config.rs                    | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/collectors/github_runners.rs b/src/collectors/github_runners.rs
index e0954a4..e59f735 100644
--- a/src/collectors/github_runners.rs
+++ b/src/collectors/github_runners.rs
@@ -41,7 +41,7 @@ pub struct GithubRunners {
 
 impl GithubRunners {
     pub async fn new(config: &Config) -> Result<Self, Error> {
-        let token = config.rust_runners_token.to_string();
+        let token = config.github_token.to_string();
         let repos: Vec<String> = config
             .gha_runners_repos
             .split(',')
diff --git a/src/config.rs b/src/config.rs
index 78a7213..c405889 100644
--- a/src/config.rs
+++ b/src/config.rs
@@ -16,7 +16,7 @@ pub struct Config {
     pub gh_rate_limit_stats_cache_refresh: u64,
     // github api token to be used when querying for gha runner's status
     // note: token must have (repo scope) authorization
-    pub rust_runners_token: String,
+    pub github_token: String,
     // gh runner's repos to track they status. multiple repos are allowed
     // ex. "rust,cargo,docs.rs"
     pub gha_runners_repos: String,
@@ -31,7 +31,7 @@ impl Config {
             port: default_env("PORT", 3001)?,
             gh_rate_limit_tokens: require_env("RATE_LIMIT_TOKENS")?,
             gh_rate_limit_stats_cache_refresh: default_env("GH_RATE_LIMIT_STATS_REFRESH", 120)?,
-            rust_runners_token: require_env("RUST_ORG_TOKEN")?,
+            github_token: require_env("GITHUB_TOKEN")?,
             gha_runners_repos: require_env("RUNNERS_REPOS")?,
             gha_runners_cache_refresh: default_env("GHA_RUNNERS_REFRESH", 120)?,
         })

From c4d76044853e55e1cd4b78f7e8673502fc70249b Mon Sep 17 00:00:00 2001
From: Nelson J Morais <njmorais@gmail.com>
Date: Fri, 16 Apr 2021 01:44:52 +0100
Subject: [PATCH 5/8] added ratelimit-remaining guard and pagination with
 parse_link_header crate

---
 Cargo.lock                       | 11 ++++++
 Cargo.toml                       |  1 +
 src/collectors/github_runners.rs | 66 ++++++++++++++++++++++++--------
 3 files changed, 62 insertions(+), 16 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 4224e37..c4a2dae 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -595,6 +595,7 @@ dependencies = [
  "futures",
  "hyper",
  "log",
+ "parse_link_header",
  "prometheus",
  "reqwest",
  "serde",
@@ -706,6 +707,16 @@ dependencies = [
  "winapi 0.3.9",
 ]
 
+[[package]]
+name = "parse_link_header"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9b4f88f9547da526c2670628a21a1749599b79b65ffc2b8609c874fe012f58a0"
+dependencies = [
+ "http",
+ "regex",
+]
+
 [[package]]
 name = "percent-encoding"
 version = "2.1.0"
diff --git a/Cargo.toml b/Cargo.toml
index 8d5705d..36aa587 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -18,3 +18,4 @@ futures = "0.3"
 anyhow = "1.0"
 log = "0.4"
 env_logger = { version = "0.8", features = ["termcolor", "humantime"] }
+parse_link_header = "0.2"
diff --git a/src/collectors/github_runners.rs b/src/collectors/github_runners.rs
index e59f735..c5fd103 100644
--- a/src/collectors/github_runners.rs
+++ b/src/collectors/github_runners.rs
@@ -1,16 +1,18 @@
 use crate::{http, Config};
-use anyhow::Error;
+use anyhow::{bail, Result};
 use log::{debug, error, warn};
 use prometheus::core::AtomicI64;
 use prometheus::core::{Desc, GenericGauge};
 use prometheus::proto::MetricFamily;
 use prometheus::{core::Collector, IntGauge, Opts};
-use reqwest::header::LINK;
+use reqwest::header::{HeaderValue, LINK};
+use reqwest::{Response, StatusCode};
 use std::collections::HashMap;
 use std::sync::{Arc, RwLock};
 use tokio::time::Duration;
 
-const GH_RUNNERS_ENDPOINT: &str = "https://api.github.com/repos/{owner_repo}/actions/runners";
+const GH_RUNNERS_ENDPOINT: &str =
+    "https://api.github.com/repos/{owner_repo}/actions/runners?per_page=100";
 
 #[derive(Debug, serde::Deserialize)]
 struct ApiResponse {
@@ -40,7 +42,7 @@ pub struct GithubRunners {
 }
 
 impl GithubRunners {
-    pub async fn new(config: &Config) -> Result<Self, Error> {
+    pub async fn new(config: &Config) -> Result<Self> {
         let token = config.github_token.to_string();
         let repos: Vec<String> = config
             .gha_runners_repos
@@ -76,8 +78,8 @@ impl GithubRunners {
         Ok(rv)
     }
 
-    async fn update_stats(&mut self) -> Result<(), Error> {
-        let mut gauges = Vec::new();
+    async fn update_stats(&mut self) -> Result<()> {
+        let mut gauges = Vec::with_capacity(self.repos.len() * 2);
         for repo in self.repos.iter() {
             let mut url: Option<String> = String::from(GH_RUNNERS_ENDPOINT)
                 .replace("{owner_repo}", repo)
@@ -85,15 +87,15 @@ impl GithubRunners {
 
             while url.is_some() {
                 let response = http::get(&self.token, &url.unwrap()).send().await?;
-                url = match response.headers().get(LINK) {
-                    Some(data) => {
-                        //TODO: parse Link header for rel=next
-                        warn!("{:#?}", data);
-                        None
-                    }
-                    _ => None,
-                };
+                debug!(
+                    "http response status ({:#?}) and headers: {:#?}",
+                    response.status(),
+                    response.headers()
+                );
 
+                guard_rate_limited(&response)?;
+
+                url = next_uri(response.headers().get(LINK));
                 let resp = response.json::<ApiResponse>().await?;
                 debug!("{:#?}", resp);
 
@@ -105,11 +107,9 @@ impl GithubRunners {
                         &repo,
                         &runner.name,
                     );
-
                     online.set(if runner.status == "online" { 1 } else { 0 });
                     gauges.push(online);
 
-                    // busy
                     let busy = metric_factory(
                         "busy",
                         "runner is busy",
@@ -152,6 +152,40 @@ impl Collector for GithubRunners {
     }
 }
 
+fn guard_rate_limited(response: &Response) -> Result<()> {
+    if response.status() == StatusCode::FORBIDDEN {
+        warn!("403 FORBIDDEN response status");
+
+        let rate_limited = match response.headers().get("x-ratelimit-remaining") {
+            Some(rl) => rl.to_str()?.parse::<usize>()? == 0,
+            None => unreachable!(),
+        };
+
+        if rate_limited {
+            bail!("We've hit the rate limit");
+        }
+    }
+
+    Ok(())
+}
+
+fn next_uri(header: Option<&HeaderValue>) -> Option<String> {
+    if let Some(header) = header {
+        return match header.to_str() {
+            Ok(header_str) => match parse_link_header::parse(header_str) {
+                Ok(links) => match links.get(&Some("next".to_string())) {
+                    Some(link) => Some(link.uri.to_string()),
+                    None => None,
+                },
+                _ => None,
+            },
+            _ => None,
+        };
+    }
+
+    None
+}
+
 fn metric_factory<S: Into<String>>(
     name: S,
     help: S,

From cde1547ee407a34ecf54c0bf0412a4a203b10a51 Mon Sep 17 00:00:00 2001
From: Nelson J Morais <njmorais@gmail.com>
Date: Sat, 1 May 2021 16:09:37 +0100
Subject: [PATCH 6/8] added http client to gh runners collector

---
 src/collectors/github_runners.rs | 24 ++++++++++++++----------
 src/collectors/mod.rs            | 21 ++++++++++++++++++++-
 src/http.rs                      | 13 -------------
 src/lib.rs                       |  1 -
 4 files changed, 34 insertions(+), 25 deletions(-)
 delete mode 100644 src/http.rs

diff --git a/src/collectors/github_runners.rs b/src/collectors/github_runners.rs
index c5fd103..307f87a 100644
--- a/src/collectors/github_runners.rs
+++ b/src/collectors/github_runners.rs
@@ -1,4 +1,5 @@
-use crate::{http, Config};
+use super::default_headers;
+use crate::Config;
 use anyhow::{bail, Result};
 use log::{debug, error, warn};
 use prometheus::core::AtomicI64;
@@ -6,7 +7,7 @@ use prometheus::core::{Desc, GenericGauge};
 use prometheus::proto::MetricFamily;
 use prometheus::{core::Collector, IntGauge, Opts};
 use reqwest::header::{HeaderValue, LINK};
-use reqwest::{Response, StatusCode};
+use reqwest::{Client, Response, StatusCode};
 use std::collections::HashMap;
 use std::sync::{Arc, RwLock};
 use tokio::time::Duration;
@@ -39,10 +40,11 @@ pub struct GithubRunners {
     metrics: Arc<RwLock<Vec<IntGauge>>>,
     // default metric description
     desc: Desc,
+    http: Client,
 }
 
 impl GithubRunners {
-    pub async fn new(config: &Config) -> Result<Self> {
+    pub async fn new(config: &Config, http: Client) -> Result<Self> {
         let token = config.github_token.to_string();
         let repos: Vec<String> = config
             .gha_runners_repos
@@ -53,6 +55,7 @@ impl GithubRunners {
         let rv = Self {
             token,
             repos,
+            http,
             metrics: Arc::new(RwLock::new(Vec::new())),
             desc: Desc::new(
                 String::from("gha_runner"),
@@ -85,19 +88,20 @@ impl GithubRunners {
                 .replace("{owner_repo}", repo)
                 .into();
 
+            debug!("Updating runner's stats");
+
             while url.is_some() {
-                let response = http::get(&self.token, &url.unwrap()).send().await?;
-                debug!(
-                    "http response status ({:#?}) and headers: {:#?}",
-                    response.status(),
-                    response.headers()
-                );
+                let response = self
+                    .http
+                    .get(&url.unwrap())
+                    .headers(default_headers(&self.token))
+                    .send()
+                    .await?;
 
                 guard_rate_limited(&response)?;
 
                 url = next_uri(response.headers().get(LINK));
                 let resp = response.json::<ApiResponse>().await?;
-                debug!("{:#?}", resp);
 
                 for runner in resp.runners.iter() {
                     let online = metric_factory(
diff --git a/src/collectors/mod.rs b/src/collectors/mod.rs
index 1b45196..cceadf1 100644
--- a/src/collectors/mod.rs
+++ b/src/collectors/mod.rs
@@ -8,9 +8,12 @@ use crate::MetricProvider;
 use anyhow::{Error, Result};
 use futures::TryFutureExt;
 use log::info;
+use reqwest::header::{HeaderMap, ACCEPT, AUTHORIZATION, USER_AGENT};
+use reqwest::Client;
 
 // register collectors for metrics gathering
 pub async fn register_collectors(p: &MetricProvider) -> Result<(), Error> {
+    let http = Client::new();
     GitHubRateLimit::new(&p.config)
         .and_then(|rl| async {
             info!("Registering GitHubRateLimit collector");
@@ -18,10 +21,26 @@ pub async fn register_collectors(p: &MetricProvider) -> Result<(), Error> {
         })
         .await?;
 
-    GithubRunners::new(&p.config)
+    GithubRunners::new(&p.config, http)
         .and_then(|gr| async {
             info!("Registering GitHubActionsRunners collector");
             p.register_collector(gr)
         })
         .await
 }
+
+fn default_headers(token: &str) -> HeaderMap {
+    let mut headers = HeaderMap::new();
+    headers.insert(
+        USER_AGENT,
+        "https://github.com/rust-lang/monitorbot (infra@rust-lang.org)"
+            .parse()
+            .unwrap(),
+    );
+    headers.insert(
+        AUTHORIZATION,
+        format!("{} {}", "token", token).parse().unwrap(),
+    );
+    headers.insert(ACCEPT, "application/vnd.github.v3+json".parse().unwrap());
+    headers
+}
diff --git a/src/http.rs b/src/http.rs
deleted file mode 100644
index 0307001..0000000
--- a/src/http.rs
+++ /dev/null
@@ -1,13 +0,0 @@
-use reqwest::header::{ACCEPT, AUTHORIZATION, USER_AGENT};
-use reqwest::{Client, Method, RequestBuilder};
-
-pub(crate) fn get(token: &str, url: &str) -> RequestBuilder {
-    Client::new()
-        .request(Method::GET, url)
-        .header(
-            USER_AGENT,
-            "https://github.com/rust-lang/monitorbot (infra@rust-lang.org)",
-        )
-        .header(AUTHORIZATION, format!("{} {}", "token", token))
-        .header(ACCEPT, "application/vnd.github.v3+json")
-}
diff --git a/src/lib.rs b/src/lib.rs
index fcc1c04..208ee87 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -2,7 +2,6 @@
 
 pub mod collectors;
 mod config;
-pub(crate) mod http;
 
 pub use config::Config;
 

From 22cc3ed3c1dd06bf1a6c1e3006f41a690dbf123e Mon Sep 17 00:00:00 2001
From: Nelson J Morais <njmorais@gmail.com>
Date: Mon, 10 May 2021 23:35:15 +0100
Subject: [PATCH 7/8] injecting http client on collector registration added.
 fix bugs and address concerns

---
 src/collectors/github_runners.rs | 37 ++++++++++++++++----------------
 src/collectors/mod.rs            | 15 ++++++-------
 2 files changed, 24 insertions(+), 28 deletions(-)

diff --git a/src/collectors/github_runners.rs b/src/collectors/github_runners.rs
index 307f87a..7e30523 100644
--- a/src/collectors/github_runners.rs
+++ b/src/collectors/github_runners.rs
@@ -1,13 +1,13 @@
 use super::default_headers;
 use crate::Config;
-use anyhow::{bail, Result};
-use log::{debug, error, warn};
+use anyhow::{Context, Result};
+use log::{debug, error};
 use prometheus::core::AtomicI64;
 use prometheus::core::{Desc, GenericGauge};
 use prometheus::proto::MetricFamily;
 use prometheus::{core::Collector, IntGauge, Opts};
 use reqwest::header::{HeaderValue, LINK};
-use reqwest::{Client, Response, StatusCode};
+use reqwest::{Client, Response};
 use std::collections::HashMap;
 use std::sync::{Arc, RwLock};
 use tokio::time::Duration;
@@ -90,17 +90,18 @@ impl GithubRunners {
 
             debug!("Updating runner's stats");
 
-            while url.is_some() {
+            while let Some(endpoint) = url.take() {
                 let response = self
                     .http
-                    .get(&url.unwrap())
+                    .get(&endpoint)
                     .headers(default_headers(&self.token))
                     .send()
                     .await?;
 
-                guard_rate_limited(&response)?;
+                url = guard_rate_limited(&response)?
+                    .error_for_status_ref()
+                    .map(|res| next_uri(res.headers().get(LINK)))?;
 
-                url = next_uri(response.headers().get(LINK));
                 let resp = response.json::<ApiResponse>().await?;
 
                 for runner in resp.runners.iter() {
@@ -156,21 +157,19 @@ impl Collector for GithubRunners {
     }
 }
 
-fn guard_rate_limited(response: &Response) -> Result<()> {
-    if response.status() == StatusCode::FORBIDDEN {
-        warn!("403 FORBIDDEN response status");
+fn guard_rate_limited(response: &Response) -> Result<&Response> {
+    let rate_limited = match response.headers().get("x-ratelimit-remaining") {
+        Some(rl) => rl.to_str()?.parse::<usize>()? == 0,
+        None => unreachable!(),
+    };
 
-        let rate_limited = match response.headers().get("x-ratelimit-remaining") {
-            Some(rl) => rl.to_str()?.parse::<usize>()? == 0,
-            None => unreachable!(),
-        };
-
-        if rate_limited {
-            bail!("We've hit the rate limit");
-        }
+    if rate_limited {
+        return response
+            .error_for_status_ref()
+            .context("We've hit the rate limit");
     }
 
-    Ok(())
+    Ok(response)
 }
 
 fn next_uri(header: Option<&HeaderValue>) -> Option<String> {
diff --git a/src/collectors/mod.rs b/src/collectors/mod.rs
index cceadf1..358f583 100644
--- a/src/collectors/mod.rs
+++ b/src/collectors/mod.rs
@@ -8,12 +8,15 @@ use crate::MetricProvider;
 use anyhow::{Error, Result};
 use futures::TryFutureExt;
 use log::info;
-use reqwest::header::{HeaderMap, ACCEPT, AUTHORIZATION, USER_AGENT};
-use reqwest::Client;
+use reqwest::header::{HeaderMap, ACCEPT, AUTHORIZATION};
+use reqwest::ClientBuilder;
 
 // register collectors for metrics gathering
 pub async fn register_collectors(p: &MetricProvider) -> Result<(), Error> {
-    let http = Client::new();
+    let http = ClientBuilder::new()
+        .user_agent("https://github.com/rust-lang/monitorbot (infra@rust-lang.org)")
+        .build()?;
+
     GitHubRateLimit::new(&p.config)
         .and_then(|rl| async {
             info!("Registering GitHubRateLimit collector");
@@ -31,12 +34,6 @@ pub async fn register_collectors(p: &MetricProvider) -> Result<(), Error> {
 
 fn default_headers(token: &str) -> HeaderMap {
     let mut headers = HeaderMap::new();
-    headers.insert(
-        USER_AGENT,
-        "https://github.com/rust-lang/monitorbot (infra@rust-lang.org)"
-            .parse()
-            .unwrap(),
-    );
     headers.insert(
         AUTHORIZATION,
         format!("{} {}", "token", token).parse().unwrap(),

From a612b79e6fd1b8fac243dbaecc099a7f43bfd53f Mon Sep 17 00:00:00 2001
From: Nelson J Morais <njmorais@gmail.com>
Date: Mon, 10 May 2021 23:54:40 +0100
Subject: [PATCH 8/8] update toolchain, and addressed clippy `manual_map` lint

---
 src/collectors/github_runners.rs | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/collectors/github_runners.rs b/src/collectors/github_runners.rs
index 7e30523..e1ee2f8 100644
--- a/src/collectors/github_runners.rs
+++ b/src/collectors/github_runners.rs
@@ -176,10 +176,9 @@ fn next_uri(header: Option<&HeaderValue>) -> Option<String> {
     if let Some(header) = header {
         return match header.to_str() {
             Ok(header_str) => match parse_link_header::parse(header_str) {
-                Ok(links) => match links.get(&Some("next".to_string())) {
-                    Some(link) => Some(link.uri.to_string()),
-                    None => None,
-                },
+                Ok(links) => links
+                    .get(&Some("next".to_string()))
+                    .map(|next| next.uri.to_string()),
                 _ => None,
             },
             _ => None,