From dbea071f161c12f998392631ae73305ed5d6b5c3 Mon Sep 17 00:00:00 2001 From: keyvan Date: Mon, 8 Apr 2024 15:37:14 -0700 Subject: [PATCH 01/27] feat: add holiday_hours module --- src/agent.rs | 1 + src/agent/holiday_hours.rs | 376 +++++++++++++++++++++++++++++++++++++ 2 files changed, 377 insertions(+) create mode 100644 src/agent/holiday_hours.rs diff --git a/src/agent.rs b/src/agent.rs index b23db867..341805c7 100644 --- a/src/agent.rs +++ b/src/agent.rs @@ -63,6 +63,7 @@ Note that there is an Oracle and Exporter for each network, but only one Local S ################################################################################################################################## */ pub mod dashboard; +pub mod holiday_hours; pub mod market_hours; pub mod metrics; pub mod pythd; diff --git a/src/agent/holiday_hours.rs b/src/agent/holiday_hours.rs new file mode 100644 index 00000000..40295354 --- /dev/null +++ b/src/agent/holiday_hours.rs @@ -0,0 +1,376 @@ +//! Holiday hours metadata parsing and evaluation logic + +use { + anyhow::{ + anyhow, + Context, + Result, + }, + chrono::{ + naive::NaiveTime, + DateTime, + Datelike, + Duration, + Utc, + }, + chrono_tz::Tz, + lazy_static::lazy_static, + std::str::FromStr, +}; + +lazy_static! { + /// Helper time value representing 24:00:00 as 00:00:00 minus 1 + /// nanosecond (underflowing to 23:59:59.999(...) ). While chrono + /// has this value internally exposed as NaiveTime::MAX, it is not + /// exposed outside the crate. + static ref MAX_TIME_INSTANT: NaiveTime = NaiveTime::MIN.overflowing_sub_signed(Duration::nanoseconds(1)).0; +} + +/// Holiday hours schedule +#[derive(Clone, Default, Debug, Eq, PartialEq)] +pub struct HolidaySchedule { + pub timezone: Option, + pub days: Vec, +} + +impl HolidaySchedule { + pub fn all_closed() -> Self { + Self { + timezone: Default::default(), + days: vec![], + } + } + + pub fn can_publish_at(&self, when: DateTime) -> bool { + // Convert to time local to the market + let when_local = when.with_timezone(&self.timezone.unwrap()); + + let market_month = when_local.date_naive().month0() + 1; + let market_day = when_local.date_naive().day0() + 1; + + let market_time = when_local.time(); + + for day in &self.days { + // Check if the day matches + if day.month == market_month && day.day == market_day { + return day.kind.can_publish_at(market_time); + } + } + true + } +} + +impl FromStr for HolidaySchedule { + type Err = anyhow::Error; + fn from_str(s: &str) -> Result { + let split_by_commas = s.split(","); + let mut days = Vec::new(); + + for day_str in split_by_commas { + let day = day_str.parse()?; + days.push(day); + } + + Ok(HolidaySchedule { + days, + timezone: None, + }) + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct HolidayDaySchedule { + pub month: u32, + pub day: u32, + pub kind: HolidayDayKind, +} + +impl FromStr for HolidayDaySchedule { + type Err = anyhow::Error; + fn from_str(s: &str) -> Result { + let date_time_parts: Vec<&str> = s.split("/").collect(); + if date_time_parts.len() != 2 { + return Err(anyhow!("Invalid format")); + } + + if date_time_parts[0].len() != 4 { + return Err(anyhow!("Invalid date format")); + } + + let month: u32 = date_time_parts[0][..2] + .parse() + .map_err(|_| anyhow!("Invalid month"))?; + let day: u32 = date_time_parts[0][2..] + .parse() + .map_err(|_| anyhow!("Invalid day"))?; + let kind: HolidayDayKind = date_time_parts[1].parse()?; + + Ok(HolidayDaySchedule { month, day, kind }) + } +} + +/// Helper enum for denoting per-day schedules: time range, all-day open and all-day closed. +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum HolidayDayKind { + Open, + Closed, + TimeRange(NaiveTime, NaiveTime), +} + +impl HolidayDayKind { + pub fn can_publish_at(&self, when_market_local: NaiveTime) -> bool { + match self { + Self::Open => true, + Self::Closed => false, + Self::TimeRange(start, end) => start <= &when_market_local && &when_market_local <= end, + } + } +} + +impl Default for HolidayDayKind { + fn default() -> Self { + Self::Open + } +} + +impl FromStr for HolidayDayKind { + type Err = anyhow::Error; + fn from_str(s: &str) -> Result { + match s { + "O" => Ok(HolidayDayKind::Open), + "C" => Ok(HolidayDayKind::Closed), + other => { + let (start_str, end_str) = other.split_once("-").ok_or(anyhow!( + "Missing '-' delimiter between start and end of range" + ))?; + + let start = NaiveTime::parse_from_str(start_str, "%H%M") + .context("start time does not match HHMM format")?; + + // The chrono crate is unable to parse 24:00 as + // previous day's perspective of midnight, so we use + // the next best thing - see MAX_TIME_INSTANT for + // details. + let end = if end_str.contains("2400") { + MAX_TIME_INSTANT.clone() + } else { + NaiveTime::parse_from_str(end_str, "%H%M") + .context("end time does not match HHMM format")? + }; + + if start < end { + Ok(HolidayDayKind::TimeRange(start, end)) + } else { + Err(anyhow!("Incorrect time range: start must come before end")) + } + } + } + } +} + +#[cfg(test)] +mod tests { + use { + super::*, + chrono::NaiveDateTime, + }; + + #[test] + fn test_parsing_single_day() -> Result<()> { + let s = "0422/0900-1700"; + + let parsed: HolidaySchedule = s.parse()?; + + let expected = HolidaySchedule { + timezone: None, + days: vec![HolidayDaySchedule { + month: 4, + day: 22, + kind: HolidayDayKind::TimeRange( + NaiveTime::from_hms_opt(9, 0, 0).unwrap(), + NaiveTime::from_hms_opt(17, 0, 0).unwrap(), + ), + }], + }; + + assert_eq!(parsed, expected); + + Ok(()) + } + + #[test] + fn test_parsing_multiple_days() -> Result<()> { + let s = "0422/0900-1700,1109/0930-1730,1201/O,1225/C,1231/0900-1700"; + + let parsed: HolidaySchedule = s.parse()?; + + let expected = HolidaySchedule { + timezone: None, + days: vec![ + HolidayDaySchedule { + month: 4, + day: 22, + kind: HolidayDayKind::TimeRange( + NaiveTime::from_hms_opt(9, 0, 0).unwrap(), + NaiveTime::from_hms_opt(17, 0, 0).unwrap(), + ), + }, + HolidayDaySchedule { + month: 11, + day: 9, + kind: HolidayDayKind::TimeRange( + NaiveTime::from_hms_opt(9, 30, 0).unwrap(), + NaiveTime::from_hms_opt(17, 30, 0).unwrap(), + ), + }, + HolidayDaySchedule { + month: 12, + day: 1, + kind: HolidayDayKind::Open, + }, + HolidayDaySchedule { + month: 12, + day: 25, + kind: HolidayDayKind::Closed, + }, + HolidayDaySchedule { + month: 12, + day: 31, + kind: HolidayDayKind::TimeRange( + NaiveTime::from_hms_opt(9, 0, 0).unwrap(), + NaiveTime::from_hms_opt(17, 0, 0).unwrap(), + ), + }, + ], + }; + + assert_eq!(parsed, expected); + + Ok(()) + } + + #[test] + fn test_parsing_month_without_leading_zero_is_error() { + let s = "422/0900-1700"; + + let parsing_result: Result = s.parse(); + assert!(parsing_result.is_err()); + } + + #[test] + fn test_parsing_hour_without_leading_zero_is_error() { + let s = "0422/900-1700"; + + let parsing_result: Result = s.parse(); + assert!(parsing_result.is_err()); + } + + #[test] + fn test_parsing_wrong_delimiter_is_error() { + let s = "0422-0900/1700"; + + let parsing_result: Result = s.parse(); + assert!(parsing_result.is_err()); + } + + #[test] + fn test_parsing_wrong_format_is_error() { + let s = "0422/09:00-17:00"; + + let parsing_result: Result = s.parse(); + assert!(parsing_result.is_err()); + } + + #[test] + fn test_parsing_wrong_time_range_is_error() { + let s = "0422/1700-0900"; + + let parsing_result: Result = s.parse(); + assert!(parsing_result.is_err()); + } + + #[test] + fn test_parse_24_hour() { + let s = "0422/0900-2400"; + + let parsed: HolidaySchedule = s.parse().unwrap(); + let expected = HolidaySchedule { + timezone: None, + days: vec![HolidayDaySchedule { + month: 4, + day: 22, + kind: HolidayDayKind::TimeRange( + NaiveTime::from_hms_opt(9, 0, 0).unwrap(), + MAX_TIME_INSTANT.clone(), + ), + }], + }; + assert_eq!(parsed, expected); + } + + #[test] + fn test_holiday_schedule_can_publish_at() -> Result<()> { + // Prepare a schedule of narrow ranges + let mut holiday_hours: HolidaySchedule = + "0422/0900-1700,1109/0930-1730,1201/O,1225/C,1231/0900-1700" + .parse() + .unwrap(); + + holiday_hours.timezone = Some(Tz::UTC); + let format = "%Y-%m-%d %H:%M"; + + // Date no match + assert!(holiday_hours + .can_publish_at(NaiveDateTime::parse_from_str("2023-11-20 05:30", format)?.and_utc())); + + // Date match before range + assert!(!holiday_hours + .can_publish_at(NaiveDateTime::parse_from_str("2023-04-22 08:59", format)?.and_utc())); + + // Date match at start of range + assert!(holiday_hours + .can_publish_at(NaiveDateTime::parse_from_str("2023-04-22 09:00", format)?.and_utc())); + + // Date match in range + assert!(holiday_hours + .can_publish_at(NaiveDateTime::parse_from_str("2023-04-22 12:00", format)?.and_utc())); + + // Date match at end of range + assert!(holiday_hours + .can_publish_at(NaiveDateTime::parse_from_str("2023-04-22 17:00", format)?.and_utc())); + + // Date match after range + assert!(!holiday_hours + .can_publish_at(NaiveDateTime::parse_from_str("2023-04-22 17:01", format)?.and_utc())); + + Ok(()) + } + + /// Verify desired 24:00 behavior. + #[test] + fn test_market_hours_midnight_00_24() -> Result<()> { + // Prepare a schedule of midnight-neighboring ranges + let mut holiday_schedule: HolidaySchedule = "0422/0900-2400".parse()?; + + holiday_schedule.timezone = Some(Tz::UTC); + + let format = "%Y-%m-%d %H:%M"; + // Date match before range + assert!(!holiday_schedule + .can_publish_at(NaiveDateTime::parse_from_str("2023-04-22 08:59", format)?.and_utc())); + + // Date match at start of range + assert!(holiday_schedule + .can_publish_at(NaiveDateTime::parse_from_str("2023-04-22 09:00", format)?.and_utc())); + + // Date match in range + assert!(holiday_schedule + .can_publish_at(NaiveDateTime::parse_from_str("2023-04-22 12:00", format)?.and_utc())); + + // Date match at end of range + assert!(holiday_schedule + .can_publish_at(NaiveDateTime::parse_from_str("2023-04-22 23:59", format)?.and_utc())); + + Ok(()) + } +} From 574a44440d625db0190379123f7d12f1bb950ddd Mon Sep 17 00:00:00 2001 From: keyvan Date: Tue, 9 Apr 2024 15:40:21 -0700 Subject: [PATCH 02/27] feat: integrate holiday schedule --- integration-tests/tests/test_integration.py | 53 ++++++++++++++++++- src/agent.rs | 3 +- src/agent/pythd/adapter.rs | 12 ++--- src/agent/schedule.rs | 14 ++++++ src/agent/{ => schedule}/holiday_hours.rs | 0 src/agent/{ => schedule}/market_hours.rs | 0 src/agent/solana/exporter.rs | 20 ++++---- src/agent/solana/oracle.rs | 56 +++++++++++++++------ 8 files changed, 124 insertions(+), 34 deletions(-) create mode 100644 src/agent/schedule.rs rename src/agent/{ => schedule}/holiday_hours.rs (100%) rename src/agent/{ => schedule}/market_hours.rs (100%) diff --git a/integration-tests/tests/test_integration.py b/integration-tests/tests/test_integration.py index 0b8f81dd..abb91eaa 100644 --- a/integration-tests/tests/test_integration.py +++ b/integration-tests/tests/test_integration.py @@ -1,4 +1,5 @@ import asyncio +from datetime import datetime import json import os import requests @@ -63,9 +64,23 @@ "quote_currency": "USD", "generic_symbol": "BTCUSD", "description": "BTC/USD", + "holidays": f"{datetime.now().strftime('%m%d')}/O" }, "metadata": {"jump_id": "78876709", "jump_symbol": "BTCUSD", "price_exp": -8, "min_publishers": 1}, } +SOL_USD = { + "account": "", + "attr_dict": { + "symbol": "Crypto.SOL/USD", + "asset_type": "Crypto", + "base": "SOL", + "quote_currency": "USD", + "generic_symbol": "SOLUSD", + "description": "SOL/USD", + "holidays": f"{datetime.now().strftime('%m%d')}/C" + }, + "metadata": {"jump_id": "78876711", "jump_symbol": "SOLUSD", "price_exp": -8, "min_publishers": 1}, +} AAPL_USD = { "account": "", "attr_dict": { @@ -95,7 +110,7 @@ }, "metadata": {"jump_id": "78876710", "jump_symbol": "ETHUSD", "price_exp": -8, "min_publishers": 1}, } -ALL_PRODUCTS=[BTC_USD, AAPL_USD, ETH_USD] +ALL_PRODUCTS=[BTC_USD, AAPL_USD, ETH_USD, SOL_USD] asyncio.set_event_loop(asyncio.new_event_loop()) @@ -277,6 +292,7 @@ def refdata_permissions(self, refdata_path): "AAPL": {"price": ["some_publisher_a"]}, "BTCUSD": {"price": ["some_publisher_b", "some_publisher_a"]}, # Reversed order helps ensure permission discovery works correctly for publisher A "ETHUSD": {"price": ["some_publisher_b"]}, + "SOLUSD": {"price": ["some_publisher_a"]}, })) f.flush() yield f.name @@ -769,3 +785,38 @@ async def test_agent_respects_market_hours(self, client: PythAgentClient): assert final_price_account["price"] == 0 assert final_price_account["conf"] == 0 assert final_price_account["status"] == "unknown" + + @pytest.mark.asyncio + async def test_agent_respects_holiday_hours(self, client: PythAgentClient): + ''' + Similar to test_agent_respects_market_hours, but using SOL_USD and + asserting that nothing is published due to the symbol's all-closed holiday. + ''' + + # Fetch all products + products = {product["attr_dict"]["symbol"]: product for product in await client.get_all_products()} + + # Find the product account ID corresponding to the AAPL/USD symbol + product = products[SOL_USD["attr_dict"]["symbol"]] + product_account = product["account"] + + # Get the price account with which to send updates + price_account = product["price_accounts"][0]["account"] + + # Send an "update_price" request + await client.update_price(price_account, 42, 2, "trading") + time.sleep(2) + + # Send another update_price request to "trigger" aggregation + # (aggregation would happen if market hours were to fail, but + # we want to catch that happening if there's a problem) + await client.update_price(price_account, 81, 1, "trading") + time.sleep(2) + + # Confirm that the price account has not been updated + final_product_state = await client.get_product(product_account) + + final_price_account = final_product_state["price_accounts"][0] + assert final_price_account["price"] == 0 + assert final_price_account["conf"] == 0 + assert final_price_account["status"] == "unknown" diff --git a/src/agent.rs b/src/agent.rs index 341805c7..ce21640a 100644 --- a/src/agent.rs +++ b/src/agent.rs @@ -63,11 +63,10 @@ Note that there is an Oracle and Exporter for each network, but only one Local S ################################################################################################################################## */ pub mod dashboard; -pub mod holiday_hours; -pub mod market_hours; pub mod metrics; pub mod pythd; pub mod remote_keypair_loader; +pub mod schedule; pub mod solana; pub mod store; use { diff --git a/src/agent/pythd/adapter.rs b/src/agent/pythd/adapter.rs index ebea11dd..a4b34283 100644 --- a/src/agent/pythd/adapter.rs +++ b/src/agent/pythd/adapter.rs @@ -960,7 +960,7 @@ mod tests { ) .unwrap(), solana::oracle::ProductEntry { - account_data: pyth_sdk_solana::state::ProductAccount { + account_data: pyth_sdk_solana::state::ProductAccount { magic: 0xa1b2c3d4, ver: 6, atype: 4, @@ -997,8 +997,8 @@ mod tests { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ], }, - weekly_schedule: Default::default(), - price_accounts: vec![ + schedule: Default::default(), + price_accounts: vec![ solana_sdk::pubkey::Pubkey::from_str( "GVXRSBjFk6e6J3NbVPXohDJetcTjaeeuykUpbQF8UoMU", ) @@ -1020,7 +1020,7 @@ mod tests { ) .unwrap(), solana::oracle::ProductEntry { - account_data: pyth_sdk_solana::state::ProductAccount { + account_data: pyth_sdk_solana::state::ProductAccount { magic: 0xa1b2c3d4, ver: 5, atype: 3, @@ -1057,8 +1057,8 @@ mod tests { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ], }, - weekly_schedule: Default::default(), - price_accounts: vec![ + schedule: Default::default(), + price_accounts: vec![ solana_sdk::pubkey::Pubkey::from_str( "GG3FTE7xhc9Diy7dn9P6BWzoCrAEE4D3p5NBYrDAm5DD", ) diff --git a/src/agent/schedule.rs b/src/agent/schedule.rs new file mode 100644 index 00000000..d78d065e --- /dev/null +++ b/src/agent/schedule.rs @@ -0,0 +1,14 @@ +pub mod holiday_hours; +pub mod market_hours; + + +use crate::agent::schedule::{ + holiday_hours::HolidaySchedule, + market_hours::WeeklySchedule, +}; + +#[derive(Debug, Clone, Default)] +pub struct Schedule { + pub market_hours: WeeklySchedule, + pub holiday_hours: HolidaySchedule, +} diff --git a/src/agent/holiday_hours.rs b/src/agent/schedule/holiday_hours.rs similarity index 100% rename from src/agent/holiday_hours.rs rename to src/agent/schedule/holiday_hours.rs diff --git a/src/agent/market_hours.rs b/src/agent/schedule/market_hours.rs similarity index 100% rename from src/agent/market_hours.rs rename to src/agent/schedule/market_hours.rs diff --git a/src/agent/solana/exporter.rs b/src/agent/solana/exporter.rs index 3231a7b6..4ff98151 100644 --- a/src/agent/solana/exporter.rs +++ b/src/agent/solana/exporter.rs @@ -10,11 +10,11 @@ use { network::Network, }, crate::agent::{ - market_hours::WeeklySchedule, remote_keypair_loader::{ KeypairRequest, RemoteKeypairLoader, }, + schedule::Schedule, }, anyhow::{ anyhow, @@ -66,8 +66,8 @@ use { }, tokio::{ sync::{ - mpsc, mpsc::{ + self, error::TryRecvError, Sender, }, @@ -172,7 +172,7 @@ pub fn spawn_exporter( network: Network, rpc_url: &str, rpc_timeout: Duration, - publisher_permissions_rx: mpsc::Receiver>>, + publisher_permissions_rx: mpsc::Receiver>>, key_store: KeyStore, local_store_tx: Sender, global_store_tx: Sender, @@ -260,10 +260,10 @@ pub struct Exporter { inflight_transactions_tx: Sender, /// publisher => { permissioned_price => market hours } as read by the oracle module - publisher_permissions_rx: mpsc::Receiver>>, + publisher_permissions_rx: mpsc::Receiver>>, /// Currently known permissioned prices of this publisher along with their market hours - our_prices: HashMap, + our_prices: HashMap, /// Interval to update the dynamic price (if enabled) dynamic_compute_unit_price_update_interval: Interval, @@ -287,7 +287,7 @@ impl Exporter { global_store_tx: Sender, network_state_rx: watch::Receiver, inflight_transactions_tx: Sender, - publisher_permissions_rx: mpsc::Receiver>>, + publisher_permissions_rx: mpsc::Receiver>>, keypair_request_tx: mpsc::Sender, logger: Logger, ) -> Self { @@ -474,13 +474,15 @@ impl Exporter { .into_iter() .filter(|(id, _data)| { let key_from_id = Pubkey::from((*id).clone().to_bytes()); - if let Some(weekly_schedule) = self.our_prices.get(&key_from_id) { - let ret = weekly_schedule.can_publish_at(&now); + if let Some(schedule) = self.our_prices.get_mut(&key_from_id) { + // let ret = schedule.market_hours.can_publish_at(&now); + schedule.holiday_hours.timezone = Some(schedule.market_hours.timezone); + let ret = schedule.market_hours.can_publish_at(&now) && schedule.holiday_hours.can_publish_at(now); if !ret { debug!(self.logger, "Exporter: Attempted to publish price outside market hours"; "price_account" => key_from_id.to_string(), - "weekly_schedule" => format!("{:?}", weekly_schedule), + "schedule" => format!("{:?}", schedule), "utc_time" => now.format("%c").to_string(), ); } diff --git a/src/agent/solana/oracle.rs b/src/agent/solana/oracle.rs index 20c8d437..8e377d5a 100644 --- a/src/agent/solana/oracle.rs +++ b/src/agent/solana/oracle.rs @@ -4,7 +4,11 @@ use { self::subscriber::Subscriber, super::key_store::KeyStore, crate::agent::{ - market_hours::WeeklySchedule, + schedule::{ + holiday_hours::HolidaySchedule, + market_hours::WeeklySchedule, + Schedule, + }, store::global, }, anyhow::{ @@ -117,7 +121,7 @@ pub struct Data { pub product_accounts: HashMap, pub price_accounts: HashMap, /// publisher => {their permissioned price accounts => market hours} - pub publisher_permissions: HashMap>, + pub publisher_permissions: HashMap>, } impl Data { @@ -125,7 +129,7 @@ impl Data { mapping_accounts: HashMap, product_accounts: HashMap, price_accounts: HashMap, - publisher_permissions: HashMap>, + publisher_permissions: HashMap>, ) -> Self { Data { mapping_accounts, @@ -139,9 +143,9 @@ impl Data { pub type MappingAccount = pyth_sdk_solana::state::MappingAccount; #[derive(Debug, Clone)] pub struct ProductEntry { - pub account_data: pyth_sdk_solana::state::ProductAccount, - pub weekly_schedule: WeeklySchedule, - pub price_accounts: Vec, + pub account_data: pyth_sdk_solana::state::ProductAccount, + pub schedule: Schedule, + pub price_accounts: Vec, } // Oracle is responsible for fetching Solana account data stored in the Pyth on-chain Oracle. @@ -203,7 +207,7 @@ pub fn spawn_oracle( wss_url: &str, rpc_timeout: Duration, global_store_update_tx: mpsc::Sender, - publisher_permissions_tx: mpsc::Sender>>, + publisher_permissions_tx: mpsc::Sender>>, key_store: KeyStore, logger: Logger, ) -> Vec> { @@ -418,7 +422,7 @@ struct Poller { data_tx: mpsc::Sender, /// Updates about permissioned price accounts from oracle to exporter - publisher_permissions_tx: mpsc::Sender>>, + publisher_permissions_tx: mpsc::Sender>>, /// The RPC client to use to poll data from the RPC node rpc_client: RpcClient, @@ -438,7 +442,7 @@ struct Poller { impl Poller { pub fn new( data_tx: mpsc::Sender, - publisher_permissions_tx: mpsc::Sender>>, + publisher_permissions_tx: mpsc::Sender>>, rpc_url: &str, rpc_timeout: Duration, commitment: CommitmentLevel, @@ -510,10 +514,8 @@ impl Poller { .entry(component.publisher) .or_insert(HashMap::new()); - let weekly_schedule = if let Some(prod_entry) = - product_accounts.get(&price_entry.prod) - { - prod_entry.weekly_schedule.clone() + let schedule = if let Some(prod_entry) = product_accounts.get(&price_entry.prod) { + prod_entry.schedule.clone() } else { warn!(&self.logger, "Oracle: INTERNAL: could not find product from price `prod` field, market hours falling back to 24/7."; "price" => price_key.to_string(), @@ -522,7 +524,7 @@ impl Poller { Default::default() }; - component_pub_entry.insert(*price_key, weekly_schedule); + component_pub_entry.insert(*price_key, schedule); } } @@ -627,11 +629,33 @@ impl Poller { Default::default() // No market hours specified, meaning 24/7 publishing }; + let holiday_schedule: HolidaySchedule = if let Some((_hsched_key, hsched_val)) = + product.iter().find(|(k, _v)| *k == "holidays") + { + hsched_val.parse().unwrap_or_else(|err| { + warn!( + self.logger, + "Oracle: Product has weekly_schedule defined but it could not be parsed. Falling back to 24/7 publishing."; + "product_key" => product_key.to_string(), + "holiday_schedule" => hsched_val, + ); + debug!(self.logger, "parsing error context"; "context" => format!("{:?}", err)); + Default::default() + }) + } else { + Default::default() // No market hours specified, meaning 24/7 publishing + }; + product_entries.insert( *product_key, ProductEntry { - account_data: *product, - weekly_schedule, + account_data: *product, + schedule: { + Schedule { + market_hours: weekly_schedule, + holiday_hours: holiday_schedule, + } + }, price_accounts: vec![], }, ); From 2982eae8ebfaac393d70f11a868333ca5954f751 Mon Sep 17 00:00:00 2001 From: keyvan Date: Tue, 9 Apr 2024 15:41:57 -0700 Subject: [PATCH 03/27] feat: update gitignore to ignore .DS_Store --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 41b55c9f..3920c3a7 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,6 @@ result **/*.rs.bk __pycache__ keystore + +# Mac OS +.DS_Store \ No newline at end of file From 3c9ee2ef7eef1084b67bf13963514e722808f331 Mon Sep 17 00:00:00 2001 From: keyvan Date: Wed, 10 Apr 2024 09:28:27 -0700 Subject: [PATCH 04/27] refactor: imports --- src/agent/schedule.rs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/agent/schedule.rs b/src/agent/schedule.rs index d78d065e..500dc443 100644 --- a/src/agent/schedule.rs +++ b/src/agent/schedule.rs @@ -1,12 +1,9 @@ +use holiday_hours::HolidaySchedule; +use market_hours::WeeklySchedule; + pub mod holiday_hours; pub mod market_hours; - -use crate::agent::schedule::{ - holiday_hours::HolidaySchedule, - market_hours::WeeklySchedule, -}; - #[derive(Debug, Clone, Default)] pub struct Schedule { pub market_hours: WeeklySchedule, From 5c57bec700f244cc05d43c635b04732423841bb7 Mon Sep 17 00:00:00 2001 From: keyvan Date: Wed, 10 Apr 2024 12:26:41 -0700 Subject: [PATCH 05/27] fix: pre-commit --- .gitignore | 2 +- src/agent/schedule.rs | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 3920c3a7..48d41e13 100644 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,4 @@ __pycache__ keystore # Mac OS -.DS_Store \ No newline at end of file +.DS_Store diff --git a/src/agent/schedule.rs b/src/agent/schedule.rs index 500dc443..07579f4a 100644 --- a/src/agent/schedule.rs +++ b/src/agent/schedule.rs @@ -1,5 +1,7 @@ -use holiday_hours::HolidaySchedule; -use market_hours::WeeklySchedule; +use { + holiday_hours::HolidaySchedule, + market_hours::WeeklySchedule, +}; pub mod holiday_hours; pub mod market_hours; From fea4d63fd8984a82cab1df597d1dc831ac9aaa18 Mon Sep 17 00:00:00 2001 From: keyvan Date: Wed, 10 Apr 2024 15:46:49 -0700 Subject: [PATCH 06/27] feat: wip market schedule using winnow parser --- Cargo.lock | 1 + Cargo.toml | 1 + src/agent.rs | 1 + src/agent/market_schedule.rs | 125 +++++++++++++++++++++++++++++++++++ 4 files changed, 128 insertions(+) create mode 100644 src/agent/market_schedule.rs diff --git a/Cargo.lock b/Cargo.lock index f0224dde..e69c2579 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3285,6 +3285,7 @@ dependencies = [ "toml_edit 0.22.9", "typed-html", "warp", + "winnow 0.6.5", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 9bd9deb0..702974b1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -52,6 +52,7 @@ prometheus-client = "0.22.2" lazy_static = "1.4.0" toml_edit = "0.22.9" slog-bunyan = "2.5.0" +winnow = "0.6.5" [dev-dependencies] tokio-util = { version = "0.7.10", features = ["full"] } diff --git a/src/agent.rs b/src/agent.rs index ce21640a..195ea6aa 100644 --- a/src/agent.rs +++ b/src/agent.rs @@ -63,6 +63,7 @@ Note that there is an Oracle and Exporter for each network, but only one Local S ################################################################################################################################## */ pub mod dashboard; +pub mod market_schedule; pub mod metrics; pub mod pythd; pub mod remote_keypair_loader; diff --git a/src/agent/market_schedule.rs b/src/agent/market_schedule.rs new file mode 100644 index 00000000..3bd60112 --- /dev/null +++ b/src/agent/market_schedule.rs @@ -0,0 +1,125 @@ +//! Holiday hours metadata parsing and evaluation logic + +use { + anyhow::{ + anyhow, + Context, + Result, + }, + chrono::{ + naive::NaiveTime, + DateTime, + Datelike, + Duration, + Utc, + }, + chrono_tz::Tz, + lazy_static::lazy_static, + std::str::FromStr, + winnow::{ + ascii::digit1, + combinator::{ + alt, + repeat, + separated_pair, + }, + error::ContextError, + token::take, + PResult, + Parser, + }, +}; + + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum ScheduleDayKind { + Open, + Closed, + TimeRange(NaiveTime, NaiveTime), +} + +impl ScheduleDayKind { + pub fn can_publish_at(&self, when_market_local: NaiveTime) -> bool { + match self { + Self::Open => true, + Self::Closed => false, + Self::TimeRange(start, end) => start <= &when_market_local && &when_market_local <= end, + } + } +} + +impl Default for ScheduleDayKind { + fn default() -> Self { + Self::Open + } +} + +fn time_range_parser<'s>(input: &mut &'s str) -> PResult { + let (start_str, end_str) = separated_pair(take(2usize), "-", take(2usize)).parse_next(input)?; + + let start_time = NaiveTime::parse_from_str(start_str, "%H%M").unwrap(); + let end_time = NaiveTime::parse_from_str(end_str, "%H%M").unwrap(); + + Ok(ScheduleDayKind::TimeRange(start_time, end_time)) +} + +fn schedule_day_kind_parser<'s>(input: &mut &'s str) -> PResult { + alt(( + "C".map(|_| ScheduleDayKind::Closed), + "O".map(|_| ScheduleDayKind::Open), + time_range_parser, + )) + .parse_next(input) +} + +impl FromStr for ScheduleDayKind { + type Err = anyhow::Error; + fn from_str(s: &str) -> Result { + schedule_day_kind_parser + .parse_next(&mut s.to_owned().as_str()) + .map_err(|e| anyhow!(e)) + } +} + + +#[cfg(test)] +mod tests { + use { + super::*, + crate::agent::schedule::holiday_hours::HolidayDayKind, + chrono::{ + NaiveDate, + NaiveDateTime, + }, + }; + + #[test] + fn test_parsing_schedule_day_kind() -> Result<()> { + // Mon-Fri 9-5, inconsistent leading space on Tuesday, leading 0 on Friday (expected to be fine) + let open = "O"; + let closed = "C"; + let valid = "1234-1347"; + let invalid = "1234-5668"; + let invalid_format = "1234-56"; + + assert_eq!( + open.parse::().unwrap(), + HolidayDayKind::Open + ); + assert_eq!( + closed.parse::().unwrap(), + HolidayDayKind::Closed + ); + assert_eq!( + valid.parse::().unwrap(), + HolidayDayKind::TimeRange( + NaiveTime::from_hms(12, 34, 0), + NaiveTime::from_hms(13, 47, 0) + ) + ); + assert!(invalid.parse::().is_err()); + assert!(invalid_format.parse::().is_err()); + + Ok(()) + } +} From 63ef212c7a1d731b75cf8cb7b09fb855195c329e Mon Sep 17 00:00:00 2001 From: keyvan Date: Wed, 10 Apr 2024 17:18:53 -0700 Subject: [PATCH 07/27] feat: wip add holiday day schedule parser --- src/agent/market_schedule.rs | 169 ++++++++++++++++++++++++++++++----- 1 file changed, 145 insertions(+), 24 deletions(-) diff --git a/src/agent/market_schedule.rs b/src/agent/market_schedule.rs index 3bd60112..39fd5ece 100644 --- a/src/agent/market_schedule.rs +++ b/src/agent/market_schedule.rs @@ -3,34 +3,116 @@ use { anyhow::{ anyhow, - Context, Result, }, chrono::{ naive::NaiveTime, DateTime, Datelike, - Duration, Utc, }, chrono_tz::Tz, - lazy_static::lazy_static, std::str::FromStr, winnow::{ - ascii::digit1, combinator::{ alt, repeat, separated_pair, }, - error::ContextError, - token::take, + error::{ + ErrMode, + ErrorKind, + ParserError, + }, + stream::ToUsize, + token::{ + take, + take_till, + }, PResult, Parser, }, }; +#[derive(Clone, Debug)] +pub struct MarketSchedule { + pub timezone: Tz, + pub weekly_schedule: Vec, + pub holidays: Vec, +} + +impl MarketSchedule { + pub fn can_publish_at(&self, when: &DateTime) -> bool { + let when_local = when.with_timezone(&self.timezone); + + let month = when_local.date_naive().month0() + 1; + let day = when_local.date_naive().day0() + 1; + let time = when_local.time(); + let weekday = when_local.weekday().number_from_monday().to_usize(); + + for holiday in &self.holidays { + // Check if the day matches + if holiday.month == month && holiday.day == day { + return holiday.kind.can_publish_at(time); + } + } + + self.weekly_schedule[weekday].can_publish_at(time) + } +} + +fn market_schedule_parser<'s>(input: &mut &'s str) -> PResult { + let timezone: Tz = take_till(0.., ';').parse_next(input)?.parse().unwrap(); + let weekly_schedule = repeat(7, schedule_day_kind_parser).parse_next(input)?; + let holidays = repeat(0.., holiday_day_schedule_parser).parse_next(input)?; + + Ok(MarketSchedule { + timezone, + weekly_schedule, + holidays, + }) +} + +impl FromStr for MarketSchedule { + type Err = anyhow::Error; + fn from_str(s: &str) -> Result { + market_schedule_parser + .parse_next(&mut s.to_owned().as_str()) + .map_err(|e| anyhow!(e)) + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct HolidayDaySchedule { + pub month: u32, + pub day: u32, + pub kind: ScheduleDayKind, +} + + +fn holiday_day_schedule_parser<'s>(input: &mut &'s str) -> PResult { + let ((month_str, day_str), kind) = + separated_pair((take(2usize), take(2usize)), "/", schedule_day_kind_parser) + .parse_next(input)?; + + dbg!(month_str, day_str, kind.clone()); + + let month = month_str.parse::().unwrap(); + let day = day_str.parse::().unwrap(); + + Ok(HolidayDaySchedule { month, day, kind }) +} + +impl FromStr for HolidayDaySchedule { + type Err = anyhow::Error; + fn from_str(s: &str) -> Result { + holiday_day_schedule_parser + .parse_next(&mut s.to_owned().as_str()) + .map_err(|e| anyhow!(e)) + } +} + #[derive(Clone, Debug, Eq, PartialEq)] pub enum ScheduleDayKind { Open, @@ -39,11 +121,11 @@ pub enum ScheduleDayKind { } impl ScheduleDayKind { - pub fn can_publish_at(&self, when_market_local: NaiveTime) -> bool { + pub fn can_publish_at(&self, when_local: NaiveTime) -> bool { match self { Self::Open => true, Self::Closed => false, - Self::TimeRange(start, end) => start <= &when_market_local && &when_market_local <= end, + Self::TimeRange(start, end) => start <= &when_local && &when_local <= end, } } } @@ -55,10 +137,12 @@ impl Default for ScheduleDayKind { } fn time_range_parser<'s>(input: &mut &'s str) -> PResult { - let (start_str, end_str) = separated_pair(take(2usize), "-", take(2usize)).parse_next(input)?; + let (start_str, end_str) = separated_pair(take(4usize), "-", take(4usize)).parse_next(input)?; - let start_time = NaiveTime::parse_from_str(start_str, "%H%M").unwrap(); - let end_time = NaiveTime::parse_from_str(end_str, "%H%M").unwrap(); + let start_time = NaiveTime::parse_from_str(start_str, "%H%M") + .map_err(|e| ErrMode::from_error_kind(input, ErrorKind::Verify))?; + let end_time = NaiveTime::parse_from_str(end_str, "%H%M") + .map_err(|e| ErrMode::from_error_kind(input, ErrorKind::Verify))?; Ok(ScheduleDayKind::TimeRange(start_time, end_time)) } @@ -86,11 +170,6 @@ impl FromStr for ScheduleDayKind { mod tests { use { super::*, - crate::agent::schedule::holiday_hours::HolidayDayKind, - chrono::{ - NaiveDate, - NaiveDateTime, - }, }; #[test] @@ -103,22 +182,64 @@ mod tests { let invalid_format = "1234-56"; assert_eq!( - open.parse::().unwrap(), - HolidayDayKind::Open + open.parse::().unwrap(), + ScheduleDayKind::Open ); assert_eq!( - closed.parse::().unwrap(), - HolidayDayKind::Closed + closed.parse::().unwrap(), + ScheduleDayKind::Closed ); assert_eq!( - valid.parse::().unwrap(), - HolidayDayKind::TimeRange( + valid.parse::().unwrap(), + ScheduleDayKind::TimeRange( NaiveTime::from_hms(12, 34, 0), NaiveTime::from_hms(13, 47, 0) ) ); - assert!(invalid.parse::().is_err()); - assert!(invalid_format.parse::().is_err()); + assert!(invalid.parse::().is_err()); + assert!(invalid_format.parse::().is_err()); + + Ok(()) + } + + #[test] + fn test_parsing_holiday_day_schedule() -> Result<()> { + let input = "0412/O"; + let expected = HolidayDaySchedule { + month: 04, + day: 12, + kind: ScheduleDayKind::Open, + }; + + let parsed = input.parse::()?; + assert_eq!(parsed, expected); + + let input = "0412/C"; + let expected = HolidayDaySchedule { + month: 04, + day: 12, + kind: ScheduleDayKind::Closed, + }; + let parsed = input.parse::()?; + assert_eq!(parsed, expected); + + let input = "0412/1234-1347"; + let expected = HolidayDaySchedule { + month: 04, + day: 12, + kind: ScheduleDayKind::TimeRange( + NaiveTime::from_hms_opt(12, 34, 0).unwrap(), + NaiveTime::from_hms_opt(13, 47, 0).unwrap(), + ), + }; + let parsed = input.parse::()?; + assert_eq!(parsed, expected); + + let input = "0412/1234-5332"; + assert!(input.parse::().is_err()); + + let input = "0412/1234-53"; + assert!(input.parse::().is_err()); Ok(()) } From 82f5eba1f27832dc289ea6db8e1694b89d408b9b Mon Sep 17 00:00:00 2001 From: keyvan Date: Wed, 10 Apr 2024 17:20:01 -0700 Subject: [PATCH 08/27] fix: format --- src/agent/market_schedule.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/agent/market_schedule.rs b/src/agent/market_schedule.rs index 39fd5ece..982e06a4 100644 --- a/src/agent/market_schedule.rs +++ b/src/agent/market_schedule.rs @@ -168,9 +168,7 @@ impl FromStr for ScheduleDayKind { #[cfg(test)] mod tests { - use { - super::*, - }; + use super::*; #[test] fn test_parsing_schedule_day_kind() -> Result<()> { @@ -192,8 +190,8 @@ mod tests { assert_eq!( valid.parse::().unwrap(), ScheduleDayKind::TimeRange( - NaiveTime::from_hms(12, 34, 0), - NaiveTime::from_hms(13, 47, 0) + NaiveTime::from_hms_opt(12, 34, 0).unwrap(), + NaiveTime::from_hms_opt(13, 47, 0).unwrap(), ) ); assert!(invalid.parse::().is_err()); From 02b1d82f08e7d25af2f860875643946a263502e0 Mon Sep 17 00:00:00 2001 From: keyvan Date: Wed, 10 Apr 2024 18:00:10 -0700 Subject: [PATCH 09/27] feat: wip implement market_schedule parser --- src/agent/market_schedule.rs | 76 ++++++++++++++++++++++++++++++------ 1 file changed, 65 insertions(+), 11 deletions(-) diff --git a/src/agent/market_schedule.rs b/src/agent/market_schedule.rs index 982e06a4..2233385a 100644 --- a/src/agent/market_schedule.rs +++ b/src/agent/market_schedule.rs @@ -16,8 +16,10 @@ use { winnow::{ combinator::{ alt, - repeat, + separated, separated_pair, + seq, + terminated, }, error::{ ErrMode, @@ -35,7 +37,7 @@ use { }; -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Eq, PartialEq)] pub struct MarketSchedule { pub timezone: Tz, pub weekly_schedule: Vec, @@ -63,15 +65,16 @@ impl MarketSchedule { } fn market_schedule_parser<'s>(input: &mut &'s str) -> PResult { - let timezone: Tz = take_till(0.., ';').parse_next(input)?.parse().unwrap(); - let weekly_schedule = repeat(7, schedule_day_kind_parser).parse_next(input)?; - let holidays = repeat(0.., holiday_day_schedule_parser).parse_next(input)?; - - Ok(MarketSchedule { - timezone, - weekly_schedule, - holidays, - }) + seq!( + MarketSchedule { + timezone: take_till(0.., ';').verify(|s| Tz::from_str(s).is_ok()).map(|s| Tz::from_str(s).unwrap()), + _: ';', + weekly_schedule: separated(7, schedule_day_kind_parser, ","), + _: ';', + holidays: separated(0.., holiday_day_schedule_parser, ","), + } + ) + .parse_next(input) } impl FromStr for MarketSchedule { @@ -241,4 +244,55 @@ mod tests { Ok(()) } + + #[test] + fn test_parsing_market_schedule() -> Result<()> { + let input = "America/New_York;O,1234-1347,C,C,C,C,O;0412/O,0413/C,0414/1234-1347"; + let expected = MarketSchedule { + timezone: Tz::America__New_York, + weekly_schedule: vec![ + ScheduleDayKind::Open, + ScheduleDayKind::TimeRange( + NaiveTime::from_hms_opt(12, 34, 0).unwrap(), + NaiveTime::from_hms_opt(13, 47, 0).unwrap(), + ), + ScheduleDayKind::Closed, + ScheduleDayKind::Closed, + ScheduleDayKind::Closed, + ScheduleDayKind::Closed, + ScheduleDayKind::Open, + ], + holidays: vec![ + HolidayDaySchedule { + month: 04, + day: 12, + kind: ScheduleDayKind::Open, + }, + HolidayDaySchedule { + month: 04, + day: 13, + kind: ScheduleDayKind::Closed, + }, + HolidayDaySchedule { + month: 04, + day: 14, + kind: ScheduleDayKind::TimeRange( + NaiveTime::from_hms_opt(12, 34, 0).unwrap(), + NaiveTime::from_hms_opt(13, 47, 0).unwrap(), + ), + }, + ], + }; + + let parsed = input.parse::()?; + assert_eq!(parsed, expected); + + Ok(()) + } + + #[test] + fn invalid_timezone_is_err() { + let input = "Invalid/Timezone;O,C,C,C,C,C,O;0412/O,0413/C,0414/1234-1347"; + assert!(input.parse::().is_err()); + } } From f26ecea9b25c2ab2668ac534aa63bf7d48adc859 Mon Sep 17 00:00:00 2001 From: keyvan Date: Wed, 10 Apr 2024 18:00:41 -0700 Subject: [PATCH 10/27] fix: format --- src/agent/market_schedule.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/agent/market_schedule.rs b/src/agent/market_schedule.rs index 2233385a..3ae5b851 100644 --- a/src/agent/market_schedule.rs +++ b/src/agent/market_schedule.rs @@ -19,7 +19,6 @@ use { separated, separated_pair, seq, - terminated, }, error::{ ErrMode, From ab96151da9826ebe62d5e93515708b5137b92d9e Mon Sep 17 00:00:00 2001 From: keyvan Date: Wed, 10 Apr 2024 18:26:27 -0700 Subject: [PATCH 11/27] feat: add market schedule can publish at test --- src/agent/market_schedule.rs | 40 ++++++++++++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/src/agent/market_schedule.rs b/src/agent/market_schedule.rs index 3ae5b851..d155628c 100644 --- a/src/agent/market_schedule.rs +++ b/src/agent/market_schedule.rs @@ -98,8 +98,6 @@ fn holiday_day_schedule_parser<'s>(input: &mut &'s str) -> PResult().unwrap(); let day = day_str.parse::().unwrap(); @@ -170,6 +168,7 @@ impl FromStr for ScheduleDayKind { #[cfg(test)] mod tests { + use chrono::NaiveDateTime; use super::*; #[test] @@ -294,4 +293,41 @@ mod tests { let input = "Invalid/Timezone;O,C,C,C,C,C,O;0412/O,0413/C,0414/1234-1347"; assert!(input.parse::().is_err()); } + + #[test] + fn test_market_schedule_can_publish_at() -> Result<()> { + // Prepare a schedule of narrow ranges + let market_schedule: MarketSchedule = + "UTC;O,O,O,O,O,O,O;0422/0900-1700,1109/0930-1730,1201/O,1225/C,1231/0900-1700" + .parse() + .unwrap(); + + let format = "%Y-%m-%d %H:%M"; + + // Date no match + assert!(market_schedule + .can_publish_at(&NaiveDateTime::parse_from_str("2023-11-20 05:30", format)?.and_utc())); + + // Date match before range + assert!(!market_schedule + .can_publish_at(&NaiveDateTime::parse_from_str("2023-04-22 08:59", format)?.and_utc())); + + // Date match at start of range + assert!(market_schedule + .can_publish_at(&NaiveDateTime::parse_from_str("2023-04-22 09:00", format)?.and_utc())); + + // Date match in range + assert!(market_schedule + .can_publish_at(&NaiveDateTime::parse_from_str("2023-04-22 12:00", format)?.and_utc())); + + // Date match at end of range + assert!(market_schedule + .can_publish_at(&NaiveDateTime::parse_from_str("2023-04-22 17:00", format)?.and_utc())); + + // Date match after range + assert!(!market_schedule + .can_publish_at(&NaiveDateTime::parse_from_str("2023-04-22 17:01", format)?.and_utc())); + + Ok(()) + } } From 33521d95b25857ecf4634b77ddc6ac517cd1aa0e Mon Sep 17 00:00:00 2001 From: keyvan Date: Wed, 10 Apr 2024 19:11:09 -0700 Subject: [PATCH 12/27] feat: use new market hours in pyth agent --- integration-tests/tests/test_integration.py | 4 +- src/agent/market_schedule.rs | 25 +++++++- src/agent/solana/exporter.rs | 16 +++--- src/agent/solana/oracle.rs | 64 ++++++++++++--------- 4 files changed, 70 insertions(+), 39 deletions(-) diff --git a/integration-tests/tests/test_integration.py b/integration-tests/tests/test_integration.py index abb91eaa..20d6d29d 100644 --- a/integration-tests/tests/test_integration.py +++ b/integration-tests/tests/test_integration.py @@ -64,7 +64,7 @@ "quote_currency": "USD", "generic_symbol": "BTCUSD", "description": "BTC/USD", - "holidays": f"{datetime.now().strftime('%m%d')}/O" + "schedule": f"America/New_York;O,O,O,O,O,O,O;{datetime.now().strftime('%m%d')}/O" }, "metadata": {"jump_id": "78876709", "jump_symbol": "BTCUSD", "price_exp": -8, "min_publishers": 1}, } @@ -77,7 +77,7 @@ "quote_currency": "USD", "generic_symbol": "SOLUSD", "description": "SOL/USD", - "holidays": f"{datetime.now().strftime('%m%d')}/C" + "schedule": f"America/New_York;O,O,O,O,O,O,O;{datetime.now().strftime('%m%d')}/C" }, "metadata": {"jump_id": "78876711", "jump_symbol": "SOLUSD", "price_exp": -8, "min_publishers": 1}, } diff --git a/src/agent/market_schedule.rs b/src/agent/market_schedule.rs index d155628c..b4dfc79d 100644 --- a/src/agent/market_schedule.rs +++ b/src/agent/market_schedule.rs @@ -1,6 +1,7 @@ //! Holiday hours metadata parsing and evaluation logic use { + super::schedule::market_hours::MHKind, anyhow::{ anyhow, Result, @@ -43,6 +44,16 @@ pub struct MarketSchedule { pub holidays: Vec, } +impl Default for MarketSchedule { + fn default() -> Self { + Self { + timezone: Tz::UTC, + weekly_schedule: vec![ScheduleDayKind::Open; 7], + holidays: vec![], + } + } +} + impl MarketSchedule { pub fn can_publish_at(&self, when: &DateTime) -> bool { let when_local = when.with_timezone(&self.timezone); @@ -128,6 +139,14 @@ impl ScheduleDayKind { Self::TimeRange(start, end) => start <= &when_local && &when_local <= end, } } + + pub fn from_mhkind(mhkind: MHKind) -> Self { + match mhkind { + MHKind::Open => ScheduleDayKind::Open, + MHKind::Closed => ScheduleDayKind::Closed, + MHKind::TimeRange(start, end) => ScheduleDayKind::TimeRange(start, end), + } + } } impl Default for ScheduleDayKind { @@ -168,8 +187,10 @@ impl FromStr for ScheduleDayKind { #[cfg(test)] mod tests { - use chrono::NaiveDateTime; - use super::*; + use { + super::*, + chrono::NaiveDateTime, + }; #[test] fn test_parsing_schedule_day_kind() -> Result<()> { diff --git a/src/agent/solana/exporter.rs b/src/agent/solana/exporter.rs index 4ff98151..5784cc6d 100644 --- a/src/agent/solana/exporter.rs +++ b/src/agent/solana/exporter.rs @@ -10,11 +10,11 @@ use { network::Network, }, crate::agent::{ + market_schedule::MarketSchedule, remote_keypair_loader::{ KeypairRequest, RemoteKeypairLoader, }, - schedule::Schedule, }, anyhow::{ anyhow, @@ -172,7 +172,7 @@ pub fn spawn_exporter( network: Network, rpc_url: &str, rpc_timeout: Duration, - publisher_permissions_rx: mpsc::Receiver>>, + publisher_permissions_rx: mpsc::Receiver>>, key_store: KeyStore, local_store_tx: Sender, global_store_tx: Sender, @@ -260,10 +260,10 @@ pub struct Exporter { inflight_transactions_tx: Sender, /// publisher => { permissioned_price => market hours } as read by the oracle module - publisher_permissions_rx: mpsc::Receiver>>, + publisher_permissions_rx: mpsc::Receiver>>, /// Currently known permissioned prices of this publisher along with their market hours - our_prices: HashMap, + our_prices: HashMap, /// Interval to update the dynamic price (if enabled) dynamic_compute_unit_price_update_interval: Interval, @@ -287,7 +287,7 @@ impl Exporter { global_store_tx: Sender, network_state_rx: watch::Receiver, inflight_transactions_tx: Sender, - publisher_permissions_rx: mpsc::Receiver>>, + publisher_permissions_rx: mpsc::Receiver>>, keypair_request_tx: mpsc::Sender, logger: Logger, ) -> Self { @@ -474,10 +474,8 @@ impl Exporter { .into_iter() .filter(|(id, _data)| { let key_from_id = Pubkey::from((*id).clone().to_bytes()); - if let Some(schedule) = self.our_prices.get_mut(&key_from_id) { - // let ret = schedule.market_hours.can_publish_at(&now); - schedule.holiday_hours.timezone = Some(schedule.market_hours.timezone); - let ret = schedule.market_hours.can_publish_at(&now) && schedule.holiday_hours.can_publish_at(now); + if let Some(schedule) = self.our_prices.get(&key_from_id) { + let ret = schedule.can_publish_at(&now); if !ret { debug!(self.logger, "Exporter: Attempted to publish price outside market hours"; diff --git a/src/agent/solana/oracle.rs b/src/agent/solana/oracle.rs index 8e377d5a..078d206f 100644 --- a/src/agent/solana/oracle.rs +++ b/src/agent/solana/oracle.rs @@ -4,11 +4,11 @@ use { self::subscriber::Subscriber, super::key_store::KeyStore, crate::agent::{ - schedule::{ - holiday_hours::HolidaySchedule, - market_hours::WeeklySchedule, - Schedule, + market_schedule::{ + MarketSchedule, + ScheduleDayKind, }, + schedule::market_hours::WeeklySchedule, store::global, }, anyhow::{ @@ -121,7 +121,7 @@ pub struct Data { pub product_accounts: HashMap, pub price_accounts: HashMap, /// publisher => {their permissioned price accounts => market hours} - pub publisher_permissions: HashMap>, + pub publisher_permissions: HashMap>, } impl Data { @@ -129,7 +129,7 @@ impl Data { mapping_accounts: HashMap, product_accounts: HashMap, price_accounts: HashMap, - publisher_permissions: HashMap>, + publisher_permissions: HashMap>, ) -> Self { Data { mapping_accounts, @@ -144,7 +144,7 @@ pub type MappingAccount = pyth_sdk_solana::state::MappingAccount; #[derive(Debug, Clone)] pub struct ProductEntry { pub account_data: pyth_sdk_solana::state::ProductAccount, - pub schedule: Schedule, + pub schedule: MarketSchedule, pub price_accounts: Vec, } @@ -207,7 +207,7 @@ pub fn spawn_oracle( wss_url: &str, rpc_timeout: Duration, global_store_update_tx: mpsc::Sender, - publisher_permissions_tx: mpsc::Sender>>, + publisher_permissions_tx: mpsc::Sender>>, key_store: KeyStore, logger: Logger, ) -> Vec> { @@ -422,7 +422,7 @@ struct Poller { data_tx: mpsc::Sender, /// Updates about permissioned price accounts from oracle to exporter - publisher_permissions_tx: mpsc::Sender>>, + publisher_permissions_tx: mpsc::Sender>>, /// The RPC client to use to poll data from the RPC node rpc_client: RpcClient, @@ -442,7 +442,7 @@ struct Poller { impl Poller { pub fn new( data_tx: mpsc::Sender, - publisher_permissions_tx: mpsc::Sender>>, + publisher_permissions_tx: mpsc::Sender>>, rpc_url: &str, rpc_timeout: Duration, commitment: CommitmentLevel, @@ -612,7 +612,7 @@ impl Poller { let product = load_product_account(prod_acc.data.as_slice()) .context(format!("Could not parse product account {}", product_key))?; - let weekly_schedule: WeeklySchedule = if let Some((_wsched_key, wsched_val)) = + let legacy_schedule: WeeklySchedule = if let Some((_wsched_key, wsched_val)) = product.iter().find(|(k, _v)| *k == "weekly_schedule") { wsched_val.parse().unwrap_or_else(|err| { @@ -629,33 +629,45 @@ impl Poller { Default::default() // No market hours specified, meaning 24/7 publishing }; - let holiday_schedule: HolidaySchedule = if let Some((_hsched_key, hsched_val)) = - product.iter().find(|(k, _v)| *k == "holidays") + let market_schedule: Option = if let Some(( + _msched_key, + msched_val, + )) = + product.iter().find(|(k, _v)| *k == "schedule") { - hsched_val.parse().unwrap_or_else(|err| { + let schedule = msched_val.parse::(); + if schedule.is_ok() { + Some(schedule.unwrap()) + } else { warn!( self.logger, - "Oracle: Product has weekly_schedule defined but it could not be parsed. Falling back to 24/7 publishing."; + "Oracle: Product has schedule defined but it could not be parsed. Falling back to legacy schedule."; "product_key" => product_key.to_string(), - "holiday_schedule" => hsched_val, + "schedule" => msched_val, ); - debug!(self.logger, "parsing error context"; "context" => format!("{:?}", err)); - Default::default() - }) + None + } } else { - Default::default() // No market hours specified, meaning 24/7 publishing + None }; product_entries.insert( *product_key, ProductEntry { account_data: *product, - schedule: { - Schedule { - market_hours: weekly_schedule, - holiday_hours: holiday_schedule, - } - }, + schedule: market_schedule.unwrap_or_else(|| MarketSchedule { + timezone: legacy_schedule.timezone, + weekly_schedule: vec![ + ScheduleDayKind::from_mhkind(legacy_schedule.mon), + ScheduleDayKind::from_mhkind(legacy_schedule.tue), + ScheduleDayKind::from_mhkind(legacy_schedule.wed), + ScheduleDayKind::from_mhkind(legacy_schedule.thu), + ScheduleDayKind::from_mhkind(legacy_schedule.fri), + ScheduleDayKind::from_mhkind(legacy_schedule.sat), + ScheduleDayKind::from_mhkind(legacy_schedule.sun), + ], + holidays: vec![], + }), price_accounts: vec![], }, ); From b4ecb3f858250840b1bcd626fd952e92dabfb61d Mon Sep 17 00:00:00 2001 From: keyvan Date: Thu, 11 Apr 2024 07:20:54 -0700 Subject: [PATCH 13/27] fix: format --- src/agent/market_schedule.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/agent/market_schedule.rs b/src/agent/market_schedule.rs index b4dfc79d..c51678a6 100644 --- a/src/agent/market_schedule.rs +++ b/src/agent/market_schedule.rs @@ -159,9 +159,9 @@ fn time_range_parser<'s>(input: &mut &'s str) -> PResult { let (start_str, end_str) = separated_pair(take(4usize), "-", take(4usize)).parse_next(input)?; let start_time = NaiveTime::parse_from_str(start_str, "%H%M") - .map_err(|e| ErrMode::from_error_kind(input, ErrorKind::Verify))?; + .map_err(|_| ErrMode::from_error_kind(input, ErrorKind::Verify))?; let end_time = NaiveTime::parse_from_str(end_str, "%H%M") - .map_err(|e| ErrMode::from_error_kind(input, ErrorKind::Verify))?; + .map_err(|_| ErrMode::from_error_kind(input, ErrorKind::Verify))?; Ok(ScheduleDayKind::TimeRange(start_time, end_time)) } @@ -348,7 +348,6 @@ mod tests { // Date match after range assert!(!market_schedule .can_publish_at(&NaiveDateTime::parse_from_str("2023-04-22 17:01", format)?.and_utc())); - Ok(()) } } From e5d41213040959aeb45a30104d0d8ae92025a071 Mon Sep 17 00:00:00 2001 From: keyvan Date: Thu, 11 Apr 2024 07:23:33 -0700 Subject: [PATCH 14/27] refactor: rollback module restructure --- src/agent.rs | 2 +- src/agent/{schedule => }/market_hours.rs | 0 src/agent/market_schedule.rs | 2 +- src/agent/schedule.rs | 13 - src/agent/schedule/holiday_hours.rs | 376 ----------------------- src/agent/solana/oracle.rs | 2 +- 6 files changed, 3 insertions(+), 392 deletions(-) rename src/agent/{schedule => }/market_hours.rs (100%) delete mode 100644 src/agent/schedule.rs delete mode 100644 src/agent/schedule/holiday_hours.rs diff --git a/src/agent.rs b/src/agent.rs index 195ea6aa..8b48bf17 100644 --- a/src/agent.rs +++ b/src/agent.rs @@ -63,11 +63,11 @@ Note that there is an Oracle and Exporter for each network, but only one Local S ################################################################################################################################## */ pub mod dashboard; +pub mod market_hours; pub mod market_schedule; pub mod metrics; pub mod pythd; pub mod remote_keypair_loader; -pub mod schedule; pub mod solana; pub mod store; use { diff --git a/src/agent/schedule/market_hours.rs b/src/agent/market_hours.rs similarity index 100% rename from src/agent/schedule/market_hours.rs rename to src/agent/market_hours.rs diff --git a/src/agent/market_schedule.rs b/src/agent/market_schedule.rs index c51678a6..3aa64b66 100644 --- a/src/agent/market_schedule.rs +++ b/src/agent/market_schedule.rs @@ -1,7 +1,7 @@ //! Holiday hours metadata parsing and evaluation logic use { - super::schedule::market_hours::MHKind, + super::market_hours::MHKind, anyhow::{ anyhow, Result, diff --git a/src/agent/schedule.rs b/src/agent/schedule.rs deleted file mode 100644 index 07579f4a..00000000 --- a/src/agent/schedule.rs +++ /dev/null @@ -1,13 +0,0 @@ -use { - holiday_hours::HolidaySchedule, - market_hours::WeeklySchedule, -}; - -pub mod holiday_hours; -pub mod market_hours; - -#[derive(Debug, Clone, Default)] -pub struct Schedule { - pub market_hours: WeeklySchedule, - pub holiday_hours: HolidaySchedule, -} diff --git a/src/agent/schedule/holiday_hours.rs b/src/agent/schedule/holiday_hours.rs deleted file mode 100644 index 40295354..00000000 --- a/src/agent/schedule/holiday_hours.rs +++ /dev/null @@ -1,376 +0,0 @@ -//! Holiday hours metadata parsing and evaluation logic - -use { - anyhow::{ - anyhow, - Context, - Result, - }, - chrono::{ - naive::NaiveTime, - DateTime, - Datelike, - Duration, - Utc, - }, - chrono_tz::Tz, - lazy_static::lazy_static, - std::str::FromStr, -}; - -lazy_static! { - /// Helper time value representing 24:00:00 as 00:00:00 minus 1 - /// nanosecond (underflowing to 23:59:59.999(...) ). While chrono - /// has this value internally exposed as NaiveTime::MAX, it is not - /// exposed outside the crate. - static ref MAX_TIME_INSTANT: NaiveTime = NaiveTime::MIN.overflowing_sub_signed(Duration::nanoseconds(1)).0; -} - -/// Holiday hours schedule -#[derive(Clone, Default, Debug, Eq, PartialEq)] -pub struct HolidaySchedule { - pub timezone: Option, - pub days: Vec, -} - -impl HolidaySchedule { - pub fn all_closed() -> Self { - Self { - timezone: Default::default(), - days: vec![], - } - } - - pub fn can_publish_at(&self, when: DateTime) -> bool { - // Convert to time local to the market - let when_local = when.with_timezone(&self.timezone.unwrap()); - - let market_month = when_local.date_naive().month0() + 1; - let market_day = when_local.date_naive().day0() + 1; - - let market_time = when_local.time(); - - for day in &self.days { - // Check if the day matches - if day.month == market_month && day.day == market_day { - return day.kind.can_publish_at(market_time); - } - } - true - } -} - -impl FromStr for HolidaySchedule { - type Err = anyhow::Error; - fn from_str(s: &str) -> Result { - let split_by_commas = s.split(","); - let mut days = Vec::new(); - - for day_str in split_by_commas { - let day = day_str.parse()?; - days.push(day); - } - - Ok(HolidaySchedule { - days, - timezone: None, - }) - } -} - -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct HolidayDaySchedule { - pub month: u32, - pub day: u32, - pub kind: HolidayDayKind, -} - -impl FromStr for HolidayDaySchedule { - type Err = anyhow::Error; - fn from_str(s: &str) -> Result { - let date_time_parts: Vec<&str> = s.split("/").collect(); - if date_time_parts.len() != 2 { - return Err(anyhow!("Invalid format")); - } - - if date_time_parts[0].len() != 4 { - return Err(anyhow!("Invalid date format")); - } - - let month: u32 = date_time_parts[0][..2] - .parse() - .map_err(|_| anyhow!("Invalid month"))?; - let day: u32 = date_time_parts[0][2..] - .parse() - .map_err(|_| anyhow!("Invalid day"))?; - let kind: HolidayDayKind = date_time_parts[1].parse()?; - - Ok(HolidayDaySchedule { month, day, kind }) - } -} - -/// Helper enum for denoting per-day schedules: time range, all-day open and all-day closed. -#[derive(Clone, Debug, Eq, PartialEq)] -pub enum HolidayDayKind { - Open, - Closed, - TimeRange(NaiveTime, NaiveTime), -} - -impl HolidayDayKind { - pub fn can_publish_at(&self, when_market_local: NaiveTime) -> bool { - match self { - Self::Open => true, - Self::Closed => false, - Self::TimeRange(start, end) => start <= &when_market_local && &when_market_local <= end, - } - } -} - -impl Default for HolidayDayKind { - fn default() -> Self { - Self::Open - } -} - -impl FromStr for HolidayDayKind { - type Err = anyhow::Error; - fn from_str(s: &str) -> Result { - match s { - "O" => Ok(HolidayDayKind::Open), - "C" => Ok(HolidayDayKind::Closed), - other => { - let (start_str, end_str) = other.split_once("-").ok_or(anyhow!( - "Missing '-' delimiter between start and end of range" - ))?; - - let start = NaiveTime::parse_from_str(start_str, "%H%M") - .context("start time does not match HHMM format")?; - - // The chrono crate is unable to parse 24:00 as - // previous day's perspective of midnight, so we use - // the next best thing - see MAX_TIME_INSTANT for - // details. - let end = if end_str.contains("2400") { - MAX_TIME_INSTANT.clone() - } else { - NaiveTime::parse_from_str(end_str, "%H%M") - .context("end time does not match HHMM format")? - }; - - if start < end { - Ok(HolidayDayKind::TimeRange(start, end)) - } else { - Err(anyhow!("Incorrect time range: start must come before end")) - } - } - } - } -} - -#[cfg(test)] -mod tests { - use { - super::*, - chrono::NaiveDateTime, - }; - - #[test] - fn test_parsing_single_day() -> Result<()> { - let s = "0422/0900-1700"; - - let parsed: HolidaySchedule = s.parse()?; - - let expected = HolidaySchedule { - timezone: None, - days: vec![HolidayDaySchedule { - month: 4, - day: 22, - kind: HolidayDayKind::TimeRange( - NaiveTime::from_hms_opt(9, 0, 0).unwrap(), - NaiveTime::from_hms_opt(17, 0, 0).unwrap(), - ), - }], - }; - - assert_eq!(parsed, expected); - - Ok(()) - } - - #[test] - fn test_parsing_multiple_days() -> Result<()> { - let s = "0422/0900-1700,1109/0930-1730,1201/O,1225/C,1231/0900-1700"; - - let parsed: HolidaySchedule = s.parse()?; - - let expected = HolidaySchedule { - timezone: None, - days: vec![ - HolidayDaySchedule { - month: 4, - day: 22, - kind: HolidayDayKind::TimeRange( - NaiveTime::from_hms_opt(9, 0, 0).unwrap(), - NaiveTime::from_hms_opt(17, 0, 0).unwrap(), - ), - }, - HolidayDaySchedule { - month: 11, - day: 9, - kind: HolidayDayKind::TimeRange( - NaiveTime::from_hms_opt(9, 30, 0).unwrap(), - NaiveTime::from_hms_opt(17, 30, 0).unwrap(), - ), - }, - HolidayDaySchedule { - month: 12, - day: 1, - kind: HolidayDayKind::Open, - }, - HolidayDaySchedule { - month: 12, - day: 25, - kind: HolidayDayKind::Closed, - }, - HolidayDaySchedule { - month: 12, - day: 31, - kind: HolidayDayKind::TimeRange( - NaiveTime::from_hms_opt(9, 0, 0).unwrap(), - NaiveTime::from_hms_opt(17, 0, 0).unwrap(), - ), - }, - ], - }; - - assert_eq!(parsed, expected); - - Ok(()) - } - - #[test] - fn test_parsing_month_without_leading_zero_is_error() { - let s = "422/0900-1700"; - - let parsing_result: Result = s.parse(); - assert!(parsing_result.is_err()); - } - - #[test] - fn test_parsing_hour_without_leading_zero_is_error() { - let s = "0422/900-1700"; - - let parsing_result: Result = s.parse(); - assert!(parsing_result.is_err()); - } - - #[test] - fn test_parsing_wrong_delimiter_is_error() { - let s = "0422-0900/1700"; - - let parsing_result: Result = s.parse(); - assert!(parsing_result.is_err()); - } - - #[test] - fn test_parsing_wrong_format_is_error() { - let s = "0422/09:00-17:00"; - - let parsing_result: Result = s.parse(); - assert!(parsing_result.is_err()); - } - - #[test] - fn test_parsing_wrong_time_range_is_error() { - let s = "0422/1700-0900"; - - let parsing_result: Result = s.parse(); - assert!(parsing_result.is_err()); - } - - #[test] - fn test_parse_24_hour() { - let s = "0422/0900-2400"; - - let parsed: HolidaySchedule = s.parse().unwrap(); - let expected = HolidaySchedule { - timezone: None, - days: vec![HolidayDaySchedule { - month: 4, - day: 22, - kind: HolidayDayKind::TimeRange( - NaiveTime::from_hms_opt(9, 0, 0).unwrap(), - MAX_TIME_INSTANT.clone(), - ), - }], - }; - assert_eq!(parsed, expected); - } - - #[test] - fn test_holiday_schedule_can_publish_at() -> Result<()> { - // Prepare a schedule of narrow ranges - let mut holiday_hours: HolidaySchedule = - "0422/0900-1700,1109/0930-1730,1201/O,1225/C,1231/0900-1700" - .parse() - .unwrap(); - - holiday_hours.timezone = Some(Tz::UTC); - let format = "%Y-%m-%d %H:%M"; - - // Date no match - assert!(holiday_hours - .can_publish_at(NaiveDateTime::parse_from_str("2023-11-20 05:30", format)?.and_utc())); - - // Date match before range - assert!(!holiday_hours - .can_publish_at(NaiveDateTime::parse_from_str("2023-04-22 08:59", format)?.and_utc())); - - // Date match at start of range - assert!(holiday_hours - .can_publish_at(NaiveDateTime::parse_from_str("2023-04-22 09:00", format)?.and_utc())); - - // Date match in range - assert!(holiday_hours - .can_publish_at(NaiveDateTime::parse_from_str("2023-04-22 12:00", format)?.and_utc())); - - // Date match at end of range - assert!(holiday_hours - .can_publish_at(NaiveDateTime::parse_from_str("2023-04-22 17:00", format)?.and_utc())); - - // Date match after range - assert!(!holiday_hours - .can_publish_at(NaiveDateTime::parse_from_str("2023-04-22 17:01", format)?.and_utc())); - - Ok(()) - } - - /// Verify desired 24:00 behavior. - #[test] - fn test_market_hours_midnight_00_24() -> Result<()> { - // Prepare a schedule of midnight-neighboring ranges - let mut holiday_schedule: HolidaySchedule = "0422/0900-2400".parse()?; - - holiday_schedule.timezone = Some(Tz::UTC); - - let format = "%Y-%m-%d %H:%M"; - // Date match before range - assert!(!holiday_schedule - .can_publish_at(NaiveDateTime::parse_from_str("2023-04-22 08:59", format)?.and_utc())); - - // Date match at start of range - assert!(holiday_schedule - .can_publish_at(NaiveDateTime::parse_from_str("2023-04-22 09:00", format)?.and_utc())); - - // Date match in range - assert!(holiday_schedule - .can_publish_at(NaiveDateTime::parse_from_str("2023-04-22 12:00", format)?.and_utc())); - - // Date match at end of range - assert!(holiday_schedule - .can_publish_at(NaiveDateTime::parse_from_str("2023-04-22 23:59", format)?.and_utc())); - - Ok(()) - } -} diff --git a/src/agent/solana/oracle.rs b/src/agent/solana/oracle.rs index 078d206f..c5a3b6d3 100644 --- a/src/agent/solana/oracle.rs +++ b/src/agent/solana/oracle.rs @@ -4,11 +4,11 @@ use { self::subscriber::Subscriber, super::key_store::KeyStore, crate::agent::{ + market_hours::WeeklySchedule, market_schedule::{ MarketSchedule, ScheduleDayKind, }, - schedule::market_hours::WeeklySchedule, store::global, }, anyhow::{ From aa325a3a9250ed8c45915ec32b1a23d04ae481f2 Mon Sep 17 00:00:00 2001 From: keyvan Date: Thu, 11 Apr 2024 07:28:52 -0700 Subject: [PATCH 15/27] rename market hours to legacy schedule --- src/agent.rs | 2 +- .../{market_hours.rs => legacy_schedule.rs} | 30 ++++++++++--------- src/agent/market_schedule.rs | 2 +- src/agent/solana/oracle.rs | 4 +-- 4 files changed, 20 insertions(+), 18 deletions(-) rename src/agent/{market_hours.rs => legacy_schedule.rs} (95%) diff --git a/src/agent.rs b/src/agent.rs index 8b48bf17..738e0a30 100644 --- a/src/agent.rs +++ b/src/agent.rs @@ -63,7 +63,7 @@ Note that there is an Oracle and Exporter for each network, but only one Local S ################################################################################################################################## */ pub mod dashboard; -pub mod market_hours; +pub mod legacy_schedule; pub mod market_schedule; pub mod metrics; pub mod pythd; diff --git a/src/agent/market_hours.rs b/src/agent/legacy_schedule.rs similarity index 95% rename from src/agent/market_hours.rs rename to src/agent/legacy_schedule.rs index 28e7f881..4aeabec2 100644 --- a/src/agent/market_hours.rs +++ b/src/agent/legacy_schedule.rs @@ -28,8 +28,10 @@ lazy_static! { } /// Weekly market hours schedule +/// TODO: Remove after the migration #[derive(Clone, Default, Debug, Eq, PartialEq)] -pub struct WeeklySchedule { +#[deprecated(note = "This struct is deprecated, use MarketSchedule instead.")] +pub struct LegacySchedule { pub timezone: Tz, pub mon: MHKind, pub tue: MHKind, @@ -40,7 +42,7 @@ pub struct WeeklySchedule { pub sun: MHKind, } -impl WeeklySchedule { +impl LegacySchedule { pub fn all_closed() -> Self { Self { timezone: Default::default(), @@ -76,7 +78,7 @@ impl WeeklySchedule { } } -impl FromStr for WeeklySchedule { +impl FromStr for LegacySchedule { type Err = anyhow::Error; fn from_str(s: &str) -> Result { let mut split_by_commas = s.split(","); @@ -235,9 +237,9 @@ mod tests { // Mon-Fri 9-5, inconsistent leading space on Tuesday, leading 0 on Friday (expected to be fine) let s = "Europe/Warsaw,9:00-17:00, 9:00-17:00,9:00-17:00,9:00-17:00,09:00-17:00,C,C"; - let parsed: WeeklySchedule = s.parse()?; + let parsed: LegacySchedule = s.parse()?; - let expected = WeeklySchedule { + let expected = LegacySchedule { timezone: Tz::Europe__Warsaw, mon: MHKind::TimeRange( NaiveTime::from_hms_opt(9, 0, 0).unwrap(), @@ -273,7 +275,7 @@ mod tests { // Valid but missing a timezone let s = "O,C,O,C,O,C,O"; - let parsing_result: Result = s.parse(); + let parsing_result: Result = s.parse(); dbg!(&parsing_result); assert!(parsing_result.is_err()); @@ -284,7 +286,7 @@ mod tests { // One day short let s = "Asia/Hong_Kong,C,O,C,O,C,O"; - let parsing_result: Result = s.parse(); + let parsing_result: Result = s.parse(); dbg!(&parsing_result); assert!(parsing_result.is_err()); @@ -294,7 +296,7 @@ mod tests { fn test_parsing_gibberish_timezone_is_error() { // Pretty sure that one's extinct let s = "Pangea/New_Dino_City,O,O,O,O,O,O,O"; - let parsing_result: Result = s.parse(); + let parsing_result: Result = s.parse(); dbg!(&parsing_result); assert!(parsing_result.is_err()); @@ -303,7 +305,7 @@ mod tests { #[test] fn test_parsing_gibberish_day_schedule_is_error() { let s = "Europe/Amsterdam,mondays are alright I guess,O,O,O,O,O,O"; - let parsing_result: Result = s.parse(); + let parsing_result: Result = s.parse(); dbg!(&parsing_result); assert!(parsing_result.is_err()); @@ -313,7 +315,7 @@ mod tests { fn test_parsing_too_many_days_is_error() { // One day too many let s = "Europe/Lisbon,O,O,O,O,O,O,O,O,C"; - let parsing_result: Result = s.parse(); + let parsing_result: Result = s.parse(); dbg!(&parsing_result); assert!(parsing_result.is_err()); @@ -322,7 +324,7 @@ mod tests { #[test] fn test_market_hours_happy_path() -> Result<()> { // Prepare a schedule of narrow ranges - let wsched: WeeklySchedule = "America/New_York,00:00-1:00,1:00-2:00,2:00-3:00,3:00-4:00,4:00-5:00,5:00-6:00,6:00-7:00".parse()?; + let wsched: LegacySchedule = "America/New_York,00:00-1:00,1:00-2:00,2:00-3:00,3:00-4:00,4:00-5:00,5:00-6:00,6:00-7:00".parse()?; // Prepare UTC datetimes that fall before, within and after market hours let format = "%Y-%m-%d %H:%M"; @@ -379,7 +381,7 @@ mod tests { #[test] fn test_market_hours_midnight_00_24() -> Result<()> { // Prepare a schedule of midnight-neighboring ranges - let wsched: WeeklySchedule = + let wsched: LegacySchedule = "Europe/Amsterdam,23:00-24:00,00:00-01:00,O,C,C,C,C".parse()?; let format = "%Y-%m-%d %H:%M"; @@ -433,8 +435,8 @@ mod tests { // CDT/CET 6h offset in use for 2 weeks, CDT/CEST 7h offset after) // * Autumn 2023: Oct29(EU)-Nov5(US) (clocks go back 1h, // CDT/CET 6h offset in use 1 week, CST/CET 7h offset after) - let wsched_eu: WeeklySchedule = "Europe/Amsterdam,9:00-17:00,O,O,O,O,O,O".parse()?; - let wsched_us: WeeklySchedule = "America/Chicago,2:00-10:00,O,O,O,O,O,O".parse()?; + let wsched_eu: LegacySchedule = "Europe/Amsterdam,9:00-17:00,O,O,O,O,O,O".parse()?; + let wsched_us: LegacySchedule = "America/Chicago,2:00-10:00,O,O,O,O,O,O".parse()?; let format = "%Y-%m-%d %H:%M"; diff --git a/src/agent/market_schedule.rs b/src/agent/market_schedule.rs index 3aa64b66..5233282f 100644 --- a/src/agent/market_schedule.rs +++ b/src/agent/market_schedule.rs @@ -1,7 +1,7 @@ //! Holiday hours metadata parsing and evaluation logic use { - super::market_hours::MHKind, + super::legacy_schedule::MHKind, anyhow::{ anyhow, Result, diff --git a/src/agent/solana/oracle.rs b/src/agent/solana/oracle.rs index c5a3b6d3..0c93076d 100644 --- a/src/agent/solana/oracle.rs +++ b/src/agent/solana/oracle.rs @@ -4,7 +4,7 @@ use { self::subscriber::Subscriber, super::key_store::KeyStore, crate::agent::{ - market_hours::WeeklySchedule, + legacy_schedule::LegacySchedule, market_schedule::{ MarketSchedule, ScheduleDayKind, @@ -612,7 +612,7 @@ impl Poller { let product = load_product_account(prod_acc.data.as_slice()) .context(format!("Could not parse product account {}", product_key))?; - let legacy_schedule: WeeklySchedule = if let Some((_wsched_key, wsched_val)) = + let legacy_schedule: LegacySchedule = if let Some((_wsched_key, wsched_val)) = product.iter().find(|(k, _v)| *k == "weekly_schedule") { wsched_val.parse().unwrap_or_else(|err| { From 9b625b5c6466151708ed292cd527a8fb6427c714 Mon Sep 17 00:00:00 2001 From: keyvan Date: Thu, 11 Apr 2024 07:31:53 -0700 Subject: [PATCH 16/27] chore: add comment --- src/agent/market_schedule.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/agent/market_schedule.rs b/src/agent/market_schedule.rs index 5233282f..77073687 100644 --- a/src/agent/market_schedule.rs +++ b/src/agent/market_schedule.rs @@ -109,6 +109,9 @@ fn holiday_day_schedule_parser<'s>(input: &mut &'s str) -> PResult().unwrap(); let day = day_str.parse::().unwrap(); From a551547d985b38d4d6b7ce8ee6dabf297a12c26a Mon Sep 17 00:00:00 2001 From: keyvan Date: Thu, 11 Apr 2024 11:12:12 -0700 Subject: [PATCH 17/27] feat: add support for 24:00 --- src/agent/market_schedule.rs | 58 ++++++++++++++++++++++++++++++------ 1 file changed, 49 insertions(+), 9 deletions(-) diff --git a/src/agent/market_schedule.rs b/src/agent/market_schedule.rs index 77073687..4b006450 100644 --- a/src/agent/market_schedule.rs +++ b/src/agent/market_schedule.rs @@ -10,6 +10,7 @@ use { naive::NaiveTime, DateTime, Datelike, + Duration, Utc, }, chrono_tz::Tz, @@ -37,6 +38,15 @@ use { }; +/// Helper time value representing 24:00:00 as 00:00:00 minus 1 +/// nanosecond (underflowing to 23:59:59.999(...) ). While chrono +/// has this value internally exposed as NaiveTime::MAX, it is not +/// exposed outside the crate. +const MAX_TIME_INSTANT: NaiveTime = NaiveTime::MIN + .overflowing_sub_signed(Duration::nanoseconds(1)) + .0; + + #[derive(Clone, Debug, Eq, PartialEq)] pub struct MarketSchedule { pub timezone: Tz, @@ -158,13 +168,20 @@ impl Default for ScheduleDayKind { } } -fn time_range_parser<'s>(input: &mut &'s str) -> PResult { - let (start_str, end_str) = separated_pair(take(4usize), "-", take(4usize)).parse_next(input)?; +fn time_parser<'s>(input: &mut &'s str) -> PResult { + alt(( + "2400", + take(4usize).verify(|s| NaiveTime::parse_from_str(s, "%H%M").is_ok()), + )) + .map(|time_str| match time_str { + "2400" => MAX_TIME_INSTANT, + _ => NaiveTime::parse_from_str(time_str, "%H%M").unwrap(), + }) + .parse_next(input) +} - let start_time = NaiveTime::parse_from_str(start_str, "%H%M") - .map_err(|_| ErrMode::from_error_kind(input, ErrorKind::Verify))?; - let end_time = NaiveTime::parse_from_str(end_str, "%H%M") - .map_err(|_| ErrMode::from_error_kind(input, ErrorKind::Verify))?; +fn time_range_parser<'s>(input: &mut &'s str) -> PResult { + let (start_time, end_time) = separated_pair(time_parser, "-", time_parser).parse_next(input)?; Ok(ScheduleDayKind::TimeRange(start_time, end_time)) } @@ -201,6 +218,7 @@ mod tests { let open = "O"; let closed = "C"; let valid = "1234-1347"; + let valid2400 = "1234-2400"; let invalid = "1234-5668"; let invalid_format = "1234-56"; @@ -219,6 +237,13 @@ mod tests { NaiveTime::from_hms_opt(13, 47, 0).unwrap(), ) ); + assert_eq!( + valid2400.parse::().unwrap(), + ScheduleDayKind::TimeRange( + NaiveTime::from_hms_opt(12, 34, 0).unwrap(), + MAX_TIME_INSTANT, + ) + ); assert!(invalid.parse::().is_err()); assert!(invalid_format.parse::().is_err()); @@ -269,7 +294,7 @@ mod tests { #[test] fn test_parsing_market_schedule() -> Result<()> { - let input = "America/New_York;O,1234-1347,C,C,C,C,O;0412/O,0413/C,0414/1234-1347"; + let input = "America/New_York;O,1234-1347,0930-2400,C,C,C,O;0412/O,0413/C,0414/1234-1347,1230/0930-2400"; let expected = MarketSchedule { timezone: Tz::America__New_York, weekly_schedule: vec![ @@ -278,7 +303,10 @@ mod tests { NaiveTime::from_hms_opt(12, 34, 0).unwrap(), NaiveTime::from_hms_opt(13, 47, 0).unwrap(), ), - ScheduleDayKind::Closed, + ScheduleDayKind::TimeRange( + NaiveTime::from_hms_opt(09, 30, 0).unwrap(), + MAX_TIME_INSTANT, + ), ScheduleDayKind::Closed, ScheduleDayKind::Closed, ScheduleDayKind::Closed, @@ -303,6 +331,14 @@ mod tests { NaiveTime::from_hms_opt(13, 47, 0).unwrap(), ), }, + HolidayDaySchedule { + month: 12, + day: 30, + kind: ScheduleDayKind::TimeRange( + NaiveTime::from_hms_opt(09, 30, 0).unwrap(), + MAX_TIME_INSTANT, + ), + }, ], }; @@ -322,7 +358,7 @@ mod tests { fn test_market_schedule_can_publish_at() -> Result<()> { // Prepare a schedule of narrow ranges let market_schedule: MarketSchedule = - "UTC;O,O,O,O,O,O,O;0422/0900-1700,1109/0930-1730,1201/O,1225/C,1231/0900-1700" + "UTC;O,O,O,O,O,O,O;0422/0900-1700,1109/0930-1730,1201/O,1225/C,1231/0930-2400" .parse() .unwrap(); @@ -351,6 +387,10 @@ mod tests { // Date match after range assert!(!market_schedule .can_publish_at(&NaiveDateTime::parse_from_str("2023-04-22 17:01", format)?.and_utc())); + + // Date 2400 range + assert!(market_schedule + .can_publish_at(&NaiveDateTime::parse_from_str("2023-12-31 23:59", format)?.and_utc())); Ok(()) } } From 7b5e1b5106e369cb8ca2aa83f0834106c86a2110 Mon Sep 17 00:00:00 2001 From: keyvan Date: Thu, 11 Apr 2024 11:29:48 -0700 Subject: [PATCH 18/27] fix: avoid parsing twice for verification --- src/agent/market_schedule.rs | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/src/agent/market_schedule.rs b/src/agent/market_schedule.rs index 4b006450..1d67c78e 100644 --- a/src/agent/market_schedule.rs +++ b/src/agent/market_schedule.rs @@ -22,11 +22,6 @@ use { separated_pair, seq, }, - error::{ - ErrMode, - ErrorKind, - ParserError, - }, stream::ToUsize, token::{ take, @@ -87,7 +82,7 @@ impl MarketSchedule { fn market_schedule_parser<'s>(input: &mut &'s str) -> PResult { seq!( MarketSchedule { - timezone: take_till(0.., ';').verify(|s| Tz::from_str(s).is_ok()).map(|s| Tz::from_str(s).unwrap()), + timezone: take_till(0.., ';').map(|s| Tz::from_str(s)).verify(|s| s.is_ok()).map(|s| s.unwrap()), _: ';', weekly_schedule: separated(7, schedule_day_kind_parser, ","), _: ';', @@ -171,12 +166,14 @@ impl Default for ScheduleDayKind { fn time_parser<'s>(input: &mut &'s str) -> PResult { alt(( "2400", - take(4usize).verify(|s| NaiveTime::parse_from_str(s, "%H%M").is_ok()), + take(4usize), )) .map(|time_str| match time_str { - "2400" => MAX_TIME_INSTANT, - _ => NaiveTime::parse_from_str(time_str, "%H%M").unwrap(), + "2400" => Ok(MAX_TIME_INSTANT), + _ => NaiveTime::parse_from_str(time_str, "%H%M"), }) + .verify(|time| time.is_ok()) + .map(|time| time.unwrap()) .parse_next(input) } From ce9417563698cb2575a2714b2426ab6eedd94a0b Mon Sep 17 00:00:00 2001 From: keyvan Date: Thu, 11 Apr 2024 11:31:59 -0700 Subject: [PATCH 19/27] refactor: use match instead of if --- src/agent/solana/oracle.rs | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/agent/solana/oracle.rs b/src/agent/solana/oracle.rs index 0c93076d..1e798fbb 100644 --- a/src/agent/solana/oracle.rs +++ b/src/agent/solana/oracle.rs @@ -635,17 +635,18 @@ impl Poller { )) = product.iter().find(|(k, _v)| *k == "schedule") { - let schedule = msched_val.parse::(); - if schedule.is_ok() { - Some(schedule.unwrap()) - } else { - warn!( - self.logger, - "Oracle: Product has schedule defined but it could not be parsed. Falling back to legacy schedule."; - "product_key" => product_key.to_string(), - "schedule" => msched_val, - ); - None + match msched_val.parse::() { + Ok(schedule) => Some(schedule), + Err(err) => { + warn!( + self.logger, + "Oracle: Product has schedule defined but it could not be parsed. Falling back to legacy schedule."; + "product_key" => product_key.to_string(), + "schedule" => msched_val, + ); + debug!(self.logger, "parsing error context"; "context" => format!("{:?}", err)); + None + } } } else { None From e5b8be310fb80b1875c959f820cac4e8dc53109b Mon Sep 17 00:00:00 2001 From: keyvan Date: Thu, 11 Apr 2024 11:59:18 -0700 Subject: [PATCH 20/27] refactor: use seq for time range parser --- src/agent/market_schedule.rs | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/src/agent/market_schedule.rs b/src/agent/market_schedule.rs index 1d67c78e..bf9f3adf 100644 --- a/src/agent/market_schedule.rs +++ b/src/agent/market_schedule.rs @@ -164,23 +164,24 @@ impl Default for ScheduleDayKind { } fn time_parser<'s>(input: &mut &'s str) -> PResult { - alt(( - "2400", - take(4usize), - )) - .map(|time_str| match time_str { - "2400" => Ok(MAX_TIME_INSTANT), - _ => NaiveTime::parse_from_str(time_str, "%H%M"), - }) - .verify(|time| time.is_ok()) - .map(|time| time.unwrap()) - .parse_next(input) + alt(("2400", take(4usize))) + .map(|time_str| match time_str { + "2400" => Ok(MAX_TIME_INSTANT), + _ => NaiveTime::parse_from_str(time_str, "%H%M"), + }) + .verify(|time| time.is_ok()) + .map(|time| time.unwrap()) + .parse_next(input) } fn time_range_parser<'s>(input: &mut &'s str) -> PResult { - let (start_time, end_time) = separated_pair(time_parser, "-", time_parser).parse_next(input)?; - - Ok(ScheduleDayKind::TimeRange(start_time, end_time)) + seq!( + time_parser, + _: "-", + time_parser, + ) + .map(|s| ScheduleDayKind::TimeRange(s.0, s.1)) + .parse_next(input) } fn schedule_day_kind_parser<'s>(input: &mut &'s str) -> PResult { From d7ff65b36880a1007959c9b1b41fed95c78b4a75 Mon Sep 17 00:00:00 2001 From: keyvan Date: Thu, 11 Apr 2024 12:34:20 -0700 Subject: [PATCH 21/27] refactor: improve parser --- src/agent/market_schedule.rs | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/src/agent/market_schedule.rs b/src/agent/market_schedule.rs index bf9f3adf..5ee46a13 100644 --- a/src/agent/market_schedule.rs +++ b/src/agent/market_schedule.rs @@ -16,6 +16,7 @@ use { chrono_tz::Tz, std::str::FromStr, winnow::{ + ascii::dec_uint, combinator::{ alt, separated, @@ -109,18 +110,25 @@ pub struct HolidayDaySchedule { } -fn holiday_day_schedule_parser<'s>(input: &mut &'s str) -> PResult { - let ((month_str, day_str), kind) = - separated_pair((take(2usize), take(2usize)), "/", schedule_day_kind_parser) - .parse_next(input)?; +fn two_digit_parser<'s>(input: &mut &'s str) -> PResult { + take(2usize) + .verify_map(|s| u32::from_str(s).ok()) + .parse_next(input) +} +fn holiday_day_schedule_parser<'s>(input: &mut &'s str) -> PResult { // day and month are not validated to be correct dates // if they are invalid, it will be ignored since there // are no real dates that match the invalid input - let month = month_str.parse::().unwrap(); - let day = day_str.parse::().unwrap(); - - Ok(HolidayDaySchedule { month, day, kind }) + seq!( + HolidayDaySchedule { + month: two_digit_parser, + day: two_digit_parser, + _: "/", + kind: schedule_day_kind_parser, + } + ) + .parse_next(input) } impl FromStr for HolidayDaySchedule { From 3d6120382f53929bf83845d9b712633999d113e5 Mon Sep 17 00:00:00 2001 From: keyvan Date: Thu, 11 Apr 2024 12:42:18 -0700 Subject: [PATCH 22/27] refactor: improve parser --- src/agent/market_schedule.rs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/agent/market_schedule.rs b/src/agent/market_schedule.rs index 5ee46a13..4987739e 100644 --- a/src/agent/market_schedule.rs +++ b/src/agent/market_schedule.rs @@ -83,7 +83,7 @@ impl MarketSchedule { fn market_schedule_parser<'s>(input: &mut &'s str) -> PResult { seq!( MarketSchedule { - timezone: take_till(0.., ';').map(|s| Tz::from_str(s)).verify(|s| s.is_ok()).map(|s| s.unwrap()), + timezone: take_till(0.., ';').verify_map(|s| Tz::from_str(s).ok()), _: ';', weekly_schedule: separated(7, schedule_day_kind_parser, ","), _: ';', @@ -102,6 +102,7 @@ impl FromStr for MarketSchedule { } } + #[derive(Clone, Debug, Eq, PartialEq)] pub struct HolidayDaySchedule { pub month: u32, @@ -109,7 +110,6 @@ pub struct HolidayDaySchedule { pub kind: ScheduleDayKind, } - fn two_digit_parser<'s>(input: &mut &'s str) -> PResult { take(2usize) .verify_map(|s| u32::from_str(s).ok()) @@ -140,6 +140,7 @@ impl FromStr for HolidayDaySchedule { } } + #[derive(Clone, Debug, Eq, PartialEq)] pub enum ScheduleDayKind { Open, @@ -173,12 +174,10 @@ impl Default for ScheduleDayKind { fn time_parser<'s>(input: &mut &'s str) -> PResult { alt(("2400", take(4usize))) - .map(|time_str| match time_str { - "2400" => Ok(MAX_TIME_INSTANT), - _ => NaiveTime::parse_from_str(time_str, "%H%M"), + .verify_map(|time_str| match time_str { + "2400" => Some(MAX_TIME_INSTANT), + _ => NaiveTime::parse_from_str(time_str, "%H%M").ok(), }) - .verify(|time| time.is_ok()) - .map(|time| time.unwrap()) .parse_next(input) } From 091074802ad2474afad6206c8f72a8bc77e1b23c Mon Sep 17 00:00:00 2001 From: keyvan Date: Thu, 11 Apr 2024 12:48:58 -0700 Subject: [PATCH 23/27] refactor: implement from trait --- src/agent/market_schedule.rs | 40 ++++++++++++++++++++++++++++-------- src/agent/solana/oracle.rs | 14 +------------ 2 files changed, 32 insertions(+), 22 deletions(-) diff --git a/src/agent/market_schedule.rs b/src/agent/market_schedule.rs index 4987739e..f0f6f6c1 100644 --- a/src/agent/market_schedule.rs +++ b/src/agent/market_schedule.rs @@ -1,7 +1,10 @@ //! Holiday hours metadata parsing and evaluation logic use { - super::legacy_schedule::MHKind, + super::legacy_schedule::{ + LegacySchedule, + MHKind, + }, anyhow::{ anyhow, Result, @@ -102,6 +105,24 @@ impl FromStr for MarketSchedule { } } +impl From for MarketSchedule { + fn from(legacy: LegacySchedule) -> Self { + Self { + timezone: legacy.timezone, + weekly_schedule: vec![ + legacy.mon.into(), + legacy.tue.into(), + legacy.wed.into(), + legacy.thu.into(), + legacy.fri.into(), + legacy.sat.into(), + legacy.sun.into(), + ], + holidays: vec![], + } + } +} + #[derive(Clone, Debug, Eq, PartialEq)] pub struct HolidayDaySchedule { @@ -156,14 +177,6 @@ impl ScheduleDayKind { Self::TimeRange(start, end) => start <= &when_local && &when_local <= end, } } - - pub fn from_mhkind(mhkind: MHKind) -> Self { - match mhkind { - MHKind::Open => ScheduleDayKind::Open, - MHKind::Closed => ScheduleDayKind::Closed, - MHKind::TimeRange(start, end) => ScheduleDayKind::TimeRange(start, end), - } - } } impl Default for ScheduleDayKind { @@ -209,6 +222,15 @@ impl FromStr for ScheduleDayKind { } } +impl From for ScheduleDayKind { + fn from(mhkind: MHKind) -> Self { + match mhkind { + MHKind::Open => ScheduleDayKind::Open, + MHKind::Closed => ScheduleDayKind::Closed, + MHKind::TimeRange(start, end) => ScheduleDayKind::TimeRange(start, end), + } + } +} #[cfg(test)] mod tests { diff --git a/src/agent/solana/oracle.rs b/src/agent/solana/oracle.rs index 1e798fbb..57027ec3 100644 --- a/src/agent/solana/oracle.rs +++ b/src/agent/solana/oracle.rs @@ -656,19 +656,7 @@ impl Poller { *product_key, ProductEntry { account_data: *product, - schedule: market_schedule.unwrap_or_else(|| MarketSchedule { - timezone: legacy_schedule.timezone, - weekly_schedule: vec![ - ScheduleDayKind::from_mhkind(legacy_schedule.mon), - ScheduleDayKind::from_mhkind(legacy_schedule.tue), - ScheduleDayKind::from_mhkind(legacy_schedule.wed), - ScheduleDayKind::from_mhkind(legacy_schedule.thu), - ScheduleDayKind::from_mhkind(legacy_schedule.fri), - ScheduleDayKind::from_mhkind(legacy_schedule.sat), - ScheduleDayKind::from_mhkind(legacy_schedule.sun), - ], - holidays: vec![], - }), + schedule: market_schedule.unwrap_or_else(|| legacy_schedule.into()), price_accounts: vec![], }, ); From c806e92c99b112fb26d7a08d06e7e05148bc6133 Mon Sep 17 00:00:00 2001 From: keyvan Date: Thu, 11 Apr 2024 14:03:16 -0700 Subject: [PATCH 24/27] feat: add proptest --- Cargo.lock | 72 ++++++++- Cargo.toml | 1 + .../agent/market_schedule.txt | 8 + src/agent/market_schedule.rs | 144 +++++++++++++++++- 4 files changed, 220 insertions(+), 5 deletions(-) create mode 100644 proptest-regressions/agent/market_schedule.txt diff --git a/Cargo.lock b/Cargo.lock index e69c2579..95876a46 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2363,6 +2363,12 @@ version = "0.2.153" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" +[[package]] +name = "libm" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" + [[package]] name = "libredox" version = "0.0.1" @@ -2715,6 +2721,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a" dependencies = [ "autocfg 1.2.0", + "libm", ] [[package]] @@ -3241,6 +3248,26 @@ dependencies = [ "syn 2.0.55", ] +[[package]] +name = "proptest" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31b476131c3c86cb68032fdc5cb6d5a1045e3e42d96b69fa599fd77701e1f5bf" +dependencies = [ + "bit-set", + "bit-vec", + "bitflags 2.5.0", + "lazy_static", + "num-traits", + "rand 0.8.5", + "rand_chacha 0.3.1", + "rand_xorshift 0.3.0", + "regex-syntax 0.8.3", + "rusty-fork", + "tempfile", + "unarray", +] + [[package]] name = "pyth-agent" version = "2.5.2" @@ -3261,6 +3288,7 @@ dependencies = [ "parking_lot", "portpicker", "prometheus-client", + "proptest", "pyth-sdk", "pyth-sdk-solana", "rand 0.8.5", @@ -3339,6 +3367,12 @@ dependencies = [ "syn 2.0.55", ] +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + [[package]] name = "quinn" version = "0.10.2" @@ -3411,7 +3445,7 @@ dependencies = [ "rand_jitter", "rand_os", "rand_pcg", - "rand_xorshift", + "rand_xorshift 0.1.1", "winapi", ] @@ -3573,6 +3607,15 @@ dependencies = [ "rand_core 0.3.1", ] +[[package]] +name = "rand_xorshift" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d25bf25ec5ae4a3f1b92f929810509a2f53d7dca2f50b794ff57e3face536c8f" +dependencies = [ + "rand_core 0.6.4", +] + [[package]] name = "rand_xoshiro" version = "0.6.0" @@ -3915,6 +3958,18 @@ version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" +[[package]] +name = "rusty-fork" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb3dcc6e454c328bb824492db107ab7c0ae8fcffe4ad210136ef014458c1bc4f" +dependencies = [ + "fnv", + "quick-error", + "tempfile", + "wait-timeout", +] + [[package]] name = "ryu" version = "1.0.17" @@ -5942,6 +5997,12 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed646292ffc8188ef8ea4d1e0e0150fb15a5c2e12ad9b8fc191ae7a8a7f3c4b9" +[[package]] +name = "unarray" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + [[package]] name = "unicase" version = "2.7.0" @@ -6078,6 +6139,15 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" +[[package]] +name = "wait-timeout" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6" +dependencies = [ + "libc", +] + [[package]] name = "want" version = "0.3.1" diff --git a/Cargo.toml b/Cargo.toml index 702974b1..5732ad82 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -53,6 +53,7 @@ lazy_static = "1.4.0" toml_edit = "0.22.9" slog-bunyan = "2.5.0" winnow = "0.6.5" +proptest = "1.4.0" [dev-dependencies] tokio-util = { version = "0.7.10", features = ["full"] } diff --git a/proptest-regressions/agent/market_schedule.txt b/proptest-regressions/agent/market_schedule.txt new file mode 100644 index 00000000..5d866125 --- /dev/null +++ b/proptest-regressions/agent/market_schedule.txt @@ -0,0 +1,8 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc 173b9a862e3ad1149b0fdef292a11164ecab5b67b395857178f63294c3c9c0b7 # shrinks to s = "0000-0060" +cc 6cf32e18287cb6de4b40f4326d1e9fd3be409086af3ccf75eac6f980c1f67052 # shrinks to s = TimeRange(00:00:00, 00:00:01) diff --git a/src/agent/market_schedule.rs b/src/agent/market_schedule.rs index f0f6f6c1..d67ab3d0 100644 --- a/src/agent/market_schedule.rs +++ b/src/agent/market_schedule.rs @@ -17,13 +17,19 @@ use { Utc, }, chrono_tz::Tz, - std::str::FromStr, + proptest::{ + arbitrary::any, + prop_compose, + proptest, + }, + std::{ + fmt::Display, + str::FromStr, + }, winnow::{ - ascii::dec_uint, combinator::{ alt, separated, - separated_pair, seq, }, stream::ToUsize, @@ -63,6 +69,26 @@ impl Default for MarketSchedule { } } +impl Display for MarketSchedule { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{};", self.timezone)?; + for (i, day) in self.weekly_schedule.iter().enumerate() { + write!(f, "{}", day)?; + if i < 6 { + write!(f, ",")?; + } + } + write!(f, ";")?; + for (i, holiday) in self.holidays.iter().enumerate() { + write!(f, "{}", holiday)?; + if i < self.holidays.len() - 1 { + write!(f, ",")?; + } + } + Ok(()) + } +} + impl MarketSchedule { pub fn can_publish_at(&self, when: &DateTime) -> bool { let when_local = when.with_timezone(&self.timezone); @@ -161,8 +187,14 @@ impl FromStr for HolidayDaySchedule { } } +impl Display for HolidayDaySchedule { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:02}{:02}/{}", self.month, self.day, self.kind) + } +} -#[derive(Clone, Debug, Eq, PartialEq)] + +#[derive(Clone, Debug, Eq, PartialEq, Copy)] pub enum ScheduleDayKind { Open, Closed, @@ -185,6 +217,18 @@ impl Default for ScheduleDayKind { } } +impl Display for ScheduleDayKind { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Open => write!(f, "O"), + Self::Closed => write!(f, "C"), + Self::TimeRange(start, end) => { + write!(f, "{}-{}", start.format("%H%M"), end.format("%H%M")) + } + } + } +} + fn time_parser<'s>(input: &mut &'s str) -> PResult { alt(("2400", take(4usize))) .verify_map(|time_str| match time_str { @@ -421,3 +465,95 @@ mod tests { Ok(()) } } + +prop_compose! { + fn schedule_day_kind()( + r in any::(), + t1 in any::(), + t2 in any::(), + ) -> ScheduleDayKind { + match r % 3 { + 0 => ScheduleDayKind::Open, + 1 => ScheduleDayKind::Closed, + _ => ScheduleDayKind::TimeRange( + NaiveTime::from_hms_opt(t1 % 24, t1 / 24 % 60, 0).unwrap(), + NaiveTime::from_hms_opt(t2 % 24, t2 / 24 % 60, 0).unwrap(), + ), + } + } +} + +prop_compose! { + fn holiday_day_schedule()( + m in 1..=12u32, + d in 1..=31u32, + s in schedule_day_kind(), + ) -> HolidayDaySchedule { + HolidayDaySchedule { + month: m, + day: d, + kind: s, + } + } +} + +prop_compose! { + fn market_schedule()( + tz in proptest::sample::select(vec![ + Tz::UTC, + Tz::America__New_York, + Tz::America__Los_Angeles, + Tz::America__Chicago, + Tz::Singapore, + Tz::Australia__Sydney, + ]), + weekly_schedule in proptest::collection::vec(schedule_day_kind(), 7..=7), + holidays in proptest::collection::vec(holiday_day_schedule(), 0..12), + ) -> MarketSchedule { + MarketSchedule { + timezone: tz, + weekly_schedule, + holidays, + } + } +} + +const VALID_SCHEDULE_DAY_KIND_REGEX: &str = + "C|O|(0[1-9]|1[0-2])([0-5][0-9])-(0[1-9]|1[0-2])([0-5][0-9])"; + +const VALID_MONTH_DAY_REGEX: &str = "(0[1-9]|1[0-2])(0[1-9]|[12][0-9]|3[01])"; + +proptest!( + #[test] + fn doesnt_crash(s in "\\PC*") { + _ = s.parse::(); + _ = s.parse::(); + _ = s.parse::(); + } + + #[test] + fn parse_valid_schedule_day_kind(s in VALID_SCHEDULE_DAY_KIND_REGEX) { + assert!(s.parse::().is_ok()); + } + + #[test] + fn test_valid_schedule_day_kind(s in schedule_day_kind()) { + assert_eq!(s, s.to_string().parse::().unwrap()); + } + + #[test] + fn parse_valid_holiday_day_schedule(s in VALID_SCHEDULE_DAY_KIND_REGEX, d in VALID_MONTH_DAY_REGEX) { + let valid_holiday_day = format!("{}/{}", d, s); + assert!(valid_holiday_day.parse::().is_ok()); + } + + #[test] + fn test_valid_holiday_day_schedule(s in holiday_day_schedule()) { + assert_eq!(s, s.to_string().parse::().unwrap()); + } + + #[test] + fn test_valid_market_schedule(s in market_schedule()) { + assert_eq!(s, s.to_string().parse::().unwrap()); + } +); From 28b2d16dd6aeac46561466e982d18027f9fdfe7e Mon Sep 17 00:00:00 2001 From: keyvan Date: Fri, 12 Apr 2024 09:04:42 -0700 Subject: [PATCH 25/27] fix: day kind regex and add comments --- src/agent/market_schedule.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/agent/market_schedule.rs b/src/agent/market_schedule.rs index d67ab3d0..e55c799c 100644 --- a/src/agent/market_schedule.rs +++ b/src/agent/market_schedule.rs @@ -518,9 +518,11 @@ prop_compose! { } } +// Matches C or O or hhmm-hhmm with 24-hour time const VALID_SCHEDULE_DAY_KIND_REGEX: &str = - "C|O|(0[1-9]|1[0-2])([0-5][0-9])-(0[1-9]|1[0-2])([0-5][0-9])"; + "C|O|([01][1-9]|2[0-3])([0-5][0-9])-([01][1-9]|2[0-3])([0-5][0-9])"; +// Matches MMDD with MM and DD being 01-12 and 01-31 respectively const VALID_MONTH_DAY_REGEX: &str = "(0[1-9]|1[0-2])(0[1-9]|[12][0-9]|3[01])"; proptest!( From 06bd95415d8e421d03c117aea635e4938e8df294 Mon Sep 17 00:00:00 2001 From: keyvan Date: Fri, 12 Apr 2024 09:50:16 -0700 Subject: [PATCH 26/27] refactor: improve comment --- src/agent/legacy_schedule.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/agent/legacy_schedule.rs b/src/agent/legacy_schedule.rs index 4aeabec2..83600216 100644 --- a/src/agent/legacy_schedule.rs +++ b/src/agent/legacy_schedule.rs @@ -28,7 +28,7 @@ lazy_static! { } /// Weekly market hours schedule -/// TODO: Remove after the migration +/// TODO: Remove after all publishers have upgraded to support the new schedule format #[derive(Clone, Default, Debug, Eq, PartialEq)] #[deprecated(note = "This struct is deprecated, use MarketSchedule instead.")] pub struct LegacySchedule { From 80f2b1463795ff090c9ce7445104a5f80f99797f Mon Sep 17 00:00:00 2001 From: keyvan Date: Fri, 12 Apr 2024 10:04:44 -0700 Subject: [PATCH 27/27] chore: increase pyth agent minor version --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 95876a46..cd1f188c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3270,7 +3270,7 @@ dependencies = [ [[package]] name = "pyth-agent" -version = "2.5.2" +version = "2.6.0" dependencies = [ "anyhow", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index 5732ad82..8c28965c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyth-agent" -version = "2.5.2" +version = "2.6.0" edition = "2021" [[bin]]