diff --git a/program/c/src/oracle/oracle.h b/program/c/src/oracle/oracle.h index 097cbf0f3..dc7cfc5ad 100644 --- a/program/c/src/oracle/oracle.h +++ b/program/c/src/oracle/oracle.h @@ -9,9 +9,9 @@ extern "C" { // The size of the "time machine" account defined in the // Rust portion of the codebase. -const uint64_t TIME_MACHINE_STRUCT_SIZE = 1864ULL; +const uint64_t TIME_MACHINE_STRUCT_SIZE = 1200ULL; -const uint64_t EXTRA_PUBLISHER_SPACE = 1000ULL; +const uint64_t EXTRA_PUBLISHER_SPACE = 3072ULL; // magic number at head of account diff --git a/program/rust/Cargo.toml b/program/rust/Cargo.toml index bdbc266ea..67c0ef6fd 100644 --- a/program/rust/Cargo.toml +++ b/program/rust/Cargo.toml @@ -21,6 +21,8 @@ solana-program-test = "=1.10.29" solana-sdk = "=1.10.29" tokio = "1.14.1" hex = "0.3.1" +quickcheck = "1" +quickcheck_macros = "1" [features] debug = [] diff --git a/program/rust/src/tests/mod.rs b/program/rust/src/tests/mod.rs index 120079ebc..b2fa4f277 100644 --- a/program/rust/src/tests/mod.rs +++ b/program/rust/src/tests/mod.rs @@ -11,9 +11,12 @@ mod test_init_price; mod test_resize_account; mod test_set_min_pub; mod test_sizes; +mod test_sma; +mod test_sma_epoch_transition; mod test_upd_aggregate; mod test_upd_permissions; mod test_upd_price; mod test_upd_price_no_fail_on_error; mod test_upd_product; +mod test_upd_sma; mod test_utils; diff --git a/program/rust/src/tests/test_resize_account.rs b/program/rust/src/tests/test_resize_account.rs index a1d5b46bb..15c598173 100644 --- a/program/rust/src/tests/test_resize_account.rs +++ b/program/rust/src/tests/test_resize_account.rs @@ -28,4 +28,10 @@ async fn test_resize_account() { assert!(sim.resize_price_account(&price1).await.is_ok()); let price1_account = sim.get_account(price1.pubkey()).await.unwrap(); assert_eq!(price1_account.data.len(), size_of::()); + let price1_account_data = sim + .get_account_data_as::(price1.pubkey()) + .await + .unwrap(); + assert_eq!(price1_account_data.time_machine.granularity, 0); + assert_eq!(price1_account_data.time_machine.threshold, 0); } diff --git a/program/rust/src/tests/test_sma.rs b/program/rust/src/tests/test_sma.rs new file mode 100644 index 000000000..2019fd441 --- /dev/null +++ b/program/rust/src/tests/test_sma.rs @@ -0,0 +1,179 @@ +use quickcheck::Arbitrary; +use quickcheck_macros::quickcheck; + +use crate::time_machine_types::{ + DataPoint, + SmaTracker, + NUM_BUCKETS_THIRTY_MIN, +}; + +#[derive(Clone, Debug, Copy)] +struct DataEvent { + time_gap: i64, + slot_gap: u64, + price: i64, +} + +impl Arbitrary for DataEvent { + fn arbitrary(g: &mut quickcheck::Gen) -> Self { + DataEvent { + time_gap: i64::from(u8::arbitrary(g)), + slot_gap: u64::from(u8::arbitrary(g)) + 1, /* Slot gap is always > 1, because there + * has been a succesful aggregation */ + price: i64::arbitrary(g), + } + } +} + +/// This is a generative test for the sma struct. quickcheck will generate a series of +/// vectors of DataEvents of different length. The generation is based on the arbitrary trait +/// above. +/// For each DataEvent : +/// - time_gap is a random number between 0 and u8::MAX (255) +/// - slot_gap is a random number between 1 and u8::MAX + 1 (256) +/// - price is a random i64 +#[quickcheck] +fn test_sma(input: Vec) -> bool { + // No gaps, no skipped epochs + let mut tracker1 = SmaTracker::::zero(); + tracker1.initialize(i64::from(u8::MAX), u64::from(u8::MAX)); + + // Skipped and gaps + let mut tracker2 = SmaTracker::::zero(); + tracker2.initialize(i64::from(u8::MAX / 5), u64::from(u8::MAX / 5)); + + // Gaps, no skips + let mut tracker3 = SmaTracker::::zero(); + tracker3.initialize(i64::from(u8::MAX), u64::from(u8::MAX / 5)); + + // No skips, gaps + let mut tracker4 = SmaTracker::::zero(); + tracker4.initialize(i64::from(u8::MAX), u64::from(u8::MAX / 5) * 4); + + // Each epoch is 1 second + let mut tracker5 = SmaTracker::::zero(); + tracker5.initialize(1, u64::from(u8::MAX / 5)); + + let mut data = Vec::::new(); + + let mut current_time = 0i64; + for data_event in input.clone() { + let datapoint = DataPoint { + previous_timestamp: current_time, + current_timestamp: current_time + data_event.time_gap, + slot_gap: data_event.slot_gap, + price: data_event.price, + }; + + tracker1.add_datapoint(&datapoint).unwrap(); + tracker2.add_datapoint(&datapoint).unwrap(); + tracker3.add_datapoint(&datapoint).unwrap(); + tracker4.add_datapoint(&datapoint).unwrap(); + tracker5.add_datapoint(&datapoint).unwrap(); + data.push(datapoint); + current_time += data_event.time_gap; + + tracker1.check_current_epoch_fields(&data, current_time); + tracker2.check_current_epoch_fields(&data, current_time); + tracker3.check_current_epoch_fields(&data, current_time); + tracker4.check_current_epoch_fields(&data, current_time); + tracker5.check_current_epoch_fields(&data, current_time); + tracker1.check_array_fields(&data, current_time); + tracker2.check_array_fields(&data, current_time); + tracker3.check_array_fields(&data, current_time); + tracker4.check_array_fields(&data, current_time); + tracker5.check_array_fields(&data, current_time); + } + + return true; +} + + +impl SmaTracker { + pub fn zero() -> Self { + return SmaTracker:: { + granularity: 0, + threshold: 0, + current_epoch_denominator: 0, + current_epoch_is_valid: false, + current_epoch_numerator: 0, + running_valid_epoch_counter: [0u64; NUM_ENTRIES], + running_sum_of_price_averages: [0i128; NUM_ENTRIES], + }; + } + + pub fn check_current_epoch_fields(&self, data: &Vec, time: i64) { + let curent_epoch = self.time_to_epoch(time).unwrap(); + + let result = self.compute_epoch_expected_values(data, curent_epoch); + assert_eq!(self.current_epoch_denominator, result.0); + assert_eq!(self.current_epoch_numerator, result.1); + assert_eq!(self.current_epoch_is_valid, result.2); + } + + pub fn check_array_fields(&self, data: &Vec, time: i64) { + let current_epoch = self.time_to_epoch(time).unwrap(); + let mut values = vec![]; + + // Compute all epoch averages + for i in 0..current_epoch { + values.push(self.compute_epoch_expected_values(data, i)); + } + + // Get running sums + let running_sum_price_iter = values.iter().scan((0, 0), |res, &y| { + res.0 = res.0 + y.1 / i128::from(y.0); + res.1 = res.1 + u64::from(y.2); + Some(*res) + }); + + // Compare to running_sum_of_price_averages + let mut i = (current_epoch + NUM_ENTRIES - 1).rem_euclid(NUM_ENTRIES); + for x in running_sum_price_iter + .collect::>() + .iter() + .rev() + .take(NUM_ENTRIES) + { + assert_eq!(self.running_sum_of_price_averages[i], x.0); + assert_eq!(self.running_valid_epoch_counter[i], x.1); + i = (i + NUM_ENTRIES - 1).rem_euclid(NUM_ENTRIES); + } + } + + pub fn compute_epoch_expected_values( + &self, + data: &Vec, + epoch_number: usize, + ) -> (u64, i128, bool) { + let left_bound = self + .granularity + .checked_mul(epoch_number.try_into().unwrap()) + .unwrap(); + + let right_bound = self + .granularity + .checked_mul((epoch_number + 1).try_into().unwrap()) + .unwrap(); + + + let mut result = data.iter().fold((0, 0, true), |x: (u64, i128, bool), y| { + if !((left_bound > y.current_timestamp) || (right_bound <= y.previous_timestamp)) + //Check interval intersection + { + let is_valid = y.slot_gap <= self.threshold; + return ( + x.0 + y.slot_gap, + x.1 + i128::from(y.slot_gap) * i128::from(y.price), + x.2 && is_valid, + ); + } + return x; + }); + + if epoch_number == 0 { + result.2 = false; + } + return result; + } +} diff --git a/program/rust/src/tests/test_sma_epoch_transition.rs b/program/rust/src/tests/test_sma_epoch_transition.rs new file mode 100644 index 000000000..fc022c61c --- /dev/null +++ b/program/rust/src/tests/test_sma_epoch_transition.rs @@ -0,0 +1,459 @@ +use solana_program::pubkey::Pubkey; +use std::mem::size_of; + +use crate::c_oracle_header::{ + PC_MAX_SEND_LATENCY, + PC_STATUS_TRADING, + PC_VERSION, +}; + +use crate::deserialize::{ + initialize_pyth_account_checked, + load_checked, + load_mut, +}; +use crate::instruction::{ + OracleCommand, + UpdPriceArgs, +}; +use crate::processor::process_instruction; +use crate::tests::test_utils::{ + update_clock_slot, + update_clock_timestamp, + AccountSetup, +}; +use crate::time_machine_types::{ + PriceAccountWrapper, + NUM_BUCKETS_THIRTY_MIN, + THIRTY_MINUTES, +}; + +/// Manually test some epoch transitions +#[test] +fn test_sma_epoch_transition() { + let mut instruction_data = [0u8; size_of::()]; + + let program_id = Pubkey::new_unique(); + + let mut funding_setup = AccountSetup::new_funding(); + let funding_account = funding_setup.to_account_info(); + + let mut price_setup = AccountSetup::new::(&program_id); + let mut price_account = price_setup.to_account_info(); + price_account.is_signer = false; + initialize_pyth_account_checked::(&price_account, PC_VERSION).unwrap(); + + + { + let mut price_data = + load_checked::(&price_account, PC_VERSION).unwrap(); + price_data + .time_machine + .initialize(THIRTY_MINUTES, PC_MAX_SEND_LATENCY as u64); + + price_data.price_data.num_ = 1; + price_data.price_data.comp_[0].pub_ = *funding_account.key; + } + + let mut clock_setup = AccountSetup::new_clock(); + let mut clock_account = clock_setup.to_account_info(); + clock_account.is_signer = false; + clock_account.is_writable = false; + + update_clock_slot(&mut clock_account, 1); + update_clock_timestamp(&mut clock_account, 1); + populate_instruction(&mut instruction_data, 42, 2, 1); + + assert!(process_instruction( + &program_id, + &[ + funding_account.clone(), + price_account.clone(), + clock_account.clone() + ], + &instruction_data + ) + .is_ok()); + + { + let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); + + // No successful aggregation yet, so everything is 0 + assert_eq!( + price_data.time_machine.threshold, + PC_MAX_SEND_LATENCY as u64 + ); + assert_eq!(price_data.time_machine.granularity, THIRTY_MINUTES); + assert_eq!(price_data.time_machine.current_epoch_numerator, 0); + assert_eq!(price_data.time_machine.current_epoch_is_valid, false); + assert_eq!(price_data.time_machine.current_epoch_denominator, 0); + for i in 0..NUM_BUCKETS_THIRTY_MIN { + assert_eq!(price_data.time_machine.running_sum_of_price_averages[i], 0); + assert_eq!(price_data.time_machine.running_valid_epoch_counter[i], 0); + } + } + + // Same epoch, valid slot gap + update_clock_slot(&mut clock_account, 2); + update_clock_timestamp(&mut clock_account, 2); + populate_instruction(&mut instruction_data, 80, 2, 2); + + assert!(process_instruction( + &program_id, + &[ + funding_account.clone(), + price_account.clone(), + clock_account.clone() + ], + &instruction_data + ) + .is_ok()); + + { + let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); + // successful aggregation, price update is average between 0 and 42 + assert_eq!( + price_data.time_machine.threshold, + PC_MAX_SEND_LATENCY as u64 + ); + assert_eq!(price_data.time_machine.granularity, THIRTY_MINUTES); + assert_eq!(price_data.time_machine.current_epoch_numerator, 42 / 2 * 2); + assert_eq!(price_data.time_machine.current_epoch_is_valid, false); + assert_eq!(price_data.time_machine.current_epoch_denominator, 2); + for i in 0..NUM_BUCKETS_THIRTY_MIN { + assert_eq!(price_data.time_machine.running_sum_of_price_averages[i], 0); + assert_eq!(price_data.time_machine.running_valid_epoch_counter[i], 0); + } + } + + // Next epoch, valid slot gap + update_clock_slot(&mut clock_account, 3); + update_clock_timestamp(&mut clock_account, THIRTY_MINUTES); + populate_instruction(&mut instruction_data, 120, 1, 3); + + assert!(process_instruction( + &program_id, + &[ + funding_account.clone(), + price_account.clone(), + clock_account.clone() + ], + &instruction_data + ) + .is_ok()); + + { + let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); + + // Slot gap is valid, so successful aggregation and 1 epoch transition should happen + assert_eq!( + price_data.time_machine.threshold, + PC_MAX_SEND_LATENCY as u64 + ); + assert_eq!(price_data.time_machine.granularity, THIRTY_MINUTES); + assert_eq!( + price_data.time_machine.current_epoch_numerator, + (80 + 42) / 2 + ); + assert_eq!(price_data.time_machine.current_epoch_is_valid, true); + assert_eq!(price_data.time_machine.current_epoch_denominator, 1); + + for i in 1..NUM_BUCKETS_THIRTY_MIN { + assert_eq!(price_data.time_machine.running_sum_of_price_averages[i], 0); + assert_eq!(price_data.time_machine.running_valid_epoch_counter[i], 0); + } + + assert_eq!( + price_data.time_machine.running_sum_of_price_averages[0], + ((42 + (80 + 42) / 2) / 3) + ); + assert_eq!(price_data.time_machine.running_valid_epoch_counter[0], 0); + } + + // Same epoch, invalid slot gap + update_clock_slot(&mut clock_account, 30); + update_clock_timestamp(&mut clock_account, THIRTY_MINUTES + 1); + populate_instruction(&mut instruction_data, 40, 1, 30); + + // Unsuccessful aggregation + assert!(process_instruction( + &program_id, + &[ + funding_account.clone(), + price_account.clone(), + clock_account.clone() + ], + &instruction_data + ) + .is_ok()); + + { + let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); + // Slot gap is invalid, so aggregation didn't take place and smas are not updated + assert_eq!( + price_data.time_machine.threshold, + PC_MAX_SEND_LATENCY as u64 + ); + assert_eq!(price_data.time_machine.granularity, THIRTY_MINUTES); + assert_eq!( + price_data.time_machine.current_epoch_numerator, + (80 + 42) / 2 + ); + assert_eq!(price_data.time_machine.current_epoch_is_valid, true); + assert_eq!(price_data.time_machine.current_epoch_denominator, 1); + + for i in 1..NUM_BUCKETS_THIRTY_MIN { + assert_eq!(price_data.time_machine.running_sum_of_price_averages[i], 0); + assert_eq!(price_data.time_machine.running_valid_epoch_counter[i], 0); + } + + assert_eq!( + price_data.time_machine.running_sum_of_price_averages[0], + ((42 + (80 + 42) / 2) / 3) + ); + assert_eq!(price_data.time_machine.running_valid_epoch_counter[0], 0); + } + + // Next epoch, valid slot gap + update_clock_slot(&mut clock_account, 31); + update_clock_timestamp(&mut clock_account, 2 * THIRTY_MINUTES + 1); + populate_instruction(&mut instruction_data, 41, 1, 31); + + // Triggers aggregation! + assert!(process_instruction( + &program_id, + &[ + funding_account.clone(), + price_account.clone(), + clock_account.clone() + ], + &instruction_data + ) + .is_ok()); + + { + let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); + // Aggregation is successful and sma is computed, update is invalid because slot_gap from + // previous successful aggregation is big + assert_eq!( + price_data.time_machine.threshold, + PC_MAX_SEND_LATENCY as u64 + ); + assert_eq!(price_data.time_machine.granularity, THIRTY_MINUTES); + assert_eq!( + price_data.time_machine.current_epoch_numerator, + (40 + 80) / 2 * 28 + ); + assert_eq!(price_data.time_machine.current_epoch_is_valid, false); + assert_eq!(price_data.time_machine.current_epoch_denominator, 28); + + for i in 2..NUM_BUCKETS_THIRTY_MIN { + assert_eq!(price_data.time_machine.running_sum_of_price_averages[i], 0); + assert_eq!(price_data.time_machine.running_valid_epoch_counter[i], 0); + } + + assert_eq!( + price_data.time_machine.running_sum_of_price_averages[0], + ((42 + (80 + 42) / 2) / 3) + ); + assert_eq!(price_data.time_machine.running_valid_epoch_counter[0], 0); + + assert_eq!( + price_data.time_machine.running_sum_of_price_averages[1], + price_data.time_machine.running_sum_of_price_averages[0] + 60 + ); + assert_eq!(price_data.time_machine.running_valid_epoch_counter[1], 0); + } + + // A big gap in time (more than 1 epoch) but not in slots, all skipped epochs are "valid" + update_clock_slot(&mut clock_account, 32); + update_clock_timestamp(&mut clock_account, 5 * THIRTY_MINUTES + 1); + populate_instruction(&mut instruction_data, 30, 1, 32); + + // Triggers aggregation! + assert!(process_instruction( + &program_id, + &[ + funding_account.clone(), + price_account.clone(), + clock_account.clone() + ], + &instruction_data + ) + .is_ok()); + + { + let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); + // Aggregation was successful, check that all skipped buckets got updated + assert_eq!( + price_data.time_machine.threshold, + PC_MAX_SEND_LATENCY as u64 + ); + assert_eq!(price_data.time_machine.granularity, THIRTY_MINUTES); + assert_eq!( + price_data.time_machine.current_epoch_numerator, + (40 + 41) / 2 + ); + assert_eq!(price_data.time_machine.current_epoch_is_valid, true); + assert_eq!(price_data.time_machine.current_epoch_denominator, 1); + + for i in 5..NUM_BUCKETS_THIRTY_MIN { + assert_eq!(price_data.time_machine.running_sum_of_price_averages[i], 0); + assert_eq!(price_data.time_machine.running_valid_epoch_counter[i], 0); + } + + assert_eq!( + price_data.time_machine.running_sum_of_price_averages[0], + ((42 + (80 + 42) / 2) / 3) + ); + assert_eq!(price_data.time_machine.running_valid_epoch_counter[0], 0); + + assert_eq!( + price_data.time_machine.running_sum_of_price_averages[1], + price_data.time_machine.running_sum_of_price_averages[0] + 60 + ); + assert_eq!(price_data.time_machine.running_valid_epoch_counter[1], 0); + + assert_eq!( + price_data.time_machine.running_sum_of_price_averages[2], + price_data.time_machine.running_sum_of_price_averages[1] + (60 * 28 + 1 * 41) / 29 + ); + assert_eq!(price_data.time_machine.running_valid_epoch_counter[2], 0); + + assert_eq!( + price_data.time_machine.running_sum_of_price_averages[3], + price_data.time_machine.running_sum_of_price_averages[2] + 40 + ); + assert_eq!(price_data.time_machine.running_valid_epoch_counter[3], 1); + + assert_eq!( + price_data.time_machine.running_sum_of_price_averages[4], + price_data.time_machine.running_sum_of_price_averages[3] + 40 + ); + assert_eq!(price_data.time_machine.running_valid_epoch_counter[4], 2); + } + + // Really big gap both in slots and epochs (the entire buffers gets rewritten) + update_clock_slot(&mut clock_account, 100); + update_clock_timestamp(&mut clock_account, 100 * THIRTY_MINUTES + 1); + populate_instruction(&mut instruction_data, 100, 1, 100); + + // Unsuccessful aggregation + assert!(process_instruction( + &program_id, + &[ + funding_account.clone(), + price_account.clone(), + clock_account.clone() + ], + &instruction_data + ) + .is_ok()); + + { + let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); + // Nothing got updated, since slot gap was too big, so aggregation was not successful + + assert_eq!( + price_data.time_machine.threshold, + PC_MAX_SEND_LATENCY as u64 + ); + assert_eq!(price_data.time_machine.granularity, THIRTY_MINUTES); + assert_eq!( + price_data.time_machine.current_epoch_numerator, + (40 + 41) / 2 + ); + assert_eq!(price_data.time_machine.current_epoch_is_valid, true); + assert_eq!(price_data.time_machine.current_epoch_denominator, 1); + + for i in 5..NUM_BUCKETS_THIRTY_MIN { + assert_eq!(price_data.time_machine.running_sum_of_price_averages[i], 0); + assert_eq!(price_data.time_machine.running_valid_epoch_counter[i], 0); + } + + assert_eq!( + price_data.time_machine.running_sum_of_price_averages[0], + ((42 + (80 + 42) / 2) / 3) + ); + assert_eq!(price_data.time_machine.running_valid_epoch_counter[0], 0); + + assert_eq!( + price_data.time_machine.running_sum_of_price_averages[1], + price_data.time_machine.running_sum_of_price_averages[0] + 60 + ); + assert_eq!(price_data.time_machine.running_valid_epoch_counter[1], 0); + + assert_eq!( + price_data.time_machine.running_sum_of_price_averages[2], + price_data.time_machine.running_sum_of_price_averages[1] + (60 * 28 + 1 * 41) / 29 + ); + assert_eq!(price_data.time_machine.running_valid_epoch_counter[2], 0); + + assert_eq!( + price_data.time_machine.running_sum_of_price_averages[3], + price_data.time_machine.running_sum_of_price_averages[2] + 40 + ); + assert_eq!(price_data.time_machine.running_valid_epoch_counter[3], 1); + + assert_eq!( + price_data.time_machine.running_sum_of_price_averages[4], + price_data.time_machine.running_sum_of_price_averages[3] + 40 + ); + assert_eq!(price_data.time_machine.running_valid_epoch_counter[4], 2); + } + + update_clock_slot(&mut clock_account, 101); + update_clock_timestamp(&mut clock_account, 100 * THIRTY_MINUTES + 2); + populate_instruction(&mut instruction_data, 100, 1, 101); + + // Aggregation triggered + assert!(process_instruction( + &program_id, + &[ + funding_account.clone(), + price_account.clone(), + clock_account.clone() + ], + &instruction_data + ) + .is_ok()); + + { + let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); + // The entire buffer got rewritten + assert_eq!( + price_data.time_machine.threshold, + PC_MAX_SEND_LATENCY as u64 + ); + assert_eq!(price_data.time_machine.granularity, THIRTY_MINUTES); + assert_eq!( + price_data.time_machine.current_epoch_numerator, + (41 + 100) / 2 * 69 + ); + assert_eq!(price_data.time_machine.current_epoch_is_valid, false); + assert_eq!(price_data.time_machine.current_epoch_denominator, 69); + for i in 0..NUM_BUCKETS_THIRTY_MIN { + assert_eq!( + price_data.time_machine.running_sum_of_price_averages + [(i + 4) % NUM_BUCKETS_THIRTY_MIN], + 233 + 69 + 70 * (NUM_BUCKETS_THIRTY_MIN as i128 - 1) + 70 * i as i128 + ); + assert_eq!( + price_data.time_machine.running_valid_epoch_counter + [(i + 4) % NUM_BUCKETS_THIRTY_MIN], + 2 + ); + } + } +} + +// Create an upd_price instruction with the provided parameters +fn populate_instruction(instruction_data: &mut [u8], price: i64, conf: u64, pub_slot: u64) -> () { + let mut cmd = load_mut::(instruction_data).unwrap(); + cmd.header = OracleCommand::UpdPrice.into(); + cmd.status = PC_STATUS_TRADING; + cmd.price = price; + cmd.confidence = conf; + cmd.publishing_slot = pub_slot; + cmd.unused_ = 0; +} diff --git a/program/rust/src/tests/test_upd_sma.rs b/program/rust/src/tests/test_upd_sma.rs new file mode 100644 index 000000000..044d4b62f --- /dev/null +++ b/program/rust/src/tests/test_upd_sma.rs @@ -0,0 +1,376 @@ +use solana_program::program_error::ProgramError; +use solana_program::pubkey::Pubkey; +use std::mem::size_of; + +use crate::c_oracle_header::{ + PC_MAX_SEND_LATENCY, + PC_STATUS_TRADING, + PC_STATUS_UNKNOWN, + PC_VERSION, +}; + +use crate::deserialize::{ + initialize_pyth_account_checked, + load_checked, + load_mut, +}; +use crate::instruction::{ + OracleCommand, + UpdPriceArgs, +}; +use crate::processor::process_instruction; +// use crate::processor::process_instruction; +use crate::tests::test_utils::{ + update_clock_slot, + AccountSetup, +}; +use crate::time_machine_types::{ + PriceAccountWrapper, + THIRTY_MINUTES, +}; + +/// Clone of test_upd_price that also checks sma fields +#[test] +fn test_upd_sma() { + let mut instruction_data = [0u8; size_of::()]; + populate_instruction(&mut instruction_data, 42, 2, 1); + + let program_id = Pubkey::new_unique(); + + let mut funding_setup = AccountSetup::new_funding(); + let funding_account = funding_setup.to_account_info(); + + let mut price_setup = AccountSetup::new::(&program_id); + let mut price_account = price_setup.to_account_info(); + price_account.is_signer = false; + initialize_pyth_account_checked::(&price_account, PC_VERSION).unwrap(); + + + { + let mut price_data = + load_checked::(&price_account, PC_VERSION).unwrap(); + price_data + .time_machine + .initialize(THIRTY_MINUTES, PC_MAX_SEND_LATENCY as u64); + + price_data.price_data.num_ = 1; + price_data.price_data.comp_[0].pub_ = *funding_account.key; + } + + let mut clock_setup = AccountSetup::new_clock(); + let mut clock_account = clock_setup.to_account_info(); + clock_account.is_signer = false; + clock_account.is_writable = false; + + update_clock_slot(&mut clock_account, 1); + + assert!(process_instruction( + &program_id, + &[ + funding_account.clone(), + price_account.clone(), + clock_account.clone() + ], + &instruction_data + ) + .is_ok()); + + { + let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); + assert_eq!(price_data.price_data.comp_[0].latest_.price_, 42); + assert_eq!(price_data.price_data.comp_[0].latest_.conf_, 2); + assert_eq!(price_data.price_data.comp_[0].latest_.pub_slot_, 1); + assert_eq!( + price_data.price_data.comp_[0].latest_.status_, + PC_STATUS_TRADING + ); + assert_eq!(price_data.price_data.valid_slot_, 0); + assert_eq!(price_data.price_data.agg_.pub_slot_, 1); + assert_eq!(price_data.price_data.agg_.price_, 0); + assert_eq!(price_data.price_data.agg_.status_, PC_STATUS_UNKNOWN); + + assert_eq!(price_data.time_machine.current_epoch_numerator, 0); + assert_eq!(price_data.time_machine.current_epoch_is_valid, false); + assert_eq!(price_data.time_machine.current_epoch_denominator, 0); + } + + // add some prices for current slot - get rejected + populate_instruction(&mut instruction_data, 43, 2, 1); + + assert_eq!( + process_instruction( + &program_id, + &[ + funding_account.clone(), + price_account.clone(), + clock_account.clone() + ], + &instruction_data + ), + Err(ProgramError::InvalidArgument) + ); + + { + let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); + assert_eq!(price_data.price_data.comp_[0].latest_.price_, 42); + assert_eq!(price_data.price_data.comp_[0].latest_.conf_, 2); + assert_eq!(price_data.price_data.comp_[0].latest_.pub_slot_, 1); + assert_eq!( + price_data.price_data.comp_[0].latest_.status_, + PC_STATUS_TRADING + ); + assert_eq!(price_data.price_data.valid_slot_, 0); + assert_eq!(price_data.price_data.agg_.pub_slot_, 1); + assert_eq!(price_data.price_data.agg_.price_, 0); + assert_eq!(price_data.price_data.agg_.status_, PC_STATUS_UNKNOWN); + + assert_eq!(price_data.time_machine.current_epoch_numerator, 0); + assert_eq!(price_data.time_machine.current_epoch_is_valid, false); + assert_eq!(price_data.time_machine.current_epoch_denominator, 0); + } + + // add next price in new slot triggering snapshot and aggregate calc + populate_instruction(&mut instruction_data, 81, 2, 2); + update_clock_slot(&mut clock_account, 3); + + assert!(process_instruction( + &program_id, + &[ + funding_account.clone(), + price_account.clone(), + clock_account.clone() + ], + &instruction_data + ) + .is_ok()); + + { + let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); + assert_eq!(price_data.price_data.comp_[0].latest_.price_, 81); + assert_eq!(price_data.price_data.comp_[0].latest_.conf_, 2); + assert_eq!(price_data.price_data.comp_[0].latest_.pub_slot_, 2); + assert_eq!( + price_data.price_data.comp_[0].latest_.status_, + PC_STATUS_TRADING + ); + assert_eq!(price_data.price_data.valid_slot_, 1); + assert_eq!(price_data.price_data.agg_.pub_slot_, 3); + assert_eq!(price_data.price_data.agg_.price_, 42); + assert_eq!(price_data.price_data.agg_.status_, PC_STATUS_TRADING); + + assert_eq!(price_data.time_machine.current_epoch_numerator, 42 / 2 * 3); + assert_eq!(price_data.time_machine.current_epoch_is_valid, false); + assert_eq!(price_data.time_machine.current_epoch_denominator, 3); + } + + // next price doesnt change but slot does + populate_instruction(&mut instruction_data, 81, 2, 3); + update_clock_slot(&mut clock_account, 4); + assert!(process_instruction( + &program_id, + &[ + funding_account.clone(), + price_account.clone(), + clock_account.clone() + ], + &instruction_data + ) + .is_ok()); + + { + let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); + assert_eq!(price_data.price_data.comp_[0].latest_.price_, 81); + assert_eq!(price_data.price_data.comp_[0].latest_.conf_, 2); + assert_eq!(price_data.price_data.comp_[0].latest_.pub_slot_, 3); + assert_eq!( + price_data.price_data.comp_[0].latest_.status_, + PC_STATUS_TRADING + ); + assert_eq!(price_data.price_data.valid_slot_, 3); + assert_eq!(price_data.price_data.agg_.pub_slot_, 4); + assert_eq!(price_data.price_data.agg_.price_, 81); + assert_eq!(price_data.price_data.agg_.status_, PC_STATUS_TRADING); + + assert_eq!( + price_data.time_machine.current_epoch_numerator, + 42 / 2 * 3 + (81 + 42) / 2 + ); + assert_eq!(price_data.time_machine.current_epoch_is_valid, false); + assert_eq!(price_data.time_machine.current_epoch_denominator, 3 + 1); + } + + // next price doesnt change and neither does aggregate but slot does + populate_instruction(&mut instruction_data, 81, 2, 4); + update_clock_slot(&mut clock_account, 5); + assert!(process_instruction( + &program_id, + &[ + funding_account.clone(), + price_account.clone(), + clock_account.clone() + ], + &instruction_data + ) + .is_ok()); + + { + let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); + assert_eq!(price_data.price_data.comp_[0].latest_.price_, 81); + assert_eq!(price_data.price_data.comp_[0].latest_.conf_, 2); + assert_eq!(price_data.price_data.comp_[0].latest_.pub_slot_, 4); + assert_eq!( + price_data.price_data.comp_[0].latest_.status_, + PC_STATUS_TRADING + ); + assert_eq!(price_data.price_data.valid_slot_, 4); + assert_eq!(price_data.price_data.agg_.pub_slot_, 5); + assert_eq!(price_data.price_data.agg_.price_, 81); + assert_eq!(price_data.price_data.agg_.status_, PC_STATUS_TRADING); + + assert_eq!( + price_data.time_machine.current_epoch_numerator, + 42 / 2 * 3 + (81 + 42) / 2 + 81 + ); + assert_eq!(price_data.time_machine.current_epoch_is_valid, false); + assert_eq!(price_data.time_machine.current_epoch_denominator, 3 + 1 + 1); + } + + // try to publish back-in-time + populate_instruction(&mut instruction_data, 81, 2, 1); + update_clock_slot(&mut clock_account, 5); + assert_eq!( + process_instruction( + &program_id, + &[ + funding_account.clone(), + price_account.clone(), + clock_account.clone() + ], + &instruction_data + ), + Err(ProgramError::InvalidArgument) + ); + + { + let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); + assert_eq!(price_data.price_data.comp_[0].latest_.price_, 81); + assert_eq!(price_data.price_data.comp_[0].latest_.conf_, 2); + assert_eq!(price_data.price_data.comp_[0].latest_.pub_slot_, 4); + assert_eq!( + price_data.price_data.comp_[0].latest_.status_, + PC_STATUS_TRADING + ); + assert_eq!(price_data.price_data.valid_slot_, 4); + assert_eq!(price_data.price_data.agg_.pub_slot_, 5); + assert_eq!(price_data.price_data.agg_.price_, 81); + assert_eq!(price_data.price_data.agg_.status_, PC_STATUS_TRADING); + + assert_eq!( + price_data.time_machine.current_epoch_numerator, + 42 / 2 * 3 + (81 + 42) / 2 + 81 + ); + assert_eq!(price_data.time_machine.current_epoch_is_valid, false); + assert_eq!(price_data.time_machine.current_epoch_denominator, 3 + 1 + 1); + } + + populate_instruction(&mut instruction_data, 50, 6, 5); + update_clock_slot(&mut clock_account, 6); + + // Publishing a wide CI results in a status of unknown. + + // check that someone doesn't accidentally break the test. + { + let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); + assert_eq!( + price_data.price_data.comp_[0].latest_.status_, + PC_STATUS_TRADING + ); + } + + assert!(process_instruction( + &program_id, + &[ + funding_account.clone(), + price_account.clone(), + clock_account.clone() + ], + &instruction_data + ) + .is_ok()); + + { + let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); + assert_eq!(price_data.price_data.comp_[0].latest_.price_, 50); + assert_eq!(price_data.price_data.comp_[0].latest_.conf_, 6); + assert_eq!(price_data.price_data.comp_[0].latest_.pub_slot_, 5); + assert_eq!( + price_data.price_data.comp_[0].latest_.status_, + PC_STATUS_UNKNOWN + ); + assert_eq!(price_data.price_data.valid_slot_, 5); + assert_eq!(price_data.price_data.agg_.pub_slot_, 6); + assert_eq!(price_data.price_data.agg_.price_, 81); + assert_eq!(price_data.price_data.agg_.status_, PC_STATUS_TRADING); + + assert_eq!( + price_data.time_machine.current_epoch_numerator, + 42 / 2 * 3 + (81 + 42) / 2 + 81 + 81 + ); + assert_eq!(price_data.time_machine.current_epoch_is_valid, false); + assert_eq!( + price_data.time_machine.current_epoch_denominator, + 3 + 1 + 1 + 1 + ); + } + + // Crank one more time and aggregate should be unknown + populate_instruction(&mut instruction_data, 50, 6, 6); + update_clock_slot(&mut clock_account, 7); + + assert!(process_instruction( + &program_id, + &[ + funding_account.clone(), + price_account.clone(), + clock_account.clone() + ], + &instruction_data + ) + .is_ok()); + + { + let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); + assert_eq!(price_data.price_data.comp_[0].latest_.price_, 50); + assert_eq!(price_data.price_data.comp_[0].latest_.conf_, 6); + assert_eq!(price_data.price_data.comp_[0].latest_.pub_slot_, 6); + assert_eq!( + price_data.price_data.comp_[0].latest_.status_, + PC_STATUS_UNKNOWN + ); + assert_eq!(price_data.price_data.valid_slot_, 6); + assert_eq!(price_data.price_data.agg_.pub_slot_, 7); + assert_eq!(price_data.price_data.agg_.price_, 81); + assert_eq!(price_data.price_data.agg_.status_, PC_STATUS_UNKNOWN); + + assert_eq!( + price_data.time_machine.current_epoch_numerator, + 42 / 2 * 3 + (81 + 42) / 2 + 81 + 81 + ); + assert_eq!(price_data.time_machine.current_epoch_is_valid, false); + assert_eq!( + price_data.time_machine.current_epoch_denominator, + 3 + 1 + 1 + 1 + ); + } +} + +// Create an upd_price instruction with the provided parameters +fn populate_instruction(instruction_data: &mut [u8], price: i64, conf: u64, pub_slot: u64) -> () { + let mut cmd = load_mut::(instruction_data).unwrap(); + cmd.header = OracleCommand::UpdPrice.into(); + cmd.status = PC_STATUS_TRADING; + cmd.price = price; + cmd.confidence = conf; + cmd.publishing_slot = pub_slot; + cmd.unused_ = 0; +} diff --git a/program/rust/src/tests/test_utils.rs b/program/rust/src/tests/test_utils.rs index a03021c70..dc6e4f716 100644 --- a/program/rust/src/tests/test_utils.rs +++ b/program/rust/src/tests/test_utils.rs @@ -35,6 +35,7 @@ const UPPER_BOUND_OF_ALL_ACCOUNT_SIZES: usize = 20536; /// After instantiating the setup `AccountSetup` with `new` (that line will transfer the fields to /// the outer scope), `to_account_info` gives the user an `AccountInfo` pointing to the fields of /// the AccountSetup. +#[repr(align(16))] // On Apple systems this is needed to support u128 in the struct pub struct AccountSetup { key: Pubkey, owner: Pubkey, @@ -109,6 +110,12 @@ pub fn update_clock_slot(clock_account: &mut AccountInfo, slot: u64) { clock_data.to_account_info(clock_account); } +pub fn update_clock_timestamp(clock_account: &mut AccountInfo, timestamp: i64) { + let mut clock_data = clock::Clock::from_account_info(clock_account).unwrap(); + clock_data.unix_timestamp = timestamp; + clock_data.to_account_info(clock_account); +} + impl Into for OracleCommand { fn into(self) -> CommandHeader { return CommandHeader { diff --git a/program/rust/src/time_machine_types.rs b/program/rust/src/time_machine_types.rs index 27d1c79f4..a186cd1a6 100644 --- a/program/rust/src/time_machine_types.rs +++ b/program/rust/src/time_machine_types.rs @@ -1,45 +1,197 @@ +#![allow(unused_imports)] +#![allow(dead_code)] use crate::c_oracle_header::{ PriceAccount, PythAccount, EXTRA_PUBLISHER_SPACE, PC_ACCTYPE_PRICE, + PC_MAX_SEND_LATENCY, PC_PRICE_T_COMP_OFFSET, }; use crate::error::OracleError; +use crate::utils::try_convert; use bytemuck::{ Pod, Zeroable, }; - -#[derive(Debug, Clone, Copy)] -#[repr(C)] -/// this wraps multiple SMA and tick trackers, and includes all the state -/// used by the time machine -pub struct TimeMachineWrapper { - //Place holder with the size of the fields I am planning to add - place_holder: [u8; 1864], -} - +pub const THIRTY_MINUTES: i64 = 30 * 60; +pub const NUM_BUCKETS_THIRTY_MIN: usize = 48; #[derive(Copy, Clone)] #[repr(C)] -/// wraps everything stored in a price account pub struct PriceAccountWrapper { - //an instance of the c price_t type + // An instance of price account pub price_data: PriceAccount, - //space for more publishers + /// Space for more publishers pub extra_publisher_space: [u8; EXTRA_PUBLISHER_SPACE as usize], - //TimeMachine - pub time_machine: TimeMachineWrapper, + // TimeMachine + pub time_machine: SmaTracker, +} + + +#[derive(Debug)] +pub struct DataPoint { + pub previous_timestamp: i64, + pub current_timestamp: i64, + pub slot_gap: u64, + pub price: i64, } + impl PriceAccountWrapper { pub fn initialize_time_machine(&mut self) -> Result<(), OracleError> { - // TO DO + // This is only enabled in tests while in development + #[cfg(test)] + self.time_machine + .initialize(THIRTY_MINUTES, PC_MAX_SEND_LATENCY.into()); Ok(()) } pub fn add_price_to_time_machine(&mut self) -> Result<(), OracleError> { - // TO DO + // This is only enabled in tests while in development + #[cfg(test)] + self.time_machine.add_datapoint( &DataPoint{ + previous_timestamp : self.price_data.prev_timestamp_, + current_timestamp: self.price_data.timestamp_, + slot_gap : (self.price_data.last_slot_ - self.price_data.prev_slot_), + price: self.price_data.prev_price_ /2 + self.price_data.agg_.price_ / 2 + (self.price_data.prev_price_ % 2) * (self.price_data.agg_.price_ % 2), // Hack to avoid overflow + } + )?; + Ok(()) + } +} + +#[derive(Debug, Copy, Clone)] +#[repr(C)] +pub struct SmaTracker { + /// The maximum gap in slots for an epoch's sma to be valid + pub threshold: u64, + /// The length of one sma epoch. + pub granularity: i64, + /// Numerator for the current epoch (ex : slot_gap1 * price_1 + slot_gap2 * price_2 + slot_gap3 + /// * price_3) + /// It can never overflow because : + /// slot_gap1 * price_1 + slot_gap2 * price_2 + slot_gap3 * price_3 <= (slot_gap1 + slot_gap2 + + /// slot_gap3) * i64::MAX <= clock_slot * i64::MAX <= i128::MAX + pub current_epoch_numerator: i128, + /// Denominator for the current epoch (ex : slot_gap1 + slot_gap2 + slot_gap3) + /// It can never overflow because : (slot_gap1 + slot_gap2 + slot_gap3) <= clock_slot + pub current_epoch_denominator: u64, + /// Whether the current epoch is valid (i.e. max slot_gap <= threshold) for all the epochs + /// updates + pub current_epoch_is_valid: bool, + /// Stores the running sum of individual epoch averages ( = current_epoch_numerator / + /// current_epoch_denominator). Each term of the running sum is an average price, at most + /// i64::MAX. If N is the number of epochs elapsed, as long as N <= u64::MAX we have + /// running_sum_of_price_averages[i] <= N * i64::MAX <= u64::MAX * i64::MAX <= i128::MAX. + /// Therefore we can support at least u64::MAX epochs. + pub running_sum_of_price_averages: [i128; NUM_BUCKETS], + /// Stores the number of valid epochs since inception. + /// Each time, at most 1 is added to the sum, therefore we can support at least u64::MAX epochs + /// without overflow. + pub running_valid_epoch_counter: [u64; NUM_BUCKETS], +} + +impl SmaTracker { + pub fn time_to_epoch(&self, time: i64) -> Result { + try_convert::(time / self.granularity) // Can never fail because usize is u64 + // and time is positive + } + + pub fn initialize(&mut self, granularity: i64, threshold: u64) { + self.threshold = threshold; + self.granularity = granularity; + self.current_epoch_is_valid = false; + } + + pub fn add_datapoint(&mut self, datapoint: &DataPoint) -> Result<(), OracleError> { + let epoch_0 = self.time_to_epoch(datapoint.previous_timestamp)?; + let epoch_1 = self.time_to_epoch(datapoint.current_timestamp)?; + + let datapoint_numerator = i128::from(datapoint.slot_gap) * i128::from(datapoint.price); + let datapoint_denominator = datapoint.slot_gap; + + self.current_epoch_denominator += datapoint_denominator; + self.current_epoch_numerator += datapoint_numerator; + + self.current_epoch_is_valid = + self.current_epoch_is_valid && datapoint_denominator <= self.threshold; + + // If epoch changed + if epoch_0 < epoch_1 { + let index = epoch_0.rem_euclid(NUM_BUCKETS); + + // This addition can only overflow if NUM_BUCKETS ~ u64::MAX / 2 + let prev_index = (epoch_0 + NUM_BUCKETS - 1).rem_euclid(NUM_BUCKETS); + + self.running_sum_of_price_averages[index] = self.running_sum_of_price_averages + [prev_index] + + self.current_epoch_numerator / i128::from(self.current_epoch_denominator); + + if self.current_epoch_is_valid { + self.running_valid_epoch_counter[index] = + self.running_valid_epoch_counter[prev_index] + 1; + } else { + self.running_valid_epoch_counter[index] = + self.running_valid_epoch_counter[prev_index] + }; + + self.current_epoch_denominator = datapoint_denominator; + self.current_epoch_numerator = datapoint_numerator; + self.current_epoch_is_valid = datapoint_denominator <= self.threshold; + } + + // If at least one epoch has been skipped + if epoch_0 + 1 < epoch_1 { + let one_point_average = datapoint_numerator / i128::from(datapoint_denominator); + let mut i = 1; + let mut current_bucket = (epoch_0 + 1) % NUM_BUCKETS; + let bucket_0 = epoch_0 % NUM_BUCKETS; + let bucket_1 = epoch_1 % NUM_BUCKETS; + + // Number of times we have wrapped around the buffer + let number_of_full_wraparound = (epoch_1 - 1 - epoch_0) / NUM_BUCKETS; + + // These buckets are "inside" (bucket_0, bucket_1) + while current_bucket != bucket_1 { + self.running_sum_of_price_averages[current_bucket] = self + .running_sum_of_price_averages[bucket_0] + + ((i + number_of_full_wraparound * NUM_BUCKETS) as i128) * one_point_average; + + if self.current_epoch_is_valid { + self.running_valid_epoch_counter[current_bucket] = self + .running_valid_epoch_counter[bucket_0] + + ((i + number_of_full_wraparound * NUM_BUCKETS) as u64); + } else { + self.running_valid_epoch_counter[current_bucket] = + self.running_valid_epoch_counter[bucket_0]; + } + + i += 1; + current_bucket = (current_bucket + 1) % NUM_BUCKETS; + } + + if number_of_full_wraparound > 0 { + // These buckets are "outside" (bucket_0, bucket_1) + while i != NUM_BUCKETS + 1 { + self.running_sum_of_price_averages[current_bucket] = self + .running_sum_of_price_averages[bucket_0] + + ((i + (number_of_full_wraparound - 1) * NUM_BUCKETS) as i128) + * one_point_average; + + if self.current_epoch_is_valid { + self.running_valid_epoch_counter[current_bucket] = self + .running_valid_epoch_counter[bucket_0] + + ((i + (number_of_full_wraparound - 1) * NUM_BUCKETS) as u64); + } else { + self.running_valid_epoch_counter[current_bucket] = + self.running_valid_epoch_counter[bucket_0]; + } + i += 1; + current_bucket = (current_bucket + 1) % NUM_BUCKETS; + } + } + } + Ok(()) } } @@ -65,7 +217,8 @@ pub mod tests { }; use crate::time_machine_types::{ PriceAccountWrapper, - TimeMachineWrapper, + SmaTracker, + NUM_BUCKETS_THIRTY_MIN, }; use std::mem::size_of; #[test] @@ -73,12 +226,9 @@ pub mod tests { ///defined in Rust fn c_time_machine_size_is_correct() { assert_eq!( - size_of::(), - TIME_MACHINE_STRUCT_SIZE as usize, - "expected TIME_MACHINE_STRUCT_SIZE ({}) in oracle.h to the same as the size of TimeMachineWrapper ({})", - TIME_MACHINE_STRUCT_SIZE, - size_of::() - ); + TIME_MACHINE_STRUCT_SIZE as usize, + size_of::>() + ); } #[test] ///test that priceAccountWrapper has a correct size diff --git a/pyth/tests/test_publish.py b/pyth/tests/test_publish.py index c33a6caf3..88a0088fa 100644 --- a/pyth/tests/test_publish.py +++ b/pyth/tests/test_publish.py @@ -120,7 +120,7 @@ def get_account_size(acc_address): resize_account(pyth_init_price['LTC']) time.sleep(20) #defined in oracle.h - new_account_size = 6176 + new_account_size = 7584 assert get_account_size(pyth_init_price['LTC']) == new_account_size