From 188e8710a052df2f430e7d753c4d38d03a3aaab5 Mon Sep 17 00:00:00 2001 From: Trevor Gross Date: Sun, 29 Dec 2024 08:45:08 +0000 Subject: [PATCH 1/5] Add a way for tests to log to a file Occasionally it is useful to see some information from running tests without making everything noisy from `--nocapture`. Add a function to log this kind of output to a file, and print the file as part of CI. --- .github/workflows/main.yml | 5 ++++ configure.rs | 15 +++++++++++ crates/libm-test/src/lib.rs | 53 +++++++++++++++++++++++++++++++++++++ 3 files changed, 73 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0f5becf73..023ec58c0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -113,6 +113,11 @@ jobs: rustup target add x86_64-unknown-linux-musl cargo generate-lockfile && ./ci/run-docker.sh ${{ matrix.target }} + - name: Print test logs if available + if: always() + run: if [ -f "target/test-log.txt" ]; then cat target/test-log.txt; fi + shell: bash + clippy: name: Clippy runs-on: ubuntu-24.04 diff --git a/configure.rs b/configure.rs index 389e86c33..a18937c3c 100644 --- a/configure.rs +++ b/configure.rs @@ -8,6 +8,7 @@ pub struct Config { pub manifest_dir: PathBuf, pub out_dir: PathBuf, pub opt_level: u8, + pub cargo_features: Vec, pub target_arch: String, pub target_env: String, pub target_family: Option, @@ -22,11 +23,16 @@ impl Config { let target_features = env::var("CARGO_CFG_TARGET_FEATURE") .map(|feats| feats.split(',').map(ToOwned::to_owned).collect()) .unwrap_or_default(); + let cargo_features = env::vars() + .filter_map(|(name, _value)| name.strip_prefix("CARGO_FEATURE_").map(ToOwned::to_owned)) + .map(|s| s.to_lowercase().replace("_", "-")) + .collect(); Self { manifest_dir: PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()), out_dir: PathBuf::from(env::var("OUT_DIR").unwrap()), opt_level: env::var("OPT_LEVEL").unwrap().parse().unwrap(), + cargo_features, target_arch: env::var("CARGO_CFG_TARGET_ARCH").unwrap(), target_env: env::var("CARGO_CFG_TARGET_ENV").unwrap(), target_family: env::var("CARGO_CFG_TARGET_FAMILY").ok(), @@ -45,6 +51,7 @@ pub fn emit_libm_config(cfg: &Config) { emit_arch_cfg(); emit_optimization_cfg(cfg); emit_cfg_shorthands(cfg); + emit_cfg_env(cfg); emit_f16_f128_cfg(cfg); } @@ -53,6 +60,7 @@ pub fn emit_libm_config(cfg: &Config) { pub fn emit_test_config(cfg: &Config) { emit_optimization_cfg(cfg); emit_cfg_shorthands(cfg); + emit_cfg_env(cfg); emit_f16_f128_cfg(cfg); } @@ -97,6 +105,13 @@ fn emit_cfg_shorthands(cfg: &Config) { } } +/// Reemit config that we make use of for test logging. +fn emit_cfg_env(cfg: &Config) { + println!("cargo:rustc-env=CFG_CARGO_FEATURES={:?}", cfg.cargo_features); + println!("cargo:rustc-env=CFG_OPT_LEVEL={}", cfg.opt_level); + println!("cargo:rustc-env=CFG_TARGET_FEATURES={:?}", cfg.target_features); +} + /// Configure whether or not `f16` and `f128` support should be enabled. fn emit_f16_f128_cfg(cfg: &Config) { println!("cargo:rustc-check-cfg=cfg(f16_enabled)"); diff --git a/crates/libm-test/src/lib.rs b/crates/libm-test/src/lib.rs index 97907b2a1..c1aec0230 100644 --- a/crates/libm-test/src/lib.rs +++ b/crates/libm-test/src/lib.rs @@ -13,6 +13,13 @@ mod precision; mod run_cfg; mod test_traits; +use std::env; +use std::fs::File; +use std::io::Write; +use std::path::PathBuf; +use std::sync::LazyLock; +use std::time::SystemTime; + pub use f8_impl::f8; pub use libm::support::{Float, Int, IntTy, MinInt}; pub use num::{FloatExt, logspace}; @@ -42,3 +49,49 @@ pub const fn ci() -> bool { Some(_) => true, } } + +/// Print to stderr and additionally log it to `target/test-log.txt`. This is useful for saving +/// output that would otherwise be consumed by the test harness. +pub fn test_log(s: &str) { + // Handle to a file opened in append mode, unless a suitable path can't be determined. + static OUTFILE: LazyLock> = LazyLock::new(|| { + // If the target directory is overridden, use that environment variable. Otherwise, save + // at the default path `{workspace_root}/target`. + let target_dir = match env::var("CARGO_TARGET_DIR") { + Ok(s) => PathBuf::from(s), + Err(_) => { + let Ok(x) = env::var("CARGO_MANIFEST_DIR") else { + return None; + }; + + PathBuf::from(x).parent().unwrap().parent().unwrap().join("target") + } + }; + let outfile = target_dir.join("test-log.txt"); + + let mut f = File::options() + .create(true) + .append(true) + .open(outfile) + .expect("failed to open logfile"); + let now = SystemTime::now().duration_since(SystemTime::UNIX_EPOCH).unwrap(); + + writeln!(f, "\n\nTest run at {}", now.as_secs()).unwrap(); + writeln!(f, "arch: {}", env::consts::ARCH).unwrap(); + writeln!(f, "os: {}", env::consts::OS).unwrap(); + writeln!(f, "bits: {}", usize::BITS).unwrap(); + writeln!(f, "emulated: {}", emulated()).unwrap(); + writeln!(f, "ci: {}", ci()).unwrap(); + writeln!(f, "cargo features: {}", env!("CFG_CARGO_FEATURES")).unwrap(); + writeln!(f, "opt level: {}", env!("CFG_OPT_LEVEL")).unwrap(); + writeln!(f, "target features: {}", env!("CFG_TARGET_FEATURES")).unwrap(); + + Some(f) + }); + + eprintln!("{s}"); + + if let Some(mut f) = OUTFILE.as_ref() { + writeln!(f, "{s}").unwrap(); + } +} From 0d486fe62a55ae409ee96ee58854ae0efaa57914 Mon Sep 17 00:00:00 2001 From: Trevor Gross Date: Thu, 26 Dec 2024 07:44:54 +0000 Subject: [PATCH 2/5] Streamline the way that test iteration count is determined Currently, tests use a handful of constants to determine how many iterations to perform: `NTESTS`, `AROUND`, and `MAX_CHECK_POINTS`. This configuration is not very straightforward to adjust and needs to be repeated everywhere it is used. Replace this with new functions in the `run_cfg` module that determine iteration counts in a more reusable and documented way. This only updates `edge_cases` and `domain_logspace`, `random` is refactored in a later commit. --- crates/libm-test/src/gen/domain_logspace.rs | 31 +--- crates/libm-test/src/gen/edge_cases.rs | 54 +++--- crates/libm-test/src/gen/random.rs | 1 + crates/libm-test/src/lib.rs | 2 +- crates/libm-test/src/run_cfg.rs | 177 +++++++++++++++++++- 5 files changed, 208 insertions(+), 57 deletions(-) diff --git a/crates/libm-test/src/gen/domain_logspace.rs b/crates/libm-test/src/gen/domain_logspace.rs index 5e37170fa..3d8a3e7fe 100644 --- a/crates/libm-test/src/gen/domain_logspace.rs +++ b/crates/libm-test/src/gen/domain_logspace.rs @@ -6,41 +6,26 @@ use libm::support::{IntTy, MinInt}; use crate::domain::HasDomain; use crate::op::OpITy; +use crate::run_cfg::{GeneratorKind, iteration_count}; use crate::{CheckCtx, MathOp, logspace}; -/// Number of tests to run. -// FIXME(ntests): replace this with a more logical algorithm -const NTESTS: usize = { - if cfg!(optimizations_enabled) { - if crate::emulated() - || !cfg!(target_pointer_width = "64") - || cfg!(all(target_arch = "x86_64", target_vendor = "apple")) - { - // Tests are pretty slow on non-64-bit targets, x86 MacOS, and targets that run - // in QEMU. - 100_000 - } else { - 5_000_000 - } - } else { - // Without optimizations just run a quick check - 800 - } -}; - /// Create a range of logarithmically spaced inputs within a function's domain. /// /// This allows us to get reasonably thorough coverage without wasting time on values that are /// NaN or out of range. Random tests will still cover values that are excluded here. -pub fn get_test_cases(_ctx: &CheckCtx) -> impl Iterator +pub fn get_test_cases(ctx: &CheckCtx) -> impl Iterator where Op: MathOp + HasDomain, - IntTy: TryFrom, + IntTy: TryFrom, RangeInclusive>: Iterator, { let domain = Op::DOMAIN; + let ntests = iteration_count(ctx, GeneratorKind::Domain, 0); + + // We generate logspaced inputs within a specific range, excluding values that are out of + // range in order to make iterations useful (random tests still cover the full range). let start = domain.range_start(); let end = domain.range_end(); - let steps = OpITy::::try_from(NTESTS).unwrap_or(OpITy::::MAX); + let steps = OpITy::::try_from(ntests).unwrap_or(OpITy::::MAX); logspace(start, end, steps).map(|v| (v,)) } diff --git a/crates/libm-test/src/gen/edge_cases.rs b/crates/libm-test/src/gen/edge_cases.rs index 3387f6c48..1f27c1467 100644 --- a/crates/libm-test/src/gen/edge_cases.rs +++ b/crates/libm-test/src/gen/edge_cases.rs @@ -3,18 +3,11 @@ use libm::support::Float; use crate::domain::HasDomain; +use crate::run_cfg::{check_near_count, check_point_count}; use crate::{CheckCtx, FloatExt, MathOp}; -/// Number of values near an interesting point to check. -// FIXME(ntests): replace this with a more logical algorithm -const AROUND: usize = 100; - -/// Functions have infinite asymptotes, limit how many we check. -// FIXME(ntests): replace this with a more logical algorithm -const MAX_CHECK_POINTS: usize = 10; - /// Create a list of values around interesting points (infinities, zeroes, NaNs). -pub fn get_test_cases(_ctx: &CheckCtx) -> impl Iterator +pub fn get_test_cases(ctx: &CheckCtx) -> impl Iterator where Op: MathOp + HasDomain, F: Float, @@ -25,23 +18,26 @@ where let domain_start = domain.range_start(); let domain_end = domain.range_end(); + let check_points = check_point_count(ctx); + let near_points = check_near_count(ctx); + // Check near some notable constants - count_up(F::ONE, values); - count_up(F::ZERO, values); - count_up(F::NEG_ONE, values); - count_down(F::ONE, values); - count_down(F::ZERO, values); - count_down(F::NEG_ONE, values); + count_up(F::ONE, near_points, values); + count_up(F::ZERO, near_points, values); + count_up(F::NEG_ONE, near_points, values); + count_down(F::ONE, near_points, values); + count_down(F::ZERO, near_points, values); + count_down(F::NEG_ONE, near_points, values); values.push(F::NEG_ZERO); // Check values near the extremes - count_up(F::NEG_INFINITY, values); - count_down(F::INFINITY, values); - count_down(domain_end, values); - count_up(domain_start, values); - count_down(domain_start, values); - count_up(domain_end, values); - count_down(domain_end, values); + count_up(F::NEG_INFINITY, near_points, values); + count_down(F::INFINITY, near_points, values); + count_down(domain_end, near_points, values); + count_up(domain_start, near_points, values); + count_down(domain_start, near_points, values); + count_up(domain_end, near_points, values); + count_down(domain_end, near_points, values); // Check some special values that aren't included in the above ranges values.push(F::NAN); @@ -50,9 +46,9 @@ where // Check around asymptotes if let Some(f) = domain.check_points { let iter = f(); - for x in iter.take(MAX_CHECK_POINTS) { - count_up(x, values); - count_down(x, values); + for x in iter.take(check_points) { + count_up(x, near_points, values); + count_down(x, near_points, values); } } @@ -65,11 +61,11 @@ where /// Add `AROUND` values starting at and including `x` and counting up. Uses the smallest possible /// increments (1 ULP). -fn count_up(mut x: F, values: &mut Vec) { +fn count_up(mut x: F, points: u64, values: &mut Vec) { assert!(!x.is_nan()); let mut count = 0; - while x < F::INFINITY && count < AROUND { + while x < F::INFINITY && count < points { values.push(x); x = x.next_up(); count += 1; @@ -78,11 +74,11 @@ fn count_up(mut x: F, values: &mut Vec) { /// Add `AROUND` values starting at and including `x` and counting down. Uses the smallest possible /// increments (1 ULP). -fn count_down(mut x: F, values: &mut Vec) { +fn count_down(mut x: F, points: u64, values: &mut Vec) { assert!(!x.is_nan()); let mut count = 0; - while x > F::NEG_INFINITY && count < AROUND { + while x > F::NEG_INFINITY && count < points { values.push(x); x = x.next_down(); count += 1; diff --git a/crates/libm-test/src/gen/random.rs b/crates/libm-test/src/gen/random.rs index 4f75da07b..a30a3674e 100644 --- a/crates/libm-test/src/gen/random.rs +++ b/crates/libm-test/src/gen/random.rs @@ -12,6 +12,7 @@ use crate::{BaseName, CheckCtx, GenerateInput}; const SEED: [u8; 32] = *b"3.141592653589793238462643383279"; /// Number of tests to run. +// FIXME(ntests): clean this up when possible const NTESTS: usize = { if cfg!(optimizations_enabled) { if crate::emulated() diff --git a/crates/libm-test/src/lib.rs b/crates/libm-test/src/lib.rs index c1aec0230..80ec23736 100644 --- a/crates/libm-test/src/lib.rs +++ b/crates/libm-test/src/lib.rs @@ -25,7 +25,7 @@ pub use libm::support::{Float, Int, IntTy, MinInt}; pub use num::{FloatExt, logspace}; pub use op::{BaseName, FloatTy, Identifier, MathOp, OpCFn, OpFTy, OpRustFn, OpRustRet, Ty}; pub use precision::{MaybeOverride, SpecialCase, default_ulp}; -pub use run_cfg::{CheckBasis, CheckCtx}; +pub use run_cfg::{CheckBasis, CheckCtx, EXTENSIVE_ENV, GeneratorKind}; pub use test_traits::{CheckOutput, GenerateInput, Hex, TupleCall}; /// Result type for tests is usually from `anyhow`. Most times there is no success value to diff --git a/crates/libm-test/src/run_cfg.rs b/crates/libm-test/src/run_cfg.rs index eb7e0e2c1..46a6a1fad 100644 --- a/crates/libm-test/src/run_cfg.rs +++ b/crates/libm-test/src/run_cfg.rs @@ -1,13 +1,11 @@ //! Configuration for how tests get run. -#![allow(unused)] - -use std::collections::BTreeMap; use std::env; use std::sync::LazyLock; -use crate::{BaseName, FloatTy, Identifier, op}; +use crate::{BaseName, FloatTy, Identifier, test_log}; +/// The environment variable indicating which extensive tests should be run. pub const EXTENSIVE_ENV: &str = "LIBM_EXTENSIVE_TESTS"; /// Context passed to [`CheckOutput`]. @@ -49,3 +47,174 @@ pub enum CheckBasis { /// Check against infinite precision (MPFR). Mpfr, } + +/// The different kinds of generators that provide test input. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum GeneratorKind { + Domain, + Random, +} + +/// A list of all functions that should get extensive tests. +/// +/// This also supports the special test name `all` to run all tests, as well as `all_f16`, +/// `all_f32`, `all_f64`, and `all_f128` to run all tests for a specific float type. +static EXTENSIVE: LazyLock> = LazyLock::new(|| { + let var = env::var(EXTENSIVE_ENV).unwrap_or_default(); + let list = var.split(",").filter(|s| !s.is_empty()).collect::>(); + let mut ret = Vec::new(); + + let append_ty_ops = |ret: &mut Vec<_>, fty: FloatTy| { + let iter = Identifier::ALL.iter().filter(move |id| id.math_op().float_ty == fty).copied(); + ret.extend(iter); + }; + + for item in list { + match item { + "all" => ret = Identifier::ALL.to_owned(), + "all_f16" => append_ty_ops(&mut ret, FloatTy::F16), + "all_f32" => append_ty_ops(&mut ret, FloatTy::F32), + "all_f64" => append_ty_ops(&mut ret, FloatTy::F64), + "all_f128" => append_ty_ops(&mut ret, FloatTy::F128), + s => { + let id = Identifier::from_str(s) + .unwrap_or_else(|| panic!("unrecognized test name `{s}`")); + ret.push(id); + } + } + } + + ret +}); + +/// Information about the function to be tested. +#[derive(Debug)] +struct TestEnv { + /// Tests should be reduced because the platform is slow. E.g. 32-bit or emulated. + slow_platform: bool, + /// The float cannot be tested exhaustively, `f64` or `f128`. + large_float_ty: bool, + /// Env indicates that an extensive test should be run. + should_run_extensive: bool, + /// Multiprecision tests will be run. + mp_tests_enabled: bool, + /// The number of inputs to the function. + input_count: usize, +} + +impl TestEnv { + fn from_env(ctx: &CheckCtx) -> Self { + let id = ctx.fn_ident; + let op = id.math_op(); + + let will_run_mp = cfg!(feature = "test-multiprecision"); + + // Tests are pretty slow on non-64-bit targets, x86 MacOS, and targets that run in QEMU. Start + // with a reduced number on these platforms. + let slow_on_ci = crate::emulated() + || usize::BITS < 64 + || cfg!(all(target_arch = "x86_64", target_vendor = "apple")); + let slow_platform = slow_on_ci && crate::ci(); + + let large_float_ty = match op.float_ty { + FloatTy::F16 | FloatTy::F32 => false, + FloatTy::F64 | FloatTy::F128 => true, + }; + + let will_run_extensive = EXTENSIVE.contains(&id); + + let input_count = op.rust_sig.args.len(); + + Self { + slow_platform, + large_float_ty, + should_run_extensive: will_run_extensive, + mp_tests_enabled: will_run_mp, + input_count, + } + } +} + +/// The number of iterations to run for a given test. +pub fn iteration_count(ctx: &CheckCtx, gen_kind: GeneratorKind, argnum: usize) -> u64 { + let t_env = TestEnv::from_env(ctx); + + // Ideally run 5M tests + let mut domain_iter_count: u64 = 4_000_000; + + // Start with a reduced number of tests on slow platforms. + if t_env.slow_platform { + domain_iter_count = 100_000; + } + + // Larger float types get more iterations. + if t_env.large_float_ty { + domain_iter_count *= 4; + } + + // Functions with more arguments get more iterations. + let arg_multiplier = 1 << (t_env.input_count - 1); + domain_iter_count *= arg_multiplier; + + // If we will be running tests against MPFR, we don't need to test as much against musl. + // However, there are some platforms where we have to test against musl since MPFR can't be + // built. + if t_env.mp_tests_enabled && ctx.basis == CheckBasis::Musl { + domain_iter_count /= 100; + } + + // Run fewer random tests than domain tests. + let random_iter_count = domain_iter_count / 100; + + let mut total_iterations = match gen_kind { + GeneratorKind::Domain => domain_iter_count, + GeneratorKind::Random => random_iter_count, + }; + + if cfg!(optimizations_enabled) { + // Always run at least 10,000 tests. + total_iterations = total_iterations.max(10_000); + } else { + // Without optimizations, just run a quick check regardless of other parameters. + total_iterations = 800; + } + + // Adjust for the number of inputs + let ntests = match t_env.input_count { + 1 => total_iterations, + 2 => (total_iterations as f64).sqrt().ceil() as u64, + 3 => (total_iterations as f64).cbrt().ceil() as u64, + _ => panic!("test has more than three arguments"), + }; + let total = ntests.pow(t_env.input_count.try_into().unwrap()); + + test_log(&format!( + "{gen_kind:?} {basis:?} {fn_ident} arg {arg}/{args}: {ntests} iterations \ + ({total} total)", + basis = ctx.basis, + fn_ident = ctx.fn_ident, + arg = argnum + 1, + args = t_env.input_count, + )); + + ntests +} + +/// For domain tests, limit how many asymptotes or specified check points we test. +pub fn check_point_count(ctx: &CheckCtx) -> usize { + let t_env = TestEnv::from_env(ctx); + if t_env.slow_platform || !cfg!(optimizations_enabled) { 4 } else { 10 } +} + +/// When validating points of interest (e.g. asymptotes, inflection points, extremes), also check +/// this many surrounding values. +pub fn check_near_count(_ctx: &CheckCtx) -> u64 { + if cfg!(optimizations_enabled) { 100 } else { 10 } +} + +/// Check whether extensive actions should be run or skipped. +#[expect(dead_code, reason = "extensive tests have not yet been added")] +pub fn skip_extensive_test(ctx: &CheckCtx) -> bool { + let t_env = TestEnv::from_env(ctx); + !t_env.should_run_extensive +} From ae8bf8c29d7f4a55cd1337ef260b512618a8c55b Mon Sep 17 00:00:00 2001 From: Trevor Gross Date: Mon, 30 Dec 2024 06:12:16 +0000 Subject: [PATCH 3/5] Add an iterator that ensures known size Introduce the `KnownSize` iterator wrapper, which allows providing the size at construction time. This provides an `ExactSizeIterator` implemenation so we can check a generator's value count during testing. --- crates/libm-test/src/gen.rs | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/crates/libm-test/src/gen.rs b/crates/libm-test/src/gen.rs index 2d15915d9..2305d2a23 100644 --- a/crates/libm-test/src/gen.rs +++ b/crates/libm-test/src/gen.rs @@ -5,6 +5,43 @@ pub mod domain_logspace; pub mod edge_cases; pub mod random; +/// A wrapper to turn any iterator into an `ExactSizeIterator`. Asserts the final result to ensure +/// the provided size was correct. +#[derive(Debug)] +pub struct KnownSize { + total: u64, + current: u64, + iter: I, +} + +impl KnownSize { + pub fn new(iter: I, total: u64) -> Self { + Self { total, current: 0, iter } + } +} + +impl Iterator for KnownSize { + type Item = I::Item; + + fn next(&mut self) -> Option { + let next = self.iter.next(); + if next.is_some() { + self.current += 1; + return next; + } + + assert_eq!(self.current, self.total, "total items did not match expected"); + None + } + + fn size_hint(&self) -> (usize, Option) { + let remaining = usize::try_from(self.total - self.current).unwrap(); + (remaining, Some(remaining)) + } +} + +impl ExactSizeIterator for KnownSize {} + /// Helper type to turn any reusable input into a generator. #[derive(Clone, Debug, Default)] pub struct CachedInput { From addbb18eec710d29382e1e85034190f398274aaa Mon Sep 17 00:00:00 2001 From: Trevor Gross Date: Sun, 29 Dec 2024 11:23:08 +0000 Subject: [PATCH 4/5] Rewrite the random test generator Currently, all inputs are generated and then cached. This works reasonably well but it isn't very configurable or extensible (adding `f16` and `f128` is awkward). Replace this with a trait for generating random sequences of tuples. This also removes possible storage limitations of caching all inputs. --- crates/libm-test/benches/random.rs | 7 +- crates/libm-test/src/gen.rs | 69 ------- crates/libm-test/src/gen/random.rs | 206 +++++++++---------- crates/libm-test/src/lib.rs | 2 +- crates/libm-test/src/run_cfg.rs | 32 ++- crates/libm-test/src/test_traits.rs | 8 +- crates/libm-test/tests/compare_built_musl.rs | 11 +- crates/libm-test/tests/multiprecision.rs | 9 +- 8 files changed, 148 insertions(+), 196 deletions(-) diff --git a/crates/libm-test/benches/random.rs b/crates/libm-test/benches/random.rs index 06997cd36..23f429455 100644 --- a/crates/libm-test/benches/random.rs +++ b/crates/libm-test/benches/random.rs @@ -2,8 +2,9 @@ use std::hint::black_box; use std::time::Duration; use criterion::{Criterion, criterion_main}; -use libm_test::gen::{CachedInput, random}; -use libm_test::{CheckBasis, CheckCtx, GenerateInput, MathOp, TupleCall}; +use libm_test::gen::random; +use libm_test::gen::random::RandomInput; +use libm_test::{CheckBasis, CheckCtx, MathOp, TupleCall}; /// Benchmark with this many items to get a variety const BENCH_ITER_ITEMS: usize = if cfg!(feature = "short-benchmarks") { 50 } else { 500 }; @@ -47,7 +48,7 @@ macro_rules! musl_rand_benches { fn bench_one(c: &mut Criterion, musl_extra: MuslExtra) where Op: MathOp, - CachedInput: GenerateInput, + Op::RustArgs: RandomInput, { let name = Op::NAME; diff --git a/crates/libm-test/src/gen.rs b/crates/libm-test/src/gen.rs index 2305d2a23..83e00f31d 100644 --- a/crates/libm-test/src/gen.rs +++ b/crates/libm-test/src/gen.rs @@ -1,6 +1,5 @@ //! Different generators that can create random or systematic bit patterns. -use crate::GenerateInput; pub mod domain_logspace; pub mod edge_cases; pub mod random; @@ -41,71 +40,3 @@ impl Iterator for KnownSize { } impl ExactSizeIterator for KnownSize {} - -/// Helper type to turn any reusable input into a generator. -#[derive(Clone, Debug, Default)] -pub struct CachedInput { - pub inputs_f32: Vec<(f32, f32, f32)>, - pub inputs_f64: Vec<(f64, f64, f64)>, - pub inputs_i32: Vec<(i32, i32, i32)>, -} - -impl GenerateInput<(f32,)> for CachedInput { - fn get_cases(&self) -> impl Iterator { - self.inputs_f32.iter().map(|f| (f.0,)) - } -} - -impl GenerateInput<(f32, f32)> for CachedInput { - fn get_cases(&self) -> impl Iterator { - self.inputs_f32.iter().map(|f| (f.0, f.1)) - } -} - -impl GenerateInput<(i32, f32)> for CachedInput { - fn get_cases(&self) -> impl Iterator { - self.inputs_i32.iter().zip(self.inputs_f32.iter()).map(|(i, f)| (i.0, f.0)) - } -} - -impl GenerateInput<(f32, i32)> for CachedInput { - fn get_cases(&self) -> impl Iterator { - GenerateInput::<(i32, f32)>::get_cases(self).map(|(i, f)| (f, i)) - } -} - -impl GenerateInput<(f32, f32, f32)> for CachedInput { - fn get_cases(&self) -> impl Iterator { - self.inputs_f32.iter().copied() - } -} - -impl GenerateInput<(f64,)> for CachedInput { - fn get_cases(&self) -> impl Iterator { - self.inputs_f64.iter().map(|f| (f.0,)) - } -} - -impl GenerateInput<(f64, f64)> for CachedInput { - fn get_cases(&self) -> impl Iterator { - self.inputs_f64.iter().map(|f| (f.0, f.1)) - } -} - -impl GenerateInput<(i32, f64)> for CachedInput { - fn get_cases(&self) -> impl Iterator { - self.inputs_i32.iter().zip(self.inputs_f64.iter()).map(|(i, f)| (i.0, f.0)) - } -} - -impl GenerateInput<(f64, i32)> for CachedInput { - fn get_cases(&self) -> impl Iterator { - GenerateInput::<(i32, f64)>::get_cases(self).map(|(i, f)| (f, i)) - } -} - -impl GenerateInput<(f64, f64, f64)> for CachedInput { - fn get_cases(&self) -> impl Iterator { - self.inputs_f64.iter().copied() - } -} diff --git a/crates/libm-test/src/gen/random.rs b/crates/libm-test/src/gen/random.rs index a30a3674e..6df944317 100644 --- a/crates/libm-test/src/gen/random.rs +++ b/crates/libm-test/src/gen/random.rs @@ -1,120 +1,118 @@ -//! A simple generator that produces deterministic random input, caching to use the same -//! inputs for all functions. - +use std::env; +use std::ops::RangeInclusive; use std::sync::LazyLock; +use libm::support::Float; +use rand::distributions::{Alphanumeric, Standard}; +use rand::prelude::Distribution; use rand::{Rng, SeedableRng}; use rand_chacha::ChaCha8Rng; -use super::CachedInput; -use crate::{BaseName, CheckCtx, GenerateInput}; - -const SEED: [u8; 32] = *b"3.141592653589793238462643383279"; - -/// Number of tests to run. -// FIXME(ntests): clean this up when possible -const NTESTS: usize = { - if cfg!(optimizations_enabled) { - if crate::emulated() - || !cfg!(target_pointer_width = "64") - || cfg!(all(target_arch = "x86_64", target_vendor = "apple")) - { - // Tests are pretty slow on non-64-bit targets, x86 MacOS, and targets that run - // in QEMU. - 100_000 - } else { - 5_000_000 - } - } else { - // Without optimizations just run a quick check - 800 - } -}; - -/// Tested inputs. -static TEST_CASES: LazyLock = LazyLock::new(|| make_test_cases(NTESTS)); - -/// The first argument to `jn` and `jnf` is the number of iterations. Make this a reasonable -/// value so tests don't run forever. -static TEST_CASES_JN: LazyLock = LazyLock::new(|| { - // Start with regular test cases - let mut cases = (*TEST_CASES).clone(); - - // These functions are extremely slow, limit them - let ntests_jn = (NTESTS / 1000).max(80); - cases.inputs_i32.truncate(ntests_jn); - cases.inputs_f32.truncate(ntests_jn); - cases.inputs_f64.truncate(ntests_jn); - - // It is easy to overflow the stack with these in debug mode - let max_iterations = if cfg!(optimizations_enabled) && cfg!(target_pointer_width = "64") { - 0xffff - } else if cfg!(windows) { - 0x00ff - } else { - 0x0fff - }; +use super::KnownSize; +use crate::run_cfg::{int_range, iteration_count}; +use crate::{CheckCtx, GeneratorKind}; - let mut rng = ChaCha8Rng::from_seed(SEED); +pub(crate) const SEED_ENV: &str = "LIBM_SEED"; - for case in cases.inputs_i32.iter_mut() { - case.0 = rng.gen_range(3..=max_iterations); - } +pub(crate) static SEED: LazyLock<[u8; 32]> = LazyLock::new(|| { + let s = env::var(SEED_ENV).unwrap_or_else(|_| { + let mut rng = rand::thread_rng(); + (0..32).map(|_| rng.sample(Alphanumeric) as char).collect() + }); - cases + s.as_bytes().try_into().unwrap_or_else(|_| { + panic!("Seed must be 32 characters, got `{s}`"); + }) }); -fn make_test_cases(ntests: usize) -> CachedInput { - let mut rng = ChaCha8Rng::from_seed(SEED); - - // make sure we include some basic cases - let mut inputs_i32 = vec![(0, 0, 0), (1, 1, 1), (-1, -1, -1)]; - let mut inputs_f32 = vec![ - (0.0, 0.0, 0.0), - (f32::EPSILON, f32::EPSILON, f32::EPSILON), - (f32::INFINITY, f32::INFINITY, f32::INFINITY), - (f32::NEG_INFINITY, f32::NEG_INFINITY, f32::NEG_INFINITY), - (f32::MAX, f32::MAX, f32::MAX), - (f32::MIN, f32::MIN, f32::MIN), - (f32::MIN_POSITIVE, f32::MIN_POSITIVE, f32::MIN_POSITIVE), - (f32::NAN, f32::NAN, f32::NAN), - ]; - let mut inputs_f64 = vec![ - (0.0, 0.0, 0.0), - (f64::EPSILON, f64::EPSILON, f64::EPSILON), - (f64::INFINITY, f64::INFINITY, f64::INFINITY), - (f64::NEG_INFINITY, f64::NEG_INFINITY, f64::NEG_INFINITY), - (f64::MAX, f64::MAX, f64::MAX), - (f64::MIN, f64::MIN, f64::MIN), - (f64::MIN_POSITIVE, f64::MIN_POSITIVE, f64::MIN_POSITIVE), - (f64::NAN, f64::NAN, f64::NAN), - ]; - - inputs_i32.extend((0..(ntests - inputs_i32.len())).map(|_| rng.gen::<(i32, i32, i32)>())); - - // Generate integers to get a full range of bitpatterns, then convert back to - // floats. - inputs_f32.extend((0..(ntests - inputs_f32.len())).map(|_| { - let ints = rng.gen::<(u32, u32, u32)>(); - (f32::from_bits(ints.0), f32::from_bits(ints.1), f32::from_bits(ints.2)) - })); - inputs_f64.extend((0..(ntests - inputs_f64.len())).map(|_| { - let ints = rng.gen::<(u64, u64, u64)>(); - (f64::from_bits(ints.0), f64::from_bits(ints.1), f64::from_bits(ints.2)) - })); - - CachedInput { inputs_f32, inputs_f64, inputs_i32 } +/// Generate a sequence of random values of this type. +pub trait RandomInput { + fn get_cases(ctx: &CheckCtx) -> impl ExactSizeIterator; } -/// Create a test case iterator. -pub fn get_test_cases(ctx: &CheckCtx) -> impl Iterator +/// Generate a sequence of deterministically random floats. +fn random_floats(count: u64) -> impl Iterator where - CachedInput: GenerateInput, + Standard: Distribution, { - let inputs = if ctx.base_name == BaseName::Jn || ctx.base_name == BaseName::Yn { - &TEST_CASES_JN - } else { - &TEST_CASES + let mut rng = ChaCha8Rng::from_seed(*SEED); + + // Generate integers to get a full range of bitpatterns (including NaNs), then convert back + // to the float type. + (0..count).map(move |_| F::from_bits(rng.gen::())) +} + +/// Generate a sequence of deterministically random `i32`s within a specified range. +fn random_ints(count: u64, range: RangeInclusive) -> impl Iterator { + let mut rng = ChaCha8Rng::from_seed(*SEED); + (0..count).map(move |_| rng.gen_range::(range.clone())) +} + +macro_rules! impl_random_input { + ($fty:ty) => { + impl RandomInput for ($fty,) { + fn get_cases(ctx: &CheckCtx) -> impl ExactSizeIterator { + let count = iteration_count(ctx, GeneratorKind::Random, 0); + let iter = random_floats(count).map(|f: $fty| (f,)); + KnownSize::new(iter, count) + } + } + + impl RandomInput for ($fty, $fty) { + fn get_cases(ctx: &CheckCtx) -> impl ExactSizeIterator { + let count0 = iteration_count(ctx, GeneratorKind::Random, 0); + let count1 = iteration_count(ctx, GeneratorKind::Random, 1); + let iter = random_floats(count0) + .flat_map(move |f1: $fty| random_floats(count1).map(move |f2: $fty| (f1, f2))); + KnownSize::new(iter, count0 * count1) + } + } + + impl RandomInput for ($fty, $fty, $fty) { + fn get_cases(ctx: &CheckCtx) -> impl ExactSizeIterator { + let count0 = iteration_count(ctx, GeneratorKind::Random, 0); + let count1 = iteration_count(ctx, GeneratorKind::Random, 1); + let count2 = iteration_count(ctx, GeneratorKind::Random, 2); + let iter = random_floats(count0).flat_map(move |f1: $fty| { + random_floats(count1).flat_map(move |f2: $fty| { + random_floats(count2).map(move |f3: $fty| (f1, f2, f3)) + }) + }); + KnownSize::new(iter, count0 * count1 * count2) + } + } + + impl RandomInput for (i32, $fty) { + fn get_cases(ctx: &CheckCtx) -> impl ExactSizeIterator { + let count0 = iteration_count(ctx, GeneratorKind::Random, 0); + let count1 = iteration_count(ctx, GeneratorKind::Random, 1); + let range0 = int_range(ctx, 0); + let iter = random_ints(count0, range0) + .flat_map(move |f1: i32| random_floats(count1).map(move |f2: $fty| (f1, f2))); + KnownSize::new(iter, count0 * count1) + } + } + + impl RandomInput for ($fty, i32) { + fn get_cases(ctx: &CheckCtx) -> impl ExactSizeIterator { + let count0 = iteration_count(ctx, GeneratorKind::Random, 0); + let count1 = iteration_count(ctx, GeneratorKind::Random, 1); + let range1 = int_range(ctx, 1); + let iter = random_floats(count0).flat_map(move |f1: $fty| { + random_ints(count1, range1.clone()).map(move |f2: i32| (f1, f2)) + }); + KnownSize::new(iter, count0 * count1) + } + } }; - inputs.get_cases() +} + +impl_random_input!(f32); +impl_random_input!(f64); + +/// Create a test case iterator. +pub fn get_test_cases( + ctx: &CheckCtx, +) -> impl Iterator + use<'_, RustArgs> { + RustArgs::get_cases(ctx) } diff --git a/crates/libm-test/src/lib.rs b/crates/libm-test/src/lib.rs index 80ec23736..8a4e782df 100644 --- a/crates/libm-test/src/lib.rs +++ b/crates/libm-test/src/lib.rs @@ -26,7 +26,7 @@ pub use num::{FloatExt, logspace}; pub use op::{BaseName, FloatTy, Identifier, MathOp, OpCFn, OpFTy, OpRustFn, OpRustRet, Ty}; pub use precision::{MaybeOverride, SpecialCase, default_ulp}; pub use run_cfg::{CheckBasis, CheckCtx, EXTENSIVE_ENV, GeneratorKind}; -pub use test_traits::{CheckOutput, GenerateInput, Hex, TupleCall}; +pub use test_traits::{CheckOutput, Hex, TupleCall}; /// Result type for tests is usually from `anyhow`. Most times there is no success value to /// propagate. diff --git a/crates/libm-test/src/run_cfg.rs b/crates/libm-test/src/run_cfg.rs index 46a6a1fad..9cede0cc7 100644 --- a/crates/libm-test/src/run_cfg.rs +++ b/crates/libm-test/src/run_cfg.rs @@ -1,8 +1,10 @@ //! Configuration for how tests get run. -use std::env; +use std::ops::RangeInclusive; use std::sync::LazyLock; +use std::{env, str}; +use crate::gen::random::{SEED, SEED_ENV}; use crate::{BaseName, FloatTy, Identifier, test_log}; /// The environment variable indicating which extensive tests should be run. @@ -188,9 +190,16 @@ pub fn iteration_count(ctx: &CheckCtx, gen_kind: GeneratorKind, argnum: usize) - }; let total = ntests.pow(t_env.input_count.try_into().unwrap()); + let seed_msg = match gen_kind { + GeneratorKind::Domain => String::new(), + GeneratorKind::Random => { + format!(" using `{SEED_ENV}={}`", str::from_utf8(SEED.as_slice()).unwrap()) + } + }; + test_log(&format!( "{gen_kind:?} {basis:?} {fn_ident} arg {arg}/{args}: {ntests} iterations \ - ({total} total)", + ({total} total){seed_msg}", basis = ctx.basis, fn_ident = ctx.fn_ident, arg = argnum + 1, @@ -200,6 +209,25 @@ pub fn iteration_count(ctx: &CheckCtx, gen_kind: GeneratorKind, argnum: usize) - ntests } +/// Some tests require that an integer be kept within reasonable limits; generate that here. +pub fn int_range(ctx: &CheckCtx, argnum: usize) -> RangeInclusive { + let t_env = TestEnv::from_env(ctx); + + if !matches!(ctx.base_name, BaseName::Jn | BaseName::Yn) { + return i32::MIN..=i32::MAX; + } + + assert_eq!(argnum, 0, "For `jn`/`yn`, only the first argument takes an integer"); + + // The integer argument to `jn` is an iteration count. Limit this to ensure tests can be + // completed in a reasonable amount of time. + if t_env.slow_platform || !cfg!(optimizations_enabled) { + (-0xf)..=0xff + } else { + (-0xff)..=0xffff + } +} + /// For domain tests, limit how many asymptotes or specified check points we test. pub fn check_point_count(ctx: &CheckCtx) -> usize { let t_env = TestEnv::from_env(ctx); diff --git a/crates/libm-test/src/test_traits.rs b/crates/libm-test/src/test_traits.rs index 6b833dfb5..261d1f254 100644 --- a/crates/libm-test/src/test_traits.rs +++ b/crates/libm-test/src/test_traits.rs @@ -1,8 +1,7 @@ //! Traits related to testing. //! -//! There are three main traits in this module: +//! There are two main traits in this module: //! -//! - `GenerateInput`: implemented on any types that create test cases. //! - `TupleCall`: implemented on tuples to allow calling them as function arguments. //! - `CheckOutput`: implemented on anything that is an output type for validation against an //! expected value. @@ -13,11 +12,6 @@ use anyhow::{Context, bail, ensure}; use crate::{CheckCtx, Float, Int, MaybeOverride, SpecialCase, TestResult}; -/// Implement this on types that can generate a sequence of tuples for test input. -pub trait GenerateInput { - fn get_cases(&self) -> impl Iterator; -} - /// Trait for calling a function with a tuple as arguments. /// /// Implemented on the tuple with the function signature as the generic (so we can use the same diff --git a/crates/libm-test/tests/compare_built_musl.rs b/crates/libm-test/tests/compare_built_musl.rs index 71f080ab1..ecd379a0a 100644 --- a/crates/libm-test/tests/compare_built_musl.rs +++ b/crates/libm-test/tests/compare_built_musl.rs @@ -9,8 +9,9 @@ // There are some targets we can't build musl for #![cfg(feature = "build-musl")] -use libm_test::gen::{CachedInput, random}; -use libm_test::{CheckBasis, CheckCtx, CheckOutput, GenerateInput, MathOp, TupleCall}; +use libm_test::gen::random; +use libm_test::gen::random::RandomInput; +use libm_test::{CheckBasis, CheckCtx, CheckOutput, MathOp, TupleCall}; macro_rules! musl_rand_tests { ( @@ -21,16 +22,16 @@ macro_rules! musl_rand_tests { #[test] $(#[$attr])* fn [< musl_random_ $fn_name >]() { - test_one::(musl_math_sys::$fn_name); + test_one_random::(musl_math_sys::$fn_name); } } }; } -fn test_one(musl_fn: Op::CFn) +fn test_one_random(musl_fn: Op::CFn) where Op: MathOp, - CachedInput: GenerateInput, + Op::RustArgs: RandomInput, { let ctx = CheckCtx::new(Op::IDENTIFIER, CheckBasis::Musl); let cases = random::get_test_cases::(&ctx); diff --git a/crates/libm-test/tests/multiprecision.rs b/crates/libm-test/tests/multiprecision.rs index 4cdba0942..960c370d4 100644 --- a/crates/libm-test/tests/multiprecision.rs +++ b/crates/libm-test/tests/multiprecision.rs @@ -3,11 +3,10 @@ #![cfg(feature = "test-multiprecision")] use libm_test::domain::HasDomain; -use libm_test::gen::{CachedInput, domain_logspace, edge_cases, random}; +use libm_test::gen::random::RandomInput; +use libm_test::gen::{domain_logspace, edge_cases, random}; use libm_test::mpfloat::MpOp; -use libm_test::{ - CheckBasis, CheckCtx, CheckOutput, GenerateInput, MathOp, OpFTy, OpRustFn, OpRustRet, TupleCall, -}; +use libm_test::{CheckBasis, CheckCtx, CheckOutput, MathOp, OpFTy, OpRustFn, OpRustRet, TupleCall}; /// Test against MPFR with random inputs. macro_rules! mp_rand_tests { @@ -29,7 +28,7 @@ macro_rules! mp_rand_tests { fn test_one_random() where Op: MathOp + MpOp, - CachedInput: GenerateInput, + Op::RustArgs: RandomInput, { let mut mp_vals = Op::new_mp(); let ctx = CheckCtx::new(Op::IDENTIFIER, CheckBasis::Mpfr); From ac3ff8cfdcab83a523b40c55330d36b470e9205b Mon Sep 17 00:00:00 2001 From: Trevor Gross Date: Mon, 6 Jan 2025 00:36:18 +0000 Subject: [PATCH 5/5] Update precision based on new test results --- crates/libm-test/src/precision.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/crates/libm-test/src/precision.rs b/crates/libm-test/src/precision.rs index 8bedcde44..a8efe1015 100644 --- a/crates/libm-test/src/precision.rs +++ b/crates/libm-test/src/precision.rs @@ -90,8 +90,15 @@ pub fn default_ulp(ctx: &CheckCtx) -> u32 { Bn::Exp10 if usize::BITS < 64 => ulp = 4, Bn::Lgamma | Bn::LgammaR => ulp = 400, Bn::Tanh => ulp = 4, - _ if ctx.fn_ident == Id::Sincosf => ulp = 500, - _ if ctx.fn_ident == Id::Tgamma => ulp = 20, + _ => (), + } + + match ctx.fn_ident { + // FIXME(#401): musl has an incorrect result here. + Id::Fdim => ulp = 2, + Id::Jnf | Id::Ynf => ulp = 4000, + Id::Sincosf => ulp = 500, + Id::Tgamma => ulp = 20, _ => (), } } @@ -99,6 +106,8 @@ pub fn default_ulp(ctx: &CheckCtx) -> u32 { // In some cases, our implementation is less accurate than musl on i586. if cfg!(x86_no_sse) { match ctx.fn_ident { + Id::Asinh => ulp = 3, + Id::Asinhf => ulp = 3, Id::Log1p | Id::Log1pf => ulp = 2, Id::Round => ulp = 1, Id::Tan => ulp = 2,