From a123fc16873d64376ce7ad718d08d586a65d313f Mon Sep 17 00:00:00 2001 From: marvinlanhenke Date: Mon, 1 Apr 2024 07:23:46 +0200 Subject: [PATCH 1/3] basic fix --- crates/iceberg/src/transform/temporal.rs | 36 ++++++++++++++++++++---- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/crates/iceberg/src/transform/temporal.rs b/crates/iceberg/src/transform/temporal.rs index 2a79db3009..c1c1bb8275 100644 --- a/crates/iceberg/src/transform/temporal.rs +++ b/crates/iceberg/src/transform/temporal.rs @@ -24,15 +24,15 @@ use arrow_array::{ types::Date32Type, Array, ArrayRef, Date32Array, Int32Array, TimestampMicrosecondArray, }; use arrow_schema::{DataType, TimeUnit}; -use chrono::{DateTime, Datelike}; +use chrono::{DateTime, Datelike, Duration}; use std::sync::Arc; /// Hour in one second. const HOUR_PER_SECOND: f64 = 1.0_f64 / 3600.0_f64; -/// Day in one second. -const DAY_PER_SECOND: f64 = 1.0_f64 / 24.0_f64 / 3600.0_f64; /// Year of unix epoch. const UNIX_EPOCH_YEAR: i32 = 1970; +/// One second in micros. +const MICROS_PER_SECOND: i64 = 1_000_000; /// Extract a date or timestamp year, as years from 1970 #[derive(Debug)] @@ -164,7 +164,33 @@ pub struct Day; impl Day { #[inline] fn day_timestamp_micro(v: i64) -> i32 { - (v as f64 / 1000.0 / 1000.0 * DAY_PER_SECOND) as i32 + let secs = v / MICROS_PER_SECOND; + + let (nanos, offset) = if v >= 0 { + let nanos = (v.rem_euclid(MICROS_PER_SECOND) * 1_000) as u32; + let offset = 0i64; + (nanos, offset) + } else { + let v = v + 1; + let nanos = (v.rem_euclid(MICROS_PER_SECOND) * 1_000) as u32; + let offset = 1i64; + (nanos, offset) + }; + + // TODO: Handle unwrap, return Result + let delta = Duration::new(secs, nanos) + .ok_or_else(|| { + Error::new( + ErrorKind::DataInvalid, + format!( + "Failed to create 'TimeDelta' from seconds {} and nanos {}", + secs, nanos + ), + ) + }) + .unwrap(); + + (delta.num_days() - offset) as i32 } } @@ -584,7 +610,7 @@ mod test { // Test TimestampMicrosecond test_timestamp_and_tz_transform_using_i64(1512151975038194, &day, Datum::int(17501)); - test_timestamp_and_tz_transform_using_i64(-115200000000, &day, Datum::int(-1)); + test_timestamp_and_tz_transform_using_i64(-115200000000, &day, Datum::int(-2)); test_timestamp_and_tz_transform("2017-12-01 10:30:42.123", &day, Datum::int(17501)); } From 3483f331425215fbd2bac677971ddf5495fa9459 Mon Sep 17 00:00:00 2001 From: marvinlanhenke Date: Mon, 1 Apr 2024 07:33:18 +0200 Subject: [PATCH 2/3] change to Result --- crates/iceberg/src/transform/temporal.rs | 33 ++++++++++++------------ 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/crates/iceberg/src/transform/temporal.rs b/crates/iceberg/src/transform/temporal.rs index c1c1bb8275..5933837994 100644 --- a/crates/iceberg/src/transform/temporal.rs +++ b/crates/iceberg/src/transform/temporal.rs @@ -163,7 +163,7 @@ pub struct Day; impl Day { #[inline] - fn day_timestamp_micro(v: i64) -> i32 { + fn day_timestamp_micro(v: i64) -> Result { let secs = v / MICROS_PER_SECOND; let (nanos, offset) = if v >= 0 { @@ -177,20 +177,19 @@ impl Day { (nanos, offset) }; - // TODO: Handle unwrap, return Result - let delta = Duration::new(secs, nanos) - .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!( - "Failed to create 'TimeDelta' from seconds {} and nanos {}", - secs, nanos - ), - ) - }) - .unwrap(); + let delta = Duration::new(secs, nanos).ok_or_else(|| { + Error::new( + ErrorKind::DataInvalid, + format!( + "Failed to create 'TimeDelta' from seconds {} and nanos {}", + secs, nanos + ), + ) + })?; + + let days = (delta.num_days() - offset) as i32; - (delta.num_days() - offset) as i32 + Ok(days) } } @@ -201,7 +200,7 @@ impl TransformFunction for Day { .as_any() .downcast_ref::() .unwrap() - .unary(|v| -> i32 { Self::day_timestamp_micro(v) }), + .unary(|v| -> i32 { Self::day_timestamp_micro(v).unwrap() }), DataType::Date32 => input .as_any() .downcast_ref::() @@ -223,8 +222,8 @@ impl TransformFunction for Day { fn transform_literal(&self, input: &crate::spec::Datum) -> Result> { let val = match input.literal() { PrimitiveLiteral::Date(v) => *v, - PrimitiveLiteral::Timestamp(v) => Self::day_timestamp_micro(*v), - PrimitiveLiteral::TimestampTZ(v) => Self::day_timestamp_micro(*v), + PrimitiveLiteral::Timestamp(v) => Self::day_timestamp_micro(*v)?, + PrimitiveLiteral::TimestampTZ(v) => Self::day_timestamp_micro(*v)?, _ => { return Err(crate::Error::new( crate::ErrorKind::FeatureUnsupported, From a55be8fbab81bcca364145fbd9879f9dee0280b4 Mon Sep 17 00:00:00 2001 From: marvinlanhenke Date: Mon, 1 Apr 2024 07:45:46 +0200 Subject: [PATCH 3/3] use try_unary --- crates/iceberg/src/transform/temporal.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/iceberg/src/transform/temporal.rs b/crates/iceberg/src/transform/temporal.rs index 5933837994..9c6489e916 100644 --- a/crates/iceberg/src/transform/temporal.rs +++ b/crates/iceberg/src/transform/temporal.rs @@ -200,7 +200,7 @@ impl TransformFunction for Day { .as_any() .downcast_ref::() .unwrap() - .unary(|v| -> i32 { Self::day_timestamp_micro(v).unwrap() }), + .try_unary(|v| -> Result { Self::day_timestamp_micro(v) })?, DataType::Date32 => input .as_any() .downcast_ref::()