From 26f2f9683dd8c6e02bc1c77b9d3d86f75ac7ab0d Mon Sep 17 00:00:00 2001 From: osipovartem Date: Wed, 19 Feb 2025 12:21:27 +0300 Subject: [PATCH 1/2] fix timestamp format --- crates/control_plane/src/models/mod.rs | 3 ++- crates/control_plane/src/utils.rs | 34 +++++++++++++++++--------- 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/crates/control_plane/src/models/mod.rs b/crates/control_plane/src/models/mod.rs index 403ed99f7..b44d00837 100644 --- a/crates/control_plane/src/models/mod.rs +++ b/crates/control_plane/src/models/mod.rs @@ -454,7 +454,8 @@ impl ColumnInfo { | DataType::UInt8 | DataType::UInt16 | DataType::UInt32 - | DataType::UInt64 => { + | DataType::UInt64 + | DataType::Float32 => { column_info.r#type = "fixed".to_string(); column_info.precision = Some(38); column_info.scale = Some(0); diff --git a/crates/control_plane/src/utils.rs b/crates/control_plane/src/utils.rs index 5b553f53a..a9ab724b4 100644 --- a/crates/control_plane/src/utils.rs +++ b/crates/control_plane/src/utils.rs @@ -11,6 +11,8 @@ use datafusion::arrow::datatypes::DataType; use datafusion::common::Result as DataFusionResult; use std::sync::Arc; +const TIMESTAMP_FORMAT: &str = "%Y-%m-%d-%H:%M:%S%.6f"; + #[must_use] pub fn first_non_empty_type(union_array: &UnionArray) -> Option<(DataType, ArrayRef)> { for i in 0..union_array.type_ids().len() { @@ -73,8 +75,6 @@ pub fn convert_record_batches( columns.push(converted_column); } let new_schema = Arc::new(Schema::new(fields)); - //println!("new schema: {:?}", new_schema); - //println!("columns: {:?}", columns); let converted_batch = RecordBatch::try_new(new_schema, columns)?; converted_batches.push(converted_batch); } @@ -97,7 +97,7 @@ fn convert_timestamp_to_struct(column: &ArrayRef, unit: TimeUnit) -> ArrayRef { .map(|x| { x.map(|ts| { let ts = DateTime::from_timestamp(ts, 0).unwrap(); - format!("{}", ts.timestamp()) + ts.format(TIMESTAMP_FORMAT).to_string() }) }) .collect(), @@ -109,7 +109,7 @@ fn convert_timestamp_to_struct(column: &ArrayRef, unit: TimeUnit) -> ArrayRef { .map(|x| { x.map(|ts| { let ts = DateTime::from_timestamp_millis(ts).unwrap(); - format!("{}.{}", ts.timestamp(), ts.timestamp_subsec_millis()) + ts.format(TIMESTAMP_FORMAT).to_string() }) }) .collect(), @@ -121,7 +121,7 @@ fn convert_timestamp_to_struct(column: &ArrayRef, unit: TimeUnit) -> ArrayRef { .map(|x| { x.map(|ts| { let ts = DateTime::from_timestamp_micros(ts).unwrap(); - format!("{}.{}", ts.timestamp(), ts.timestamp_subsec_micros()) + ts.format(TIMESTAMP_FORMAT).to_string() }) }) .collect(), @@ -133,7 +133,7 @@ fn convert_timestamp_to_struct(column: &ArrayRef, unit: TimeUnit) -> ArrayRef { .map(|x| { x.map(|ts| { let ts = DateTime::from_timestamp_nanos(ts); - format!("{}.{}", ts.timestamp(), ts.timestamp_subsec_nanos()) + ts.format(TIMESTAMP_FORMAT).to_string() }) }) .collect(), @@ -178,21 +178,25 @@ mod tests { #[test] fn test_convert_timestamp_to_struct() { let cases = [ - (TimeUnit::Second, Some(1_627_846_261), "1627846261"), + ( + TimeUnit::Second, + Some(1_627_846_261), + "2021-08-01-19:31:01.000000", + ), ( TimeUnit::Millisecond, Some(1_627_846_261_233), - "1627846261.233", + "2021-08-01-19:31:01.233000", ), ( TimeUnit::Microsecond, Some(1_627_846_261_233_222), - "1627846261.233222", + "2021-08-01-19:31:01.233222", ), ( TimeUnit::Nanosecond, Some(1_627_846_261_233_222_111), - "1627846261.233222111", + "2021-08-01-19:31:01.233222", ), ]; for (unit, timestamp, expected) in &cases { @@ -247,9 +251,15 @@ mod tests { .as_any() .downcast_ref::() .unwrap(); - assert_eq!(converted_timestamp_array.value(0), "1627846261"); + assert_eq!( + converted_timestamp_array.value(0), + "2021-08-01-19:31:01.000000" + ); assert!(converted_timestamp_array.is_null(1)); - assert_eq!(converted_timestamp_array.value(2), "1627846262"); + assert_eq!( + converted_timestamp_array.value(2), + "2021-08-01-19:31:02.000000" + ); assert_eq!(column_infos[0].name, "int_col"); assert_eq!(column_infos[0].r#type, "fixed"); From dd1e3d79e4b7f17f228f68608636b00a3aca40ee Mon Sep 17 00:00:00 2001 From: osipovartem Date: Wed, 19 Feb 2025 13:15:28 +0300 Subject: [PATCH 2/2] Revert timestamp change --- crates/control_plane/src/utils.rs | 32 ++++++++++--------------------- 1 file changed, 10 insertions(+), 22 deletions(-) diff --git a/crates/control_plane/src/utils.rs b/crates/control_plane/src/utils.rs index a9ab724b4..5529b5bda 100644 --- a/crates/control_plane/src/utils.rs +++ b/crates/control_plane/src/utils.rs @@ -11,8 +11,6 @@ use datafusion::arrow::datatypes::DataType; use datafusion::common::Result as DataFusionResult; use std::sync::Arc; -const TIMESTAMP_FORMAT: &str = "%Y-%m-%d-%H:%M:%S%.6f"; - #[must_use] pub fn first_non_empty_type(union_array: &UnionArray) -> Option<(DataType, ArrayRef)> { for i in 0..union_array.type_ids().len() { @@ -97,7 +95,7 @@ fn convert_timestamp_to_struct(column: &ArrayRef, unit: TimeUnit) -> ArrayRef { .map(|x| { x.map(|ts| { let ts = DateTime::from_timestamp(ts, 0).unwrap(); - ts.format(TIMESTAMP_FORMAT).to_string() + format!("{}", ts.timestamp()) }) }) .collect(), @@ -109,7 +107,7 @@ fn convert_timestamp_to_struct(column: &ArrayRef, unit: TimeUnit) -> ArrayRef { .map(|x| { x.map(|ts| { let ts = DateTime::from_timestamp_millis(ts).unwrap(); - ts.format(TIMESTAMP_FORMAT).to_string() + format!("{}.{}", ts.timestamp(), ts.timestamp_subsec_millis()) }) }) .collect(), @@ -121,7 +119,7 @@ fn convert_timestamp_to_struct(column: &ArrayRef, unit: TimeUnit) -> ArrayRef { .map(|x| { x.map(|ts| { let ts = DateTime::from_timestamp_micros(ts).unwrap(); - ts.format(TIMESTAMP_FORMAT).to_string() + format!("{}.{}", ts.timestamp(), ts.timestamp_subsec_micros()) }) }) .collect(), @@ -133,7 +131,7 @@ fn convert_timestamp_to_struct(column: &ArrayRef, unit: TimeUnit) -> ArrayRef { .map(|x| { x.map(|ts| { let ts = DateTime::from_timestamp_nanos(ts); - ts.format(TIMESTAMP_FORMAT).to_string() + format!("{}.{}", ts.timestamp(), ts.timestamp_subsec_nanos()) }) }) .collect(), @@ -178,25 +176,21 @@ mod tests { #[test] fn test_convert_timestamp_to_struct() { let cases = [ - ( - TimeUnit::Second, - Some(1_627_846_261), - "2021-08-01-19:31:01.000000", - ), + (TimeUnit::Second, Some(1_627_846_261), "1627846261"), ( TimeUnit::Millisecond, Some(1_627_846_261_233), - "2021-08-01-19:31:01.233000", + "1627846261.233", ), ( TimeUnit::Microsecond, Some(1_627_846_261_233_222), - "2021-08-01-19:31:01.233222", + "1627846261.233222", ), ( TimeUnit::Nanosecond, Some(1_627_846_261_233_222_111), - "2021-08-01-19:31:01.233222", + "1627846261.233222111", ), ]; for (unit, timestamp, expected) in &cases { @@ -251,15 +245,9 @@ mod tests { .as_any() .downcast_ref::() .unwrap(); - assert_eq!( - converted_timestamp_array.value(0), - "2021-08-01-19:31:01.000000" - ); + assert_eq!(converted_timestamp_array.value(0), "1627846261"); assert!(converted_timestamp_array.is_null(1)); - assert_eq!( - converted_timestamp_array.value(2), - "2021-08-01-19:31:02.000000" - ); + assert_eq!(converted_timestamp_array.value(2), "1627846262"); assert_eq!(column_infos[0].name, "int_col"); assert_eq!(column_infos[0].r#type, "fixed");