From 9d68ed25fbf855273d507bd8703fe749f40851a6 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Thu, 25 Jul 2024 14:00:35 +0800 Subject: [PATCH 1/3] feat(iceberg): Add memory file IO support Signed-off-by: Xuanwo --- crates/iceberg/Cargo.toml | 5 +-- crates/iceberg/src/io/mod.rs | 4 +++ crates/iceberg/src/io/storage.rs | 19 +++++++++++ crates/iceberg/src/io/storage_memory.rs | 43 +++++++++++++++++++++++++ 4 files changed, 69 insertions(+), 2 deletions(-) create mode 100644 crates/iceberg/src/io/storage_memory.rs diff --git a/crates/iceberg/Cargo.toml b/crates/iceberg/Cargo.toml index 9098456df1..de5b7cdc5e 100644 --- a/crates/iceberg/Cargo.toml +++ b/crates/iceberg/Cargo.toml @@ -29,9 +29,10 @@ license = { workspace = true } keywords = ["iceberg"] [features] -default = ["storage-fs", "storage-s3", "tokio"] -storage-all = ["storage-fs", "storage-s3"] +default = ["storage-memory", "storage-fs", "storage-s3", "tokio"] +storage-all = ["storage-memory", "storage-fs", "storage-s3"] +storage-memory = ["opendal/services-memory"] storage-fs = ["opendal/services-fs"] storage-s3 = ["opendal/services-s3"] diff --git a/crates/iceberg/src/io/mod.rs b/crates/iceberg/src/io/mod.rs index 914293da3d..d299f069e8 100644 --- a/crates/iceberg/src/io/mod.rs +++ b/crates/iceberg/src/io/mod.rs @@ -52,6 +52,10 @@ mod file_io; pub use file_io::*; mod storage; +#[cfg(feature = "storage-memory")] +mod storage_memory; +#[cfg(feature = "storage-memory")] +use storage_memory::*; #[cfg(feature = "storage-s3")] mod storage_s3; #[cfg(feature = "storage-s3")] diff --git a/crates/iceberg/src/io/storage.rs b/crates/iceberg/src/io/storage.rs index 8d7df45b88..53bf5ba598 100644 --- a/crates/iceberg/src/io/storage.rs +++ b/crates/iceberg/src/io/storage.rs @@ -18,6 +18,8 @@ use super::FileIOBuilder; #[cfg(feature = "storage-fs")] use super::FsConfig; +#[cfg(feature = "storage-memory")] +use super::MemoryConfig; #[cfg(feature = "storage-s3")] use super::S3Config; use crate::{Error, ErrorKind}; @@ -26,6 +28,8 @@ use opendal::{Operator, Scheme}; /// The storage carries all supported storage services in iceberg #[derive(Debug)] pub(crate) enum Storage { + #[cfg(feature = "storage-memory")] + Memory { config: MemoryConfig }, #[cfg(feature = "storage-fs")] LocalFs { config: FsConfig }, #[cfg(feature = "storage-s3")] @@ -44,6 +48,10 @@ impl Storage { let scheme = Self::parse_scheme(&scheme_str)?; match scheme { + #[cfg(feature = "storage-memory")] + Scheme::Memory => Ok(Self::Memory { + config: MemoryConfig::new(props), + }), #[cfg(feature = "storage-fs")] Scheme::Fs => Ok(Self::LocalFs { config: FsConfig::new(props), @@ -79,6 +87,16 @@ impl Storage { ) -> crate::Result<(Operator, &'a str)> { let path = path.as_ref(); match self { + #[cfg(feature = "storage-memory")] + Storage::Memory { config } => { + let op = config.build(path)?; + + if let Some(stripped) = path.strip_prefix("file:/") { + Ok((op, stripped)) + } else { + Ok((op, &path[1..])) + } + } #[cfg(feature = "storage-fs")] Storage::LocalFs { config } => { let op = config.build(path)?; @@ -116,6 +134,7 @@ impl Storage { /// Parse scheme. fn parse_scheme(scheme: &str) -> crate::Result { match scheme { + "memory" => Ok(Scheme::Memory), "file" | "" => Ok(Scheme::Fs), "s3" | "s3a" => Ok(Scheme::S3), s => Ok(s.parse::()?), diff --git a/crates/iceberg/src/io/storage_memory.rs b/crates/iceberg/src/io/storage_memory.rs new file mode 100644 index 0000000000..07a576ab59 --- /dev/null +++ b/crates/iceberg/src/io/storage_memory.rs @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::Result; +use opendal::{Operator, Scheme}; +use std::collections::HashMap; +use std::fmt::{Debug, Formatter}; + +#[derive(Default, Clone)] +pub(crate) struct MemoryConfig {} + +impl Debug for MemoryConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("MemoryConfig").finish() + } +} + +impl MemoryConfig { + /// Decode from iceberg props. + pub fn new(_: HashMap) -> Self { + Self::default() + } + + /// Build new opendal operator from give path. + pub fn build(&self, _: &str) -> Result { + let m = HashMap::new(); + Ok(Operator::via_map(Scheme::Memory, m)?) + } +} From c84456283e7e3f99716344c75ecf3980fb40f4cd Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Thu, 25 Jul 2024 22:01:22 +0800 Subject: [PATCH 2/3] Fix typo Signed-off-by: Xuanwo --- crates/iceberg/src/io/storage.rs | 2 +- crates/iceberg/src/io/storage_memory.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/iceberg/src/io/storage.rs b/crates/iceberg/src/io/storage.rs index 53bf5ba598..3da95c0f18 100644 --- a/crates/iceberg/src/io/storage.rs +++ b/crates/iceberg/src/io/storage.rs @@ -91,7 +91,7 @@ impl Storage { Storage::Memory { config } => { let op = config.build(path)?; - if let Some(stripped) = path.strip_prefix("file:/") { + if let Some(stripped) = path.strip_prefix("memory:/") { Ok((op, stripped)) } else { Ok((op, &path[1..])) diff --git a/crates/iceberg/src/io/storage_memory.rs b/crates/iceberg/src/io/storage_memory.rs index 07a576ab59..eca39a7e86 100644 --- a/crates/iceberg/src/io/storage_memory.rs +++ b/crates/iceberg/src/io/storage_memory.rs @@ -35,7 +35,7 @@ impl MemoryConfig { Self::default() } - /// Build new opendal operator from give path. + /// Build new opendal operator from given path. pub fn build(&self, _: &str) -> Result { let m = HashMap::new(); Ok(Operator::via_map(Scheme::Memory, m)?) From 5524cbc4960c5d33aaba4af145ebb25db2103ede Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Fri, 26 Jul 2024 20:55:25 +0800 Subject: [PATCH 3/3] Add comments for memory file io Signed-off-by: Xuanwo --- crates/iceberg/src/expr/predicate.rs | 30 ++++---- crates/iceberg/src/expr/term.rs | 14 ---- crates/iceberg/src/io/mod.rs | 30 ++++++-- crates/iceberg/src/io/storage.rs | 1 - crates/iceberg/src/spec/datatypes.rs | 2 +- crates/iceberg/src/spec/partition.rs | 2 +- crates/iceberg/src/spec/snapshot.rs | 2 +- crates/iceberg/src/spec/sort.rs | 2 +- crates/iceberg/src/spec/values.rs | 111 +++++++++++++++------------ crates/iceberg/src/table.rs | 18 +++-- 10 files changed, 119 insertions(+), 93 deletions(-) diff --git a/crates/iceberg/src/expr/predicate.rs b/crates/iceberg/src/expr/predicate.rs index 3a91d6bbc3..270772615d 100644 --- a/crates/iceberg/src/expr/predicate.rs +++ b/crates/iceberg/src/expr/predicate.rs @@ -478,10 +478,10 @@ impl Predicate { /// # Example /// /// ```rust - /// use std::ops::Bound::Unbounded; /// use iceberg::expr::BoundPredicate::Unary; /// use iceberg::expr::Reference; /// use iceberg::spec::Datum; + /// use std::ops::Bound::Unbounded; /// let expr1 = Reference::new("a").less_than(Datum::long(10)); /// /// let expr2 = Reference::new("b").less_than(Datum::long(20)); @@ -505,10 +505,10 @@ impl Predicate { /// # Example /// /// ```rust - /// use std::ops::Bound::Unbounded; /// use iceberg::expr::BoundPredicate::Unary; /// use iceberg::expr::Reference; /// use iceberg::spec::Datum; + /// use std::ops::Bound::Unbounded; /// let expr1 = Reference::new("a").less_than(Datum::long(10)); /// /// let expr2 = Reference::new("b").less_than(Datum::long(20)); @@ -534,12 +534,14 @@ impl Predicate { /// # Example /// /// ```rust - /// use std::ops::Bound::Unbounded; /// use iceberg::expr::BoundPredicate::Unary; /// use iceberg::expr::{LogicalExpression, Predicate, Reference}; /// use iceberg::spec::Datum; + /// use std::ops::Bound::Unbounded; /// let expr1 = Reference::new("a").less_than(Datum::long(10)); - /// let expr2 = Reference::new("b").less_than(Datum::long(5)).and(Reference::new("c").less_than(Datum::long(10))); + /// let expr2 = Reference::new("b") + /// .less_than(Datum::long(5)) + /// .and(Reference::new("c").less_than(Datum::long(10))); /// /// let result = expr1.negate(); /// assert_eq!(&format!("{result}"), "a >= 10"); @@ -632,16 +634,16 @@ impl Not for Predicate { /// # Example /// ///```rust - ///use std::ops::Bound::Unbounded; - ///use iceberg::expr::BoundPredicate::Unary; - ///use iceberg::expr::Reference; - ///use iceberg::spec::Datum; - ///let expr1 = Reference::new("a").less_than(Datum::long(10)); - /// - ///let expr = !expr1; - /// - ///assert_eq!(&format!("{expr}"), "NOT (a < 10)"); - ///``` + /// use iceberg::expr::BoundPredicate::Unary; + /// use iceberg::expr::Reference; + /// use iceberg::spec::Datum; + /// use std::ops::Bound::Unbounded; + /// let expr1 = Reference::new("a").less_than(Datum::long(10)); + /// + /// let expr = !expr1; + /// + /// assert_eq!(&format!("{expr}"), "NOT (a < 10)"); + /// ``` fn not(self) -> Self::Output { Predicate::Not(LogicalExpression::new([Box::new(self)])) } diff --git a/crates/iceberg/src/expr/term.rs b/crates/iceberg/src/expr/term.rs index 909aa62bcd..00ace57177 100644 --- a/crates/iceberg/src/expr/term.rs +++ b/crates/iceberg/src/expr/term.rs @@ -56,7 +56,6 @@ impl Reference { /// # Example /// /// ```rust - /// /// use iceberg::expr::Reference; /// use iceberg::spec::Datum; /// let expr = Reference::new("a").less_than(Datum::long(10)); @@ -76,7 +75,6 @@ impl Reference { /// # Example /// /// ```rust - /// /// use iceberg::expr::Reference; /// use iceberg::spec::Datum; /// let expr = Reference::new("a").less_than_or_equal_to(Datum::long(10)); @@ -96,7 +94,6 @@ impl Reference { /// # Example /// /// ```rust - /// /// use iceberg::expr::Reference; /// use iceberg::spec::Datum; /// let expr = Reference::new("a").greater_than(Datum::long(10)); @@ -116,7 +113,6 @@ impl Reference { /// # Example /// /// ```rust - /// /// use iceberg::expr::Reference; /// use iceberg::spec::Datum; /// let expr = Reference::new("a").greater_than_or_equal_to(Datum::long(10)); @@ -136,7 +132,6 @@ impl Reference { /// # Example /// /// ```rust - /// /// use iceberg::expr::Reference; /// use iceberg::spec::Datum; /// let expr = Reference::new("a").equal_to(Datum::long(10)); @@ -152,7 +147,6 @@ impl Reference { /// # Example /// /// ```rust - /// /// use iceberg::expr::Reference; /// use iceberg::spec::Datum; /// let expr = Reference::new("a").not_equal_to(Datum::long(10)); @@ -168,7 +162,6 @@ impl Reference { /// # Example /// /// ```rust - /// /// use iceberg::expr::Reference; /// use iceberg::spec::Datum; /// let expr = Reference::new("a").starts_with(Datum::string("foo")); @@ -188,7 +181,6 @@ impl Reference { /// # Example /// /// ```rust - /// /// use iceberg::expr::Reference; /// use iceberg::spec::Datum; /// @@ -209,7 +201,6 @@ impl Reference { /// # Example /// /// ```rust - /// /// use iceberg::expr::Reference; /// use iceberg::spec::Datum; /// let expr = Reference::new("a").is_nan(); @@ -225,7 +216,6 @@ impl Reference { /// # Example /// /// ```rust - /// /// use iceberg::expr::Reference; /// use iceberg::spec::Datum; /// let expr = Reference::new("a").is_not_nan(); @@ -241,7 +231,6 @@ impl Reference { /// # Example /// /// ```rust - /// /// use iceberg::expr::Reference; /// use iceberg::spec::Datum; /// let expr = Reference::new("a").is_null(); @@ -257,7 +246,6 @@ impl Reference { /// # Example /// /// ```rust - /// /// use iceberg::expr::Reference; /// use iceberg::spec::Datum; /// let expr = Reference::new("a").is_not_null(); @@ -273,7 +261,6 @@ impl Reference { /// # Example /// /// ```rust - /// /// use fnv::FnvHashSet; /// use iceberg::expr::Reference; /// use iceberg::spec::Datum; @@ -295,7 +282,6 @@ impl Reference { /// # Example /// /// ```rust - /// /// use fnv::FnvHashSet; /// use iceberg::expr::Reference; /// use iceberg::spec::Datum; diff --git a/crates/iceberg/src/io/mod.rs b/crates/iceberg/src/io/mod.rs index d299f069e8..12ad8097e3 100644 --- a/crates/iceberg/src/io/mod.rs +++ b/crates/iceberg/src/io/mod.rs @@ -20,23 +20,41 @@ //! # How to build `FileIO` //! //! We provided a `FileIOBuilder` to build `FileIO` from scratch. For example: +//! //! ```rust //! use iceberg::io::{FileIOBuilder, S3_REGION}; +//! use iceberg::Result; //! +//! # fn test() -> Result<()> { +//! // Build a memory file io. +//! let file_io = FileIOBuilder::new("memory").build()?; +//! // Build an fs file io. +//! let file_io = FileIOBuilder::new("fs").build()?; +//! // Build an s3 file io. //! let file_io = FileIOBuilder::new("s3") //! .with_prop(S3_REGION, "us-east-1") -//! .build() -//! .unwrap(); +//! .build()?; +//! # Ok(()) +//! # } //! ``` //! //! Or you can pass a path to ask `FileIO` to infer schema for you: +//! //! ```rust //! use iceberg::io::{FileIO, S3_REGION}; -//! let file_io = FileIO::from_path("s3://bucket/a") -//! .unwrap() +//! use iceberg::Result; +//! +//! # fn test() -> Result<()> { +//! // Build a memory file io. +//! let file_io = FileIO::from_path("memory:///")?.build()?; +//! // Build an fs file io. +//! let file_io = FileIO::from_path("fs:///tmp")?.build()?; +//! // Build an s3 file io. +//! let file_io = FileIO::from_path("s3://bucket/a")? //! .with_prop(S3_REGION, "us-east-1") -//! .build() -//! .unwrap(); +//! .build()?; +//! # Ok(()) +//! # } //! ``` //! //! # How to use `FileIO` diff --git a/crates/iceberg/src/io/storage.rs b/crates/iceberg/src/io/storage.rs index 3da95c0f18..ad0dcb4572 100644 --- a/crates/iceberg/src/io/storage.rs +++ b/crates/iceberg/src/io/storage.rs @@ -80,7 +80,6 @@ impl Storage { /// /// * An [`opendal::Operator`] instance used to operate on file. /// * Relative path to the root uri of [`opendal::Operator`]. - /// pub(crate) fn create_operator<'a>( &self, path: &'a impl AsRef, diff --git a/crates/iceberg/src/spec/datatypes.rs b/crates/iceberg/src/spec/datatypes.rs index 80ac4a0a88..a79b8f57a8 100644 --- a/crates/iceberg/src/spec/datatypes.rs +++ b/crates/iceberg/src/spec/datatypes.rs @@ -17,7 +17,7 @@ /*! * Data Types -*/ + */ use crate::ensure_data_valid; use crate::error::Result; use crate::spec::datatypes::_decimal::{MAX_PRECISION, REQUIRED_LENGTH}; diff --git a/crates/iceberg/src/spec/partition.rs b/crates/iceberg/src/spec/partition.rs index 9388820a25..4be6311cab 100644 --- a/crates/iceberg/src/spec/partition.rs +++ b/crates/iceberg/src/spec/partition.rs @@ -17,7 +17,7 @@ /*! * Partitioning -*/ + */ use serde::{Deserialize, Serialize}; use std::sync::Arc; use typed_builder::TypedBuilder; diff --git a/crates/iceberg/src/spec/snapshot.rs b/crates/iceberg/src/spec/snapshot.rs index e0ff0794da..72df801aeb 100644 --- a/crates/iceberg/src/spec/snapshot.rs +++ b/crates/iceberg/src/spec/snapshot.rs @@ -17,7 +17,7 @@ /*! * Snapshots -*/ + */ use crate::error::Result; use chrono::{DateTime, TimeZone, Utc}; use serde::{Deserialize, Serialize}; diff --git a/crates/iceberg/src/spec/sort.rs b/crates/iceberg/src/spec/sort.rs index a4d2f7dc7a..a1311f7bf8 100644 --- a/crates/iceberg/src/spec/sort.rs +++ b/crates/iceberg/src/spec/sort.rs @@ -17,7 +17,7 @@ /*! * Sorting -*/ + */ use crate::error::Result; use crate::spec::Schema; use crate::{Error, ErrorKind}; diff --git a/crates/iceberg/src/spec/values.rs b/crates/iceberg/src/spec/values.rs index 310bec1d12..c9a99a0f8c 100644 --- a/crates/iceberg/src/spec/values.rs +++ b/crates/iceberg/src/spec/values.rs @@ -458,11 +458,14 @@ impl Datum { /// /// Example: /// ```rust - /// use iceberg::spec::{Literal, PrimitiveLiteral, Datum}; + /// use iceberg::spec::{Datum, Literal, PrimitiveLiteral}; /// let t = Datum::bool(true); /// /// assert_eq!(format!("{}", t), "true".to_string()); - /// assert_eq!(Literal::from(t), Literal::Primitive(PrimitiveLiteral::Boolean(true))); + /// assert_eq!( + /// Literal::from(t), + /// Literal::Primitive(PrimitiveLiteral::Boolean(true)) + /// ); /// ``` pub fn bool>(t: T) -> Self { Self { @@ -476,11 +479,14 @@ impl Datum { /// /// Example: /// ```rust - /// use iceberg::spec::{Literal, PrimitiveLiteral, Datum}; + /// use iceberg::spec::{Datum, Literal, PrimitiveLiteral}; /// let t = Datum::bool_from_str("false").unwrap(); /// /// assert_eq!(&format!("{}", t), "false"); - /// assert_eq!(Literal::Primitive(PrimitiveLiteral::Boolean(false)), t.into()); + /// assert_eq!( + /// Literal::Primitive(PrimitiveLiteral::Boolean(false)), + /// t.into() + /// ); /// ``` pub fn bool_from_str>(s: S) -> Result { let v = s.as_ref().parse::().map_err(|e| { @@ -493,7 +499,7 @@ impl Datum { /// /// Example: /// ```rust - /// use iceberg::spec::{Literal, PrimitiveLiteral, Datum}; + /// use iceberg::spec::{Datum, Literal, PrimitiveLiteral}; /// let t = Datum::int(23i8); /// /// assert_eq!(&format!("{}", t), "23"); @@ -510,7 +516,7 @@ impl Datum { /// /// Example: /// ```rust - /// use iceberg::spec::{Literal, PrimitiveLiteral, Datum}; + /// use iceberg::spec::{Datum, Literal, PrimitiveLiteral}; /// let t = Datum::long(24i8); /// /// assert_eq!(&format!("{t}"), "24"); @@ -527,12 +533,15 @@ impl Datum { /// /// Example: /// ```rust + /// use iceberg::spec::{Datum, Literal, PrimitiveLiteral}; /// use ordered_float::OrderedFloat; - /// use iceberg::spec::{Literal, PrimitiveLiteral, Datum}; - /// let t = Datum::float( 32.1f32 ); + /// let t = Datum::float(32.1f32); /// /// assert_eq!(&format!("{t}"), "32.1"); - /// assert_eq!(Literal::Primitive(PrimitiveLiteral::Float(OrderedFloat(32.1))), t.into()); + /// assert_eq!( + /// Literal::Primitive(PrimitiveLiteral::Float(OrderedFloat(32.1))), + /// t.into() + /// ); /// ``` pub fn float>(t: T) -> Self { Self { @@ -545,12 +554,15 @@ impl Datum { /// /// Example: /// ```rust + /// use iceberg::spec::{Datum, Literal, PrimitiveLiteral}; /// use ordered_float::OrderedFloat; - /// use iceberg::spec::{Literal, PrimitiveLiteral, Datum}; - /// let t = Datum::double( 32.1f64 ); + /// let t = Datum::double(32.1f64); /// /// assert_eq!(&format!("{t}"), "32.1"); - /// assert_eq!(Literal::Primitive(PrimitiveLiteral::Double(OrderedFloat(32.1))), t.into()); + /// assert_eq!( + /// Literal::Primitive(PrimitiveLiteral::Double(OrderedFloat(32.1))), + /// t.into() + /// ); /// ``` pub fn double>(t: T) -> Self { Self { @@ -563,8 +575,7 @@ impl Datum { /// /// Example: /// ```rust - /// - /// use iceberg::spec::{Literal, PrimitiveLiteral, Datum}; + /// use iceberg::spec::{Datum, Literal, PrimitiveLiteral}; /// // 2 days after 1970-01-01 /// let t = Datum::date(2); /// @@ -584,7 +595,7 @@ impl Datum { /// /// Example /// ```rust - /// use iceberg::spec::{Literal, Datum}; + /// use iceberg::spec::{Datum, Literal}; /// let t = Datum::date_from_str("1970-01-05").unwrap(); /// /// assert_eq!(&format!("{t}"), "1970-01-05"); @@ -609,7 +620,7 @@ impl Datum { /// Example: /// ///```rust - /// use iceberg::spec::{Literal, Datum}; + /// use iceberg::spec::{Datum, Literal}; /// let t = Datum::date_from_ymd(1970, 1, 5).unwrap(); /// /// assert_eq!(&format!("{t}"), "1970-01-05"); @@ -633,13 +644,13 @@ impl Datum { /// Example: /// /// ```rust - /// use iceberg::spec::{Literal, Datum}; + /// use iceberg::spec::{Datum, Literal}; /// let micro_secs = { /// 1 * 3600 * 1_000_000 + // 1 hour /// 2 * 60 * 1_000_000 + // 2 minutes /// 1 * 1_000_000 + // 1 second - /// 888999 // microseconds - /// }; + /// 888999 // microseconds + /// }; /// /// let t = Datum::time_micros(micro_secs).unwrap(); /// @@ -683,7 +694,7 @@ impl Datum { /// /// Example: /// ```rust - /// use iceberg::spec::{Literal, Datum}; + /// use iceberg::spec::{Datum, Literal}; /// let t = Datum::time_from_str("01:02:01.888999777").unwrap(); /// /// assert_eq!(&format!("{t}"), "01:02:01.888999"); @@ -706,8 +717,7 @@ impl Datum { /// /// Example: /// ```rust - /// - /// use iceberg::spec::{Literal, Datum}; + /// use iceberg::spec::{Datum, Literal}; /// let t = Datum::time_from_hms_micro(22, 15, 33, 111).unwrap(); /// /// assert_eq!(&format!("{t}"), "22:15:33.000111"); @@ -726,7 +736,6 @@ impl Datum { /// Example: /// /// ```rust - /// /// use iceberg::spec::Datum; /// let t = Datum::timestamp_micros(1000); /// @@ -744,14 +753,14 @@ impl Datum { /// Example: /// /// ```rust - /// /// use chrono::{NaiveDate, NaiveDateTime, TimeZone, Utc}; /// use iceberg::spec::Datum; /// let t = Datum::timestamp_from_datetime( /// NaiveDate::from_ymd_opt(1992, 3, 1) /// .unwrap() /// .and_hms_micro_opt(1, 2, 3, 88) - /// .unwrap()); + /// .unwrap(), + /// ); /// /// assert_eq!(&format!("{t}"), "1992-03-01 01:02:03.000088"); /// ``` @@ -767,7 +776,7 @@ impl Datum { /// /// ```rust /// use chrono::{DateTime, FixedOffset, NaiveDate, NaiveDateTime, NaiveTime}; - /// use iceberg::spec::{Literal, Datum}; + /// use iceberg::spec::{Datum, Literal}; /// let t = Datum::timestamp_from_str("1992-03-01T01:02:03.000088").unwrap(); /// /// assert_eq!(&format!("{t}"), "1992-03-01 01:02:03.000088"); @@ -785,7 +794,6 @@ impl Datum { /// Example: /// /// ```rust - /// /// use iceberg::spec::Datum; /// let t = Datum::timestamptz_micros(1000); /// @@ -802,7 +810,6 @@ impl Datum { /// Example: /// /// ```rust - /// /// use chrono::{TimeZone, Utc}; /// use iceberg::spec::Datum; /// let t = Datum::timestamptz_from_datetime(Utc.timestamp_opt(1000, 0).unwrap()); @@ -821,7 +828,7 @@ impl Datum { /// /// ```rust /// use chrono::{DateTime, FixedOffset, NaiveDate, NaiveDateTime, NaiveTime}; - /// use iceberg::spec::{Literal, Datum}; + /// use iceberg::spec::{Datum, Literal}; /// let t = Datum::timestamptz_from_str("1992-03-01T01:02:03.000088+08:00").unwrap(); /// /// assert_eq!(&format!("{t}"), "1992-02-29 17:02:03.000088 UTC"); @@ -856,8 +863,8 @@ impl Datum { /// Example: /// /// ```rust - /// use uuid::uuid; /// use iceberg::spec::Datum; + /// use uuid::uuid; /// let t = Datum::uuid(uuid!("a1a2a3a4-b1b2-c1c2-d1d2-d3d4d5d6d7d8")); /// /// assert_eq!(&format!("{t}"), "a1a2a3a4-b1b2-c1c2-d1d2-d3d4d5d6d7d8"); @@ -874,7 +881,7 @@ impl Datum { /// Example: /// /// ```rust - /// use iceberg::spec::{Datum}; + /// use iceberg::spec::Datum; /// let t = Datum::uuid_from_str("a1a2a3a4-b1b2-c1c2-d1d2-d3d4d5d6d7d8").unwrap(); /// /// assert_eq!(&format!("{t}"), "a1a2a3a4-b1b2-c1c2-d1d2-d3d4d5d6d7d8"); @@ -895,7 +902,7 @@ impl Datum { /// Example: /// /// ```rust - /// use iceberg::spec::{Literal, PrimitiveLiteral, Datum}; + /// use iceberg::spec::{Datum, Literal, PrimitiveLiteral}; /// let t = Datum::fixed(vec![1u8, 2u8]); /// /// assert_eq!(&format!("{t}"), "0102"); @@ -930,9 +937,9 @@ impl Datum { /// Example: /// /// ```rust + /// use iceberg::spec::Datum; /// use itertools::assert_equal; /// use rust_decimal::Decimal; - /// use iceberg::spec::Datum; /// let t = Datum::decimal_from_str("123.45").unwrap(); /// /// assert_eq!(&format!("{t}"), "123.45"); @@ -950,8 +957,8 @@ impl Datum { /// Example: /// /// ```rust - /// use rust_decimal::Decimal; /// use iceberg::spec::Datum; + /// use rust_decimal::Decimal; /// /// let t = Datum::decimal(Decimal::new(123, 2)).unwrap(); /// @@ -1194,11 +1201,14 @@ impl Literal { /// /// Example: /// ```rust - /// use ordered_float::OrderedFloat; /// use iceberg::spec::{Literal, PrimitiveLiteral}; - /// let t = Literal::float( 32.1f32 ); + /// use ordered_float::OrderedFloat; + /// let t = Literal::float(32.1f32); /// - /// assert_eq!(Literal::Primitive(PrimitiveLiteral::Float(OrderedFloat(32.1))), t); + /// assert_eq!( + /// Literal::Primitive(PrimitiveLiteral::Float(OrderedFloat(32.1))), + /// t + /// ); /// ``` pub fn float>(t: T) -> Self { Self::Primitive(PrimitiveLiteral::Float(OrderedFloat(t.into()))) @@ -1208,11 +1218,14 @@ impl Literal { /// /// Example: /// ```rust - /// use ordered_float::OrderedFloat; /// use iceberg::spec::{Literal, PrimitiveLiteral}; - /// let t = Literal::double( 32.1f64 ); + /// use ordered_float::OrderedFloat; + /// let t = Literal::double(32.1f64); /// - /// assert_eq!(Literal::Primitive(PrimitiveLiteral::Double(OrderedFloat(32.1))), t); + /// assert_eq!( + /// Literal::Primitive(PrimitiveLiteral::Double(OrderedFloat(32.1))), + /// t + /// ); /// ``` pub fn double>(t: T) -> Self { Self::Primitive(PrimitiveLiteral::Double(OrderedFloat(t.into()))) @@ -1296,7 +1309,7 @@ impl Literal { /// 1 * 3600 * 1_000_000 + // 1 hour /// 2 * 60 * 1_000_000 + // 2 minutes /// 1 * 1_000_000 + // 1 second - /// 888999 // microseconds + /// 888999 // microseconds /// }; /// assert_eq!(Literal::time(micro_secs), t); /// ``` @@ -1318,7 +1331,6 @@ impl Literal { /// /// Example: /// ```rust - /// /// use iceberg::spec::Literal; /// let t = Literal::time_from_hms_micro(22, 15, 33, 111).unwrap(); /// @@ -1365,10 +1377,13 @@ impl Literal { /// let t = Literal::timestamp_from_str("2012-12-12 12:12:12.8899-04:00").unwrap(); /// /// let t2 = { - /// let date = NaiveDate::from_ymd_opt(2012, 12, 12).unwrap(); - /// let time = NaiveTime::from_hms_micro_opt(12, 12, 12, 889900).unwrap(); - /// let dt = NaiveDateTime::new(date, time); - /// Literal::timestamp_from_datetime(DateTime::::from_local(dt, FixedOffset::west_opt(4 * 3600).unwrap())) + /// let date = NaiveDate::from_ymd_opt(2012, 12, 12).unwrap(); + /// let time = NaiveTime::from_hms_micro_opt(12, 12, 12, 889900).unwrap(); + /// let dt = NaiveDateTime::new(date, time); + /// Literal::timestamp_from_datetime(DateTime::::from_local( + /// dt, + /// FixedOffset::west_opt(4 * 3600).unwrap(), + /// )) /// }; /// /// assert_eq!(t, t2); @@ -1405,8 +1420,8 @@ impl Literal { /// Example: /// /// ```rust - /// use uuid::Uuid; /// use iceberg::spec::Literal; + /// use uuid::Uuid; /// let t1 = Literal::uuid_from_str("a1a2a3a4-b1b2-c1c2-d1d2-d3d4d5d6d7d8").unwrap(); /// let t2 = Literal::uuid(Uuid::from_u128_le(0xd8d7d6d5d4d3d2d1c2c1b2b1a4a3a2a1)); /// @@ -1463,8 +1478,8 @@ impl Literal { /// Example: /// /// ```rust - /// use rust_decimal::Decimal; /// use iceberg::spec::Literal; + /// use rust_decimal::Decimal; /// let t1 = Literal::decimal(12345); /// let t2 = Literal::decimal_from_str("123.45").unwrap(); /// diff --git a/crates/iceberg/src/table.rs b/crates/iceberg/src/table.rs index c76b286125..e804878ce4 100644 --- a/crates/iceberg/src/table.rs +++ b/crates/iceberg/src/table.rs @@ -88,14 +88,20 @@ impl Table { /// # use iceberg::TableIdent; /// # async fn example() { /// let metadata_file_location = "s3://bucket_name/path/to/metadata.json"; -/// let file_io = FileIO::from_path(&metadata_file_location).unwrap().build().unwrap(); +/// let file_io = FileIO::from_path(&metadata_file_location) +/// .unwrap() +/// .build() +/// .unwrap(); /// let static_identifier = TableIdent::from_strs(["static_ns", "static_table"]).unwrap(); -/// let static_table = StaticTable::from_metadata_file(&metadata_file_location, static_identifier, file_io).await.unwrap(); +/// let static_table = +/// StaticTable::from_metadata_file(&metadata_file_location, static_identifier, file_io) +/// .await +/// .unwrap(); /// let snapshot_id = static_table -/// .metadata() -/// .current_snapshot() -/// .unwrap() -/// .snapshot_id(); +/// .metadata() +/// .current_snapshot() +/// .unwrap() +/// .snapshot_id(); /// # } /// ``` pub struct StaticTable(Table);