From 9ed55da66886a661d29d2b90906bb81b9bced092 Mon Sep 17 00:00:00 2001 From: Sung Yun <107272191+sungwy@users.noreply.github.com> Date: Mon, 16 Dec 2024 18:40:48 +0000 Subject: [PATCH 1/3] deprecate functions --- crates/catalog/rest/src/catalog.rs | 36 ------------------- crates/iceberg/src/catalog/mod.rs | 8 ----- crates/iceberg/src/spec/snapshot.rs | 8 ----- crates/iceberg/src/spec/table_metadata.rs | 32 ++--------------- .../src/spec/table_metadata_builder.rs | 13 ++----- 5 files changed, 4 insertions(+), 93 deletions(-) diff --git a/crates/catalog/rest/src/catalog.rs b/crates/catalog/rest/src/catalog.rs index 96da5dc951..7405e5350d 100644 --- a/crates/catalog/rest/src/catalog.rs +++ b/crates/catalog/rest/src/catalog.rs @@ -73,12 +73,6 @@ impl RestCatalogConfig { pub(crate) fn get_token_endpoint(&self) -> String { if let Some(oauth2_uri) = self.props.get("oauth2-server-uri") { oauth2_uri.to_string() - } else if let Some(auth_url) = self.props.get("rest.authorization-url") { - log::warn!( - "'rest.authorization-url' is deprecated and will be removed in version 0.4.0. \ - Please use 'oauth2-server-uri' instead." - ); - auth_url.to_string() } else { [&self.uri, PATH_V1, "oauth", "tokens"].join("/") } @@ -924,36 +918,6 @@ mod tests { assert_eq!(headers, expected_headers); } - #[tokio::test] - async fn test_oauth_with_deprecated_auth_url() { - let mut server = Server::new_async().await; - let config_mock = create_config_mock(&mut server).await; - - let mut auth_server = Server::new_async().await; - let auth_server_path = "/some/path"; - let oauth_mock = create_oauth_mock_with_path(&mut auth_server, auth_server_path).await; - - let mut props = HashMap::new(); - props.insert("credential".to_string(), "client1:secret1".to_string()); - props.insert( - "rest.authorization-url".to_string(), - format!("{}{}", auth_server.url(), auth_server_path).to_string(), - ); - - let catalog = RestCatalog::new( - RestCatalogConfig::builder() - .uri(server.url()) - .props(props) - .build(), - ); - - let token = catalog.context().await.unwrap().client.token().await; - - oauth_mock.assert_async().await; - config_mock.assert_async().await; - assert_eq!(token, Some("ey000000000000".to_string())); - } - #[tokio::test] async fn test_oauth_with_oauth2_server_uri() { let mut server = Server::new_async().await; diff --git a/crates/iceberg/src/catalog/mod.rs b/crates/iceberg/src/catalog/mod.rs index deb4d2f34b..ffafc66fa4 100644 --- a/crates/iceberg/src/catalog/mod.rs +++ b/crates/iceberg/src/catalog/mod.rs @@ -367,12 +367,6 @@ pub enum TableUpdate { AddSchema { /// The schema to add. schema: Schema, - /// The last column id of the table. - #[deprecated( - since = "0.3.0", - note = "This field is handled internally, and should not be part of the update." - )] - last_column_id: Option, }, /// Set table's current schema #[serde(rename_all = "kebab-case")] @@ -1301,7 +1295,6 @@ mod tests { "#, TableUpdate::AddSchema { schema: test_schema.clone(), - last_column_id: Some(3), }, ); @@ -1340,7 +1333,6 @@ mod tests { "#, TableUpdate::AddSchema { schema: test_schema.clone(), - last_column_id: None, }, ); } diff --git a/crates/iceberg/src/spec/snapshot.rs b/crates/iceberg/src/spec/snapshot.rs index 81fd6eae66..40156a7990 100644 --- a/crates/iceberg/src/spec/snapshot.rs +++ b/crates/iceberg/src/spec/snapshot.rs @@ -26,7 +26,6 @@ use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use typed_builder::TypedBuilder; -use super::table_metadata::SnapshotLog; use crate::error::{timestamp_ms_to_utc, Result}; use crate::io::FileIO; use crate::spec::{ManifestList, SchemaId, SchemaRef, StructType, TableMetadata}; @@ -192,13 +191,6 @@ impl Snapshot { partition_type_provider, ) } - - pub(crate) fn log(&self) -> SnapshotLog { - SnapshotLog { - timestamp_ms: self.timestamp_ms, - snapshot_id: self.snapshot_id, - } - } } pub(super) mod _serde { diff --git a/crates/iceberg/src/spec/table_metadata.rs b/crates/iceberg/src/spec/table_metadata.rs index 38204fc154..7b35a6d43a 100644 --- a/crates/iceberg/src/spec/table_metadata.rs +++ b/crates/iceberg/src/spec/table_metadata.rs @@ -33,9 +33,8 @@ use uuid::Uuid; use super::snapshot::SnapshotReference; pub use super::table_metadata_builder::{TableMetadataBuildResult, TableMetadataBuilder}; use super::{ - PartitionSpecRef, PartitionStatisticsFile, SchemaId, SchemaRef, Snapshot, SnapshotRef, - SnapshotRetention, SortOrder, SortOrderRef, StatisticsFile, StructType, - DEFAULT_PARTITION_SPEC_ID, + PartitionSpecRef, PartitionStatisticsFile, SchemaId, SchemaRef, SnapshotRef, SortOrder, + SortOrderRef, StatisticsFile, StructType, DEFAULT_PARTITION_SPEC_ID, }; use crate::error::{timestamp_ms_to_utc, Result}; use crate::{Error, ErrorKind}; @@ -398,33 +397,6 @@ impl TableMetadata { self.partition_statistics.get(&snapshot_id) } - /// Append snapshot to table - #[deprecated( - since = "0.4.0", - note = "please use `TableMetadataBuilder.set_branch_snapshot` instead" - )] - pub fn append_snapshot(&mut self, snapshot: Snapshot) { - self.last_updated_ms = snapshot.timestamp_ms(); - self.last_sequence_number = snapshot.sequence_number(); - - self.refs - .entry(MAIN_BRANCH.to_string()) - .and_modify(|s| { - s.snapshot_id = snapshot.snapshot_id(); - }) - .or_insert_with(|| { - SnapshotReference::new(snapshot.snapshot_id(), SnapshotRetention::Branch { - min_snapshots_to_keep: None, - max_snapshot_age_ms: None, - max_ref_age_ms: None, - }) - }); - - self.snapshot_log.push(snapshot.log()); - self.snapshots - .insert(snapshot.snapshot_id(), Arc::new(snapshot)); - } - /// Normalize this partition spec. /// /// This is an internal method diff --git a/crates/iceberg/src/spec/table_metadata_builder.rs b/crates/iceberg/src/spec/table_metadata_builder.rs index 4f200c8b27..5b4ff1234d 100644 --- a/crates/iceberg/src/spec/table_metadata_builder.rs +++ b/crates/iceberg/src/spec/table_metadata_builder.rs @@ -585,7 +585,6 @@ impl TableMetadataBuilder { if schema_found { if self.last_added_schema_id != Some(new_schema_id) { self.changes.push(TableUpdate::AddSchema { - last_column_id: Some(self.metadata.last_column_id), schema: schema.clone(), }); self.last_added_schema_id = Some(new_schema_id); @@ -609,10 +608,7 @@ impl TableMetadataBuilder { .schemas .insert(new_schema_id, schema.clone().into()); - self.changes.push(TableUpdate::AddSchema { - schema, - last_column_id: Some(self.metadata.last_column_id), - }); + self.changes.push(TableUpdate::AddSchema { schema }); self.last_added_schema_id = Some(new_schema_id); @@ -1453,10 +1449,7 @@ mod tests { TableUpdate::SetLocation { location: TEST_LOCATION.to_string() }, - TableUpdate::AddSchema { - last_column_id: Some(LAST_ASSIGNED_COLUMN_ID), - schema: schema(), - }, + TableUpdate::AddSchema { schema: schema() }, TableUpdate::SetCurrentSchema { schema_id: -1 }, TableUpdate::AddSpec { // Because this is a new tables, field-ids are assigned @@ -1509,7 +1502,6 @@ mod tests { location: TEST_LOCATION.to_string() }, TableUpdate::AddSchema { - last_column_id: Some(0), schema: Schema::builder().build().unwrap(), }, TableUpdate::SetCurrentSchema { schema_id: -1 }, @@ -1751,7 +1743,6 @@ mod tests { Some(&Arc::new(added_schema.clone())) ); pretty_assertions::assert_eq!(build_result.changes[0], TableUpdate::AddSchema { - last_column_id: Some(4), schema: added_schema }); assert_eq!(build_result.changes[1], TableUpdate::SetCurrentSchema { From d0434080351d044cd290e20af9c136f7704916f0 Mon Sep 17 00:00:00 2001 From: Sung Yun <107272191+sungwy@users.noreply.github.com> Date: Mon, 16 Dec 2024 18:40:57 +0000 Subject: [PATCH 2/3] bump version to 0.4.0 --- Cargo.toml | 12 ++++++------ README.md | 2 -- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index c766040d7c..05f2d9073e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,7 +28,7 @@ members = [ exclude = ["bindings/python"] [workspace.package] -version = "0.3.0" +version = "0.4.0" edition = "2021" homepage = "https://rust.iceberg.apache.org/" @@ -62,11 +62,11 @@ either = "1" env_logger = "0.11.0" fnv = "1" futures = "0.3" -iceberg = { version = "0.3.0", path = "./crates/iceberg" } -iceberg-catalog-rest = { version = "0.3.0", path = "./crates/catalog/rest" } -iceberg-catalog-hms = { version = "0.3.0", path = "./crates/catalog/hms" } -iceberg-catalog-memory = { version = "0.3.0", path = "./crates/catalog/memory" } -iceberg-datafusion = { version = "0.3.0", path = "./crates/integrations/datafusion" } +iceberg = { version = "0.4.0", path = "./crates/iceberg" } +iceberg-catalog-rest = { version = "0.4.0", path = "./crates/catalog/rest" } +iceberg-catalog-hms = { version = "0.4.0", path = "./crates/catalog/hms" } +iceberg-catalog-memory = { version = "0.4.0", path = "./crates/catalog/memory" } +iceberg-datafusion = { version = "0.4.0", path = "./crates/integrations/datafusion" } itertools = "0.13" log = "0.4" mockito = "1" diff --git a/README.md b/README.md index cef0bb5dd1..d7f32e2ef2 100644 --- a/README.md +++ b/README.md @@ -23,8 +23,6 @@ Rust implementation of [Apache Iceberg™](https://iceberg.apache.org/). -Working on [v0.3.0 Release Milestone](https://github.com/apache/iceberg-rust/milestone/2) - ## Components The Apache Iceberg Rust project is composed of the following components: From e428a8281e7680146aeae85a8671cfdcdc60597e Mon Sep 17 00:00:00 2001 From: Sung Yun <107272191+sungwy@users.noreply.github.com> Date: Mon, 16 Dec 2024 20:56:06 +0000 Subject: [PATCH 3/3] revert --- crates/iceberg/src/spec/snapshot.rs | 8 ++++++ crates/iceberg/src/spec/table_metadata.rs | 32 +++++++++++++++++++++-- 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/crates/iceberg/src/spec/snapshot.rs b/crates/iceberg/src/spec/snapshot.rs index 40156a7990..81fd6eae66 100644 --- a/crates/iceberg/src/spec/snapshot.rs +++ b/crates/iceberg/src/spec/snapshot.rs @@ -26,6 +26,7 @@ use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use typed_builder::TypedBuilder; +use super::table_metadata::SnapshotLog; use crate::error::{timestamp_ms_to_utc, Result}; use crate::io::FileIO; use crate::spec::{ManifestList, SchemaId, SchemaRef, StructType, TableMetadata}; @@ -191,6 +192,13 @@ impl Snapshot { partition_type_provider, ) } + + pub(crate) fn log(&self) -> SnapshotLog { + SnapshotLog { + timestamp_ms: self.timestamp_ms, + snapshot_id: self.snapshot_id, + } + } } pub(super) mod _serde { diff --git a/crates/iceberg/src/spec/table_metadata.rs b/crates/iceberg/src/spec/table_metadata.rs index 7b35a6d43a..38204fc154 100644 --- a/crates/iceberg/src/spec/table_metadata.rs +++ b/crates/iceberg/src/spec/table_metadata.rs @@ -33,8 +33,9 @@ use uuid::Uuid; use super::snapshot::SnapshotReference; pub use super::table_metadata_builder::{TableMetadataBuildResult, TableMetadataBuilder}; use super::{ - PartitionSpecRef, PartitionStatisticsFile, SchemaId, SchemaRef, SnapshotRef, SortOrder, - SortOrderRef, StatisticsFile, StructType, DEFAULT_PARTITION_SPEC_ID, + PartitionSpecRef, PartitionStatisticsFile, SchemaId, SchemaRef, Snapshot, SnapshotRef, + SnapshotRetention, SortOrder, SortOrderRef, StatisticsFile, StructType, + DEFAULT_PARTITION_SPEC_ID, }; use crate::error::{timestamp_ms_to_utc, Result}; use crate::{Error, ErrorKind}; @@ -397,6 +398,33 @@ impl TableMetadata { self.partition_statistics.get(&snapshot_id) } + /// Append snapshot to table + #[deprecated( + since = "0.4.0", + note = "please use `TableMetadataBuilder.set_branch_snapshot` instead" + )] + pub fn append_snapshot(&mut self, snapshot: Snapshot) { + self.last_updated_ms = snapshot.timestamp_ms(); + self.last_sequence_number = snapshot.sequence_number(); + + self.refs + .entry(MAIN_BRANCH.to_string()) + .and_modify(|s| { + s.snapshot_id = snapshot.snapshot_id(); + }) + .or_insert_with(|| { + SnapshotReference::new(snapshot.snapshot_id(), SnapshotRetention::Branch { + min_snapshots_to_keep: None, + max_snapshot_age_ms: None, + max_ref_age_ms: None, + }) + }); + + self.snapshot_log.push(snapshot.log()); + self.snapshots + .insert(snapshot.snapshot_id(), Arc::new(snapshot)); + } + /// Normalize this partition spec. /// /// This is an internal method