From dd4070b1331d27dd49954b2526596f34a8809095 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Wed, 9 Apr 2025 11:10:01 +0800 Subject: [PATCH 1/6] refactor: Use tracing to replace log Signed-off-by: Xuanwo --- Cargo.lock | 101 +----------------- Cargo.toml | 6 +- crates/catalog/glue/Cargo.toml | 2 +- .../catalog/glue/tests/glue_catalog_test.rs | 3 +- crates/catalog/hms/Cargo.toml | 2 +- crates/catalog/hms/tests/hms_catalog_test.rs | 5 +- crates/catalog/rest/Cargo.toml | 2 +- .../catalog/rest/tests/rest_catalog_test.rs | 3 +- crates/test_utils/Cargo.toml | 4 +- crates/test_utils/src/cmd.rs | 10 +- crates/test_utils/src/docker.rs | 4 +- crates/test_utils/src/lib.rs | 2 +- 12 files changed, 30 insertions(+), 114 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1cc2cc42d6..3d24a0ee88 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -105,56 +105,6 @@ dependencies = [ "libc", ] -[[package]] -name = "anstream" -version = "0.6.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" -dependencies = [ - "anstyle", - "anstyle-parse", - "anstyle-query", - "anstyle-wincon", - "colorchoice", - "is_terminal_polyfill", - "utf8parse", -] - -[[package]] -name = "anstyle" -version = "1.0.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" - -[[package]] -name = "anstyle-parse" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" -dependencies = [ - "utf8parse", -] - -[[package]] -name = "anstyle-query" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" -dependencies = [ - "windows-sys 0.59.0", -] - -[[package]] -name = "anstyle-wincon" -version = "3.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3534e77181a9cc07539ad51f2141fe32f6c3ffd4df76db8ad92346b003ae4e" -dependencies = [ - "anstyle", - "once_cell", - "windows-sys 0.59.0", -] - [[package]] name = "anyhow" version = "1.0.95" @@ -1380,12 +1330,6 @@ dependencies = [ "inout", ] -[[package]] -name = "colorchoice" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" - [[package]] name = "colored" version = "2.2.0" @@ -2230,29 +2174,6 @@ dependencies = [ "serde", ] -[[package]] -name = "env_filter" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "186e05a59d4c50738528153b83b0b0194d3a29507dfec16eccd4b342903397d0" -dependencies = [ - "log", - "regex", -] - -[[package]] -name = "env_logger" -version = "0.11.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcaee3d8e3cfc3fd92428d477bc97fc29ec8716d180c0d74c643bb26166660e0" -dependencies = [ - "anstream", - "anstyle", - "env_filter", - "humantime", - "log", -] - [[package]] name = "equivalent" version = "1.0.2" @@ -3031,10 +2952,10 @@ dependencies = [ "ctor", "iceberg", "iceberg_test_utils", - "log", "port_scanner", "serde_json", "tokio", + "tracing", "typed-builder 0.20.1", "uuid", ] @@ -3050,11 +2971,11 @@ dependencies = [ "hive_metastore", "iceberg", "iceberg_test_utils", - "log", "pilota", "port_scanner", "serde_json", "tokio", + "tracing", "typed-builder 0.20.1", "uuid", "volo-thrift", @@ -3086,7 +3007,6 @@ dependencies = [ "iceberg", "iceberg_test_utils", "itertools 0.13.0", - "log", "mockito", "port_scanner", "reqwest", @@ -3094,6 +3014,7 @@ dependencies = [ "serde_derive", "serde_json", "tokio", + "tracing", "typed-builder 0.20.1", "uuid", ] @@ -3178,8 +3099,8 @@ dependencies = [ name = "iceberg_test_utils" version = "0.4.0" dependencies = [ - "env_logger", - "log", + "tracing", + "tracing-subscriber", ] [[package]] @@ -3397,12 +3318,6 @@ version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" -[[package]] -name = "is_terminal_polyfill" -version = "1.70.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" - [[package]] name = "itertools" version = "0.13.0" @@ -6488,12 +6403,6 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" -[[package]] -name = "utf8parse" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" - [[package]] name = "uuid" version = "1.16.0" diff --git a/Cargo.toml b/Cargo.toml index ed6380cb02..5eedbe32c2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -61,7 +61,7 @@ chrono = "0.4.38" ctor = "0.2.8" datafusion = "45" derive_builder = "0.20" -env_logger = "0.11.0" +tracing-subscriber = "0.3" fnv = "1.0.7" futures = "0.3" iceberg = { version = "0.4.0", path = "./crates/iceberg" } @@ -69,7 +69,7 @@ iceberg-catalog-rest = { version = "0.4.0", path = "./crates/catalog/rest" } iceberg-catalog-memory = { version = "0.4.0", path = "./crates/catalog/memory" } iceberg-datafusion = { version = "0.4.0", path = "./crates/integrations/datafusion" } itertools = "0.13" -log = "0.4.22" +tracing = "0.1" mockito = "1" murmur3 = "0.5.2" num-bigint = "0.4.6" @@ -92,7 +92,7 @@ serde_json = "1.0.138" serde_repr = "0.1.16" serde_with = "3.4" tempfile = "3.18" -thrift = "0.17.0" +thrift = "0.17.0" tokio = { version = "1.44", default-features = false } typed-builder = "0.20" url = "2.5.4" diff --git a/crates/catalog/glue/Cargo.toml b/crates/catalog/glue/Cargo.toml index 0d2e1f9833..0eb84b32c1 100644 --- a/crates/catalog/glue/Cargo.toml +++ b/crates/catalog/glue/Cargo.toml @@ -34,7 +34,7 @@ async-trait = { workspace = true } aws-config = { workspace = true } aws-sdk-glue = { workspace = true } iceberg = { workspace = true } -log = { workspace = true } +tracing = { workspace = true } serde_json = { workspace = true } tokio = { workspace = true } typed-builder = { workspace = true } diff --git a/crates/catalog/glue/tests/glue_catalog_test.rs b/crates/catalog/glue/tests/glue_catalog_test.rs index 634d7345de..e13786a792 100644 --- a/crates/catalog/glue/tests/glue_catalog_test.rs +++ b/crates/catalog/glue/tests/glue_catalog_test.rs @@ -32,6 +32,7 @@ use iceberg_test_utils::docker::DockerCompose; use iceberg_test_utils::{normalize_test_name, set_up}; use port_scanner::scan_port_addr; use tokio::time::sleep; +use tracing::info; const GLUE_CATALOG_PORT: u16 = 5000; const MINIO_PORT: u16 = 9000; @@ -68,7 +69,7 @@ async fn get_catalog() -> GlueCatalog { let glue_socket_addr = SocketAddr::new(glue_catalog_ip, GLUE_CATALOG_PORT); let minio_socket_addr = SocketAddr::new(minio_ip, MINIO_PORT); while !scan_port_addr(glue_socket_addr) { - log::info!("Waiting for 1s glue catalog to ready..."); + info!("Waiting for 1s glue catalog to ready..."); sleep(std::time::Duration::from_millis(1000)).await; } diff --git a/crates/catalog/hms/Cargo.toml b/crates/catalog/hms/Cargo.toml index e7d4ec2f39..f1969651ae 100644 --- a/crates/catalog/hms/Cargo.toml +++ b/crates/catalog/hms/Cargo.toml @@ -34,7 +34,7 @@ async-trait = { workspace = true } chrono = { workspace = true } hive_metastore = { workspace = true } iceberg = { workspace = true } -log = { workspace = true } +tracing = { workspace = true } pilota = { workspace = true } serde_json = { workspace = true } tokio = { workspace = true } diff --git a/crates/catalog/hms/tests/hms_catalog_test.rs b/crates/catalog/hms/tests/hms_catalog_test.rs index f70517ac00..a10bc1e6c2 100644 --- a/crates/catalog/hms/tests/hms_catalog_test.rs +++ b/crates/catalog/hms/tests/hms_catalog_test.rs @@ -30,6 +30,7 @@ use iceberg_test_utils::docker::DockerCompose; use iceberg_test_utils::{normalize_test_name, set_up}; use port_scanner::scan_port_addr; use tokio::time::sleep; +use tracing::info; const HMS_CATALOG_PORT: u16 = 9083; const MINIO_PORT: u16 = 9000; @@ -67,8 +68,8 @@ async fn get_catalog() -> HmsCatalog { let hms_socket_addr = SocketAddr::new(hms_catalog_ip, HMS_CATALOG_PORT); let minio_socket_addr = SocketAddr::new(minio_ip, MINIO_PORT); while !scan_port_addr(hms_socket_addr) { - log::info!("scan hms_socket_addr {} check", hms_socket_addr); - log::info!("Waiting for 1s hms catalog to ready..."); + info!("scan hms_socket_addr {} check", hms_socket_addr); + info!("Waiting for 1s hms catalog to ready..."); sleep(std::time::Duration::from_millis(1000)).await; } diff --git a/crates/catalog/rest/Cargo.toml b/crates/catalog/rest/Cargo.toml index 6915454930..b2b948ace5 100644 --- a/crates/catalog/rest/Cargo.toml +++ b/crates/catalog/rest/Cargo.toml @@ -35,7 +35,7 @@ chrono = { workspace = true } http = "1.1.0" iceberg = { workspace = true } itertools = { workspace = true } -log = { workspace = true } +tracing = { workspace = true } reqwest = { workspace = true } serde = { workspace = true } serde_derive = { workspace = true } diff --git a/crates/catalog/rest/tests/rest_catalog_test.rs b/crates/catalog/rest/tests/rest_catalog_test.rs index 71ec33b514..f08bfb6fa7 100644 --- a/crates/catalog/rest/tests/rest_catalog_test.rs +++ b/crates/catalog/rest/tests/rest_catalog_test.rs @@ -30,6 +30,7 @@ use iceberg_test_utils::docker::DockerCompose; use iceberg_test_utils::{normalize_test_name, set_up}; use port_scanner::scan_port_addr; use tokio::time::sleep; +use tracing::info; const REST_CATALOG_PORT: u16 = 8181; static DOCKER_COMPOSE_ENV: RwLock> = RwLock::new(None); @@ -62,7 +63,7 @@ async fn get_catalog() -> RestCatalog { let rest_socket_addr = SocketAddr::new(rest_catalog_ip, REST_CATALOG_PORT); while !scan_port_addr(rest_socket_addr) { - log::info!("Waiting for 1s rest catalog to ready..."); + info!("Waiting for 1s rest catalog to ready..."); sleep(std::time::Duration::from_millis(1000)).await; } diff --git a/crates/test_utils/Cargo.toml b/crates/test_utils/Cargo.toml index c98309760b..181a8f0750 100644 --- a/crates/test_utils/Cargo.toml +++ b/crates/test_utils/Cargo.toml @@ -26,8 +26,8 @@ repository = { workspace = true } license = { workspace = true } [dependencies] -env_logger = { workspace = true } -log = { workspace = true } +tracing-subscriber = { workspace = true } +tracing = { workspace = true } [features] tests = [] diff --git a/crates/test_utils/src/cmd.rs b/crates/test_utils/src/cmd.rs index 503d63d151..0e035888f6 100644 --- a/crates/test_utils/src/cmd.rs +++ b/crates/test_utils/src/cmd.rs @@ -17,12 +17,14 @@ use std::process::Command; +use tracing::info; + pub fn run_command(mut cmd: Command, desc: impl ToString) { let desc = desc.to_string(); - log::info!("Starting to {}, command: {:?}", &desc, cmd); + info!("Starting to {}, command: {:?}", &desc, cmd); let exit = cmd.status().unwrap(); if exit.success() { - log::info!("{} succeed!", desc) + info!("{} succeed!", desc) } else { panic!("{} failed: {:?}", desc, exit); } @@ -30,12 +32,12 @@ pub fn run_command(mut cmd: Command, desc: impl ToString) { pub fn get_cmd_output_result(mut cmd: Command, desc: impl ToString) -> Result { let desc = desc.to_string(); - log::info!("Starting to {}, command: {:?}", &desc, cmd); + info!("Starting to {}, command: {:?}", &desc, cmd); let result = cmd.output(); match result { Ok(output) => { if output.status.success() { - log::info!("{} succeed!", desc); + info!("{} succeed!", desc); Ok(String::from_utf8(output.stdout).unwrap()) } else { Err(format!("{} failed with rc: {:?}", desc, output.status)) diff --git a/crates/test_utils/src/docker.rs b/crates/test_utils/src/docker.rs index 42e6c39b9e..832c22240e 100644 --- a/crates/test_utils/src/docker.rs +++ b/crates/test_utils/src/docker.rs @@ -18,6 +18,8 @@ use std::net::IpAddr; use std::process::Command; +use tracing::error; + use crate::cmd::{get_cmd_output, get_cmd_output_result, run_command}; /// A utility to manage the lifecycle of `docker compose`. @@ -127,7 +129,7 @@ impl DockerCompose { match ip_result { Ok(ip) => ip, Err(e) => { - log::error!("Invalid IP, {e}"); + error!("Invalid IP, {e}"); panic!("Failed to parse IP for {container_name}") } } diff --git a/crates/test_utils/src/lib.rs b/crates/test_utils/src/lib.rs index 4f63b8dd77..6d1104a1b9 100644 --- a/crates/test_utils/src/lib.rs +++ b/crates/test_utils/src/lib.rs @@ -33,7 +33,7 @@ mod common { static INIT: Once = Once::new(); pub fn set_up() { - INIT.call_once(env_logger::init); + INIT.call_once(tracing_subscriber::fmt::init); } pub fn normalize_test_name(s: impl ToString) -> String { s.to_string().replace("::", "__").replace('.', "_") From 4333714b7e40b453cde784b32b93d18eeaf9ce96 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Wed, 9 Apr 2025 11:10:59 +0800 Subject: [PATCH 2/6] Format Signed-off-by: Xuanwo --- Cargo.toml | 14 +++++++------- deny.toml | 8 ++------ 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 5eedbe32c2..dd577718bb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,13 +18,13 @@ [workspace] resolver = "2" members = [ - "crates/catalog/*", - "crates/examples", - "crates/iceberg", - "crates/integration_tests", - "crates/integrations/*", - "crates/sqllogictest", - "crates/test_utils", + "crates/catalog/*", + "crates/examples", + "crates/iceberg", + "crates/integration_tests", + "crates/integrations/*", + "crates/sqllogictest", + "crates/test_utils", ] exclude = ["bindings/python"] diff --git a/deny.toml b/deny.toml index 9c62e0d68f..5ed9b48b99 100644 --- a/deny.toml +++ b/deny.toml @@ -24,13 +24,9 @@ allow = [ "ISC", "CC0-1.0", "Unicode-DFS-2016", - "Zlib" -] -exceptions = [ - { allow = [ - "OpenSSL", - ], name = "ring" } + "Zlib", ] +exceptions = [{ allow = ["OpenSSL"], name = "ring" }] [[licenses.clarify]] name = "ring" From 6122115c8735f76fafd638af035629d183f9b924 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Wed, 9 Apr 2025 11:11:35 +0800 Subject: [PATCH 3/6] Sort Signed-off-by: Xuanwo --- crates/catalog/glue/Cargo.toml | 2 +- crates/catalog/hms/Cargo.toml | 2 +- crates/catalog/rest/Cargo.toml | 2 +- crates/test_utils/Cargo.toml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/catalog/glue/Cargo.toml b/crates/catalog/glue/Cargo.toml index 0eb84b32c1..76b48c9c17 100644 --- a/crates/catalog/glue/Cargo.toml +++ b/crates/catalog/glue/Cargo.toml @@ -34,9 +34,9 @@ async-trait = { workspace = true } aws-config = { workspace = true } aws-sdk-glue = { workspace = true } iceberg = { workspace = true } -tracing = { workspace = true } serde_json = { workspace = true } tokio = { workspace = true } +tracing = { workspace = true } typed-builder = { workspace = true } uuid = { workspace = true } diff --git a/crates/catalog/hms/Cargo.toml b/crates/catalog/hms/Cargo.toml index f1969651ae..c54aa3f13f 100644 --- a/crates/catalog/hms/Cargo.toml +++ b/crates/catalog/hms/Cargo.toml @@ -34,10 +34,10 @@ async-trait = { workspace = true } chrono = { workspace = true } hive_metastore = { workspace = true } iceberg = { workspace = true } -tracing = { workspace = true } pilota = { workspace = true } serde_json = { workspace = true } tokio = { workspace = true } +tracing = { workspace = true } typed-builder = { workspace = true } uuid = { workspace = true } volo-thrift = { workspace = true } diff --git a/crates/catalog/rest/Cargo.toml b/crates/catalog/rest/Cargo.toml index b2b948ace5..64c56663f8 100644 --- a/crates/catalog/rest/Cargo.toml +++ b/crates/catalog/rest/Cargo.toml @@ -35,12 +35,12 @@ chrono = { workspace = true } http = "1.1.0" iceberg = { workspace = true } itertools = { workspace = true } -tracing = { workspace = true } reqwest = { workspace = true } serde = { workspace = true } serde_derive = { workspace = true } serde_json = { workspace = true } tokio = { workspace = true, features = ["sync"] } +tracing = { workspace = true } typed-builder = { workspace = true } uuid = { workspace = true, features = ["v4"] } diff --git a/crates/test_utils/Cargo.toml b/crates/test_utils/Cargo.toml index 181a8f0750..5cc63ff312 100644 --- a/crates/test_utils/Cargo.toml +++ b/crates/test_utils/Cargo.toml @@ -26,8 +26,8 @@ repository = { workspace = true } license = { workspace = true } [dependencies] -tracing-subscriber = { workspace = true } tracing = { workspace = true } +tracing-subscriber = { workspace = true } [features] tests = [] From e3120174367563917c74509410a2c33b777f5e92 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Wed, 9 Apr 2025 12:37:55 +0800 Subject: [PATCH 4/6] We need at least 0.1.37 Signed-off-by: Xuanwo --- Cargo.toml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index bdcbf72d13..41ea268ccf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,13 +18,13 @@ [workspace] resolver = "2" members = [ - "crates/catalog/*", - "crates/examples", - "crates/iceberg", - "crates/integration_tests", - "crates/integrations/*", - "crates/sqllogictest", - "crates/test_utils", + "crates/catalog/*", + "crates/examples", + "crates/iceberg", + "crates/integration_tests", + "crates/integrations/*", + "crates/sqllogictest", + "crates/test_utils", ] exclude = ["bindings/python"] @@ -69,7 +69,7 @@ iceberg-catalog-rest = { version = "0.4.0", path = "./crates/catalog/rest" } iceberg-catalog-memory = { version = "0.4.0", path = "./crates/catalog/memory" } iceberg-datafusion = { version = "0.4.0", path = "./crates/integrations/datafusion" } itertools = "0.13" -tracing = "0.1" +tracing = "0.1.37" mockito = "1" murmur3 = "0.5.2" num-bigint = "0.4.6" From b1129874a3c4e308377231c770088a8bf6ef4ddd Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Wed, 9 Apr 2025 12:44:40 +0800 Subject: [PATCH 5/6] We need at least tracing-subscriber 0.3.8 Signed-off-by: Xuanwo --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 41ea268ccf..b801bde6df 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -61,7 +61,7 @@ chrono = "0.4.38" ctor = "0.2.8" datafusion = "45" derive_builder = "0.20" -tracing-subscriber = "0.3" +tracing-subscriber = "0.3.8" fnv = "1.0.7" futures = "0.3" iceberg = { version = "0.4.0", path = "./crates/iceberg" } From 9060e0e23e6558dad7fea2f67423ec97cae70d5d Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Wed, 9 Apr 2025 13:51:46 +0800 Subject: [PATCH 6/6] Format Signed-off-by: Xuanwo --- Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 500d9347ba..7c4f84ca0f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -61,7 +61,6 @@ chrono = "0.4.38" ctor = "0.2.8" datafusion = "45" derive_builder = "0.20" -tracing-subscriber = "0.3.8" expect-test = "1" fnv = "1.0.7" futures = "0.3" @@ -72,7 +71,6 @@ iceberg-catalog-memory = { version = "0.4.0", path = "./crates/catalog/memory" } iceberg-catalog-rest = { version = "0.4.0", path = "./crates/catalog/rest" } iceberg-datafusion = { version = "0.4.0", path = "./crates/integrations/datafusion" } itertools = "0.13" -tracing = "0.1.37" mockito = "1" murmur3 = "0.5.2" num-bigint = "0.4.6" @@ -98,6 +96,8 @@ tempfile = "3.18" tera = "1" thrift = "0.17.0" tokio = { version = "1.44", default-features = false } +tracing = "0.1.37" +tracing-subscriber = "0.3.8" typed-builder = "0.20" url = "2.5.4" uuid = { version = "1.14", features = ["v7"] }