diff --git a/Cargo.lock b/Cargo.lock index cdfbbd86ab0..343ae2edd1b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -817,7 +817,7 @@ checksum = "25c7df09945d65ea8d70b3321547ed414bbc540aad5bac6883d021b970f35b04" [[package]] name = "fastfield_codecs" version = "0.1.0" -source = "git+https://github.com/quickwit-oss/tantivy?rev=46d5de9#46d5de920dd1ac86fa7a74baa0debd933bcb6574" +source = "git+https://github.com/quickwit-oss/tantivy?rev=447811c#447811c1117a4c87182077958d09761574df5e21" dependencies = [ "tantivy-bitpacker", "tantivy-common", @@ -1943,7 +1943,7 @@ dependencies = [ [[package]] name = "ownedbytes" version = "0.2.0" -source = "git+https://github.com/quickwit-oss/tantivy?rev=46d5de9#46d5de920dd1ac86fa7a74baa0debd933bcb6574" +source = "git+https://github.com/quickwit-oss/tantivy?rev=447811c#447811c1117a4c87182077958d09761574df5e21" dependencies = [ "stable_deref_trait", ] @@ -2476,7 +2476,6 @@ version = "0.2.1" dependencies = [ "anyhow", "async-trait", - "chrono", "crossbeam", "futures", "once_cell", @@ -2487,6 +2486,7 @@ dependencies = [ "tantivy", "tempfile", "thiserror", + "time 0.3.9", "tokio", "tracing", "uuid 0.8.2", @@ -2551,6 +2551,7 @@ dependencies = [ "tantivy", "tempfile", "thiserror", + "time 0.3.9", "tokio", "tokio-stream", "tracing", @@ -3606,14 +3607,13 @@ dependencies = [ [[package]] name = "tantivy" version = "0.17.0" -source = "git+https://github.com/quickwit-oss/tantivy?rev=46d5de9#46d5de920dd1ac86fa7a74baa0debd933bcb6574" +source = "git+https://github.com/quickwit-oss/tantivy?rev=447811c#447811c1117a4c87182077958d09761574df5e21" dependencies = [ "async-trait", "base64", "bitpacking", "byteorder", "census", - "chrono", "crc32fast", "crossbeam", "downcast-rs", @@ -3650,6 +3650,7 @@ dependencies = [ "tantivy-query-grammar", "tempfile", "thiserror", + "time 0.3.9", "uuid 0.8.2", "winapi 0.3.9", ] @@ -3657,12 +3658,12 @@ dependencies = [ [[package]] name = "tantivy-bitpacker" version = "0.1.1" -source = "git+https://github.com/quickwit-oss/tantivy?rev=46d5de9#46d5de920dd1ac86fa7a74baa0debd933bcb6574" +source = "git+https://github.com/quickwit-oss/tantivy?rev=447811c#447811c1117a4c87182077958d09761574df5e21" [[package]] name = "tantivy-common" version = "0.2.0" -source = "git+https://github.com/quickwit-oss/tantivy?rev=46d5de9#46d5de920dd1ac86fa7a74baa0debd933bcb6574" +source = "git+https://github.com/quickwit-oss/tantivy?rev=447811c#447811c1117a4c87182077958d09761574df5e21" dependencies = [ "byteorder", "ownedbytes", @@ -3682,7 +3683,7 @@ dependencies = [ [[package]] name = "tantivy-query-grammar" version = "0.15.0" -source = "git+https://github.com/quickwit-oss/tantivy?rev=46d5de9#46d5de920dd1ac86fa7a74baa0debd933bcb6574" +source = "git+https://github.com/quickwit-oss/tantivy?rev=447811c#447811c1117a4c87182077958d09761574df5e21" dependencies = [ "combine", "once_cell", @@ -3816,6 +3817,7 @@ dependencies = [ "itoa", "libc", "num_threads", + "serde", ] [[package]] diff --git a/quickwit-core/Cargo.toml b/quickwit-core/Cargo.toml index aa61d422197..0d0afe1d103 100644 --- a/quickwit-core/Cargo.toml +++ b/quickwit-core/Cargo.toml @@ -24,7 +24,7 @@ quickwit-storage = { version = "0.2.1", path = "../quickwit-storage" } tokio = { version = "1", features = ["full"] } tokio-util = { version = "0.7", features = ["full"] } rand = "0.8" -tantivy = { git= "https://github.com/quickwit-oss/tantivy", rev="46d5de9", default-features=false, features = ["mmap", "lz4-compression", "quickwit"] } +tantivy = { git= "https://github.com/quickwit-oss/tantivy", rev="447811c", default-features=false, features = ["mmap", "lz4-compression", "quickwit"] } futures = "0.3" futures-util = { version = "0.3.1", default-features = false } uuid = "0.8" diff --git a/quickwit-directories/Cargo.toml b/quickwit-directories/Cargo.toml index e54dd34c02d..d6176c2302d 100644 --- a/quickwit-directories/Cargo.toml +++ b/quickwit-directories/Cargo.toml @@ -16,7 +16,7 @@ futures = "0.3" serde = "1" serde_cbor = "0.11" serde_json = "1" -tantivy = { git= "https://github.com/quickwit-oss/tantivy", rev="46d5de9", default-features=false, features = ["mmap", "lz4-compression", "quickwit"] } +tantivy = { git= "https://github.com/quickwit-oss/tantivy", rev="447811c", default-features=false, features = ["mmap", "lz4-compression", "quickwit"] } quickwit-storage = { version = "0.2.1", path = "../quickwit-storage" } uuid = "0.8" once_cell = "1" @@ -25,7 +25,7 @@ tracing = "0.1.29" thiserror = "1" anyhow = "1" async-trait = "0.1" -chrono = "0.4" +time = { version = "0.3.7", features = ["std"] } [dev-dependencies] tempfile = '3' diff --git a/quickwit-directories/src/debug_proxy_directory.rs b/quickwit-directories/src/debug_proxy_directory.rs index b6f5f8ede7d..d42fe3a5176 100644 --- a/quickwit-directories/src/debug_proxy_directory.rs +++ b/quickwit-directories/src/debug_proxy_directory.rs @@ -24,10 +24,10 @@ use std::time::{Duration, Instant}; use std::{fmt, io, mem}; use async_trait::async_trait; -use tantivy::chrono::{DateTime, Utc}; use tantivy::directory::error::OpenReadError; use tantivy::directory::{FileHandle, OwnedBytes}; use tantivy::{Directory, HasLen}; +use time::OffsetDateTime; use crate::StorageDirectory; @@ -63,14 +63,14 @@ pub struct ReadOperation { pub offset: usize, /// The number of bytes fetched pub num_bytes: usize, - /// The date at which the operation was performed. - pub start_date: DateTime, + /// The date at which the operation was performed (UTC timezone). + pub start_date: OffsetDateTime, /// The elapsed time to run the read operatioon. pub duration: Duration, } struct ReadOperationBuilder { - start_date: DateTime, + start_date: OffsetDateTime, start_instant: Instant, path: PathBuf, offset: usize, @@ -79,7 +79,7 @@ struct ReadOperationBuilder { impl ReadOperationBuilder { pub fn new(path: &Path) -> Self { let start_instant = Instant::now(); - let start_date = Utc::now(); + let start_date = OffsetDateTime::now_utc(); ReadOperationBuilder { start_date, start_instant, diff --git a/quickwit-doc-mapper/Cargo.toml b/quickwit-doc-mapper/Cargo.toml index ea0f3b1022f..09bc3f96160 100644 --- a/quickwit-doc-mapper/Cargo.toml +++ b/quickwit-doc-mapper/Cargo.toml @@ -19,8 +19,8 @@ once_cell = "1.10" regex = "1" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" -tantivy = { git = "https://github.com/quickwit-oss/tantivy", rev = "46d5de9", default-features = false, features = ["mmap", "lz4-compression", "quickwit"] } -tantivy-query-grammar = { git = "https://github.com/quickwit-oss/tantivy/", rev = "46d5de9" } +tantivy = { git = "https://github.com/quickwit-oss/tantivy", rev = "447811c", default-features = false, features = ["mmap", "lz4-compression", "quickwit"] } +tantivy-query-grammar = { git = "https://github.com/quickwit-oss/tantivy/", rev = "447811c" } thiserror = "1.0" tracing = "0.1.29" typetag = "0.1" diff --git a/quickwit-doc-mapper/src/default_doc_mapper/default_mapper.rs b/quickwit-doc-mapper/src/default_doc_mapper/default_mapper.rs index bcc45076871..3cdcdd427dd 100644 --- a/quickwit-doc-mapper/src/default_doc_mapper/default_mapper.rs +++ b/quickwit-doc-mapper/src/default_doc_mapper/default_mapper.rs @@ -475,6 +475,7 @@ mod tests { { "timestamp": 1586960586000, "body": "20200415T072306-0700 INFO This is a great log", + "response_date2": "2021-12-19T16:39:57+00:00", "response_date": "2021-12-19T16:39:57Z", "response_time": 2.3, "response_payload": "YWJj", @@ -490,7 +491,7 @@ mod tests { const EXPECTED_JSON_PATHS_AND_VALUES: &str = r#"{ "timestamp": [1586960586000], "body": ["20200415T072306-0700 INFO This is a great log"], - "response_date": ["2021-12-19T16:39:57+00:00"], + "response_date": ["2021-12-19T16:39:57Z"], "response_time": [2.3], "response_payload": [[97,98,99]], "owner": ["foo"], @@ -567,6 +568,12 @@ mod tests { .iter() .map(|expected_value| format!("{}", expected_value)) .any(|expected_value| expected_value == value); + if !is_value_in_expected_values { + panic!( + "Could not find: {:?} in {:?}", + value, expected_json_paths_and_values + ); + } assert!(is_value_in_expected_values); } }); diff --git a/quickwit-doc-mapper/src/default_doc_mapper/field_mapping_entry.rs b/quickwit-doc-mapper/src/default_doc_mapper/field_mapping_entry.rs index 7c6897db91b..9c743bba0a7 100644 --- a/quickwit-doc-mapper/src/default_doc_mapper/field_mapping_entry.rs +++ b/quickwit-doc-mapper/src/default_doc_mapper/field_mapping_entry.rs @@ -20,7 +20,6 @@ use std::convert::TryFrom; use anyhow::bail; -use chrono::{FixedOffset, Utc}; use itertools::{process_results, Itertools}; use serde::{Deserialize, Serialize}; use serde_json::{self, Value as JsonValue}; @@ -28,6 +27,9 @@ use tantivy::schema::{ BytesOptions, Cardinality, DocParsingError as TantivyDocParser, FieldType, IndexRecordOption, NumericOptions, TextFieldIndexing, TextOptions, Value, }; +use tantivy::time::format_description::well_known::Rfc3339; +use tantivy::time::OffsetDateTime; +use tantivy::DateTime; use thiserror::Error; use super::{default_as_true, FieldMappingType}; @@ -335,17 +337,16 @@ impl FieldMappingEntry { )? } JsonValue::String(value_as_str) => { - let dt_with_fixed_tz: chrono::DateTime = - chrono::DateTime::parse_from_rfc3339(&value_as_str).map_err(|err| { + let date_time_utc = DateTime::new_utc( + OffsetDateTime::parse(&value_as_str, &Rfc3339).map_err(|err| { DocParsingError::ValueError( self.name.clone(), format!("Expected RFC 3339 date, got '{}'. {:?}", value_as_str, err), ) - })?; - vec![( - FieldPath::new(&self.name), - Value::Date(dt_with_fixed_tz.with_timezone(&Utc)), - )] + })?, + ); + + vec![(FieldPath::new(&self.name), Value::Date(date_time_utc))] } JsonValue::Null => { vec![] @@ -747,10 +748,11 @@ impl From for DocParsingError { #[cfg(test)] mod tests { use anyhow::bail; - use chrono::{NaiveDate, NaiveDateTime, NaiveTime, TimeZone, Utc}; use matches::matches; use serde_json::json; use tantivy::schema::{Cardinality, Value}; + use tantivy::time::{Date, Month, PrimitiveDateTime, Time}; + use tantivy::DateTime; use super::FieldMappingEntry; use crate::default_doc_mapper::FieldMappingType; @@ -1332,11 +1334,14 @@ mod tests { // Successful parsing let parsed_value = entry.parse(json!("2021-12-19T16:39:57-01:00"))?; - let datetime = NaiveDateTime::new( - NaiveDate::from_ymd(2021, 12, 19), - NaiveTime::from_hms(17, 39, 57), + + let datetime = PrimitiveDateTime::new( + Date::from_calendar_date(2021, Month::December, 19).unwrap(), + Time::from_hms(17, 39, 57).unwrap(), ); - let datetime_utc = Utc.from_utc_datetime(&datetime); + // let datetime = datetime!(2021-12-19 17:39:57); + + let datetime_utc = DateTime::new_primitive(datetime); // Utc.from_utc_datetime(&datetime); assert_eq!(parsed_value.len(), 1); assert_eq!(parsed_value[0].1, Value::Date(datetime_utc)); diff --git a/quickwit-indexing/Cargo.toml b/quickwit-indexing/Cargo.toml index 2381cb46dcb..e3a8f1f9dc9 100644 --- a/quickwit-indexing/Cargo.toml +++ b/quickwit-indexing/Cargo.toml @@ -34,7 +34,7 @@ rusoto_kinesis = { version = "0.47", default-features = false, features = ["rust serde = "1" serde_json = "1" serde_yaml = "0.8" -tantivy = { git= "https://github.com/quickwit-oss/tantivy", rev="46d5de9", default-features=false, features = ["mmap", "lz4-compression", "quickwit"] } +tantivy = { git= "https://github.com/quickwit-oss/tantivy", rev="447811c", default-features=false, features = ["mmap", "lz4-compression", "quickwit"] } tempfile = "3.3" thiserror = "1" tokio = { version = "1", features = ["sync"] } @@ -42,6 +42,8 @@ tracing = "0.1.29" ulid = "0.5" tokio-stream = "0.1" arc-swap = "1.4" +time = { version = "0.3.7", features = ["std"] } + [features] kafka = ["rdkafka"] diff --git a/quickwit-indexing/src/actors/uploader.rs b/quickwit-indexing/src/actors/uploader.rs index 2c02444f622..abeb42ea6a5 100644 --- a/quickwit-indexing/src/actors/uploader.rs +++ b/quickwit-indexing/src/actors/uploader.rs @@ -31,7 +31,7 @@ use itertools::Itertools; use quickwit_actors::{Actor, ActorContext, ActorExitStatus, Handler, Mailbox, QueueCapacity}; use quickwit_metastore::{Metastore, SplitMetadata}; use quickwit_storage::SplitPayloadBuilder; -use tantivy::chrono::Utc; +use time::OffsetDateTime; use tokio::sync::{OwnedSemaphorePermit, Semaphore}; use tracing::{info, info_span, warn, Instrument, Span}; @@ -194,7 +194,7 @@ fn create_split_metadata(split: &PackagedSplit, footer_offsets: Range) -> S num_docs: split.num_docs as usize, time_range: split.time_range.clone(), original_size_in_bytes: split.size_in_bytes, - create_timestamp: Utc::now().timestamp(), + create_timestamp: OffsetDateTime::now_utc().unix_timestamp(), tags: split.tags.clone(), demux_num_ops: split.demux_num_ops, footer_offsets, diff --git a/quickwit-indexing/src/garbage_collection.rs b/quickwit-indexing/src/garbage_collection.rs index eaa1eaeae4e..eb532980012 100644 --- a/quickwit-indexing/src/garbage_collection.rs +++ b/quickwit-indexing/src/garbage_collection.rs @@ -24,8 +24,8 @@ use futures::StreamExt; use quickwit_actors::ActorContext; use quickwit_metastore::{Metastore, MetastoreError, SplitMetadata, SplitState}; use quickwit_storage::StorageError; -use tantivy::chrono::Utc; use thiserror::Error; +use time::OffsetDateTime; use tracing::error; use crate::actors::GarbageCollector; @@ -83,7 +83,8 @@ pub async fn run_garbage_collect( ctx_opt: Option<&ActorContext>, ) -> anyhow::Result> { // Select staged splits with staging timestamp older than grace period timestamp. - let grace_period_timestamp = Utc::now().timestamp() - staged_grace_period.as_secs() as i64; + let grace_period_timestamp = + OffsetDateTime::now_utc().unix_timestamp() - staged_grace_period.as_secs() as i64; let deletable_staged_splits: Vec = metastore .list_splits(index_id, SplitState::Staged, None, None) @@ -123,7 +124,8 @@ pub async fn run_garbage_collect( .await?; // We wait another 2 minutes until the split is actually deleted. - let grace_period_deletion = Utc::now().timestamp() - deletion_grace_period.as_secs() as i64; + let grace_period_deletion = + OffsetDateTime::now_utc().unix_timestamp() - deletion_grace_period.as_secs() as i64; let splits_to_delete = metastore .list_splits(index_id, SplitState::MarkedForDeletion, None, None) .await? diff --git a/quickwit-search/Cargo.toml b/quickwit-search/Cargo.toml index 1b303671a2b..1d67563dd4e 100644 --- a/quickwit-search/Cargo.toml +++ b/quickwit-search/Cargo.toml @@ -33,7 +33,7 @@ once_cell = "1" opentelemetry = "0.17" tracing-opentelemetry = "0.17" rayon = "1" -tantivy = { git= "https://github.com/quickwit-oss/tantivy", rev="46d5de9", default-features=false, features = ["mmap", "lz4-compression", "quickwit"] } +tantivy = { git= "https://github.com/quickwit-oss/tantivy", rev="447811c", default-features=false, features = ["mmap", "lz4-compression", "quickwit"] } [dependencies.quickwit-cluster] path = '../quickwit-cluster' diff --git a/quickwit-storage/Cargo.toml b/quickwit-storage/Cargo.toml index 29c0f1fa0de..cc310422110 100644 --- a/quickwit-storage/Cargo.toml +++ b/quickwit-storage/Cargo.toml @@ -20,7 +20,7 @@ futures = '0.3' serde_json = "1" base64 = '0.13' tracing = "0.1.29" -tantivy = { git= "https://github.com/quickwit-oss/tantivy", rev="46d5de9", default-features=false, features = ["mmap", "lz4-compression", "quickwit"] } +tantivy = { git= "https://github.com/quickwit-oss/tantivy", rev="447811c", default-features=false, features = ["mmap", "lz4-compression", "quickwit"] } once_cell = '1' regex = '1' thiserror = '1'