From ed1e4b48a66d5ce68e6437be241ad2c07b850d90 Mon Sep 17 00:00:00 2001 From: Nikhil Sinha Date: Mon, 2 Dec 2024 15:13:08 -0800 Subject: [PATCH 1/3] fix: skip null data type fields skip the schema generation with `Null` data type fields fixes the issue when first event has null data type and field has valid data after the initial schema is persisted fixes the schema merge issue for such cases --- src/event/format/json.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/event/format/json.rs b/src/event/format/json.rs index cedbabaa3..972a89999 100644 --- a/src/event/format/json.rs +++ b/src/event/format/json.rs @@ -86,6 +86,7 @@ impl EventFormat for Event { infer_schema .fields .iter() + .filter(|field| *field.data_type() != DataType::Null) .cloned() .sorted_by(|a, b| a.name().cmp(b.name())) .collect() From 6113d72cd68c7ff0991ce8166ac6a74a620fb7bf Mon Sep 17 00:00:00 2001 From: Nikhil Sinha Date: Mon, 2 Dec 2024 18:28:21 -0800 Subject: [PATCH 2/3] ut: fix result count of columns --- src/handlers/http/ingest.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/handlers/http/ingest.rs b/src/handlers/http/ingest.rs index d15f6660e..9f72000b4 100644 --- a/src/handlers/http/ingest.rs +++ b/src/handlers/http/ingest.rs @@ -384,7 +384,7 @@ mod tests { let (rb, _) = into_event_batch(req, json, HashMap::default(), None, None).unwrap(); assert_eq!(rb.num_rows(), 1); - assert_eq!(rb.num_columns(), 6); + assert_eq!(rb.num_columns(), 5); assert_eq!( rb.column_by_name("a").unwrap().as_int64_arr(), &Int64Array::from_iter([1]) From 1be3b87123b88ddb2e911972842e6e684e466079 Mon Sep 17 00:00:00 2001 From: Nitish Tiwari Date: Wed, 4 Dec 2024 10:03:18 +0530 Subject: [PATCH 3/3] Update src/event/format/json.rs Co-authored-by: Devdutt Shenoi Signed-off-by: Nitish Tiwari --- src/event/format/json.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/event/format/json.rs b/src/event/format/json.rs index 972a89999..abb823c46 100644 --- a/src/event/format/json.rs +++ b/src/event/format/json.rs @@ -86,7 +86,7 @@ impl EventFormat for Event { infer_schema .fields .iter() - .filter(|field| *field.data_type() != DataType::Null) + .filter(|field| !field.data_type().is_null()) .cloned() .sorted_by(|a, b| a.name().cmp(b.name())) .collect()