fix: skip null data type fields (#1016)

skip the schema generation with `Null` data type fields fixes the issue when first event has null data type and field has valid data after the initial schema is persisted fixes the schema merge issue for such cases
---------

Signed-off-by: Nitish Tiwari <[email protected]>
Co-authored-by: Nitish Tiwari <[email protected]>
Co-authored-by: Devdutt Shenoi <[email protected]>

Author: 3 people

src/event/format/json.rs

@@ -86,6 +86,7 @@ impl EventFormat for Event {
                     infer_schema
                         .fields
                         .iter()
+                        .filter(|field| !field.data_type().is_null())
                         .cloned()
                         .sorted_by(|a, b| a.name().cmp(b.name()))
                         .collect()

src/handlers/http/ingest.rs

@@ -384,7 +384,7 @@ mod tests {
         let (rb, _) = into_event_batch(req, json, HashMap::default(), None, None).unwrap();
 
         assert_eq!(rb.num_rows(), 1);
-        assert_eq!(rb.num_columns(), 6);
+        assert_eq!(rb.num_columns(), 5);
         assert_eq!(
             rb.column_by_name("a").unwrap().as_int64_arr(),
             &Int64Array::from_iter([1])