enhancement: implement generic flattening

helps to flatten the nested json
such that all list type fields gets converted to primitive type
hence spliting the repeating array to multiple rows

eg.
`[
        {
            "id": 1,
            "name": "John Doe",
            "addresses": [
                {
                    "street": "123 Main St",
                    "city": "Springfield",
                    "state": "IL",
                    "zip": "62701"
                },
                {
                    "street": "456 Elm St",
                    "city": "Springfield",
                    "state": "IL",
                    "zip": "62702"
                }
            ]
        }
    ]
`

gets coverted to below
`
[
		{
			"id": 1,
            "name": "John Doe",
			"addresses_street": "123 Main St",
			"addresses_city": "Springfield",
			"addresses_state": "IL",
			"addresses_zip": "62701",

		},
		{
			"id": 1,
            "name": "John Doe",
			"addresses_street": "456 Elm St",
			"addresses_city": "Springfield",
			"addresses_state": "IL",
			"addresses_zip": "62702",
		}
	]
`

Author: nikhilsinhaparseable

src/utils/json/flatten.rs

@@ -304,6 +304,61 @@ pub fn flatten_array_objects(
     Ok(())
 }
 
+pub fn flatten_json(value: &Value) -> Vec<Value> {
+    match value {
+        Value::Array(arr) => {
+            let mut results = Vec::new();
+            for item in arr {
+                results.extend(flatten_json(item));
+            }
+            results
+        }
+        Value::Object(map) => {
+            let mut results = vec![map.clone()];
+            for (key, val) in map {
+                if let Value::Array(arr) = val {
+                    let mut new_results = Vec::new();
+                    for item in arr {
+                        let flattened_items = flatten_json(item);
+                        for flattened_item in flattened_items {
+                            for result in &results {
+                                let mut new_obj = result.clone();
+                                new_obj.insert(key.clone(), flattened_item.clone());
+                                new_results.push(new_obj);
+                            }
+                        }
+                    }
+                    results = new_results;
+                } else if let Value::Object(_) = val {
+                    let nested_results = flatten_json(val);
+                    let mut new_results = Vec::new();
+                    for nested_result in nested_results {
+                        for result in &results {
+                            let mut new_obj = result.clone();
+                            new_obj.insert(key.clone(), nested_result.clone());
+                            new_results.push(new_obj);
+                        }
+                    }
+                    results = new_results;
+                }
+            }
+            results.into_iter().map(Value::Object).collect()
+        }
+        _ => vec![value.clone()],
+    }
+}
+
+pub fn convert_to_array(flatterned: Vec<Value>) -> Value {
+    let mut result = Vec::new();
+    for item in flatterned {
+        let mut map = Map::new();
+        for (key, value) in item.as_object().unwrap() {
+            map.insert(key.clone(), value.clone());
+        }
+        result.push(Value::Object(map));
+    }
+    Value::Array(result)
+}
 #[cfg(test)]
 mod tests {
     use crate::utils::json::flatten::flatten_array_objects;

src/utils/json/mod.rs

@@ -28,8 +28,9 @@ pub fn flatten_json_body(
     custom_partition: Option<String>,
     validation_required: bool,
 ) -> Result<Value, anyhow::Error> {
+    let nested_value = flatten::convert_to_array(flatten::flatten_json(&body));
     flatten::flatten(
-        body,
+        nested_value,
         "_",
         time_partition,
         time_partition_limit,