fix for invalid parquet (#849)

nikhilsinhaparseable · web-flow · commit 16bf9b3d5185 · 2024-07-16T16:46:04.000+05:30
delete invalid parquets where file size is less than the length of the parquet footer Fixes: #848
diff --git a/server/src/storage/staging.rs b/server/src/storage/staging.rs
@@ -280,7 +280,7 @@ pub fn convert_disk_files_to_parquet(
         }
 
         writer.close()?;
-        if parquet_file.metadata().unwrap().len() == 0 {
+        if parquet_file.metadata().unwrap().len() < parquet::file::FOOTER_SIZE as u64 {
             log::error!(
                 "Invalid parquet file {:?} detected for stream {}, removing it",
                 &parquet_path,

Original file line number	Diff line number	Diff line change
`@@ -280,7 +280,7 @@ pub fn convert_disk_files_to_parquet(`
`280`	`280`	`}`
`281`	`281`
`282`	`282`	`writer.close()?;`
`283`		`- if parquet_file.metadata().unwrap().len() == 0 {`
	`283`	`+ if parquet_file.metadata().unwrap().len() < parquet::file::FOOTER_SIZE as u64 {`
`284`	`284`	`log::error!(`
`285`	`285`	`"Invalid parquet file {:?} detected for stream {}, removing it",`
`286`	`286`	`&parquet_path,`