From ff6c20fc102b42ce63a0885ec62c70e8ebbb042a Mon Sep 17 00:00:00 2001 From: Satyam Singh Date: Thu, 9 Mar 2023 17:48:34 +0530 Subject: [PATCH] Enable compression flag --- server/src/main.rs | 2 +- server/src/storage/object_storage.rs | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/server/src/main.rs b/server/src/main.rs index 769e029ea..0fbf490e6 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -90,7 +90,7 @@ async fn main() -> anyhow::Result<()> { { if std::env::var(DEBUG_PYROSCOPE_URL).is_ok() { let url = std::env::var(DEBUG_PYROSCOPE_URL).ok(); - Some(start_profiling(url.unwrap())); + start_profiling(url.unwrap()); } } diff --git a/server/src/storage/object_storage.rs b/server/src/storage/object_storage.rs index f4013d5ac..cf35edbae 100644 --- a/server/src/storage/object_storage.rs +++ b/server/src/storage/object_storage.rs @@ -32,7 +32,7 @@ use crate::{ use actix_web_prometheus::PrometheusMetrics; use async_trait::async_trait; use bytes::Bytes; -use datafusion::arrow::datatypes::Schema; +use datafusion::{arrow::datatypes::Schema, parquet::basic::Compression}; use datafusion::{ arrow::{ array::TimestampMillisecondArray, ipc::reader::StreamReader, record_batch::RecordBatch, @@ -321,7 +321,11 @@ pub trait ObjectStorage: Sync + 'static { fs::File::create(&parquet_path).map_err(|_| MoveDataError::Create)?; parquet_table.upsert(&parquet_path); - let props = WriterProperties::builder().build(); + let props = WriterProperties::builder() + .set_compression(Compression::SNAPPY) + .set_max_row_group_size(1024 * 256) + .build(); + let schema = Arc::new(record_reader.merged_schema()); let mut writer = ArrowWriter::try_new(parquet_file, schema.clone(), Some(props))?;