Skip to content

fix: set key_metadata to Null by default #800

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Dec 14, 2024
Merged

fix: set key_metadata to Null by default #800

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
92ab3c4
fix: set key_metadata to Null by default
feniljain Dec 14, 2024
5a7d5ab
fix: return Option<&[u8]> instead of &Option<Vec<u8>> for key_metadata
feniljain Dec 14, 2024
a86b3f6
refactor: use as_deref instead of explicit ref/deref using map
feniljain Dec 14, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions crates/iceberg/src/expr/visitors/expression_evaluator.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -338,7 +338,7 @@ mod tests {
nan_value_counts: HashMap::new(),
lower_bounds: HashMap::new(),
upper_bounds: HashMap::new(),
key_metadata: vec![],
key_metadata: None,
split_offsets: vec![],
equality_ids: vec![],
sort_order_id: None,
Expand All @@ -361,7 +361,7 @@ mod tests {
nan_value_counts: HashMap::new(),
lower_bounds: HashMap::new(),
upper_bounds: HashMap::new(),
key_metadata: vec![],
key_metadata: None,
split_offsets: vec![],
equality_ids: vec![],
sort_order_id: None,
Expand Down
12 changes: 6 additions & 6 deletions crates/iceberg/src/expr/visitors/inclusive_metrics_evaluator.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1991,7 +1991,7 @@ mod test {
nan_value_counts: Default::default(),
lower_bounds: Default::default(),
upper_bounds: Default::default(),
key_metadata: vec![],
key_metadata: None,
split_offsets: vec![],
equality_ids: vec![],
sort_order_id: None,
Expand All @@ -2012,7 +2012,7 @@ mod test {
nan_value_counts: Default::default(),
lower_bounds: Default::default(),
upper_bounds: Default::default(),
key_metadata: vec![],
key_metadata: None,
split_offsets: vec![],
equality_ids: vec![],
sort_order_id: None,
Expand Down Expand Up @@ -2069,7 +2069,7 @@ mod test {
]),

column_sizes: Default::default(),
key_metadata: vec![],
key_metadata: None,
split_offsets: vec![],
equality_ids: vec![],
sort_order_id: None,
Expand All @@ -2095,7 +2095,7 @@ mod test {
upper_bounds: HashMap::from([(3, Datum::string("dC"))]),

column_sizes: Default::default(),
key_metadata: vec![],
key_metadata: None,
split_offsets: vec![],
equality_ids: vec![],
sort_order_id: None,
Expand All @@ -2122,7 +2122,7 @@ mod test {
upper_bounds: HashMap::from([(3, Datum::string("3str3"))]),

column_sizes: Default::default(),
key_metadata: vec![],
key_metadata: None,
split_offsets: vec![],
equality_ids: vec![],
sort_order_id: None,
Expand All @@ -2149,7 +2149,7 @@ mod test {
upper_bounds: HashMap::from([(3, Datum::string("イロハニホヘト"))]),

column_sizes: Default::default(),
key_metadata: vec![],
key_metadata: None,
split_offsets: vec![],
equality_ids: vec![],
sort_order_id: None,
Expand Down
1 change: 1 addition & 0 deletions crates/iceberg/src/io/object_cache.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -278,6 +278,7 @@ mod tests {
.file_size_in_bytes(100)
.record_count(1)
.partition(Struct::from_iter([Some(Literal::long(100))]))
.key_metadata(None)
.build()
.unwrap(),
)
Expand Down
1 change: 1 addition & 0 deletions crates/iceberg/src/scan.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1073,6 +1073,7 @@ mod tests {
.file_size_in_bytes(100)
.record_count(1)
.partition(Struct::from_iter([Some(Literal::long(100))]))
.key_metadata(None)
.build()
.unwrap(),
)
Expand Down
31 changes: 16 additions & 15 deletions crates/iceberg/src/spec/manifest.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1074,7 +1074,7 @@ pub struct DataFile {
///
/// Implementation-specific key metadata for encryption
#[builder(default)]
pub(crate) key_metadata: Vec<u8>,
pub(crate) key_metadata: Option<Vec<u8>>,
/// field id: 132
/// element field id: 133
///
Expand Down Expand Up @@ -1164,8 +1164,8 @@ impl DataFile {
&self.upper_bounds
}
/// Get the Implementation-specific key metadata for the data file.
pub fn key_metadata(&self) -> &[u8] {
&self.key_metadata
pub fn key_metadata(&self) -> Option<&[u8]> {
self.key_metadata.as_deref()
}
/// Get the split offsets of the data file.
/// For example, all row group offsets in a Parquet file.
Expand Down Expand Up @@ -1378,12 +1378,13 @@ mod _serde {
nan_value_counts: Some(to_i64_entry(value.nan_value_counts)?),
lower_bounds: Some(to_bytes_entry(value.lower_bounds)?),
upper_bounds: Some(to_bytes_entry(value.upper_bounds)?),
key_metadata: Some(serde_bytes::ByteBuf::from(value.key_metadata)),
key_metadata: value.key_metadata.map(serde_bytes::ByteBuf::from),
split_offsets: Some(value.split_offsets),
equality_ids: Some(value.equality_ids),
sort_order_id: value.sort_order_id,
})
}

pub fn try_into(
self,
partition_type: &StructType,
Expand Down Expand Up @@ -1441,7 +1442,7 @@ mod _serde {
.map(|v| parse_bytes_entry(v, schema))
.transpose()?
.unwrap_or_default(),
key_metadata: self.key_metadata.map(|v| v.to_vec()).unwrap_or_default(),
key_metadata: self.key_metadata.map(|v| v.to_vec()),
split_offsets: self.split_offsets.unwrap_or_default(),
equality_ids: self.equality_ids.unwrap_or_default(),
sort_order_id: self.sort_order_id,
Expand Down Expand Up @@ -1657,7 +1658,7 @@ mod tests {
nan_value_counts: HashMap::new(),
lower_bounds: HashMap::new(),
upper_bounds: HashMap::new(),
key_metadata: Vec::new(),
key_metadata: None,
split_offsets: vec![4],
equality_ids: Vec::new(),
sort_order_id: None,
Expand Down Expand Up @@ -1813,7 +1814,7 @@ mod tests {
nan_value_counts: HashMap::new(),
lower_bounds: HashMap::new(),
upper_bounds: HashMap::new(),
key_metadata: vec![],
key_metadata: None,
split_offsets: vec![4],
equality_ids: vec![],
sort_order_id: None,
Expand Down Expand Up @@ -1880,7 +1881,7 @@ mod tests {
nan_value_counts: HashMap::new(),
lower_bounds: HashMap::from([(1,Datum::int(1)),(2,Datum::string("a")),(3,Datum::string("AC/DC"))]),
upper_bounds: HashMap::from([(1,Datum::int(1)),(2,Datum::string("a")),(3,Datum::string("AC/DC"))]),
key_metadata: vec![],
key_metadata: None,
split_offsets: vec![4],
equality_ids: vec![],
sort_order_id: Some(0),
Expand Down Expand Up @@ -1960,7 +1961,7 @@ mod tests {
(2, Datum::string("a")),
(3, Datum::string("x"))
]),
key_metadata: vec![],
key_metadata: None,
split_offsets: vec![4],
equality_ids: vec![],
sort_order_id: Some(0),
Expand Down Expand Up @@ -2035,7 +2036,7 @@ mod tests {
(2, Datum::int(2)),
(3, Datum::string("x"))
]),
key_metadata: vec![],
key_metadata: None,
split_offsets: vec![4],
equality_ids: vec![],
sort_order_id: None,
Expand Down Expand Up @@ -2105,7 +2106,7 @@ mod tests {
(1, Datum::long(1)),
(2, Datum::int(2)),
]),
key_metadata: vec![],
key_metadata: None,
split_offsets: vec![4],
equality_ids: vec![],
sort_order_id: None,
Expand Down Expand Up @@ -2183,7 +2184,7 @@ mod tests {
nan_value_counts: HashMap::new(),
lower_bounds: HashMap::new(),
upper_bounds: HashMap::new(),
key_metadata: Vec::new(),
key_metadata: None,
split_offsets: vec![4],
equality_ids: Vec::new(),
sort_order_id: None,
Expand Down Expand Up @@ -2214,7 +2215,7 @@ mod tests {
nan_value_counts: HashMap::new(),
lower_bounds: HashMap::new(),
upper_bounds: HashMap::new(),
key_metadata: Vec::new(),
key_metadata: None,
split_offsets: vec![4],
equality_ids: Vec::new(),
sort_order_id: None,
Expand Down Expand Up @@ -2246,7 +2247,7 @@ mod tests {
nan_value_counts: HashMap::new(),
lower_bounds: HashMap::new(),
upper_bounds: HashMap::new(),
key_metadata: Vec::new(),
key_metadata: None,
split_offsets: vec![4],
equality_ids: Vec::new(),
sort_order_id: None,
Expand Down Expand Up @@ -2278,7 +2279,7 @@ mod tests {
nan_value_counts: HashMap::new(),
lower_bounds: HashMap::new(),
upper_bounds: HashMap::new(),
key_metadata: Vec::new(),
key_metadata: None,
split_offsets: vec![4],
equality_ids: Vec::new(),
sort_order_id: None,
Expand Down
2 changes: 1 addition & 1 deletion crates/iceberg/src/writer/file_writer/parquet_writer.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -381,7 +381,7 @@ impl ParquetWriter {
// # TODO(#417)
// - nan_value_counts
// - distinct_counts
.key_metadata(metadata.footer_signing_key_metadata.unwrap_or_default())
.key_metadata(metadata.footer_signing_key_metadata)
.split_offsets(
metadata
.row_groups
Expand Down
Loading