Merge remote-tracking branch 'upstream/main' into issue_7743

Author: zhuqi-lucas

arrow-arith/src/numeric.rs

@@ -22,7 +22,7 @@ use crate::types::bytes::ByteArrayNativeType;
 use crate::types::{BinaryViewType, ByteViewType, StringViewType};
 use crate::{Array, ArrayAccessor, ArrayRef, GenericByteArray, OffsetSizeTrait, Scalar};
 use arrow_buffer::{ArrowNativeType, Buffer, NullBuffer, ScalarBuffer};
-use arrow_data::{ArrayData, ArrayDataBuilder, ByteView};
+use arrow_data::{ArrayData, ArrayDataBuilder, ByteView, MAX_INLINE_VIEW_LEN};
 use arrow_schema::{ArrowError, DataType};
 use core::str;
 use num::ToPrimitive;
@@ -78,8 +78,9 @@ use super::ByteArrayType;
 ///                          0    31       63      95    127
 /// ```
 ///
-/// * Strings with length <= 12 are stored directly in the view. See
-///   [`Self::inline_value`] to access the inlined prefix from a short view.
+/// * Strings with length <= 12 ([`MAX_INLINE_VIEW_LEN`]) are stored directly in
+///   the view. See [`Self::inline_value`] to access the inlined prefix from a
+///   short view.
 ///
 /// * Strings with length > 12: The first four bytes are stored inline in the
 ///   view and the entire string is stored in one of the buffers. See [`ByteView`]
@@ -129,6 +130,7 @@ use super::ByteArrayType;
 /// assert_eq!(value, "this string is also longer than 12 bytes");
 /// ```
 ///
+/// [`MAX_INLINE_VIEW_LEN`]: arrow_data::MAX_INLINE_VIEW_LEN
 /// [`arrow_compute`]: https://docs.rs/arrow/latest/arrow/compute/index.html
 ///
 /// Unlike [`GenericByteArray`], there are no constraints on the offsets other
@@ -317,7 +319,7 @@ impl<T: ByteViewType + ?Sized> GenericByteViewArray<T> {
     pub unsafe fn value_unchecked(&self, idx: usize) -> &T::Native {
         let v = self.views.get_unchecked(idx);
         let len = *v as u32;
-        let b = if len <= 12 {
+        let b = if len <= MAX_INLINE_VIEW_LEN {
             Self::inline_value(v, len as usize)
         } else {
             let view = ByteView::from(*v);
@@ -332,10 +334,10 @@ impl<T: ByteViewType + ?Sized> GenericByteViewArray<T> {
     ///
     /// # Safety
     /// - The `view` must be a valid element from `Self::views()` that adheres to the view layout.
-    /// - The `len` must be the length of the inlined value. It should never be larger than 12.
+    /// - The `len` must be the length of the inlined value. It should never be larger than [`MAX_INLINE_VIEW_LEN`].
     #[inline(always)]
     pub unsafe fn inline_value(view: &u128, len: usize) -> &[u8] {
-        debug_assert!(len <= 12);
+        debug_assert!(len <= MAX_INLINE_VIEW_LEN as usize);
         std::slice::from_raw_parts((view as *const u128 as *const u8).wrapping_add(4), len)
     }
 
@@ -348,7 +350,7 @@ impl<T: ByteViewType + ?Sized> GenericByteViewArray<T> {
     pub fn bytes_iter(&self) -> impl Iterator<Item = &[u8]> {
         self.views.iter().map(move |v| {
             let len = *v as u32;
-            if len <= 12 {
+            if len <= MAX_INLINE_VIEW_LEN {
                 unsafe { Self::inline_value(v, len as usize) }
             } else {
                 let view = ByteView::from(*v);
@@ -372,7 +374,7 @@ impl<T: ByteViewType + ?Sized> GenericByteViewArray<T> {
                 return &[] as &[u8];
             }
 
-            if prefix_len <= 4 || len <= 12 {
+            if prefix_len <= 4 || len as u32 <= MAX_INLINE_VIEW_LEN {
                 unsafe { StringViewArray::inline_value(v, prefix_len) }
             } else {
                 let view = ByteView::from(*v);
@@ -402,7 +404,7 @@ impl<T: ByteViewType + ?Sized> GenericByteViewArray<T> {
                 return &[] as &[u8];
             }
 
-            if len <= 12 {
+            if len as u32 <= MAX_INLINE_VIEW_LEN {
                 unsafe { &StringViewArray::inline_value(v, len)[len - suffix_len..] }
             } else {
                 let view = ByteView::from(*v);
@@ -496,9 +498,9 @@ impl<T: ByteViewType + ?Sized> GenericByteViewArray<T> {
         self.views()
             .iter()
             .map(|v| {
-                let len = (*v as u32) as usize;
-                if len > 12 {
-                    len
+                let len = *v as u32;
+                if len > MAX_INLINE_VIEW_LEN {
+                    len as usize
                 } else {
                     0
                 }
@@ -512,11 +514,11 @@ impl<T: ByteViewType + ?Sized> GenericByteViewArray<T> {
     /// It takes a bit of patience to understand why we don't just compare two &[u8] directly.
     ///
     /// ByteView types give us the following two advantages, and we need to be careful not to lose them:
-    /// (1) For string/byte smaller than 12 bytes, the entire data is inlined in the view.
+    /// (1) For string/byte smaller than [`MAX_INLINE_VIEW_LEN`] bytes, the entire data is inlined in the view.
     ///     Meaning that reading one array element requires only one memory access
     ///     (two memory access required for StringArray, one for offset buffer, the other for value buffer).
     ///
-    /// (2) For string/byte larger than 12 bytes, we can still be faster than (for certain operations) StringArray/ByteArray,
+    /// (2) For string/byte larger than [`MAX_INLINE_VIEW_LEN`] bytes, we can still be faster than (for certain operations) StringArray/ByteArray,
     ///     thanks to the inlined 4 bytes.
     ///     Consider equality check:
     ///     If the first four bytes of the two strings are different, we can return false immediately (with just one memory access).
@@ -526,8 +528,8 @@ impl<T: ByteViewType + ?Sized> GenericByteViewArray<T> {
     ///   e.g., if the inlined 4 bytes are different, we can directly return unequal without looking at the full string.
     ///
     /// # Order check flow
-    /// (1) if both string are smaller than 12 bytes, we can directly compare the data inlined to the view.
-    /// (2) if any of the string is larger than 12 bytes, we need to compare the full string.
+    /// (1) if both string are smaller than [`MAX_INLINE_VIEW_LEN`] bytes, we can directly compare the data inlined to the view.
+    /// (2) if any of the string is larger than [`MAX_INLINE_VIEW_LEN`] bytes, we need to compare the full string.
     ///     (2.1) if the inlined 4 bytes are different, we can return the result immediately.
     ///     (2.2) o.w., we need to compare the full string.
     ///
@@ -555,7 +557,7 @@ impl<T: ByteViewType + ?Sized> GenericByteViewArray<T> {
         // one of the string is larger than 12 bytes,
         // we then try to compare the inlined data first
 
-        // Note: In theory, ByteView is only used for views larger than 12 bytes,
+        // Note: In theory, ByteView is only used for string which is larger than 12 bytes,
         // but we can still use it to get the inlined prefix for shorter strings.
         // The prefix is always the first 4 bytes of the view, for both short and long strings.
         let l_inlined_be = l_byte_view.prefix.swap_bytes();

arrow-array/src/array/byte_view_array.rs

@@ -343,8 +343,8 @@ impl From<ArrayData> for FixedSizeListArray {
     fn from(data: ArrayData) -> Self {
         let value_length = match data.data_type() {
             DataType::FixedSizeList(_, len) => *len,
-            _ => {
-                panic!("FixedSizeListArray data should contain a FixedSizeList data type")
+            data_type => {
+                panic!("FixedSizeListArray data should contain a FixedSizeList data type, got {data_type:?}")
             }
         };
 

arrow-array/src/array/fixed_size_list_array.rs

@@ -20,7 +20,7 @@ use std::marker::PhantomData;
 use std::sync::Arc;
 
 use arrow_buffer::{Buffer, NullBufferBuilder, ScalarBuffer};
-use arrow_data::ByteView;
+use arrow_data::{ByteView, MAX_INLINE_VIEW_LEN};
 use arrow_schema::ArrowError;
 use hashbrown::hash_table::Entry;
 use hashbrown::HashTable;
@@ -68,8 +68,8 @@ impl BlockSizeGrowthStrategy {
 ///
 /// To avoid bump allocating, this builder allocates data in fixed size blocks, configurable
 /// using [`GenericByteViewBuilder::with_fixed_block_size`]. [`GenericByteViewBuilder::append_value`]
-/// writes values larger than 12 bytes to the current in-progress block, with values smaller
-/// than 12 bytes inlined into the views. If a value is appended that will not fit in the
+/// writes values larger than [`MAX_INLINE_VIEW_LEN`] bytes to the current in-progress block, with values smaller
+/// than [`MAX_INLINE_VIEW_LEN`] bytes inlined into the views. If a value is appended that will not fit in the
 /// in-progress block, it will be closed, and a new block of sufficient size allocated
 ///
 /// # Append Views
@@ -114,7 +114,7 @@ impl<T: ByteViewType + ?Sized> GenericByteViewBuilder<T> {
     /// Set a fixed buffer size for variable length strings
     ///
     /// The block size is the size of the buffer used to store values greater
-    /// than 12 bytes. The builder allocates new buffers when the current
+    /// than [`MAX_INLINE_VIEW_LEN`] bytes. The builder allocates new buffers when the current
     /// buffer is full.
     ///
     /// By default the builder balances buffer size and buffer count by
@@ -221,7 +221,7 @@ impl<T: ByteViewType + ?Sized> GenericByteViewBuilder<T> {
         } else {
             self.views_buffer.extend(array.views().iter().map(|v| {
                 let mut byte_view = ByteView::from(*v);
-                if byte_view.length > 12 {
+                if byte_view.length > MAX_INLINE_VIEW_LEN {
                     // Small views (<=12 bytes) are inlined, so only need to update large views
                     byte_view.buffer_index += starting_buffer;
                 };
@@ -289,7 +289,7 @@ impl<T: ByteViewType + ?Sized> GenericByteViewBuilder<T> {
     pub fn get_value(&self, index: usize) -> &[u8] {
         let view = self.views_buffer.as_slice().get(index).unwrap();
         let len = *view as u32;
-        if len <= 12 {
+        if len <= MAX_INLINE_VIEW_LEN {
             // # Safety
             // The view is valid from the builder
             unsafe { GenericByteViewArray::<T>::inline_value(view, len as usize) }
@@ -315,7 +315,7 @@ impl<T: ByteViewType + ?Sized> GenericByteViewBuilder<T> {
     pub fn append_value(&mut self, value: impl AsRef<T::Native>) {
         let v: &[u8] = value.as_ref().as_ref();
         let length: u32 = v.len().try_into().unwrap();
-        if length <= 12 {
+        if length <= MAX_INLINE_VIEW_LEN {
             let mut view_buffer = [0; 16];
             view_buffer[0..4].copy_from_slice(&length.to_le_bytes());
             view_buffer[4..4 + v.len()].copy_from_slice(v);