Fix reading some RDC compressed pages

Author: jonashaag

doc/source/whatsnew/v1.5.0.rst

@@ -814,6 +814,7 @@ I/O
 - Bug in :func:`read_excel` when reading a ``.ods`` file with newlines between xml elements (:issue:`45598`)
 - Bug in :func:`read_parquet` when ``engine="fastparquet"`` where the file was not closed on error (:issue:`46555`)
 - :meth:`to_html` now excludes the ``border`` attribute from ``<table>`` elements when ``border`` keyword is set to ``False``.
+- Bug in :func:`read_sas` with certain types of compressed SAS7BDAT files (:issue:`35545`)
 -
 
 Period

pandas/io/sas/sas.pyx

@@ -203,8 +203,7 @@ cdef enum ColumnTypes:
 # type the page_data types
 cdef:
     int page_meta_type = const.page_meta_type
-    int page_mix_types_0 = const.page_mix_types[0]
-    int page_mix_types_1 = const.page_mix_types[1]
+    int page_mix_type = const.page_mix_type
     int page_data_type = const.page_data_type
     int subheader_pointers_offset = const.subheader_pointers_offset
 
@@ -350,8 +349,7 @@ cdef class Parser:
                     current_subheader_pointer.offset,
                     current_subheader_pointer.length)
                 return False
-            elif (self.current_page_type == page_mix_types_0 or
-                    self.current_page_type == page_mix_types_1):
+            elif self.current_page_type == page_mix_type:
                 align_correction = (
                     bit_offset
                     + subheader_pointers_offset
@@ -369,7 +367,7 @@ cdef class Parser:
                     if done:
                         return True
                 return False
-            elif self.current_page_type & page_data_type == page_data_type:
+            elif self.current_page_type == page_data_type:
                 self.process_byte_array_with_data(
                     bit_offset
                     + subheader_pointers_offset

pandas/io/sas/sas7bdat.py

@@ -410,11 +410,11 @@ def _parse_metadata(self) -> None:
 
     def _process_page_meta(self) -> bool:
         self._read_page_header()
-        pt = [const.page_meta_type, const.page_amd_type] + const.page_mix_types
+        pt = [const.page_meta_type, const.page_amd_type, const.page_mix_type]
         if self._current_page_type in pt:
             self._process_page_metadata()
-        is_data_page = self._current_page_type & const.page_data_type
-        is_mix_page = self._current_page_type in const.page_mix_types
+        is_data_page = self._current_page_type == const.page_data_type
+        is_mix_page = self._current_page_type == const.page_mix_type
         return bool(
             is_data_page
             or is_mix_page
@@ -424,7 +424,9 @@ def _process_page_meta(self) -> bool:
     def _read_page_header(self):
         bit_offset = self._page_bit_offset
         tx = const.page_type_offset + bit_offset
-        self._current_page_type = self._read_int(tx, const.page_type_length)
+        self._current_page_type = (
+            self._read_int(tx, const.page_type_length) & const.page_type_mask2
+        )
         tx = const.block_count_offset + bit_offset
         self._current_page_block_count = self._read_int(tx, const.block_count_length)
         tx = const.subheader_count_offset + bit_offset
@@ -774,13 +776,14 @@ def _read_next_page(self):
             raise ValueError(msg)
 
         self._read_page_header()
-        page_type = self._current_page_type
-        if page_type == const.page_meta_type:
+        if self._current_page_type == const.page_meta_type:
             self._process_page_metadata()
 
-        is_data_page = page_type & const.page_data_type
-        pt = [const.page_meta_type] + const.page_mix_types
-        if not is_data_page and self._current_page_type not in pt:
+        if self._current_page_type not in [
+            const.page_data_type,
+            const.page_meta_type,
+            const.page_mix_type,
+        ]:
             return self._read_next_page()
 
         return False

pandas/io/sas/sas_constants.py

@@ -53,12 +53,14 @@
 block_count_length = 2
 subheader_count_offset = 4
 subheader_count_length = 2
-page_meta_type = 0
-page_data_type = 256
-page_amd_type = 1024
-page_metc_type = 16384
-page_comp_type = -28672
-page_mix_types = [512, 640]
+page_type_mask = 0x0F00
+# Keep "page_comp_type" bits
+page_type_mask2 = 0xF000 | page_type_mask
+page_meta_type = 0x0000
+page_data_type = 0x0100
+page_mix_type = 0x0200
+page_amd_type = 0x0400
+page_comp_type = 0x9000
 subheader_pointers_offset = 8
 truncated_subheader_id = 1
 compressed_subheader_id = 4