clean up

Author: jqin61

Makefile

@@ -40,7 +40,7 @@ test-integration:
 	docker-compose -f dev/docker-compose-integration.yml kill
 	docker-compose -f dev/docker-compose-integration.yml rm -f
 	docker-compose -f dev/docker-compose-integration.yml up -d
-	sleep 5
+	sleep 10
 	docker-compose -f dev/docker-compose-integration.yml exec -T spark-iceberg ipython ./provision.py
 	poetry run pytest tests/ -v -m integration ${PYTEST_ARGS}
 

pyiceberg/avro/file.py

@@ -276,7 +276,6 @@ def _write_header(self) -> None:
     def write_block(self, objects: List[D]) -> None:
         in_memory = io.BytesIO()
         block_content_encoder = BinaryEncoder(output_stream=in_memory)
-
         for obj in objects:
             self.writer.write(block_content_encoder, obj)
         block_content = in_memory.getvalue()

pyiceberg/io/pyarrow.py

@@ -1719,10 +1719,6 @@ def fill_parquet_file_metadata(
     data_file.split_offsets = split_offsets
 
 
-# write_file(table: Table, tasks: Iterator[WriteTask]) -> Iterator[DataFile]:
-# write_file(io = table.io, table_metadata = table.metadata, tasks: Iterator[WriteTask]) -> Iterator[DataFile]:
-
-
 def write_file(io: FileIO, table_metadata: TableMetadata, tasks: Iterator[WriteTask]) -> Iterator[DataFile]:
     for task in tasks:
         parquet_writer_kwargs = _get_parquet_writer_kwargs(table_metadata.properties)

pyiceberg/partitioning.py

@@ -231,6 +231,7 @@ def partition_to_path(self, data: Record, schema: Schema) -> str:
             partition_field = self.fields[pos]
             value_str = partition_field.transform.to_human_string(field_types[pos].field_type, value=value)
             value_str = quote(value_str, safe='')
+        
             value_strs.append(value_str)
             field_strs.append(partition_field.name)
 

pyiceberg/table/__init__.py

@@ -232,7 +232,6 @@ def property_as_int(properties: Dict[str, str], property_name: str, default: Opt
             return default
 
 
-# to do
 class PartitionProjector:
     def __init__(
         self,
@@ -2528,8 +2527,9 @@ def _write_filtered_manifest(self, file_num: int = 1) -> List[ManifestFile]:
         ) as writer:
             for manifest_entry in filtered_entries:
                 writer.add_entry(manifest_entry)
-        m = writer.to_manifest_file()
-        return [m]
+        
+        return [writer.to_manifest_file()]
+
 
     def _write_added_manifest(self) -> List[ManifestFile]:
         if self._added_data_files:

pyiceberg/table/snapshots.py

@@ -364,4 +364,5 @@ def _update_totals(total_property: str, added_property: str, removed_property: s
         added_property=ADDED_EQUALITY_DELETES,
         removed_property=REMOVED_EQUALITY_DELETES,
     )
+    
     return summary

tests/integration/test_partitioned_writes.py

@@ -170,7 +170,7 @@ def table_v1_with_null_partitioned(session_catalog: Catalog, arrow_table_with_nu
         "timestamptz",
         "timestamp",
         "binary",
-    ]  # [, "date", "timestamptz", "timestamp", "binary", "fixed"]
+    ]  
     for partition_col in partition_cols:
         identifier = f"default.arrow_table_v1_with_null_partitioned_on_col_{partition_col}"
 
@@ -555,7 +555,7 @@ def test_data_files_with_table_partitioned_with_null(
     # fourth operation of append manifest list abandons M3 since it has no existing or added entries and keeps M4 and added M5 with 3 added files
     # fifth operation of static overwrite's manifest list is linked to one filtered manifest M7 which filters and merges M5 and M6 where each has 1 entrys are deleted (int=1 matching the filter) and 2 entries marked as existed, this operation
     # also links to M6 which adds 3 entries.
-    # so we have flattened list of [[M1], [M1, M2], [M3, M4], [M4, M5], [M6, M7, M8]]
+    # so we have flattened list of [[M1], [M1, M2], [M3, M4], [M4, M5], [M6, M7]]
     # where: add      exist      delete    added_by
     # M1      3         0           0        S1
     # M2      3         0           0        S2