From e79ce1f076bc8fd97a07f3f0658277c49d1a94e1 Mon Sep 17 00:00:00 2001 From: HonahX Date: Wed, 5 Jun 2024 23:59:41 -0700 Subject: [PATCH 1/6] test --- pyiceberg/table/snapshots.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/pyiceberg/table/snapshots.py b/pyiceberg/table/snapshots.py index b21a0f5613..a810196cc4 100644 --- a/pyiceberg/table/snapshots.py +++ b/pyiceberg/table/snapshots.py @@ -19,6 +19,7 @@ import time from collections import defaultdict from enum import Enum +from functools import lru_cache from typing import TYPE_CHECKING, Any, DefaultDict, Dict, Iterable, List, Mapping, Optional from pydantic import Field, PrivateAttr, model_serializer @@ -247,12 +248,19 @@ def __str__(self) -> str: result_str = f"{operation}id={self.snapshot_id}{parent_id}{schema_id}" return result_str - def manifests(self, io: FileIO) -> List[ManifestFile]: - if self.manifest_list is not None: - file = io.new_input(self.manifest_list) + @staticmethod + @lru_cache + def _manifests(io: FileIO, manifest_list: Optional[str]) -> List[ManifestFile]: + """Return the manifests for the given snapshot.""" + if manifest_list not in (None, ""): + file = io.new_input(manifest_list) # type: ignore return list(read_manifest_list(file)) return [] + def manifests(self, io: FileIO) -> List[ManifestFile]: + """Return the manifests for the given snapshot.""" + return Snapshot._manifests(io, self.manifest_list) + class MetadataLogEntry(IcebergBaseModel): metadata_file: str = Field(alias="metadata-file") From 24571a44c371b62e42cebe70546fcc186a511def Mon Sep 17 00:00:00 2001 From: HonahX Date: Thu, 6 Jun 2024 00:15:37 -0700 Subject: [PATCH 2/6] test2 --- pyiceberg/table/snapshots.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pyiceberg/table/snapshots.py b/pyiceberg/table/snapshots.py index a810196cc4..c3c853f07b 100644 --- a/pyiceberg/table/snapshots.py +++ b/pyiceberg/table/snapshots.py @@ -19,7 +19,6 @@ import time from collections import defaultdict from enum import Enum -from functools import lru_cache from typing import TYPE_CHECKING, Any, DefaultDict, Dict, Iterable, List, Mapping, Optional from pydantic import Field, PrivateAttr, model_serializer @@ -249,7 +248,6 @@ def __str__(self) -> str: return result_str @staticmethod - @lru_cache def _manifests(io: FileIO, manifest_list: Optional[str]) -> List[ManifestFile]: """Return the manifests for the given snapshot.""" if manifest_list not in (None, ""): From 557586b885136c490b8ff9ea55a355dcca372767 Mon Sep 17 00:00:00 2001 From: HonahX Date: Thu, 6 Jun 2024 00:41:30 -0700 Subject: [PATCH 3/6] test3 --- pyiceberg/table/snapshots.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/pyiceberg/table/snapshots.py b/pyiceberg/table/snapshots.py index c3c853f07b..767f5e92f2 100644 --- a/pyiceberg/table/snapshots.py +++ b/pyiceberg/table/snapshots.py @@ -19,6 +19,7 @@ import time from collections import defaultdict from enum import Enum +from functools import lru_cache from typing import TYPE_CHECKING, Any, DefaultDict, Dict, Iterable, List, Mapping, Optional from pydantic import Field, PrivateAttr, model_serializer @@ -228,6 +229,15 @@ def __eq__(self, other: Any) -> bool: ) +@lru_cache +def _manifests(io: FileIO, manifest_list: Optional[str]) -> List[ManifestFile]: + """Return the manifests for the given snapshot.""" + if manifest_list not in (None, ""): + file = io.new_input(manifest_list) # type: ignore + return list(read_manifest_list(file)) + return [] + + class Snapshot(IcebergBaseModel): snapshot_id: int = Field(alias="snapshot-id") parent_snapshot_id: Optional[int] = Field(alias="parent-snapshot-id", default=None) @@ -247,14 +257,6 @@ def __str__(self) -> str: result_str = f"{operation}id={self.snapshot_id}{parent_id}{schema_id}" return result_str - @staticmethod - def _manifests(io: FileIO, manifest_list: Optional[str]) -> List[ManifestFile]: - """Return the manifests for the given snapshot.""" - if manifest_list not in (None, ""): - file = io.new_input(manifest_list) # type: ignore - return list(read_manifest_list(file)) - return [] - def manifests(self, io: FileIO) -> List[ManifestFile]: """Return the manifests for the given snapshot.""" return Snapshot._manifests(io, self.manifest_list) From 5d033190b6bc4984a8c9a9fb1e7d7464e03551dc Mon Sep 17 00:00:00 2001 From: HonahX Date: Thu, 6 Jun 2024 00:50:09 -0700 Subject: [PATCH 4/6] test4 --- pyiceberg/table/snapshots.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyiceberg/table/snapshots.py b/pyiceberg/table/snapshots.py index 767f5e92f2..123611a9f5 100644 --- a/pyiceberg/table/snapshots.py +++ b/pyiceberg/table/snapshots.py @@ -259,7 +259,7 @@ def __str__(self) -> str: def manifests(self, io: FileIO) -> List[ManifestFile]: """Return the manifests for the given snapshot.""" - return Snapshot._manifests(io, self.manifest_list) + return _manifests(io, self.manifest_list) class MetadataLogEntry(IcebergBaseModel): From 88f74f7fd33e2d94ee19106b8a7f5ad4bddca044 Mon Sep 17 00:00:00 2001 From: HonahX Date: Thu, 6 Jun 2024 01:12:35 -0700 Subject: [PATCH 5/6] manual load --- tests/integration/test_writes/test_writes.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_writes/test_writes.py b/tests/integration/test_writes/test_writes.py index e329adcd5c..598c5b5724 100644 --- a/tests/integration/test_writes/test_writes.py +++ b/tests/integration/test_writes/test_writes.py @@ -555,7 +555,8 @@ def test_duckdb_url_import(warehouse: Path, arrow_table_with_null: pa.Table) -> import duckdb - duckdb.sql("INSTALL iceberg; LOAD iceberg;") + duckdb.install_extension("iceberg") + duckdb.load_extension("iceberg") result = duckdb.sql( f""" SELECT * From ac04865e30394ac370347717a17cf48aa87dec56 Mon Sep 17 00:00:00 2001 From: HonahX Date: Tue, 11 Jun 2024 23:45:07 -0700 Subject: [PATCH 6/6] update --- pyiceberg/table/snapshots.py | 2 -- tests/integration/test_writes/test_writes.py | 3 +-- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/pyiceberg/table/snapshots.py b/pyiceberg/table/snapshots.py index 123611a9f5..c2820dd724 100644 --- a/pyiceberg/table/snapshots.py +++ b/pyiceberg/table/snapshots.py @@ -19,7 +19,6 @@ import time from collections import defaultdict from enum import Enum -from functools import lru_cache from typing import TYPE_CHECKING, Any, DefaultDict, Dict, Iterable, List, Mapping, Optional from pydantic import Field, PrivateAttr, model_serializer @@ -229,7 +228,6 @@ def __eq__(self, other: Any) -> bool: ) -@lru_cache def _manifests(io: FileIO, manifest_list: Optional[str]) -> List[ManifestFile]: """Return the manifests for the given snapshot.""" if manifest_list not in (None, ""): diff --git a/tests/integration/test_writes/test_writes.py b/tests/integration/test_writes/test_writes.py index 598c5b5724..e329adcd5c 100644 --- a/tests/integration/test_writes/test_writes.py +++ b/tests/integration/test_writes/test_writes.py @@ -555,8 +555,7 @@ def test_duckdb_url_import(warehouse: Path, arrow_table_with_null: pa.Table) -> import duckdb - duckdb.install_extension("iceberg") - duckdb.load_extension("iceberg") + duckdb.sql("INSTALL iceberg; LOAD iceberg;") result = duckdb.sql( f""" SELECT *