From 3c4348991e918180bc4b88e64020af727491dcfa Mon Sep 17 00:00:00 2001 From: Sergei Turukin Date: Thu, 24 Oct 2024 11:20:15 -0500 Subject: [PATCH] Fix tests --- crates/control_plane/src/models/mod.rs | 2 +- crates/nexus/src/http/catalog/router.rs | 3 +- tests/conftest.py | 49 +++++++++++++++++++++---- tests/test_catalog.py | 33 ++++++----------- 4 files changed, 55 insertions(+), 32 deletions(-) diff --git a/crates/control_plane/src/models/mod.rs b/crates/control_plane/src/models/mod.rs index f92249d74..68bc1c230 100644 --- a/crates/control_plane/src/models/mod.rs +++ b/crates/control_plane/src/models/mod.rs @@ -185,7 +185,7 @@ impl Warehouse { let location = format!("{prefix}/{id}"); let now = Utc::now().naive_utc(); Ok(Self { - id: Uuid::new_v4(), + id, prefix, name, location, diff --git a/crates/nexus/src/http/catalog/router.rs b/crates/nexus/src/http/catalog/router.rs index 1bd2b455f..deb5b575b 100644 --- a/crates/nexus/src/http/catalog/router.rs +++ b/crates/nexus/src/http/catalog/router.rs @@ -4,12 +4,13 @@ use axum::Router; use crate::http::catalog::handlers::{ commit_table, create_namespace, create_table, delete_namespace, delete_table, get_config, - get_namespace, get_table, list_namespaces, + get_namespace, get_table, list_namespaces, list_tables, }; pub fn create_router() -> Router { let table_router: Router = Router::new() .route("/", post(create_table)) + .route("/", get(list_tables)) .route("/:table", get(get_table)) .route("/:table", delete(delete_table)) .route("/:table", post(commit_table)); diff --git a/tests/conftest.py b/tests/conftest.py index fc37acf92..c304d2597 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -22,6 +22,13 @@ S3_PATH_STYLE_ACCESS = os.environ.get("S3_PATH_STYLE_ACCESS") +@dataclasses.dataclass +class Namespace: + name: [str] + properties: dict + warehouse_id: str + + @dataclasses.dataclass class Server: catalog_url: str @@ -39,12 +46,24 @@ def create_storage_profile(self, **payload) -> dict: assert response.ok, response.text return response.json() + def delete_storage_profile(self, storage_profile: dict) -> None: + sid = storage_profile.get("id", None) + assert sid + storage_profile_url = urllib.parse.urljoin( + self.management_url, f"v1/storage-profile/{sid}" + ) + response = requests.delete( + storage_profile_url, + json=storage_profile, + ) + assert response.ok, response.text + def create_warehouse( self, **payload, ) -> uuid.UUID: """Create a warehouse in this server""" - warehouse_url = self.warehouse_url + warehouse_url = urllib.parse.urljoin(self.management_url, "v1/warehouse") payload = {k.replace("_", "-"): v for k, v in payload.items()} response = requests.post( warehouse_url, @@ -53,9 +72,15 @@ def create_warehouse( assert response.ok, response.text return response.json() - @property - def warehouse_url(self) -> str: - return urllib.parse.urljoin(self.management_url, "v1/warehouse") + def delete_warehouse(self, warehouse: dict) -> None: + wid = warehouse.get("id", None) + assert wid + warehouse_url = urllib.parse.urljoin(self.management_url, f"v1/warehouse/{wid}") + response = requests.delete( + warehouse_url, + json=warehouse, + ) + assert response.ok, response.text @pytest.fixture(scope="session") @@ -73,7 +98,7 @@ def server() -> Server: @pytest.fixture(scope="session") def storage_profile(server: Server) -> dict: - return server.create_storage_profile( + data = server.create_storage_profile( type="aws", region=S3_REGION, bucket=S3_BUCKET, @@ -86,6 +111,9 @@ def storage_profile(server: Server) -> dict: endpoint=S3_ENDPOINT, ) + yield data + server.delete_storage_profile(data) + @pytest.fixture(scope="session") def warehouse(server: Server, storage_profile) -> dict: @@ -96,14 +124,19 @@ def warehouse(server: Server, storage_profile) -> dict: prefix=preix, storage_profile_id=storage_profile.get("id", None), ) - return wh + yield wh + server.delete_warehouse(wh) @pytest.fixture(scope="session") def namespace(catalog) -> dict: - namespace = "test-namespace" + namespace = ("test-namespace",) catalog.create_namespace(namespace) - return catalog.get_namespace(namespace) + return Namespace( + name=namespace, + properties={}, + warehouse_id=catalog.properties.get("warehouse", None), + ) @pytest.fixture(scope="session") diff --git a/tests/test_catalog.py b/tests/test_catalog.py index c4b5e4ec6..793f62915 100644 --- a/tests/test_catalog.py +++ b/tests/test_catalog.py @@ -6,15 +6,13 @@ import pyiceberg.io as io -def test_create_namespace(warehouse): - catalog = conftest.catalog(warehouse) +def test_create_namespace(catalog): namespace = ("test_create_namespace",) catalog.create_namespace(namespace) assert namespace in catalog.list_namespaces() -def test_list_namespaces(warehouse): - catalog = conftest.catalog(warehouse) +def test_list_namespaces(catalog): catalog.create_namespace(("test_list_namespaces_1",)) catalog.create_namespace(("test_list_namespaces_2")) namespaces = catalog.list_namespaces() @@ -22,16 +20,14 @@ def test_list_namespaces(warehouse): assert ("test_list_namespaces_2",) in namespaces -def test_default_location_for_namespace_is_set(warehouse): - catalog = conftest.catalog(warehouse) +def test_default_location_for_namespace_is_set(catalog): namespace = ("test_default_location_for_namespace",) catalog.create_namespace(namespace) loaded_properties = catalog.load_namespace_properties(namespace) assert "location" in loaded_properties -def test_namespace_properties(warehouse): - catalog = conftest.catalog(warehouse) +def test_namespace_properties(catalog): namespace = ("test_namespace_properties",) properties = {"key-1": "value-1", "key2": "value2"} catalog.create_namespace(namespace, properties=properties) @@ -40,8 +36,7 @@ def test_namespace_properties(warehouse): assert loaded_properties[key] == value -def test_drop_namespace(warehouse): - catalog = conftest.catalog(warehouse) +def test_drop_namespace(catalog): namespace = ("test_drop_namespace",) catalog.create_namespace(namespace) assert namespace in catalog.list_namespaces() @@ -49,8 +44,7 @@ def test_drop_namespace(warehouse): assert namespace not in catalog.list_namespaces() -def test_create_table(warehouse): - catalog = conftest.catalog(warehouse) +def test_create_table(catalog): namespace = ("test_create_table",) table_name = "my_table" schema = pa.schema( @@ -71,8 +65,7 @@ def test_create_table(warehouse): assert len(loaded_table.schema().fields) == 3 -def test_drop_table(namespace): - catalog = conftest.catalog(namespace.warehouse) +def test_drop_table(catalog, namespace): table_name = "my_table" schema = pa.schema( [ @@ -89,8 +82,7 @@ def test_drop_table(namespace): assert "NoSuchTableError" in str(e) -def test_drop_purge_table(namespace, storage_config): - catalog = conftest.catalog(namespace.warehouse) +def test_drop_purge_table(catalog, namespace, storage_config): table_name = "my_table" schema = pa.schema( [ @@ -133,8 +125,7 @@ def test_drop_purge_table(namespace, storage_config): assert "NoSuchTableError" in str(e) -def test_table_properties(namespace): - catalog = conftest.catalog(namespace.warehouse) +def test_table_properties(catalog, namespace): table_name = "my_table" schema = pa.schema( [ @@ -151,8 +142,7 @@ def test_table_properties(namespace): assert table.properties == properties -def test_list_tables(namespace): - catalog = conftest.catalog(namespace.warehouse) +def test_list_tables(catalog, namespace): assert len(catalog.list_tables(namespace.name)) == 0 table_name_1 = "my_table_1" table_name_2 = "my_table_2" @@ -171,8 +161,7 @@ def test_list_tables(namespace): assert (*namespace.name, table_name_2) in tables -def test_write_read(namespace): - catalog = conftest.catalog(namespace.warehouse) +def test_write_read(catalog, namespace): table_name = "my_table" schema = pa.schema( [