Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion crates/control_plane/src/models/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -185,7 +185,7 @@ impl Warehouse {
let location = format!("{prefix}/{id}");
let now = Utc::now().naive_utc();
Ok(Self {
id: Uuid::new_v4(),
id,
prefix,
name,
location,
Expand Down
3 changes: 2 additions & 1 deletion crates/nexus/src/http/catalog/router.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,13 @@ use axum::Router;

use crate::http::catalog::handlers::{
commit_table, create_namespace, create_table, delete_namespace, delete_table, get_config,
get_namespace, get_table, list_namespaces,
get_namespace, get_table, list_namespaces, list_tables,
};

pub fn create_router() -> Router<AppState> {
let table_router: Router<AppState> = Router::new()
.route("/", post(create_table))
.route("/", get(list_tables))
.route("/:table", get(get_table))
.route("/:table", delete(delete_table))
.route("/:table", post(commit_table));
Expand Down
49 changes: 41 additions & 8 deletions tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,13 @@
S3_PATH_STYLE_ACCESS = os.environ.get("S3_PATH_STYLE_ACCESS")


@dataclasses.dataclass
class Namespace:
name: [str]
properties: dict
warehouse_id: str


@dataclasses.dataclass
class Server:
catalog_url: str
Expand All @@ -39,12 +46,24 @@ def create_storage_profile(self, **payload) -> dict:
assert response.ok, response.text
return response.json()

def delete_storage_profile(self, storage_profile: dict) -> None:
sid = storage_profile.get("id", None)
assert sid
storage_profile_url = urllib.parse.urljoin(
self.management_url, f"v1/storage-profile/{sid}"
)
response = requests.delete(
storage_profile_url,
json=storage_profile,
)
assert response.ok, response.text

def create_warehouse(
self,
**payload,
) -> uuid.UUID:
"""Create a warehouse in this server"""
warehouse_url = self.warehouse_url
warehouse_url = urllib.parse.urljoin(self.management_url, "v1/warehouse")
payload = {k.replace("_", "-"): v for k, v in payload.items()}
response = requests.post(
warehouse_url,
Expand All @@ -53,9 +72,15 @@ def create_warehouse(
assert response.ok, response.text
return response.json()

@property
def warehouse_url(self) -> str:
return urllib.parse.urljoin(self.management_url, "v1/warehouse")
def delete_warehouse(self, warehouse: dict) -> None:
wid = warehouse.get("id", None)
assert wid
warehouse_url = urllib.parse.urljoin(self.management_url, f"v1/warehouse/{wid}")
response = requests.delete(
warehouse_url,
json=warehouse,
)
assert response.ok, response.text


@pytest.fixture(scope="session")
Expand All @@ -73,7 +98,7 @@ def server() -> Server:

@pytest.fixture(scope="session")
def storage_profile(server: Server) -> dict:
return server.create_storage_profile(
data = server.create_storage_profile(
type="aws",
region=S3_REGION,
bucket=S3_BUCKET,
Expand All @@ -86,6 +111,9 @@ def storage_profile(server: Server) -> dict:
endpoint=S3_ENDPOINT,
)

yield data
server.delete_storage_profile(data)


@pytest.fixture(scope="session")
def warehouse(server: Server, storage_profile) -> dict:
Expand All @@ -96,14 +124,19 @@ def warehouse(server: Server, storage_profile) -> dict:
prefix=preix,
storage_profile_id=storage_profile.get("id", None),
)
return wh
yield wh
server.delete_warehouse(wh)


@pytest.fixture(scope="session")
def namespace(catalog) -> dict:
namespace = "test-namespace"
namespace = ("test-namespace",)
catalog.create_namespace(namespace)
return catalog.get_namespace(namespace)
return Namespace(
name=namespace,
properties={},
warehouse_id=catalog.properties.get("warehouse", None),
)


@pytest.fixture(scope="session")
Expand Down
33 changes: 11 additions & 22 deletions tests/test_catalog.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,32 +6,28 @@
import pyiceberg.io as io


def test_create_namespace(warehouse):
catalog = conftest.catalog(warehouse)
def test_create_namespace(catalog):
namespace = ("test_create_namespace",)
catalog.create_namespace(namespace)
assert namespace in catalog.list_namespaces()


def test_list_namespaces(warehouse):
catalog = conftest.catalog(warehouse)
def test_list_namespaces(catalog):
catalog.create_namespace(("test_list_namespaces_1",))
catalog.create_namespace(("test_list_namespaces_2"))
namespaces = catalog.list_namespaces()
assert ("test_list_namespaces_1",) in namespaces
assert ("test_list_namespaces_2",) in namespaces


def test_default_location_for_namespace_is_set(warehouse):
catalog = conftest.catalog(warehouse)
def test_default_location_for_namespace_is_set(catalog):
namespace = ("test_default_location_for_namespace",)
catalog.create_namespace(namespace)
loaded_properties = catalog.load_namespace_properties(namespace)
assert "location" in loaded_properties


def test_namespace_properties(warehouse):
catalog = conftest.catalog(warehouse)
def test_namespace_properties(catalog):
namespace = ("test_namespace_properties",)
properties = {"key-1": "value-1", "key2": "value2"}
catalog.create_namespace(namespace, properties=properties)
Expand All @@ -40,17 +36,15 @@ def test_namespace_properties(warehouse):
assert loaded_properties[key] == value


def test_drop_namespace(warehouse):
catalog = conftest.catalog(warehouse)
def test_drop_namespace(catalog):
namespace = ("test_drop_namespace",)
catalog.create_namespace(namespace)
assert namespace in catalog.list_namespaces()
catalog.drop_namespace(namespace)
assert namespace not in catalog.list_namespaces()


def test_create_table(warehouse):
catalog = conftest.catalog(warehouse)
def test_create_table(catalog):
namespace = ("test_create_table",)
table_name = "my_table"
schema = pa.schema(
Expand All @@ -71,8 +65,7 @@ def test_create_table(warehouse):
assert len(loaded_table.schema().fields) == 3


def test_drop_table(namespace):
catalog = conftest.catalog(namespace.warehouse)
def test_drop_table(catalog, namespace):
table_name = "my_table"
schema = pa.schema(
[
Expand All @@ -89,8 +82,7 @@ def test_drop_table(namespace):
assert "NoSuchTableError" in str(e)


def test_drop_purge_table(namespace, storage_config):
catalog = conftest.catalog(namespace.warehouse)
def test_drop_purge_table(catalog, namespace, storage_config):
table_name = "my_table"
schema = pa.schema(
[
Expand Down Expand Up @@ -133,8 +125,7 @@ def test_drop_purge_table(namespace, storage_config):
assert "NoSuchTableError" in str(e)


def test_table_properties(namespace):
catalog = conftest.catalog(namespace.warehouse)
def test_table_properties(catalog, namespace):
table_name = "my_table"
schema = pa.schema(
[
Expand All @@ -151,8 +142,7 @@ def test_table_properties(namespace):
assert table.properties == properties


def test_list_tables(namespace):
catalog = conftest.catalog(namespace.warehouse)
def test_list_tables(catalog, namespace):
assert len(catalog.list_tables(namespace.name)) == 0
table_name_1 = "my_table_1"
table_name_2 = "my_table_2"
Expand All @@ -171,8 +161,7 @@ def test_list_tables(namespace):
assert (*namespace.name, table_name_2) in tables


def test_write_read(namespace):
catalog = conftest.catalog(namespace.warehouse)
def test_write_read(catalog, namespace):
table_name = "my_table"
schema = pa.schema(
[
Expand Down