Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ venv
__pycache__

.env
metastore.yaml
alloc.log

slatedb-prefix/
Expand All @@ -42,6 +43,7 @@ tests/benchmark/*.json
test/errors.log
test/test_statistics.csv
snowplow
crates/api-snowflake-rest/traces.log

*.manifest
*.sst
Expand Down
2 changes: 2 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 4 additions & 1 deletion crates/api-iceberg-rest/src/error.rs
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,10 @@ impl IntoResponse for Error {
| core_metastore::Error::Serde { .. }
| core_metastore::Error::TableMetadataBuilder { .. }
| core_metastore::Error::TableObjectStoreNotFound { .. }
| core_metastore::Error::UrlParse { .. } => http::StatusCode::INTERNAL_SERVER_ERROR,
| core_metastore::Error::UrlParse { .. }
| core_metastore::Error::NotYetImplemented { .. } => {
http::StatusCode::INTERNAL_SERVER_ERROR
}
};

// Record the result as part of the current span.
Expand Down
6 changes: 6 additions & 0 deletions crates/core-executor/src/error.rs
Original file line number Diff line number Diff line change
Expand Up @@ -449,6 +449,12 @@ pub enum Error {
#[snafu(implicit)]
location: Location,
},
#[snafu(display("Unimplemented functionality: '{name}'"))]
UnimplementedFunctionality {
name: String,
#[snafu(implicit)]
location: Location,
},
#[snafu(display("SQL parser error: {error}"))]
SqlParser {
#[snafu(source)]
Expand Down
2 changes: 2 additions & 0 deletions crates/core-metastore/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ iceberg-rust-spec = { workspace = true }
object_store = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
serde_yaml = { workspace = true }
slatedb = { workspace = true }
snafu = { workspace = true }
strum = { workspace = true }
Expand All @@ -30,6 +31,7 @@ utoipa = { workspace = true }
uuid = { workspace = true }
validator = { workspace = true }
regex = { workspace = true }
parking_lot = "0.12.5"

[dev-dependencies]
insta = { workspace = true }
Expand Down
99 changes: 99 additions & 0 deletions crates/core-metastore/src/basic/config.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
use crate::models::{Database, S3TablesVolume, Volume, VolumeIdent, VolumeType};
use serde::{Deserialize, Serialize};

/// Configuration for the basic metastore loaded from YAML
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct BasicMetastoreConfig {
pub volumes: Vec<VolumeConfig>,
}

/// Volume configuration with optional databases
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct VolumeConfig {
pub name: String,
#[serde(flatten)]
pub volume_type: S3TablesVolumeType,
#[serde(default)]
pub databases: Vec<DatabaseConfig>,
}

/// Only `S3Tables` volume type is supported in basic metastore
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum S3TablesVolumeType {
S3Tables(S3TablesVolume),
}

/// Database configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DatabaseConfig {
pub name: String,
}

impl BasicMetastoreConfig {
/// Load configuration from YAML file
pub fn from_yaml_file(path: &str) -> Result<Self, String> {
let content = std::fs::read_to_string(path)
.map_err(|e| format!("Failed to read config file: {e}"))?;
Self::from_yaml_str(&content)
}

/// Load configuration from YAML string
pub fn from_yaml_str(yaml: &str) -> Result<Self, String> {
serde_yaml::from_str(yaml).map_err(|e| format!("Failed to parse config YAML: {e}"))
}
}

impl VolumeConfig {
/// Convert to metastore Volume
#[must_use]
pub fn to_volume(&self) -> Volume {
Volume {
ident: self.name.clone(),
volume: match &self.volume_type {
S3TablesVolumeType::S3Tables(s3tables) => VolumeType::S3Tables(s3tables.clone()),
},
}
}
}

impl DatabaseConfig {
/// Convert to metastore Database
#[must_use]
pub fn to_database(&self, volume: &VolumeIdent) -> Database {
Database {
ident: self.name.clone(),
volume: volume.clone(),
properties: None,
}
}
}

#[cfg(test)]
mod tests {
use super::*;

#[test]
fn test_parse_config() {
let yaml = r#"
volumes:
- name: my_volume
type: s3_tables
arn: "arn:aws:s3tables:us-east-1:123456789012:bucket/my-bucket"
endpoint: "https://s3tables.us-east-1.amazonaws.com"
credentials:
credential_type: access_key
aws-access-key-id: "AKIAIOSFODNN7EXAMPLE"
aws-secret-access-key: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
databases:
- name: my_db
- name: another_db
"#;

let config = BasicMetastoreConfig::from_yaml_str(yaml).expect("Failed to parse config");
assert_eq!(config.volumes.len(), 1);
assert_eq!(config.volumes[0].name, "my_volume");
assert_eq!(config.volumes[0].databases.len(), 2);
assert_eq!(config.volumes[0].databases[0].name, "my_db");
}
}
Loading
Loading