diff --git a/Cargo.lock b/Cargo.lock index 59c342deb..e293a0663 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2711,6 +2711,7 @@ dependencies = [ "once_cell", "openid", "parquet", + "path-clean", "prometheus", "rand", "regex", @@ -2760,6 +2761,12 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d01a5bd0424d00070b0098dd17ebca6f961a959dead1dbcbbbc1d1cd8d3deeba" +[[package]] +name = "path-clean" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17359afc20d7ab31fdb42bb844c8b3bb1dabd7dcf7e68428492da7f16966fcef" + [[package]] name = "path-matchers" version = "1.0.2" diff --git a/server/Cargo.toml b/server/Cargo.toml index 9c151e6d1..57afe65c5 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -102,6 +102,7 @@ url = "2.4.0" http-auth-basic = "0.3.3" serde_repr = "0.1.17" hashlru = { version = "0.11.0", features = ["serde"] } +path-clean = "1.0.1" [build-dependencies] cargo_toml = "0.15" diff --git a/server/src/main.rs b/server/src/main.rs index de98cff83..954ed6ddd 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -55,9 +55,9 @@ use crate::localcache::LocalCacheManager; async fn main() -> anyhow::Result<()> { env_logger::init(); let storage = CONFIG.storage().get_object_store(); - CONFIG.validate_staging()?; migration::run_metadata_migration(&CONFIG).await?; let metadata = storage::resolve_parseable_metadata().await?; + CONFIG.validate_staging()?; banner::print(&CONFIG, &metadata).await; rbac::map::init(&metadata); metadata.set_global(); diff --git a/server/src/option.rs b/server/src/option.rs index 1104f9674..8b3983170 100644 --- a/server/src/option.rs +++ b/server/src/option.rs @@ -21,7 +21,8 @@ use clap::{command, value_parser, Arg, ArgGroup, Args, Command, FromArgMatches}; use once_cell::sync::Lazy; use parquet::basic::{BrotliLevel, GzipLevel, ZstdLevel}; -use std::path::{Path, PathBuf}; +use std::env; +use std::path::PathBuf; use std::sync::Arc; use url::Url; @@ -43,7 +44,6 @@ pub struct Config { impl Config { fn new() -> Self { let cli = parseable_cli_command().get_matches(); - match cli.subcommand() { Some(("local-store", m)) => { let server = match Server::from_arg_matches(m) { @@ -51,7 +51,7 @@ impl Config { Err(err) => err.exit(), }; let storage = match FSConfig::from_arg_matches(m) { - Ok(server) => server, + Ok(storage) => storage, Err(err) => err.exit(), }; @@ -85,7 +85,7 @@ impl Config { Err(err) => err.exit(), }; let storage = match S3Config::from_arg_matches(m) { - Ok(server) => server, + Ok(storage) => storage, Err(err) => err.exit(), }; @@ -108,7 +108,7 @@ impl Config { self.storage.clone() } - pub fn staging_dir(&self) -> &Path { + pub fn staging_dir(&self) -> &PathBuf { &self.parseable.local_staging_path } @@ -611,12 +611,14 @@ impl From for parquet::basic::Compression { pub mod validation { use std::{ - fs::{canonicalize, create_dir_all}, + env, io, net::ToSocketAddrs, - path::PathBuf, + path::{Path, PathBuf}, str::FromStr, }; + use path_clean::PathClean; + use crate::option::MIN_CACHE_SIZE_BYTES; use crate::storage::LOCAL_SYNC_INTERVAL; use human_size::{multiples, SpecificSize}; @@ -634,16 +636,22 @@ pub mod validation { Ok(path) } + pub fn absolute_path(path: impl AsRef) -> io::Result { + let path = path.as_ref(); + + let absolute_path = if path.is_absolute() { + path.to_path_buf() + } else { + env::current_dir()?.join(path) + } + .clean(); + + Ok(absolute_path) + } pub fn canonicalize_path(s: &str) -> Result { let path = PathBuf::from(s); - - create_dir_all(&path) - .map_err(|err| err.to_string()) - .and_then(|_| { - canonicalize(&path) - .map_err(|_| "Cannot use the path provided as an absolute path".to_string()) - }) + Ok(absolute_path(path).unwrap()) } pub fn socket_addr(s: &str) -> Result { diff --git a/server/src/storage/localfs.rs b/server/src/storage/localfs.rs index e1dc31ee5..df88499a9 100644 --- a/server/src/storage/localfs.rs +++ b/server/src/storage/localfs.rs @@ -187,7 +187,7 @@ impl ObjectStorage for LocalFS { }; let to_path = self.root.join(key); if let Some(path) = to_path.parent() { - fs::create_dir_all(path).await? + fs::create_dir_all(path).await?; } let _ = fs_extra::file::copy(path, to_path, &op)?; Ok(()) diff --git a/server/src/storage/store_metadata.rs b/server/src/storage/store_metadata.rs index b0ef21d6d..0e9ad955f 100644 --- a/server/src/storage/store_metadata.rs +++ b/server/src/storage/store_metadata.rs @@ -66,7 +66,7 @@ impl StorageMetadata { Self { version: "v3".to_string(), mode: CONFIG.storage_name.to_owned(), - staging: CONFIG.staging_dir().canonicalize().unwrap(), + staging: CONFIG.staging_dir().to_path_buf(), storage: CONFIG.storage().get_endpoint(), deployment_id: uid::gen(), users: Vec::new(), @@ -104,7 +104,7 @@ pub async fn resolve_parseable_metadata() -> Result EnvChange::NewStaging(remote), @@ -116,16 +116,14 @@ pub async fn resolve_parseable_metadata() -> Result { // overwrite staging anyways so that it matches remote in case of any divergence overwrite_staging = true; Ok(metadata) - } - EnvChange::DeploymentMismatch => Err(MISMATCH), + }, EnvChange::NewRemote => { - Err("Could not start the server because metadata not found in storage") + Err("Could not start the server because staging directory indicates stale data from previous deployment, please choose an empty staging directory and restart the server") } EnvChange::NewStaging(mut metadata) => { create_dir_all(CONFIG.staging_dir())?; @@ -171,9 +169,8 @@ pub async fn resolve_parseable_metadata() -> Result