From ffeca58cbdd622aa1a65c24726a999b68ce63f26 Mon Sep 17 00:00:00 2001 From: osipovartem Date: Fri, 31 Jan 2025 15:41:09 +0300 Subject: [PATCH 1/8] DFParser should skip unsupported COPY INTO --- datafusion/sql/src/parser.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/datafusion/sql/src/parser.rs b/datafusion/sql/src/parser.rs index f185d65fa194..e3a137939d18 100644 --- a/datafusion/sql/src/parser.rs +++ b/datafusion/sql/src/parser.rs @@ -354,6 +354,14 @@ impl<'a> DFParser<'a> { self.parse_create() } Keyword::COPY => { + if let Token::Word(w) = self.parser.peek_nth_token(1).token { + // use native parser for COPY INTO + if w.keyword == Keyword::INTO { + return Ok(Statement::Statement(Box::from( + self.parser.parse_statement()?, + ))); + } + } self.parser.next_token(); // COPY self.parse_copy() } From f615c1ae49196c1498f4d805288ce9d2f7f2e1f1 Mon Sep 17 00:00:00 2001 From: osipovartem Date: Fri, 31 Jan 2025 16:13:39 +0300 Subject: [PATCH 2/8] Add tests --- datafusion/sql/src/parser.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/datafusion/sql/src/parser.rs b/datafusion/sql/src/parser.rs index e3a137939d18..7f5134406114 100644 --- a/datafusion/sql/src/parser.rs +++ b/datafusion/sql/src/parser.rs @@ -885,6 +885,7 @@ mod tests { use super::*; use sqlparser::ast::Expr::Identifier; use sqlparser::ast::{BinaryOperator, DataType, Expr, Ident}; + use sqlparser::dialect::{dialect_from_str, SnowflakeDialect}; use sqlparser::tokenizer::Span; fn expect_parse_ok(sql: &str, expected: Statement) -> Result<(), ParserError> { @@ -1411,6 +1412,23 @@ mod tests { Ok(()) } + + #[test] + fn skip_copy_into_snowflake() -> Result<(), ParserError> { + let sql = "COPY INTO foo FROM @~/staged FILE_FORMAT = (FORMAT_NAME = 'mycsv');"; + let dialect = Box::new(SnowflakeDialect); + let statements = DFParser::parse_sql_with_dialect(sql, dialect.as_ref())?; + + assert_eq!( statements.len(), 1, "Expected to parse exactly one statement"); + if let Statement::CopyTo(_) = &statements[0] { + panic!( + "Expected non COPY TO statement, but was successful: {statements:?}" + ); + } + Ok(()) + } + + #[test] fn explain_copy_to_table_to_table() -> Result<(), ParserError> { let cases = vec![ From efbdd742b41344e731397a73aa640667042c4425 Mon Sep 17 00:00:00 2001 From: osipovartem Date: Fri, 31 Jan 2025 16:14:18 +0300 Subject: [PATCH 3/8] Add tests --- datafusion/sql/src/parser.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/datafusion/sql/src/parser.rs b/datafusion/sql/src/parser.rs index 7f5134406114..5da0cf764f90 100644 --- a/datafusion/sql/src/parser.rs +++ b/datafusion/sql/src/parser.rs @@ -1411,8 +1411,7 @@ mod tests { assert_eq!(verified_stmt(sql), expected); Ok(()) } - - + #[test] fn skip_copy_into_snowflake() -> Result<(), ParserError> { let sql = "COPY INTO foo FROM @~/staged FILE_FORMAT = (FORMAT_NAME = 'mycsv');"; @@ -1428,7 +1427,6 @@ mod tests { Ok(()) } - #[test] fn explain_copy_to_table_to_table() -> Result<(), ParserError> { let cases = vec![ From 154a0ca2929809be4bf447a792a660cc23e068ec Mon Sep 17 00:00:00 2001 From: osipovartem Date: Fri, 31 Jan 2025 18:31:06 +0300 Subject: [PATCH 4/8] Add tests --- datafusion/sql/src/parser.rs | 36 +++++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/datafusion/sql/src/parser.rs b/datafusion/sql/src/parser.rs index 5da0cf764f90..68d76860b6f2 100644 --- a/datafusion/sql/src/parser.rs +++ b/datafusion/sql/src/parser.rs @@ -557,7 +557,8 @@ impl<'a> DFParser<'a> { self.parser.expect_keyword(Keyword::EXTERNAL)?; self.parse_create_external_table(true) } else { - Ok(Statement::Statement(Box::from(self.parser.parse_create()?))) + self.parser.prev_token(); + Ok(Statement::Statement(Box::from(self.parser.parse_statement()?))) } } @@ -930,6 +931,22 @@ mod tests { } } + + #[test] + fn skip_create_stage_snowflake() -> Result<(), ParserError> { + let sql = "CREATE OR REPLACE STAGE stage URL='s3://data.csv' FILE_FORMAT=(TYPE=csv)"; + let dialect = Box::new(SnowflakeDialect); + let statements = DFParser::parse_sql_with_dialect(sql, dialect.as_ref())?; + + assert_eq!(statements.len(), 1, "Expected to parse exactly one statement"); + match &statements[0] { + Statement::Statement(stmt) => { + assert_eq!(stmt.to_string(), sql); + } + _ => panic!("Expected statement type"), + } + Ok(()) + } #[test] fn create_external_table() -> Result<(), ParserError> { // positive case @@ -1411,18 +1428,19 @@ mod tests { assert_eq!(verified_stmt(sql), expected); Ok(()) } - + #[test] fn skip_copy_into_snowflake() -> Result<(), ParserError> { - let sql = "COPY INTO foo FROM @~/staged FILE_FORMAT = (FORMAT_NAME = 'mycsv');"; + let sql = "COPY INTO foo FROM @~/staged FILE_FORMAT=(FORMAT_NAME='mycsv')"; let dialect = Box::new(SnowflakeDialect); let statements = DFParser::parse_sql_with_dialect(sql, dialect.as_ref())?; - - assert_eq!( statements.len(), 1, "Expected to parse exactly one statement"); - if let Statement::CopyTo(_) = &statements[0] { - panic!( - "Expected non COPY TO statement, but was successful: {statements:?}" - ); + + assert_eq!(statements.len(), 1, "Expected to parse exactly one statement"); + match &statements[0] { + Statement::Statement(stmt) => { + assert_eq!(stmt.to_string(), sql); + } + _ => panic!("Expected statement type"), } Ok(()) } From a8d795eb2f83857b041d9952cf2b4fa078052a48 Mon Sep 17 00:00:00 2001 From: osipovartem Date: Fri, 31 Jan 2025 18:33:10 +0300 Subject: [PATCH 5/8] Add tests --- datafusion/sql/src/parser.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/datafusion/sql/src/parser.rs b/datafusion/sql/src/parser.rs index 68d76860b6f2..afbc81f7be1d 100644 --- a/datafusion/sql/src/parser.rs +++ b/datafusion/sql/src/parser.rs @@ -557,6 +557,7 @@ impl<'a> DFParser<'a> { self.parser.expect_keyword(Keyword::EXTERNAL)?; self.parse_create_external_table(true) } else { + // Push back CREATE self.parser.prev_token(); Ok(Statement::Statement(Box::from(self.parser.parse_statement()?))) } From 4b74fe9a013686d0815cf976ec64adb37cd8bffe Mon Sep 17 00:00:00 2001 From: osipovartem Date: Mon, 3 Feb 2025 11:27:28 +0300 Subject: [PATCH 6/8] Fmt --- datafusion/sql/src/parser.rs | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/datafusion/sql/src/parser.rs b/datafusion/sql/src/parser.rs index afbc81f7be1d..b69d76cd6905 100644 --- a/datafusion/sql/src/parser.rs +++ b/datafusion/sql/src/parser.rs @@ -559,7 +559,9 @@ impl<'a> DFParser<'a> { } else { // Push back CREATE self.parser.prev_token(); - Ok(Statement::Statement(Box::from(self.parser.parse_statement()?))) + Ok(Statement::Statement(Box::from( + self.parser.parse_statement()?, + ))) } } @@ -932,14 +934,18 @@ mod tests { } } - #[test] fn skip_create_stage_snowflake() -> Result<(), ParserError> { - let sql = "CREATE OR REPLACE STAGE stage URL='s3://data.csv' FILE_FORMAT=(TYPE=csv)"; + let sql = + "CREATE OR REPLACE STAGE stage URL='s3://data.csv' FILE_FORMAT=(TYPE=csv)"; let dialect = Box::new(SnowflakeDialect); let statements = DFParser::parse_sql_with_dialect(sql, dialect.as_ref())?; - assert_eq!(statements.len(), 1, "Expected to parse exactly one statement"); + assert_eq!( + statements.len(), + 1, + "Expected to parse exactly one statement" + ); match &statements[0] { Statement::Statement(stmt) => { assert_eq!(stmt.to_string(), sql); @@ -1436,7 +1442,11 @@ mod tests { let dialect = Box::new(SnowflakeDialect); let statements = DFParser::parse_sql_with_dialect(sql, dialect.as_ref())?; - assert_eq!(statements.len(), 1, "Expected to parse exactly one statement"); + assert_eq!( + statements.len(), + 1, + "Expected to parse exactly one statement" + ); match &statements[0] { Statement::Statement(stmt) => { assert_eq!(stmt.to_string(), sql); From 07bae6ccf19d391fceb6c6dd9a57bcdf04460d42 Mon Sep 17 00:00:00 2001 From: osipovartem Date: Mon, 3 Feb 2025 11:30:47 +0300 Subject: [PATCH 7/8] Fmt --- datafusion/sql/src/parser.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/sql/src/parser.rs b/datafusion/sql/src/parser.rs index b69d76cd6905..2598b203f61c 100644 --- a/datafusion/sql/src/parser.rs +++ b/datafusion/sql/src/parser.rs @@ -889,7 +889,7 @@ mod tests { use super::*; use sqlparser::ast::Expr::Identifier; use sqlparser::ast::{BinaryOperator, DataType, Expr, Ident}; - use sqlparser::dialect::{dialect_from_str, SnowflakeDialect}; + use sqlparser::dialect::{SnowflakeDialect}; use sqlparser::tokenizer::Span; fn expect_parse_ok(sql: &str, expected: Statement) -> Result<(), ParserError> { From 4b29d04b384866e76f946df214ded9421348c500 Mon Sep 17 00:00:00 2001 From: osipovartem Date: Mon, 3 Feb 2025 11:57:30 +0300 Subject: [PATCH 8/8] Fmt --- datafusion/sql/src/parser.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/sql/src/parser.rs b/datafusion/sql/src/parser.rs index 2598b203f61c..cd5358c819e5 100644 --- a/datafusion/sql/src/parser.rs +++ b/datafusion/sql/src/parser.rs @@ -889,7 +889,7 @@ mod tests { use super::*; use sqlparser::ast::Expr::Identifier; use sqlparser::ast::{BinaryOperator, DataType, Expr, Ident}; - use sqlparser::dialect::{SnowflakeDialect}; + use sqlparser::dialect::SnowflakeDialect; use sqlparser::tokenizer::Span; fn expect_parse_ok(sql: &str, expected: Statement) -> Result<(), ParserError> {