From 4feac2bfe0662c3766eb93e963944ed03bb08433 Mon Sep 17 00:00:00 2001 From: panbingkun Date: Sun, 3 Jul 2022 15:23:31 +0800 Subject: [PATCH 1/6] [SPARK-39663][SQL][TESTS] Add UT for MysqlDialect listIndexes method --- .../sql/jdbc/v2/MySQLIntegrationSuite.scala | 2 ++ .../apache/spark/sql/jdbc/v2/V2JDBCTest.scala | 26 ++++++++++++++++--- .../apache/spark/sql/jdbc/MySQLDialect.scala | 2 +- 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/MySQLIntegrationSuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/MySQLIntegrationSuite.scala index 97f521a378eb7..6e76b74c7d830 100644 --- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/MySQLIntegrationSuite.scala +++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/MySQLIntegrationSuite.scala @@ -119,6 +119,8 @@ class MySQLIntegrationSuite extends DockerJDBCIntegrationV2Suite with V2JDBCTest override def supportsIndex: Boolean = true + override def supportListIndexes: Boolean = true + override def indexOptions: String = "KEY_BLOCK_SIZE=10" testVarPop() diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/V2JDBCTest.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/V2JDBCTest.scala index 5f0033490d5c5..4354aa1782192 100644 --- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/V2JDBCTest.scala +++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/V2JDBCTest.scala @@ -197,6 +197,8 @@ private[v2] trait V2JDBCTest extends SharedSparkSession with DockerIntegrationFu def supportsIndex: Boolean = false + def supportListIndexes: Boolean = false + def indexOptions: String = "" test("SPARK-36895: Test INDEX Using SQL") { @@ -219,11 +221,19 @@ private[v2] trait V2JDBCTest extends SharedSparkSession with DockerIntegrationFu s" The supported Index Types are:")) sql(s"CREATE index i1 ON $catalogName.new_table USING BTREE (col1)") + assert(jdbcTable.indexExists("i1") == true) + if (supportListIndexes) { + val indexes = jdbcTable.listIndexes() + assert(indexes.size == 1) + assert(indexes(0).indexName() == "i1") + } + sql(s"CREATE index i2 ON $catalogName.new_table (col2, col3, col5)" + s" OPTIONS ($indexOptions)") - - assert(jdbcTable.indexExists("i1") == true) assert(jdbcTable.indexExists("i2") == true) + if (supportListIndexes) { + assert(jdbcTable.listIndexes().size == 2) + } // This should pass without exception sql(s"CREATE index IF NOT EXISTS i1 ON $catalogName.new_table (col1)") @@ -234,10 +244,18 @@ private[v2] trait V2JDBCTest extends SharedSparkSession with DockerIntegrationFu assert(m.contains("Failed to create index i1 in new_table")) sql(s"DROP index i1 ON $catalogName.new_table") - sql(s"DROP index i2 ON $catalogName.new_table") - assert(jdbcTable.indexExists("i1") == false) + if (supportListIndexes) { + val indexes = jdbcTable.listIndexes() + assert(indexes.size == 1) + assert(indexes(0).indexName() == "i2") + } + + sql(s"DROP index i2 ON $catalogName.new_table") assert(jdbcTable.indexExists("i2") == false) + if (supportListIndexes) { + assert(jdbcTable.listIndexes().size == 0) + } // This should pass without exception sql(s"DROP index IF EXISTS i1 ON $catalogName.new_table") diff --git a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/MySQLDialect.scala b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/MySQLDialect.scala index 24f9bac74f86d..c4cb5369af9e7 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/MySQLDialect.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/MySQLDialect.scala @@ -206,7 +206,7 @@ private case object MySQLDialect extends JdbcDialect with SQLConfHelper { val indexName = rs.getString("key_name") val colName = rs.getString("column_name") val indexType = rs.getString("index_type") - val indexComment = rs.getString("Index_comment") + val indexComment = rs.getString("index_comment") if (indexMap.contains(indexName)) { val index = indexMap.get(indexName).get val newIndex = new TableIndex(indexName, indexType, From a837e0d898f72488d1e81f3a30d9ad46afa2b79c Mon Sep 17 00:00:00 2001 From: panbingkun Date: Tue, 5 Jul 2022 10:13:20 +0800 Subject: [PATCH 2/6] [SPARK-39663][SQL][TESTS] Add UT for MysqlDialect listIndexes method --- .../scala/org/apache/spark/sql/jdbc/v2/V2JDBCTest.scala | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/V2JDBCTest.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/V2JDBCTest.scala index 4354aa1782192..33be3009ab4c4 100644 --- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/V2JDBCTest.scala +++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/V2JDBCTest.scala @@ -221,7 +221,7 @@ private[v2] trait V2JDBCTest extends SharedSparkSession with DockerIntegrationFu s" The supported Index Types are:")) sql(s"CREATE index i1 ON $catalogName.new_table USING BTREE (col1)") - assert(jdbcTable.indexExists("i1") == true) + assert(jdbcTable.indexExists("i1")) if (supportListIndexes) { val indexes = jdbcTable.listIndexes() assert(indexes.size == 1) @@ -230,9 +230,11 @@ private[v2] trait V2JDBCTest extends SharedSparkSession with DockerIntegrationFu sql(s"CREATE index i2 ON $catalogName.new_table (col2, col3, col5)" + s" OPTIONS ($indexOptions)") - assert(jdbcTable.indexExists("i2") == true) + assert(jdbcTable.indexExists("i2")) if (supportListIndexes) { - assert(jdbcTable.listIndexes().size == 2) + val indexes = jdbcTable.listIndexes() + assert(indexes.size == 2) + assert(indexes.map(_ => _.indexName()).sortWith(_ < _) === Array("i1", "i2")) } // This should pass without exception From 1b88fb90ed5fe0e474935a762fae4ad3fcf03b38 Mon Sep 17 00:00:00 2001 From: panbingkun Date: Tue, 5 Jul 2022 10:33:16 +0800 Subject: [PATCH 3/6] [SPARK-39663][SQL][TESTS] Add UT for MysqlDialect listIndexes method --- .../test/scala/org/apache/spark/sql/jdbc/v2/V2JDBCTest.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/V2JDBCTest.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/V2JDBCTest.scala index 33be3009ab4c4..84186851a794e 100644 --- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/V2JDBCTest.scala +++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/V2JDBCTest.scala @@ -234,7 +234,7 @@ private[v2] trait V2JDBCTest extends SharedSparkSession with DockerIntegrationFu if (supportListIndexes) { val indexes = jdbcTable.listIndexes() assert(indexes.size == 2) - assert(indexes.map(_ => _.indexName()).sortWith(_ < _) === Array("i1", "i2")) + assert(indexes.map(index => index.indexName()).sortWith(_ < _) === Array("i1", "i2")) } // This should pass without exception From ea651fe3952eed7543e9d239baf1d5dab19fc768 Mon Sep 17 00:00:00 2001 From: panbingkun Date: Tue, 5 Jul 2022 15:25:02 +0800 Subject: [PATCH 4/6] [SPARK-39663][SQL][TESTS] Add UT for MysqlDialect listIndexes method --- .../test/scala/org/apache/spark/sql/jdbc/v2/V2JDBCTest.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/V2JDBCTest.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/V2JDBCTest.scala index 84186851a794e..b393e08ea1b4a 100644 --- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/V2JDBCTest.scala +++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/V2JDBCTest.scala @@ -256,7 +256,7 @@ private[v2] trait V2JDBCTest extends SharedSparkSession with DockerIntegrationFu sql(s"DROP index i2 ON $catalogName.new_table") assert(jdbcTable.indexExists("i2") == false) if (supportListIndexes) { - assert(jdbcTable.listIndexes().size == 0) + assert(jdbcTable.listIndexes().isEmpty) } // This should pass without exception From 08a04b2b05c21acf4749e51e3d54d1417d18cb9c Mon Sep 17 00:00:00 2001 From: panbingkun Date: Tue, 5 Jul 2022 15:28:17 +0800 Subject: [PATCH 5/6] [SPARK-39663][SQL][TESTS] Add UT for MysqlDialect listIndexes method --- .../test/scala/org/apache/spark/sql/jdbc/v2/V2JDBCTest.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/V2JDBCTest.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/V2JDBCTest.scala index b393e08ea1b4a..24fb71a00f8c9 100644 --- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/V2JDBCTest.scala +++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/V2JDBCTest.scala @@ -234,7 +234,7 @@ private[v2] trait V2JDBCTest extends SharedSparkSession with DockerIntegrationFu if (supportListIndexes) { val indexes = jdbcTable.listIndexes() assert(indexes.size == 2) - assert(indexes.map(index => index.indexName()).sortWith(_ < _) === Array("i1", "i2")) + assert(indexes.map(index => index.indexName()).sorted === Array("i1", "i2")) } // This should pass without exception From c3ce2ea5c3f6ad2951074b86ac446b38ceee386b Mon Sep 17 00:00:00 2001 From: panbingkun Date: Wed, 6 Jul 2022 08:27:11 +0800 Subject: [PATCH 6/6] [SPARK-39663][SQL][TESTS] Add UT for MysqlDialect listIndexes method --- .../scala/org/apache/spark/sql/jdbc/v2/V2JDBCTest.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/V2JDBCTest.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/V2JDBCTest.scala index 24fb71a00f8c9..0f85bd534c393 100644 --- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/V2JDBCTest.scala +++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/V2JDBCTest.scala @@ -225,7 +225,7 @@ private[v2] trait V2JDBCTest extends SharedSparkSession with DockerIntegrationFu if (supportListIndexes) { val indexes = jdbcTable.listIndexes() assert(indexes.size == 1) - assert(indexes(0).indexName() == "i1") + assert(indexes.head.indexName() == "i1") } sql(s"CREATE index i2 ON $catalogName.new_table (col2, col3, col5)" + @@ -234,7 +234,7 @@ private[v2] trait V2JDBCTest extends SharedSparkSession with DockerIntegrationFu if (supportListIndexes) { val indexes = jdbcTable.listIndexes() assert(indexes.size == 2) - assert(indexes.map(index => index.indexName()).sorted === Array("i1", "i2")) + assert(indexes.map(_.indexName()).sorted === Array("i1", "i2")) } // This should pass without exception @@ -250,7 +250,7 @@ private[v2] trait V2JDBCTest extends SharedSparkSession with DockerIntegrationFu if (supportListIndexes) { val indexes = jdbcTable.listIndexes() assert(indexes.size == 1) - assert(indexes(0).indexName() == "i2") + assert(indexes.head.indexName() == "i2") } sql(s"DROP index i2 ON $catalogName.new_table")