From 0c23fbec6cbfe7d84337917610005202ae35c85d Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Thu, 4 Jan 2024 16:24:06 +0800 Subject: [PATCH 1/4] fix --- .../sql/jdbc/OracleDatabaseOnDocker.scala | 60 +++++++++++++++++++ .../sql/jdbc/OracleIntegrationSuite.scala | 14 +---- .../sql/jdbc/v2/OracleIntegrationSuite.scala | 16 +---- .../sql/jdbc/v2/OracleNamespaceSuite.scala | 16 +---- 4 files changed, 65 insertions(+), 41 deletions(-) create mode 100644 connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleDatabaseOnDocker.scala diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleDatabaseOnDocker.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleDatabaseOnDocker.scala new file mode 100644 index 000000000000..c7ba25602231 --- /dev/null +++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleDatabaseOnDocker.scala @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.jdbc + +import java.io.{File, PrintWriter} + +import com.github.dockerjava.api.model.{AccessMode, Bind, ContainerConfig, HostConfig, Volume} + +import org.apache.spark.internal.Logging +import org.apache.spark.util.Utils + +class OracleDatabaseOnDocker extends DatabaseOnDocker with Logging { + lazy override val imageName = + sys.env.getOrElse("ORACLE_DOCKER_IMAGE_NAME", "gvenzl/oracle-free:23.3") + val oracle_password = "Th1s1sThe0racle#Pass" + override val env = Map( + "ORACLE_PWD" -> oracle_password, // oracle images uses this + "ORACLE_PASSWORD" -> oracle_password // gvenzl/oracle-free uses this + ) + override val usesIpc = false + override val jdbcPort: Int = 1521 + + override def getJdbcUrl(ip: String, port: Int): String = { + s"jdbc:oracle:thin:system/$oracle_password@//$ip:$port/freepdb1" + } + + override def beforeContainerStart( + hostConfigBuilder: HostConfig, + containerConfigBuilder: ContainerConfig): Unit = { + try { + val dir = Utils.createTempDir() + val writer = new PrintWriter(new File(dir, "install.sql")) + writer.write("ALTER SESSION SET DDL_LOCK_TIMEOUT = 10") + writer.close() + val newBind = new Bind( + dir.getAbsolutePath, + new Volume("/docker-entrypoint-initdb.d"), + AccessMode.ro) + hostConfigBuilder.withBinds(hostConfigBuilder.getBinds :+ newBind: _*) + } catch { + case e: Exception => + logWarning("Failed to create install.sql file", e) + } + } +} diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala index 4e13d21864fe..0cdff79ef69f 100644 --- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala +++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala @@ -65,19 +65,7 @@ import org.apache.spark.tags.DockerTest class OracleIntegrationSuite extends DockerJDBCIntegrationSuite with SharedSparkSession { import testImplicits._ - override val db = new DatabaseOnDocker { - lazy override val imageName = - sys.env.getOrElse("ORACLE_DOCKER_IMAGE_NAME", "gvenzl/oracle-free:23.3") - val oracle_password = "Th1s1sThe0racle#Pass" - override val env = Map( - "ORACLE_PWD" -> oracle_password, // oracle images uses this - "ORACLE_PASSWORD" -> oracle_password // gvenzl/oracle-free uses this - ) - override val usesIpc = false - override val jdbcPort: Int = 1521 - override def getJdbcUrl(ip: String, port: Int): String = - s"jdbc:oracle:thin:system/$oracle_password@//$ip:$port/freepdb1" - } + override val db = new OracleDatabaseOnDocker override val connectionTimeout = timeout(7.minutes) diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/OracleIntegrationSuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/OracleIntegrationSuite.scala index 0c844219aeb5..1badc110347d 100644 --- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/OracleIntegrationSuite.scala +++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/OracleIntegrationSuite.scala @@ -26,7 +26,7 @@ import org.apache.spark.{SparkConf, SparkRuntimeException} import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.catalyst.util.CharVarcharUtils.CHAR_VARCHAR_TYPE_STRING_METADATA_KEY import org.apache.spark.sql.execution.datasources.v2.jdbc.JDBCTableCatalog -import org.apache.spark.sql.jdbc.DatabaseOnDocker +import org.apache.spark.sql.jdbc.OracleDatabaseOnDocker import org.apache.spark.sql.types._ import org.apache.spark.tags.DockerTest @@ -75,19 +75,7 @@ class OracleIntegrationSuite extends DockerJDBCIntegrationV2Suite with V2JDBCTes override val catalogName: String = "oracle" override val namespaceOpt: Option[String] = Some("SYSTEM") - override val db = new DatabaseOnDocker { - lazy override val imageName = - sys.env.getOrElse("ORACLE_DOCKER_IMAGE_NAME", "gvenzl/oracle-free:23.3") - val oracle_password = "Th1s1sThe0racle#Pass" - override val env = Map( - "ORACLE_PWD" -> oracle_password, // oracle images uses this - "ORACLE_PASSWORD" -> oracle_password // gvenzl/oracle-free uses this - ) - override val usesIpc = false - override val jdbcPort: Int = 1521 - override def getJdbcUrl(ip: String, port: Int): String = - s"jdbc:oracle:thin:system/$oracle_password@//$ip:$port/freepdb1" - } + override val db = new OracleDatabaseOnDocker override val defaultMetadata: Metadata = new MetadataBuilder() .putLong("scale", 0) diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/OracleNamespaceSuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/OracleNamespaceSuite.scala index daffb5a2d4a3..05f38102d410 100644 --- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/OracleNamespaceSuite.scala +++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/OracleNamespaceSuite.scala @@ -21,7 +21,7 @@ import java.sql.Connection import scala.jdk.CollectionConverters._ -import org.apache.spark.sql.jdbc.{DatabaseOnDocker, DockerJDBCIntegrationSuite} +import org.apache.spark.sql.jdbc.{DockerJDBCIntegrationSuite, OracleDatabaseOnDocker} import org.apache.spark.sql.util.CaseInsensitiveStringMap import org.apache.spark.tags.DockerTest @@ -57,19 +57,7 @@ class OracleNamespaceSuite extends DockerJDBCIntegrationSuite with V2JDBCNamespa override def excluded: Seq[String] = Seq("listNamespaces: basic behavior", "Drop namespace") - override val db = new DatabaseOnDocker { - lazy override val imageName = - sys.env.getOrElse("ORACLE_DOCKER_IMAGE_NAME", "gvenzl/oracle-free:23.3") - val oracle_password = "Th1s1sThe0racle#Pass" - override val env = Map( - "ORACLE_PWD" -> oracle_password, // oracle images uses this - "ORACLE_PASSWORD" -> oracle_password // gvenzl/oracle-free uses this - ) - override val usesIpc = false - override val jdbcPort: Int = 1521 - override def getJdbcUrl(ip: String, port: Int): String = - s"jdbc:oracle:thin:system/$oracle_password@//$ip:$port/freepdb1" - } + override val db = new OracleDatabaseOnDocker val map = new CaseInsensitiveStringMap( Map("url" -> db.getJdbcUrl(dockerIp, externalPort), From 5e71c7da4161ff0b17eee87fe8d9c6d63a4edf31 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Thu, 4 Jan 2024 16:28:56 +0800 Subject: [PATCH 2/4] intent --- .../org/apache/spark/sql/jdbc/OracleDatabaseOnDocker.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleDatabaseOnDocker.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleDatabaseOnDocker.scala index c7ba25602231..6def4fa44bee 100644 --- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleDatabaseOnDocker.scala +++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleDatabaseOnDocker.scala @@ -45,7 +45,7 @@ class OracleDatabaseOnDocker extends DatabaseOnDocker with Logging { try { val dir = Utils.createTempDir() val writer = new PrintWriter(new File(dir, "install.sql")) - writer.write("ALTER SESSION SET DDL_LOCK_TIMEOUT = 10") + writer.write("ALTER SESSIONXX SET DDL_LOCK_TIMEOUT = 10") writer.close() val newBind = new Bind( dir.getAbsolutePath, From 9a12fb41aae1e3c9b036e66a53b81b56bd9f3a8c Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Thu, 4 Jan 2024 18:50:30 +0800 Subject: [PATCH 3/4] [SPARK-46592][DOCKER][TEST] OracleIntegrationSuite is flaky because of ORA-04021 --- .../spark/sql/jdbc/OracleDatabaseOnDocker.scala | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleDatabaseOnDocker.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleDatabaseOnDocker.scala index 6def4fa44bee..c37e5c7d8558 100644 --- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleDatabaseOnDocker.scala +++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleDatabaseOnDocker.scala @@ -19,7 +19,7 @@ package org.apache.spark.sql.jdbc import java.io.{File, PrintWriter} -import com.github.dockerjava.api.model.{AccessMode, Bind, ContainerConfig, HostConfig, Volume} +import com.github.dockerjava.api.model._ import org.apache.spark.internal.Logging import org.apache.spark.util.Utils @@ -45,7 +45,14 @@ class OracleDatabaseOnDocker extends DatabaseOnDocker with Logging { try { val dir = Utils.createTempDir() val writer = new PrintWriter(new File(dir, "install.sql")) - writer.write("ALTER SESSIONXX SET DDL_LOCK_TIMEOUT = 10") + // SPARK-46592: gvenzl/oracle-free occasionally fails to start with the following error: + // 'ORA-04021: timeout occurred while waiting to lock object', when initializing the + // SYSTEM user. This is due to the fact that the default DDL_LOCK_TIMEOUT is 0, which + // means that the lock will no wait. We set the timeout to 30 seconds to try again. + // TODO: This workaround should be removed once the issue is fixed in the image. + // https://github.com/gvenzl/oci-oracle-free/issues/35 + writer.write("ALTER SESSION SET DDL_LOCK_TIMEOUT = 30;") + writer.write(s"ALTER USER SYSTEM IDENTIFIED BY \"$oracle_password\";") writer.close() val newBind = new Bind( dir.getAbsolutePath, From 0575d04991a783b60657a3b1eeb744141430400f Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Thu, 4 Jan 2024 19:29:02 +0800 Subject: [PATCH 4/4] [SPARK-46592][DOCKER][TEST] OracleIntegrationSuite is flaky because of ORA-04021 --- .../org/apache/spark/sql/jdbc/OracleDatabaseOnDocker.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleDatabaseOnDocker.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleDatabaseOnDocker.scala index c37e5c7d8558..9b77469b2f8a 100644 --- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleDatabaseOnDocker.scala +++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleDatabaseOnDocker.scala @@ -52,7 +52,7 @@ class OracleDatabaseOnDocker extends DatabaseOnDocker with Logging { // TODO: This workaround should be removed once the issue is fixed in the image. // https://github.com/gvenzl/oci-oracle-free/issues/35 writer.write("ALTER SESSION SET DDL_LOCK_TIMEOUT = 30;") - writer.write(s"ALTER USER SYSTEM IDENTIFIED BY \"$oracle_password\";") + writer.write(s"""ALTER USER SYSTEM IDENTIFIED BY "$oracle_password";""") writer.close() val newBind = new Bind( dir.getAbsolutePath,