Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ private[hive] object SparkSQLEnv extends Logging {
metadataHive.setOut(new PrintStream(System.out, true, "UTF-8"))
metadataHive.setInfo(new PrintStream(System.err, true, "UTF-8"))
metadataHive.setError(new PrintStream(System.err, true, "UTF-8"))
sparkSession.conf.set(HiveUtils.FAKE_HIVE_VERSION.key, HiveUtils.builtinHiveVersion)
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ private[hive] class SparkSQLSessionManager(hiveServer: HiveServer2, sqlContext:
} else {
sqlContext.newSession()
}
ctx.setConf(HiveUtils.FAKE_HIVE_VERSION.key, HiveUtils.builtinHiveVersion)
if (sessionConf != null && sessionConf.containsKey("use:database")) {
ctx.sql(s"use ${sessionConf.get("use:database")}")
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -155,9 +155,9 @@ class HiveThriftBinaryServerSuite extends HiveThriftJdbcTest {

test("Checks Hive version") {
withJdbcStatement() { statement =>
val resultSet = statement.executeQuery("SET spark.sql.hive.metastore.version")
val resultSet = statement.executeQuery("SET spark.sql.hive.version")
resultSet.next()
assert(resultSet.getString(1) === "spark.sql.hive.metastore.version")
assert(resultSet.getString(1) === "spark.sql.hive.version")
assert(resultSet.getString(2) === HiveUtils.builtinHiveVersion)
}
}
Expand Down Expand Up @@ -521,7 +521,20 @@ class HiveThriftBinaryServerSuite extends HiveThriftJdbcTest {
conf += resultSet.getString(1) -> resultSet.getString(2)
}

assert(conf.get("spark.sql.hive.metastore.version") === Some("1.2.1"))
assert(conf.get("spark.sql.hive.version") === Some("1.2.1"))
}
}

test("Checks Hive version via SET") {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i might be a session level conf

withJdbcStatement() { statement =>
val resultSet = statement.executeQuery("SET")

val conf = mutable.Map.empty[String, String]
while (resultSet.next()) {
conf += resultSet.getString(1) -> resultSet.getString(2)
}

assert(conf.get("spark.sql.hive.version") === Some("1.2.1"))
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If we do not set it explicitly, this will not be returned.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

a default value will be returned, isn't it?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nope, I think. Try it.

Copy link
Member

@yaooqinn yaooqinn Nov 11, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In Hive, "SET" returns all changed properties while "SET -v" returns all properties. In Spark, SET queries all key-value pairs that are set in the SQLConf of the sparkSession.

}
}

Expand Down Expand Up @@ -708,9 +721,9 @@ class HiveThriftHttpServerSuite extends HiveThriftJdbcTest {

test("Checks Hive version") {
withJdbcStatement() { statement =>
val resultSet = statement.executeQuery("SET spark.sql.hive.metastore.version")
val resultSet = statement.executeQuery("SET spark.sql.hive.version")
resultSet.next()
assert(resultSet.getString(1) === "spark.sql.hive.metastore.version")
assert(resultSet.getString(1) === "spark.sql.hive.version")
assert(resultSet.getString(2) === HiveUtils.builtinHiveVersion)
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,14 @@ private[spark] object HiveUtils extends Logging {
.stringConf
.createWithDefault(builtinHiveVersion)

// A fake config which is only here for backward compatibility reasons. This config has no effect
// to Spark, just for reporting the builtin Hive version of Spark to existing applications that
// already rely on this config.
val FAKE_HIVE_VERSION = buildConf("spark.sql.hive.version")
.doc(s"deprecated, please use ${HIVE_METASTORE_VERSION.key} to get the Hive version in Spark.")
.stringConf
.createWithDefault(builtinHiveVersion)

val HIVE_METASTORE_JARS = buildConf("spark.sql.hive.metastore.jars")
.doc(s"""
| Location of the jars that should be used to instantiate the HiveMetastoreClient.
Expand Down