Skip to content

Commit e39c4b7

Browse files
HanumathRaomgorbov
authored andcommitted
Branch spark 69 (apache#170)
* Fixing the wrong type casting of TimeStamp to OTimeStamp when read from spark dataFrame. * SPARK-69: Problem with license when we try to read from json and write to maprdb
1 parent 177edc0 commit e39c4b7

File tree

3 files changed

+15
-2
lines changed

3 files changed

+15
-2
lines changed

external/maprdb/src/main/scala/com/mapr/db/spark/utils/MapRDBUtils.scala

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ private[spark] object MapRDBUtils {
2727
tabDesc.setAutoSplit(true)
2828
tabDesc.setPath(tableName)
2929
tabDesc.setBulkLoad(bulkMode)
30+
tabDesc.setInsertionOrder(false)
3031
if (keys.isEmpty)
3132
DBClient().createTable(tabDesc)
3233
else

external/maprdb/src/test/scala/com/mapr/db/testCases/PredicateTests.scala

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@ import org.apache.spark.rdd.RDD
1010
import org.ojai.exceptions.TypeException
1111
import org.ojai.store.QueryCondition
1212
import com.mapr.db.MapRDB
13+
import com.mapr.db.spark.dbclient.DBClient
14+
1315

1416
object PredicateTests {
1517
val tableName = "/tmp/user_profiles_predicates"
@@ -29,7 +31,11 @@ object PredicateTests {
2931
if (MapRDB.tableExists(tableName))
3032
MapRDB.deleteTable(tableName)
3133
println("table successfully create :" + tableName)
32-
MapRDB.createTable(tableName)
34+
val tabDesc = DBClient().newTableDescriptor()
35+
tabDesc.setAutoSplit(true)
36+
tabDesc.setPath(tableName)
37+
tabDesc.setInsertionOrder(false)
38+
DBClient().createTable(tabDesc)
3339
}
3440

3541
def runTests(sparkSession: SparkContext): Unit = {

external/maprdb/src/test/scala/com/mapr/db/testCases/SparkSqlAcessTests.scala

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import org.apache.spark.sql.SparkSession
1212
import org.ojai.types.{ODate, OTime, OTimestamp}
1313
import com.mapr.org.apache.hadoop.hbase.util.Bytes.ByteArrayComparator
1414
import com.mapr.db.MapRDB
15+
import com.mapr.db.spark.dbclient.DBClient
1516

1617
object SparkSqlAccessTests {
1718
lazy val conf = new SparkConf()
@@ -31,7 +32,12 @@ object SparkSqlAccessTests {
3132
def tableInitialization(tableName: String): Unit = {
3233
if (MapRDB.tableExists(tableName))
3334
MapRDB.deleteTable(tableName)
34-
val table = MapRDB.createTable(tableName)
35+
val tabDesc = DBClient().newTableDescriptor()
36+
tabDesc.setAutoSplit(true)
37+
tabDesc.setPath(tableName)
38+
tabDesc.setInsertionOrder(false)
39+
DBClient().createTable(tabDesc)
40+
val table = DBClient().getTable(tableName)
3541
table.insertOrReplace(getNullRecord())
3642
table.insertOrReplace(getBooleanRecord())
3743
table.insertOrReplace(getStringRecord())

0 commit comments

Comments
 (0)