From cf887e5fd8452f6b410aa6d816076c12f99c2166 Mon Sep 17 00:00:00 2001 From: slfan1989 <55643692+slfan1989@users.noreply.github.com> Date: Thu, 10 Oct 2024 09:27:29 -0400 Subject: [PATCH 1/7] HIVE-28191: Upgrade Hadoop Version to 3.4.0 --- .../java/org/apache/hadoop/hive/common/JvmMetrics.java | 9 --------- pom.xml | 2 +- .../tools/metatool/MetaToolTaskListExtTblLocs.java | 2 +- standalone-metastore/pom.xml | 2 +- storage-api/pom.xml | 2 +- 5 files changed, 4 insertions(+), 13 deletions(-) diff --git a/common/src/java/org/apache/hadoop/hive/common/JvmMetrics.java b/common/src/java/org/apache/hadoop/hive/common/JvmMetrics.java index c0d689d550f7..fa0d285faf4a 100644 --- a/common/src/java/org/apache/hadoop/hive/common/JvmMetrics.java +++ b/common/src/java/org/apache/hadoop/hive/common/JvmMetrics.java @@ -20,7 +20,6 @@ import static org.apache.hadoop.hive.common.JvmMetricsInfo.*; -import org.apache.hadoop.log.metrics.EventCounter; import org.apache.hadoop.metrics2.MetricsCollector; import org.apache.hadoop.metrics2.MetricsInfo; import org.apache.hadoop.metrics2.MetricsRecordBuilder; @@ -96,7 +95,6 @@ public void getMetrics(MetricsCollector collector, boolean all) { getMemoryUsage(rb); getGcUsage(rb); getThreadUsage(rb); - getEventCounters(rb); } private void getMemoryUsage(MetricsRecordBuilder rb) { @@ -201,11 +199,4 @@ public ThreadCountResult(int threadsNew, int threadsRunnable, int threadsBlocked this.threadsTerminated = threadsTerminated; } } - - private void getEventCounters(MetricsRecordBuilder rb) { - rb.addCounter(LogFatal, EventCounter.getFatal()) - .addCounter(LogError, EventCounter.getError()) - .addCounter(LogWarn, EventCounter.getWarn()) - .addCounter(LogInfo, EventCounter.getInfo()); - } } diff --git a/pom.xml b/pom.xml index f16874246174..f847be3f900d 100644 --- a/pom.xml +++ b/pom.xml @@ -140,7 +140,7 @@ 22.0 2.4.21 2.2.220 - 3.3.6 + 3.4.0 ${basedir}/${hive.path.to.root}/testutils/hadoop 1.3 2.5.6-hadoop3 diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/metatool/MetaToolTaskListExtTblLocs.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/metatool/MetaToolTaskListExtTblLocs.java index f9d34ee13f5d..6eccfb665452 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/metatool/MetaToolTaskListExtTblLocs.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/metatool/MetaToolTaskListExtTblLocs.java @@ -341,7 +341,7 @@ private void createOutputList(Set locations, String outputDir, String db * Table-name followed by "*" indicates that all partitions are inside table location. * Otherwise, we record the number of partitions covered by table location. */ - private JSONArray listOutputEntities(HashSet locations) { + private JSONArray listOutputEntities(HashSet locations) throws JSONException { List listEntities = new ArrayList<>(); for(String loc : locations) { DataLocation data = inputLocations.get(loc); diff --git a/standalone-metastore/pom.xml b/standalone-metastore/pom.xml index 81e80833a8bf..878d2e914295 100644 --- a/standalone-metastore/pom.xml +++ b/standalone-metastore/pom.xml @@ -79,7 +79,7 @@ 3.1.0 22.0 - 3.3.6 + 3.4.0 4.0.3 2.16.1 3.3 diff --git a/storage-api/pom.xml b/storage-api/pom.xml index 03ba7bb2ffee..6a43d3987ad0 100644 --- a/storage-api/pom.xml +++ b/storage-api/pom.xml @@ -30,7 +30,7 @@ 1.8 1.1.3 22.0 - 3.3.6 + 3.4.0 4.13.2 5.11.2 5.11.2 From 7c6dc557fa7105bade76c023970660b0e4015928 Mon Sep 17 00:00:00 2001 From: linghengqian Date: Thu, 10 Oct 2024 09:28:58 -0400 Subject: [PATCH 2/7] Upgrade Hbase Version to 2.5.10 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f847be3f900d..eef1a2122afb 100644 --- a/pom.xml +++ b/pom.xml @@ -143,7 +143,7 @@ 3.4.0 ${basedir}/${hive.path.to.root}/testutils/hadoop 1.3 - 2.5.6-hadoop3 + 2.5.10-hadoop3 0.7.2 3.3.7 From 6934a2c2172c8415ed3227189cd101d5dd63fdf0 Mon Sep 17 00:00:00 2001 From: zhangbutao Date: Thu, 10 Oct 2024 09:31:27 -0400 Subject: [PATCH 3/7] Remove useless Hbase profile --- itests/hcatalog-unit/pom.xml | 19 ------------------- itests/util/pom.xml | 18 ------------------ pom.xml | 8 -------- 3 files changed, 45 deletions(-) diff --git a/itests/hcatalog-unit/pom.xml b/itests/hcatalog-unit/pom.xml index 748900958db5..9448ad8dba12 100644 --- a/itests/hcatalog-unit/pom.xml +++ b/itests/hcatalog-unit/pom.xml @@ -357,23 +357,4 @@ - - - - gahbase - - - hbase.version - - - - - org.apache.hbase - hbase-zookeeper - tests - test - - - - diff --git a/itests/util/pom.xml b/itests/util/pom.xml index 5eae654d8835..77813db787ae 100644 --- a/itests/util/pom.xml +++ b/itests/util/pom.xml @@ -258,22 +258,4 @@ - - - - gahbase - - - hbase.version - - - - - org.apache.hbase - hbase-zookeeper - tests - - - - diff --git a/pom.xml b/pom.xml index eef1a2122afb..7f567802d46f 100644 --- a/pom.xml +++ b/pom.xml @@ -2120,14 +2120,6 @@ itests - - customhbase - - - hbase.version - - - dist From 95a1731ac3b49a2153789cebec0f38f606f65e73 Mon Sep 17 00:00:00 2001 From: zhangbutao Date: Thu, 10 Oct 2024 09:41:35 -0400 Subject: [PATCH 4/7] Fix Hbase issue HBASE-28908 --- .../main/java/org/apache/hadoop/hive/hbase/HBaseTestSetup.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseTestSetup.java b/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseTestSetup.java index 202420854dff..489d5b39f759 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseTestSetup.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseTestSetup.java @@ -106,6 +106,8 @@ private void setUpFixtures(HiveConf conf) throws Exception { "org.apache.hadoop.hbase.shaded."); Configuration hbaseConf = HBaseConfiguration.create(conf); + // A workaround for HBASE-28908 + hbaseConf.set("hbase.wal.provider", "filesystem"); util = new HBaseTestingUtility(hbaseConf); util.startMiniDFSCluster(1); From ab10e087f4d49a474e64d7a719a6e4652d363bac Mon Sep 17 00:00:00 2001 From: zhangbutao Date: Thu, 10 Oct 2024 09:41:58 -0400 Subject: [PATCH 5/7] Remove useless&incorrect env setting to fix cast exception --- .../hive/service/auth/jwt/TestHttpJwtAuthentication.java | 2 -- .../hive/metastore/TestRemoteHiveMetastoreWithHttpJwt.java | 5 ----- 2 files changed, 7 deletions(-) diff --git a/itests/hive-unit/src/test/java/org/apache/hive/service/auth/jwt/TestHttpJwtAuthentication.java b/itests/hive-unit/src/test/java/org/apache/hive/service/auth/jwt/TestHttpJwtAuthentication.java index 7aa7adb67e98..664bf0563c46 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/service/auth/jwt/TestHttpJwtAuthentication.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/service/auth/jwt/TestHttpJwtAuthentication.java @@ -87,9 +87,7 @@ public class TestHttpJwtAuthentication { public static void makeEnvModifiable() throws Exception { envMap = new HashMap<>(); Class envClass = Class.forName("java.lang.ProcessEnvironment"); - Field theEnvironmentField = envClass.getDeclaredField("theEnvironment"); Field theUnmodifiableEnvironmentField = envClass.getDeclaredField("theUnmodifiableEnvironment"); - removeStaticFinalAndSetValue(theEnvironmentField, envMap); removeStaticFinalAndSetValue(theUnmodifiableEnvironmentField, envMap); } diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetastoreWithHttpJwt.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetastoreWithHttpJwt.java index c228e94fd491..39009f0942f1 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetastoreWithHttpJwt.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetastoreWithHttpJwt.java @@ -49,7 +49,6 @@ import java.util.Date; import java.util.UUID; import java.util.concurrent.TimeUnit; -import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; import org.slf4j.Logger; @@ -58,8 +57,6 @@ import static com.github.tomakehurst.wiremock.client.WireMock.get; import static com.github.tomakehurst.wiremock.client.WireMock.ok; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; /* Tests JWT auth in HiveMetastore server. @@ -94,9 +91,7 @@ public class TestRemoteHiveMetastoreWithHttpJwt { public static void makeEnvModifiable() throws Exception { envMap = new HashMap<>(); Class envClass = Class.forName("java.lang.ProcessEnvironment"); - Field theEnvironmentField = envClass.getDeclaredField("theEnvironment"); Field theUnmodifiableEnvironmentField = envClass.getDeclaredField("theUnmodifiableEnvironment"); - removeStaticFinalAndSetValue(theEnvironmentField, envMap); removeStaticFinalAndSetValue(theUnmodifiableEnvironmentField, envMap); } From 737cb37f56789a2acac6ae643987b29bd7339cf1 Mon Sep 17 00:00:00 2001 From: zhangbutao Date: Mon, 11 Nov 2024 17:04:17 +0800 Subject: [PATCH 6/7] Change hadoop version to 3.4.1 --- pom.xml | 2 +- standalone-metastore/pom.xml | 2 +- storage-api/pom.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 7f567802d46f..e49b80e2e77d 100644 --- a/pom.xml +++ b/pom.xml @@ -140,7 +140,7 @@ 22.0 2.4.21 2.2.220 - 3.4.0 + 3.4.1 ${basedir}/${hive.path.to.root}/testutils/hadoop 1.3 2.5.10-hadoop3 diff --git a/standalone-metastore/pom.xml b/standalone-metastore/pom.xml index 878d2e914295..8d7b04baf42f 100644 --- a/standalone-metastore/pom.xml +++ b/standalone-metastore/pom.xml @@ -79,7 +79,7 @@ 3.1.0 22.0 - 3.4.0 + 3.4.1 4.0.3 2.16.1 3.3 diff --git a/storage-api/pom.xml b/storage-api/pom.xml index 6a43d3987ad0..dddae62a7557 100644 --- a/storage-api/pom.xml +++ b/storage-api/pom.xml @@ -30,7 +30,7 @@ 1.8 1.1.3 22.0 - 3.4.0 + 3.4.1 4.13.2 5.11.2 5.11.2 From bed92f624dbc5afe759accec686ea52925e13370 Mon Sep 17 00:00:00 2001 From: zhangbutao Date: Wed, 5 Feb 2025 14:59:25 +0800 Subject: [PATCH 7/7] Revert upgrading Hbase version --- .../main/java/org/apache/hadoop/hive/hbase/HBaseTestSetup.java | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseTestSetup.java b/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseTestSetup.java index 489d5b39f759..d7bc2880a90c 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseTestSetup.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseTestSetup.java @@ -106,7 +106,7 @@ private void setUpFixtures(HiveConf conf) throws Exception { "org.apache.hadoop.hbase.shaded."); Configuration hbaseConf = HBaseConfiguration.create(conf); - // A workaround for HBASE-28908 + // A workaround for HBASE-28908. Should be removed once HBase 2.7.0 is released & HIVE-28740 is fixed. hbaseConf.set("hbase.wal.provider", "filesystem"); util = new HBaseTestingUtility(hbaseConf); diff --git a/pom.xml b/pom.xml index e49b80e2e77d..5ef5ffe05494 100644 --- a/pom.xml +++ b/pom.xml @@ -143,7 +143,7 @@ 3.4.1 ${basedir}/${hive.path.to.root}/testutils/hadoop 1.3 - 2.5.10-hadoop3 + 2.5.6-hadoop3 0.7.2 3.3.7