[KYUUBI #1950] Remove ambiguous SPARK_HADOOP_VERSION

<!--
Thanks for sending a pull request!

Here are some tips for you:
  1. If this is your first time, please read our contributor guidelines: https://kyuubi.readthedocs.io/en/latest/community/contributions.html
  2. If the PR is related to an issue in https://github.com/apache/incubator-kyuubi/issues, add '[KYUUBI #XXXX]' in your PR title, e.g., '[KYUUBI #XXXX] Your PR title ...'.
  3. If the PR is unfinished, add '[WIP]' in your PR title, e.g., '[WIP][KYUUBI #XXXX] Your PR title ...'.
-->

### _Why are the changes needed?_
<!--
Please clarify why the changes are needed. For instance,
  1. If you add a feature, you can talk about the use case of it.
  2. If you fix a bug, you can clarify why it is a bug.
-->

The original idea of SPARK_HADOOP_VERSION is used to concat spark release names only, now we need to remove it as
- SPARK_HADOOP_VERSION is misunderstood by developers and misused somewhere like the one of kyuubi compiled
- multi-engine support now
- the release names  of spark(or something else) are very easy to get through code with different environments, prod/test/dev
- A `mvn` job is bundled with `bin/load-kyuubi-env.sh` which is truly worrisome
- SPARK_HADOOP_VERSION on spark side hass broken already for spark 3.2 which actually bundled with hadoop 3.3, see apache/spark-website#361 (comment)

### _How was this patch tested?_
- [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible

- [ ] Add screenshots for manual tests if appropriate

- [x] [Run test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests) locally before make a pull request

Closes #1950 from yaooqinn/hadoop.

Closes #1950

b47be7c [Kent Yao] Remove ambiguous SPARK_HADOOP_VERSION
3b33ee5 [Kent Yao] Remove ambiguous SPARK_HADOOP_VERSION

Authored-by: Kent Yao <[email protected]>
Signed-off-by: Kent Yao <[email protected]>

Author: yaooqinn

bin/load-kyuubi-env.sh

@@ -71,32 +71,11 @@ fi
 export KYUUBI_SCALA_VERSION="${KYUUBI_SCALA_VERSION:-"2.12"}"
 
 if [[ -f ${KYUUBI_HOME}/RELEASE ]]; then
-  KYUUBI_VERSION="$(grep "Kyuubi " "$KYUUBI_HOME/RELEASE" | awk -F ' ' '{print $2}')"
-  FLINK_VERSION_BUILD="$(grep "Flink " "$KYUUBI_HOME/RELEASE" | awk -F ' ' '{print $2}')"
-  SPARK_VERSION_BUILD="$(grep "Spark " "$KYUUBI_HOME/RELEASE" | awk -F ' ' '{print $2}' | grep -v 'Hadoop')"
-  HADOOP_VERSION_BUILD="$(grep "Spark Hadoop " "$KYUUBI_HOME/RELEASE" | awk -F ' ' '{print $3}')"
-  FLINK_BUILTIN="${KYUUBI_HOME}/externals/flink-$FLINK_VERSION_BUILD"
-  SPARK_BUILTIN="${KYUUBI_HOME}/externals/spark-$SPARK_VERSION_BUILD-bin-hadoop${HADOOP_VERSION_BUILD:0:3}"
+  FLINK_BUILTIN="$(find "$KYUUBI_HOME/externals" -name 'flink-*' -type d | head -n 1)"
+  SPARK_BUILTIN="$(find "$KYUUBI_HOME/externals" -name 'spark-*' -type d | head -n 1)"
 else
-  MVN="${MVN:-"${KYUUBI_HOME}/build/mvn"}"
-  KYUUBI_VERSION=$("$MVN" help:evaluate -Dexpression=project.version 2>/dev/null\
-    | grep -v "INFO"\
-    | grep -v "WARNING"\
-    | tail -n 1)
-  FLINK_VERSION_BUILD=$("$MVN" help:evaluate -Dexpression=flink.version 2>/dev/null\
-    | grep -v "INFO"\
-    | grep -v "WARNING"\
-    | tail -n 1)
-  SPARK_VERSION_BUILD=$("$MVN" help:evaluate -Dexpression=spark.version 2>/dev/null\
-    | grep -v "INFO"\
-    | grep -v "WARNING"\
-    | tail -n 1)
-  HADOOP_VERSION_BUILD=$("$MVN" help:evaluate -Dexpression=hadoop.binary.version 2>/dev/null\
-    | grep -v "INFO"\
-    | grep -v "WARNING"\
-    | tail -n 1)
-  FLINK_BUILTIN="${KYUUBI_HOME}/externals/kyuubi-download/target/flink-$FLINK_VERSION_BUILD"
-  SPARK_BUILTIN="${KYUUBI_HOME}/externals/kyuubi-download/target/spark-$SPARK_VERSION_BUILD-bin-hadoop${HADOOP_VERSION_BUILD}"
+  FLINK_BUILTIN="$(find "$KYUUBI_HOME/externals/kyuubi-download/target" -name 'flink-*' -type d | head -n 1)"
+  SPARK_BUILTIN="$(find "$KYUUBI_HOME/externals/kyuubi-download/target" -name 'spark-*' -type d | head -n 1)"
 fi
 
 export FLINK_HOME="${FLINK_HOME:-"${FLINK_BUILTIN}"}"
@@ -109,7 +88,6 @@ if [ $silent -eq 0 ]; then
   echo "JAVA_HOME: ${JAVA_HOME}"
 
   echo "KYUUBI_HOME: ${KYUUBI_HOME}"
-  echo "KYUUBI_VERSION: ${KYUUBI_VERSION}"
   echo "KYUUBI_CONF_DIR: ${KYUUBI_CONF_DIR}"
   echo "KYUUBI_LOG_DIR: ${KYUUBI_LOG_DIR}"
   echo "KYUUBI_PID_DIR: ${KYUUBI_PID_DIR}"

build/dist

@@ -153,11 +153,6 @@ SPARK_VERSION=$("$MVN" help:evaluate -Dexpression=spark.version $@ 2>/dev/null\
     | grep -v "WARNING"\
     | tail -n 1)
 
-SPARK_HADOOP_VERSION=$("$MVN" help:evaluate -Dexpression=hadoop.binary.version $@ 2>/dev/null\
-    | grep -v "INFO"\
-    | grep -v "WARNING"\
-    | tail -n 1)
-
 HADOOP_VERSION=$("$MVN" help:evaluate -Dexpression=hadoop.version $@ 2>/dev/null\
     | grep -v "INFO"\
     | grep -v "WARNING"\
@@ -175,7 +170,7 @@ if [[ "$NAME" == "none" ]]; then
   if [[ "$SPARK_PROVIDED" == "true" ]]; then
     SUFFIX=""
   else
-    SUFFIX="-spark-${SPARK_VERSION:0:3}-hadoop${SPARK_HADOOP_VERSION}"
+    SUFFIX="-spark-${SPARK_VERSION:0:3}"
   fi
 fi
 
@@ -219,7 +214,6 @@ echo "Java $JAVA_VERSION" >> "$DISTDIR/RELEASE"
 echo "Scala $SCALA_VERSION" >> "$DISTDIR/RELEASE"
 echo "Flink $FLINK_VERSION" >> "$DISTDIR/RELEASE"
 echo "Spark $SPARK_VERSION" >> "$DISTDIR/RELEASE"
-echo "Spark Hadoop $SPARK_HADOOP_VERSION" >> "$DISTDIR/RELEASE"
 echo "Kyuubi Hadoop $HADOOP_VERSION" >> "$DISTDIR/RELEASE"
 echo "Hive $HIVE_VERSION" >> "$DISTDIR/RELEASE"
 echo "Build flags: $@" >> "$DISTDIR/RELEASE"
@@ -275,14 +269,14 @@ done
 
 if [[ "$FLINK_PROVIDED" != "true" ]]; then
   # Copy flink binary dist
-  cp -r "$KYUUBI_HOME/externals/kyuubi-download/target/flink-$FLINK_VERSION/" \
-        "$DISTDIR/externals/flink-$FLINK_VERSION/"
+  FLINK_BUILTIN="$(find "$KYUUBI_HOME/externals/kyuubi-download/target" -name 'flink-*' -type d)"
+  cp -r "$FLINK_BUILTIN" "$DISTDIR/externals/"
 fi
 
 if [[ "$SPARK_PROVIDED" != "true" ]]; then
   # Copy spark binary dist
-  cp -r "$KYUUBI_HOME/externals/kyuubi-download/target/spark-$SPARK_VERSION-bin-hadoop${SPARK_HADOOP_VERSION}$HIVE_VERSION_SUFFIX/" \
-        "$DISTDIR/externals/spark-$SPARK_VERSION-bin-hadoop${SPARK_HADOOP_VERSION}$HIVE_VERSION_SUFFIX/"
+  SPARK_BUILTIN="$(find "$KYUUBI_HOME/externals/kyuubi-download/target" -name 'spark-*' -type d)"
+  cp -r "$SPARK_BUILTIN" "$DISTDIR/externals/"
 fi
 
 # Copy license files

pom.xml

@@ -110,7 +110,6 @@
         <guava.version>30.1-jre</guava.version>
         <guava.failureaccess.version>1.0.1</guava.failureaccess.version>
         <hadoop.version>3.3.1</hadoop.version>
-        <hadoop.binary.version>3.2</hadoop.binary.version>
         <hive.version>2.3.9</hive.version>
         <hudi.version>0.10.0</hudi.version>
         <iceberg.name>iceberg-spark-runtime-3.2_${scala.binary.version}</iceberg.name>
@@ -138,7 +137,7 @@
           `delta.version`, `iceberg.name`
           -->
         <spark.version>3.2.1</spark.version>
-        <spark.archive.name>spark-${spark.version}-bin-hadoop${hadoop.binary.version}.tgz</spark.archive.name>
+        <spark.archive.name>spark-${spark.version}-bin-hadoop3.2.tgz</spark.archive.name>
         <spark.archive.mirror>https://archive.apache.org/dist/spark/spark-${spark.version}</spark.archive.mirror>
         <spark.archive.download.skip>false</spark.archive.download.skip>
         <swagger.version>2.1.11</swagger.version>
@@ -1901,20 +1900,6 @@
             </properties>
         </profile>
 
-        <profile>
-            <id>spark-hadoop-2.7</id>
-            <properties>
-                <hadoop.binary.version>2.7</hadoop.binary.version>
-            </properties>
-        </profile>
-
-        <profile>
-            <id>spark-hadoop-3.2</id>
-            <properties>
-                <hadoop.binary.version>3.2</hadoop.binary.version>
-            </properties>
-        </profile>
-
         <profile>
             <id>spark-provided</id>
             <properties>