[KYUUBI #3677][K8S] Bump Kubernetes Client Version to 6.1.1

.github/workflows/dep.yml

@@ -56,4 +56,6 @@ jobs:
           -DskipTests
           -pl kyuubi-ctl,kyuubi-server,kyuubi-assembly -am
       - name: Check dependency list
-        run: build/dependency.sh
+        run: |
+          /usr/bin/mvn --version
+          build/dependency.sh

.github/workflows/master.yml

@@ -421,6 +421,7 @@ jobs:
         if: failure()
         run: |
           kubectl get pods
+          kubectl get pods | grep driver | awk -F " " '{print$1}' | xargs -I {} kubectl describe pods {}
           kubectl get pods | grep driver | awk -F " " '{print$1}' | xargs -I {} kubectl logs {}
       - name: Upload test logs
         if: failure()

dev/dependencyList

@@ -16,13 +16,10 @@
 #
 
 HikariCP/4.0.3//HikariCP-4.0.3.jar
-ST4/4.3.4//ST4-4.3.4.jar
 animal-sniffer-annotations/1.21//animal-sniffer-annotations-1.21.jar
 annotations/4.1.1.4//annotations-4.1.1.4.jar
-antlr-runtime/3.5.3//antlr-runtime-3.5.3.jar
 antlr4-runtime/4.9.3//antlr4-runtime-4.9.3.jar
 aopalliance-repackaged/2.6.1//aopalliance-repackaged-2.6.1.jar
-automaton/1.11-8//automaton-1.11-8.jar
 classgraph/4.8.138//classgraph-4.8.138.jar
 commons-codec/1.15//commons-codec-1.15.jar
 commons-collections/3.2.2//commons-collections-3.2.2.jar
@@ -37,7 +34,6 @@ error_prone_annotations/2.14.0//error_prone_annotations-2.14.0.jar
 failsafe/2.4.4//failsafe-2.4.4.jar
 failureaccess/1.0.1//failureaccess-1.0.1.jar
 fliptables/1.0.2//fliptables-1.0.2.jar
-generex/1.0.2//generex-1.0.2.jar
 grpc-api/1.48.0//grpc-api-1.48.0.jar
 grpc-context/1.48.0//grpc-context-1.48.0.jar
 grpc-core/1.48.0//grpc-core-1.48.0.jar
@@ -63,7 +59,6 @@ hk2-locator/2.6.1//hk2-locator-2.6.1.jar
 hk2-utils/2.6.1//hk2-utils-2.6.1.jar
 httpclient/4.5.14//httpclient-4.5.14.jar
 httpcore/4.4.16//httpcore-4.4.16.jar
-httpmime/4.5.14//httpmime-4.5.14.jar
 j2objc-annotations/1.3//j2objc-annotations-1.3.jar
 jackson-annotations/2.14.1//jackson-annotations-2.14.1.jar
 jackson-core/2.14.1//jackson-core-2.14.1.jar
@@ -104,27 +99,30 @@ jetty-util-ajax/9.4.50.v20221201//jetty-util-ajax-9.4.50.v20221201.jar
 jetty-util/9.4.50.v20221201//jetty-util-9.4.50.v20221201.jar
 jline/0.9.94//jline-0.9.94.jar
 jul-to-slf4j/1.7.36//jul-to-slf4j-1.7.36.jar
-kubernetes-client/5.12.1//kubernetes-client-5.12.1.jar
-kubernetes-model-admissionregistration/5.12.1//kubernetes-model-admissionregistration-5.12.1.jar
-kubernetes-model-apiextensions/5.12.1//kubernetes-model-apiextensions-5.12.1.jar
-kubernetes-model-apps/5.12.1//kubernetes-model-apps-5.12.1.jar
-kubernetes-model-autoscaling/5.12.1//kubernetes-model-autoscaling-5.12.1.jar
-kubernetes-model-batch/5.12.1//kubernetes-model-batch-5.12.1.jar
-kubernetes-model-certificates/5.12.1//kubernetes-model-certificates-5.12.1.jar
-kubernetes-model-common/5.12.1//kubernetes-model-common-5.12.1.jar
-kubernetes-model-coordination/5.12.1//kubernetes-model-coordination-5.12.1.jar
-kubernetes-model-core/5.12.1//kubernetes-model-core-5.12.1.jar
-kubernetes-model-discovery/5.12.1//kubernetes-model-discovery-5.12.1.jar
-kubernetes-model-events/5.12.1//kubernetes-model-events-5.12.1.jar
-kubernetes-model-extensions/5.12.1//kubernetes-model-extensions-5.12.1.jar
-kubernetes-model-flowcontrol/5.12.1//kubernetes-model-flowcontrol-5.12.1.jar
-kubernetes-model-metrics/5.12.1//kubernetes-model-metrics-5.12.1.jar
-kubernetes-model-networking/5.12.1//kubernetes-model-networking-5.12.1.jar
-kubernetes-model-node/5.12.1//kubernetes-model-node-5.12.1.jar
-kubernetes-model-policy/5.12.1//kubernetes-model-policy-5.12.1.jar
-kubernetes-model-rbac/5.12.1//kubernetes-model-rbac-5.12.1.jar
-kubernetes-model-scheduling/5.12.1//kubernetes-model-scheduling-5.12.1.jar
-kubernetes-model-storageclass/5.12.1//kubernetes-model-storageclass-5.12.1.jar
+kubernetes-client-api/6.1.1//kubernetes-client-api-6.1.1.jar
+kubernetes-client/6.1.1//kubernetes-client-6.1.1.jar
+kubernetes-httpclient-okhttp/6.1.1//kubernetes-httpclient-okhttp-6.1.1.jar
+kubernetes-model-admissionregistration/6.1.1//kubernetes-model-admissionregistration-6.1.1.jar
+kubernetes-model-apiextensions/6.1.1//kubernetes-model-apiextensions-6.1.1.jar
+kubernetes-model-apps/6.1.1//kubernetes-model-apps-6.1.1.jar
+kubernetes-model-autoscaling/6.1.1//kubernetes-model-autoscaling-6.1.1.jar
+kubernetes-model-batch/6.1.1//kubernetes-model-batch-6.1.1.jar
+kubernetes-model-certificates/6.1.1//kubernetes-model-certificates-6.1.1.jar
+kubernetes-model-common/6.1.1//kubernetes-model-common-6.1.1.jar
+kubernetes-model-coordination/6.1.1//kubernetes-model-coordination-6.1.1.jar
+kubernetes-model-core/6.1.1//kubernetes-model-core-6.1.1.jar
+kubernetes-model-discovery/6.1.1//kubernetes-model-discovery-6.1.1.jar
+kubernetes-model-events/6.1.1//kubernetes-model-events-6.1.1.jar
+kubernetes-model-extensions/6.1.1//kubernetes-model-extensions-6.1.1.jar
+kubernetes-model-flowcontrol/6.1.1//kubernetes-model-flowcontrol-6.1.1.jar
+kubernetes-model-gatewayapi/6.1.1//kubernetes-model-gatewayapi-6.1.1.jar
+kubernetes-model-metrics/6.1.1//kubernetes-model-metrics-6.1.1.jar
+kubernetes-model-networking/6.1.1//kubernetes-model-networking-6.1.1.jar
+kubernetes-model-node/6.1.1//kubernetes-model-node-6.1.1.jar
+kubernetes-model-policy/6.1.1//kubernetes-model-policy-6.1.1.jar
+kubernetes-model-rbac/6.1.1//kubernetes-model-rbac-6.1.1.jar
+kubernetes-model-scheduling/6.1.1//kubernetes-model-scheduling-6.1.1.jar
+kubernetes-model-storageclass/6.1.1//kubernetes-model-storageclass-6.1.1.jar
 libfb303/0.9.3//libfb303-0.9.3.jar
 libthrift/0.9.3//libthrift-0.9.3.jar
 log4j-1.2-api/2.19.0//log4j-1.2-api-2.19.0.jar

integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/MiniKube.scala

@@ -17,7 +17,7 @@
 
 package org.apache.kyuubi.kubernetes.test
 
-import io.fabric8.kubernetes.client.{Config, DefaultKubernetesClient}
+import io.fabric8.kubernetes.client.{Config, KubernetesClient, KubernetesClientBuilder}
 
 /**
  * This code copied from Aapache Spark
@@ -44,7 +44,7 @@ object MiniKube {
     executeMinikube(true, "ip").head
   }
 
-  def getKubernetesClient: DefaultKubernetesClient = {
+  def getKubernetesClient: KubernetesClient = {
     // only the three-part version number is matched (the optional suffix like "-beta.0" is dropped)
     val versionArrayOpt = "\\d+\\.\\d+\\.\\d+".r
       .findFirstIn(minikubeVersionString.split(VERSION_PREFIX)(1))
@@ -66,6 +66,6 @@ object MiniKube {
             "non-numeric suffix is intentionally dropped)")
     }
 
-    new DefaultKubernetesClient(Config.autoConfigure("minikube"))
+    new KubernetesClientBuilder().withConfig(Config.autoConfigure("minikube")).build()
   }
 }

integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/WithKyuubiServerOnKubernetes.scala

@@ -18,14 +18,15 @@
 package org.apache.kyuubi.kubernetes.test
 
 import io.fabric8.kubernetes.api.model.Pod
-import io.fabric8.kubernetes.client.DefaultKubernetesClient
+import io.fabric8.kubernetes.client.KubernetesClient
 
 import org.apache.kyuubi.KyuubiFunSuite
 
 trait WithKyuubiServerOnKubernetes extends KyuubiFunSuite {
   protected def connectionConf: Map[String, String] = Map.empty
 
-  lazy val miniKubernetesClient: DefaultKubernetesClient = MiniKube.getKubernetesClient
+  private val miniKubernetesClient: KubernetesClient = MiniKube.getKubernetesClient
+
   lazy val kyuubiPod: Pod = miniKubernetesClient.pods().withName("kyuubi-test").get()
   lazy val kyuubiServerIp: String = kyuubiPod.getStatus.getPodIP
   lazy val miniKubeIp: String = MiniKube.getIp

kyuubi-server/pom.xml

@@ -91,6 +91,10 @@
             <groupId>io.fabric8</groupId>
             <artifactId>kubernetes-client</artifactId>
         </dependency>
+        <dependency>
+            <groupId>io.fabric8</groupId>
+            <artifactId>kubernetes-httpclient-okhttp</artifactId>
+        </dependency>
 
         <dependency>
             <groupId>org.apache.hive</groupId>

kyuubi-server/src/main/scala/org/apache/kyuubi/engine/KubernetesApplicationOperation.scala

@@ -17,9 +17,8 @@
 
 package org.apache.kyuubi.engine
 
-import io.fabric8.kubernetes.api.model.{Pod, PodList}
+import io.fabric8.kubernetes.api.model.Pod
 import io.fabric8.kubernetes.client.KubernetesClient
-import io.fabric8.kubernetes.client.dsl.FilterWatchListDeletable
 
 import org.apache.kyuubi.Logging
 import org.apache.kyuubi.config.KyuubiConf
@@ -57,23 +56,22 @@ class KubernetesApplicationOperation extends ApplicationOperation with Logging {
     if (kubernetesClient != null) {
       debug(s"Deleting application info from Kubernetes cluster by $tag tag")
       try {
-        // Need driver only
-        val operation = findDriverPodByTag(tag)
-        val podList = operation.list().getItems
-        if (podList.size() != 0) {
-          toApplicationState(podList.get(0).getStatus.getPhase) match {
-            case FAILED | UNKNOWN =>
-              (
-                false,
-                s"Target Pod ${podList.get(0).getMetadata.getName} is in FAILED or UNKNOWN status")
-            case _ =>
-              (
-                operation.delete(),
-                s"Operation of deleted appId: ${podList.get(0).getMetadata.getName} is completed")
-          }
-        } else {
-          // client mode
-          jpsOperation.killApplicationByTag(tag)
+        findDriverPodByTag(tag) match {
+          case Some(pod) =>
+            val podName = pod.getMetadata.getName
+            toApplicationState(pod.getStatus.getPhase) match {
+              case FAILED | UNKNOWN =>
+                (false, s"Target Driver Pod $podName is in FAILED or UNKNOWN status")
+              case _ =>
+                // Imitate from fabric8io/kubernetes-client
+                // kubernetes-tests/CustomResourceTest.java#testDeleteNonExistentItem
+                (
+                  !kubernetesClient.pods().inNamespace(podName).withName(
+                    pod.getMetadata.getName).delete().isEmpty,
+                  s"Operation of deleted app: $podName is completed")
+            }
+          case None =>
+            jpsOperation.killApplicationByTag(tag)
         }
       } catch {
         case e: Exception =>
@@ -88,21 +86,18 @@ class KubernetesApplicationOperation extends ApplicationOperation with Logging {
     if (kubernetesClient != null) {
       debug(s"Getting application info from Kubernetes cluster by $tag tag")
       try {
-        val operation = findDriverPodByTag(tag)
-        val podList = operation.list().getItems
-        if (podList.size() != 0) {
-          val pod = podList.get(0)
-          val info = ApplicationInfo(
-            // spark pods always tag label `spark-app-selector:<spark-app-id>`
-            id = pod.getMetadata.getLabels.get(SPARK_APP_ID_LABEL),
-            name = pod.getMetadata.getName,
-            state = KubernetesApplicationOperation.toApplicationState(pod.getStatus.getPhase),
-            error = Option(pod.getStatus.getReason))
-          debug(s"Successfully got application info by $tag: $info")
-          info
-        } else {
-          // client mode
-          jpsOperation.getApplicationInfoByTag(tag)
+        findDriverPodByTag(tag) match {
+          case Some(pod) =>
+            val info = ApplicationInfo(
+              // spark pods always tag label `spark-app-selector:<spark-app-id>`
+              id = pod.getMetadata.getLabels.get(SPARK_APP_ID_LABEL),
+              name = pod.getMetadata.getName,
+              state = KubernetesApplicationOperation.toApplicationState(pod.getStatus.getPhase),
+              error = Option(pod.getStatus.getReason))
+            debug(s"Successfully got application info by $tag: $info")
+            info
+          case None =>
+            jpsOperation.getApplicationInfoByTag(tag)
         }
       } catch {
         case e: Exception =>
@@ -114,14 +109,21 @@ class KubernetesApplicationOperation extends ApplicationOperation with Logging {
     }
   }
 
-  private def findDriverPodByTag(tag: String): FilterWatchListDeletable[Pod, PodList] = {
+  private def findDriverPodByTag(tag: String): Option[Pod] = {
     val operation = kubernetesClient.pods()
       .withLabel(KubernetesApplicationOperation.LABEL_KYUUBI_UNIQUE_KEY, tag)
-    val size = operation.list().getItems.size()
-    if (size != 1) {
-      warn(s"Get Tag: ${tag} Driver Pod In Kubernetes size: ${size}, we expect 1")
+    val podList = operation.list().getItems
+    val size = podList.size()
+    size match {
+      case 0 =>
+        warn(s"Can't find Driver pod with tag $tag")
+        None
+      case 1 =>
+        Some(podList.get(0))
+      case _ =>
+        warn(s"Get Tag: $tag Driver Pod In Kubernetes size: $size, we expect 1")
+        Some(podList.get(0))
     }
-    operation
   }
 
   override def stop(): Unit = {

kyuubi-server/src/main/scala/org/apache/kyuubi/util/KubernetesUtils.scala

@@ -22,7 +22,7 @@ import java.io.File
 import com.fasterxml.jackson.databind.ObjectMapper
 import com.google.common.base.Charsets
 import com.google.common.io.Files
-import io.fabric8.kubernetes.client.{Config, ConfigBuilder, DefaultKubernetesClient, KubernetesClient}
+import io.fabric8.kubernetes.client.{Config, ConfigBuilder, KubernetesClient, KubernetesClientBuilder}
 import io.fabric8.kubernetes.client.Config.autoConfigure
 import io.fabric8.kubernetes.client.okhttp.OkHttpClientFactory
 import okhttp3.{Dispatcher, OkHttpClient}
@@ -93,7 +93,10 @@ object KubernetesUtils extends Logging {
 
     debug("Kubernetes client config: " +
       new ObjectMapper().writerWithDefaultPrettyPrinter().writeValueAsString(config))
-    Some(new DefaultKubernetesClient(factoryWithCustomDispatcher.createHttpClient(config), config))
+    Some(new KubernetesClientBuilder()
+      .withConfig(config)
+      .withHttpClientFactory(factoryWithCustomDispatcher)
+      .build())
   }
 
   implicit private class OptionConfigurableConfigBuilder(val configBuilder: ConfigBuilder)

pom.xml

@@ -166,7 +166,7 @@
         <jetty.version>9.4.50.v20221201</jetty.version>
         <jline.version>0.9.94</jline.version>
         <junit.version>4.13.1</junit.version>
-        <kubernetes-client.version>5.12.1</kubernetes-client.version>
+        <kubernetes-client.version>6.1.1</kubernetes-client.version>
         <kudu.version>1.15.0</kudu.version>
         <ldapsdk.version>6.0.5</ldapsdk.version>
         <log4j.version>2.19.0</log4j.version>
@@ -587,6 +587,16 @@
                 <artifactId>kubernetes-client</artifactId>
                 <version>${kubernetes-client.version}</version>
             </dependency>
+            <!--
+              according to kubernetes-client MIGRATION-v6.md, we should include this dependency
+              for doing any customization to OkHttp clients.
+              https://github.com/fabric8io/kubernetes-client/blob/master/doc/MIGRATION-v6.md#okhttp-httpclient
+            -->
+            <dependency>
+                <groupId>io.fabric8</groupId>
+                <artifactId>kubernetes-httpclient-okhttp</artifactId>
+                <version>${kubernetes-client.version}</version>
+            </dependency>
 
             <!--
               because of THRIFT-4805, we don't upgrade to libthrift:0.12.0,