[SPARK-52010] Do not generate API docs for internal classes

### What changes were proposed in this pull request?

While reviewing the API doc of 4.0 RC4, I found that there are some newly added internal classes appear in the API doc. This PR fixes them.

### Why are the changes needed?

API doc should not include internal classes

### Does this PR introduce _any_ user-facing change?

no

### How was this patch tested?

N/A

### Was this patch authored or co-authored using generative AI tooling?

no

Closes #50797 from cloud-fan/api.

Authored-by: Wenchen Fan <[email protected]>
Signed-off-by: Kent Yao <[email protected]>
(cherry picked from commit 9f5ae88)
Signed-off-by: Kent Yao <[email protected]>

Author: cloud-fan

common/utils/src/main/scala/org/apache/spark/util/SparkStringUtils.scala

@@ -16,11 +16,11 @@
  */
 package org.apache.spark.util
 
-trait SparkStringUtils {
+private[spark] trait SparkStringUtils {
   def stringToSeq(str: String): Seq[String] = {
     import org.apache.spark.util.ArrayImplicits._
     str.split(",").map(_.trim()).filter(_.nonEmpty).toImmutableArraySeq
   }
 }
 
-object SparkStringUtils extends SparkStringUtils
+private[spark] object SparkStringUtils extends SparkStringUtils

common/utils/src/main/scala/org/apache/spark/util/SparkTestUtils.scala

@@ -25,7 +25,7 @@ import javax.tools.{JavaFileObject, SimpleJavaFileObject, ToolProvider}
 
 import scala.jdk.CollectionConverters._
 
-trait SparkTestUtils {
+private[spark] trait SparkTestUtils {
   // Adapted from the JavaCompiler.java doc examples
   private val SOURCE = JavaFileObject.Kind.SOURCE
 
@@ -103,4 +103,4 @@ trait SparkTestUtils {
 
 }
 
-object SparkTestUtils extends SparkTestUtils
+private[spark] object SparkTestUtils extends SparkTestUtils

connector/avro/src/main/scala/org/apache/spark/sql/avro/AvroDataToCatalyst.scala

@@ -30,7 +30,7 @@ import org.apache.spark.sql.catalyst.util.{FailFastMode, ParseMode, PermissiveMo
 import org.apache.spark.sql.errors.QueryCompilationErrors
 import org.apache.spark.sql.types._
 
-private[sql] case class AvroDataToCatalyst(
+case class AvroDataToCatalyst(
     child: Expression,
     jsonFormatSchema: String,
     options: Map[String, String])

connector/avro/src/main/scala/org/apache/spark/sql/avro/CatalystDataToAvro.scala

@@ -27,7 +27,7 @@ import org.apache.spark.sql.catalyst.expressions.{Expression, UnaryExpression}
 import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode}
 import org.apache.spark.sql.types.{BinaryType, DataType}
 
-private[sql] case class CatalystDataToAvro(
+case class CatalystDataToAvro(
     child: Expression,
     jsonFormatSchema: Option[String]) extends UnaryExpression {
 

connector/avro/src/main/scala/org/apache/spark/sql/avro/SchemaOfAvro.scala

@@ -25,7 +25,7 @@ import org.apache.spark.sql.catalyst.util.{FailFastMode, ParseMode, PermissiveMo
 import org.apache.spark.sql.errors.QueryCompilationErrors
 import org.apache.spark.sql.types.{DataType, ObjectType, StringType}
 
-private[sql] case class SchemaOfAvro(
+case class SchemaOfAvro(
     jsonFormatSchema: String,
     options: Map[String, String])
   extends LeafExpression with RuntimeReplaceable {

connector/kafka-0-10-token-provider/src/main/scala/org/apache/spark/kafka010/KafkaConfigUpdater.scala

@@ -30,7 +30,7 @@ import org.apache.spark.internal.Logging
 /**
  * Class to conveniently update Kafka config params, while logging the changes
  */
-private[spark] case class KafkaConfigUpdater(module: String, kafkaParams: Map[String, Object])
+case class KafkaConfigUpdater(module: String, kafkaParams: Map[String, Object])
     extends Logging {
   private val map = new ju.HashMap[String, Object](kafkaParams.asJava)
 

connector/kafka-0-10-token-provider/src/main/scala/org/apache/spark/kafka010/KafkaDelegationTokenProvider.scala

@@ -28,7 +28,7 @@ import org.apache.spark.internal.{Logging, MDC}
 import org.apache.spark.internal.LogKeys.{CLUSTER_ID, SERVICE_NAME}
 import org.apache.spark.security.HadoopDelegationTokenProvider
 
-private[spark] class KafkaDelegationTokenProvider
+class KafkaDelegationTokenProvider
   extends HadoopDelegationTokenProvider with Logging {
 
   override def serviceName: String = "kafka"

connector/kafka-0-10-token-provider/src/main/scala/org/apache/spark/kafka010/KafkaRedactionUtil.scala

@@ -24,8 +24,8 @@ import org.apache.spark.internal.Logging
 import org.apache.spark.internal.config.SECRET_REDACTION_PATTERN
 import org.apache.spark.util.Utils.{redact, REDACTION_REPLACEMENT_TEXT}
 
-private[spark] object KafkaRedactionUtil extends Logging {
-  private[spark] def redactParams(params: Seq[(String, Object)]): Seq[(String, String)] = {
+object KafkaRedactionUtil extends Logging {
+  def redactParams(params: Seq[(String, Object)]): Seq[(String, String)] = {
     val redactionPattern = Some(Option(SparkEnv.get).map(_.conf)
       .getOrElse(new SparkConf()).get(SECRET_REDACTION_PATTERN))
     params.map { case (key, value) =>
@@ -42,7 +42,7 @@ private[spark] object KafkaRedactionUtil extends Logging {
     }
   }
 
-  private[kafka010] def redactJaasParam(param: String): String = {
+  def redactJaasParam(param: String): String = {
     if (param != null && !param.isEmpty) {
       param.replaceAll("password=\".*\"", s"""password="$REDACTION_REPLACEMENT_TEXT"""")
     } else {

connector/kafka-0-10-token-provider/src/main/scala/org/apache/spark/kafka010/KafkaTokenSparkConf.scala

@@ -25,7 +25,7 @@ import org.apache.spark.SparkConf
 import org.apache.spark.internal.Logging
 import org.apache.spark.util.Utils.REDACTION_REPLACEMENT_TEXT
 
-private[spark] case class KafkaTokenClusterConf(
+case class KafkaTokenClusterConf(
     identifier: String,
     authBootstrapServers: String,
     targetServersRegex: String,
@@ -57,7 +57,7 @@ private[spark] case class KafkaTokenClusterConf(
     s"specifiedKafkaParams=${KafkaRedactionUtil.redactParams(specifiedKafkaParams.toSeq)}}"
 }
 
-private [kafka010] object KafkaTokenSparkConf extends Logging {
+object KafkaTokenSparkConf extends Logging {
   val CLUSTERS_CONFIG_PREFIX = "spark.kafka.clusters."
   val DEFAULT_TARGET_SERVERS_REGEX = ".*"
   val DEFAULT_SASL_KERBEROS_SERVICE_NAME = "kafka"

connector/kafka-0-10-token-provider/src/main/scala/org/apache/spark/kafka010/KafkaTokenUtil.scala

@@ -44,25 +44,25 @@ import org.apache.spark.internal.config._
 import org.apache.spark.util.{SecurityUtils, Utils}
 import org.apache.spark.util.Utils.REDACTION_REPLACEMENT_TEXT
 
-private[spark] object KafkaTokenUtil extends Logging {
+object KafkaTokenUtil extends Logging {
   val TOKEN_KIND = new Text("KAFKA_DELEGATION_TOKEN")
   private val TOKEN_SERVICE_PREFIX = "kafka.server.delegation.token"
   private val DATE_TIME_FORMATTER =
     DateTimeFormatter
       .ofPattern("yyyy-MM-dd'T'HH:mm")
       .withZone(ZoneId.systemDefault())
 
-  private[kafka010] def getTokenService(identifier: String): Text =
+  def getTokenService(identifier: String): Text =
     new Text(s"$TOKEN_SERVICE_PREFIX.$identifier")
 
   private def getClusterIdentifier(service: Text): String =
     service.toString().replace(s"$TOKEN_SERVICE_PREFIX.", "")
 
-  private[spark] class KafkaDelegationTokenIdentifier extends AbstractDelegationTokenIdentifier {
+  class KafkaDelegationTokenIdentifier extends AbstractDelegationTokenIdentifier {
     override def getKind: Text = TOKEN_KIND
   }
 
-  private[kafka010] def obtainToken(
+  def obtainToken(
       sparkConf: SparkConf,
       clusterConf: KafkaTokenClusterConf): (Token[KafkaDelegationTokenIdentifier], Long) = {
     checkProxyUser()
@@ -81,15 +81,15 @@ private[spark] object KafkaTokenUtil extends Logging {
     ), token.tokenInfo.expiryTimestamp)
   }
 
-  private[kafka010] def checkProxyUser(): Unit = {
+  def checkProxyUser(): Unit = {
     val currentUser = UserGroupInformation.getCurrentUser()
     // Obtaining delegation token for proxy user is planned but not yet implemented
     // See https://issues.apache.org/jira/browse/KAFKA-6945
     require(!SparkHadoopUtil.get.isProxyUser(currentUser), "Obtaining delegation token for proxy " +
       "user is not yet supported.")
   }
 
-  private[kafka010] def createAdminClientProperties(
+  def createAdminClientProperties(
       sparkConf: SparkConf,
       clusterConf: KafkaTokenClusterConf): ju.Properties = {
     val adminClientProperties = new ju.Properties