WIP buildLocationMetadata-show-remaining-number-of-paths

Author: HeartSaVioR

core/src/main/scala/org/apache/spark/util/Utils.scala

@@ -2989,6 +2989,9 @@ private[spark] object Utils extends Logging {
       metadata.append(paths(index).toString)
       index += 1
     }
+    if (paths.length > index) {
+      metadata.append(s", ... ${paths.length - index} more")
+    }
     metadata.append("]")
     metadata.toString
   }

core/src/test/scala/org/apache/spark/util/UtilsSuite.scala

@@ -1304,16 +1304,16 @@ class UtilsSuite extends SparkFunSuite with ResetSystemProperties with Logging {
 
   test("pathsToMetadata") {
     val paths = (0 to 4).map(i => new Path(s"path$i"))
-    assert(Utils.buildLocationMetadata(paths, 5) == "[path0]")
-    assert(Utils.buildLocationMetadata(paths, 10) == "[path0, path1]")
-    assert(Utils.buildLocationMetadata(paths, 15) == "[path0, path1, path2]")
-    assert(Utils.buildLocationMetadata(paths, 25) == "[path0, path1, path2, path3]")
+    assert(Utils.buildLocationMetadata(paths, 5) == "[path0, ... 4 more]")
+    assert(Utils.buildLocationMetadata(paths, 10) == "[path0, path1, ... 3 more]")
+    assert(Utils.buildLocationMetadata(paths, 15) == "[path0, path1, path2, ... 2 more]")
+    assert(Utils.buildLocationMetadata(paths, 25) == "[path0, path1, path2, path3, ... 1 more]")
 
     // edge-case: we should consider the fact non-path chars including '[' and ", " are accounted
     // 1. second path is not added due to the addition of '['
-    assert(Utils.buildLocationMetadata(paths, 6) == "[path0]")
+    assert(Utils.buildLocationMetadata(paths, 6) == "[path0, ... 4 more]")
     // 2. third path is not added due to the addition of ", "
-    assert(Utils.buildLocationMetadata(paths, 13) == "[path0, path1]")
+    assert(Utils.buildLocationMetadata(paths, 13) == "[path0, path1, ... 3 more]")
   }
 
   test("checkHost supports both IPV4 and IPV6") {

sql/core/src/test/scala/org/apache/spark/sql/execution/DataSourceScanExecRedactionSuite.scala

@@ -122,8 +122,6 @@ class DataSourceScanExecRedactionSuite extends DataSourceScanRedactionTest {
 
   test("SPARK-31793: FileSourceScanExec metadata should contain limited file paths") {
     withTempPath { path =>
-      val dir = path.getCanonicalPath
-
       // create a sub-directory with long name so that each root path will always exceed the limit
       // this is to ensure we always test the case for the path truncation
       val dataDirName = Random.alphanumeric.take(100).toList.mkString
@@ -155,7 +153,9 @@ class DataSourceScanExecRedactionSuite extends DataSourceScanRedactionTest {
         location.get.indexOf('[') + 1, location.get.indexOf(']')).split(", ").toSeq
 
       // the only one path should be available
-      assert(pathsInLocation.size == 1)
+      assert(pathsInLocation.size == 2)
+      // indicator ("... N more") should be available
+      assert(pathsInLocation.exists(_.contains("... ")))
     }
   }
 }