Do not let Yarn Shuffle Server retry its server port.

Author: SaintBacchus

network/yarn/src/main/java/org/apache/spark/network/yarn/YarnShuffleService.java

@@ -98,6 +98,13 @@ private boolean isAuthenticationEnabled() {
    */
   @Override
   protected void serviceInit(Configuration conf) {
+
+    // It's better to let the NodeManager get down rather than take a port retry
+    // when `spark.shuffle.service.port` has been conflicted during starting
+    // the Spark Yarn Shuffle Server, because the retry mechanism will make the
+    // inconsistency of shuffle port and also make client fail to find the port.
+    conf.setInt("spark.port.maxRetries", 0);
+
     TransportConf transportConf = new TransportConf(new HadoopConfigProvider(conf));
     // If authentication is enabled, set up the shuffle server to use a
     // special RPC handler that filters out unauthenticated fetch requests