xds: support multiple xDS servers in bootstrap file (#6493)

Support bootstrap file containing multiple xDS servers, with each has its own server URI and channel credential options. Multiple xDS servers are provided in case of one not reachable. For now, we would only use the first one.

This change also formats JSON strings in bootstrap related tests and add several tests for parsing bootstrap JSON as completeness.

Implementation of XdsClient is changed to take in a list of xDS servers. But still, we only use the first one.

Author: voidzcy

xds/src/main/java/io/grpc/xds/Bootstrapper.java

@@ -73,28 +73,33 @@ static BootstrapInfo parseConfig(String rawData) throws IOException {
     @SuppressWarnings("unchecked")
     Map<String, ?> rawBootstrap = (Map<String, ?>) JsonParser.parse(rawData);
 
-    Map<String, ?> rawServerConfig = JsonUtil.getObject(rawBootstrap, "xds_server");
-    if (rawServerConfig == null) {
-      throw new IOException("Invalid bootstrap: 'xds_server' does not exist.");
+    List<ServerInfo> servers = new ArrayList<>();
+    List<?> rawServerConfigs = JsonUtil.getList(rawBootstrap, "xds_servers");
+    if (rawServerConfigs == null) {
+      throw new IOException("Invalid bootstrap: 'xds_servers' does not exist.");
     }
-    // Field "server_uri" is required.
-    String serverUri = JsonUtil.getString(rawServerConfig, "server_uri");
-    if (serverUri == null) {
-      throw new IOException("Invalid bootstrap: 'xds_server : server_uri' does not exist.");
-    }
-    List<ChannelCreds> channelCredsOptions = new ArrayList<>();
-    List<?> rawChannelCredsList = JsonUtil.getList(rawServerConfig, "channel_creds");
-    // List of channel creds is optional.
-    if (rawChannelCredsList != null) {
-      List<Map<String, ?>> channelCredsList = JsonUtil.checkObjectList(rawChannelCredsList);
-      for (Map<String, ?> channelCreds : channelCredsList) {
-        String type = JsonUtil.getString(channelCreds, "type");
-        if (type == null) {
-          throw new IOException("Invalid bootstrap: 'channel_creds' contains unknown type.");
+    List<Map<String, ?>> serverConfigList = JsonUtil.checkObjectList(rawServerConfigs);
+    for (Map<String, ?> serverConfig : serverConfigList) {
+      String serverUri = JsonUtil.getString(serverConfig, "server_uri");
+      if (serverUri == null) {
+        throw new IOException("Invalid bootstrap: 'xds_servers' contains unknown server.");
+      }
+      List<ChannelCreds> channelCredsOptions = new ArrayList<>();
+      List<?> rawChannelCredsList = JsonUtil.getList(serverConfig, "channel_creds");
+      // List of channel creds is optional.
+      if (rawChannelCredsList != null) {
+        List<Map<String, ?>> channelCredsList = JsonUtil.checkObjectList(rawChannelCredsList);
+        for (Map<String, ?> channelCreds : channelCredsList) {
+          String type = JsonUtil.getString(channelCreds, "type");
+          if (type == null) {
+            throw new IOException("Invalid bootstrap: 'xds_servers' contains server with "
+                + "unknown type 'channel_creds'.");
+          }
+          ChannelCreds creds = new ChannelCreds(type, JsonUtil.getObject(channelCreds, "config"));
+          channelCredsOptions.add(creds);
         }
-        ChannelCreds creds = new ChannelCreds(type, JsonUtil.getObject(channelCreds, "config"));
-        channelCredsOptions.add(creds);
       }
+      servers.add(new ServerInfo(serverUri, channelCredsOptions));
     }
 
     Node.Builder nodeBuilder = Node.newBuilder();
@@ -133,7 +138,7 @@ static BootstrapInfo parseConfig(String rawData) throws IOException {
     }
     nodeBuilder.setBuildVersion(GrpcUtil.getGrpcBuildVersion());
 
-    return new BootstrapInfo(serverUri, channelCredsOptions, nodeBuilder.build());
+    return new BootstrapInfo(servers, nodeBuilder.build());
   }
 
   /**
@@ -203,26 +208,48 @@ String getType() {
   }
 
   /**
-   * Data class containing the results of reading bootstrap.
+   * Data class containing xDS server information, such as server URI and channel credential
+   * options to be used for communication.
    */
   @Immutable
-  public static class BootstrapInfo {
+  static class ServerInfo {
     private final String serverUri;
     private final List<ChannelCreds> channelCredsList;
-    private final Node node;
 
     @VisibleForTesting
-    BootstrapInfo(String serverUri, List<ChannelCreds> channelCredsList, Node node) {
+    ServerInfo(String serverUri, List<ChannelCreds> channelCredsList) {
       this.serverUri = serverUri;
       this.channelCredsList = channelCredsList;
+    }
+
+    String getServerUri() {
+      return serverUri;
+    }
+
+    List<ChannelCreds> getChannelCredentials() {
+      return Collections.unmodifiableList(channelCredsList);
+    }
+  }
+
+  /**
+   * Data class containing the results of reading bootstrap.
+   */
+  @Immutable
+  public static class BootstrapInfo {
+    private List<ServerInfo> servers;
+    private final Node node;
+
+    @VisibleForTesting
+    BootstrapInfo(List<ServerInfo> servers, Node node) {
+      this.servers = servers;
       this.node = node;
     }
 
     /**
-     * Returns the URI the traffic director to be connected to.
+     * Returns the list of xDS servers to be connected to.
      */
-    String getServerUri() {
-      return serverUri;
+    List<ServerInfo> getServers() {
+      return Collections.unmodifiableList(servers);
     }
 
     /**
@@ -232,11 +259,5 @@ public Node getNode() {
       return node;
     }
 
-    /**
-     * Returns the credentials to use when communicating with the xDS server.
-     */
-    List<ChannelCreds> getChannelCredentials() {
-      return Collections.unmodifiableList(channelCredsList);
-    }
   }
 }

xds/src/main/java/io/grpc/xds/LookasideLb.java

@@ -41,6 +41,7 @@
 import io.grpc.util.GracefulSwitchLoadBalancer;
 import io.grpc.xds.Bootstrapper.BootstrapInfo;
 import io.grpc.xds.Bootstrapper.ChannelCreds;
+import io.grpc.xds.Bootstrapper.ServerInfo;
 import io.grpc.xds.EnvoyProtoData.DropOverload;
 import io.grpc.xds.EnvoyProtoData.Locality;
 import io.grpc.xds.EnvoyProtoData.LocalityLbEndpoints;
@@ -204,9 +205,21 @@ XdsClient createXdsClient() {
               new ErrorPicker(Status.UNAVAILABLE.withCause(e)));
           return;
         }
+
+        List<ServerInfo> serverList = bootstrapInfo.getServers();
+        if (serverList.isEmpty()) {
+          lookasideLbHelper.updateBalancingState(
+              TRANSIENT_FAILURE,
+              new ErrorPicker(
+                  Status.UNAVAILABLE
+                      .withDescription("No traffic director provided by bootstrap")));
+          return;
+        }
+        // Currently we only support using the first server from bootstrap.
+        ServerInfo serverInfo = serverList.get(0);
         channel = initLbChannel(
-            lookasideLbHelper, bootstrapInfo.getServerUri(),
-            bootstrapInfo.getChannelCredentials());
+            lookasideLbHelper, serverInfo.getServerUri(),
+            serverInfo.getChannelCredentials());
         xdsClientRef = new RefCountedXdsClientObjectPool(new XdsClientFactory() {
           @Override
           XdsClient createXdsClient() {

xds/src/main/java/io/grpc/xds/XdsClientImpl.java

@@ -49,6 +49,7 @@
 import io.grpc.internal.BackoffPolicy;
 import io.grpc.stub.StreamObserver;
 import io.grpc.xds.Bootstrapper.ChannelCreds;
+import io.grpc.xds.Bootstrapper.ServerInfo;
 import io.grpc.xds.EnvoyProtoData.DropOverload;
 import io.grpc.xds.EnvoyProtoData.Locality;
 import io.grpc.xds.EnvoyProtoData.LocalityLbEndpoints;
@@ -135,19 +136,14 @@ final class XdsClientImpl extends XdsClient {
   private String ldsResourceName;
 
   XdsClientImpl(
-      // URI of the management server to be connected to.
-      String serverUri,
+      List<ServerInfo> servers,  // list of management servers
       Node node,
-      // List of channel credential configurations for the channel to management server.
-      // Should pick the first supported one.
-      List<ChannelCreds> channelCredsList,
       SynchronizationContext syncContext,
       ScheduledExecutorService timeService,
       BackoffPolicy.Provider backoffPolicyProvider,
       Stopwatch stopwatch) {
     this(
-        buildChannel(checkNotNull(serverUri, "serverUri"),
-            checkNotNull(channelCredsList, "channelCredsList")),
+        buildChannel(checkNotNull(servers, "servers")),
         node,
         syncContext,
         timeService,
@@ -319,9 +315,15 @@ void cancelEndpointDataWatch(String clusterName, EndpointWatcher watcher) {
   }
 
   /**
-   * Builds a channel to the given server URI with the first supported channel creds config.
+   * Builds a channel to one of the provided management servers.
+   *
+   * <p>Note: currently we only support using the first server.
    */
-  private static ManagedChannel buildChannel(String serverUri,List<ChannelCreds> channelCredsList) {
+  private static ManagedChannel buildChannel(List<ServerInfo> servers) {
+    checkArgument(!servers.isEmpty(), "No management server provided.");
+    ServerInfo serverInfo = servers.get(0);
+    String serverUri = serverInfo.getServerUri();
+    List<ChannelCreds> channelCredsList = serverInfo.getChannelCredentials();
     ManagedChannel ch = null;
     // Use the first supported channel credentials configuration.
     // Currently, only "google_default" is supported.

xds/src/main/java/io/grpc/xds/XdsNameResolver.java

@@ -29,6 +29,7 @@
 import io.grpc.internal.JsonParser;
 import io.grpc.xds.Bootstrapper.BootstrapInfo;
 import io.grpc.xds.Bootstrapper.ChannelCreds;
+import io.grpc.xds.Bootstrapper.ServerInfo;
 import java.io.IOException;
 import java.net.URI;
 import java.util.Collections;
@@ -89,13 +90,26 @@ public void start(final Listener2 listener) {
       return;
     }
 
-    String serviceConfig = "{"
-        + "\"loadBalancingConfig\": ["
-        + "{\"xds_experimental\" : {"
-        + "\"balancerName\" : \"" + bootstrapInfo.getServerUri() + "\","
-        + "\"childPolicy\" : [{\"round_robin\" : {}}]"
-        + "}}"
-        + "]}";
+    List<ServerInfo> serverList = bootstrapInfo.getServers();
+    if (serverList.isEmpty()) {
+      listener.onError(
+          Status.UNAVAILABLE.withDescription("No traffic director provided by bootstrap"));
+      return;
+    }
+
+    // Currently we only support using the first server from bootstrap.
+    ServerInfo serverInfo = serverList.get(0);
+
+    String serviceConfig = "{\n"
+        + "  \"loadBalancingConfig\": [\n"
+        + "    {\n"
+        + "      \"xds_experimental\": {\n"
+        + "        \"balancerName\": \"" + serverInfo.getServerUri() + "\",\n"
+        + "        \"childPolicy\": [ {\"round_robin\": {} } ]\n"
+        + "      }\n"
+        + "    }"
+        + "  ]\n"
+        + "}";
     Map<String, ?> config;
     try {
       config = (Map<String, ?>) JsonParser.parse(serviceConfig);
@@ -108,7 +122,7 @@ public void start(final Listener2 listener) {
         Attributes.newBuilder()
             .set(GrpcAttributes.NAME_RESOLVER_SERVICE_CONFIG, config)
             .set(XDS_NODE, bootstrapInfo.getNode())
-            .set(XDS_CHANNEL_CREDS_LIST, bootstrapInfo.getChannelCredentials())
+            .set(XDS_CHANNEL_CREDS_LIST, serverInfo.getChannelCredentials())
             .build();
     ResolutionResult result =
         ResolutionResult.newBuilder()