Upgrade Azure SDK

modules/repository-azure/build.gradle

@@ -23,50 +23,47 @@ esplugin {
 }
 
 versions << [
-  'azure': '12.9.0',
-  'azureCore': '1.10.0',
-  'azureCoreHttpNetty': '1.6.3',
+  'azure': '12.14.4',
+  'azureCommon': '12.14.3',
+  'azureCore': '1.25.0',
+  'azureCoreHttpNetty': '1.11.7',
+  'azureJackson': '2.13.2',
+  'azureJacksonDatabind': '2.13.2.2',
 
   'jakartaActivation': '1.2.1',
   'jakartaXMLBind': '2.3.2',
-  'stax2API': '4.2',
-  'woodstox': '6.0.2',
+  'stax2API': '4.2.1',
+  'woodstox': '6.2.7',
 
-  'reactorNetty': '0.9.12.RELEASE',
-  'reactorCore': '3.3.10.RELEASE',
+  'reactorNetty': '1.0.14',
+  'reactorCore': '3.4.13',
   'reactiveStreams': '1.0.3',
 ]
 
 dependencies {
-  api project(path: 'azure-storage-blob', configuration: 'shadow')
-  if (isEclipse) {
-    /*
-     * Eclipse can't pick up the shadow dependency so we point it at *something*
-     * so it can compile things.
-     */
-    api project(path: 'azure-storage-blob')
-  }
-  api "com.azure:azure-storage-common:${versions.azure}"
+  api "com.azure:azure-storage-blob:${versions.azure}"
+  api "com.azure:azure-storage-common:${versions.azureCommon}"
   api "com.azure:azure-core-http-netty:${versions.azureCoreHttpNetty}"
   api "com.azure:azure-core:${versions.azureCore}"
 
   // jackson
-  api "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
-  api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}"
-  api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
-  api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}"
+  api "com.fasterxml.jackson.core:jackson-core:${versions.azureJackson}"
+  api "com.fasterxml.jackson.core:jackson-databind:${versions.azureJacksonDatabind}"
+  api "com.fasterxml.jackson.core:jackson-annotations:${versions.azureJackson}"
 
   // jackson xml
-  api "com.fasterxml.jackson.dataformat:jackson-dataformat-xml:${versions.jackson}"
-  api "com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${versions.jackson}"
-  api "com.fasterxml.jackson.module:jackson-module-jaxb-annotations:${versions.jackson}"
+  api "com.fasterxml.jackson.dataformat:jackson-dataformat-xml:${versions.azureJackson}"
+  api "com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${versions.azureJackson}"
+  api "com.fasterxml.jackson.module:jackson-module-jaxb-annotations:${versions.azureJackson}"
   api "jakarta.activation:jakarta.activation-api:${versions.jakartaActivation}"
+  // The SDK uses javax.xml bindings
   api "jakarta.xml.bind:jakarta.xml.bind-api:${versions.jakartaXMLBind}"
   api "org.codehaus.woodstox:stax2-api:${versions.stax2API}"
   api "com.fasterxml.woodstox:woodstox-core:${versions.woodstox}"
 
   // netty
   api "io.netty:netty-buffer:${versions.netty}"
+  api "io.netty:netty-codec-dns:${versions.netty}"
   api "io.netty:netty-codec-http:${versions.netty}"
   api "io.netty:netty-codec-http2:${versions.netty}"
   api "io.netty:netty-codec-socks:${versions.netty}"
@@ -75,10 +72,13 @@ dependencies {
   api "io.netty:netty-handler-proxy:${versions.netty}"
   api "io.netty:netty-handler:${versions.netty}"
   api "io.netty:netty-resolver:${versions.netty}"
+  api "io.netty:netty-resolver-dns:${versions.netty}"
   api "io.netty:netty-transport:${versions.netty}"
+  api "io.netty:netty-transport-native-unix-common:${versions.netty}"
 
   // reactor
-  api "io.projectreactor.netty:reactor-netty:${versions.reactorNetty}"
+  api "io.projectreactor.netty:reactor-netty-core:${versions.reactorNetty}"
+  api "io.projectreactor.netty:reactor-netty-http:${versions.reactorNetty}"
   api "io.projectreactor:reactor-core:${versions.reactorCore}"
   api "org.reactivestreams:reactive-streams:${versions.reactiveStreams}"
 
@@ -110,6 +110,7 @@ tasks.named("dependencyLicenses").configure {
   mapping from: /netty-.*/, to: 'netty'
   mapping from: /jaxb-.*/, to: 'jaxb'
   mapping from: /stax-.*/, to: 'stax'
+  mapping from: /reactor-netty-.*/, to: 'reactor-netty'
   mapping from: /reactive-streams.*/, to: 'reactive-streams'
 }
 
@@ -122,6 +123,11 @@ tasks.named("thirdPartyAudit").configure {
     'com.aayushatharva.brotli4j.encoder.Encoder$Mode',
     'com.aayushatharva.brotli4j.encoder.Encoder$Parameters',
 
+    'com.azure.storage.internal.avro.implementation.AvroObject',
+    'com.azure.storage.internal.avro.implementation.AvroReader',
+    'com.azure.storage.internal.avro.implementation.AvroReaderFactory',
+    'com.azure.storage.internal.avro.implementation.schema.AvroSchema',
+
     // from reactory-netty metric collection
     'io.micrometer.core.instrument.Clock',
     'io.micrometer.core.instrument.Counter',
@@ -255,11 +261,17 @@ tasks.named("thirdPartyAudit").configure {
     'io.netty.channel.kqueue.KQueueDatagramChannel',
     'io.netty.channel.kqueue.KQueueServerSocketChannel',
     'io.netty.channel.kqueue.KQueueSocketChannel',
+
     'io.netty.channel.epoll.Epoll',
     'io.netty.channel.epoll.EpollDatagramChannel',
     'io.netty.channel.epoll.EpollServerSocketChannel',
     'io.netty.channel.epoll.EpollSocketChannel',
 
+    'io.netty.incubator.channel.uring.IOUring',
+    'io.netty.incubator.channel.uring.IOUringDatagramChannel',
+    'io.netty.incubator.channel.uring.IOUringServerSocketChannel',
+    'io.netty.incubator.channel.uring.IOUringSocketChannel',
+
     // from reactor.netty.http.server.HttpServer (reactor-netty)
     'io.netty.handler.codec.haproxy.HAProxyMessage',
     'io.netty.handler.codec.haproxy.HAProxyMessageDecoder',
@@ -269,8 +281,7 @@ tasks.named("thirdPartyAudit").configure {
     'org.osgi.framework.BundleContext',
 
     // from com.ctc.wstx.shaded.msv_core.driver.textui.Driver (woodstox-core)
-    'com.sun.org.apache.xml.internal.resolver.Catalog',
-    'com.sun.org.apache.xml.internal.resolver.tools.CatalogResolver',
+    'com.ctc.wstx.shaded.msv_core.driver.textui.Driver',
     // [missing classes] SLF4j includes an optional class that depends on an extension class. see Log4jLogger#createConverter
     'org.slf4j.ext.EventData'
   )
@@ -297,12 +308,7 @@ tasks.named("thirdPartyAudit").configure {
 
     'javax.activation.MailcapCommandMap',
     'javax.activation.MimetypesFileTypeMap',
-
-    'reactor.core.publisher.MultiProducerRingBuffer',
-    'reactor.core.publisher.RingBufferFields',
     'reactor.core.publisher.Traces$SharedSecretsCallSiteSupplierFactory$TracingException',
-    'reactor.core.publisher.UnsafeSequence',
-    'reactor.core.publisher.UnsafeSupport'
   )
 }
 boolean useFixture = false

modules/repository-azure/licenses/azure-core-1.25.0.jar.sha1

@@ -0,0 +1 @@
+85db86803538e6d01051ae9ee21f56684ea9995b

modules/repository-azure/licenses/azure-core-http-netty-1.11.7.jar.sha1

@@ -0,0 +1 @@
+c6b14fcca3e75acc8dbe07ac101afd05d48a1647

modules/repository-azure/licenses/azure-storage-blob-12.14.4.jar.sha1

@@ -0,0 +1 @@
+2b92020693d09e4980b96d278e8038a1087afea0

modules/repository-azure/licenses/azure-storage-common-12.14.3.jar.sha1

@@ -0,0 +1 @@
+e8d6258aa8bf1594980c01294e60de74d13a815f

modules/repository-azure/licenses/jackson-annotations-2.13.2.jar.sha1

@@ -0,0 +1 @@
+ec18851f1976d5b810ae1a5fcc32520d2d38f77a

modules/repository-azure/licenses/jackson-core-2.13.2.jar.sha1

@@ -0,0 +1 @@
+0a6a0e0620d51833feffc67bccb51937b2345763

modules/repository-azure/licenses/jackson-databind-2.13.2.2.jar.sha1

@@ -0,0 +1 @@
+ffeb635597d093509f33e1e94274d14be610f933

modules/repository-azure/licenses/jackson-dataformat-xml-2.13.2.jar.sha1

@@ -0,0 +1 @@
+cb6a722f128ff0ce2494384d419b6ff20fad25ab

modules/repository-azure/licenses/jackson-datatype-jsr310-2.13.2.jar.sha1

@@ -0,0 +1 @@
+cddd9380efd4b81ea01e98be8fbdc9765a81793b

modules/repository-azure/licenses/jackson-module-jaxb-annotations-2.13.2.jar.sha1

@@ -0,0 +1 @@
+e2f198c512f0f0ccbd6d618baecc9dde9975eadf

modules/repository-azure/licenses/netty-codec-dns-4.1.74.Final.jar.sha1

@@ -0,0 +1 @@
+2f8d808c01eb3a49435c8c8e9b3b47d0d616a75c

modules/repository-azure/licenses/netty-resolver-dns-4.1.74.Final.jar.sha1

@@ -0,0 +1 @@
+8ec11e228f827dbccf79573a18c4831785aab8be

modules/repository-azure/licenses/netty-transport-native-unix-common-4.1.74.Final.jar.sha1

@@ -0,0 +1 @@
+6b173e3fde38290859446db7889422e6ff0980d9

modules/repository-azure/licenses/reactor-core-3.4.13.jar.sha1

@@ -0,0 +1 @@
+7066cb27d2eee153b07b10647598cec75e817d2d

modules/repository-azure/licenses/reactor-netty-core-1.0.14.jar.sha1

@@ -0,0 +1 @@
+02e945beb0e38e2f8ea2da0ffa09e845ba74d5df

modules/repository-azure/licenses/reactor-netty-http-1.0.14.jar.sha1

@@ -0,0 +1 @@
+bca6b7b9a183e3fb7c223ff3a18c1ea94ca318d6

modules/repository-azure/licenses/stax2-api-4.2.1.jar.sha1

@@ -0,0 +1 @@
+a3f7325c52240418c2ba257b103c3c550e140c83

modules/repository-azure/licenses/woodstox-core-6.2.7.jar.sha1

@@ -0,0 +1 @@
+86622cfd0a9933628b6b876d0c92589148d3b42e

modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java

@@ -93,12 +93,15 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
         final MockSecureSettings secureSettings = new MockSecureSettings();
         String accountName = DEFAULT_ACCOUNT_NAME;
         secureSettings.setString(AzureStorageSettings.ACCOUNT_SETTING.getConcreteSettingForNamespace("test").getKey(), accountName);
-        secureSettings.setString(
-            (randomBoolean() ? AzureStorageSettings.KEY_SETTING : AzureStorageSettings.SAS_TOKEN_SETTING).getConcreteSettingForNamespace(
-                "test"
-            ).getKey(),
-            key
-        );
+        if (randomBoolean()) {
+            secureSettings.setString(AzureStorageSettings.KEY_SETTING.getConcreteSettingForNamespace("test").getKey(), key);
+        } else {
+            // The SDK expects a valid SAS TOKEN
+            secureSettings.setString(
+                AzureStorageSettings.SAS_TOKEN_SETTING.getConcreteSettingForNamespace("test").getKey(),
+                "se=2021-07-20T13%3A21Z&sp=rwdl&sv=2018-11-09&sr=c&sig=random"
+            );
+        }
 
         // see com.azure.storage.blob.BlobUrlParts.parseIpUrl
         final String endpoint = "ignored;DefaultEndpointsProtocol=http;BlobEndpoint=" + httpServerUrl() + "/" + accountName;

modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureClientProvider.java

@@ -13,10 +13,12 @@
 import io.netty.channel.ChannelOption;
 import io.netty.channel.EventLoopGroup;
 import io.netty.channel.nio.NioEventLoopGroup;
+import io.netty.resolver.DefaultAddressResolverGroup;
 import reactor.core.publisher.Mono;
 import reactor.core.scheduler.Scheduler;
 import reactor.core.scheduler.Schedulers;
 import reactor.netty.resources.ConnectionProvider;
+import reactor.netty.resources.LoopResources;
 
 import com.azure.core.http.HttpClient;
 import com.azure.core.http.HttpMethod;
@@ -94,6 +96,7 @@ class AzureClientProvider extends AbstractLifecycleComponent {
     private final ConnectionProvider connectionProvider;
     private final ByteBufAllocator byteBufAllocator;
     private final ClientLogger clientLogger = new ClientLogger(AzureClientProvider.class);
+    private final LoopResources nioLoopResources;
     private volatile boolean closed = false;
 
     AzureClientProvider(
@@ -108,6 +111,10 @@ class AzureClientProvider extends AbstractLifecycleComponent {
         this.eventLoopGroup = eventLoopGroup;
         this.connectionProvider = connectionProvider;
         this.byteBufAllocator = byteBufAllocator;
+        // The underlying http client uses this as part of the connection pool key,
+        // hence we need to use the same instance across all the client instances
+        // to avoid creating multiple connection pools.
+        this.nioLoopResources = useNative -> eventLoopGroup;
     }
 
     static int eventLoopThreadsFromSettings(Settings settings) {
@@ -164,13 +171,11 @@ AzureBlobServiceClient createClient(
         }
 
         reactor.netty.http.client.HttpClient nettyHttpClient = reactor.netty.http.client.HttpClient.create(connectionProvider);
-        nettyHttpClient = nettyHttpClient.port(80).wiretap(false);
-
-        nettyHttpClient = nettyHttpClient.tcpConfiguration(tcpClient -> {
-            tcpClient = tcpClient.runOn(eventLoopGroup);
-            tcpClient = tcpClient.option(ChannelOption.ALLOCATOR, byteBufAllocator);
-            return tcpClient;
-        });
+        nettyHttpClient = nettyHttpClient.port(80)
+            .wiretap(false)
+            .resolver(DefaultAddressResolverGroup.INSTANCE)
+            .runOn(nioLoopResources)
+            .option(ChannelOption.ALLOCATOR, byteBufAllocator);
 
         final HttpClient httpClient = new NettyAsyncHttpClientBuilder(nettyHttpClient).disableBufferCopy(true).proxy(proxyOptions).build();