Add generic interface for loading service providers from plugins (#88082)

Relates to 86224

Author: grcevski

server/src/main/java/org/elasticsearch/plugins/PluginsService.java

@@ -52,6 +52,7 @@
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
+import java.util.ServiceLoader;
 import java.util.Set;
 import java.util.function.Consumer;
 import java.util.function.Function;
@@ -279,7 +280,6 @@ private Map<String, LoadedPlugin> loadBundles(Set<PluginBundle> bundles) {
 
     // package-private for test visibility
     static void loadExtensions(Collection<LoadedPlugin> plugins) {
-
         Map<String, List<Plugin>> extendingPluginsByName = plugins.stream()
             .flatMap(t -> t.descriptor().getExtendedPlugins().stream().map(extendedPlugin -> Tuple.tuple(extendedPlugin, t.instance())))
             .collect(Collectors.groupingBy(Tuple::v1, Collectors.mapping(Tuple::v2, Collectors.toList())));
@@ -293,6 +293,28 @@ static void loadExtensions(Collection<LoadedPlugin> plugins) {
         }
     }
 
+    /**
+     * SPI convenience method that uses the {@link ServiceLoader} JDK class to load various SPI providers
+     * from plugins/modules.
+     * <p>
+     * For example:
+     *
+     * <pre>
+     * var pluginHandlers = pluginsService.loadServiceProviders(OperatorHandlerProvider.class);
+     * </pre>
+     * @param service A templated service class to look for providers in plugins
+     * @return an immutable {@link List} of discovered providers in the plugins/modules
+     */
+    public <T> List<? extends T> loadServiceProviders(Class<T> service) {
+        List<T> result = new ArrayList<>();
+
+        for (LoadedPlugin pluginTuple : plugins()) {
+            ServiceLoader.load(service, pluginTuple.loader()).iterator().forEachRemaining(c -> result.add(c));
+        }
+
+        return Collections.unmodifiableList(result);
+    }
+
     private static void loadExtensionsForPlugin(ExtensiblePlugin extensiblePlugin, List<Plugin> extendingPlugins) {
         ExtensiblePlugin.ExtensionLoader extensionLoader = new ExtensiblePlugin.ExtensionLoader() {
             @Override

server/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java

@@ -12,23 +12,32 @@
 import org.apache.lucene.util.Constants;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.TestEnvironment;
 import org.elasticsearch.index.IndexModule;
+import org.elasticsearch.plugins.spi.TestService;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.compiler.InMemoryJavaCompiler;
+import org.elasticsearch.test.jar.JarUtils;
 
 import java.io.IOException;
 import java.io.InputStream;
 import java.lang.reflect.InvocationTargetException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.FileSystemException;
 import java.nio.file.Files;
 import java.nio.file.NoSuchFileException;
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
+import java.util.Map;
 import java.util.Set;
 
 import static org.hamcrest.Matchers.containsInAnyOrder;
@@ -613,6 +622,70 @@ public void testThrowingConstructor() {
         assertThat(e.getCause().getCause(), hasToString(containsString("test constructor failure")));
     }
 
+    private ClassLoader buildTestProviderPlugin(String name) throws Exception {
+        Map<String, CharSequence> sources = Map.of("r.FooPlugin", """
+            package r;
+            import org.elasticsearch.plugins.ActionPlugin;
+            import org.elasticsearch.plugins.Plugin;
+            public final class FooPlugin extends Plugin implements ActionPlugin { }
+            """, "r.FooTestService", Strings.format("""
+            package r;
+            import org.elasticsearch.plugins.spi.TestService;
+            public final class FooTestService implements TestService {
+                @Override
+                public String name() {
+                    return "%s";
+                }
+            }
+            """, name));
+
+        var classToBytes = InMemoryJavaCompiler.compile(sources);
+
+        Map<String, byte[]> jarEntries = new HashMap<>();
+        jarEntries.put("r/FooPlugin.class", classToBytes.get("r.FooPlugin"));
+        jarEntries.put("r/FooTestService.class", classToBytes.get("r.FooTestService"));
+        jarEntries.put("META-INF/services/org.elasticsearch.plugins.spi.TestService", "r.FooTestService".getBytes(StandardCharsets.UTF_8));
+
+        Path topLevelDir = createTempDir(getTestName());
+        Path jar = topLevelDir.resolve(Strings.format("custom_plugin_%s.jar", name));
+        JarUtils.createJarWithEntries(jar, jarEntries);
+        URL[] urls = new URL[] { jar.toUri().toURL() };
+
+        URLClassLoader loader = URLClassLoader.newInstance(urls, this.getClass().getClassLoader());
+        return loader;
+    }
+
+    public void testLoadServiceProviders() throws Exception {
+        ClassLoader fakeClassLoader = buildTestProviderPlugin("integer");
+        @SuppressWarnings("unchecked")
+        Class<? extends Plugin> fakePluginClass = (Class<? extends Plugin>) fakeClassLoader.loadClass("r.FooPlugin");
+
+        ClassLoader fakeClassLoader1 = buildTestProviderPlugin("string");
+        @SuppressWarnings("unchecked")
+        Class<? extends Plugin> fakePluginClass1 = (Class<? extends Plugin>) fakeClassLoader1.loadClass("r.FooPlugin");
+
+        assertFalse(fakePluginClass.getClassLoader().equals(fakePluginClass1.getClassLoader()));
+
+        getClass().getModule().addUses(TestService.class);
+
+        PluginsService service = newMockPluginsService(List.of(fakePluginClass, fakePluginClass1));
+
+        List<? extends TestService> providers = service.loadServiceProviders(TestService.class);
+        assertEquals(2, providers.size());
+        assertThat(providers.stream().map(p -> p.name()).toList(), containsInAnyOrder("string", "integer"));
+
+        service = newMockPluginsService(List.of(fakePluginClass));
+        providers = service.loadServiceProviders(TestService.class);
+
+        assertEquals(1, providers.size());
+        assertThat(providers.stream().map(p -> p.name()).toList(), containsInAnyOrder("integer"));
+
+        service = newMockPluginsService(new ArrayList<>());
+        providers = service.loadServiceProviders(TestService.class);
+
+        assertEquals(0, providers.size());
+    }
+
     private static class TestExtensiblePlugin extends Plugin implements ExtensiblePlugin {
         private List<TestExtensionPoint> extensions;
 

server/src/test/java/org/elasticsearch/plugins/spi/TestService.java

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.plugins.spi;
+
+public interface TestService {
+    String name();
+}

test/framework/src/main/java/org/elasticsearch/plugins/MockPluginsService.java

@@ -58,7 +58,7 @@ public MockPluginsService(Settings settings, Environment environment, Collection
             if (logger.isTraceEnabled()) {
                 logger.trace("plugin loaded from classpath [{}]", pluginInfo);
             }
-            pluginsLoaded.add(new LoadedPlugin(pluginInfo, plugin));
+            pluginsLoaded.add(new LoadedPlugin(pluginInfo, plugin, pluginClass.getClassLoader(), ModuleLayer.boot()));
         }
 
         this.classpathPlugins = List.copyOf(pluginsLoaded);