Merge branch 'master' into netty4

* master:
  Fix REST test documentation
  [Test] move methods from bwc test to test package for use in plugins (elastic#19738)
  package-info.java should be in src/main only.
  Split regular histograms from date histograms. elastic#19551
  Tighten up concurrent store metadata listing and engine writes (elastic#19684)
  Plugins: Make NamedWriteableRegistry immutable and add extenion point for named writeables
  Add documentation for the 'elasticsearch-translog' tool
  [TEST] Increase time waiting for all shards to move off/on to a node
  Fixes the active shard count check in the case of (elastic#19760)
  Fixes cat tasks operation in detailed mode
  ignore some docker craziness in scccomp environment checks

Author: jasontedor

TESTING.asciidoc

@@ -285,18 +285,18 @@ REST tests use the following command:
 
 ---------------------------------------------------------------------------
 gradle :distribution:integ-test-zip:integTest   \
-  -Dtests.class=org.elasticsearch.test.rest.RestIT
+  -Dtests.class=org.elasticsearch.test.rest.*Yaml*IT
 ---------------------------------------------------------------------------
 
 A specific test case can be run with
 
 ---------------------------------------------------------------------------
 gradle :distribution:integ-test-zip:integTest \
-  -Dtests.class=org.elasticsearch.test.rest.RestIT \
+  -Dtests.class=org.elasticsearch.test.rest.*Yaml*IT \
   -Dtests.method="test {p0=cat.shards/10_basic/Help}"
 ---------------------------------------------------------------------------
 
-`RestIT` are the executable test classes that runs all the
+`*Yaml*IT` are the executable test classes that runs all the
 yaml suites available within the `rest-api-spec` folder.
 
 The REST tests support all the options provided by the randomized runner, plus the following:

core/src/main/java/org/elasticsearch/action/support/ActiveShardCount.java

@@ -160,12 +160,15 @@ public boolean enoughShardsActive(final ClusterState clusterState, final String
      * to meet the required shard count represented by this instance.
      */
     public boolean enoughShardsActive(final IndexShardRoutingTable shardRoutingTable) {
+        final int activeShardCount = shardRoutingTable.activeShards().size();
         if (this == ActiveShardCount.ALL) {
-            return shardRoutingTable.allShardsStarted();
+            // adding 1 for the primary in addition to the total number of replicas,
+            // which gives us the total number of shard copies
+            return activeShardCount == shardRoutingTable.replicaShards().size() + 1;
         } else if (this == ActiveShardCount.DEFAULT) {
-            return shardRoutingTable.primaryShard().started();
+            return activeShardCount >= 1;
         } else {
-            return shardRoutingTable.activeShards().size() >= value;
+            return activeShardCount >= value;
         }
     }
 

core/src/main/java/org/elasticsearch/bootstrap/Seccomp.java

@@ -200,6 +200,7 @@ static SockFilter BPF_JUMP(int code, int k, int jt, int jf) {
     static final int SECCOMP_RET_ALLOW = 0x7FFF0000;
 
     // some errno constants for error checking/handling
+    static final int EPERM  = 0x01;
     static final int EACCES = 0x0D;
     static final int EFAULT = 0x0E;
     static final int EINVAL = 0x16;
@@ -275,10 +276,23 @@ private static int linuxImpl() {
 
         // check that unimplemented syscalls actually return ENOSYS
         // you never know (e.g. https://code.google.com/p/chromium/issues/detail?id=439795)
-        if (linux_syscall(999) >= 0 || Native.getLastError() != ENOSYS) {
+        if (linux_syscall(999) >= 0) {
             throw new UnsupportedOperationException("seccomp unavailable: your kernel is buggy and you should upgrade");
         }
 
+        switch (Native.getLastError()) {
+            case ENOSYS:
+                break; // ok
+            case EPERM:
+                // NOT ok, but likely a docker container
+                if (logger.isDebugEnabled()) {
+                    logger.debug("syscall(BOGUS) bogusly gets EPERM instead of ENOSYS");
+                }
+                break;
+            default:
+                throw new UnsupportedOperationException("seccomp unavailable: your kernel is buggy and you should upgrade");
+        }
+
         // try to check system calls really are who they claim
         // you never know (e.g. https://chromium.googlesource.com/chromium/src.git/+/master/sandbox/linux/seccomp-bpf/sandbox_bpf.cc#57)
         final int bogusArg = 0xf7a46a5c;

core/src/main/java/org/elasticsearch/client/transport/TransportClient.java

@@ -109,7 +109,6 @@ private static ClientTemplate buildTemplate(Settings providedSettings, Settings
         final ThreadPool threadPool = new ThreadPool(settings);
         resourcesToClose.add(() -> ThreadPool.terminate(threadPool, 10, TimeUnit.SECONDS));
         final NetworkService networkService = new NetworkService(settings, Collections.emptyList());
-        NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry();
         try {
             final List<Setting<?>> additionalSettings = new ArrayList<>();
             final List<String> additionalSettingsFilter = new ArrayList<>();
@@ -120,14 +119,21 @@ private static ClientTemplate buildTemplate(Settings providedSettings, Settings
             }
             SettingsModule settingsModule = new SettingsModule(settings, additionalSettings, additionalSettingsFilter);
 
+            NetworkModule networkModule = new NetworkModule(networkService, settings, true);
+            SearchModule searchModule = new SearchModule(settings, true, pluginsService.filterPlugins(SearchPlugin.class));
+            List<NamedWriteableRegistry.Entry> entries = new ArrayList<>();
+            entries.addAll(networkModule.getNamedWriteables());
+            entries.addAll(searchModule.getNamedWriteables());
+            NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(entries);
+
             ModulesBuilder modules = new ModulesBuilder();
             // plugin modules must be added here, before others or we can get crazy injection errors...
             for (Module pluginModule : pluginsService.createGuiceModules()) {
                 modules.add(pluginModule);
             }
-            modules.add(new NetworkModule(networkService, settings, true, namedWriteableRegistry));
+            modules.add(networkModule);
             modules.add(b -> b.bind(ThreadPool.class).toInstance(threadPool));
-            modules.add(new SearchModule(settings, namedWriteableRegistry, true, pluginsService.filterPlugins(SearchPlugin.class)));
+            modules.add(searchModule);
             ActionModule actionModule = new ActionModule(false, true, settings, null, settingsModule.getClusterSettings(),
                 pluginsService.filterPlugins(ActionPlugin.class));
             modules.add(actionModule);
@@ -143,6 +149,7 @@ private static ClientTemplate buildTemplate(Settings providedSettings, Settings
                 b.bind(BigArrays.class).toInstance(bigArrays);
                 b.bind(PluginsService.class).toInstance(pluginsService);
                 b.bind(CircuitBreakerService.class).toInstance(circuitBreakerService);
+                b.bind(NamedWriteableRegistry.class).toInstance(namedWriteableRegistry);
             }));
 
             Injector injector = modules.createInjector();

core/src/main/java/org/elasticsearch/common/geo/builders/ShapeBuilders.java

@@ -19,12 +19,11 @@
 
 package org.elasticsearch.common.geo.builders;
 
-import com.vividsolutions.jts.geom.Coordinate;
-
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-
 import java.util.List;
 
+import com.vividsolutions.jts.geom.Coordinate;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry.Entry;
+
 /**
  * A collection of static methods for creating ShapeBuilders.
  */
@@ -140,15 +139,15 @@ public static EnvelopeBuilder newEnvelope(Coordinate topLeft, Coordinate bottomR
         return new EnvelopeBuilder(topLeft, bottomRight);
     }
 
-    public static void register(NamedWriteableRegistry namedWriteableRegistry) {
-        namedWriteableRegistry.register(ShapeBuilder.class, PointBuilder.TYPE.shapeName(), PointBuilder::new);
-        namedWriteableRegistry.register(ShapeBuilder.class, CircleBuilder.TYPE.shapeName(), CircleBuilder::new);
-        namedWriteableRegistry.register(ShapeBuilder.class, EnvelopeBuilder.TYPE.shapeName(), EnvelopeBuilder::new);
-        namedWriteableRegistry.register(ShapeBuilder.class, MultiPointBuilder.TYPE.shapeName(), MultiPointBuilder::new);
-        namedWriteableRegistry.register(ShapeBuilder.class, LineStringBuilder.TYPE.shapeName(), LineStringBuilder::new);
-        namedWriteableRegistry.register(ShapeBuilder.class, MultiLineStringBuilder.TYPE.shapeName(), MultiLineStringBuilder::new);
-        namedWriteableRegistry.register(ShapeBuilder.class, PolygonBuilder.TYPE.shapeName(), PolygonBuilder::new);
-        namedWriteableRegistry.register(ShapeBuilder.class, MultiPolygonBuilder.TYPE.shapeName(), MultiPolygonBuilder::new);
-        namedWriteableRegistry.register(ShapeBuilder.class, GeometryCollectionBuilder.TYPE.shapeName(), GeometryCollectionBuilder::new);
+    public static void register(List<Entry> namedWriteables) {
+        namedWriteables.add(new Entry(ShapeBuilder.class, PointBuilder.TYPE.shapeName(), PointBuilder::new));
+        namedWriteables.add(new Entry(ShapeBuilder.class, CircleBuilder.TYPE.shapeName(), CircleBuilder::new));
+        namedWriteables.add(new Entry(ShapeBuilder.class, EnvelopeBuilder.TYPE.shapeName(), EnvelopeBuilder::new));
+        namedWriteables.add(new Entry(ShapeBuilder.class, MultiPointBuilder.TYPE.shapeName(), MultiPointBuilder::new));
+        namedWriteables.add(new Entry(ShapeBuilder.class, LineStringBuilder.TYPE.shapeName(), LineStringBuilder::new));
+        namedWriteables.add(new Entry(ShapeBuilder.class, MultiLineStringBuilder.TYPE.shapeName(), MultiLineStringBuilder::new));
+        namedWriteables.add(new Entry(ShapeBuilder.class, PolygonBuilder.TYPE.shapeName(), PolygonBuilder::new));
+        namedWriteables.add(new Entry(ShapeBuilder.class, MultiPolygonBuilder.TYPE.shapeName(), MultiPolygonBuilder::new));
+        namedWriteables.add(new Entry(ShapeBuilder.class, GeometryCollectionBuilder.TYPE.shapeName(), GeometryCollectionBuilder::new));
     }
 }

core/src/main/java/org/elasticsearch/common/io/stream/NamedWriteableRegistry.java

@@ -19,70 +19,102 @@
 
 package org.elasticsearch.common.io.stream;
 
+import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.elasticsearch.plugins.Plugin;
 
 /**
- * Registry for {@link NamedWriteable} objects. Allows to register and retrieve prototype instances of writeable objects
- * given their name.
+ * A registry for {@link org.elasticsearch.common.io.stream.Writeable.Reader} readers of {@link NamedWriteable}.
+ *
+ * The registration is keyed by the combination of the category class of {@link NamedWriteable}, and a name unique
+ * to that category.
  */
 public class NamedWriteableRegistry {
 
-    private final Map<Class<?>, InnerRegistry<?>> registry = new HashMap<>();
+    /** An entry in the registry, made up of a category class and name, and a reader for that category class. */
+    public static class Entry {
 
-    /**
-     * Register a {@link NamedWriteable} given its category, its name, and a function to read it from the stream.
-     *
-     * This method suppresses the rawtypes warning because it intentionally using NamedWriteable instead of {@code NamedWriteable<T>} so it
-     * is easier to use and because we might be able to drop the type parameter from NamedWriteable entirely some day.
-     */
-    public synchronized <T extends NamedWriteable> void register(Class<T> categoryClass, String name,
-            Writeable.Reader<? extends T> reader) {
-        @SuppressWarnings("unchecked")
-        InnerRegistry<T> innerRegistry = (InnerRegistry<T>) registry.get(categoryClass);
-        if (innerRegistry == null) {
-            innerRegistry = new InnerRegistry<>(categoryClass);
-            registry.put(categoryClass, innerRegistry);
+        /** The superclass of a {@link NamedWriteable} which will be read by {@link #reader}. */
+        public final Class<?> categoryClass;
+
+        /** A name for the writeable which is unique to the {@link #categoryClass}. */
+        public final String name;
+
+        /** A reader captability of reading*/
+        public final Writeable.Reader<?> reader;
+
+        /** Creates a new entry which can be stored by the registry. */
+        public <T extends NamedWriteable> Entry(Class<T> categoryClass, String name, Writeable.Reader<? extends T> reader) {
+            this.categoryClass = Objects.requireNonNull(categoryClass);
+            this.name = Objects.requireNonNull(name);
+            this.reader = Objects.requireNonNull(reader);
         }
-        innerRegistry.register(name, reader);
     }
 
     /**
-     * Returns a prototype of the {@link NamedWriteable} object identified by the name provided as argument and its category
+     * The underlying data of the registry maps from the category to an inner
+     * map of name unique to that category, to the actual reader.
      */
-    public synchronized <T> Writeable.Reader<? extends T> getReader(Class<T> categoryClass, String name) {
-        @SuppressWarnings("unchecked")
-        InnerRegistry<T> innerRegistry = (InnerRegistry<T>)registry.get(categoryClass);
-        if (innerRegistry == null) {
-            throw new IllegalArgumentException("unknown named writeable category [" + categoryClass.getName() + "]");
-        }
-        return innerRegistry.getReader(name);
-    }
-
-    private static class InnerRegistry<T> {
+    private final Map<Class<?>, Map<String, Writeable.Reader<?>>> registry;
 
-        private final Map<String, Writeable.Reader<? extends T>> registry = new HashMap<>();
-        private final Class<T> categoryClass;
-
-        private InnerRegistry(Class<T> categoryClass) {
-            this.categoryClass = categoryClass;
+    /**
+     * Constructs a new registry from the given entries.
+     */
+    public NamedWriteableRegistry(List<Entry> entries) {
+        if (entries.isEmpty()) {
+            registry = Collections.emptyMap();
+            return;
         }
+        entries = new ArrayList<>(entries);
+        entries.sort((e1, e2) -> e1.categoryClass.getName().compareTo(e2.categoryClass.getName()));
 
-        private void register(String name, Writeable.Reader<? extends T> reader) {
-            Writeable.Reader<? extends T> existingReader = registry.get(name);
-            if (existingReader != null) {
-                throw new IllegalArgumentException(
-                        "named writeable [" + categoryClass.getName() + "][" + name + "] is already registered by [" + reader + "]");
+        Map<Class<?>, Map<String, Writeable.Reader<?>>> registry = new HashMap<>();
+        Map<String, Writeable.Reader<?>> readers = null;
+        Class currentCategory = null;
+        for (Entry entry : entries) {
+            if (currentCategory != entry.categoryClass) {
+                if (currentCategory != null) {
+                    // we've seen the last of this category, put it into the big map
+                    registry.put(currentCategory, Collections.unmodifiableMap(readers));
+                }
+                readers = new HashMap<>();
+                currentCategory = entry.categoryClass;
             }
-            registry.put(name, reader);
-        }
 
-        private Writeable.Reader<? extends T> getReader(String name) {
-            Writeable.Reader<? extends T> reader = registry.get(name);
-            if (reader == null) {
-                throw new IllegalArgumentException("unknown named writeable [" + categoryClass.getName() + "][" + name + "]");
+            Writeable.Reader<?> oldReader = readers.put(entry.name, entry.reader);
+            if (oldReader != null) {
+                throw new IllegalArgumentException("NamedWriteable [" + currentCategory.getName() + "][" + entry.name + "]" +
+                    " is already registered for [" + oldReader.getClass().getName() + "]," +
+                    " cannot register [" + entry.reader.getClass().getName() + "]");
             }
-            return reader;
         }
+        // handle the last category
+        registry.put(currentCategory, Collections.unmodifiableMap(readers));
+
+        this.registry = Collections.unmodifiableMap(registry);
+    }
+
+    /**
+     * Returns a reader for a {@link NamedWriteable} object identified by the
+     * name provided as argument and its category.
+     */
+    public <T> Writeable.Reader<? extends T> getReader(Class<T> categoryClass, String name) {
+        Map<String, Writeable.Reader<?>> readers = registry.get(categoryClass);
+        if (readers == null) {
+            throw new IllegalArgumentException("Unknown NamedWriteable category [" + categoryClass.getName() + "]");
+        }
+        @SuppressWarnings("unchecked")
+        Writeable.Reader<? extends T> reader = (Writeable.Reader<? extends T>)readers.get(name);
+        if (reader == null) {
+            throw new IllegalArgumentException("Unknown NamedWriteable [" + categoryClass.getName() + "][" + name + "]");
+        }
+        return reader;
     }
 }