Merge branch 'master' into cleanup/transport_bulk

Author: areek

.gitignore

@@ -20,6 +20,11 @@ nbactions.xml
 .gradle/
 build/
 
+# gradle wrapper 
+/gradle/
+gradlew
+gradlew.bat
+
 # maven stuff (to be removed when trunk becomes 4.x)
 *-execution-hints.log
 target/

CONTRIBUTING.md

@@ -120,7 +120,8 @@ Please follow these formatting guidelines:
 * The rest is left to Java coding standards
 * Disable “auto-format on save” to prevent unnecessary format changes. This makes reviews much harder as it generates unnecessary formatting changes. If your IDE supports formatting only modified chunks that is fine to do.
 * Wildcard imports (`import foo.bar.baz.*`) are forbidden and will cause the build to fail. Please attempt to tame your IDE so it doesn't make them and please send a PR against this document with instructions for your IDE if it doesn't contain them.
- * Eclipse: Preferences->Java->Code Style->Organize Imports. There are two boxes labeled "`Number of (static )? imports needed for .*`". Set their values to 99999 or some other absurdly high value.
+ * Eclipse: `Preferences->Java->Code Style->Organize Imports`. There are two boxes labeled "`Number of (static )? imports needed for .*`". Set their values to 99999 or some other absurdly high value.
+ * IntelliJ: `Preferences->Editor->Code Style->Java->Imports`. There are two configuration options: `Class count to use import with '*'` and `Names count to use static import with '*'`. Set their values to 99999 or some other absurdly high value.
 * Don't worry too much about import order. Try not to change it but don't worry about fighting your IDE to stop it from doing so.
 
 To create a distribution from the source, simply run:

GRADLE.CHEATSHEET

@@ -4,4 +4,4 @@ test -> test
 verify -> check
 verify -Dskip.unit.tests -> integTest
 package -DskipTests -> assemble
-install -DskipTests -> install
+install -DskipTests -> publishToMavenLocal

TESTING.asciidoc

@@ -364,10 +364,12 @@ These are the linux flavors the Vagrantfile currently supports:
 * ubuntu-1204 aka precise
 * ubuntu-1404 aka trusty
 * ubuntu-1504 aka vivid
+* ubuntu-1604 aka xenial
 * debian-8 aka jessie, the current debian stable distribution
 * centos-6
 * centos-7
-* fedora-22
+* fedora-24
+* oel-6 aka Oracle Enterprise Linux 6
 * oel-7 aka Oracle Enterprise Linux 7
 * sles-12
 * opensuse-13
@@ -376,7 +378,6 @@ We're missing the following from the support matrix because there aren't high
 quality boxes available in vagrant atlas:
 
 * sles-11
-* oel-6
 
 We're missing the follow because our tests are very linux/bash centric:
 

Vagrantfile

@@ -37,6 +37,13 @@ Vagrant.configure(2) do |config|
       [ -f /usr/share/java/jayatanaag.jar ] || install jayatana
     SHELL
   end
+  config.vm.define "ubuntu-1604" do |config|
+    config.vm.box = "elastic/ubuntu-16.04-x86_64"
+    ubuntu_common config, extra: <<-SHELL
+      # Install Jayatana so we can work around it being present.
+      [ -f /usr/share/java/jayatanaag.jar ] || install jayatana
+    SHELL
+  end
   # Wheezy's backports don't contain Openjdk 8 and the backflips required to
   # get the sun jdk on there just aren't worth it. We have jessie for testing
   # debian and it works fine.
@@ -78,8 +85,8 @@ Vagrant.configure(2) do |config|
   config.vm.synced_folder ".", "/vagrant", disabled: true
   config.vm.synced_folder ".", "/elasticsearch"
   config.vm.provider "virtualbox" do |v|
-    # Give the boxes 2GB so they can run our tests if they have to.
-    v.memory = 2048
+    # Give the boxes 3GB because Elasticsearch defaults to using 2GB
+    v.memory = 3072
   end
   if Vagrant.has_plugin?("vagrant-cachier")
     config.cache.scope = :box

benchmarks/build.gradle

@@ -57,6 +57,9 @@ dependencies {
 }
 
 compileJava.options.compilerArgs << "-Xlint:-cast,-deprecation,-rawtypes,-try,-unchecked"
+// enable the JMH's BenchmarkProcessor to generate the final benchmark classes
+// needs to be added separately otherwise Gradle will quote it and javac will fail
+compileJava.options.compilerArgs.addAll(["-processor", "org.openjdk.jmh.generators.BenchmarkProcessor"])
 compileTestJava.options.compilerArgs << "-Xlint:-cast,-deprecation,-rawtypes,-try,-unchecked"
 
 forbiddenApis {

benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/AllocationBenchmark.java

@@ -27,7 +27,6 @@
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
-import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.common.settings.Settings;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -160,11 +159,9 @@ private int toInt(String v) {
     public ClusterState measureAllocation() {
         ClusterState clusterState = initialClusterState;
         while (clusterState.getRoutingNodes().hasUnassignedShards()) {
-            RoutingAllocation.Result result = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes()
+            clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes()
                     .shardsWithState(ShardRoutingState.INITIALIZING));
-            clusterState = ClusterState.builder(clusterState).routingResult(result).build();
-            result = strategy.reroute(clusterState, "reroute");
-            clusterState = ClusterState.builder(clusterState).routingResult(result).build();
+            clusterState = strategy.reroute(clusterState, "reroute");
         }
         return clusterState;
     }

benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/Allocators.java

@@ -22,10 +22,10 @@
 import org.elasticsearch.cluster.ClusterModule;
 import org.elasticsearch.cluster.EmptyClusterInfoService;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
-import org.elasticsearch.cluster.routing.allocation.FailedRerouteAllocation;
+import org.elasticsearch.cluster.routing.allocation.FailedShard;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
-import org.elasticsearch.cluster.routing.allocation.StartedRerouteAllocation;
 import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator;
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
@@ -35,9 +35,9 @@
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.gateway.GatewayAllocator;
 
-import java.lang.reflect.Constructor;
 import java.lang.reflect.InvocationTargetException;
-import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 
@@ -50,12 +50,12 @@ protected NoopGatewayAllocator() {
         }
 
         @Override
-        public void applyStartedShards(StartedRerouteAllocation allocation) {
+        public void applyStartedShards(RoutingAllocation allocation, List<ShardRouting> startedShards) {
             // noop
         }
 
         @Override
-        public void applyFailedShards(FailedRerouteAllocation allocation) {
+        public void applyFailedShards(RoutingAllocation allocation, List<FailedShard> failedShards) {
             // noop
         }
 
@@ -72,7 +72,7 @@ private Allocators() {
 
     public static AllocationService createAllocationService(Settings settings) throws NoSuchMethodException, InstantiationException,
         IllegalAccessException, InvocationTargetException {
-        return createAllocationService(settings, new ClusterSettings(Settings.Builder.EMPTY_SETTINGS, ClusterSettings
+        return createAllocationService(settings, new ClusterSettings(Settings.EMPTY, ClusterSettings
             .BUILT_IN_CLUSTER_SETTINGS));
     }
 
@@ -85,19 +85,9 @@ public static AllocationService createAllocationService(Settings settings, Clust
 
     public static AllocationDeciders defaultAllocationDeciders(Settings settings, ClusterSettings clusterSettings) throws
         IllegalAccessException, InvocationTargetException, InstantiationException, NoSuchMethodException {
-        List<AllocationDecider> list = new ArrayList<>();
-        // Keep a deterministic order of allocation deciders for the benchmark
-        for (Class<? extends AllocationDecider> deciderClass : ClusterModule.DEFAULT_ALLOCATION_DECIDERS) {
-            try {
-                Constructor<? extends AllocationDecider> constructor = deciderClass.getConstructor(Settings.class, ClusterSettings
-                    .class);
-                list.add(constructor.newInstance(settings, clusterSettings));
-            } catch (NoSuchMethodException e) {
-                Constructor<? extends AllocationDecider> constructor = deciderClass.getConstructor(Settings.class);
-                list.add(constructor.newInstance(settings));
-            }
-        }
-        return new AllocationDeciders(settings, list.toArray(new AllocationDecider[0]));
+        Collection<AllocationDecider> deciders =
+            ClusterModule.createAllocationDeciders(settings, clusterSettings, Collections.emptyList());
+        return new AllocationDeciders(settings, deciders);
 
     }
 

benchmarks/src/main/resources/log4j.properties

@@ -0,0 +1,8 @@
+appender.console.type = Console
+appender.console.name = console
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%m%n
+
+# Do not log at all if it is not really critical - we're in a benchmark
+rootLogger.level = error
+rootLogger.appenderRef.console.ref = console