[Build] Update build scripts to support downloading ES >= 6.3 (#799)

* Update build scripts to support downloading ES >= 6.3

* Update travis build target
* Update artifact downloads to pull oss and standard versions of Elasticsearch when version >= 6.3.0
* Update method to add users for ES > 6.3.0
* Remove redundant test for 6.2.4
* Use rake `12.2.1` for builds using jruby-1.7.x
* Tidy up, and fix secure integration tests
* Adds a DISTRIBUTION environment variable, which can be:
    legacy: OSS < 6.3
    legacyxpack: < 6.3 with xpack plugin
    oss: OSS >= 6.3
    default: standard distribution >= 6.3
* Removes the exclusion filter for `secure_integration` - this was stopping
  secure integration tests from running even with tags on
* Updates logstash branch in travis.yml to 6.4
* Update version of Gradle used to build es
  Later versions of Elasticsearch require a later version of Gradle
  in order to build
* Fix for flaky tests
  Build the client, and set the mocks on that client before calling
    register to avoid race conditions
* Remove unnecessary add-on from Travis build

Author: robbavey

.travis.yml

@@ -4,13 +4,15 @@ language: ruby
 cache: bundler
 env:
 - INTEGRATION=false
-- INTEGRATION=true ES_VERSION=1.7.6 TEST_DEBUG=true
-- INTEGRATION=true ES_VERSION=2.4.4 TEST_DEBUG=true
-- INTEGRATION=true ES_VERSION=5.6.9 TEST_DEBUG=true
-- INTEGRATION=true ES_VERSION=6.2.4 TEST_DEBUG=true
-- INTEGRATION=true ES_VERSION=master TEST_DEBUG=true
-- SECURE_INTEGRATION=true INTEGRATION=true ES_VERSION=5.6.9 TEST_DEBUG=true
-- SECURE_INTEGRATION=true INTEGRATION=true ES_VERSION=6.2.4 TEST_DEBUG=true
+- DISTRIBUTION=legacy INTEGRATION=true ES_VERSION=1.7.6 TEST_DEBUG=true
+- DISTRIBUTION=legacy INTEGRATION=true ES_VERSION=2.4.4 TEST_DEBUG=true
+- DISTRIBUTION=legacy INTEGRATION=true ES_VERSION=5.6.9 TEST_DEBUG=true
+- DISTRIBUTION=oss INTEGRATION=true ES_VERSION=6.4.2 TEST_DEBUG=true
+- DISTRIBUTION=default INTEGRATION=true ES_VERSION=6.4.2 TEST_DEBUG=true
+- DISTRIBUTION=default INTEGRATION=true ES_VERSION=master TEST_DEBUG=true
+- DISTRIBUTION=legacyxpack SECURE_INTEGRATION=true INTEGRATION=true ES_VERSION=5.6.9 TEST_DEBUG=true
+- DISTRIBUTION=default SECURE_INTEGRATION=true INTEGRATION=true ES_VERSION=6.4.2 TEST_DEBUG=true
+
 rvm:
 - jruby-1.7.25
 matrix:
@@ -20,16 +22,13 @@ matrix:
   - rvm: jruby-9.1.13.0
     env: LOGSTASH_BRANCH=6.x
   - rvm: jruby-9.1.13.0
-    env: LOGSTASH_BRANCH=6.2
+    env: LOGSTASH_BRANCH=6.4
   - rvm: jruby-1.7.27
     env: LOGSTASH_BRANCH=5.6
   allow_failures:
-  - env: INTEGRATION=true ES_VERSION=master TEST_DEBUG=true
+  - env: DISTRIBUTION=default INTEGRATION=true ES_VERSION=master TEST_DEBUG=true
   fast_finish: true
 install: true
 script: ci/build.sh
 jdk: oraclejdk8
-addons:
-  apt:
-    packages:
-    - oracle-java8-installer
+

Gemfile

@@ -9,3 +9,7 @@ if Dir.exist?(logstash_path) && use_logstash_source
   gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
   gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
 end
+
+if RUBY_VERSION == "1.9.3"
+  gem 'rake', '12.2.1'
+end

ci/run.sh

@@ -30,14 +30,18 @@ setup_es() {
   cp $BUILD_DIR/spec/fixtures/scripts/painless/* elasticsearch/config/scripts
 
   # If we're running with xpack SSL/Users enabled...
-  if [[ "$SECURE_INTEGRATION" == "true" ]]; then
+  # Note that 6.3.0 releases and above do not require an x-pack plugin install
+
+  if [[ "$DISTRIBUTION" == "legacyxpack" ]]; then
     if [[ "$xpack_download_url" == "" ]]; then
       yes y | elasticsearch/bin/elasticsearch-plugin install x-pack
     else
       curl -sL $xpack_download_url > elasticsearch/xpack.zip
       yes y | elasticsearch/bin/elasticsearch-plugin install file://$BUILD_DIR/elasticsearch/xpack.zip
     fi
+  fi
 
+  if [[ "$SECURE_INTEGRATION" == "true" ]]; then
     es_yml=elasticsearch/config/elasticsearch.yml
     cp -rv $BUILD_DIR/spec/fixtures/test_certs elasticsearch/config/test_certs
     echo "xpack.security.http.ssl.enabled: true" >> $es_yml
@@ -55,12 +59,17 @@ start_es() {
   es_url="http://localhost:9200"
   if [[ "$SECURE_INTEGRATION" == "true" ]]; then
     es_url="https://localhost:9200 -k"
-
-    # Needed for travis. On travis the `users` script will fail because it will first try and write
-    # to /etc/elasticsearch
-    export CONF_DIR=$BUILD_DIR/elasticsearch/config
-    elasticsearch/bin/x-pack/users useradd simpleuser -p abc123 -r superuser
-    elasticsearch/bin/x-pack/users useradd 'f@ncyuser' -p 'ab%12#' -r superuser
+  fi
+  # Needed for travis. On travis the `users` script will fail because it will first try and write
+  # to /etc/elasticsearch
+  export CONF_DIR=$BUILD_DIR/elasticsearch/config
+
+  if [[ "$DISTRIBUTION" == "default" ]]; then
+      elasticsearch/bin/elasticsearch-users useradd simpleuser -p abc123 -r superuser
+      elasticsearch/bin/elasticsearch-users useradd 'f@ncyuser' -p 'ab%12#' -r superuser
+  elif [[ "$DISTRIBUTION" == "legacyxpack" ]]; then
+      elasticsearch/bin/x-pack/users useradd simpleuser -p abc123 -r superuser
+      elasticsearch/bin/x-pack/users useradd 'f@ncyuser' -p 'ab%12#' -r superuser
   fi
 
   while ! curl --silent $es_url && [[ $count -ne 0 ]]; do
@@ -83,7 +92,7 @@ get_es_distribution_version() {
 # not here because this script runs in a different bash shell.
 download_gradle() {
   echo $PWD
-  local version="4.3"
+  local version="4.10"
   curl -sL https://services.gradle.org/distributions/gradle-$version-bin.zip > gradle.zip
   unzip -d . gradle.zip
   mv gradle-* gradle
@@ -123,12 +132,23 @@ else
   fi
 
   case "$ES_VERSION" in
-    6.*)
+    6.[0-2]*)
       setup_es https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-${ES_VERSION}.tar.gz https://artifacts.elastic.co/downloads/packs/x-pack/x-pack-${ES_VERSION}.zip
       es_distribution_version=$(get_es_distribution_version)
       start_es
       bundle exec rspec -fd $extra_tag_args --tag update_tests:painless --tag update_tests:groovy --tag es_version:$es_distribution_version $spec_path
       ;;
+    6.*)
+      if [[ "$DISTRIBUTION" == "oss" ]]; then
+        setup_es https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-oss-${ES_VERSION}.tar.gz
+      elif [[ "$DISTRIBUTION" == "default" ]]; then
+        setup_es https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-${ES_VERSION}.tar.gz
+      fi
+      es_distribution_version=$(get_es_distribution_version)
+      start_es
+      bundle exec rspec -fd $extra_tag_args --tag update_tests:painless --tag update_tests:groovy --tag es_version:$es_distribution_version $spec_path
+      ;;
+
     5.*)
       setup_es https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-${ES_VERSION}.tar.gz
       es_distribution_version=$(get_es_distribution_version)

spec/es_spec_helper.rb

@@ -2,12 +2,6 @@
 require 'manticore'
 require 'elasticsearch'
 
-# by default exclude secure_integration tests unless requested
-# normal integration specs are already excluded by devutils' spec helper
-RSpec.configure do |config|
-   config.filter_run_excluding config.exclusion_filter.add(:secure_integration => true)
-end
-
 module ESHelper
   def get_host_port
     "127.0.0.1:9200"

spec/unit/outputs/elasticsearch_spec.rb

@@ -11,13 +11,15 @@
 
   before(:each) do
     if do_register
-      subject.register
+      # Build the client and set mocks before calling register to avoid races.
+      subject.build_client
 
       # Rspec mocks can't handle background threads, so... we can't use any
       allow(subject.client.pool).to receive(:start_resurrectionist)
       allow(subject.client.pool).to receive(:start_sniffer)
       allow(subject.client.pool).to receive(:healthcheck!)
       allow(subject.client).to receive(:maximum_seen_major_version).at_least(:once).and_return(maximum_seen_major_version)
+      subject.register
       subject.client.pool.adapter.manticore.respond_with(:body => "{}")
     end
   end