Merge pull request #178 from github/kpaulisse-split-tests

Split the puppet versions in hopes the tests don't timeout

Author: kpaulisse

.travis.yml

@@ -7,14 +7,22 @@ script: "script/cibuild"
 matrix:
   include:
     # Build with latest ruby
-    - rvm: 2.4
-      env: RUBOCOP_TEST="true" RSPEC_TEST="true"
+    - rvm: 2.5
+      env: RUBOCOP_TEST="true" RSPEC_TEST="true" PUPPET_VERSIONS="3.8.7"
+    - rvm: 2.5
+      env: RUBOCOP_TEST="false" RSPEC_TEST="true" PUPPET_VERSIONS="4.10.8"
+    - rvm: 2.5
+      env: RUBOCOP_TEST="false" RSPEC_TEST="true" PUPPET_VERSIONS="5.0.0"
     # Build with older ruby versions
+    - rvm: 2.4
+      env: RUBOCOP_TEST="false" RSPEC_TEST="true" PUPPET_VERSIONS="3.8.7"
     - rvm: 2.3.2
-      env: RUBOCOP_TEST="false" RSPEC_TEST="true"
+      env: RUBOCOP_TEST="false" RSPEC_TEST="true" PUPPET_VERSIONS="4.10.8"
     - rvm: 2.2.3
-      env: RUBOCOP_TEST="false" RSPEC_TEST="true"
+      env: RUBOCOP_TEST="false" RSPEC_TEST="true" PUPPET_VERSIONS="5.0.0"
     - rvm: 2.1
-      env: RUBOCOP_TEST="false" RSPEC_TEST="true"
+      env: RUBOCOP_TEST="false" RSPEC_TEST="true" PUPPET_VERSIONS="3.8.7"
+    - rvm: 2.0
+      env: RUBOCOP_TEST="false" RSPEC_TEST="true" PUPPET_VERSIONS="3.8.7"
     - rvm: 2.0
-      env: RUBOCOP_TEST="false" RSPEC_TEST="true"
+      env: RUBOCOP_TEST="false" RSPEC_TEST="true" PUPPET_VERSIONS="5.0.0"

lib/octocatalog-diff/util/util.rb

@@ -27,7 +27,9 @@ def self.object_is_any_of?(object, classes)
       def self.safe_dup(object)
         object.dup
       rescue TypeError
+        # :nocov:
         object
+        # :nocov:
       end
 
       # Utility Method!

script/cibuild

@@ -6,7 +6,11 @@
 # with one or more Puppet versions, with PUPPET_VERSIONS set to a space-separated list
 # of versions to test.
 
-[ -z "$PUPPET_VERSIONS" ] && export PUPPET_VERSIONS='3.8.7 4.10.8 5.0.0'
+if [ -z "$PUPPET_VERSIONS" ]; then
+  echo "Required PUPPET_VERSIONS!"
+  exit 255
+fi
+
 [ -z "$RUBOCOP_TEST" ] && export RUBOCOP_TEST='true'
 [ -z "$RSPEC_TEST" ] && export RSPEC_TEST='true'
 
@@ -19,9 +23,7 @@ env
 echo "travis_fold:end:cibuild-environment-dump"
 
 # Create a temporary file to capture output of various steps.
-export OUTPUT_FILE="$(mktemp)"
 function cleanup() {
-  rm -rf "$OUTPUT_FILE"
   rm -f "${DIR}/.ruby-version"
 }
 trap cleanup EXIT
@@ -95,12 +97,39 @@ if [ "$RSPEC_TEST" = "true" ]; then
     fi
 
     # Run the tests
-    echo "Running tests"
-    time bundle exec rake test
+    echo "Running rspec unit tests"
+    export COVERAGE=true
+    time bundle exec rspec "${DIR}/spec/octocatalog-diff/tests"
     exitcode=$?
+    unset COVERAGE
     if [ "$exitcode" -ne 0 ]; then RSPEC_EXITCODE="$exitcode"; fi
-    cat "$OUTPUT_FILE"
+
+    # Quick coverage report
+    "$DIR/script/display-coverage-report" "$DIR/coverage/coverage.json"
     echo ""
+
+    # To avoid travis getting hung if it gets confused, we'll run each of these
+    # scripts individually with a timeout. This will hopefully address the problem
+    # of hung builds.
+    echo "Running rspec integration tests"
+    for file in "${DIR}"/spec/octocatalog-diff/integration/*_spec.rb; do
+      retry=1
+      for try in 1 2 3 ; do
+        if [ $retry -eq 1 ]; then
+          retry=0
+          echo "$(basename "$file") try ${try}"
+          "$DIR/script/timeout" 180 bundle exec rspec "$file"
+          exitcode=$?
+          if [ $exitcode -eq 124 ] && [ $try -eq 3 ]; then
+            RSPEC_EXITCODE="255"
+          elif [ $exitcode -eq 124 ] && [ $try -lt 3 ]; then
+            retry=1
+          elif [ $exitcode -ne 0 ]; then
+            RSPEC_EXITCODE="$exitcode"
+          fi
+        fi
+      done
+    done
   done
   export PATH="$SAVED_PATH"
   unset PUPPET_VERSION

script/display-coverage-report

@@ -0,0 +1,51 @@
+#!/usr/bin/env ruby
+
+# Quick hacky script to parse the coverage JSON report and display it in human-readable terms.
+# Usage: display-coverage-report <JSON_FILE>
+
+require 'json'
+require 'pathname'
+
+# Arg handling and validation
+json_file = ARGV.first
+unless json_file
+  raise "Usage: #{__FILE__} <json_coverage_file>"
+end
+
+unless File.file?(json_file)
+  raise "Error: Provided file #{json_file} does not exist"
+end
+
+# Find files with < 100% coverage
+result = {}
+data = JSON.parse(File.read(json_file))
+data['files'].each do |data|
+  next if data['covered_percent'] == 100
+
+  result[data['filename']] = { covered: data['covered_percent'], lines: [] }
+  data['coverage'].each_with_index do |cov, index|
+    next if cov.nil?
+    result[data['filename']][:lines] << (index + 1) if cov == 0
+  end
+end
+
+# Display results
+if result.empty?
+  puts "100% Coverage - You're all set, friend! :sparkles:"
+  exit 0
+end
+
+puts ""
+puts "-------------------------------------------------------------"
+puts "Test coverage report"
+puts "-------------------------------------------------------------"
+
+basedir = Pathname.new(File.expand_path('..', File.dirname(__FILE__)))
+result.keys.sort.each do |filename|
+  data = result[filename]
+  relative_path = Pathname.new(filename).relative_path_from(basedir).to_s
+  printf "* %s: %0.02f%% (missed: %s)", relative_path, data[:covered], data[:lines].join(",")
+end
+
+puts ""
+exit 1

script/timeout

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+
+require 'shellwords'
+require 'timeout'
+
+seconds = ARGV.shift
+raise "Usage: #{__FILE__} <seconds> <command>" unless seconds
+begin
+  Timeout::timeout(seconds.to_i) do
+    system ARGV.map { |i| Shellwords.escape(i) }.join(" ")
+    exit $?.exitstatus
+  end
+rescue Timeout::Error
+  STDERR.puts "Timed out after #{seconds} seconds"
+  exit 124
+end