add quick mutation testing to travis

Author: tomato42

.travis.yml

@@ -6,9 +6,13 @@ language: python
 cache: pip
 before_cache:
   - rm -f $HOME/.cache/pip/log/debug.log
-# place the slowest (instrumental and py2.6) first
+# place the slowest (instrumental, mutation and py2.6) first
 matrix:
   include:
+      - python: 3.7
+        dist: xenial
+        sudo: true
+        env: MUTATION=yes
       - python: 2.7
         env: INSTRUMENTAL=yes
       - python: 2.6
@@ -74,6 +78,13 @@ install:
         travis_retry pip install -r build-requirements.txt;
       fi
   - if [[ $INSTRUMENTAL ]]; then travis_retry pip install instrumental; fi
+  - if [[ $MUTATION ]]; then travis_retry pip install cosmic-ray; fi
+  # temporary, until https://github.com/sixty-north/cosmic-ray/pull/491 is merged
+  - pushd $(mktemp -d)
+  - git clone --branch sampling --depth 1 https://github.com/tomato42/cosmic-ray.git
+  - cd cosmic-ray
+  - python setup.py install
+  - popd
   - pip list
 script:
   - if [[ $TOX_ENV ]]; then tox -e $TOX_ENV; fi
@@ -101,6 +112,25 @@ script:
       if [[ $INSTRUMENTAL && $TRAVIS_PULL_REQUEST != "false" ]]; then
           instrumental -f .instrumental.cov -s | python diff-instrumental.py --read .diff-instrumental --fail-under 70 --max-difference -0.1
       fi
+  # cosmic-ray (mutation testing) runs
+  - if [[ $MUTATION ]]; then cosmic-ray init cosmic-ray.toml session.sqlite; fi
+  - if [[ $MUTATION ]]; then cosmic-ray baseline --report session.sqlite; fi
+  - if [[ $MUTATION ]]; then cr-report --show-output session.baseline.sqlite; fi
+  - |
+      if [[ $MUTATION ]]; then
+          cosmic-ray exec session.sqlite &
+          COSMIC_PID=$!
+          # make sure that we output something every 10 minutes (otherwise travis will kill us)
+          while kill -s 0 $COSMIC_PID; do
+            sleep 300
+            cr-report session.sqlite | tail -n 3;
+          done &
+          REPORT_PID=$!
+          # kill exec after 20 minutes
+          (sleep $((60*20)); kill $COSMIC_PID) &
+      fi
+  - if [[ $MUTATION ]]; then wait $COSMIC_PID ; kill $REPORT_PID ; true; fi
+  - if [[ $MUTATION ]]; then cr-rate --estimate --fail-over 25 --confidence 99.9 session.sqlite; fi
 after_success:
-  - if [[ -z $INSTRUMENTAL ]]; then coveralls; fi
+  - if [[ -z $INSTRUMENTAL && -z $MUTATION ]]; then coveralls; fi
 

cosmic-ray.sh

@@ -7,3 +7,4 @@ cr-report --show-output session.baseline.sqlite
 cosmic-ray exec session.sqlite
 cr-report session.sqlite
 cr-html session.sqlite > session.html
+cr-rate --estimate --fail-over 25 --confidence 99.9 session.sqlite