New TravisCI setup, using build stages

.travis.yml

@@ -1,37 +1,89 @@
 sudo: false
 language: python
-python:
+conditions: v1
-- 2.7
-- 3.4
-- 3.5
-- 3.6
-- pypy
 
 # There are two accounts made specifically for Travis, and the passwords are really only encrypted for obscurity
-# The global env variables `client1_email`, `client1_password`, `client2_email`, `client2_password` and `group_id` are set on the Travis Settings page
+# The global env variables `client1_email`, `client1_password`, `client2_email`, `client2_password` and `group_id`
+# are set on the Travis Settings page
+
+# The tests are run with `Limit concurrent jobs = 1`, since the tests can't use the clients simultaneously
 
 install:
   - pip install -U -r requirements.txt
   - pip install -U -r dev-requirements.txt
 
-before_script:
-  - if [[ "$TRAVIS_PYTHON_VERSION" = "2.7" ]]; then export PYTEST_ADDOPTS='-m ""'; fi; # expensive tests (otherwise disabled in pytest.ini)
-  - if [[ "$TRAVIS_PULL_REQUEST" != false ]]; then export PYTEST_ADDOPTS='-m offline'; fi; # offline tests only
-
-script: python -m pytest || python -m pytest --lf; # Run failed tests twice
-
 cache:
   pip: true
   directories:
     - .pytest_cache
 
-deploy:
+jobs:
+  include:
+    # The tests are split into online and offline versions.
+    # The online tests are only run against the master branch.
+    # Because:
+    # Travis caching is per-branch and per-job, so even though we cache the Facebook sessions via. `.pytest_cache`
+    # and in `tests.utils.load_client`, we need 6 new sessions per branch. This is usually the point where Facebook
+    # starts complaining, and we have to manually fix it
+
+    - &test-online
+      if: (branch = master OR tag IS present) AND type != pull_request
+      stage: online tests
+      script: scripts/travis-online
+
+    # Run online tests in all the supported python versions
+      python: 2.7
+    - <<: *test-online
+      python: 3.4
+    - <<: *test-online
+      python: 3.5
+    - <<: *test-online
+      python: 3.6
+    - <<: *test-online
+      python: pypy
+
+    # Run the expensive tests, with the python version most likely to break, aka. 2
+    - <<: *test-online
+      # Only run if the commit message includes [ci all] or [all ci]
+      if: commit_message =~ /\[ci\s+all\]|\[all\s+ci\]/
+      python: 2.7
+      env: PYTEST_ADDOPTS='-m expensive'
+
+    - &test-offline
+      # Ideally, it'd be nice to run the offline tests in every build, but since we can't run jobs concurrently (yet),
+      # we'll disable them when they're not needed, and include them inside the online tests instead
+      if: not ((branch = master OR tag IS present) AND type != pull_request)
+      stage: offline tests
+      script: scripts/travis-offline
+      env: PYTEST_ADDOPTS='-m offline'
+
+    # Run offline tests in all the supported python versions
+      python: 2.7
+    - <<: *test-offline
+      python: 3.4
+    - <<: *test-offline
+      python: 3.5
+    - <<: *test-offline
+      python: 3.6
+    - <<: *test-offline
+      python: 3.6
+    - <<: *test-offline
+      python: pypy
+
+    # Deploy to PyPI
+    - &deploy
+      stage: deploy
+      if: branch = master AND tag IS present
+      install: skip
+      deploy:
         provider: pypi
         user: madsmtm
         password:
           secure: "VA0MLSrwIW/T2KjMwjLZCzrLHw8pJT6tAvb48t7qpBdm8x192hax61pz1TaBZoJvlzyBPFKvluftuclTc7yEFwzXe7Gjqgd/ODKZl/wXDr36hQ7BBOLPZujdwmWLvTzMh3eJZlvkgcLCzrvK3j2oW8cM/+FZeVi/5/FhVuJ4ofs="
         distributions: sdist bdist_wheel
         skip_existing: true
-  on:
+
-    branch: master
+    # We need the bdist_wheels from both Python 2 and 3
-    tags: true
+      python: 3.6
+    - <<: *deploy
+      python: 2.7

scripts/travis-offline

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -ex
+
+echo travis_fold:start:pytest
+
+python -m pytest -m offline
+
+echo travis_fold:end:pytest

scripts/travis-online

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -ex
+
+echo travis_fold:start:pytest
+
+if ! python -m pytest --tb=no --color=yes; then # --tb=no -> Don't print stack traces
+    echo 'Some tests failed. Rerunning them, since they can be kinda flaky.'
+    python -m pytest --last-failed --color=yes
+fi
+
+echo travis_fold:end:pytest