Do not run tutorial tests by default (#668)

This turns off running tests of tutorials by default.  Tutorial tests are run by calling `pytest --run-tutorials`.

Testing of tutorials remains on in the GitHub action, currently for all pushes.

---------

Co-authored-by: Theodore Turocy <ted.turocy@gmail.com>

Author: edwardchalstrey1

.github/workflows/python.yml

@@ -32,7 +32,7 @@ jobs:
           cd dist
           pip install -v pygambit*.tar.gz
       - name: Run tests
-        run: pytest
+        run: pytest --run-tutorials
 
   macos-13:
     runs-on: macos-13
@@ -52,7 +52,6 @@ jobs:
           python -m pip install --upgrade pip
           pip install setuptools build cython wheel
           pip install -r tests/requirements.txt
-          pip install -r doc/requirements.txt
       - name: Build extension
         run: |
           python -m pip install -v .
@@ -82,7 +81,7 @@ jobs:
         run: |
           python -m pip install -v .
       - name: Run tests
-        run: pytest
+        run: pytest --run-tutorials
 
   windows:
     runs-on: windows-latest
@@ -102,9 +101,9 @@ jobs:
           python -m pip install --upgrade pip
           pip install setuptools build cython wheel
           pip install -r tests/requirements.txt
-          pip install nbformat nbclient ipykernel jupyter matplotlib
+          pip install -r doc/requirements.txt
       - name: Build extension
         run: |
           python -m pip install -v .
       - name: Run tests
-        run: pytest
+        run: pytest --run-tutorials

doc/developer.contributing.rst

@@ -99,6 +99,11 @@ You can also run the tests locally before submitting your pull request, using `p
 
     pytest
 
+3. [Optional] If you wish to run the tutorial notebook tests, you will need to add the ``--run-tutorials`` flag, which require the `doc` dependencies: ::
+
+    pip install -r doc/requirements.txt
+    pytest --run-tutorials
+
 Adding to the test suite
 ^^^^^^^^^^^^^^^^^^^^^^^^
 

doc/requirements.txt

@@ -9,4 +9,4 @@ ipython
 matplotlib
 pickleshare
 jupyter
-open_spiel
+open_spiel; sys_platform != 'win32'

tests/conftest.py

@@ -0,0 +1,33 @@
+import pytest
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--run-tutorials",
+        action="store_true",
+        default=False,
+        help="Run tutorial notebook tests",
+    )
+
+
+def pytest_configure(config):
+    config.addinivalue_line(
+        "markers", "tutorials: mark test as a tutorial notebook test"
+    )
+
+
+def pytest_collection_modifyitems(config, items):
+    if config.getoption("--run-tutorials"):
+        # --run-tutorials given in cli: do not skip tutorial tests
+        return
+
+    # Check if test_tutorials.py was explicitly specified
+    args = config.invocation_params.args
+    if any("test_tutorials.py" in str(arg) for arg in args):
+        # test_tutorials.py explicitly run: do not skip
+        return
+
+    skip_tutorials = pytest.mark.skip(reason="need --run-tutorials option to run")
+    for item in items:
+        if "tutorials" in item.keywords:
+            item.add_marker(skip_tutorials)

tests/test_tutorials.py

@@ -39,6 +39,7 @@ def _find_tutorial_notebooks():
 _NOTEBOOKS = _find_tutorial_notebooks()
 
 
+@pytest.mark.tutorials
 @pytest.mark.parametrize("nb_path", _NOTEBOOKS, ids=[p.name for p in _NOTEBOOKS])
 def test_execute_notebook(nb_path):
     """Execute a single Jupyter notebook and fail if any cell errors occur.