diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 85ace93445b6..f301021ec30c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -40,54 +40,16 @@ jobs: ) permissions: contents: read - name: "Python ${{ matrix.python-version }} on ${{ matrix.os }} ${{ matrix.name-suffix }}" + name: "Python ${{ matrix.python-version }} on ${{ matrix.os }}, ${{ matrix.test_type}} ${{ matrix.name-suffix }}" runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: include: - - name-suffix: "(Minimum Versions)" - os: ubuntu-22.04 - python-version: '3.11' - extra-requirements: '-c requirements/testing/minver.txt' - delete-font-cache: true - # https://github.com/matplotlib/matplotlib/issues/29844 - pygobject-ver: '<3.52.0' - - os: ubuntu-22.04 - python-version: '3.11' - CFLAGS: "-fno-lto" # Ensure that disabling LTO works. - extra-requirements: '-r requirements/testing/extra.txt' - # https://github.com/matplotlib/matplotlib/issues/29844 - pygobject-ver: '<3.52.0' - - os: ubuntu-22.04-arm - python-version: '3.12' - # https://github.com/matplotlib/matplotlib/issues/29844 - pygobject-ver: '<3.52.0' - - name-suffix: "(Extra TeX packages)" - os: ubuntu-22.04 - python-version: '3.13' - extra-packages: 'texlive-fonts-extra texlive-lang-cyrillic' - # https://github.com/matplotlib/matplotlib/issues/29844 - pygobject-ver: '<3.52.0' - - name-suffix: "Free-threaded" - os: ubuntu-22.04 - python-version: '3.13t' - # https://github.com/matplotlib/matplotlib/issues/29844 - pygobject-ver: '<3.52.0' - os: ubuntu-24.04 python-version: '3.12' - - os: macos-13 # This runner is on Intel chips. - # merge numpy and pandas install in nighties test when this runner is dropped - python-version: '3.11' - - os: macos-14 # This runner is on M1 (arm64) chips. - python-version: '3.12' - # https://github.com/matplotlib/matplotlib/issues/29732 - pygobject-ver: '<3.52.0' - - os: macos-14 # This runner is on M1 (arm64) chips. - python-version: '3.13' - # https://github.com/matplotlib/matplotlib/issues/29732 - pygobject-ver: '<3.52.0' + test_type: serial steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -337,9 +299,21 @@ jobs: if [[ "${{ matrix.python-version }}" == '3.13t' ]]; then export PYTHON_GIL=0 fi - pytest -rfEsXR -n auto \ - --maxfail=50 --timeout=300 --durations=25 \ - --cov-report=xml --cov=lib --log-level=DEBUG --color=yes + FLAGS=( + -rfEsXR + --maxfail=5 + --timeout=300 + --durations=25 + --cov-report=xml + --cov=lib + --log-level=DEBUG + --color=yes + ) + if [[ "${{ matrix.test_type }}" == 'serial' ]]; then + pytest "${FLAGS[@]}" -m subprocess -n 0 -vv -s + else + pytest "${FLAGS[@]}" -m 'not subprocess' -n auto + fi - name: Cleanup non-failed image files if: failure() diff --git a/azure-pipelines.yml b/azure-pipelines.yml index d68a9d36f0d3..0f52752292c2 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -49,15 +49,30 @@ stages: - job: Pytest strategy: matrix: - Windows_py311: + Windows_py311_parallel: vmImage: 'windows-2022' # Keep one job pinned to the oldest image python.version: '3.11' - Windows_py312: + test.type: 'parallel' + Windows_py311_serial: + vmImage: 'windows-2022' # Keep one job pinned to the oldest image + python.version: '3.11' + test.type: 'serial' + Windows_py312_parallel: vmImage: 'windows-latest' python.version: '3.12' - Windows_py313: + test.type: 'parallel' + Windows_py312_serial: + vmImage: 'windows-latest' + python.version: '3.12' + test.type: 'serial' + Windows_py313_parallel: vmImage: 'windows-latest' python.version: '3.13' + test.type: 'parallel' + Windows_py313_serial: + vmImage: 'windows-latest' + python.version: '3.13' + test.type: 'serial' maxParallel: 4 pool: vmImage: '$(vmImage)' @@ -120,10 +135,20 @@ stages: echo "##vso[task.setvariable variable=VS_COVERAGE_TOOL]$TOOL" - PYTHONFAULTHANDLER=1 pytest -rfEsXR -n 2 \ - --maxfail=50 --timeout=300 --durations=25 \ - --junitxml=junit/test-results.xml --cov-report=xml --cov=lib - + FLAGS=( + -rfEsXR + --maxfail=50 + --timeout=300 + --durations=25 + --cov-report=xml + --cov=lib + --junitxml=junit/test-results.xml + ) + if [[ "$TEST_TYPE" == serial ]]; then + PYTHONFAULTHANDLER=1 pytest "${FLAGS[@]}" -m subprocess -n 0 + else + PYTHONFAULTHANDLER=1 pytest "${FLAGS[@]}" -m 'not subprocess' -n 2 + fi if [[ $VS_VER == 2022 ]]; then "$TOOL" shutdown $SESSION_ID echo "Coverage collection log" @@ -153,7 +178,8 @@ stages: - task: PublishTestResults@2 inputs: - testResultsFiles: '**/test-results.xml' + mergeTestResults: true + testResultsFiles: '**/test-results*.xml' testRunTitle: 'Python $(python.version)' condition: succeededOrFailed() diff --git a/lib/matplotlib/testing/conftest.py b/lib/matplotlib/testing/conftest.py index 2961e7f02f3f..4efc10766be7 100644 --- a/lib/matplotlib/testing/conftest.py +++ b/lib/matplotlib/testing/conftest.py @@ -15,6 +15,7 @@ def pytest_configure(config): ("markers", "backend: Set alternate Matplotlib backend temporarily."), ("markers", "baseline_images: Compare output against references."), ("markers", "pytz: Tests that require pytz to be installed."), + ("markers", "subprocess: Tests that start a subprocess."), ("filterwarnings", "error"), ("filterwarnings", "ignore:.*The py23 module has been deprecated:DeprecationWarning"), diff --git a/lib/matplotlib/tests/test_backend_inline.py b/lib/matplotlib/tests/test_backend_inline.py index 997e1e7186b1..2cc1022b7939 100644 --- a/lib/matplotlib/tests/test_backend_inline.py +++ b/lib/matplotlib/tests/test_backend_inline.py @@ -10,6 +10,7 @@ pytest.importorskip('nbconvert') pytest.importorskip('ipykernel') pytest.importorskip('matplotlib_inline') +pytestmark = pytest.mark.subprocess def test_ipynb(): diff --git a/lib/matplotlib/tests/test_backend_nbagg.py b/lib/matplotlib/tests/test_backend_nbagg.py index ccf74df20aab..5b082f159c36 100644 --- a/lib/matplotlib/tests/test_backend_nbagg.py +++ b/lib/matplotlib/tests/test_backend_nbagg.py @@ -9,6 +9,7 @@ nbformat = pytest.importorskip('nbformat') pytest.importorskip('nbconvert') pytest.importorskip('ipykernel') +pytestmark = pytest.mark.subprocess # From https://blog.thedataincubator.com/2016/06/testing-jupyter-notebooks/ diff --git a/lib/matplotlib/tests/test_backend_webagg.py b/lib/matplotlib/tests/test_backend_webagg.py index 1d6769494ef9..e09d85ce2e26 100644 --- a/lib/matplotlib/tests/test_backend_webagg.py +++ b/lib/matplotlib/tests/test_backend_webagg.py @@ -6,6 +6,7 @@ from matplotlib.testing import subprocess_run_for_testing +@pytest.mark.subprocess @pytest.mark.parametrize("backend", ["webagg", "nbagg"]) def test_webagg_fallback(backend): pytest.importorskip("tornado") diff --git a/lib/matplotlib/tests/test_backends_interactive.py b/lib/matplotlib/tests/test_backends_interactive.py index 9f8522a9df4a..1e0efd4c5966 100644 --- a/lib/matplotlib/tests/test_backends_interactive.py +++ b/lib/matplotlib/tests/test_backends_interactive.py @@ -22,6 +22,9 @@ from matplotlib.testing import subprocess_run_helper as _run_helper, is_ci_environment +pytestmark = pytest.mark.subprocess + + class _WaitForStringPopen(subprocess.Popen): """ A Popen that passes flags that allow triggering KeyboardInterrupt. @@ -33,8 +36,10 @@ def __init__(self, *args, **kwargs): super().__init__( *args, **kwargs, # Force Agg so that each test can switch to its desired backend. - env={**os.environ, "MPLBACKEND": "Agg", "SOURCE_DATE_EPOCH": "0"}, - stdout=subprocess.PIPE, universal_newlines=True) + env={**os.environ, "MPLBACKEND": "Agg", "SOURCE_DATE_EPOCH": "0", + "PYTHONUNBUFFERED": "1"}, + bufsize=0, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, universal_newlines=True) def wait_for(self, terminator): """Read until the terminator is reached.""" @@ -42,8 +47,12 @@ def wait_for(self, terminator): while True: c = self.stdout.read(1) if not c: - raise RuntimeError( - f'Subprocess died before emitting expected {terminator!r}') + if self.poll() is None: + os.sched_yield() + else: + raise RuntimeError( + f'Subprocess died before emitting expected {terminator!r}\n' + f'\nSubprocess output:\n{buf}') buf += c if buf.endswith(terminator): return @@ -237,7 +246,7 @@ def check_alt_backend(alt_backend): @pytest.mark.parametrize("env", _get_testable_interactive_backends()) @pytest.mark.parametrize("toolbar", ["toolbar2", "toolmanager"]) -@pytest.mark.flaky(reruns=3) +#@pytest.mark.flaky(reruns=3) def test_interactive_backend(env, toolbar): if env["MPLBACKEND"] == "macosx": if toolbar == "toolmanager": @@ -329,7 +338,7 @@ def _test_thread_impl(): @pytest.mark.parametrize("env", _thread_safe_backends) -@pytest.mark.flaky(reruns=3) +#@pytest.mark.flaky(reruns=3) def test_interactive_thread_safety(env): proc = _run_helper(_test_thread_impl, timeout=_test_timeout, extra_env=env) assert proc.stdout.count("CloseEvent") == 1 @@ -617,7 +626,7 @@ def _test_number_of_draws_script(): @pytest.mark.parametrize("env", _blit_backends) # subprocesses can struggle to get the display, so rerun a few times -@pytest.mark.flaky(reruns=4) +#@pytest.mark.flaky(reruns=4) def test_blitting_events(env): proc = _run_helper( _test_number_of_draws_script, timeout=_test_timeout, extra_env=env) diff --git a/lib/matplotlib/tests/test_basic.py b/lib/matplotlib/tests/test_basic.py index f6aa1e458555..10e94ee3be9d 100644 --- a/lib/matplotlib/tests/test_basic.py +++ b/lib/matplotlib/tests/test_basic.py @@ -5,6 +5,8 @@ from matplotlib.testing import subprocess_run_for_testing +import pytest + def test_simple(): assert 1 + 1 == 2 @@ -28,6 +30,7 @@ def test_override_builtins(): assert overridden <= ok_to_override +@pytest.mark.subprocess def test_lazy_imports(): source = textwrap.dedent(""" import sys diff --git a/lib/matplotlib/tests/test_determinism.py b/lib/matplotlib/tests/test_determinism.py index 2ecc40dbd3c0..dd1669894d98 100644 --- a/lib/matplotlib/tests/test_determinism.py +++ b/lib/matplotlib/tests/test_determinism.py @@ -21,6 +21,8 @@ from matplotlib.text import TextPath from matplotlib.transforms import IdentityTransform +pytestmark = pytest.mark.subprocess + def _save_figure(objects='mhip', fmt="pdf", usetex=False): mpl.use(fmt) diff --git a/lib/matplotlib/tests/test_font_manager.py b/lib/matplotlib/tests/test_font_manager.py index 97ee8672b1d4..3e9b95f4fd5a 100644 --- a/lib/matplotlib/tests/test_font_manager.py +++ b/lib/matplotlib/tests/test_font_manager.py @@ -288,6 +288,7 @@ def test_fontcache_thread_safe(): subprocess_run_helper(_test_threading, timeout=10) +@pytest.mark.subprocess def test_lockfilefailure(tmp_path): # The logic here: # 1. get a temp directory from pytest diff --git a/lib/matplotlib/tests/test_matplotlib.py b/lib/matplotlib/tests/test_matplotlib.py index d0a3f8c617e1..3a28ecb985a7 100644 --- a/lib/matplotlib/tests/test_matplotlib.py +++ b/lib/matplotlib/tests/test_matplotlib.py @@ -19,6 +19,7 @@ def test_parse_to_version_info(version_str, version_tuple): assert matplotlib._parse_to_version_info(version_str) == version_tuple +@pytest.mark.subprocess @pytest.mark.skipif(sys.platform == "win32", reason="chmod() doesn't work as is on Windows") @pytest.mark.skipif(sys.platform != "win32" and os.geteuid() == 0, @@ -37,6 +38,7 @@ def test_tmpconfigdir_warning(tmp_path): os.chmod(tmp_path, mode) +@pytest.mark.subprocess def test_importable_with_no_home(tmp_path): subprocess_run_for_testing( [sys.executable, "-c", diff --git a/lib/matplotlib/tests/test_preprocess_data.py b/lib/matplotlib/tests/test_preprocess_data.py index c983d78786e1..3825a0aecd09 100644 --- a/lib/matplotlib/tests/test_preprocess_data.py +++ b/lib/matplotlib/tests/test_preprocess_data.py @@ -245,6 +245,7 @@ def funcy(ax, x, y, z, t=None): funcy.__doc__) +@pytest.mark.subprocess def test_data_parameter_replacement(): """ Test that the docstring contains the correct *data* parameter stub diff --git a/lib/matplotlib/tests/test_pyplot.py b/lib/matplotlib/tests/test_pyplot.py index 55f7c33cb52e..336847f4b7ef 100644 --- a/lib/matplotlib/tests/test_pyplot.py +++ b/lib/matplotlib/tests/test_pyplot.py @@ -12,6 +12,7 @@ from matplotlib import pyplot as plt +@pytest.mark.subprocess def test_pyplot_up_to_date(tmp_path): pytest.importorskip("black") diff --git a/lib/matplotlib/tests/test_rcparams.py b/lib/matplotlib/tests/test_rcparams.py index 2235f98b720f..ee9a6476703a 100644 --- a/lib/matplotlib/tests/test_rcparams.py +++ b/lib/matplotlib/tests/test_rcparams.py @@ -528,6 +528,7 @@ def test_rcparams_reset_after_fail(): assert mpl.rcParams['text.usetex'] is False +@pytest.mark.subprocess @pytest.mark.skipif(sys.platform != "linux", reason="Linux only") def test_backend_fallback_headless_invalid_backend(tmp_path): env = {**os.environ, @@ -545,6 +546,7 @@ def test_backend_fallback_headless_invalid_backend(tmp_path): env=env, check=True, stderr=subprocess.DEVNULL) +@pytest.mark.subprocess @pytest.mark.skipif(sys.platform != "linux", reason="Linux only") def test_backend_fallback_headless_auto_backend(tmp_path): # specify a headless mpl environment, but request a graphical (tk) backend @@ -567,6 +569,7 @@ def test_backend_fallback_headless_auto_backend(tmp_path): assert backend.strip().lower() == "agg" +@pytest.mark.subprocess @pytest.mark.skipif( sys.platform == "linux" and not _c_internal_utils.xdisplay_is_valid(), reason="headless") diff --git a/lib/matplotlib/tests/test_sphinxext.py b/lib/matplotlib/tests/test_sphinxext.py index ede3166a2e1b..5b6e2e6b6e70 100644 --- a/lib/matplotlib/tests/test_sphinxext.py +++ b/lib/matplotlib/tests/test_sphinxext.py @@ -16,6 +16,7 @@ tinypages = Path(__file__).parent / 'data/tinypages' +@pytest.mark.subprocess def build_sphinx_html(source_dir, doctree_dir, html_dir, extra_args=None): # Build the pages with warnings turned into errors extra_args = [] if extra_args is None else extra_args diff --git a/lib/matplotlib/tests/test_texmanager.py b/lib/matplotlib/tests/test_texmanager.py index 64dcbf46456d..d7adda36e615 100644 --- a/lib/matplotlib/tests/test_texmanager.py +++ b/lib/matplotlib/tests/test_texmanager.py @@ -63,6 +63,7 @@ def test_unicode_characters(): fig.canvas.draw() +@pytest.mark.subprocess @needs_usetex def test_openin_any_paranoid(): completed = subprocess_run_for_testing(