Test

setup: Use cog to automate the annual chore of updating Python versions #701

Sign in to view logs

Workflow file for this run

.github/workflows/test.yml at 9c31fe5

	# The "test" workflow is run on every PR and runs tests across all
	# supported python versions and a range of configurations
	# specified in tox.ini. Also see the "build" workflow which is only
	# run for release branches and covers platforms other than linux-amd64
	# (Platform-specific issues are rare these days so we don't want to
	# take that time on every build).

	name: Test

	on: pull_request

	permissions: {}

	jobs:
	# Before starting the full build matrix, run one test configuration
	# and the linter (the `black` linter is especially likely to catch
	# first-time contributors).
	test_quick:
	name: Run quick tests
	runs-on: ubuntu-24.04
	steps:
	- uses: actions/checkout@v4
	with:
	persist-credentials: false
	- uses: actions/setup-python@v5
	name: Install Python
	with:
	# [[[cog cog.outl(f"python-version: '3.{default_python_minor}'")]]]
	python-version: '3.13'
	# [[[end]]]
	- name: Install tox
	run: python -m pip install tox -c requirements.txt

	- name: Run test suite
	# [[[cog cog.outl(f"run: python -m tox -e py3{default_python_minor},lint")]]]
	run: python -m tox -e py313,lint
	# [[[end]]]

	test_tox:
	name: Run full tests
	needs: test_quick
	runs-on: ubuntu-24.04
	strategy:
	matrix:
	include:
	# [[[cog
	# configs = []
	# for minor in range(int(min_python_minor), int(max_python_minor) + 1):
	# if minor == int(dev_python_minor):
	# # For python versions in development, specify a range so we install
	# # a final version if available, otherwise a prerelease.
	# # The .0 is necessary here for unclear reasons.
	# configs.append((f"3.{minor}.0-dev - 3.{minor}", f"py3{minor}"))
	# else:
	# configs.append((f"3.{minor}", f"py3{minor}-full"))
	# if int(min_python_threaded_minor) <= minor <= int(max_python_threaded_minor):
	# if minor == int(dev_python_minor):
	# # The range trick above doesn't work for threaded builds, so we'll
	# # just be stuck with last prerelease until we update dev_python_minor.
	# configs.append((f"3.{minor}.0t-dev", f"py3{minor}"))
	# else:
	# configs.append((f"3.{minor}t", f"py3{minor}"))
	# # Early versions of 3.10 and 3.11 had different deprecation
	# # warnings in asyncio. Test with them too to make sure everything
	# # works the same way.
	# configs.append(("3.10.8", "py310-full"))
	# configs.append(("3.11.0", "py311-full"))
	# # Pypy is a lot slower due to jit warmup costs, so don't run the
	# # "full" test config there.
	# configs.append(("pypy-3.10", "pypy3"))
	# # Docs python version must be synced with tox.ini
	# configs.append((f"3.{default_python_minor}", "docs"))
	# for version, tox_env in configs:
	# cog.outl(f" - python: '{version}'")
	# cog.outl(f" tox_env: {tox_env}")
	# ]]]
	- python: '3.10'
	tox_env: py310-full
	- python: '3.11'
	tox_env: py311-full
	- python: '3.12'
	tox_env: py312-full
	- python: '3.13'
	tox_env: py313-full
	- python: '3.14.0-dev - 3.14'
	tox_env: py314
	- python: '3.14.0t-dev'
	tox_env: py314
	- python: '3.10.8'
	tox_env: py310-full
	- python: '3.11.0'
	tox_env: py311-full
	- python: 'pypy-3.10'
	tox_env: pypy3
	- python: '3.13'
	tox_env: docs
	# [[[end]]]

	steps:
	- uses: actions/checkout@v4
	with:
	persist-credentials: false
	- uses: actions/setup-python@v5
	name: Install Python
	with:
	python-version: ${{ matrix.python}}
	- name: Install apt packages
	run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev
	- name: Install tox
	run: python -m pip install tox -c requirements.txt

	- name: Run test suite
	run: python -m tox -e ${{ matrix.tox_env }}

	test_win:
	# Windows tests are fairly slow, so only run one configuration here.
	# We test on windows but not mac because even though mac is a more
	# fully-supported platform, it's similar enough to linux that we
	# don't generally need to test it separately. Windows is different
	# enough that we'll break it if we don't test it in CI.
	name: Run windows tests
	needs: test_quick
	runs-on: windows-2025
	steps:
	- uses: actions/checkout@v4
	with:
	persist-credentials: false
	- uses: actions/setup-python@v5
	name: Install Python
	with:
	# [[[cog cog.outl(f"python-version: '3.{default_python_minor}'")]]]
	python-version: '3.13'
	# [[[end]]]
	- name: Run test suite
	# TODO: figure out what's up with these log messages
	run: py -m tornado.test --fail-if-logs=false

	zizmor:
	name: Analyze action configs with zizmor
	runs-on: ubuntu-24.04
	needs: test_quick
	steps:
	- uses: actions/checkout@v4
	with:
	persist-credentials: false
	- uses: astral-sh/setup-uv@v6
	name: Install uv
	- name: Run zizmor
	run: uvx zizmor .github/workflows

	test_cibw:
	# cibuildwheel is the tool that we use for release builds in build.yml.
	# Run it in the every-PR workflow because it's slightly different from our
	# regular build and this gives us easier ways to test freethreading changes.
	#
	# Note that test_cibw and test_tox both take about a minute to run, but test_tox runs
	# more tests; test_cibw spends a lot of its time installing dependencies. Replacing
	# test_tox with test_cibw would entail either increasing test runtime or reducing
	# test coverage.
	name: Test with cibuildwheel
	runs-on: ubuntu-24.04
	needs: test_quick
	steps:
	- uses: actions/checkout@v4
	with:
	persist-credentials: false
	- name: Run cibuildwheel
	uses: pypa/cibuildwheel@v2.23
	env:
	# For speed, we only build one python version and one arch. We throw away the wheels
	# built here; the real build is defined in build.yml.
	CIBW_ARCHS: native
	# [[[cog cog.outl(f"CIBW_BUILD: cp3{default_python_minor}-manylinux*")]]]
	CIBW_BUILD: cp313-manylinux*
	# [[[end]]]