Build/test in parallel - now with proper coverage reporting (#535)

* ci: did a roleback and reimplemented changes

* chore: added coverage for debugging

* disable xdist

* minor fix

* degging with tolerance

* updated test data

* reverted test

* chore: lint

* test cli in seriel

* we never dispatched xdist as loadgroup

* forced groups and extras

* ci: more typo fixing

* run grouped by file

* test: added pytest-rerunfailures for flaky tests

* test: removed mark as the we use file based distribution

---------

Co-authored-by: Jakob Guldberg Aaes <jakob1379@gmali.com>

.github/workflows/test.yaml

@@ -28,6 +28,8 @@ jobs:
 
       - name: Install uv
         uses: astral-sh/setup-uv@v7
+        with:
+          python-version: "${{ matrix.python-version}}"
 
       - name: Install just
         uses: taiki-e/install-action@just

justfile

@@ -15,7 +15,7 @@ pre-commit:
 
 count-lines:
   wc -l `find src -name '*.py'`
-  
+
 update-schema:
   uv run scripts/update_schema.py
 
@@ -44,10 +44,10 @@ test:
   uv run pytest
 
 test-with-coverage:
-  uv run -- coverage run -m pytest; uv run -- coverage combine
+  uv run -- pytest --cov
 
 report-coverage:
-  uv run -- coverage report && uv run -- coverage html --show-contexts
+  uv run -- pytest --cov --cov-report=html
 
 open video_path:
   uv run scripts/open.py {{video_path}}
@@ -56,4 +56,4 @@ profile target:
   sudo -E uv run py-spy record -o profile.svg python {{target}}
 
 pull-data:
-  uv run scripts/pull_data.py
+  uv run scripts/pull_data.py

pyproject.toml

@@ -104,11 +104,13 @@ dev = [
     'pyright>=1.1.406', # Type checking
     'pre-commit>=4.3.0', # Run checks before committing
     'pytest>=8.4.2', # Run tests
-    'coverage>=7.11.0', # Generate coverage reports
     'pypdf>=6.1.3', # Read PDF files
     'snakeviz>=2.2.2', # Profiling
     'pyinstaller>=6.16.0', # Build executables
     "pytest-xdist>=3.8.0",
+    "pytest-cov>=7.0.0",
+    "coverage>=7.11.0",
+    "pytest-rerunfailures>=16.1",
 ]
 docs = [
     'mkdocs-material>=9.6.20',
@@ -206,6 +208,8 @@ addopts = [
     '--strict-markers',    # Disallow unknown markers
     '--strict-config',     # Fail on unknown config options
     '--numprocesses=auto', # Number of processes in parallel
+    '--dist=loadfile',     # Make sure specific jobs are grouped
+    '--reruns=5',          # Rerun flaky tests
 ]
 testpaths = ['tests']
 

tests/conftest.py

@@ -4,6 +4,7 @@ import copy
 import filecmp
 import itertools
 import json
+import os
 import pathlib
 import shutil
 import typing
@@ -24,7 +25,7 @@ from rendercv.renderer import templater
 # the output. Setting update_testdata to True will update the reference files with
 # the latest RenderCV. This should be done with caution, as it will overwrite the
 # reference files with the latest output.
-update_testdata = False
+update_testdata = os.getenv("UPDATE_TESTDATA", "false").lower() == "true"
 
 education_entry_dictionary = {
     "institution": "Boğaziçi University",

tests/test_renderer.py

@@ -15,7 +15,6 @@ folder_name_dictionary = {
     "rendercv_filled_curriculum_vitae_data_model": "filled",
 }
 
-
 def test_typst_file_class(tmp_path, rendercv_data_model, jinja2_environment):
     typst_file = templater.TypstFile(rendercv_data_model, jinja2_environment)
     typst_file.get_full_code()

uv.lock

@@ -305,6 +305,11 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/5f/04/642c1d8a448ae5ea1369eac8495740a79eb4e581a9fb0cbdce56bbf56da1/coverage-7.11.0-py3-none-any.whl", hash = "sha256:4b7589765348d78fb4e5fb6ea35d07564e387da2fc5efff62e0222971f155f68", size = 207761, upload-time = "2025-10-15T15:15:06.439Z" },
 ]
 
+[package.optional-dependencies]
+toml = [
+    { name = "tomli", marker = "python_full_version <= '3.11'" },
+]
+
 [[package]]
 name = "distlib"
 version = "0.4.0"
@@ -1195,6 +1200,33 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/a8/a4/20da314d277121d6534b3a980b29035dcd51e6744bd79075a6ce8fa4eb8d/pytest-8.4.2-py3-none-any.whl", hash = "sha256:872f880de3fc3a5bdc88a11b39c9710c3497a547cfa9320bc3c5e62fbf272e79", size = 365750, upload-time = "2025-09-04T14:34:20.226Z" },
 ]
 
+[[package]]
+name = "pytest-cov"
+version = "7.0.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "coverage", extra = ["toml"] },
+    { name = "pluggy" },
+    { name = "pytest" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/5e/f7/c933acc76f5208b3b00089573cf6a2bc26dc80a8aece8f52bb7d6b1855ca/pytest_cov-7.0.0.tar.gz", hash = "sha256:33c97eda2e049a0c5298e91f519302a1334c26ac65c1a483d6206fd458361af1", size = 54328, upload-time = "2025-09-09T10:57:02.113Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/ee/49/1377b49de7d0c1ce41292161ea0f721913fa8722c19fb9c1e3aa0367eecb/pytest_cov-7.0.0-py3-none-any.whl", hash = "sha256:3b8e9558b16cc1479da72058bdecf8073661c7f57f7d3c5f22a1c23507f2d861", size = 22424, upload-time = "2025-09-09T10:57:00.695Z" },
+]
+
+[[package]]
+name = "pytest-rerunfailures"
+version = "16.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "packaging" },
+    { name = "pytest" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/de/04/71e9520551fc8fe2cf5c1a1842e4e600265b0815f2016b7c27ec85688682/pytest_rerunfailures-16.1.tar.gz", hash = "sha256:c38b266db8a808953ebd71ac25c381cb1981a78ff9340a14bcb9f1b9bff1899e", size = 30889, upload-time = "2025-10-10T07:06:01.238Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/77/54/60eabb34445e3db3d3d874dc1dfa72751bfec3265bd611cb13c8b290adea/pytest_rerunfailures-16.1-py3-none-any.whl", hash = "sha256:5d11b12c0ca9a1665b5054052fcc1084f8deadd9328962745ef6b04e26382e86", size = 14093, upload-time = "2025-10-10T07:06:00.019Z" },
+]
+
 [[package]]
 name = "pytest-xdist"
 version = "3.8.0"
@@ -1347,6 +1379,8 @@ dev = [
     { name = "pypdf" },
     { name = "pyright" },
     { name = "pytest" },
+    { name = "pytest-cov" },
+    { name = "pytest-rerunfailures" },
     { name = "pytest-xdist" },
     { name = "ruff" },
     { name = "snakeviz" },
@@ -1390,6 +1424,8 @@ dev = [
     { name = "pypdf", specifier = ">=6.1.3" },
     { name = "pyright", specifier = ">=1.1.406" },
     { name = "pytest", specifier = ">=8.4.2" },
+    { name = "pytest-cov", specifier = ">=7.0.0" },
+    { name = "pytest-rerunfailures", specifier = ">=16.1" },
     { name = "pytest-xdist", specifier = ">=3.8.0" },
     { name = "ruff", specifier = ">=0.14.1" },
     { name = "snakeviz", specifier = ">=2.2.2" },